Push current state: v3 checkpoints + metrics + batch2 scripts

v3 trained on complete 1647-repo dataset (best cr_val 2.5995); batch2 novel
dataset scripts (enhanced QA + orchestrator).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

data/batch2_novel_repos.txt

@@ -0,0 +1,500 @@
+openstack/akanda-rug	python
+rgardner/ouimeaux	python
+plainas/tq	python
+timraasveld/ansible-string-split-filter	python
+adampiskorski/lpr_poc	python
+ranisalt/enigma	python
+claudiopastorini/claudiopastorini.github.io	python
+bshaffer/appengine-python-vm-hello	python
+RainCity471/lyCompiler	python
+chiphogg/vim-vtd	python
+egafford/sahara	python
+VirusTotal/misp-modules	python
+benedfit/SublimeLinter-contrib-pug-lint	python
+thebinarypenguin/SublimeLinter-contrib-raml-cop	python
+monostable/haskell-kicad-data	python
+Dybov/real_estate_agency	python
+nagilum/script.rndmov	python
+ianfieldhouse/number_to_words	python
+andylytical/brewpi-scripts	python
+HHS-IntroProgramming/Multiplication-table	python
+potatolondon/contentious	python
+sassoftware/mint	python
+ikaruswill/vector-space-model	python
+NewKnowledge/punk	python
+igstan/redis-grep	python
+gviot/nadis	python
+kevgathuku/top40	python
+MichaelYusko/Bot-Chucky	python
+associazionepoltronieri/blender-ap	python
+jefrailey/basic-scraper	python
+haridsv/fabric	python
+gadventures/gapipy	python
+sk-/python2.7-type-annotator	python
+mythmon/kitsune	python
+j4mie/django-activelink	python
+jwg4/flask-autodoc	python
+EmadMokhtar/halaqat	python
+EDUlib/edx-platform	python
+Graylog2/graylog-ansible-role	python
+armab/st2contrib	python
+KrzysztofSendor/dactyl	python
+nurav/balrog	python
+Pierre-Sassoulas/django-survey	python
+microsoft/dpu-utils	python
+PersonalGenomesOrg/open-humans	python
+vayan/external-video	python
+simonjbeaumont/xcp-rrdd	python
+lozadaOmr/ansible-admin	python
+ulikoehler/UliEngineering	python
+kxxoling/fabric	python
+EmilStenstrom/nephele	python
+chingc/DJRivals	python
+docsbox/docsbox	python
+uva-financial-engineering/cifer-tournament	python
+OSGConnect/freesurfer_workflow	python
+aaronkurtz/bricky	python
+worldforge/libwfut	python
+churchlab/millstone	python
+asteroide/immo_spider	python
+odoo-chile/l10n_cl_account_vat_ledger	python
+jeenalee/sklearn_pmml	python
+attenzione/SublimeLinter-scss-lint	python
+pudo/spendb	python
+hackebrot/cibopath	python
+mokieyue/mopidy	python
+textbook/flask-forecaster	python
+ncareol/spatialdb	python
+fna/owning-a-home-api	python
+gitpython-developers/GitPython	python
+spyder-ide/qtawesome	python
+ravel-net/ravel	python
+humangeo/preflyt	python
+ucb-sts/sts	python
+jlaunonen/kirppu	python
+matthiask/towel	python
+jmiserez/sts	python
+teonlamont/mne-python	python
+EinsamHauer/graphite-web-iow	python
+OCA/contract	python
+spreadflow/spreadflow-core	python
+djangogirlstaipei/eshop	python
+99designs/colorific	python
+revng/llvmcpy	python
+Antrikshy/reddit2Kindle	python
+foliant-docs/foliant	python
+estebistec/django-twitter-bootstrap	python
+Antojitos/chalupas	python
+MasonM/hssonline-conference	python
+2baOrNot2ba/dreamBeam	python
+k4nar/inbox	python
+ctsit/nacculator	python
+kytos/python-openflow	python
+victorpoluceno/xwing	python
+Stratoscale/skipper	python
+SUSE/azurectl	python
+pinax/django-waitinglist	python
+CA-APM/ca-apm-fieldpack-asm	java
+emicklei/melrose	go
+dev-brutus/427	javascript
+sabre-io/vobject	php
+logentries/le_ruby	ruby
+chunye/azure-functions-ux	typescript
+jhelwig/homers	rust
+matayoshi/tools	c
+philipphager/disclosure-android-app	java
+hashicorp/go-getter	go
+jlord/git-it-electron	javascript
+armadito/armadito-glpi	php
+kolorahl/acls	ruby
+neild3r/vscode-php-docblocker	typescript
+LesPepitos/gist	rust
+ROCm-Developer-Tools/HIP	c
+veyndan/paper-for-reddit	java
+torinos-io/api	go
+alphagov/notifications-admin	javascript
+ProjetPP/PPP-Wikidata	php
+uhoh-itsmaciek/pg-quilter	ruby
+Ragg-/Delir	typescript
+cobalt-org/cobalt.rs	rust
+k4rtik/btp-gvn	c
+kryptnostic/rhizome	java
+resumic/schema	go
+topherauyeung/portfolio	javascript
+EOL/eol_php_code	php
+jiajiawang/load_more	ruby
+jchakra/discord-raider	typescript
+hoodie/rust-email	rust
+GNOME/telepathy-account-widgets	c
+Hypersonic/Starlorn	java
+declanshanaghy/bbqberry	go
+var-bin/reactjs-training	javascript
+joshmassey/sga-itracker	php
+spatchcock/calc-json-ruby	ruby
+what3words/w3w-node-wrapper	typescript
+bitwalker/rustcalc	rust
+fergul/Cache	c
+Monospark/ActionControl	java
+qskycolor/gofrontend	go
+slightly-askew/portfolio-2017	javascript
+digitickets/omnipay-verifone-web-service	php
+emodeqidao/Kiwi	ruby
+sdeleuze/mixit	typescript
+aopicier/cryptopals-rust	rust
+bbockelm/condor-network-accounting	c
+steria/skuldsku	java
+mattn/go-mastodon	go
+cristiana214/cristianachavez214-cristianachavez	javascript
+ob-ivan/sd-currency	php
+chrishunt/github-auth	ruby
+thecjharries/slim-ace	typescript
+tum-rt/emoji-feedback	rust
+ytaben/WakeOnLanRelay	c
+yingyun001/ovirt-engine	java
+soudy/eparser	go
+mg4tv/mg4tv-web	javascript
+tux-rampage/zend-di	php
+chrisvanheuveln/cisco-network-node-utils	ruby
+jordwest/news-feed-eradicator	typescript
+DaGenix/rust-crypto	rust
+hikoLab/pebcessing	c
+bogdansolga/nokia-spring-boot-training	java
+btobolaski/terraform-provider-linode	go
+testlnord/wurfelspiel	javascript
+fullybaked/pslackr	php
+pch/lastfm-client	ruby
+cookingfox/stibble-api-client-angular	typescript
+oakes/neovim-rs	rust
+reupen/mmh	c
+Charling-Huang/birt	java
+anthonyrego/gosmf	go
+hellsgate1001/waypoints	javascript
+renyuneyun/core	php
+Validic/validic	ruby
+artsy/force-public	typescript
+mijnadres/serde_elm	rust
+pressel/mpi4py	c
+RoboZonky/robozonky	java
+ftcjeff/AirlineCode-go	go
+dmitriiabramov/esfmt	javascript
+artesaos/laravel-linkedin	php
+aruprakshit/sync	ruby
+dkocich/osm-pt-ngx-leaflet	typescript
+zslayton/cron	rust
+aaronriekenberg/openbsd_cproxy	c
+rajdavies/fabric8	java
+ziutek/thread	go
+vigetlabs/microcosm	javascript
+whathejoe/lms-m4g4	php
+mongodb/mongoid	ruby
+wishtack/wishtack-steroids	typescript
+burtonageo/rebind	rust
+thoughtbot/BotKit	c
+smalldatalab/omh-dsu	java
+Versent/saml2aws	go
+dimkarakostas/unimeet	javascript
+symfony/security-bundle	php
+moneyadviceservice/middleman-i18n-markdown	ruby
+xtina-starr/reaction	typescript
+tzkhan/roman-rust	rust
+ferrous26/cs452-flaming-meme	c
+QuickBlox/quickblox-android-sdk	java
+willfaught/gockle	go
+coryhouse/react-slingshot	javascript
+LogicAndTrick/twhl	php
+eric/metriks_log_webhook	ruby
+BillWagner/WorkingWithTypeScript	typescript
+mulkieran/stratisd	rust
+abusalimov/SublimeCImproved	c
+danshannon/javastrava-test	java
+projectatomic/buildah	go
+shioyang/EmotionalReader	javascript
+sparkframework/spark	php
+r7kamura/altria-simple_cov	ruby
+murmur76/beyond.ts	typescript
+nathanrosspowell/rust_guessing_game	rust
+deniscostadsc/playground	c
+helun/Ektorp	java
+guildencrantz/ask_nest	go
+sgmap/mes-aides-ui	javascript
+webfactory/content-mapping-sourceadapter-propel	php
+benwbrum/fromthepage	ruby
+artivilla/desktop	typescript
+adRichter/plantex	rust
+coreboot/chrome-ec	c
+hpautonomy/find	java
+go-playground/generate	go
+mrkmarron/ChakraCore	javascript
+atehnix/laravel-stubs	php
+costolo/chain	ruby
+github/webauthn-json	typescript
+pacman82/odbc-sys	rust
+mundue/MMTrackingController	c
+proxer/ProxerLibJava	java
+flynn-examples/go-flynn-example	go
+eJRF/ejrf	javascript
+jolantis/artlantis	php
+rasnom/sorryyoufeelthatway	ruby
+formigio/angular-frontend	typescript
+dtolnay/cxx	rust
+dslab-epfl/asap	c
+Microsoft/ApplicationInsights-Android	java
+buchgr/bazel-remote	go
+amos-ws16/amos-ws16-arrowjs-server	javascript
+engram-design/FieldManager	php
+gina-alaska/ttnmanager	ruby
+cannoneyed/fiddle	typescript
+Boddlnagg/rustup.rs	rust
+thomaskeck/root	c
+pilot51/voicenotify	java
+mongrelion/gapp	go
+virtool/virtool	javascript
+stof/symfony	php
+twitter/twurl	ruby
+amalshehu/angular-tour-of-heroes	typescript
+dylanmckay/protocol	rust
+mlaz/mynewt-core	c
+openmhealth/sample-data-generator	java
+opencontainers/runc	go
+dtekcth/dfotose	javascript
+edvinaskrucas/counter-laravel	php
+amatsuda/nested_scaffold	ruby
+sgtoj/cw-manage-slack-bot	typescript
+tsion/miri	rust
+HalCanary/skia-hc	c
+kits-ab/gakusei	java
+didip/tollbooth	go
+pd4d10/tiza	javascript
+phalcon/zephir	php
+slate-studio/openapi-rails	ruby
+dsebastien/DefinitelyTyped	typescript
+lowRISC/opentitan	rust
+rubenk/burp	c
+smalecki/motech	java
+mrekucci/euler	go
+bitclaw/apptmnt	javascript
+mdlayher/wavepipe	php
+dhensby/silverstripe-cms	ruby
+goloveychuk/tsruntime	typescript
+exe-dealer/rust-md5	rust
+mk12/morse	c
+HubSpot/jinjava	java
+elwinar/adeptus	go
+tidepool-org/blip	javascript
+listabierta/census-ahoraencomun	php
+backus/guard-rubocop	ruby
+dl988/quedro	typescript
+ggez/ggez	rust
+robertmaynard/Sandbox	c
+kaltsimon/Chasing-Pictures-front-end	java
+StroblIndustries/codeutilsShared	go
+TripleSpeeder/StandingOrderDapp	javascript
+jeffersonmartin/code-examples	php
+julik/mal	ruby
+thehyve/glowing-bear	typescript
+mneumann/expression-rs	rust
+Gluttton/PslRK	c
+inbloom/secure-data-service	java
+Dirbaio/btcd	go
+intervention-engine/frontend	javascript
+burnflare/CrowdBuy	php
+postmodern/kramdown-man	ruby
+zentol/flink	typescript
+oxcable/oxcable	rust
+CollegeBart/bart-sdl-engine-e16	c
+outofcoffee/testcontainers-java	java
+scode/saltybox	go
+MacPaw/eslint-config-webservices	javascript
+jimmythongtran/recipe_tracker	php
+samvera/hyrax	ruby
+j-f1/forked-desktop	typescript
+habitat-sh/habitat	rust
+anagromataf/Fountain	c
+sake/bouncycastle-java	java
+liam-middlebrook/csh-plug	go
+benefitcloud/ember-uploader	javascript
+siad007/json-schema	php
+solidusio-contrib/solidus_easypost	ruby
+influxdb/influxdb	typescript
+bolinfest/serde-jsonrc	rust
+UrbanCompass/RMGallery	c
+TehSAUCE/imagej	java
+asobti/kube-monkey	go
+CartoDB/Windshaft-cartodb	javascript
+ponkrit/developer-test	php
+lukeredpath/xcodesnippets	ruby
+trustpilot/skift	typescript
+stepancheg/rust-ide-rust	rust
+cxa/MenuItemKit	c
+VinnyQ/blueflood	java
+melange-app/melange	go
+mzrimsek/resume_site	javascript
+wesley1001/Kurogo-Mobile-Web	php
+giedomak/table_cloth	ruby
+t123yh/simple-sandbox	typescript
+dati91/servo	rust
+bespider/STTweetLabel	c
+Team694/joebot	java
+v4lproik/no-name	go
+bassettmb/slack-bot-dev	javascript
+biplobice/concrete5	php
+heavenstudio/spree_pag_seguro	ruby
+BitGo/BitGoJS	typescript
+keeperofdakeys/asn1-rs	rust
+BGWoodward/FishDetector	c
+rwinch/spring-security	java
+wrapp/wrapplog	go
+kiboit/phast	javascript
+alexweissman/UserFrosting	php
+chronogolf/lightspeed_restaurant	ruby
+gund/ng-http-interceptor	typescript
+ajroetker/rustc-serialize	rust
+greenplum-db/gpdb	c
+malucs-developer/Android-Simulator	java
+otoolep/raft	go
+escaip/smart_bike	javascript
+imabug/raddb	php
+ruby/spec	ruby
+SaschaNaz/TypeScript	typescript
+dinfuehr/rust	rust
+kazmasaurus/CRToast	c
+apache/jmeter	java
+ccirello/gochatbot	go
+rowanoulton/prefix-matches	javascript
+vanderlee/PHPSwaggerGen	php
+dtan4/c2y	ruby
+tsvetie/nativescript-cli	typescript
+qdot/systray-rs	rust
+filom/ASN1Decoder	c
+Arabidopsis-Information-Portal/intermine	java
+lstolyarov/terraform	go
+squarewave/addons-frontend	javascript
+symfony-cmf/Resource	php
+syncrou/manageiq	ruby
+arthot/calque	typescript
+dwrensha/capnproto-rust	rust
+cxd4/trig	c
+Syncleus/aparapi	java
+rancher/k3s	go
+mozilla/webmaker-app-cordova	javascript
+LaurentEsc/Laravel-Subdomain-Localization	php
+mahimahi42/microblogger	ruby
+robak86/neography	typescript
+minijackson/INF-4301A	rust
+k0gaMSX/kcc	c
+holandajunior/workaday	java
+myodc/go-micro	go
+sigurdga/barteby	javascript
+anqh/anqh	php
+ValiMail/griddler	ruby
+polyaxon/polyaxon	typescript
+gchp/termbox-sys	rust
+leovegetium/Trabalho-P	c
+pzn/clipapp	java
+ephemeralsnow/packer	go
+dkrathi457/app	javascript
+GrottoPress/jentil	php
+crepe/creperie	ruby
+Dkendal/battle_snake	typescript
+snewt/bnf	rust
+netzimme/mbed-os	c
+mkarneim/luamod	java
+resourced/resourced-master	go
+manifoldco/torus-cli	javascript
+kjhoerr/august-offensive	php
+matiaskorhonen/libpixel-ruby	ruby
+tjoskar/ng2-lazyload-image	typescript
+jonas-schievink/sneeze	rust
+adobe-rnd/cordova-osx	c
+mrdon/AMPS	java
+bernardolins/clustereasy	go
+CVBDL/EagleEye-App	javascript
+hernantas/MangaReader	php
+k0nserv/Humus	ruby
+paypac/node-concierge	typescript
+jzhu98/telescope	rust
+IFTTT/JazzHands	c
+fhoeben/hsac-fitnesse-fixtures	java
+spacedock-io/registry	go
+weepower/wee-core	javascript
+DNepovim/kraj-praha	php
+ifosch/skyed	ruby
+shiftkey/desktop	typescript
+aptos-labs/aptos-core	rust
+willpatterson/HOPSTACK	c
+Steveice10/MCProtocolLib	java
+go-gl/gl	go
+burningtomatoes/Bitsmashers	javascript
+tmingos/sassy-roots	php
+mirego/hanzo	ruby
+bradyholt/aspnet-core-react-template	typescript
+Logicalshift/tame-tree	rust
+qsctr/vex-4194b-2016	c
+gabrielavara/music-player	java
+ianopolous/go-ipfs	go
+SentinelsOfMagic/SentinelsOfMagic	javascript
+dxw/php-missing	php
+moneyadviceservice/mas-feedback	ruby
+nrlquaker/nfov	typescript
+crumblingstatue/rust-libnotify-sys	rust
+darkbuck/Halide	c
+scottfrederick/spring-boot	java
+firestudios/qor-example	go
+etopian/codebox-package-menubar	javascript
+dkgndec/DrupalConsole	php
+Teino1978-Corp/Teino1978-Corp-kaminari	ruby
+rtsjs/rts	typescript
+fanderzon/rocket-api	rust
+isbadawi/badavi	c
+ryanbrainard/richsobjects	java
+rancher/rancher	go
+jamen/craze	javascript
+unl/UNL_UCBCN_System	php
+joeyates/ruby-sun-times	ruby
+larsvanbraam/vue-transition-component	typescript
+terminalcloud/libnetfilter_queue	rust
+qtmediahub/sasquatch	c
+InnovateUKGitHub/innovation-funding-service	java
+PufferPanel/PufferPanel	go
+npmcomponent/ramitos-midway	javascript
+Radvance/Radvance	php
+sandro/ephemeral_response	ruby
+molstar/molstar	typescript
+spacejam/sled	rust
+AAChartModel/AAChartKit	c
+microprofile/microprofile-conference	java
+monzo/typhon	go
+formidable-coffee/masterfully	javascript
+facile-it/paraunit	php
+zurb/inky-rb	ruby
+dragma/styled-bootstrap-grid	typescript
+musoke/inspirer	rust
+echonest/libechonest	c
+aleph-zero/presto	java
+cloudfoundry-incubator/nsync	go
+MichaelKohler/mozilla-watcher	javascript
+nordsoftware/lumen-core	php
+dtan4/organicat	ruby
+Flatline4/Flatline4	typescript
+Zamicol/Challenge	rust
+ryepdx/keyphrase	c
+robertoschwald/cas	java
+zombiezen/go-log	go
+ONSdigital/florence	javascript
+dweidner/laravel-goutte	php
+pinzolo/redmine_persist_wfmt	ruby
+sheldarr/Votenger	typescript
+adeschamps/lcm	rust
+jpoirier/picoc	c
+Suseika/collections-utils	java
+qasim/what-class-is-this	go
+nissoh/core	javascript
+dafik/CodeGen	php
+braintree/braintree-web-drop-in	ruby
+kawamon/hue	typescript
+GuzTech/tray_rust	rust
+crontab/libobjc2	c
+aNNiMON/JECP	java
+Urethramancer/lbapi	go
+facebook/draft-js	javascript
+alfaproject/omnipay-skrill	php

runs/sixview_v2/head.push_latest.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6674db62c7284efdf40170b3b00dec0159c81bb6bc3081e78028bf434e84e58
+oid sha256:8d9dd24838eff6b9f40342b93de86fbed724bdc8f35eb9a8b579cc869ae384e3
 size 754581395

runs/sixview_v2/metrics.jsonl

@@ -4,3 +4,6 @@
 {"step": 600, "epoch": 0, "end_of_epoch": false, "eval_loss": 2.647629145788428, "suites": {"cr_val": {"eval_loss": 2.647629145788428, "n_docs": 40, "n_tokens": 7696}, "cr_test": {"eval_loss": 2.5539493785098246, "n_docs": 40, "n_tokens": 7924}, "ir_test": {"eval_loss": 2.71551365456855, "n_docs": 37, "n_tokens": 1706}}}
 {"step": 900, "epoch": 0, "end_of_epoch": false, "eval_loss": 2.6266876440766325, "suites": {"cr_val": {"eval_loss": 2.6266876440766325, "n_docs": 40, "n_tokens": 7696}, "cr_test": {"eval_loss": 2.523330387534787, "n_docs": 40, "n_tokens": 7924}, "ir_test": {"eval_loss": 2.7079888427802574, "n_docs": 37, "n_tokens": 1706}}}
 {"step": 1200, "epoch": 0, "end_of_epoch": false, "eval_loss": 2.599496544259378, "suites": {"cr_val": {"eval_loss": 2.599496544259378, "n_docs": 40, "n_tokens": 7696}, "cr_test": {"eval_loss": 2.501433642448529, "n_docs": 40, "n_tokens": 7924}, "ir_test": {"eval_loss": 2.7031463161372913, "n_docs": 37, "n_tokens": 1706}}}
+{"step": 1341, "epoch": 0, "end_of_epoch": true, "eval_loss": 2.606481142984916, "suites": {"cr_val": {"eval_loss": 2.606481142984916, "n_docs": 40, "n_tokens": 7696}, "cr_test": {"eval_loss": 2.4960094789914686, "n_docs": 40, "n_tokens": 7924}, "ir_test": {"eval_loss": 2.644830758169136, "n_docs": 37, "n_tokens": 1706}}}
+{"step": 1500, "epoch": 1, "end_of_epoch": false, "eval_loss": 2.6363723853877947, "suites": {"cr_val": {"eval_loss": 2.6363723853877947, "n_docs": 40, "n_tokens": 7696}, "cr_test": {"eval_loss": 2.521588217604347, "n_docs": 40, "n_tokens": 7924}, "ir_test": {"eval_loss": 2.67671262980346, "n_docs": 37, "n_tokens": 1706}}}
+{"step": 1800, "epoch": 1, "end_of_epoch": false, "eval_loss": 2.632204555261594, "suites": {"cr_val": {"eval_loss": 2.632204555261594, "n_docs": 40, "n_tokens": 7696}, "cr_test": {"eval_loss": 2.508357783750354, "n_docs": 40, "n_tokens": 7924}, "ir_test": {"eval_loss": 2.6669074774906476, "n_docs": 37, "n_tokens": 1706}}}

scripts/generate_repo_scoped_qa_v2.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""ENHANCED repo-scoped QA generator (batch2 / novelty pass).
+
+Same 6-view input as generate_repo_scoped_qa.py, but the prompt asks for a
+BROADER, NOVEL set of judgment aspects that the earlier passes under-covered --
+so the new repos teach the model NEW capabilities, not just more of the same:
+
+  existing aspects : architecture, data_flow, why, contracts, conventions, impact, ops
+  NEW aspects here : debugging (failure modes / how to diagnose),
+                     security (attack surface / trust boundaries / validation),
+                     performance (hot paths / scaling / cost),
+                     concurrency (races / ordering / data consistency),
+                     migration (how to evolve/refactor safely / deprecation),
+                     testing_strategy (what a good test here asserts)
+
+Still Tier-A/B discipline: short judgment answers, NO exact file/line lists.
+Appends to the SAME data/qna/repo_scoped_qa.jsonl (resume-safe by doc_id) so the
+next-day assemble picks it up automatically.
+
+Usage:
+  ./venv/bin/python scripts/generate_repo_scoped_qa_v2.py \
+      --sources data/docs/multiview_sources.jsonl \
+      --model google/gemma-4-31b-it --workers 10 --only-repos data/batch2_novel_repos.txt
+"""
+from __future__ import annotations
+import argparse, hashlib, json, os, re, sys, time
+from pathlib import Path
+from typing import Dict, List
+from openai import OpenAI
+
+HERE = Path(__file__).resolve().parent; REPO_ROOT = HERE.parent
+sys.path.insert(0, str(REPO_ROOT))
+from memory_lora.data_paths import QNA_DIR, DOCS_DIR, CACHE_DIR, ensure_dirs  # noqa: E402
+
+def _load_dotenv(p: Path):
+    if p.exists():
+        for line in p.read_text().splitlines():
+            if "=" in line and not line.strip().startswith("#"):
+                k, v = line.split("=", 1); os.environ.setdefault(k.strip(), v.strip())
+_load_dotenv(REPO_ROOT / ".env")
+MODEL = "google/gemma-4-31b-it"; BASE_URL = "https://openrouter.ai/api/v1"
+
+SYSTEM = (
+ "You are a 20-year staff engineer writing ADVANCED onboarding Q&A for a senior "
+ "engineer joining a repository. You get 6 views: call/import graph, architecture "
+ "(readme/tree), git history, test contracts, code conventions, and ops/build "
+ "config. Produce a JSON array of 10-14 objects with keys 'aspect','question',"
+ "'answer'. aspect in: debugging, security, performance, concurrency, migration, "
+ "testing_strategy, data_flow, impact. Rules: (1) questions are REPO-LEVEL and "
+ "test HARD JUDGMENT a senior would reason about -- 'where would a bug in X most "
+ "likely surface', 'what is the trust boundary / attack surface here', 'what is "
+ "the likely hot path or scaling bottleneck', 'what ordering/consistency "
+ "assumption could break under concurrency', 'how would you evolve Y without "
+ "breaking callers', 'what must a good test here assert'. (2) answers SHORT -- one "
+ "phrase or sentence, copyable, NO file-path or line-number lists. (3) ground "
+ "every answer in the provided views; invent nothing; if a view is thin, reason "
+ "from what IS present. Output ONLY the JSON array."
+)
+
+def ckey(*p): return hashlib.sha256("||".join(p).encode()).hexdigest()[:24]
+
+def gen(client, ctx, cache_dir, retries=4) -> List[Dict]:
+    cf = cache_dir / f"rsqa2_{ckey(MODEL, ctx)}.json"
+    if cf.exists():
+        raw = cf.read_text()
+    else:
+        for a in range(retries):
+            try:
+                r = client.chat.completions.create(model=MODEL,
+                    messages=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": ctx}],
+                    max_tokens=4000, temperature=0.6)
+                raw = r.choices[0].message.content or ""; cf.write_text(raw); break
+            except Exception:  # noqa: BLE001
+                time.sleep(2 ** a)
+        else:
+            return []
+    m = re.search(r"\[.*\]", raw, re.DOTALL)
+    if not m: return []
+    try: items = json.loads(m.group(0))
+    except json.JSONDecodeError: return []
+    out = []
+    for it in items:
+        q = (it.get("question") or "").strip(); a = (it.get("answer") or "").strip()
+        asp = (it.get("aspect") or "general").strip()
+        if q and a and len(a) < 240: out.append({"aspect": asp, "question": q, "answer": a})
+    return out
+
+def build_ctx(repo: str, vt: Dict[str, str]) -> str:
+    parts = [f"REPOSITORY: {repo}\n"]
+    for k in ["v_arch", "v_graph", "v_history", "v_contracts", "v_conventions", "v_ops"]:
+        parts.append(f"=== {k} ===\n{vt.get(k,'')}\n")
+    return "\n".join(parts)
+
+def main():
+    import threading
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--sources", default=str(DOCS_DIR / "multiview_sources.jsonl"))
+    ap.add_argument("--only-repos", default="", help="file of repo\\tlang -- restrict to these repos (the novel batch)")
+    ap.add_argument("--model", default=MODEL)
+    ap.add_argument("--workers", type=int, default=10)
+    args = ap.parse_args()
+    globals()["MODEL"] = args.model
+    ensure_dirs()
+    key = os.environ.get("OPENROUTER_API_KEY")
+    if not key: raise SystemExit("OPENROUTER_API_KEY not set")
+    client = OpenAI(base_url=BASE_URL, api_key=key)
+
+    only = None
+    if args.only_repos and Path(args.only_repos).exists():
+        only = {l.split("\t")[0].split()[0].strip() for l in open(args.only_repos) if l.strip()}
+
+    src = [json.loads(l) for l in open(args.sources)]
+    if only is not None:
+        src = [r for r in src if r["doc_id"] in only or r.get("repo") in only]
+    outp = QNA_DIR / "repo_scoped_qa.jsonl"
+    done = set()
+    if outp.exists():
+        done = {json.loads(l)["doc_id"] for l in open(outp)}
+    todo = [r for r in src if r["doc_id"] not in done]
+    print(f"enhanced QA: {len(todo)} repos to do (of {len(src)} selected)", flush=True)
+    out_f = outp.open("a")
+    lock = threading.Lock()
+    n_docs = [0]; n_qa = [0]; t0 = time.time()
+
+    def work(row):
+        doc_id = row["doc_id"]
+        qas = gen(client, build_ctx(row["repo"], row["view_text"]), CACHE_DIR)
+        if not qas: return
+        with lock:
+            for qa in qas:
+                out_f.write(json.dumps({"doc_id": doc_id, "doc_version": "head",
+                    "split": "train", "qna_split": "train", "aspect": qa["aspect"],
+                    "question": qa["question"], "prefix": f"Q: {qa['question']}\nA:",
+                    "target": " " + qa["answer"], "lang": row.get("lang", "unknown"),
+                    "source": "batch2_novel"}) + "\n")
+                n_qa[0] += 1
+            n_docs[0] += 1
+            if n_docs[0] % 25 == 0:
+                print(f"  {n_docs[0]}/{len(todo)} repos, {n_qa[0]} QA "
+                      f"({n_docs[0]/max(1e-9,(time.time()-t0)/60):.1f}/min)", flush=True)
+
+    with ThreadPoolExecutor(max_workers=args.workers) as ex:
+        for _ in as_completed([ex.submit(work, r) for r in todo]):
+            pass
+    out_f.close()
+    print(f"\nDone. {n_docs[0]} novel repos -> {n_qa[0]} enhanced (debug/security/perf/"
+          f"concurrency/migration) QA.", flush=True)
+
+if __name__ == "__main__":
+    main()

scripts/orchestrate_batch2.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Independent batch2 pipeline: wait for the novel-repo CPU build to finish, then
+# run the ENHANCED QA pass on exactly those repos. Never touches training.
+# Leaves everything staged for a next-day assemble (does NOT re-assemble aligned6).
+set -u
+cd /Users/moncif/gemma4-hack
+BL=runs/build_batch2.log
+QL=runs/qa_batch2.log
+OL=runs/batch2_orchestrator.log
+log(){ echo "$(date '+%F %T'): $*" >> "$OL"; }
+
+log "batch2 orchestrator started; waiting for build to finish"
+# wait for the build process to end
+while pgrep -f "build_repo_multiview.py --repos-file data/batch2_novel_repos.txt" >/dev/null 2>&1; do
+  sleep 30
+done
+log "build finished: $(grep -oE 'Done\. [0-9]+ repos.*' "$BL" | tail -1)"
+
+# run enhanced QA on just the novel batch repos (resume-safe, appends to main qna)
+log "launching enhanced QA (debug/security/perf/concurrency/migration)"
+./venv/bin/python scripts/generate_repo_scoped_qa_v2.py \
+  --sources data/docs/multiview_sources.jsonl \
+  --only-repos data/batch2_novel_repos.txt \
+  --model google/gemma-4-31b-it --workers 10 >> "$QL" 2>&1
+log "enhanced QA done: $(grep -oE 'Done\. [0-9]+ novel repos.*' "$QL" | tail -1)"
+log "batch2 dataset READY for next-day assemble (novel repos in main pool; enhanced QA appended)"