Spaces:
Running
Running
Sync platform from lab mainline: dev instance ready to deploy
Browse filesFull feature set: accounts (email/password + feature-flagged Google via
Supabase PKCE), anonymous tiers (3 runs/day, delete-after-analysis, 24h
purge), saved-run cap, construct upload from CSV/XLSX, structured
warnings + language detection, model registry (MiniLM/E5), per-run
reproduction scripts, corpus-embedding cache, storage interface (local/
R2), sample-data test kit, MANUAL_TESTING.md.
Rights guard: only the 5 original seed constructs ship in this public
repo; the imported lab collection stays out pending the redistribution
decision. HF remote URL detokenized. 64 tests passing.
This view is limited to 50 files because it contains too many changes. See raw diff
- .env.example +56 -0
- .gitignore +1 -0
- DEPLOY.md +36 -68
- Dockerfile +27 -16
- MANUAL_TESTING.md +163 -0
- README.md +40 -78
- backend/app/auth.py +159 -0
- backend/app/auth_google.py +99 -0
- backend/app/ccr.py +57 -27
- backend/app/construct_files.py +110 -0
- backend/app/construct_lib.py +121 -0
- backend/app/db.py +45 -1
- backend/app/ingest.py +54 -32
- backend/app/jobs.py +167 -25
- backend/app/main.py +473 -40
- backend/app/models.py +25 -3
- backend/app/registry.py +134 -0
- backend/app/reproducibility.py +118 -0
- backend/app/retention.py +123 -0
- backend/app/schemas.py +29 -1
- backend/app/seed_constructs.py +0 -66
- backend/app/storage.py +140 -0
- backend/app/warnings_engine.py +151 -0
- backend/requirements.txt +3 -0
- backend/static/assets/index-BUzS6usZ.js +0 -0
- backend/static/assets/index-C356-0ZT.css +0 -1
- backend/static/assets/index-CN_FzJfm.css +1 -0
- backend/static/assets/index-DrFcy6bH.js +0 -0
- backend/static/index.html +3 -3
- backend/tests/conftest.py +3 -0
- backend/tests/test_accounts_limits_retention.py +311 -0
- backend/tests/test_api.py +111 -5
- backend/tests/test_auth_tiers_and_lifecycle.py +138 -0
- backend/tests/test_auto_migration.py +42 -0
- backend/tests/test_ccr.py +1 -1
- backend/tests/test_google_auth.py +89 -0
- backend/tests/test_registry_and_prefixes.py +110 -0
- backend/tests/test_storage_backends.py +154 -0
- frontend/src/App.jsx +280 -34
- frontend/src/ConstructPicker.jsx +206 -0
- frontend/src/ResultsView.jsx +35 -21
- frontend/src/Workspace.jsx +278 -77
- frontend/src/api.js +15 -0
- frontend/src/styles.css +317 -11
- packages/ccr_engine/README.md +11 -0
- packages/construct_library/constructs/collectivism_horizontal.yaml +21 -0
- packages/construct_library/constructs/individualism_horizontal.yaml +21 -0
- packages/construct_library/constructs/mfq_care.yaml +21 -0
- packages/construct_library/constructs/mfq_fairness.yaml +21 -0
- packages/construct_library/constructs/satisfaction_with_life.yaml +25 -0
.env.example
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CCR Platform environment. Everything has a safe local-dev default; the
|
| 2 |
+
# variables under "Deployment" should be set explicitly on a real instance.
|
| 3 |
+
|
| 4 |
+
# ---- Deployment (set these in production) ----
|
| 5 |
+
# REQUIRED in production: sessions survive restarts only with a fixed secret.
|
| 6 |
+
# Generate one: python -c "import secrets; print(secrets.token_hex(32))"
|
| 7 |
+
# CCR_SESSION_SECRET=
|
| 8 |
+
|
| 9 |
+
# Set to 1 when serving over HTTPS (marks cookies Secure).
|
| 10 |
+
# CCR_COOKIE_SECURE=1
|
| 11 |
+
|
| 12 |
+
# Anonymous-data TTL purge in hours. 0 disables (local-dev default);
|
| 13 |
+
# deployments should set 24 (PI retention decision, 2026-07-10).
|
| 14 |
+
# CCR_ANON_TTL_HOURS=24
|
| 15 |
+
|
| 16 |
+
# Preload the default embedding model at startup so the first run is fast.
|
| 17 |
+
# CCR_WARM_MODEL=1
|
| 18 |
+
|
| 19 |
+
# ---- Tiers and limits (defaults shown) ----
|
| 20 |
+
# CCR_ANON_MAX_BYTES=2097152 # 2 MB anonymous upload cap
|
| 21 |
+
# CCR_ANON_MAX_ROWS=500 # anonymous row cap per file
|
| 22 |
+
# CCR_ANON_MAX_RUNS_PER_DAY=3 # anonymous runs/day, then sign-in
|
| 23 |
+
# CCR_USER_MAX_SAVED_RUNS=15 # saved-run cap for signed-in users
|
| 24 |
+
|
| 25 |
+
# ---- Storage and processing ----
|
| 26 |
+
# File storage backend: "local" (default; files under CCR_DATA_DIR) or "s3"
|
| 27 |
+
# (any S3-compatible store; Cloudflare R2 recommended - zero egress fees).
|
| 28 |
+
# The s3 path is production-ready; enabling it is config, not development.
|
| 29 |
+
# CCR_STORAGE=s3
|
| 30 |
+
# CCR_S3_ENDPOINT=https://<accountid>.r2.cloudflarestorage.com
|
| 31 |
+
# CCR_S3_BUCKET=ccr-platform
|
| 32 |
+
# CCR_S3_ACCESS_KEY_ID=
|
| 33 |
+
# CCR_S3_SECRET_ACCESS_KEY=
|
| 34 |
+
|
| 35 |
+
# Where the DB, uploaded corpora, results, and embedding cache live (default: backend/data)
|
| 36 |
+
# CCR_DATA_DIR=/absolute/path
|
| 37 |
+
# Row ceiling for uploads (default 100000; hosted demo uses 20000)
|
| 38 |
+
# CCR_MAX_ROWS=20000
|
| 39 |
+
# Corpus-embedding cache (default on; set 0 to disable)
|
| 40 |
+
# CCR_EMB_CACHE=1
|
| 41 |
+
|
| 42 |
+
# ---- Development / CI only ----
|
| 43 |
+
# Force the deterministic fake embedder (never production)
|
| 44 |
+
# CCR_FAKE_EMBEDDINGS=1
|
| 45 |
+
|
| 46 |
+
# ---- Google sign-in via Supabase (optional; button hidden when unset) ----
|
| 47 |
+
# Supabase dashboard > Project Settings > API. The anon key is public-facing
|
| 48 |
+
# by design; the service_role key is never used and never leaves the dashboard.
|
| 49 |
+
# SUPABASE_URL=https://YOUR_PROJECT_REF.supabase.co
|
| 50 |
+
# SUPABASE_ANON_KEY=
|
| 51 |
+
# Public base URL of this app (Google redirect target).
|
| 52 |
+
# CCR_APP_URL=http://127.0.0.1:8000
|
| 53 |
+
|
| 54 |
+
# ---- Phase 2 (not read yet; reserved names) ----
|
| 55 |
+
# DATABASE_URL=postgresql://...
|
| 56 |
+
# ADMIN_EMAILS=devaanand@umass.edu,matari@umass.edu
|
.gitignore
CHANGED
|
@@ -12,3 +12,4 @@ node_modules/
|
|
| 12 |
|
| 13 |
# os
|
| 14 |
.DS_Store
|
|
|
|
|
|
| 12 |
|
| 13 |
# os
|
| 14 |
.DS_Store
|
| 15 |
+
.env
|
DEPLOY.md
CHANGED
|
@@ -1,81 +1,49 @@
|
|
| 1 |
-
# Deploying the
|
| 2 |
|
| 3 |
-
|
|
|
|
| 4 |
|
| 5 |
-
|
| 6 |
-
CCR online tool lives on HF Spaces, so the prototype sits where the CCR
|
| 7 |
-
community already works.
|
| 8 |
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
-
|
| 13 |
-
|
| 14 |
|
| 15 |
-
|
| 16 |
-
---
|
| 17 |
-
title: CCR Platform
|
| 18 |
-
emoji: 🧭
|
| 19 |
-
colorFrom: red
|
| 20 |
-
colorTo: gray
|
| 21 |
-
sdk: docker
|
| 22 |
-
app_port: 7860
|
| 23 |
-
pinned: false
|
| 24 |
-
---
|
| 25 |
-
```
|
| 26 |
|
| 27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
-
|
| 30 |
-
git remote add hf https://huggingface.co/spaces/<your-username>/ccr-platform
|
| 31 |
-
git push hf main
|
| 32 |
-
```
|
| 33 |
-
|
| 34 |
-
First build takes ~5–10 min (model bakes into the image). Watch the
|
| 35 |
-
build logs in the Space's "Logs" tab.
|
| 36 |
-
|
| 37 |
-
4. Optional hardening for the public instance — in Space Settings →
|
| 38 |
-
Variables, set `CCR_MAX_ROWS=20000` (tighter ceiling than the
|
| 39 |
-
100k default while strangers can reach it).
|
| 40 |
-
|
| 41 |
-
Notes:
|
| 42 |
-
- Storage is **ephemeral** — uploads/results vanish on restart or rebuild.
|
| 43 |
-
Fine for a demo; the email and the in-app welcome text both say so.
|
| 44 |
-
- Free Spaces sleep after ~48h without traffic. Visit the URL the
|
| 45 |
-
evening before and the morning of the interview so it's warm.
|
| 46 |
-
- The direct app URL (no HF frame) is
|
| 47 |
-
`https://<username>-ccr-platform.hf.space` — send that one.
|
| 48 |
-
|
| 49 |
-
## Option B — Google Cloud Run (few dollars, more "prod-like" URL)
|
| 50 |
|
| 51 |
```bash
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
--region us-central1 \
|
| 55 |
-
--allow-unauthenticated \
|
| 56 |
-
--memory 2Gi \
|
| 57 |
-
--cpu 2 \
|
| 58 |
-
--min-instances 1 \
|
| 59 |
-
--max-instances 1 \
|
| 60 |
-
--concurrency 20
|
| 61 |
```
|
| 62 |
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
- `--max-instances 1`: SQLite + in-process queue assume one instance —
|
| 66 |
-
documented demo trade-off, not an oversight.
|
| 67 |
-
|
| 68 |
-
## Pre-send checklist (either host)
|
| 69 |
|
| 70 |
-
|
| 71 |
-
2. Full run: new project → upload `sample_data/sample_corpus.csv` →
|
| 72 |
-
Satisfaction with Life → Run → results render → Export CSV downloads.
|
| 73 |
-
3. Second run (Individualism) finishes in seconds (model + item cache warm).
|
| 74 |
-
4. Upload your own messy CSV (Excel export with a BOM, or semicolon-
|
| 75 |
-
delimited) — parses, and any fallback is flagged in the UI.
|
| 76 |
-
5. Upload rejects a bogus file (.txt/.exe) with a clean error.
|
| 77 |
-
6. Refresh the page — SPA loads, project still listed.
|
| 78 |
-
7. Morning of the interview: open the URL once (warm the instance),
|
| 79 |
-
re-run step 2 quickly.
|
| 80 |
|
| 81 |
-
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Deploying the dev instance (Hugging Face Space)
|
| 2 |
|
| 3 |
+
The Space builds from this repo's Dockerfile. One-time setup lives in the
|
| 4 |
+
Space settings; after that, deploys are just `git push hf main`.
|
| 5 |
|
| 6 |
+
## Space secrets (Settings > Variables and secrets)
|
|
|
|
|
|
|
| 7 |
|
| 8 |
+
| Secret | Value |
|
| 9 |
+
|---|---|
|
| 10 |
+
| CCR_SESSION_SECRET | `python3 -c "import secrets; print(secrets.token_hex(32))"` |
|
| 11 |
+
| SUPABASE_URL | from Supabase > Project Settings > API |
|
| 12 |
+
| SUPABASE_ANON_KEY | from the same page (anon public key, NOT service_role) |
|
| 13 |
+
| CCR_APP_URL | https://devaanand-ccr-platform.hf.space |
|
| 14 |
+
| CCR_COOKIE_SECURE | 1 |
|
| 15 |
|
| 16 |
+
Retention (CCR_ANON_TTL_HOURS=24) and model pre-warm are already defaults in
|
| 17 |
+
the Dockerfile.
|
| 18 |
|
| 19 |
+
## Supabase setup for Google sign-in (one time)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
+
1. supabase.com > New project (free tier).
|
| 22 |
+
2. Authentication > Providers > Google > Enable. Copy the shown callback URL
|
| 23 |
+
(https://PROJECT_REF.supabase.co/auth/v1/callback).
|
| 24 |
+
3. console.cloud.google.com > OAuth consent screen (External) > Credentials >
|
| 25 |
+
Create OAuth client ID (Web application) > add the Supabase callback URL as
|
| 26 |
+
an authorized redirect URI. Paste client id/secret back into the Supabase
|
| 27 |
+
Google provider form.
|
| 28 |
+
4. Authentication > URL Configuration: add BOTH redirect URLs:
|
| 29 |
+
- http://127.0.0.1:8000/api/auth/google/callback
|
| 30 |
+
- https://devaanand-ccr-platform.hf.space/api/auth/google/callback
|
| 31 |
+
5. Project Settings > API: copy the Project URL and anon key into the Space
|
| 32 |
+
secrets (and your local .env).
|
| 33 |
|
| 34 |
+
## Push
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
```bash
|
| 37 |
+
git push origin main # GitHub
|
| 38 |
+
git push hf main # Hugging Face Space (rebuilds + redeploys)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
```
|
| 40 |
|
| 41 |
+
The hf remote has no stored token; use your HF username and a WRITE token as
|
| 42 |
+
the password when prompted (or a credential helper).
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
+
## Caveats of the free dev instance
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
+
- Ephemeral disk: SQLite resets on rebuild/restart. Google users are recreated
|
| 47 |
+
on next sign-in automatically; password accounts must re-register. Fine for
|
| 48 |
+
feedback; a persistent volume or Postgres arrives with the launch decision.
|
| 49 |
+
- The Space sleeps after ~48 h idle; first visit wakes it (~1 min).
|
Dockerfile
CHANGED
|
@@ -1,35 +1,46 @@
|
|
| 1 |
-
# CCR Platform
|
| 2 |
-
#
|
| 3 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
FROM python:3.11-slim
|
| 6 |
|
| 7 |
-
#
|
| 8 |
-
#
|
|
|
|
| 9 |
ENV PYTHONUNBUFFERED=1 \
|
| 10 |
HF_HOME=/opt/hf-cache \
|
| 11 |
-
CCR_DATA_DIR=/tmp/ccr-data
|
|
|
|
|
|
|
| 12 |
|
| 13 |
WORKDIR /srv
|
| 14 |
|
| 15 |
COPY backend/requirements.txt .
|
| 16 |
RUN pip install --no-cache-dir -r requirements.txt
|
| 17 |
|
| 18 |
-
#
|
| 19 |
-
#
|
| 20 |
-
# (
|
|
|
|
| 21 |
RUN python -c "from sentence_transformers import SentenceTransformer; \
|
| 22 |
-
|
| 23 |
-
'sentence-transformers/all-MiniLM-L6-v2', \
|
| 24 |
-
'sentence-transformers/all-mpnet-base-v2', \
|
| 25 |
-
'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2')]" \
|
| 26 |
&& chmod -R 777 /opt/hf-cache
|
| 27 |
|
| 28 |
COPY backend/app ./app
|
| 29 |
COPY backend/static ./static
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
-
# Demo note: SQLite + uploads live on the container's ephemeral disk —
|
| 32 |
-
# data resets on restart/redeploy. Acceptable for a demo; use
|
| 33 |
-
# Postgres + S3/GCS object storage before any real use.
|
| 34 |
EXPOSE 7860
|
| 35 |
CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-7860}"]
|
|
|
|
| 1 |
+
# CCR Platform - single-container deployment (one deployable unit: FastAPI
|
| 2 |
+
# serves both the JSON API and the prebuilt React SPA from backend/static).
|
| 3 |
+
#
|
| 4 |
+
# Build: docker build -t ccr-platform .
|
| 5 |
+
# Ephemeral demo: docker run -p 7860:7860 ccr-platform
|
| 6 |
+
# Persistent: docker run -p 7860:7860 \
|
| 7 |
+
# -e CCR_SESSION_SECRET=$(python -c "import secrets;print(secrets.token_hex(32))") \
|
| 8 |
+
# -e CCR_DATA_DIR=/data -e CCR_COOKIE_SECURE=1 \
|
| 9 |
+
# -v ccr_data:/data ccr-platform
|
| 10 |
+
#
|
| 11 |
+
# NOTE: run `npm run build` in frontend/ before building the image - the
|
| 12 |
+
# committed backend/static is what ships (no Node stage; keeps HF Spaces
|
| 13 |
+
# builds fast and the image small).
|
| 14 |
|
| 15 |
FROM python:3.11-slim
|
| 16 |
|
| 17 |
+
# Defaults favor a hosted instance: retention purge on, model pre-warmed.
|
| 18 |
+
# Data dir defaults to /tmp so the container runs under any UID (HF Spaces);
|
| 19 |
+
# persistent deployments override CCR_DATA_DIR to a mounted volume.
|
| 20 |
ENV PYTHONUNBUFFERED=1 \
|
| 21 |
HF_HOME=/opt/hf-cache \
|
| 22 |
+
CCR_DATA_DIR=/tmp/ccr-data \
|
| 23 |
+
CCR_ANON_TTL_HOURS=24 \
|
| 24 |
+
CCR_WARM_MODEL=1
|
| 25 |
|
| 26 |
WORKDIR /srv
|
| 27 |
|
| 28 |
COPY backend/requirements.txt .
|
| 29 |
RUN pip install --no-cache-dir -r requirements.txt
|
| 30 |
|
| 31 |
+
# Bake the DEFAULT model (MiniLM, ~90 MB - the CCR reference model) into the
|
| 32 |
+
# image so the first run never stalls on a download. The E5 models are large
|
| 33 |
+
# (1+ GB) and lazy-load into HF_HOME on first use instead; the dir stays
|
| 34 |
+
# writable for any runtime UID.
|
| 35 |
RUN python -c "from sentence_transformers import SentenceTransformer; \
|
| 36 |
+
SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')" \
|
|
|
|
|
|
|
|
|
|
| 37 |
&& chmod -R 777 /opt/hf-cache
|
| 38 |
|
| 39 |
COPY backend/app ./app
|
| 40 |
COPY backend/static ./static
|
| 41 |
+
# registry.py/construct_lib.py resolve packages/ two levels above app/
|
| 42 |
+
# (= "/" here), so /packages is exactly where they look.
|
| 43 |
+
COPY packages /packages
|
| 44 |
|
|
|
|
|
|
|
|
|
|
| 45 |
EXPOSE 7860
|
| 46 |
CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-7860}"]
|
MANUAL_TESTING.md
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Manual testing guide
|
| 2 |
+
|
| 3 |
+
Everything built so far, as click-through scenarios. Each scenario says what to
|
| 4 |
+
do and exactly what you should see. Files referenced live in `sample_data/`
|
| 5 |
+
(see `sample_data/README.md` for what each one triggers).
|
| 6 |
+
|
| 7 |
+
## 0. Setup
|
| 8 |
+
|
| 9 |
+
```bash
|
| 10 |
+
cd backend
|
| 11 |
+
pip install -r requirements.txt
|
| 12 |
+
uvicorn app.main:app --reload --port 8000
|
| 13 |
+
```
|
| 14 |
+
|
| 15 |
+
Open http://127.0.0.1:8000. First start downloads MiniLM (~90 MB) on the first
|
| 16 |
+
real run; set `CCR_WARM_MODEL=1` to preload it at startup instead.
|
| 17 |
+
|
| 18 |
+
To test quickly without models: `CCR_FAKE_EMBEDDINGS=1 uvicorn ...` (scores are
|
| 19 |
+
fake but every flow works; never use for real analysis).
|
| 20 |
+
|
| 21 |
+
## 1. Projects and sidebar
|
| 22 |
+
|
| 23 |
+
1. Create three projects. They appear under "Today", newest activity first.
|
| 24 |
+
2. Type in the sidebar search box: list filters as you type.
|
| 25 |
+
3. Archive a project (project header > Archive): it moves into the collapsed
|
| 26 |
+
"Archived" group; Unarchive brings it back. No data is lost either way.
|
| 27 |
+
4. Delete a project: requires typing the project name; removes its datasets,
|
| 28 |
+
runs, and files permanently.
|
| 29 |
+
|
| 30 |
+
## 2. Upload paths (Step 1 card)
|
| 31 |
+
|
| 32 |
+
| Upload | Expect |
|
| 33 |
+
|---|---|
|
| 34 |
+
| `sample_corpus.csv` | Parses, 60 rows, `text` column suggested |
|
| 35 |
+
| `multi_column_demo.csv` | 5 columns; `comment_text` marked "(suggested)" |
|
| 36 |
+
| `semicolon_delimited_demo.csv` | Parses into exactly id + text (commas inside texts intact) |
|
| 37 |
+
| `latin1_encoding_demo.csv` | Parses with a ⚠ note: decoded as latin-1; fiancée/café render correctly |
|
| 38 |
+
| `xlsx_upload_demo.xlsx` | Parses like a CSV |
|
| 39 |
+
| a `.txt` or `.pdf` file | Rejected: unsupported file type |
|
| 40 |
+
|
| 41 |
+
Anonymous limits (signed out): the Step 1 hint shows 2 MB / 500 rows and says
|
| 42 |
+
uploads are deleted after analysis. Upload `large_demo.csv` (800 rows): rejected
|
| 43 |
+
with a "Sign in (top right)" message. Sign in and retry: accepted.
|
| 44 |
+
|
| 45 |
+
## 3. Construct selection (Step 2 card)
|
| 46 |
+
|
| 47 |
+
1. Open the picker: search field + panel below it, library grouped by category,
|
| 48 |
+
with "Recently used" pinned on top after your first runs.
|
| 49 |
+
2. Type "GAD" or "empathy": matches by name and category; Arrow keys + Enter work.
|
| 50 |
+
3. Select any imported construct: items listed, plus the "not yet verified
|
| 51 |
+
verbatim" notice (expected for the whole imported library for now).
|
| 52 |
+
|
| 53 |
+
### Custom construct, typed
|
| 54 |
+
|
| 55 |
+
1. "+ Custom construct" > name it, paste items one per line.
|
| 56 |
+
2. Append `(R)` to one line: the form shows "1 item(s) marked reverse-scored".
|
| 57 |
+
3. Save: it appears in the picker under "My custom constructs"; run metadata
|
| 58 |
+
will carry the reverse flag (check via Results > metadata download).
|
| 59 |
+
|
| 60 |
+
### Custom construct, from file (new)
|
| 61 |
+
|
| 62 |
+
1. "+ Custom construct" > "Upload items from CSV/XLSX".
|
| 63 |
+
2. Try a CSV with `item,reverse` columns (1/true/yes/R = reverse) or a
|
| 64 |
+
single-column file with `(R)` markers.
|
| 65 |
+
3. Expect: items fill the textarea ((R) appended where flagged), the filename
|
| 66 |
+
becomes the suggested name, and parse notes list skipped blanks/duplicates.
|
| 67 |
+
Nothing is saved until you review and press Save. Item files are never
|
| 68 |
+
retained on the server.
|
| 69 |
+
|
| 70 |
+
## 4. Language, models, and warnings (Step 3 card + results)
|
| 71 |
+
|
| 72 |
+
Run each of these and open the results page; the amber warnings panel should
|
| 73 |
+
show exactly:
|
| 74 |
+
|
| 75 |
+
| Corpus | Selection | Expected warnings |
|
| 76 |
+
|---|---|---|
|
| 77 |
+
| `warnings_showcase.csv` | en + MiniLM | EMPTY_ROWS_DROPPED (2), DUPLICATE_TEXTS (2), TEXT_TOO_SHORT (3), TEXTS_MAYBE_TRUNCATED (2); no language warnings |
|
| 78 |
+
| `french_demo.csv` | en + MiniLM | LANGUAGE_MISMATCH (detected fr, 100%) |
|
| 79 |
+
| `french_demo.csv` | fr + MiniLM | MODEL_LANGUAGE_UNSUPPORTED |
|
| 80 |
+
| `french_demo.csv` | fr + Multilingual E5 | no language warnings |
|
| 81 |
+
| `mixed_language_demo.csv` | en + MiniLM | LANGUAGE_UNCERTAIN (majority 50%) |
|
| 82 |
+
| `long_documents_demo.csv` | en + MiniLM | TEXTS_MAYBE_TRUNCATED (4) + LANGUAGE_UNCERTAIN (only 10 rows, below the 20-row minimum - by design) |
|
| 83 |
+
|
| 84 |
+
Warnings are per-run snapshots: changing language/model requires a NEW run;
|
| 85 |
+
old result pages don't update.
|
| 86 |
+
|
| 87 |
+
## 5. Results and reproducibility
|
| 88 |
+
|
| 89 |
+
1. Run `moral_foundations_demo.csv` against two different MFQ-2 foundations:
|
| 90 |
+
top texts change per foundation; the 6 neutral rows sink to the bottom.
|
| 91 |
+
2. Results page: histogram, mean/SD/min/max, per-item loadings, top/bottom texts.
|
| 92 |
+
3. Downloads: results CSV (input columns + sim_item_N + ccr_score), metadata
|
| 93 |
+
JSON (model revision, construct snapshot + item hash, language block,
|
| 94 |
+
environment pins), reproduction script + requirements file.
|
| 95 |
+
4. Reproduction check: `pip install -r requirements-repro.txt`, then
|
| 96 |
+
`python reproduce_analysis.py your_corpus.csv` on a machine with no platform
|
| 97 |
+
access; values should match the export (target ~1e-5 with real models).
|
| 98 |
+
|
| 99 |
+
## 6. Accounts
|
| 100 |
+
|
| 101 |
+
1. Sign in (top right) > "Create a free account" > email + password (min 8 chars).
|
| 102 |
+
2. You're signed in immediately; header shows your name.
|
| 103 |
+
3. Sign out, sign back in; wrong password gives "Incorrect email or password";
|
| 104 |
+
registering the same email again gives "already exists".
|
| 105 |
+
4. Email is case-insensitive. There is no self-service password reset yet
|
| 106 |
+
(interim local accounts; Google/Supabase swap planned) - reset = admin action.
|
| 107 |
+
|
| 108 |
+
## 7. Anonymous tiers (test signed OUT)
|
| 109 |
+
|
| 110 |
+
1. Upload caps: see section 2.
|
| 111 |
+
2. Run limit: run 3 analyses (default). The Step 3 card counts "X of 3 free
|
| 112 |
+
runs used today". The 4th run is refused with a sign-in prompt (HTTP 429).
|
| 113 |
+
Counter resets next day (UTC). Signing in removes the limit.
|
| 114 |
+
3. Delete-after-analysis: run any corpus, open results (fine, downloadable),
|
| 115 |
+
note the info warning "uploaded file was deleted after this analysis".
|
| 116 |
+
Re-running that same corpus: refused ("upload again, or sign in").
|
| 117 |
+
4. TTL purge: with `CCR_ANON_TTL_HOURS=24` (deployment default; 0 = off in
|
| 118 |
+
local dev), anonymous projects older than 24h are deleted entirely,
|
| 119 |
+
startup + hourly.
|
| 120 |
+
|
| 121 |
+
## 8. Signed-in tier
|
| 122 |
+
|
| 123 |
+
1. Sign in, upload, run: no ANONYMOUS_DATA_REMOVED warning; re-running the same
|
| 124 |
+
corpus works (file kept).
|
| 125 |
+
2. Saved-run cap: Step 3 card shows "N of 15 saved runs used". At the cap, new
|
| 126 |
+
runs are refused until you delete old runs/projects (nothing is auto-deleted).
|
| 127 |
+
3. Ownership: your projects are invisible to signed-out visitors and other
|
| 128 |
+
accounts (they get 403 on any modification). Anonymous projects stay shared.
|
| 129 |
+
|
| 130 |
+
## 9. Performance behaviors
|
| 131 |
+
|
| 132 |
+
1. Corpus-embedding cache: run the SAME corpus with a second construct
|
| 133 |
+
(signed in, same model): the run skips document embedding and completes in
|
| 134 |
+
seconds; metadata shows `"doc_embeddings_from_cache": true`.
|
| 135 |
+
2. Duplicate texts are embedded once (`warnings_showcase.csv` has 2 dupes):
|
| 136 |
+
identical scores for identical texts, less compute.
|
| 137 |
+
3. API responses are gzip-compressed (check the response headers).
|
| 138 |
+
|
| 139 |
+
## 10. Robustness
|
| 140 |
+
|
| 141 |
+
1. Restart the server mid-run: the orphaned job is marked failed with an
|
| 142 |
+
explanation, never stuck at "running".
|
| 143 |
+
2. A DB from an older version gains new columns automatically at startup
|
| 144 |
+
(additive auto-migration) - no more "no such column" 500s.
|
| 145 |
+
3. Tampered session cookie = treated as signed out, no error.
|
| 146 |
+
|
| 147 |
+
## 11. Deployment (container)
|
| 148 |
+
|
| 149 |
+
```bash
|
| 150 |
+
cd frontend && npm run build && cd ..
|
| 151 |
+
docker build -t ccr-platform .
|
| 152 |
+
docker run -p 7860:7860 \
|
| 153 |
+
-e CCR_SESSION_SECRET=$(python3 -c "import secrets;print(secrets.token_hex(32))") \
|
| 154 |
+
-e CCR_DATA_DIR=/data -e CCR_COOKIE_SECURE=1 \
|
| 155 |
+
-v ccr_data:/data ccr-platform
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
Checklist before giving the URL to real users:
|
| 159 |
+
- [ ] `CCR_SESSION_SECRET` set (sessions survive restarts)
|
| 160 |
+
- [ ] `CCR_COOKIE_SECURE=1` (HTTPS only)
|
| 161 |
+
- [ ] `CCR_DATA_DIR` on a persistent volume (default /tmp is ephemeral)
|
| 162 |
+
- [ ] `CCR_ANON_TTL_HOURS=24` (default in the image)
|
| 163 |
+
- [ ] Smoke test: sections 2, 4, 6, 7 above
|
README.md
CHANGED
|
@@ -10,87 +10,49 @@ pinned: false
|
|
| 10 |
|
| 11 |
# CCR Platform
|
| 12 |
|
| 13 |
-
A web platform for **Contextualized Construct Representations (CCR)**
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
```bash
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
**Try it:** create a project → upload `sample_data/sample_corpus.csv` (60 synthetic texts) → choose *Satisfaction with Life* → Run. Then re-run the same corpus against *Individualism* vs *Collectivism* and compare the top-scoring texts.
|
| 36 |
-
|
| 37 |
-
## What the platform adds over the existing CCR tools
|
| 38 |
-
|
| 39 |
-
The published [R/Python packages](https://github.com/Ali-Omrani/CCR) and the single-run web demo cover one-off analyses. This platform adds the workflow around the method: **projects** that persist corpora and runs, a **construct library** of validated scales (plus custom constructs), **async jobs** with live progress on large corpora, a **results dashboard** (score distribution, per-item loadings, highest/lowest-scoring texts for face-validity checks), **CSV export** in the same shape as `ccr_wrapper` output, and a **reproducibility record** (model + version, item hash, package versions, timestamps) downloadable per run.
|
| 40 |
-
|
| 41 |
-
## Architecture
|
| 42 |
-
|
| 43 |
-
```
|
| 44 |
-
Browser (React SPA, prebuilt → served by FastAPI)
|
| 45 |
-
│ REST /api/*
|
| 46 |
-
FastAPI (backend/app/main.py)
|
| 47 |
-
│── SQLite (projects, corpora, constructs, jobs) backend/data/ccr.db
|
| 48 |
-
│── File storage (uploaded corpora, result CSVs) backend/data/
|
| 49 |
-
└── Background jobs (backend/app/jobs.py)
|
| 50 |
-
└── CCR engine (backend/app/ccr.py)
|
| 51 |
-
└── sentence-transformers (local, pinned)
|
| 52 |
```
|
| 53 |
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
| Database | SQLite | Right-sized for single-node, few writers. Schema is Postgres-portable; the upgrade trigger is concurrent multi-user writes. |
|
| 58 |
-
| Job execution | FastAPI `BackgroundTasks`, state in DB | Zero extra infrastructure; lab-scale corpora embed in seconds–minutes. Known limits (no restart survival, no retries) are accepted MVP trade-offs; upgrade trigger to Celery+Redis is long/frequent jobs — the API contract doesn't change because job state already lives in the DB. |
|
| 59 |
-
| Model dependency | Injected `EmbeddingBackend` interface | Tests/CI run a deterministic hash embedder (no torch), so the full pipeline is testable in seconds. Production backend is swappable per job. |
|
| 60 |
-
| Frontend | React (Vite), served as static files by the API | Single deployable, no CORS in production, no Node required to run. |
|
| 61 |
|
| 62 |
-
##
|
| 63 |
-
|
| 64 |
-
Research files are messy, so ingestion is tolerant by design: encoding fallback (UTF-8 with BOM → latin-1, with a user-facing note when fallback was needed), delimiter sniffing (`,` `;` tab `|`), ragged-row skipping, a configurable row ceiling (`CCR_MAX_ROWS`, default 100k), and a text-column suggestion heuristic. The exact parse configuration (format, encoding, delimiter) is stored per corpus and echoed into each run's reproducibility record. Runs execute on a dedicated worker queue with persisted state — jobs orphaned by a restart are marked failed with an explanation instead of hanging — and results carry data-quality notes (empty rows dropped, duplicates detected, texts likely truncated by the model's token window) so silent data issues become visible ones.
|
| 65 |
-
|
| 66 |
-
## Reproducibility & data handling
|
| 67 |
-
|
| 68 |
-
Every run records: model name + `sentence-transformers` version, embedding dimension, SHA-256 of the exact item wordings, text column, row counts (including empty rows dropped), timestamps, and library versions — downloadable as JSON next to the results CSV. Exports mirror the `ccr_wrapper` output shape (input columns + `sim_item_i` + `ccr_score`) so they drop into existing CCR workflows.
|
| 69 |
-
|
| 70 |
-
Processing is self-contained: embeddings are computed on the server running the app — text is never sent to third-party AI APIs. Run locally (`./run.sh`) and corpora never leave your machine, which is the recommended mode for sensitive data. Uploaded corpora and results live in the data directory (`backend/data/` locally; ephemeral on the hosted demo, which may reset at any time — don't upload sensitive or identifiable data there).
|
| 71 |
-
|
| 72 |
-
## Construct library — verify before research use
|
| 73 |
-
|
| 74 |
-
The seeded scales (SWLS; MFQ Care & Fairness; Triandis & Gelfand Individualism/Collectivism) carry citations, but item wordings must be **verified verbatim against the original publications before research use** — CCR's validity rests on using the validated instrument as published.
|
| 75 |
-
|
| 76 |
-
## Known limitations / roadmap
|
| 77 |
-
|
| 78 |
-
- **Method nuance:** cosine similarity captures construct *relatedness* more than stance — a text lamenting life dissatisfaction can sit near SWLS items in embedding space. Reverse-scored items need care, and results should be validated against human-annotated subsets for new constructs/corpora.
|
| 79 |
-
- No auth/multi-user yet (single-lab, local deployment); add before any public hosting, along with per-user quotas.
|
| 80 |
-
- `BackgroundTasks` → Celery+Redis when corpora grow; SQLite → Postgres with multi-user concurrency; local files → S3/GCS if deployed off-machine.
|
| 81 |
-
- Embedding cache keyed on (model, item-set hash) to make repeated runs on the same construct instant.
|
| 82 |
-
|
| 83 |
-
## Development
|
| 84 |
-
|
| 85 |
-
```bash
|
| 86 |
-
# backend tests (fast — no ML deps needed)
|
| 87 |
-
cd backend && pip install -r requirements-dev.txt && python -m pytest tests/ -q
|
| 88 |
-
|
| 89 |
-
# frontend dev server (proxies /api to :8000)
|
| 90 |
-
cd frontend && npm install && npm run dev
|
| 91 |
-
|
| 92 |
-
# rebuild the shipped dashboard
|
| 93 |
-
cd frontend && npm run build # outputs to backend/static/
|
| 94 |
-
```
|
| 95 |
|
| 96 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
# CCR Platform
|
| 12 |
|
| 13 |
+
A web platform for **Contextualized Construct Representations (CCR)** - theory-driven
|
| 14 |
+
psychological text analysis ([Atari, Omrani, et al.](https://github.com/Ali-Omrani/CCR);
|
| 15 |
+
[EMNLP 2024](https://aclanthology.org/2024.emnlp-main.151/)). Built for the Culture and
|
| 16 |
+
Morality Lab (UMass Amherst); this instance is the lab's **dev/testing environment**.
|
| 17 |
+
|
| 18 |
+
Upload a corpus (CSV/XLSX), pick a validated construct from the library or define your
|
| 19 |
+
own (typed or uploaded from a file), choose a language and embedding model, run the
|
| 20 |
+
analysis, inspect results (distributions, per-item loadings, top/bottom texts,
|
| 21 |
+
data-quality warnings), and export everything - including a Python script that
|
| 22 |
+
reproduces the run on any machine.
|
| 23 |
+
|
| 24 |
+
## Features
|
| 25 |
+
|
| 26 |
+
- Anonymous try-it tier: 3 runs/day, uploads deleted right after analysis, sessions
|
| 27 |
+
purged after 24 h. Free accounts (email/password, optional Google sign-in) lift
|
| 28 |
+
limits and keep your work (15 saved runs).
|
| 29 |
+
- Construct library (versioned, append-only, item-hashed) + custom constructs with
|
| 30 |
+
reverse-scored flags; searchable grouped picker.
|
| 31 |
+
- Model registry: MiniLM default (the CCR reference model), E5-large-v2,
|
| 32 |
+
Multilingual-E5; E5 prefix policy handled automatically; language coverage warnings.
|
| 33 |
+
- Structured data-quality warnings (language mismatch/uncertainty, short texts,
|
| 34 |
+
truncation, duplicates, encoding fallback) - stable machine-readable codes.
|
| 35 |
+
- Per-run reproducibility: metadata JSON + offline-runnable script + pinned
|
| 36 |
+
requirements. Corpus-embedding cache makes re-runs on the same corpus near-instant.
|
| 37 |
+
- Storage: local disk by default; S3-compatible (Cloudflare R2) via env config.
|
| 38 |
+
|
| 39 |
+
## Run locally
|
| 40 |
|
| 41 |
```bash
|
| 42 |
+
cd backend
|
| 43 |
+
pip install -r requirements.txt
|
| 44 |
+
uvicorn app.main:app --reload --port 8000 --env-file ../.env
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
```
|
| 46 |
|
| 47 |
+
Open http://127.0.0.1:8000. See `MANUAL_TESTING.md` for a full click-through test
|
| 48 |
+
script and `sample_data/README.md` for what each sample file demonstrates.
|
| 49 |
+
Configuration: copy `.env.example` to `.env` and fill what you need.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
+
## Notes
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
+
- Do not upload sensitive or identifiable data to this shared dev instance; anonymous
|
| 54 |
+
storage is ephemeral and the instance may reset.
|
| 55 |
+
- The construct library here carries the 5 original seed scales; the lab's full
|
| 56 |
+
imported collection ships separately pending a redistribution-rights decision.
|
| 57 |
+
- Tests: `cd backend && CCR_FAKE_EMBEDDINGS=1 python -m pytest -q` (64 tests, no ML
|
| 58 |
+
downloads needed).
|
backend/app/auth.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Accounts, sessions, and usage tiers.
|
| 2 |
+
|
| 3 |
+
Local email+password accounts - the "best cheap option available now" (Deva,
|
| 4 |
+
2026-07-11): zero external dependencies, zero cost, real password security via
|
| 5 |
+
stdlib scrypt. This deliberately does NOT implement email verification or
|
| 6 |
+
self-service password reset; at lab scale a reset is an admin action. The
|
| 7 |
+
managed-provider swap (Supabase: Google + email/password, design doc §8)
|
| 8 |
+
replaces token creation/verification here - get_current_user() stays the only
|
| 9 |
+
integration point the rest of the app knows about.
|
| 10 |
+
|
| 11 |
+
Sessions: HMAC-signed cookie carrying {uid, email, name}. Secret from
|
| 12 |
+
CCR_SESSION_SECRET (REQUIRED in production - random per process otherwise,
|
| 13 |
+
which signs everyone out on restart).
|
| 14 |
+
|
| 15 |
+
Anonymous usage tiers (PI decisions, 2026-07-10):
|
| 16 |
+
* upload caps (bytes/rows),
|
| 17 |
+
* run limit per day (signed cookie counter - a nudge toward accounts, not a
|
| 18 |
+
security boundary; clearing cookies evades it and that is acceptable),
|
| 19 |
+
* data removed after analysis (see retention.py).
|
| 20 |
+
Signed-in users: caps lifted, runs persist up to a saved-run cap.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import base64
|
| 26 |
+
import hashlib
|
| 27 |
+
import hmac
|
| 28 |
+
import json
|
| 29 |
+
import os
|
| 30 |
+
import re
|
| 31 |
+
import secrets
|
| 32 |
+
from datetime import datetime, timezone
|
| 33 |
+
|
| 34 |
+
from fastapi import Request
|
| 35 |
+
|
| 36 |
+
COOKIE_NAME = "ccr_session"
|
| 37 |
+
RUNS_COOKIE_NAME = "ccr_runs"
|
| 38 |
+
_SECRET = (os.environ.get("CCR_SESSION_SECRET") or secrets.token_hex(32)).encode()
|
| 39 |
+
|
| 40 |
+
ANON_MAX_BYTES_DEFAULT = 2 * 1024 * 1024
|
| 41 |
+
ANON_MAX_ROWS_DEFAULT = 500
|
| 42 |
+
ANON_MAX_RUNS_PER_DAY_DEFAULT = 3
|
| 43 |
+
USER_MAX_SAVED_RUNS_DEFAULT = 15
|
| 44 |
+
ANON_TTL_HOURS_DEFAULT = 0 # 0 = purge disabled (local dev); deployments set 24
|
| 45 |
+
|
| 46 |
+
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
| 47 |
+
MIN_PASSWORD_LEN = 8
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
# ------------------------------------------------------------- env knobs
|
| 51 |
+
def anon_max_bytes() -> int:
|
| 52 |
+
return int(os.environ.get("CCR_ANON_MAX_BYTES", ANON_MAX_BYTES_DEFAULT))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def anon_max_rows() -> int:
|
| 56 |
+
return int(os.environ.get("CCR_ANON_MAX_ROWS", ANON_MAX_ROWS_DEFAULT))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def anon_max_runs_per_day() -> int:
|
| 60 |
+
return int(os.environ.get("CCR_ANON_MAX_RUNS_PER_DAY", ANON_MAX_RUNS_PER_DAY_DEFAULT))
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def user_max_saved_runs() -> int:
|
| 64 |
+
return int(os.environ.get("CCR_USER_MAX_SAVED_RUNS", USER_MAX_SAVED_RUNS_DEFAULT))
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def anon_ttl_hours() -> int:
|
| 68 |
+
return int(os.environ.get("CCR_ANON_TTL_HOURS", ANON_TTL_HOURS_DEFAULT))
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def cookies_secure() -> bool:
|
| 72 |
+
"""Set CCR_COOKIE_SECURE=1 behind HTTPS in production."""
|
| 73 |
+
return os.environ.get("CCR_COOKIE_SECURE") == "1"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# ---------------------------------------------------------- passwords
|
| 77 |
+
def hash_password(password: str) -> str:
|
| 78 |
+
salt = secrets.token_bytes(16)
|
| 79 |
+
digest = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=64)
|
| 80 |
+
return f"scrypt${salt.hex()}${digest.hex()}"
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def verify_password(password: str, stored: str) -> bool:
|
| 84 |
+
try:
|
| 85 |
+
algo, salt_hex, digest_hex = stored.split("$")
|
| 86 |
+
if algo != "scrypt":
|
| 87 |
+
return False
|
| 88 |
+
digest = hashlib.scrypt(
|
| 89 |
+
password.encode(), salt=bytes.fromhex(salt_hex), n=16384, r=8, p=1, dklen=64
|
| 90 |
+
)
|
| 91 |
+
return hmac.compare_digest(digest.hex(), digest_hex)
|
| 92 |
+
except Exception:
|
| 93 |
+
return False
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def valid_email(email: str) -> bool:
|
| 97 |
+
return bool(_EMAIL_RE.match(email.strip().lower()))
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ------------------------------------------------- signed cookie payloads
|
| 101 |
+
def _sign(payload: bytes) -> str:
|
| 102 |
+
return hmac.new(_SECRET, payload, hashlib.sha256).hexdigest()
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def sign_payload(data: dict) -> str:
|
| 106 |
+
payload = base64.urlsafe_b64encode(json.dumps(data, separators=(",", ":")).encode()).decode()
|
| 107 |
+
return f"{payload}.{_sign(payload.encode())}"
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def verify_payload(token: str | None) -> dict | None:
|
| 111 |
+
if not token or "." not in token:
|
| 112 |
+
return None
|
| 113 |
+
payload, signature = token.rsplit(".", 1)
|
| 114 |
+
if not hmac.compare_digest(signature, _sign(payload.encode())):
|
| 115 |
+
return None
|
| 116 |
+
try:
|
| 117 |
+
data = json.loads(base64.urlsafe_b64decode(payload.encode()).decode())
|
| 118 |
+
return data if isinstance(data, dict) else None
|
| 119 |
+
except Exception:
|
| 120 |
+
return None
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
# ------------------------------------------------------------- sessions
|
| 124 |
+
def create_session_token(user_id: str, email: str, name: str) -> str:
|
| 125 |
+
return sign_payload({"uid": user_id, "email": email, "name": name})
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def get_current_user(request: Request) -> dict | None:
|
| 129 |
+
"""THE auth integration point (design doc §8). A managed provider (Supabase)
|
| 130 |
+
replaces this body with provider-session verification; callers only ever see
|
| 131 |
+
{"id", "email", "name", "tier"} or None."""
|
| 132 |
+
data = verify_payload(request.cookies.get(COOKIE_NAME))
|
| 133 |
+
if not data or "uid" not in data:
|
| 134 |
+
return None
|
| 135 |
+
return {
|
| 136 |
+
"id": data["uid"],
|
| 137 |
+
"email": data.get("email", ""),
|
| 138 |
+
"name": data.get("name", ""),
|
| 139 |
+
"tier": "member",
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
# ------------------------------------------- anonymous daily run counter
|
| 144 |
+
def _today() -> str:
|
| 145 |
+
return datetime.now(timezone.utc).date().isoformat()
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def runs_used_today(request: Request) -> int:
|
| 149 |
+
data = verify_payload(request.cookies.get(RUNS_COOKIE_NAME))
|
| 150 |
+
if not data or data.get("d") != _today():
|
| 151 |
+
return 0 # missing, tampered, or from a previous day - counter resets
|
| 152 |
+
try:
|
| 153 |
+
return max(0, int(data.get("n", 0)))
|
| 154 |
+
except (TypeError, ValueError):
|
| 155 |
+
return 0
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def run_counter_token(count: int) -> str:
|
| 159 |
+
return sign_payload({"d": _today(), "n": int(count)})
|
backend/app/auth_google.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Google sign-in via Supabase Auth (server-side PKCE flow).
|
| 2 |
+
|
| 3 |
+
Feature-flagged: everything here is inert until SUPABASE_URL and
|
| 4 |
+
SUPABASE_ANON_KEY are set, so local dev and tests run unchanged without any
|
| 5 |
+
Supabase project. When configured, the flow is:
|
| 6 |
+
|
| 7 |
+
1. GET /api/auth/google/login -> redirect to Supabase's Google authorize
|
| 8 |
+
URL with a PKCE challenge; the verifier rides in a short-lived signed
|
| 9 |
+
cookie (never stored server-side).
|
| 10 |
+
2. Google -> Supabase -> GET /api/auth/google/callback?code=...
|
| 11 |
+
3. The backend exchanges code+verifier for the Supabase user (stdlib
|
| 12 |
+
urllib - no new dependencies), finds-or-creates a local User row by
|
| 13 |
+
email, and issues OUR normal session cookie (auth.py).
|
| 14 |
+
|
| 15 |
+
Design consequence: Supabase verifies identity at sign-in time only; the
|
| 16 |
+
session, tiers, and ownership model are exactly the same as email/password
|
| 17 |
+
accounts. Google users have an empty password_hash and cannot password-login
|
| 18 |
+
(a clear message says to use Google). Because users are re-created on next
|
| 19 |
+
sign-in by email, an ephemeral-disk dev instance losing its SQLite file is a
|
| 20 |
+
nuisance, not a lockout.
|
| 21 |
+
|
| 22 |
+
No frontend SDK: the button is a plain link, keeping the react+react-dom-only
|
| 23 |
+
dependency rule intact.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
from __future__ import annotations
|
| 27 |
+
|
| 28 |
+
import base64
|
| 29 |
+
import hashlib
|
| 30 |
+
import json
|
| 31 |
+
import os
|
| 32 |
+
import secrets
|
| 33 |
+
import urllib.error
|
| 34 |
+
import urllib.parse
|
| 35 |
+
import urllib.request
|
| 36 |
+
|
| 37 |
+
VERIFIER_COOKIE = "ccr_pkce"
|
| 38 |
+
VERIFIER_TTL_SECONDS = 600
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def configured() -> bool:
|
| 42 |
+
return bool(os.environ.get("SUPABASE_URL") and os.environ.get("SUPABASE_ANON_KEY"))
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _supabase_url() -> str:
|
| 46 |
+
return os.environ["SUPABASE_URL"].rstrip("/")
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def app_url() -> str:
|
| 50 |
+
"""Public base URL of THIS app (redirect target). Local default matches
|
| 51 |
+
the dev server; deployments set CCR_APP_URL."""
|
| 52 |
+
return os.environ.get("CCR_APP_URL", "http://127.0.0.1:8000").rstrip("/")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def begin() -> tuple[str, str]:
|
| 56 |
+
"""Return (authorize_url, code_verifier)."""
|
| 57 |
+
verifier = secrets.token_urlsafe(64)
|
| 58 |
+
challenge = (
|
| 59 |
+
base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
|
| 60 |
+
.decode()
|
| 61 |
+
.rstrip("=")
|
| 62 |
+
)
|
| 63 |
+
params = urllib.parse.urlencode(
|
| 64 |
+
{
|
| 65 |
+
"provider": "google",
|
| 66 |
+
"redirect_to": f"{app_url()}/api/auth/google/callback",
|
| 67 |
+
"code_challenge": challenge,
|
| 68 |
+
"code_challenge_method": "s256",
|
| 69 |
+
}
|
| 70 |
+
)
|
| 71 |
+
return f"{_supabase_url()}/auth/v1/authorize?{params}", verifier
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def exchange(code: str, verifier: str) -> dict:
|
| 75 |
+
"""Exchange the PKCE code for the Supabase user. Returns {email, name}.
|
| 76 |
+
Raises ValueError with a user-safe message on any failure."""
|
| 77 |
+
body = json.dumps({"auth_code": code, "code_verifier": verifier}).encode()
|
| 78 |
+
req = urllib.request.Request(
|
| 79 |
+
f"{_supabase_url()}/auth/v1/token?grant_type=pkce",
|
| 80 |
+
data=body,
|
| 81 |
+
headers={
|
| 82 |
+
"apikey": os.environ["SUPABASE_ANON_KEY"],
|
| 83 |
+
"Content-Type": "application/json",
|
| 84 |
+
},
|
| 85 |
+
method="POST",
|
| 86 |
+
)
|
| 87 |
+
try:
|
| 88 |
+
with urllib.request.urlopen(req, timeout=15) as resp:
|
| 89 |
+
payload = json.load(resp)
|
| 90 |
+
except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError) as exc:
|
| 91 |
+
raise ValueError("Google sign-in could not be completed. Please try again.") from exc
|
| 92 |
+
|
| 93 |
+
user = payload.get("user") or {}
|
| 94 |
+
email = (user.get("email") or "").strip().lower()
|
| 95 |
+
if not email:
|
| 96 |
+
raise ValueError("Google sign-in returned no email address.")
|
| 97 |
+
meta = user.get("user_metadata") or {}
|
| 98 |
+
name = (meta.get("full_name") or meta.get("name") or email.split("@")[0]).strip()
|
| 99 |
+
return {"email": email, "name": name}
|
backend/app/ccr.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""CCR engine
|
| 2 |
|
| 3 |
Method (Atari, Omrani et al.): embed validated questionnaire items and the
|
| 4 |
texts to be analyzed with a contextual sentence-embedding model, then take
|
|
@@ -7,7 +7,7 @@ similarities are the text's "loadings" on the construct; their mean is the
|
|
| 7 |
overall CCR score.
|
| 8 |
|
| 9 |
The embedding model is injected behind a small interface so that:
|
| 10 |
-
* production uses sentence-transformers (local, pinned, reproducible
|
| 11 |
corpora never leave the machine), and
|
| 12 |
* tests/CI use a deterministic hash-based embedder with no ML dependency.
|
| 13 |
"""
|
|
@@ -28,21 +28,6 @@ ProgressCb = Callable[[float], None]
|
|
| 28 |
|
| 29 |
FAKE_MODEL_NAME = "fake-deterministic"
|
| 30 |
|
| 31 |
-
AVAILABLE_MODELS = [
|
| 32 |
-
{
|
| 33 |
-
"name": "sentence-transformers/all-MiniLM-L6-v2",
|
| 34 |
-
"label": "all-MiniLM-L6-v2 (default — fast, CCR reference model)",
|
| 35 |
-
},
|
| 36 |
-
{
|
| 37 |
-
"name": "sentence-transformers/all-mpnet-base-v2",
|
| 38 |
-
"label": "all-mpnet-base-v2 (higher quality, slower)",
|
| 39 |
-
},
|
| 40 |
-
{
|
| 41 |
-
"name": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
|
| 42 |
-
"label": "paraphrase-multilingual-MiniLM-L12-v2 (50+ languages)",
|
| 43 |
-
},
|
| 44 |
-
]
|
| 45 |
-
|
| 46 |
|
| 47 |
class EmbeddingBackend(Protocol):
|
| 48 |
name: str
|
|
@@ -57,14 +42,16 @@ class SentenceTransformerBackend:
|
|
| 57 |
|
| 58 |
_cache: dict[str, object] = {}
|
| 59 |
|
| 60 |
-
def __init__(self, model_name: str):
|
| 61 |
self.name = model_name
|
|
|
|
| 62 |
|
| 63 |
def _model(self):
|
| 64 |
if self.name not in self._cache:
|
| 65 |
from sentence_transformers import SentenceTransformer # lazy: heavy import
|
| 66 |
|
| 67 |
-
self.
|
|
|
|
| 68 |
return self._cache[self.name]
|
| 69 |
|
| 70 |
@property
|
|
@@ -121,10 +108,14 @@ class HashEmbeddingBackend:
|
|
| 121 |
return out
|
| 122 |
|
| 123 |
|
| 124 |
-
def get_backend(
|
| 125 |
-
|
|
|
|
| 126 |
return HashEmbeddingBackend()
|
| 127 |
-
|
|
|
|
|
|
|
|
|
|
| 128 |
|
| 129 |
|
| 130 |
@dataclass
|
|
@@ -132,10 +123,31 @@ class CCRResult:
|
|
| 132 |
similarities: np.ndarray # (n_docs, n_items)
|
| 133 |
scores: np.ndarray # (n_docs,) mean over items
|
| 134 |
metadata: dict
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
|
| 136 |
|
| 137 |
# Item-set embeddings are tiny and constantly reused (same construct run
|
| 138 |
-
# against many corpora)
|
| 139 |
_item_embedding_cache: dict[tuple[str, str], np.ndarray] = {}
|
| 140 |
|
| 141 |
|
|
@@ -154,8 +166,16 @@ def run_ccr(
|
|
| 154 |
items: list[str],
|
| 155 |
backend: EmbeddingBackend,
|
| 156 |
progress_cb: ProgressCb | None = None,
|
|
|
|
|
|
|
|
|
|
| 157 |
) -> CCRResult:
|
| 158 |
-
"""Compute CCR loadings: cosine(text, item) for every text × item pair.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
if not texts:
|
| 160 |
raise ValueError("Corpus contains no non-empty texts.")
|
| 161 |
if not items:
|
|
@@ -163,7 +183,8 @@ def run_ccr(
|
|
| 163 |
|
| 164 |
started = datetime.now(timezone.utc)
|
| 165 |
|
| 166 |
-
|
|
|
|
| 167 |
if progress_cb:
|
| 168 |
progress_cb(0.02)
|
| 169 |
|
|
@@ -171,7 +192,13 @@ def run_ccr(
|
|
| 171 |
if progress_cb:
|
| 172 |
progress_cb(0.02 + 0.93 * frac)
|
| 173 |
|
| 174 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
|
| 176 |
# Both matrices are L2-normalized -> cosine similarity is a dot product.
|
| 177 |
sims = doc_emb @ item_emb.T
|
|
@@ -186,9 +213,12 @@ def run_ccr(
|
|
| 186 |
"embedding_dim": int(doc_emb.shape[1]),
|
| 187 |
"model_max_seq_length": getattr(backend, "max_seq_length", None),
|
| 188 |
"item_embeddings_from_cache": items_cached,
|
|
|
|
| 189 |
"n_texts": len(texts),
|
| 190 |
"n_items": len(items),
|
| 191 |
"items_sha256_16": items_hash,
|
|
|
|
|
|
|
| 192 |
"similarity": "cosine",
|
| 193 |
"score": "mean of per-item cosine similarities",
|
| 194 |
"python": sys.version.split()[0],
|
|
@@ -206,4 +236,4 @@ def run_ccr(
|
|
| 206 |
|
| 207 |
if progress_cb:
|
| 208 |
progress_cb(0.97)
|
| 209 |
-
return CCRResult(similarities=sims, scores=scores, metadata=metadata)
|
|
|
|
| 1 |
+
"""CCR engine - Contextualized Construct Representations.
|
| 2 |
|
| 3 |
Method (Atari, Omrani et al.): embed validated questionnaire items and the
|
| 4 |
texts to be analyzed with a contextual sentence-embedding model, then take
|
|
|
|
| 7 |
overall CCR score.
|
| 8 |
|
| 9 |
The embedding model is injected behind a small interface so that:
|
| 10 |
+
* production uses sentence-transformers (local, pinned, reproducible -
|
| 11 |
corpora never leave the machine), and
|
| 12 |
* tests/CI use a deterministic hash-based embedder with no ML dependency.
|
| 13 |
"""
|
|
|
|
| 28 |
|
| 29 |
FAKE_MODEL_NAME = "fake-deterministic"
|
| 30 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
class EmbeddingBackend(Protocol):
|
| 33 |
name: str
|
|
|
|
| 42 |
|
| 43 |
_cache: dict[str, object] = {}
|
| 44 |
|
| 45 |
+
def __init__(self, model_name: str, revision: str | None = None):
|
| 46 |
self.name = model_name
|
| 47 |
+
self.revision = revision
|
| 48 |
|
| 49 |
def _model(self):
|
| 50 |
if self.name not in self._cache:
|
| 51 |
from sentence_transformers import SentenceTransformer # lazy: heavy import
|
| 52 |
|
| 53 |
+
kwargs = {"revision": self.revision} if self.revision else {}
|
| 54 |
+
self._cache[self.name] = SentenceTransformer(self.name, **kwargs)
|
| 55 |
return self._cache[self.name]
|
| 56 |
|
| 57 |
@property
|
|
|
|
| 108 |
return out
|
| 109 |
|
| 110 |
|
| 111 |
+
def get_backend(model_id: str) -> EmbeddingBackend:
|
| 112 |
+
"""Resolve a REGISTRY model id (or the test fake) to an embedding backend."""
|
| 113 |
+
if model_id == FAKE_MODEL_NAME or os.environ.get("CCR_FAKE_EMBEDDINGS") == "1":
|
| 114 |
return HashEmbeddingBackend()
|
| 115 |
+
from . import registry # local import: engine stays importable without yaml deps
|
| 116 |
+
|
| 117 |
+
cfg = registry.get_model(model_id)
|
| 118 |
+
return SentenceTransformerBackend(cfg.provider_model_id, revision=cfg.pinned_revision)
|
| 119 |
|
| 120 |
|
| 121 |
@dataclass
|
|
|
|
| 123 |
similarities: np.ndarray # (n_docs, n_items)
|
| 124 |
scores: np.ndarray # (n_docs,) mean over items
|
| 125 |
metadata: dict
|
| 126 |
+
doc_embeddings: np.ndarray | None = None # exposed so jobs.py can cache them
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def encode_unique(backend: EmbeddingBackend, texts: list[str],
|
| 130 |
+
progress_cb: ProgressCb | None = None) -> np.ndarray:
|
| 131 |
+
"""Encode only unique texts, then scatter back to full row order.
|
| 132 |
+
|
| 133 |
+
Duplicate rows are common in social-media corpora; embeddings are
|
| 134 |
+
deterministic per text, so encoding each unique text once is a pure
|
| 135 |
+
speedup with bit-identical output.
|
| 136 |
+
"""
|
| 137 |
+
unique: dict[str, int] = {}
|
| 138 |
+
for t in texts:
|
| 139 |
+
if t not in unique:
|
| 140 |
+
unique[t] = len(unique)
|
| 141 |
+
if len(unique) == len(texts):
|
| 142 |
+
return backend.encode(texts, progress_cb=progress_cb)
|
| 143 |
+
unique_texts = list(unique.keys())
|
| 144 |
+
unique_emb = backend.encode(unique_texts, progress_cb=progress_cb)
|
| 145 |
+
idx = np.fromiter((unique[t] for t in texts), dtype=np.int64, count=len(texts))
|
| 146 |
+
return unique_emb[idx]
|
| 147 |
|
| 148 |
|
| 149 |
# Item-set embeddings are tiny and constantly reused (same construct run
|
| 150 |
+
# against many corpora) - cache them per (model, exact item wording).
|
| 151 |
_item_embedding_cache: dict[tuple[str, str], np.ndarray] = {}
|
| 152 |
|
| 153 |
|
|
|
|
| 166 |
items: list[str],
|
| 167 |
backend: EmbeddingBackend,
|
| 168 |
progress_cb: ProgressCb | None = None,
|
| 169 |
+
item_prefix: str = "",
|
| 170 |
+
text_prefix: str = "",
|
| 171 |
+
doc_embeddings: np.ndarray | None = None,
|
| 172 |
) -> CCRResult:
|
| 173 |
+
"""Compute CCR loadings: cosine(text, item) for every text × item pair.
|
| 174 |
+
|
| 175 |
+
Prefixes come from the model registry's usage_config - E5-family models require
|
| 176 |
+
"query: " on BOTH sides for symmetric similarity. Prefixed strings feed the
|
| 177 |
+
encoder only; raw wording is what gets hashed and exported.
|
| 178 |
+
"""
|
| 179 |
if not texts:
|
| 180 |
raise ValueError("Corpus contains no non-empty texts.")
|
| 181 |
if not items:
|
|
|
|
| 183 |
|
| 184 |
started = datetime.now(timezone.utc)
|
| 185 |
|
| 186 |
+
items_for_encoding = [item_prefix + i for i in items] if item_prefix else items
|
| 187 |
+
item_emb, items_cached = encode_items_cached(backend, items_for_encoding)
|
| 188 |
if progress_cb:
|
| 189 |
progress_cb(0.02)
|
| 190 |
|
|
|
|
| 192 |
if progress_cb:
|
| 193 |
progress_cb(0.02 + 0.93 * frac)
|
| 194 |
|
| 195 |
+
embeddings_from_cache = doc_embeddings is not None and len(doc_embeddings) == len(texts)
|
| 196 |
+
if embeddings_from_cache:
|
| 197 |
+
doc_emb = doc_embeddings # precomputed for this exact corpus+model+prefix (jobs.py cache)
|
| 198 |
+
doc_progress(1.0)
|
| 199 |
+
else:
|
| 200 |
+
texts_for_encoding = [text_prefix + t for t in texts] if text_prefix else texts
|
| 201 |
+
doc_emb = encode_unique(backend, texts_for_encoding, progress_cb=doc_progress)
|
| 202 |
|
| 203 |
# Both matrices are L2-normalized -> cosine similarity is a dot product.
|
| 204 |
sims = doc_emb @ item_emb.T
|
|
|
|
| 213 |
"embedding_dim": int(doc_emb.shape[1]),
|
| 214 |
"model_max_seq_length": getattr(backend, "max_seq_length", None),
|
| 215 |
"item_embeddings_from_cache": items_cached,
|
| 216 |
+
"doc_embeddings_from_cache": embeddings_from_cache,
|
| 217 |
"n_texts": len(texts),
|
| 218 |
"n_items": len(items),
|
| 219 |
"items_sha256_16": items_hash,
|
| 220 |
+
"item_prefix": item_prefix,
|
| 221 |
+
"text_prefix": text_prefix,
|
| 222 |
"similarity": "cosine",
|
| 223 |
"score": "mean of per-item cosine similarities",
|
| 224 |
"python": sys.version.split()[0],
|
|
|
|
| 236 |
|
| 237 |
if progress_cb:
|
| 238 |
progress_cb(0.97)
|
| 239 |
+
return CCRResult(similarities=sims, scores=scores, metadata=metadata, doc_embeddings=doc_emb)
|
backend/app/construct_files.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Parse a construct's items from an uploaded CSV/XLSX file.
|
| 2 |
+
|
| 3 |
+
Design (Deva, 2026-07-11): parse -> preview -> confirm. The file is parsed
|
| 4 |
+
into items + reverse flags and returned for the researcher to REVIEW AND EDIT
|
| 5 |
+
before saving - never silently imported, because in CCR the item wording IS
|
| 6 |
+
the instrument.
|
| 7 |
+
|
| 8 |
+
Accepted shapes (tolerant, reusing the corpus ingest loaders):
|
| 9 |
+
* an "item" / "items" / "text" / "statement" / "question" column (case-
|
| 10 |
+
insensitive), else a single-column file, else the longest-string column;
|
| 11 |
+
* optional reverse-scoring either as a column ("reverse", "reversed",
|
| 12 |
+
"reverse_scored", "rev", "r"; truthy = 1/true/yes/y/r) or as a trailing
|
| 13 |
+
"(R)" / "(rev)" / "(reversed)" marker in the item text (the lab's own
|
| 14 |
+
spreadsheet convention - packages/construct_library/import_from_xlsx.py);
|
| 15 |
+
* blank rows dropped, exact duplicates dropped with a warning.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import re
|
| 21 |
+
|
| 22 |
+
import pandas as pd
|
| 23 |
+
|
| 24 |
+
from .ingest import IngestError, load_corpus
|
| 25 |
+
|
| 26 |
+
ITEM_COLUMNS = ("item", "items", "text", "statement", "question", "item_text")
|
| 27 |
+
REVERSE_COLUMNS = ("reverse", "reversed", "reverse_scored", "reverse-scored", "rev", "r")
|
| 28 |
+
TRUTHY = {"1", "true", "yes", "y", "r", "reverse", "reversed"}
|
| 29 |
+
REVERSE_MARKER = re.compile(r"\s*\((r|rev|reversed)\)\s*$", re.IGNORECASE)
|
| 30 |
+
MAX_ITEMS = 200
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def parse_construct_file(path: str) -> dict:
|
| 34 |
+
"""Return {items: [{text, reverse_scored}], warnings: [str], source_column: str}."""
|
| 35 |
+
try:
|
| 36 |
+
df, _info = load_corpus(path)
|
| 37 |
+
except IngestError as exc:
|
| 38 |
+
raise ValueError(str(exc)) from exc
|
| 39 |
+
if df.empty:
|
| 40 |
+
raise ValueError("The file contains no rows.")
|
| 41 |
+
|
| 42 |
+
lower = {str(c).strip().lower(): c for c in df.columns}
|
| 43 |
+
|
| 44 |
+
item_col = next((lower[c] for c in ITEM_COLUMNS if c in lower), None)
|
| 45 |
+
if item_col is None:
|
| 46 |
+
if len(df.columns) == 1:
|
| 47 |
+
item_col = df.columns[0]
|
| 48 |
+
else: # longest average string wins - same heuristic family as corpora
|
| 49 |
+
def avg_len(col):
|
| 50 |
+
s = df[col].astype("string").dropna()
|
| 51 |
+
return s.str.len().mean() if len(s) else 0
|
| 52 |
+
item_col = max(df.columns, key=avg_len)
|
| 53 |
+
|
| 54 |
+
reverse_col = next((lower[c] for c in REVERSE_COLUMNS if c in lower), None)
|
| 55 |
+
if reverse_col == item_col:
|
| 56 |
+
reverse_col = None
|
| 57 |
+
|
| 58 |
+
warnings: list[str] = []
|
| 59 |
+
items: list[dict] = []
|
| 60 |
+
seen: set[str] = set()
|
| 61 |
+
n_blank = n_dupes = 0
|
| 62 |
+
|
| 63 |
+
for _, row in df.iterrows():
|
| 64 |
+
raw = row[item_col]
|
| 65 |
+
text = "" if pd.isna(raw) else str(raw).strip()
|
| 66 |
+
if not text:
|
| 67 |
+
n_blank += 1
|
| 68 |
+
continue
|
| 69 |
+
|
| 70 |
+
reverse = False
|
| 71 |
+
if reverse_col is not None:
|
| 72 |
+
flag = row[reverse_col]
|
| 73 |
+
if not pd.isna(flag):
|
| 74 |
+
s = str(flag).strip().lower()
|
| 75 |
+
try: # pandas floats an int column containing blanks: 1 -> "1.0"
|
| 76 |
+
reverse = float(s) != 0
|
| 77 |
+
except ValueError:
|
| 78 |
+
reverse = s in TRUTHY
|
| 79 |
+
if REVERSE_MARKER.search(text):
|
| 80 |
+
reverse = True
|
| 81 |
+
text = REVERSE_MARKER.sub("", text).strip()
|
| 82 |
+
|
| 83 |
+
if text in seen:
|
| 84 |
+
n_dupes += 1
|
| 85 |
+
continue
|
| 86 |
+
seen.add(text)
|
| 87 |
+
items.append({"text": text, "reverse_scored": reverse})
|
| 88 |
+
|
| 89 |
+
if not items:
|
| 90 |
+
raise ValueError(f"No usable items found in column '{item_col}'.")
|
| 91 |
+
if len(items) > MAX_ITEMS:
|
| 92 |
+
raise ValueError(
|
| 93 |
+
f"{len(items)} items found; a construct is capped at {MAX_ITEMS}. "
|
| 94 |
+
"If this file holds multiple scales, split it per construct."
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
if n_blank:
|
| 98 |
+
warnings.append(f"{n_blank} blank row(s) skipped.")
|
| 99 |
+
if n_dupes:
|
| 100 |
+
warnings.append(f"{n_dupes} duplicate item(s) skipped.")
|
| 101 |
+
if reverse_col is None and not any(i["reverse_scored"] for i in items):
|
| 102 |
+
warnings.append(
|
| 103 |
+
"No reverse-scoring information found. Mark reverse-scored items by "
|
| 104 |
+
"appending (R) to the item text, or include a 'reverse' column."
|
| 105 |
+
)
|
| 106 |
+
warnings.append(
|
| 107 |
+
"Review each item against the original publication before research use - "
|
| 108 |
+
"the item wording IS the instrument."
|
| 109 |
+
)
|
| 110 |
+
return {"items": items, "warnings": warnings, "source_column": str(item_col)}
|
backend/app/construct_lib.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Construct library loader - seeds the DB from packages/construct_library/constructs/.
|
| 2 |
+
|
| 3 |
+
Source of truth is the versioned YAML files (spec 0004, design doc §10.1). Rules:
|
| 4 |
+
* append-only: (construct_id, version) is immutable - same version with changed
|
| 5 |
+
items is a hard error, never a silent update;
|
| 6 |
+
* item_hash uses the REFERENCE implementation from validate_constructs.py (loaded
|
| 7 |
+
by file path) so validator, seeder, and metadata always agree;
|
| 8 |
+
* verification_status flows to the UI - unverified wording is visibly flagged.
|
| 9 |
+
|
| 10 |
+
New questionnaires from the lab land as new YAML files; `python packages/construct_library/
|
| 11 |
+
validate_constructs.py` first, then restart the app (or call sync) to pick them up.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import importlib.util
|
| 17 |
+
import json
|
| 18 |
+
import logging
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
import yaml
|
| 22 |
+
from sqlalchemy.orm import Session
|
| 23 |
+
|
| 24 |
+
from .models import Construct
|
| 25 |
+
|
| 26 |
+
logger = logging.getLogger("ccr.constructs")
|
| 27 |
+
|
| 28 |
+
REPO_ROOT = Path(__file__).resolve().parents[2]
|
| 29 |
+
CONSTRUCTS_DIR = REPO_ROOT / "packages" / "construct_library" / "constructs"
|
| 30 |
+
_VALIDATOR_PY = REPO_ROOT / "packages" / "construct_library" / "validate_constructs.py"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _reference_item_hash():
|
| 34 |
+
spec = importlib.util.spec_from_file_location("ccr_construct_validator", _VALIDATOR_PY)
|
| 35 |
+
module = importlib.util.module_from_spec(spec)
|
| 36 |
+
spec.loader.exec_module(module)
|
| 37 |
+
return module.item_hash
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def load_yaml_constructs() -> list[dict]:
|
| 41 |
+
files = sorted(CONSTRUCTS_DIR.glob("*.yaml"))
|
| 42 |
+
out = []
|
| 43 |
+
for f in files:
|
| 44 |
+
data = yaml.safe_load(f.read_text())
|
| 45 |
+
data["_file"] = f.name
|
| 46 |
+
out.append(data)
|
| 47 |
+
return out
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def sync_library(db: Session) -> dict:
|
| 51 |
+
"""Idempotent seed/update of library constructs. Returns a small report."""
|
| 52 |
+
item_hash = _reference_item_hash()
|
| 53 |
+
report = {"inserted": 0, "unchanged": 0, "errors": []}
|
| 54 |
+
|
| 55 |
+
for c in load_yaml_constructs():
|
| 56 |
+
slug, version = c["construct_id"], int(c["version"])
|
| 57 |
+
computed_hash = item_hash(c)
|
| 58 |
+
|
| 59 |
+
existing = (
|
| 60 |
+
db.query(Construct)
|
| 61 |
+
.filter_by(construct_slug=slug, version=version, is_seed=True)
|
| 62 |
+
.one_or_none()
|
| 63 |
+
)
|
| 64 |
+
if existing:
|
| 65 |
+
if existing.item_hash != computed_hash:
|
| 66 |
+
# Append-only violation: same version, different wording. Refuse loudly.
|
| 67 |
+
report["errors"].append(
|
| 68 |
+
f"{c['_file']}: items changed under existing version {version} "
|
| 69 |
+
f"(hash {existing.item_hash[:12]} -> {computed_hash[:12]}). "
|
| 70 |
+
"Create a NEW version instead of editing this one."
|
| 71 |
+
)
|
| 72 |
+
else:
|
| 73 |
+
report["unchanged"] += 1
|
| 74 |
+
continue
|
| 75 |
+
|
| 76 |
+
db.add(
|
| 77 |
+
Construct(
|
| 78 |
+
name=c["name"],
|
| 79 |
+
description=c.get("description", ""),
|
| 80 |
+
reference=c.get("citation", ""),
|
| 81 |
+
items_json=json.dumps([str(i["text"]) for i in c["items"]]),
|
| 82 |
+
reverse_flags_json=json.dumps([bool(i.get("reverse_scored", False)) for i in c["items"]]),
|
| 83 |
+
is_seed=True,
|
| 84 |
+
construct_slug=slug,
|
| 85 |
+
version=version,
|
| 86 |
+
item_hash=computed_hash,
|
| 87 |
+
verification_status=c.get("verification_status", "needs_verification"),
|
| 88 |
+
language=c.get("language", "en"),
|
| 89 |
+
category=c.get("category", ""),
|
| 90 |
+
)
|
| 91 |
+
)
|
| 92 |
+
report["inserted"] += 1
|
| 93 |
+
|
| 94 |
+
db.commit()
|
| 95 |
+
if report["errors"]:
|
| 96 |
+
for e in report["errors"]:
|
| 97 |
+
logger.error("construct library: %s", e)
|
| 98 |
+
raise RuntimeError(
|
| 99 |
+
"Construct library append-only violation(s): " + " | ".join(report["errors"])
|
| 100 |
+
)
|
| 101 |
+
logger.info("construct library sync: %s", report)
|
| 102 |
+
return report
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def construct_snapshot(construct: Construct) -> dict:
|
| 106 |
+
"""Immutable snapshot embedded in every run's metadata (design §10.1)."""
|
| 107 |
+
items = json.loads(construct.items_json)
|
| 108 |
+
flags = json.loads(construct.reverse_flags_json or "[]") or [False] * len(items)
|
| 109 |
+
return {
|
| 110 |
+
"construct_id": construct.construct_slug or f"custom_{construct.id[:8]}",
|
| 111 |
+
"version": construct.version or 1,
|
| 112 |
+
"name": construct.name,
|
| 113 |
+
"language": construct.language or "en",
|
| 114 |
+
"items": [
|
| 115 |
+
{"text": t, "reverse_scored": bool(f)} for t, f in zip(items, flags)
|
| 116 |
+
],
|
| 117 |
+
"item_hash": construct.item_hash or "",
|
| 118 |
+
"citation": construct.reference or "",
|
| 119 |
+
"verification_status": construct.verification_status or "draft",
|
| 120 |
+
"source_type": "predefined" if construct.is_seed else "user_custom",
|
| 121 |
+
}
|
backend/app/db.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Database setup
|
| 2 |
|
| 3 |
SQLite is a deliberate choice for this deployment size (single-node, few
|
| 4 |
concurrent writers). The models use no SQLite-specific features, so moving
|
|
@@ -54,3 +54,47 @@ def get_db():
|
|
| 54 |
yield db
|
| 55 |
finally:
|
| 56 |
db.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Database setup - SQLite via SQLAlchemy.
|
| 2 |
|
| 3 |
SQLite is a deliberate choice for this deployment size (single-node, few
|
| 4 |
concurrent writers). The models use no SQLite-specific features, so moving
|
|
|
|
| 54 |
yield db
|
| 55 |
finally:
|
| 56 |
db.close()
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _sqlite_literal(value) -> str:
|
| 60 |
+
if isinstance(value, bool):
|
| 61 |
+
return "1" if value else "0"
|
| 62 |
+
if isinstance(value, (int, float)):
|
| 63 |
+
return str(value)
|
| 64 |
+
return "'" + str(value).replace("'", "''") + "'"
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def auto_migrate_sqlite(target_engine, metadata) -> list[str]:
|
| 68 |
+
"""Add ORM columns missing from existing SQLite tables (additive only).
|
| 69 |
+
|
| 70 |
+
create_all() creates missing tables but never alters existing ones, so a
|
| 71 |
+
dev DB from last week 500s on this week's new column. This closes that gap
|
| 72 |
+
for the additive changes we make; anything non-additive (renames, drops,
|
| 73 |
+
type changes) waits for Alembic, which replaces this in Phase 2 alongside
|
| 74 |
+
Postgres. Columns with scalar defaults get that default; callable defaults
|
| 75 |
+
(uuid/now) are added nullable and filled by the ORM on new rows.
|
| 76 |
+
"""
|
| 77 |
+
import logging
|
| 78 |
+
|
| 79 |
+
from sqlalchemy import inspect, text
|
| 80 |
+
|
| 81 |
+
added: list[str] = []
|
| 82 |
+
inspector = inspect(target_engine)
|
| 83 |
+
with target_engine.begin() as conn:
|
| 84 |
+
for table in metadata.sorted_tables:
|
| 85 |
+
if table.name not in inspector.get_table_names():
|
| 86 |
+
continue # create_all handles brand-new tables
|
| 87 |
+
existing = {c["name"] for c in inspector.get_columns(table.name)}
|
| 88 |
+
for column in table.columns:
|
| 89 |
+
if column.name in existing:
|
| 90 |
+
continue
|
| 91 |
+
col_type = column.type.compile(target_engine.dialect)
|
| 92 |
+
ddl = f'ALTER TABLE {table.name} ADD COLUMN "{column.name}" {col_type}'
|
| 93 |
+
default = getattr(column.default, "arg", None)
|
| 94 |
+
if default is not None and not callable(default):
|
| 95 |
+
ddl += f" DEFAULT {_sqlite_literal(default)}"
|
| 96 |
+
conn.execute(text(ddl))
|
| 97 |
+
added.append(f"{table.name}.{column.name}")
|
| 98 |
+
if added:
|
| 99 |
+
logging.getLogger("ccr.db").warning("auto-migrated columns: %s", ", ".join(added))
|
| 100 |
+
return added
|
backend/app/ingest.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Corpus ingestion
|
| 2 |
|
| 3 |
Researchers upload CSVs exported from Qualtrics, Excel, R, SPSS, and
|
| 4 |
scrapers: BOMs, latin-1 encodings, semicolon/tab delimiters, ragged rows.
|
|
@@ -34,7 +34,7 @@ def load_corpus(path: str | Path) -> tuple[pd.DataFrame, dict]:
|
|
| 34 |
"""Parse CSV/XLSX into a DataFrame.
|
| 35 |
|
| 36 |
Returns (df, parse_info) where parse_info records the format,
|
| 37 |
-
encoding, and delimiter actually used
|
| 38 |
echoed into every run's reproducibility metadata.
|
| 39 |
"""
|
| 40 |
p = Path(path)
|
|
@@ -48,40 +48,62 @@ def load_corpus(path: str | Path) -> tuple[pd.DataFrame, dict]:
|
|
| 48 |
|
| 49 |
last_error: Exception | None = None
|
| 50 |
for encoding in _ENCODINGS:
|
| 51 |
-
#
|
| 52 |
-
#
|
| 53 |
-
#
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
)
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
if encoding == "latin-1":
|
| 70 |
-
info["note"] = (
|
| 71 |
-
"File was not valid UTF-8; decoded as latin-1. "
|
| 72 |
-
"Verify non-ASCII characters rendered correctly."
|
| 73 |
-
)
|
| 74 |
-
return _validate(df), info
|
| 75 |
-
except IngestError:
|
| 76 |
-
raise
|
| 77 |
-
except Exception as exc: # try next (sep, encoding) combination
|
| 78 |
-
last_error = exc
|
| 79 |
|
| 80 |
raise IngestError(f"Could not parse CSV file: {last_error}")
|
| 81 |
|
| 82 |
|
| 83 |
-
|
| 84 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
|
| 87 |
def _validate(df: pd.DataFrame) -> pd.DataFrame:
|
|
@@ -89,7 +111,7 @@ def _validate(df: pd.DataFrame) -> pd.DataFrame:
|
|
| 89 |
raise IngestError("The file parsed but contains no data rows.")
|
| 90 |
if len(df) > max_rows():
|
| 91 |
raise IngestError(
|
| 92 |
-
f"File has {len(df):,} rows
|
| 93 |
f"{max_rows():,}-row limit. Split the corpus or run locally."
|
| 94 |
)
|
| 95 |
df.columns = [str(c) for c in df.columns]
|
|
|
|
| 1 |
+
"""Corpus ingestion - tolerant of real-world research files.
|
| 2 |
|
| 3 |
Researchers upload CSVs exported from Qualtrics, Excel, R, SPSS, and
|
| 4 |
scrapers: BOMs, latin-1 encodings, semicolon/tab delimiters, ragged rows.
|
|
|
|
| 34 |
"""Parse CSV/XLSX into a DataFrame.
|
| 35 |
|
| 36 |
Returns (df, parse_info) where parse_info records the format,
|
| 37 |
+
encoding, and delimiter actually used - stored with the corpus and
|
| 38 |
echoed into every run's reproducibility metadata.
|
| 39 |
"""
|
| 40 |
p = Path(path)
|
|
|
|
| 48 |
|
| 49 |
last_error: Exception | None = None
|
| 50 |
for encoding in _ENCODINGS:
|
| 51 |
+
# Delimiter detection restricted to REAL delimiter candidates (, ; tab |).
|
| 52 |
+
# Unrestricted sniffing famously "detects" spaces in single-column files
|
| 53 |
+
# of natural-language sentences, exploding the header into word-columns.
|
| 54 |
+
try:
|
| 55 |
+
sep = _detect_delimiter(p, encoding)
|
| 56 |
+
df = pd.read_csv(
|
| 57 |
+
p,
|
| 58 |
+
encoding=encoding,
|
| 59 |
+
sep=sep,
|
| 60 |
+
engine="python",
|
| 61 |
+
on_bad_lines="skip",
|
| 62 |
+
)
|
| 63 |
+
info = {
|
| 64 |
+
"format": "csv",
|
| 65 |
+
"encoding": encoding,
|
| 66 |
+
"delimiter": {"\t": "tab"}.get(sep, sep),
|
| 67 |
+
}
|
| 68 |
+
if encoding == "latin-1":
|
| 69 |
+
info["note"] = (
|
| 70 |
+
"File was not valid UTF-8; decoded as latin-1. "
|
| 71 |
+
"Verify non-ASCII characters rendered correctly."
|
| 72 |
)
|
| 73 |
+
return _validate(df), info
|
| 74 |
+
except IngestError:
|
| 75 |
+
raise
|
| 76 |
+
except Exception as exc: # try next encoding
|
| 77 |
+
last_error = exc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
|
| 79 |
raise IngestError(f"Could not parse CSV file: {last_error}")
|
| 80 |
|
| 81 |
|
| 82 |
+
_DELIMITER_CANDIDATES = (",", ";", "\t", "|")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _detect_delimiter(p: Path, encoding: str) -> str:
|
| 86 |
+
"""Pick the candidate delimiter most consistent across the first lines.
|
| 87 |
+
|
| 88 |
+
Single-column files (no candidate present) default to ',' - a comma parse
|
| 89 |
+
of a delimiter-free file yields one column, which is exactly right.
|
| 90 |
+
"""
|
| 91 |
+
try:
|
| 92 |
+
with open(p, encoding=encoding, errors="strict") as fh:
|
| 93 |
+
lines = [line for line, _ in zip(fh, range(20)) if line.strip()]
|
| 94 |
+
except UnicodeDecodeError:
|
| 95 |
+
raise ValueError(f"not decodable as {encoding}")
|
| 96 |
+
if not lines:
|
| 97 |
+
return ","
|
| 98 |
+
|
| 99 |
+
def score(delim: str) -> tuple[int, int]:
|
| 100 |
+
counts = [line.count(delim) for line in lines]
|
| 101 |
+
present = min(counts) > 0
|
| 102 |
+
consistent = len(set(counts)) == 1
|
| 103 |
+
return (int(present) + int(present and consistent), counts[0])
|
| 104 |
+
|
| 105 |
+
best = max(_DELIMITER_CANDIDATES, key=score)
|
| 106 |
+
return best if score(best)[0] > 0 else ","
|
| 107 |
|
| 108 |
|
| 109 |
def _validate(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
|
| 111 |
raise IngestError("The file parsed but contains no data rows.")
|
| 112 |
if len(df) > max_rows():
|
| 113 |
raise IngestError(
|
| 114 |
+
f"File has {len(df):,} rows - above this instance's "
|
| 115 |
f"{max_rows():,}-row limit. Split the corpus or run locally."
|
| 116 |
)
|
| 117 |
df.columns = [str(c) for c in df.columns]
|
backend/app/jobs.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""Background job runner.
|
| 2 |
|
| 3 |
-
Jobs run on a dedicated single-worker executor
|
| 4 |
embedding is CPU-bound, so running jobs sequentially protects the instance's
|
| 5 |
memory and keeps per-job throughput predictable, while job state lives in
|
| 6 |
the DB (queued → running → completed/failed) so the API and UI never depend
|
|
@@ -15,18 +15,28 @@ rather than hanging forever in the UI.
|
|
| 15 |
|
| 16 |
from __future__ import annotations
|
| 17 |
|
|
|
|
| 18 |
import json
|
| 19 |
import logging
|
|
|
|
| 20 |
import traceback
|
| 21 |
from concurrent.futures import ThreadPoolExecutor
|
| 22 |
from datetime import datetime, timezone
|
| 23 |
|
| 24 |
import numpy as np
|
| 25 |
|
| 26 |
-
from .
|
|
|
|
|
|
|
| 27 |
from .db import DATA_DIR, SessionLocal
|
| 28 |
from .ingest import load_corpus
|
| 29 |
-
from .models import Construct, Corpus, Job
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
logger = logging.getLogger("ccr.jobs")
|
| 32 |
|
|
@@ -36,15 +46,32 @@ TOP_N = 10
|
|
| 36 |
SNIPPET_LEN = 220
|
| 37 |
|
| 38 |
# Single worker: sequential jobs, bounded memory. See module docstring.
|
| 39 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
|
| 42 |
def submit_job(job_id: str) -> None:
|
| 43 |
-
|
| 44 |
|
| 45 |
|
| 46 |
def shutdown_executor() -> None:
|
| 47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
|
| 50 |
def recover_orphaned_jobs() -> int:
|
|
@@ -99,7 +126,14 @@ def run_job(job_id: str) -> None:
|
|
| 99 |
items = json.loads(construct.items_json)
|
| 100 |
parse_info = json.loads(corpus.parse_info_json or "{}")
|
| 101 |
|
| 102 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
if job.text_column not in df.columns:
|
| 104 |
raise ValueError(f"Column '{job.text_column}' not found in corpus.")
|
| 105 |
|
|
@@ -109,33 +143,109 @@ def run_job(job_id: str) -> None:
|
|
| 109 |
work_df = df.loc[mask].reset_index(drop=True)
|
| 110 |
texts = work_df[job.text_column].astype(str).tolist()
|
| 111 |
|
|
|
|
|
|
|
| 112 |
def progress(frac: float):
|
| 113 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
|
| 115 |
backend = get_backend(job.model_name)
|
| 116 |
-
result = run_ccr(
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
if dropped:
|
| 121 |
-
warnings.append(
|
|
|
|
|
|
|
|
|
|
| 122 |
n_dupes = len(texts) - len(set(texts))
|
| 123 |
if n_dupes:
|
| 124 |
-
warnings.append(
|
| 125 |
-
|
| 126 |
-
"
|
| 127 |
-
|
| 128 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
if max_seq:
|
| 130 |
char_budget = int(max_seq) * 4 # rough chars-per-token heuristic
|
| 131 |
n_long = sum(1 for t in texts if len(t) > char_budget)
|
| 132 |
if n_long:
|
| 133 |
-
warnings.append(
|
| 134 |
-
|
| 135 |
-
"
|
| 136 |
-
|
|
|
|
| 137 |
if parse_info.get("note"):
|
| 138 |
-
warnings.append(parse_info["note"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
|
| 140 |
# Export mirrors ccr_wrapper's shape: input columns + per-item
|
| 141 |
# similarity columns + overall score, so it drops into existing
|
|
@@ -144,8 +254,9 @@ def run_job(job_id: str) -> None:
|
|
| 144 |
for j in range(result.similarities.shape[1]):
|
| 145 |
out[f"sim_item_{j + 1}"] = np.round(result.similarities[:, j], 6)
|
| 146 |
out["ccr_score"] = np.round(result.scores, 6)
|
| 147 |
-
|
| 148 |
-
out.to_csv(
|
|
|
|
| 149 |
|
| 150 |
scores = result.scores
|
| 151 |
order = np.argsort(scores)
|
|
@@ -181,14 +292,45 @@ def run_job(job_id: str) -> None:
|
|
| 181 |
metadata = {
|
| 182 |
**result.metadata,
|
| 183 |
"job_id": job.id,
|
|
|
|
|
|
|
| 184 |
"corpus_file": corpus.filename,
|
| 185 |
"corpus_parse_info": parse_info,
|
| 186 |
"text_column": job.text_column,
|
|
|
|
| 187 |
"construct": construct.name,
|
| 188 |
"construct_reference": construct.reference,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
"n_rows_input": int(corpus.n_rows),
|
| 190 |
"n_rows_dropped_empty": dropped,
|
| 191 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
|
| 193 |
_set(
|
| 194 |
db,
|
|
|
|
| 1 |
"""Background job runner.
|
| 2 |
|
| 3 |
+
Jobs run on a dedicated single-worker executor - a deliberate right-sizing:
|
| 4 |
embedding is CPU-bound, so running jobs sequentially protects the instance's
|
| 5 |
memory and keeps per-job throughput predictable, while job state lives in
|
| 6 |
the DB (queued → running → completed/failed) so the API and UI never depend
|
|
|
|
| 15 |
|
| 16 |
from __future__ import annotations
|
| 17 |
|
| 18 |
+
import hashlib
|
| 19 |
import json
|
| 20 |
import logging
|
| 21 |
+
import os
|
| 22 |
import traceback
|
| 23 |
from concurrent.futures import ThreadPoolExecutor
|
| 24 |
from datetime import datetime, timezone
|
| 25 |
|
| 26 |
import numpy as np
|
| 27 |
|
| 28 |
+
from . import registry, warnings_engine
|
| 29 |
+
from .ccr import FAKE_MODEL_NAME, get_backend, run_ccr
|
| 30 |
+
from .construct_lib import construct_snapshot
|
| 31 |
from .db import DATA_DIR, SessionLocal
|
| 32 |
from .ingest import load_corpus
|
| 33 |
+
from .models import Construct, Corpus, Job, Project
|
| 34 |
+
from .reproducibility import record_environment
|
| 35 |
+
from .retention import EMB_CACHE_DIR, remove_corpus_files
|
| 36 |
+
from . import storage
|
| 37 |
+
|
| 38 |
+
PLATFORM_VERSION = "0.2.0"
|
| 39 |
+
OUTPUT_SCHEMA_VERSION = "1.0" # bump on ANY export-column change (CLAUDE.md hard rule)
|
| 40 |
|
| 41 |
logger = logging.getLogger("ccr.jobs")
|
| 42 |
|
|
|
|
| 46 |
SNIPPET_LEN = 220
|
| 47 |
|
| 48 |
# Single worker: sequential jobs, bounded memory. See module docstring.
|
| 49 |
+
# Created lazily and re-creatable: a lifespan shutdown (dev reload, test
|
| 50 |
+
# client closing) must not permanently kill job submission for the process.
|
| 51 |
+
import threading
|
| 52 |
+
|
| 53 |
+
_executor: ThreadPoolExecutor | None = None
|
| 54 |
+
_executor_lock = threading.Lock()
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _get_executor() -> ThreadPoolExecutor:
|
| 58 |
+
global _executor
|
| 59 |
+
with _executor_lock:
|
| 60 |
+
if _executor is None:
|
| 61 |
+
_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="ccr-job")
|
| 62 |
+
return _executor
|
| 63 |
|
| 64 |
|
| 65 |
def submit_job(job_id: str) -> None:
|
| 66 |
+
_get_executor().submit(_run_job_logged, job_id)
|
| 67 |
|
| 68 |
|
| 69 |
def shutdown_executor() -> None:
|
| 70 |
+
global _executor
|
| 71 |
+
with _executor_lock:
|
| 72 |
+
if _executor is not None:
|
| 73 |
+
_executor.shutdown(wait=False, cancel_futures=True)
|
| 74 |
+
_executor = None
|
| 75 |
|
| 76 |
|
| 77 |
def recover_orphaned_jobs() -> int:
|
|
|
|
| 126 |
items = json.loads(construct.items_json)
|
| 127 |
parse_info = json.loads(corpus.parse_info_json or "{}")
|
| 128 |
|
| 129 |
+
# Materialize the corpus locally (a no-op on the local backend; a
|
| 130 |
+
# temp download when files live in object storage).
|
| 131 |
+
local_corpus, corpus_is_temp = storage.fetch_to_local(corpus.path)
|
| 132 |
+
try:
|
| 133 |
+
df, _ = load_corpus(str(local_corpus))
|
| 134 |
+
finally:
|
| 135 |
+
if corpus_is_temp:
|
| 136 |
+
local_corpus.unlink(missing_ok=True)
|
| 137 |
if job.text_column not in df.columns:
|
| 138 |
raise ValueError(f"Column '{job.text_column}' not found in corpus.")
|
| 139 |
|
|
|
|
| 143 |
work_df = df.loc[mask].reset_index(drop=True)
|
| 144 |
texts = work_df[job.text_column].astype(str).tolist()
|
| 145 |
|
| 146 |
+
last_progress = -1.0
|
| 147 |
+
|
| 148 |
def progress(frac: float):
|
| 149 |
+
# Throttled: commit only on >=1% movement (or completion) so large
|
| 150 |
+
# corpora don't turn the progress bar into a DB write hotspot.
|
| 151 |
+
nonlocal last_progress
|
| 152 |
+
frac = round(float(frac), 3)
|
| 153 |
+
if frac - last_progress >= 0.01 or frac >= 1.0:
|
| 154 |
+
last_progress = frac
|
| 155 |
+
_set(db, job, progress=frac)
|
| 156 |
+
|
| 157 |
+
# Model config from the registry (spec 0003); the test fake has none.
|
| 158 |
+
model_cfg = None if job.model_name == FAKE_MODEL_NAME else registry.get_model(job.model_name)
|
| 159 |
+
item_prefix = model_cfg.item_prefix if (model_cfg and model_cfg.requires_prefix) else ""
|
| 160 |
+
text_prefix = model_cfg.text_prefix if (model_cfg and model_cfg.requires_prefix) else ""
|
| 161 |
+
|
| 162 |
+
project = db.get(Project, job.project_id)
|
| 163 |
+
is_anonymous = not (project and project.owner_user_id)
|
| 164 |
+
|
| 165 |
+
# Corpus-embedding cache: the CCR workflow is many constructs against
|
| 166 |
+
# the SAME corpus, and ~97% of a run is embedding the documents. Corpora
|
| 167 |
+
# are immutable after upload, so (corpus, column, model, revision,
|
| 168 |
+
# prefix) fully determines the embeddings - reusing them is bit-identical.
|
| 169 |
+
# Disabled for the test fake (unless forced) and skipped for anonymous
|
| 170 |
+
# runs (their files are removed right after the run anyway).
|
| 171 |
+
cache_enabled = os.environ.get("CCR_EMB_CACHE", "1") == "1" and (
|
| 172 |
+
model_cfg is not None or os.environ.get("CCR_EMB_CACHE_FORCE") == "1"
|
| 173 |
+
)
|
| 174 |
+
cache_path = None
|
| 175 |
+
cached_embeddings = None
|
| 176 |
+
if cache_enabled:
|
| 177 |
+
key = hashlib.sha256(
|
| 178 |
+
f"{job.text_column}|{job.model_name}|"
|
| 179 |
+
f"{model_cfg.revision if model_cfg else 'fake'}|{text_prefix}".encode()
|
| 180 |
+
).hexdigest()[:20]
|
| 181 |
+
cache_path = EMB_CACHE_DIR / f"{corpus.id}_{key}.npy"
|
| 182 |
+
if cache_path.exists():
|
| 183 |
+
try:
|
| 184 |
+
candidate = np.load(cache_path)
|
| 185 |
+
if candidate.shape[0] == len(texts):
|
| 186 |
+
cached_embeddings = candidate
|
| 187 |
+
except Exception:
|
| 188 |
+
cache_path.unlink(missing_ok=True) # unreadable cache: recompute
|
| 189 |
|
| 190 |
backend = get_backend(job.model_name)
|
| 191 |
+
result = run_ccr(
|
| 192 |
+
texts, items, backend,
|
| 193 |
+
progress_cb=progress, item_prefix=item_prefix, text_prefix=text_prefix,
|
| 194 |
+
doc_embeddings=cached_embeddings,
|
| 195 |
+
)
|
| 196 |
+
if (
|
| 197 |
+
cache_enabled and cache_path is not None and cached_embeddings is None
|
| 198 |
+
and not is_anonymous and result.doc_embeddings is not None
|
| 199 |
+
):
|
| 200 |
+
try:
|
| 201 |
+
np.save(cache_path, result.doc_embeddings)
|
| 202 |
+
except Exception:
|
| 203 |
+
logger.warning("could not write embedding cache %s", cache_path)
|
| 204 |
+
|
| 205 |
+
# Structured data-quality warnings (spec 0001) - objects, never bare strings.
|
| 206 |
+
W = warnings_engine.warning
|
| 207 |
+
warnings: list[dict] = []
|
| 208 |
if dropped:
|
| 209 |
+
warnings.append(W(
|
| 210 |
+
"EMPTY_ROWS_DROPPED", "info",
|
| 211 |
+
f"{dropped} empty text row(s) were dropped before analysis.", count=dropped,
|
| 212 |
+
))
|
| 213 |
n_dupes = len(texts) - len(set(texts))
|
| 214 |
if n_dupes:
|
| 215 |
+
warnings.append(W(
|
| 216 |
+
"DUPLICATE_TEXTS", "warning",
|
| 217 |
+
f"{n_dupes} duplicate text(s) detected - each is scored independently; "
|
| 218 |
+
"deduplicate upstream if unintended.", count=n_dupes,
|
| 219 |
+
))
|
| 220 |
+
short = warnings_engine.short_text_warning(texts)
|
| 221 |
+
if short:
|
| 222 |
+
warnings.append(short)
|
| 223 |
+
max_seq = model_cfg.max_seq_length if model_cfg else result.metadata.get("model_max_seq_length")
|
| 224 |
if max_seq:
|
| 225 |
char_budget = int(max_seq) * 4 # rough chars-per-token heuristic
|
| 226 |
n_long = sum(1 for t in texts if len(t) > char_budget)
|
| 227 |
if n_long:
|
| 228 |
+
warnings.append(W(
|
| 229 |
+
"TEXTS_MAYBE_TRUNCATED", "warning",
|
| 230 |
+
f"{n_long} text(s) likely exceed the model's {max_seq}-token window and "
|
| 231 |
+
"were truncated; consider splitting long documents.", count=n_long,
|
| 232 |
+
))
|
| 233 |
if parse_info.get("note"):
|
| 234 |
+
warnings.append(W("ENCODING_FALLBACK", "warning", parse_info["note"]))
|
| 235 |
+
|
| 236 |
+
# Language checks: corpus-level detection + model-coverage (spec 0001, design §12).
|
| 237 |
+
selected_language = (job.language or "en").lower()
|
| 238 |
+
lang_result, lang_warnings = warnings_engine.detect_corpus_language(texts, selected_language)
|
| 239 |
+
warnings.extend(lang_warnings)
|
| 240 |
+
if model_cfg:
|
| 241 |
+
mlw = warnings_engine.model_language_warning(
|
| 242 |
+
selected_language, model_cfg.id, model_cfg.supported_languages,
|
| 243 |
+
model_cfg.language_set_name,
|
| 244 |
+
)
|
| 245 |
+
if mlw:
|
| 246 |
+
warnings.append(mlw)
|
| 247 |
+
for user_warning in (model_cfg.user_warnings if model_cfg else ()):
|
| 248 |
+
warnings.append(W("MODEL_NOTE", "info", user_warning))
|
| 249 |
|
| 250 |
# Export mirrors ccr_wrapper's shape: input columns + per-item
|
| 251 |
# similarity columns + overall score, so it drops into existing
|
|
|
|
| 254 |
for j in range(result.similarities.shape[1]):
|
| 255 |
out[f"sim_item_{j + 1}"] = np.round(result.similarities[:, j], 6)
|
| 256 |
out["ccr_score"] = np.round(result.scores, 6)
|
| 257 |
+
local_result = RESULTS_DIR / f"{job.id}.csv"
|
| 258 |
+
out.to_csv(local_result, index=False)
|
| 259 |
+
result_path = storage.move_local_into_storage("results", f"{job.id}.csv", local_result)
|
| 260 |
|
| 261 |
scores = result.scores
|
| 262 |
order = np.argsort(scores)
|
|
|
|
| 292 |
metadata = {
|
| 293 |
**result.metadata,
|
| 294 |
"job_id": job.id,
|
| 295 |
+
"platform_version": PLATFORM_VERSION,
|
| 296 |
+
"output_schema_version": OUTPUT_SCHEMA_VERSION,
|
| 297 |
"corpus_file": corpus.filename,
|
| 298 |
"corpus_parse_info": parse_info,
|
| 299 |
"text_column": job.text_column,
|
| 300 |
+
"language": lang_result.as_metadata(),
|
| 301 |
"construct": construct.name,
|
| 302 |
"construct_reference": construct.reference,
|
| 303 |
+
"construct_snapshot": construct_snapshot(construct),
|
| 304 |
+
"model_registry_id": model_cfg.id if model_cfg else job.model_name,
|
| 305 |
+
"provider_model_id": model_cfg.provider_model_id if model_cfg else job.model_name,
|
| 306 |
+
"model_revision": model_cfg.revision if model_cfg else None,
|
| 307 |
+
"scoring": {"adjustment_strategy": "none", "aggregate": "mean_all_items"},
|
| 308 |
+
"output_schema": (
|
| 309 |
+
list(work_df.columns)
|
| 310 |
+
+ [f"sim_item_{j + 1}" for j in range(result.similarities.shape[1])]
|
| 311 |
+
+ ["ccr_score"]
|
| 312 |
+
),
|
| 313 |
+
"warnings": warnings,
|
| 314 |
"n_rows_input": int(corpus.n_rows),
|
| 315 |
"n_rows_dropped_empty": dropped,
|
| 316 |
}
|
| 317 |
+
record_environment(metadata) # pins exact package versions for the repro bundle
|
| 318 |
+
|
| 319 |
+
# Retention (PI decision 2026-07-10): anonymous uploads are removed the
|
| 320 |
+
# moment analysis finishes. The results summary/CSV stay downloadable
|
| 321 |
+
# until the anonymous project's TTL purge; the raw upload does not.
|
| 322 |
+
if is_anonymous:
|
| 323 |
+
remove_corpus_files(corpus)
|
| 324 |
+
corpus.path = ""
|
| 325 |
+
metadata["anonymous_corpus_removed"] = True
|
| 326 |
+
warnings.append(W(
|
| 327 |
+
"ANONYMOUS_DATA_REMOVED", "info",
|
| 328 |
+
"The uploaded file was deleted after this analysis (anonymous runs "
|
| 329 |
+
"keep no raw data). Re-running requires uploading again, or sign in "
|
| 330 |
+
"to keep datasets.",
|
| 331 |
+
))
|
| 332 |
+
summary["warnings"] = warnings
|
| 333 |
+
metadata["warnings"] = warnings
|
| 334 |
|
| 335 |
_set(
|
| 336 |
db,
|
backend/app/main.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""CCR Platform
|
| 2 |
|
| 3 |
Single deployable: serves the JSON API under /api and the prebuilt React
|
| 4 |
dashboard as static files at /. Local-first by design: corpora, embeddings,
|
|
@@ -10,61 +10,81 @@ against pinned model weights.
|
|
| 10 |
from __future__ import annotations
|
| 11 |
|
| 12 |
import json
|
|
|
|
| 13 |
from contextlib import asynccontextmanager
|
| 14 |
from pathlib import Path
|
| 15 |
|
| 16 |
-
from fastapi import Depends, FastAPI, HTTPException, UploadFile
|
| 17 |
from fastapi.middleware.cors import CORSMiddleware
|
| 18 |
-
from fastapi.
|
|
|
|
| 19 |
from fastapi.staticfiles import StaticFiles
|
| 20 |
from sqlalchemy.orm import Session
|
| 21 |
|
|
|
|
| 22 |
from . import jobs as jobs_module
|
| 23 |
-
from .
|
| 24 |
-
from .
|
|
|
|
|
|
|
|
|
|
| 25 |
from .ingest import IngestError, load_corpus, suggest_text_column
|
| 26 |
-
from .models import Construct, Corpus, Job, Project
|
|
|
|
| 27 |
from .schemas import (
|
| 28 |
ConstructCreate,
|
| 29 |
ConstructOut,
|
| 30 |
CorpusOut,
|
| 31 |
JobCreate,
|
| 32 |
JobOut,
|
|
|
|
| 33 |
ProjectCreate,
|
| 34 |
ProjectOut,
|
|
|
|
|
|
|
| 35 |
)
|
| 36 |
-
from .seed_constructs import SEED_CONSTRUCTS
|
| 37 |
|
| 38 |
MAX_UPLOAD_BYTES = 25 * 1024 * 1024 # sane lab-scale ceiling; raise deliberately
|
| 39 |
ALLOWED_SUFFIXES = (".csv", ".xlsx", ".xls")
|
| 40 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
@asynccontextmanager
|
| 42 |
async def lifespan(_: FastAPI):
|
| 43 |
-
"""Create tables and
|
| 44 |
Base.metadata.create_all(engine)
|
|
|
|
|
|
|
| 45 |
db = SessionLocal()
|
| 46 |
try:
|
| 47 |
-
|
| 48 |
-
for seed in SEED_CONSTRUCTS:
|
| 49 |
-
db.add(
|
| 50 |
-
Construct(
|
| 51 |
-
name=seed["name"],
|
| 52 |
-
description=seed["description"],
|
| 53 |
-
reference=seed["reference"],
|
| 54 |
-
items_json=json.dumps(seed["items"]),
|
| 55 |
-
is_seed=True,
|
| 56 |
-
)
|
| 57 |
-
)
|
| 58 |
-
db.commit()
|
| 59 |
finally:
|
| 60 |
db.close()
|
| 61 |
jobs_module.recover_orphaned_jobs()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
yield
|
|
|
|
| 63 |
jobs_module.shutdown_executor()
|
| 64 |
|
| 65 |
|
| 66 |
app = FastAPI(title="CCR Platform", version="0.1.0", lifespan=lifespan)
|
| 67 |
|
|
|
|
| 68 |
app.add_middleware(
|
| 69 |
CORSMiddleware,
|
| 70 |
allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"], # Vite dev server
|
|
@@ -75,13 +95,21 @@ app.add_middleware(
|
|
| 75 |
|
| 76 |
# ---------------------------------------------------------------- helpers
|
| 77 |
def _construct_out(c: Construct) -> ConstructOut:
|
|
|
|
|
|
|
| 78 |
return ConstructOut(
|
| 79 |
id=c.id,
|
| 80 |
name=c.name,
|
| 81 |
description=c.description,
|
| 82 |
reference=c.reference,
|
| 83 |
-
items=
|
|
|
|
| 84 |
is_seed=c.is_seed,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
)
|
| 86 |
|
| 87 |
|
|
@@ -97,6 +125,7 @@ def _job_out(db: Session, j: Job) -> JobOut:
|
|
| 97 |
corpus_filename=corpus.filename if corpus else "",
|
| 98 |
text_column=j.text_column,
|
| 99 |
model_name=j.model_name,
|
|
|
|
| 100 |
status=j.status,
|
| 101 |
progress=j.progress,
|
| 102 |
error=j.error,
|
|
@@ -127,21 +156,279 @@ def health():
|
|
| 127 |
|
| 128 |
@app.get("/api/models")
|
| 129 |
def list_models():
|
| 130 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
|
| 132 |
|
| 133 |
# --------------------------------------------------------------- projects
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
@app.get("/api/projects", response_model=list[ProjectOut])
|
| 135 |
-
def list_projects(db: Session = Depends(get_db)):
|
| 136 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
|
| 138 |
|
| 139 |
@app.post("/api/projects", response_model=ProjectOut, status_code=201)
|
| 140 |
-
def create_project(
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
db.add(project)
|
| 143 |
db.commit()
|
| 144 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
|
| 146 |
|
| 147 |
# ----------------------------------------------------------------- corpora
|
|
@@ -170,8 +457,14 @@ def list_corpora(project_id: str, db: Session = Depends(get_db)):
|
|
| 170 |
|
| 171 |
|
| 172 |
@app.post("/api/projects/{project_id}/corpora", response_model=CorpusOut, status_code=201)
|
| 173 |
-
async def upload_corpus(
|
| 174 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
|
| 176 |
suffix = Path(file.filename or "upload.csv").suffix.lower()
|
| 177 |
if suffix not in ALLOWED_SUFFIXES:
|
|
@@ -181,19 +474,41 @@ async def upload_corpus(project_id: str, file: UploadFile, db: Session = Depends
|
|
| 181 |
if len(payload) > MAX_UPLOAD_BYTES:
|
| 182 |
raise HTTPException(413, "File exceeds the 25 MB upload limit.")
|
| 183 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
corpus = Corpus(
|
| 185 |
project_id=project_id, filename=file.filename, path="", n_rows=0, columns_json="[]"
|
| 186 |
)
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
|
|
|
|
|
|
|
|
|
| 190 |
|
| 191 |
try:
|
| 192 |
-
df, parse_info = load_corpus(str(
|
| 193 |
except IngestError as exc:
|
| 194 |
-
|
| 195 |
raise HTTPException(400, str(exc)) from exc
|
| 196 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
corpus.n_rows = int(len(df))
|
| 198 |
corpus.columns_json = json.dumps(list(df.columns))
|
| 199 |
corpus.parse_info_json = json.dumps(parse_info)
|
|
@@ -227,30 +542,101 @@ def create_construct(body: ConstructCreate, db: Session = Depends(get_db)):
|
|
| 227 |
items = [i.strip() for i in body.items if i.strip()]
|
| 228 |
if not items:
|
| 229 |
raise HTTPException(400, "Construct needs at least one non-empty item.")
|
|
|
|
|
|
|
|
|
|
| 230 |
construct = Construct(
|
| 231 |
name=body.name.strip(),
|
| 232 |
description=body.description.strip(),
|
| 233 |
reference=body.reference.strip(),
|
| 234 |
items_json=json.dumps(items),
|
|
|
|
| 235 |
is_seed=False,
|
|
|
|
|
|
|
| 236 |
)
|
| 237 |
db.add(construct)
|
| 238 |
db.commit()
|
| 239 |
return _construct_out(construct)
|
| 240 |
|
| 241 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
# -------------------------------------------------------------------- jobs
|
| 243 |
@app.post("/api/jobs", response_model=JobOut, status_code=201)
|
| 244 |
-
def create_job(
|
| 245 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
corpus = _get_or_404(db, Corpus, body.corpus_id)
|
| 247 |
_get_or_404(db, Construct, body.construct_id)
|
| 248 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
if body.text_column not in json.loads(corpus.columns_json):
|
| 250 |
raise HTTPException(400, f"Column '{body.text_column}' not in corpus columns.")
|
| 251 |
-
allowed =
|
| 252 |
if body.model_name not in allowed:
|
| 253 |
raise HTTPException(400, f"Unknown model '{body.model_name}'.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
|
| 255 |
job = Job(
|
| 256 |
project_id=body.project_id,
|
|
@@ -258,10 +644,21 @@ def create_job(body: JobCreate, db: Session = Depends(get_db)):
|
|
| 258 |
construct_id=body.construct_id,
|
| 259 |
text_column=body.text_column,
|
| 260 |
model_name=body.model_name,
|
|
|
|
| 261 |
)
|
| 262 |
db.add(job)
|
| 263 |
db.commit()
|
| 264 |
jobs_module.submit_job(job.id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
return _job_out(db, job)
|
| 266 |
|
| 267 |
|
|
@@ -294,9 +691,16 @@ def export_results(job_id: str, db: Session = Depends(get_db)):
|
|
| 294 |
job = _get_or_404(db, Job, job_id)
|
| 295 |
if job.status != "completed" or not job.result_path:
|
| 296 |
raise HTTPException(409, "Results not available.")
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
|
| 301 |
|
| 302 |
@app.get("/api/jobs/{job_id}/metadata")
|
|
@@ -312,6 +716,35 @@ def export_metadata(job_id: str, db: Session = Depends(get_db)):
|
|
| 312 |
)
|
| 313 |
|
| 314 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
# ------------------------------------------------------------ static (SPA)
|
| 316 |
STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
|
| 317 |
if STATIC_DIR.exists():
|
|
|
|
| 1 |
+
"""CCR Platform - FastAPI application.
|
| 2 |
|
| 3 |
Single deployable: serves the JSON API under /api and the prebuilt React
|
| 4 |
dashboard as static files at /. Local-first by design: corpora, embeddings,
|
|
|
|
| 10 |
from __future__ import annotations
|
| 11 |
|
| 12 |
import json
|
| 13 |
+
import os
|
| 14 |
from contextlib import asynccontextmanager
|
| 15 |
from pathlib import Path
|
| 16 |
|
| 17 |
+
from fastapi import Depends, FastAPI, HTTPException, Request, Response, UploadFile
|
| 18 |
from fastapi.middleware.cors import CORSMiddleware
|
| 19 |
+
from fastapi.middleware.gzip import GZipMiddleware
|
| 20 |
+
from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse
|
| 21 |
from fastapi.staticfiles import StaticFiles
|
| 22 |
from sqlalchemy.orm import Session
|
| 23 |
|
| 24 |
+
from . import auth, auth_google, retention, storage
|
| 25 |
from . import jobs as jobs_module
|
| 26 |
+
from . import registry
|
| 27 |
+
from .ccr import FAKE_MODEL_NAME
|
| 28 |
+
from .construct_files import parse_construct_file
|
| 29 |
+
from .construct_lib import sync_library
|
| 30 |
+
from .db import DATA_DIR, Base, SessionLocal, auto_migrate_sqlite, engine, get_db
|
| 31 |
from .ingest import IngestError, load_corpus, suggest_text_column
|
| 32 |
+
from .models import Construct, Corpus, Job, Project, User
|
| 33 |
+
from .reproducibility import requirements_text, script_text
|
| 34 |
from .schemas import (
|
| 35 |
ConstructCreate,
|
| 36 |
ConstructOut,
|
| 37 |
CorpusOut,
|
| 38 |
JobCreate,
|
| 39 |
JobOut,
|
| 40 |
+
LoginIn,
|
| 41 |
ProjectCreate,
|
| 42 |
ProjectOut,
|
| 43 |
+
ProjectPatch,
|
| 44 |
+
RegisterIn,
|
| 45 |
)
|
|
|
|
| 46 |
|
| 47 |
MAX_UPLOAD_BYTES = 25 * 1024 * 1024 # sane lab-scale ceiling; raise deliberately
|
| 48 |
ALLOWED_SUFFIXES = (".csv", ".xlsx", ".xls")
|
| 49 |
|
| 50 |
+
# Languages offered in the UI selector; detection may report others (ISO 639-1).
|
| 51 |
+
SELECTABLE_LANGUAGES = [
|
| 52 |
+
"en", "es", "fr", "de", "it", "pt", "nl", "ru", "zh", "ja", "ko", "ar", "hi", "tr", "fa",
|
| 53 |
+
]
|
| 54 |
+
|
| 55 |
@asynccontextmanager
|
| 56 |
async def lifespan(_: FastAPI):
|
| 57 |
+
"""Create tables and sync the construct library (YAML source of truth) at startup."""
|
| 58 |
Base.metadata.create_all(engine)
|
| 59 |
+
auto_migrate_sqlite(engine, Base.metadata) # additive column adds for existing dev DBs
|
| 60 |
+
registry.list_models() # fail fast on an invalid models.yaml
|
| 61 |
db = SessionLocal()
|
| 62 |
try:
|
| 63 |
+
sync_library(db)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
finally:
|
| 65 |
db.close()
|
| 66 |
jobs_module.recover_orphaned_jobs()
|
| 67 |
+
retention.start_cleanup() # anonymous-data TTL purge (no-op if CCR_ANON_TTL_HOURS=0)
|
| 68 |
+
if os.environ.get("CCR_WARM_MODEL") == "1" and os.environ.get("CCR_FAKE_EMBEDDINGS") != "1":
|
| 69 |
+
import threading
|
| 70 |
+
|
| 71 |
+
def _warm():
|
| 72 |
+
try:
|
| 73 |
+
from .ccr import get_backend
|
| 74 |
+
|
| 75 |
+
get_backend(registry.default_model().id).encode(["warm up"])
|
| 76 |
+
except Exception:
|
| 77 |
+
pass # first real run will load the model instead
|
| 78 |
+
|
| 79 |
+
threading.Thread(target=_warm, daemon=True, name="ccr-warmup").start()
|
| 80 |
yield
|
| 81 |
+
retention.stop_cleanup()
|
| 82 |
jobs_module.shutdown_executor()
|
| 83 |
|
| 84 |
|
| 85 |
app = FastAPI(title="CCR Platform", version="0.1.0", lifespan=lifespan)
|
| 86 |
|
| 87 |
+
app.add_middleware(GZipMiddleware, minimum_size=1024) # constructs payload + SPA compress ~4-5x
|
| 88 |
app.add_middleware(
|
| 89 |
CORSMiddleware,
|
| 90 |
allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"], # Vite dev server
|
|
|
|
| 95 |
|
| 96 |
# ---------------------------------------------------------------- helpers
|
| 97 |
def _construct_out(c: Construct) -> ConstructOut:
|
| 98 |
+
items = json.loads(c.items_json)
|
| 99 |
+
flags = json.loads(c.reverse_flags_json or "[]") or [False] * len(items)
|
| 100 |
return ConstructOut(
|
| 101 |
id=c.id,
|
| 102 |
name=c.name,
|
| 103 |
description=c.description,
|
| 104 |
reference=c.reference,
|
| 105 |
+
items=items,
|
| 106 |
+
reverse_scored=flags,
|
| 107 |
is_seed=c.is_seed,
|
| 108 |
+
version=c.version or 1,
|
| 109 |
+
verification_status=c.verification_status or "draft",
|
| 110 |
+
language=c.language or "en",
|
| 111 |
+
category=c.category or "",
|
| 112 |
+
item_hash=(c.item_hash or "")[:16],
|
| 113 |
)
|
| 114 |
|
| 115 |
|
|
|
|
| 125 |
corpus_filename=corpus.filename if corpus else "",
|
| 126 |
text_column=j.text_column,
|
| 127 |
model_name=j.model_name,
|
| 128 |
+
language=j.language or "en",
|
| 129 |
status=j.status,
|
| 130 |
progress=j.progress,
|
| 131 |
error=j.error,
|
|
|
|
| 156 |
|
| 157 |
@app.get("/api/models")
|
| 158 |
def list_models():
|
| 159 |
+
"""Model options from the registry (spec 0003) - never hardcoded."""
|
| 160 |
+
return [
|
| 161 |
+
{
|
| 162 |
+
"id": m.id,
|
| 163 |
+
"label": m.display_name,
|
| 164 |
+
"default": m.default,
|
| 165 |
+
"languages": (m.language_set_name or ", ".join(sorted(m.supported_languages)) or "unspecified"),
|
| 166 |
+
"speed_tier": m.speed_tier,
|
| 167 |
+
"quality_tier": m.quality_tier,
|
| 168 |
+
"warnings": list(m.user_warnings),
|
| 169 |
+
}
|
| 170 |
+
for m in registry.list_models()
|
| 171 |
+
]
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
@app.get("/api/languages")
|
| 175 |
+
def list_languages():
|
| 176 |
+
return SELECTABLE_LANGUAGES
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
# ------------------------------------------------------------------ accounts
|
| 180 |
+
# Local email+password accounts (auth.py) - the free interim provider. The
|
| 181 |
+
# managed swap (Supabase: Google + email/password) replaces token issuance
|
| 182 |
+
# only; every other endpoint just depends on auth.get_current_user.
|
| 183 |
+
def _saved_runs_used(db: Session, user_id: str) -> int:
|
| 184 |
+
return (
|
| 185 |
+
db.query(Job)
|
| 186 |
+
.join(Project, Job.project_id == Project.id)
|
| 187 |
+
.filter(Project.owner_user_id == user_id, Job.status.in_(("queued", "running", "completed")))
|
| 188 |
+
.count()
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _set_session_cookie(response: Response, user: User) -> None:
|
| 193 |
+
response.set_cookie(
|
| 194 |
+
auth.COOKIE_NAME,
|
| 195 |
+
auth.create_session_token(user.id, user.email, user.name),
|
| 196 |
+
httponly=True,
|
| 197 |
+
samesite="lax",
|
| 198 |
+
secure=auth.cookies_secure(),
|
| 199 |
+
max_age=30 * 24 * 3600,
|
| 200 |
+
)
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
@app.get("/api/auth/me")
|
| 204 |
+
def auth_me(
|
| 205 |
+
request: Request,
|
| 206 |
+
db: Session = Depends(get_db),
|
| 207 |
+
user: dict | None = Depends(auth.get_current_user),
|
| 208 |
+
):
|
| 209 |
+
if user:
|
| 210 |
+
return {
|
| 211 |
+
"signed_in": True,
|
| 212 |
+
"name": user["name"],
|
| 213 |
+
"email": user["email"],
|
| 214 |
+
"limits": {"max_bytes": MAX_UPLOAD_BYTES, "max_rows": None},
|
| 215 |
+
"usage": {
|
| 216 |
+
"saved_runs": _saved_runs_used(db, user["id"]),
|
| 217 |
+
"max_saved_runs": auth.user_max_saved_runs(),
|
| 218 |
+
},
|
| 219 |
+
}
|
| 220 |
+
return {
|
| 221 |
+
"signed_in": False,
|
| 222 |
+
"name": None,
|
| 223 |
+
"email": None,
|
| 224 |
+
"google_available": auth_google.configured(),
|
| 225 |
+
"limits": {"max_bytes": auth.anon_max_bytes(), "max_rows": auth.anon_max_rows()},
|
| 226 |
+
"usage": {
|
| 227 |
+
"runs_used_today": auth.runs_used_today(request),
|
| 228 |
+
"max_runs_per_day": auth.anon_max_runs_per_day(),
|
| 229 |
+
},
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
@app.post("/api/auth/register", status_code=201)
|
| 234 |
+
def register(body: RegisterIn, response: Response, db: Session = Depends(get_db)):
|
| 235 |
+
email = body.email.strip().lower()
|
| 236 |
+
if not auth.valid_email(email):
|
| 237 |
+
raise HTTPException(400, "Please enter a valid email address.")
|
| 238 |
+
if len(body.password) < auth.MIN_PASSWORD_LEN:
|
| 239 |
+
raise HTTPException(400, f"Password must be at least {auth.MIN_PASSWORD_LEN} characters.")
|
| 240 |
+
if db.query(User).filter_by(email=email).first():
|
| 241 |
+
raise HTTPException(409, "An account with this email already exists. Sign in instead.")
|
| 242 |
+
user = User(email=email, name=body.name.strip(), password_hash=auth.hash_password(body.password))
|
| 243 |
+
db.add(user)
|
| 244 |
+
db.commit()
|
| 245 |
+
_set_session_cookie(response, user)
|
| 246 |
+
return {"signed_in": True, "name": user.name, "email": user.email}
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
@app.post("/api/auth/login")
|
| 250 |
+
def login(body: LoginIn, response: Response, db: Session = Depends(get_db)):
|
| 251 |
+
email = body.email.strip().lower()
|
| 252 |
+
user = db.query(User).filter_by(email=email).first()
|
| 253 |
+
if user is not None and not user.password_hash:
|
| 254 |
+
raise HTTPException(401, "This account uses Google sign-in - use the Google button.")
|
| 255 |
+
if user is None or not auth.verify_password(body.password, user.password_hash):
|
| 256 |
+
raise HTTPException(401, "Incorrect email or password.")
|
| 257 |
+
_set_session_cookie(response, user)
|
| 258 |
+
return {"signed_in": True, "name": user.name, "email": user.email}
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
@app.get("/api/auth/google/login")
|
| 262 |
+
def google_login():
|
| 263 |
+
"""Start the Google sign-in flow (Supabase PKCE). Plain redirect - the
|
| 264 |
+
frontend links here directly, no SDK involved."""
|
| 265 |
+
if not auth_google.configured():
|
| 266 |
+
raise HTTPException(503, "Google sign-in is not configured on this instance.")
|
| 267 |
+
from fastapi.responses import RedirectResponse
|
| 268 |
+
|
| 269 |
+
url, verifier = auth_google.begin()
|
| 270 |
+
resp = RedirectResponse(url, status_code=307)
|
| 271 |
+
resp.set_cookie(
|
| 272 |
+
auth_google.VERIFIER_COOKIE,
|
| 273 |
+
auth.sign_payload({"v": verifier}),
|
| 274 |
+
httponly=True,
|
| 275 |
+
samesite="lax",
|
| 276 |
+
secure=auth.cookies_secure(),
|
| 277 |
+
max_age=auth_google.VERIFIER_TTL_SECONDS,
|
| 278 |
+
)
|
| 279 |
+
return resp
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
@app.get("/api/auth/google/callback")
|
| 283 |
+
def google_callback(request: Request, code: str = "", db: Session = Depends(get_db)):
|
| 284 |
+
from fastapi.responses import RedirectResponse
|
| 285 |
+
|
| 286 |
+
def fail(msg: str):
|
| 287 |
+
return RedirectResponse(f"/?auth_error={msg}", status_code=307)
|
| 288 |
+
|
| 289 |
+
if not auth_google.configured():
|
| 290 |
+
return fail("google-not-configured")
|
| 291 |
+
payload = auth.verify_payload(request.cookies.get(auth_google.VERIFIER_COOKIE))
|
| 292 |
+
if not code or not payload or "v" not in payload:
|
| 293 |
+
return fail("sign-in-expired-try-again")
|
| 294 |
+
try:
|
| 295 |
+
info = auth_google.exchange(code, payload["v"])
|
| 296 |
+
except ValueError:
|
| 297 |
+
return fail("google-exchange-failed")
|
| 298 |
+
|
| 299 |
+
user = db.query(User).filter_by(email=info["email"]).first()
|
| 300 |
+
if user is None:
|
| 301 |
+
# Google-verified account: no local password (password login is refused
|
| 302 |
+
# with a pointer to the Google button).
|
| 303 |
+
user = User(email=info["email"], name=info["name"], password_hash="")
|
| 304 |
+
db.add(user)
|
| 305 |
+
db.commit()
|
| 306 |
+
|
| 307 |
+
resp = RedirectResponse("/", status_code=307)
|
| 308 |
+
resp.delete_cookie(auth_google.VERIFIER_COOKIE)
|
| 309 |
+
_set_session_cookie(resp, user)
|
| 310 |
+
return resp
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
@app.post("/api/auth/logout")
|
| 314 |
+
def logout(response: Response):
|
| 315 |
+
response.delete_cookie(auth.COOKIE_NAME)
|
| 316 |
+
return {"signed_in": False}
|
| 317 |
|
| 318 |
|
| 319 |
# --------------------------------------------------------------- projects
|
| 320 |
+
def _visible_owners(user: dict | None) -> tuple[str, ...]:
|
| 321 |
+
"""Anonymous viewers see anonymous projects; signed-in users additionally
|
| 322 |
+
see their own. Other users' projects are invisible (and untouchable)."""
|
| 323 |
+
return ("",) if user is None else ("", user["id"])
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def _require_project_access(project: Project, user: dict | None) -> None:
|
| 327 |
+
if project.owner_user_id and (user is None or project.owner_user_id != user["id"]):
|
| 328 |
+
raise HTTPException(403, "This project belongs to another account.")
|
| 329 |
+
|
| 330 |
+
|
| 331 |
@app.get("/api/projects", response_model=list[ProjectOut])
|
| 332 |
+
def list_projects(db: Session = Depends(get_db), user: dict | None = Depends(auth.get_current_user)):
|
| 333 |
+
"""Projects ordered by last activity (latest run, else creation) - the
|
| 334 |
+
project a researcher wants is almost always the one they last worked on."""
|
| 335 |
+
from sqlalchemy import func
|
| 336 |
+
|
| 337 |
+
activity = {
|
| 338 |
+
pid: (last, count)
|
| 339 |
+
for pid, last, count in db.query(
|
| 340 |
+
Job.project_id, func.max(Job.created_at), func.count(Job.id)
|
| 341 |
+
)
|
| 342 |
+
.group_by(Job.project_id)
|
| 343 |
+
.all()
|
| 344 |
+
}
|
| 345 |
+
rows = db.query(Project).filter(Project.owner_user_id.in_(_visible_owners(user))).all()
|
| 346 |
+
out = []
|
| 347 |
+
for p in rows:
|
| 348 |
+
last, count = activity.get(p.id, (None, 0))
|
| 349 |
+
out.append(
|
| 350 |
+
ProjectOut(
|
| 351 |
+
id=p.id,
|
| 352 |
+
name=p.name,
|
| 353 |
+
description=p.description,
|
| 354 |
+
created_at=p.created_at,
|
| 355 |
+
last_activity_at=last or p.created_at,
|
| 356 |
+
n_runs=count,
|
| 357 |
+
archived=bool(p.archived),
|
| 358 |
+
)
|
| 359 |
+
)
|
| 360 |
+
out.sort(key=lambda x: x.last_activity_at, reverse=True)
|
| 361 |
+
return out
|
| 362 |
|
| 363 |
|
| 364 |
@app.post("/api/projects", response_model=ProjectOut, status_code=201)
|
| 365 |
+
def create_project(
|
| 366 |
+
body: ProjectCreate,
|
| 367 |
+
db: Session = Depends(get_db),
|
| 368 |
+
user: dict | None = Depends(auth.get_current_user),
|
| 369 |
+
):
|
| 370 |
+
project = Project(
|
| 371 |
+
name=body.name.strip(),
|
| 372 |
+
description=body.description.strip(),
|
| 373 |
+
owner_user_id=user["id"] if user else "", # "" = anonymous (TTL purge applies)
|
| 374 |
+
)
|
| 375 |
db.add(project)
|
| 376 |
db.commit()
|
| 377 |
+
return ProjectOut(
|
| 378 |
+
id=project.id,
|
| 379 |
+
name=project.name,
|
| 380 |
+
description=project.description,
|
| 381 |
+
created_at=project.created_at,
|
| 382 |
+
last_activity_at=project.created_at,
|
| 383 |
+
n_runs=0,
|
| 384 |
+
archived=False,
|
| 385 |
+
)
|
| 386 |
+
|
| 387 |
+
|
| 388 |
+
@app.patch("/api/projects/{project_id}", response_model=ProjectOut)
|
| 389 |
+
def patch_project(
|
| 390 |
+
project_id: str,
|
| 391 |
+
body: ProjectPatch,
|
| 392 |
+
db: Session = Depends(get_db),
|
| 393 |
+
user: dict | None = Depends(auth.get_current_user),
|
| 394 |
+
):
|
| 395 |
+
"""Archive/unarchive - reversible, no data loss. Archived projects collapse
|
| 396 |
+
into the sidebar's Archived section and keep all datasets and runs."""
|
| 397 |
+
project = _get_or_404(db, Project, project_id)
|
| 398 |
+
_require_project_access(project, user)
|
| 399 |
+
if body.archived is not None:
|
| 400 |
+
project.archived = bool(body.archived)
|
| 401 |
+
db.commit()
|
| 402 |
+
return ProjectOut(
|
| 403 |
+
id=project.id,
|
| 404 |
+
name=project.name,
|
| 405 |
+
description=project.description,
|
| 406 |
+
created_at=project.created_at,
|
| 407 |
+
last_activity_at=project.created_at,
|
| 408 |
+
n_runs=0,
|
| 409 |
+
archived=bool(project.archived),
|
| 410 |
+
)
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
@app.delete("/api/projects/{project_id}", status_code=204)
|
| 414 |
+
def delete_project(
|
| 415 |
+
project_id: str,
|
| 416 |
+
db: Session = Depends(get_db),
|
| 417 |
+
user: dict | None = Depends(auth.get_current_user),
|
| 418 |
+
):
|
| 419 |
+
"""Permanent delete: removes the project, its datasets, runs, uploaded
|
| 420 |
+
files, result files, and cached embeddings. Logged without retaining any
|
| 421 |
+
uploaded text (design doc §9)."""
|
| 422 |
+
import logging
|
| 423 |
+
|
| 424 |
+
project = _get_or_404(db, Project, project_id)
|
| 425 |
+
_require_project_access(project, user)
|
| 426 |
+
counts = retention.delete_project_cascade(db, project)
|
| 427 |
+
logging.getLogger("ccr.projects").info(
|
| 428 |
+
"project deleted: id=%s name=%r corpora=%d runs=%d",
|
| 429 |
+
project_id, project.name, counts["corpora"], counts["runs"],
|
| 430 |
+
)
|
| 431 |
+
return Response(status_code=204)
|
| 432 |
|
| 433 |
|
| 434 |
# ----------------------------------------------------------------- corpora
|
|
|
|
| 457 |
|
| 458 |
|
| 459 |
@app.post("/api/projects/{project_id}/corpora", response_model=CorpusOut, status_code=201)
|
| 460 |
+
async def upload_corpus(
|
| 461 |
+
project_id: str,
|
| 462 |
+
file: UploadFile,
|
| 463 |
+
db: Session = Depends(get_db),
|
| 464 |
+
user: dict | None = Depends(auth.get_current_user),
|
| 465 |
+
):
|
| 466 |
+
project = _get_or_404(db, Project, project_id)
|
| 467 |
+
_require_project_access(project, user)
|
| 468 |
|
| 469 |
suffix = Path(file.filename or "upload.csv").suffix.lower()
|
| 470 |
if suffix not in ALLOWED_SUFFIXES:
|
|
|
|
| 474 |
if len(payload) > MAX_UPLOAD_BYTES:
|
| 475 |
raise HTTPException(413, "File exceeds the 25 MB upload limit.")
|
| 476 |
|
| 477 |
+
# Tier gate (design §5.1): anonymous users get strict caps; signing in
|
| 478 |
+
# lifts them.
|
| 479 |
+
if user is None and len(payload) > auth.anon_max_bytes():
|
| 480 |
+
mb = auth.anon_max_bytes() // (1024 * 1024)
|
| 481 |
+
raise HTTPException(
|
| 482 |
+
413,
|
| 483 |
+
f"Anonymous uploads are limited to {mb} MB. Sign in (top right) to upload larger files.",
|
| 484 |
+
)
|
| 485 |
+
|
| 486 |
corpus = Corpus(
|
| 487 |
project_id=project_id, filename=file.filename, path="", n_rows=0, columns_json="[]"
|
| 488 |
)
|
| 489 |
+
# Parse from a local temp file, then hand the bytes to the storage backend
|
| 490 |
+
# (local disk by default; S3/R2 when CCR_STORAGE=s3 in production).
|
| 491 |
+
tmp_dir = DATA_DIR / "tmp"
|
| 492 |
+
tmp_dir.mkdir(exist_ok=True)
|
| 493 |
+
tmp = tmp_dir / f"{corpus.id}{suffix}"
|
| 494 |
+
tmp.write_bytes(payload)
|
| 495 |
|
| 496 |
try:
|
| 497 |
+
df, parse_info = load_corpus(str(tmp))
|
| 498 |
except IngestError as exc:
|
| 499 |
+
tmp.unlink(missing_ok=True)
|
| 500 |
raise HTTPException(400, str(exc)) from exc
|
| 501 |
|
| 502 |
+
if user is None and len(df) > auth.anon_max_rows():
|
| 503 |
+
tmp.unlink(missing_ok=True)
|
| 504 |
+
raise HTTPException(
|
| 505 |
+
400,
|
| 506 |
+
f"Anonymous uploads are limited to {auth.anon_max_rows():,} rows "
|
| 507 |
+
f"(this file has {len(df):,}). Sign in (top right) to upload larger corpora.",
|
| 508 |
+
)
|
| 509 |
+
|
| 510 |
+
corpus.path = storage.move_local_into_storage("corpora", f"{corpus.id}{suffix}", tmp)
|
| 511 |
+
|
| 512 |
corpus.n_rows = int(len(df))
|
| 513 |
corpus.columns_json = json.dumps(list(df.columns))
|
| 514 |
corpus.parse_info_json = json.dumps(parse_info)
|
|
|
|
| 542 |
items = [i.strip() for i in body.items if i.strip()]
|
| 543 |
if not items:
|
| 544 |
raise HTTPException(400, "Construct needs at least one non-empty item.")
|
| 545 |
+
flags = body.reverse_scored or [False] * len(items)
|
| 546 |
+
if len(flags) != len(items):
|
| 547 |
+
raise HTTPException(400, "reverse_scored must have one flag per item.")
|
| 548 |
construct = Construct(
|
| 549 |
name=body.name.strip(),
|
| 550 |
description=body.description.strip(),
|
| 551 |
reference=body.reference.strip(),
|
| 552 |
items_json=json.dumps(items),
|
| 553 |
+
reverse_flags_json=json.dumps([bool(f) for f in flags]),
|
| 554 |
is_seed=False,
|
| 555 |
+
verification_status="draft", # user-defined research tools, not validated scales
|
| 556 |
+
language=(body.language or "en").lower(),
|
| 557 |
)
|
| 558 |
db.add(construct)
|
| 559 |
db.commit()
|
| 560 |
return _construct_out(construct)
|
| 561 |
|
| 562 |
|
| 563 |
+
@app.post("/api/constructs/parse-file")
|
| 564 |
+
async def parse_construct_upload(file: UploadFile):
|
| 565 |
+
"""Parse a CSV/XLSX of scale items into a PREVIEW (nothing is saved).
|
| 566 |
+
The researcher reviews/edits, then saves via POST /api/constructs."""
|
| 567 |
+
suffix = Path(file.filename or "items.csv").suffix.lower()
|
| 568 |
+
if suffix not in ALLOWED_SUFFIXES:
|
| 569 |
+
raise HTTPException(400, f"Unsupported file type '{suffix}'. Use CSV or XLSX.")
|
| 570 |
+
payload = await file.read()
|
| 571 |
+
if len(payload) > 1024 * 1024:
|
| 572 |
+
raise HTTPException(413, "Item files are capped at 1 MB (a scale is a short list).")
|
| 573 |
+
|
| 574 |
+
tmp_dir = DATA_DIR / "tmp"
|
| 575 |
+
tmp_dir.mkdir(exist_ok=True)
|
| 576 |
+
tmp = tmp_dir / f"construct_upload_{os.urandom(6).hex()}{suffix}"
|
| 577 |
+
tmp.write_bytes(payload)
|
| 578 |
+
try:
|
| 579 |
+
parsed = parse_construct_file(str(tmp))
|
| 580 |
+
except ValueError as exc:
|
| 581 |
+
raise HTTPException(400, str(exc)) from exc
|
| 582 |
+
finally:
|
| 583 |
+
tmp.unlink(missing_ok=True) # item files are never retained
|
| 584 |
+
|
| 585 |
+
stem = Path(file.filename or "").stem.replace("_", " ").replace("-", " ").strip()
|
| 586 |
+
parsed["suggested_name"] = stem.title() if stem else ""
|
| 587 |
+
return parsed
|
| 588 |
+
|
| 589 |
+
|
| 590 |
# -------------------------------------------------------------------- jobs
|
| 591 |
@app.post("/api/jobs", response_model=JobOut, status_code=201)
|
| 592 |
+
def create_job(
|
| 593 |
+
body: JobCreate,
|
| 594 |
+
request: Request,
|
| 595 |
+
response: Response,
|
| 596 |
+
db: Session = Depends(get_db),
|
| 597 |
+
user: dict | None = Depends(auth.get_current_user),
|
| 598 |
+
):
|
| 599 |
+
project = _get_or_404(db, Project, body.project_id)
|
| 600 |
+
_require_project_access(project, user)
|
| 601 |
corpus = _get_or_404(db, Corpus, body.corpus_id)
|
| 602 |
_get_or_404(db, Construct, body.construct_id)
|
| 603 |
|
| 604 |
+
# Anonymous tier: N runs per day, then sign-in (PI decision 2026-07-10).
|
| 605 |
+
# Cookie counter = a nudge, not a security boundary (recorded in DECISIONS.md).
|
| 606 |
+
if user is None:
|
| 607 |
+
used = auth.runs_used_today(request)
|
| 608 |
+
if used >= auth.anon_max_runs_per_day():
|
| 609 |
+
raise HTTPException(
|
| 610 |
+
429,
|
| 611 |
+
f"Anonymous limit reached ({auth.anon_max_runs_per_day()} runs/day). "
|
| 612 |
+
"Sign in (top right) to keep running - accounts are free.",
|
| 613 |
+
)
|
| 614 |
+
else:
|
| 615 |
+
# Signed-in tier: saved-run cap instead of deletion (their data, their call).
|
| 616 |
+
if _saved_runs_used(db, user["id"]) >= auth.user_max_saved_runs():
|
| 617 |
+
raise HTTPException(
|
| 618 |
+
409,
|
| 619 |
+
f"You have {auth.user_max_saved_runs()} saved runs (the maximum). "
|
| 620 |
+
"Delete a project or old runs to start a new analysis.",
|
| 621 |
+
)
|
| 622 |
+
|
| 623 |
if body.text_column not in json.loads(corpus.columns_json):
|
| 624 |
raise HTTPException(400, f"Column '{body.text_column}' not in corpus columns.")
|
| 625 |
+
allowed = registry.known_ids() | {FAKE_MODEL_NAME}
|
| 626 |
if body.model_name not in allowed:
|
| 627 |
raise HTTPException(400, f"Unknown model '{body.model_name}'.")
|
| 628 |
+
language = (body.language or "en").strip().lower()
|
| 629 |
+
if not (2 <= len(language) <= 8 and language.replace("-", "").isalpha()):
|
| 630 |
+
raise HTTPException(400, f"Invalid language code '{body.language}'.")
|
| 631 |
+
|
| 632 |
+
# Retention: anonymous uploads are deleted after their analysis, so a
|
| 633 |
+
# re-run needs a fresh upload (or an account, where data persists).
|
| 634 |
+
if not corpus.path or not storage.exists(corpus.path):
|
| 635 |
+
raise HTTPException(
|
| 636 |
+
410,
|
| 637 |
+
"This dataset's file was removed after analysis (anonymous uploads are "
|
| 638 |
+
"not kept). Upload the file again, or sign in to keep datasets.",
|
| 639 |
+
)
|
| 640 |
|
| 641 |
job = Job(
|
| 642 |
project_id=body.project_id,
|
|
|
|
| 644 |
construct_id=body.construct_id,
|
| 645 |
text_column=body.text_column,
|
| 646 |
model_name=body.model_name,
|
| 647 |
+
language=language,
|
| 648 |
)
|
| 649 |
db.add(job)
|
| 650 |
db.commit()
|
| 651 |
jobs_module.submit_job(job.id)
|
| 652 |
+
|
| 653 |
+
if user is None: # advance the daily counter only after the job is accepted
|
| 654 |
+
response.set_cookie(
|
| 655 |
+
auth.RUNS_COOKIE_NAME,
|
| 656 |
+
auth.run_counter_token(auth.runs_used_today(request) + 1),
|
| 657 |
+
httponly=True,
|
| 658 |
+
samesite="lax",
|
| 659 |
+
secure=auth.cookies_secure(),
|
| 660 |
+
max_age=24 * 3600,
|
| 661 |
+
)
|
| 662 |
return _job_out(db, job)
|
| 663 |
|
| 664 |
|
|
|
|
| 691 |
job = _get_or_404(db, Job, job_id)
|
| 692 |
if job.status != "completed" or not job.result_path:
|
| 693 |
raise HTTPException(409, "Results not available.")
|
| 694 |
+
filename = f"ccr_results_{job_id[:8]}.csv"
|
| 695 |
+
if storage.is_s3(job.result_path):
|
| 696 |
+
from fastapi.responses import StreamingResponse
|
| 697 |
+
|
| 698 |
+
return StreamingResponse(
|
| 699 |
+
storage.open_stream(job.result_path),
|
| 700 |
+
media_type="text/csv",
|
| 701 |
+
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
| 702 |
+
)
|
| 703 |
+
return FileResponse(job.result_path, media_type="text/csv", filename=filename)
|
| 704 |
|
| 705 |
|
| 706 |
@app.get("/api/jobs/{job_id}/metadata")
|
|
|
|
| 716 |
)
|
| 717 |
|
| 718 |
|
| 719 |
+
@app.get("/api/jobs/{job_id}/script")
|
| 720 |
+
def export_script(job_id: str, db: Session = Depends(get_db)):
|
| 721 |
+
"""Offline-runnable reproduction script generated from run metadata (spec 0002)."""
|
| 722 |
+
job = _get_or_404(db, Job, job_id)
|
| 723 |
+
if job.status != "completed":
|
| 724 |
+
raise HTTPException(409, "Script not available until the run completes.")
|
| 725 |
+
return PlainTextResponse(
|
| 726 |
+
script_text(json.loads(job.metadata_json)),
|
| 727 |
+
media_type="text/x-python",
|
| 728 |
+
headers={
|
| 729 |
+
"Content-Disposition": f'attachment; filename="reproduce_analysis_{job_id[:8]}.py"'
|
| 730 |
+
},
|
| 731 |
+
)
|
| 732 |
+
|
| 733 |
+
|
| 734 |
+
@app.get("/api/jobs/{job_id}/script-requirements")
|
| 735 |
+
def export_script_requirements(job_id: str, db: Session = Depends(get_db)):
|
| 736 |
+
job = _get_or_404(db, Job, job_id)
|
| 737 |
+
if job.status != "completed":
|
| 738 |
+
raise HTTPException(409, "Requirements not available until the run completes.")
|
| 739 |
+
return PlainTextResponse(
|
| 740 |
+
requirements_text(json.loads(job.metadata_json)),
|
| 741 |
+
media_type="text/plain",
|
| 742 |
+
headers={
|
| 743 |
+
"Content-Disposition": f'attachment; filename="requirements-repro_{job_id[:8]}.txt"'
|
| 744 |
+
},
|
| 745 |
+
)
|
| 746 |
+
|
| 747 |
+
|
| 748 |
# ------------------------------------------------------------ static (SPA)
|
| 749 |
STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
|
| 750 |
if STATIC_DIR.exists():
|
backend/app/models.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
|
| 3 |
IDs are UUID strings (portable across SQLite/Postgres). JSON-ish payloads
|
| 4 |
(column lists, construct items, job metadata/summaries) are stored as JSON
|
| 5 |
-
text
|
| 6 |
"""
|
| 7 |
|
| 8 |
import uuid
|
|
@@ -22,12 +22,24 @@ def _now() -> str:
|
|
| 22 |
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 23 |
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
class Project(Base):
|
| 26 |
__tablename__ = "projects"
|
| 27 |
|
| 28 |
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uuid)
|
| 29 |
name: Mapped[str] = mapped_column(String(200))
|
| 30 |
description: Mapped[str] = mapped_column(Text, default="")
|
|
|
|
|
|
|
| 31 |
created_at: Mapped[str] = mapped_column(String(32), default=_now)
|
| 32 |
|
| 33 |
|
|
@@ -51,9 +63,18 @@ class Construct(Base):
|
|
| 51 |
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uuid)
|
| 52 |
name: Mapped[str] = mapped_column(String(200))
|
| 53 |
description: Mapped[str] = mapped_column(Text, default="")
|
| 54 |
-
reference: Mapped[str] = mapped_column(Text, default="")
|
| 55 |
items_json: Mapped[str] = mapped_column(Text) # list[str]
|
|
|
|
| 56 |
is_seed: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
created_at: Mapped[str] = mapped_column(String(32), default=_now)
|
| 58 |
|
| 59 |
|
|
@@ -65,7 +86,8 @@ class Job(Base):
|
|
| 65 |
corpus_id: Mapped[str] = mapped_column(ForeignKey("corpora.id"))
|
| 66 |
construct_id: Mapped[str] = mapped_column(ForeignKey("constructs.id"))
|
| 67 |
text_column: Mapped[str] = mapped_column(String(200))
|
| 68 |
-
model_name: Mapped[str] = mapped_column(String(200))
|
|
|
|
| 69 |
status: Mapped[str] = mapped_column(String(20), default="queued")
|
| 70 |
# queued -> running -> completed | failed
|
| 71 |
progress: Mapped[float] = mapped_column(Float, default=0.0) # 0..1
|
|
|
|
| 2 |
|
| 3 |
IDs are UUID strings (portable across SQLite/Postgres). JSON-ish payloads
|
| 4 |
(column lists, construct items, job metadata/summaries) are stored as JSON
|
| 5 |
+
text - they are read-mostly blobs, not queried relationally.
|
| 6 |
"""
|
| 7 |
|
| 8 |
import uuid
|
|
|
|
| 22 |
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
| 23 |
|
| 24 |
|
| 25 |
+
class User(Base):
|
| 26 |
+
__tablename__ = "users"
|
| 27 |
+
|
| 28 |
+
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uuid)
|
| 29 |
+
email: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
| 30 |
+
name: Mapped[str] = mapped_column(String(120), default="")
|
| 31 |
+
password_hash: Mapped[str] = mapped_column(Text) # scrypt$salt$digest (auth.py)
|
| 32 |
+
created_at: Mapped[str] = mapped_column(String(32), default=_now)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
class Project(Base):
|
| 36 |
__tablename__ = "projects"
|
| 37 |
|
| 38 |
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uuid)
|
| 39 |
name: Mapped[str] = mapped_column(String(200))
|
| 40 |
description: Mapped[str] = mapped_column(Text, default="")
|
| 41 |
+
archived: Mapped[bool] = mapped_column(Boolean, default=False)
|
| 42 |
+
owner_user_id: Mapped[str] = mapped_column(String(32), default="") # "" = anonymous
|
| 43 |
created_at: Mapped[str] = mapped_column(String(32), default=_now)
|
| 44 |
|
| 45 |
|
|
|
|
| 63 |
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_uuid)
|
| 64 |
name: Mapped[str] = mapped_column(String(200))
|
| 65 |
description: Mapped[str] = mapped_column(Text, default="")
|
| 66 |
+
reference: Mapped[str] = mapped_column(Text, default="") # citation
|
| 67 |
items_json: Mapped[str] = mapped_column(Text) # list[str]
|
| 68 |
+
reverse_flags_json: Mapped[str] = mapped_column(Text, default="[]") # list[bool], parallel to items
|
| 69 |
is_seed: Mapped[bool] = mapped_column(Boolean, default=False)
|
| 70 |
+
# Library identity (spec 0004): versioned append-only; hash via reference algorithm.
|
| 71 |
+
construct_slug: Mapped[str] = mapped_column(String(120), default="")
|
| 72 |
+
version: Mapped[int] = mapped_column(default=1)
|
| 73 |
+
item_hash: Mapped[str] = mapped_column(String(64), default="")
|
| 74 |
+
verification_status: Mapped[str] = mapped_column(String(24), default="draft")
|
| 75 |
+
# draft | needs_verification | verified | archived
|
| 76 |
+
language: Mapped[str] = mapped_column(String(12), default="en")
|
| 77 |
+
category: Mapped[str] = mapped_column(String(80), default="")
|
| 78 |
created_at: Mapped[str] = mapped_column(String(32), default=_now)
|
| 79 |
|
| 80 |
|
|
|
|
| 86 |
corpus_id: Mapped[str] = mapped_column(ForeignKey("corpora.id"))
|
| 87 |
construct_id: Mapped[str] = mapped_column(ForeignKey("constructs.id"))
|
| 88 |
text_column: Mapped[str] = mapped_column(String(200))
|
| 89 |
+
model_name: Mapped[str] = mapped_column(String(200)) # registry id (or test fake)
|
| 90 |
+
language: Mapped[str] = mapped_column(String(12), default="en") # selected analysis language
|
| 91 |
status: Mapped[str] = mapped_column(String(20), default="queued")
|
| 92 |
# queued -> running -> completed | failed
|
| 93 |
progress: Mapped[float] = mapped_column(Float, default=0.0) # 0..1
|
backend/app/registry.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model registry loader - the app-side reader of packages/model_registry/models.yaml.
|
| 2 |
+
|
| 3 |
+
Single source of model truth (design doc §13): the UI dropdown, backend validation,
|
| 4 |
+
prefix handling, language-support warnings, run metadata, and generated reproduction
|
| 5 |
+
scripts all read the SAME config through this module. No model behavior is hardcoded
|
| 6 |
+
anywhere else (CLAUDE.md hard rule).
|
| 7 |
+
|
| 8 |
+
Language sets are resolved through packages/model_registry/language_sets.py, loaded
|
| 9 |
+
by explicit file path (editable-install wiring arrives with Phase 1 packaging).
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import importlib.util
|
| 15 |
+
import os
|
| 16 |
+
from dataclasses import dataclass, field
|
| 17 |
+
from functools import lru_cache
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import yaml
|
| 21 |
+
|
| 22 |
+
REPO_ROOT = Path(__file__).resolve().parents[2]
|
| 23 |
+
MODELS_YAML = Path(os.environ.get("CCR_MODELS_YAML", REPO_ROOT / "packages" / "model_registry" / "models.yaml"))
|
| 24 |
+
_LANGUAGE_SETS_PY = REPO_ROOT / "packages" / "model_registry" / "language_sets.py"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _load_language_sets():
|
| 28 |
+
spec = importlib.util.spec_from_file_location("ccr_language_sets", _LANGUAGE_SETS_PY)
|
| 29 |
+
module = importlib.util.module_from_spec(spec)
|
| 30 |
+
spec.loader.exec_module(module)
|
| 31 |
+
return module
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@dataclass(frozen=True)
|
| 35 |
+
class ModelConfig:
|
| 36 |
+
id: str
|
| 37 |
+
provider_model_id: str
|
| 38 |
+
display_name: str
|
| 39 |
+
revision: str
|
| 40 |
+
default: bool
|
| 41 |
+
supported_languages: frozenset[str] # resolved ISO codes; frozenset() = unknown/any
|
| 42 |
+
language_set_name: str | None
|
| 43 |
+
embedding_dimension: int
|
| 44 |
+
max_seq_length: int
|
| 45 |
+
quality_tier: str
|
| 46 |
+
speed_tier: str
|
| 47 |
+
requires_prefix: bool
|
| 48 |
+
item_prefix: str
|
| 49 |
+
text_prefix: str
|
| 50 |
+
normalize_embeddings: bool
|
| 51 |
+
lazy_load: bool
|
| 52 |
+
user_warnings: tuple[str, ...] = field(default_factory=tuple)
|
| 53 |
+
|
| 54 |
+
@property
|
| 55 |
+
def pinned_revision(self) -> str | None:
|
| 56 |
+
"""Revision to pass to the model loader; None while unpinned (PIN_ME)."""
|
| 57 |
+
return None if self.revision in ("PIN_ME", "", None) else self.revision
|
| 58 |
+
|
| 59 |
+
def supports_language(self, iso_code: str) -> bool:
|
| 60 |
+
if not self.supported_languages: # unknown coverage - never block/warn on it
|
| 61 |
+
return True
|
| 62 |
+
return iso_code.lower() in self.supported_languages
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _parse_model(raw: dict, lang_sets) -> ModelConfig:
|
| 66 |
+
usage = raw.get("usage_config", {})
|
| 67 |
+
ops = raw.get("operational_config", {})
|
| 68 |
+
set_name = raw.get("supported_language_set")
|
| 69 |
+
if set_name:
|
| 70 |
+
languages = frozenset(lang_sets.resolve(set_name))
|
| 71 |
+
else:
|
| 72 |
+
languages = frozenset(str(c).lower() for c in raw.get("supported_languages", []))
|
| 73 |
+
|
| 74 |
+
sym = usage.get("symmetric_similarity_prefix") or ""
|
| 75 |
+
return ModelConfig(
|
| 76 |
+
id=raw["id"],
|
| 77 |
+
provider_model_id=raw["provider_model_id"],
|
| 78 |
+
display_name=raw.get("display_name", raw["id"]),
|
| 79 |
+
revision=str(raw.get("revision", "PIN_ME")),
|
| 80 |
+
default=bool(raw.get("default", False)),
|
| 81 |
+
supported_languages=languages,
|
| 82 |
+
language_set_name=set_name,
|
| 83 |
+
embedding_dimension=int(raw["embedding_dimension"]),
|
| 84 |
+
max_seq_length=int(raw["max_seq_length"]),
|
| 85 |
+
quality_tier=str(raw.get("quality_tier", "unknown")),
|
| 86 |
+
speed_tier=str(raw.get("speed_tier", "unknown")),
|
| 87 |
+
requires_prefix=bool(usage.get("requires_prefix", False)),
|
| 88 |
+
item_prefix=usage.get("construct_prefix") or sym or "",
|
| 89 |
+
text_prefix=usage.get("text_prefix") or sym or "",
|
| 90 |
+
normalize_embeddings=bool(usage.get("normalize_embeddings", True)),
|
| 91 |
+
lazy_load=bool(ops.get("lazy_load", False)),
|
| 92 |
+
user_warnings=tuple(raw.get("warnings", []) or []),
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
@lru_cache(maxsize=1)
|
| 97 |
+
def _registry() -> dict[str, ModelConfig]:
|
| 98 |
+
data = yaml.safe_load(MODELS_YAML.read_text())
|
| 99 |
+
lang_sets = _load_language_sets()
|
| 100 |
+
models = {}
|
| 101 |
+
for raw in data.get("models", []):
|
| 102 |
+
cfg = _parse_model(raw, lang_sets)
|
| 103 |
+
if cfg.requires_prefix and not (cfg.item_prefix and cfg.text_prefix):
|
| 104 |
+
raise ValueError(f"models.yaml: {cfg.id} requires_prefix but prefixes missing.")
|
| 105 |
+
models[cfg.id] = cfg
|
| 106 |
+
defaults = [m for m in models.values() if m.default]
|
| 107 |
+
if len(defaults) != 1:
|
| 108 |
+
raise ValueError(f"models.yaml must define exactly one default model, found {len(defaults)}.")
|
| 109 |
+
return models
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def reload() -> None:
|
| 113 |
+
"""Clear the cache (tests / config changes)."""
|
| 114 |
+
_registry.cache_clear()
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def list_models() -> list[ModelConfig]:
|
| 118 |
+
ordered = sorted(_registry().values(), key=lambda m: (not m.default, m.id))
|
| 119 |
+
return ordered
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def get_model(model_id: str) -> ModelConfig:
|
| 123 |
+
reg = _registry()
|
| 124 |
+
if model_id not in reg:
|
| 125 |
+
raise KeyError(f"Unknown model id '{model_id}'. Known: {sorted(reg)}")
|
| 126 |
+
return reg[model_id]
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def default_model() -> ModelConfig:
|
| 130 |
+
return next(m for m in _registry().values() if m.default)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def known_ids() -> set[str]:
|
| 134 |
+
return set(_registry().keys())
|
backend/app/reproducibility.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reproduction-script generation (spec 0002, design doc §14).
|
| 2 |
+
|
| 3 |
+
The generated script is built ONLY from the run's stored metadata - never from live
|
| 4 |
+
state - so it reproduces what actually ran. It must be runnable outside the platform:
|
| 5 |
+
input CSV + Python + internet for the (pinned) model download. No platform credentials.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import json
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _pinned_requirements(metadata: dict) -> list[str]:
|
| 14 |
+
"""Exact versions of the packages the analysis math depends on."""
|
| 15 |
+
from importlib.metadata import PackageNotFoundError, version
|
| 16 |
+
|
| 17 |
+
pins = []
|
| 18 |
+
for pkg in ("sentence-transformers", "torch", "numpy", "pandas"):
|
| 19 |
+
try:
|
| 20 |
+
pins.append(f"{pkg}=={version(pkg)}")
|
| 21 |
+
except PackageNotFoundError:
|
| 22 |
+
continue
|
| 23 |
+
return pins
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def requirements_text(metadata: dict) -> str:
|
| 27 |
+
lines = [
|
| 28 |
+
"# Reproduction environment for CCR run " + metadata.get("job_id", "?"),
|
| 29 |
+
"# Install: pip install -r requirements-repro.txt",
|
| 30 |
+
]
|
| 31 |
+
recorded = metadata.get("environment_pins")
|
| 32 |
+
lines += recorded if recorded else _pinned_requirements(metadata)
|
| 33 |
+
return "\n".join(lines) + "\n"
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def record_environment(metadata: dict) -> dict:
|
| 37 |
+
"""Store pins in metadata at run time so exports match the executing environment."""
|
| 38 |
+
metadata["environment_pins"] = _pinned_requirements(metadata)
|
| 39 |
+
return metadata
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def script_text(metadata: dict) -> str:
|
| 43 |
+
"""Standalone Python script reproducing the run's similarities and scores."""
|
| 44 |
+
construct = metadata.get("construct_snapshot", {})
|
| 45 |
+
items = construct.get("items", [])
|
| 46 |
+
model_id = metadata.get("model_registry_id", metadata.get("model", ""))
|
| 47 |
+
provider = metadata.get("provider_model_id", metadata.get("model", ""))
|
| 48 |
+
revision = metadata.get("model_revision")
|
| 49 |
+
revision_arg = f", revision={revision!r}" if revision and revision != "PIN_ME" else ""
|
| 50 |
+
item_prefix = metadata.get("item_prefix", "")
|
| 51 |
+
text_prefix = metadata.get("text_prefix", "")
|
| 52 |
+
text_column = metadata.get("text_column", "text")
|
| 53 |
+
scoring = metadata.get("scoring", {})
|
| 54 |
+
|
| 55 |
+
items_literal = json.dumps(
|
| 56 |
+
[{"text": i["text"], "reverse_scored": i.get("reverse_scored", False)} for i in items],
|
| 57 |
+
indent=4, ensure_ascii=False,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
return f'''#!/usr/bin/env python3
|
| 61 |
+
"""Reproduce CCR analysis independently of the platform.
|
| 62 |
+
|
| 63 |
+
run_id: {metadata.get("job_id", "?")}
|
| 64 |
+
created_at: {metadata.get("started_at", "?")}
|
| 65 |
+
platform_version: {metadata.get("platform_version", "?")}
|
| 66 |
+
output_schema_version: {metadata.get("output_schema_version", "1.0")}
|
| 67 |
+
construct: {construct.get("name", "?")} (v{construct.get("version", "?")}, hash {construct.get("item_hash", "?")[:16]})
|
| 68 |
+
model: {model_id} -> {provider} (revision: {revision or "unpinned"})
|
| 69 |
+
scoring: adjustment_strategy={scoring.get("adjustment_strategy", "none")}, aggregate={scoring.get("aggregate", "mean_all_items")}
|
| 70 |
+
|
| 71 |
+
Usage:
|
| 72 |
+
pip install -r requirements-repro.txt
|
| 73 |
+
python reproduce_analysis.py your_corpus.csv
|
| 74 |
+
Outputs reproduced_results.csv with the same similarity columns as the platform export.
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
import sys
|
| 78 |
+
|
| 79 |
+
import numpy as np
|
| 80 |
+
import pandas as pd
|
| 81 |
+
from sentence_transformers import SentenceTransformer
|
| 82 |
+
|
| 83 |
+
TEXT_COLUMN = {text_column!r}
|
| 84 |
+
ITEM_PREFIX = {item_prefix!r} # model-required prefix (E5 family); empty = none
|
| 85 |
+
TEXT_PREFIX = {text_prefix!r}
|
| 86 |
+
|
| 87 |
+
ITEMS = {items_literal}
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def main(csv_path: str) -> None:
|
| 91 |
+
df = pd.read_csv(csv_path)
|
| 92 |
+
texts_all = df[TEXT_COLUMN].astype("string")
|
| 93 |
+
mask = texts_all.notna() & (texts_all.str.strip() != "")
|
| 94 |
+
work = df.loc[mask].reset_index(drop=True) # platform drops empty rows the same way
|
| 95 |
+
texts = work[TEXT_COLUMN].astype(str).tolist()
|
| 96 |
+
|
| 97 |
+
model = SentenceTransformer({provider!r}{revision_arg})
|
| 98 |
+
item_texts = [ITEM_PREFIX + i["text"] for i in ITEMS]
|
| 99 |
+
doc_texts = [TEXT_PREFIX + t for t in texts]
|
| 100 |
+
|
| 101 |
+
item_emb = model.encode(item_texts, convert_to_numpy=True, normalize_embeddings=True)
|
| 102 |
+
doc_emb = model.encode(doc_texts, convert_to_numpy=True, normalize_embeddings=True)
|
| 103 |
+
|
| 104 |
+
sims = doc_emb @ item_emb.T # normalized -> cosine similarity
|
| 105 |
+
for j in range(sims.shape[1]):
|
| 106 |
+
work[f"sim_item_{{j + 1}}"] = np.round(sims[:, j], 6)
|
| 107 |
+
work["ccr_score"] = np.round(sims.mean(axis=1), 6)
|
| 108 |
+
|
| 109 |
+
work.to_csv("reproduced_results.csv", index=False)
|
| 110 |
+
print(f"Wrote reproduced_results.csv ({{len(work)}} rows, {{sims.shape[1]}} items).")
|
| 111 |
+
print("Compare against the platform export - values should match to ~1e-5.")
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
if __name__ == "__main__":
|
| 115 |
+
if len(sys.argv) != 2:
|
| 116 |
+
raise SystemExit("Usage: python reproduce_analysis.py <corpus.csv>")
|
| 117 |
+
main(sys.argv[1])
|
| 118 |
+
'''
|
backend/app/retention.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Data retention (PI decision, 2026-07-10: "remove temp data after analysis").
|
| 2 |
+
|
| 3 |
+
Policy:
|
| 4 |
+
* ANONYMOUS runs: the uploaded corpus file (and its embedding cache) is
|
| 5 |
+
deleted the moment the run finishes (jobs.py calls remove_corpus_files).
|
| 6 |
+
Result summaries/CSVs stick around so the person can download them, then
|
| 7 |
+
the whole anonymous project is purged after CCR_ANON_TTL_HOURS.
|
| 8 |
+
* SIGNED-IN runs: nothing is auto-deleted; a saved-run cap applies instead
|
| 9 |
+
(enforced at job creation in main.py - the user chooses what to delete).
|
| 10 |
+
|
| 11 |
+
The purge loop runs in a daemon thread (startup + hourly). TTL of 0 disables
|
| 12 |
+
purging entirely - the local-dev default, so nobody's dev projects vanish
|
| 13 |
+
overnight. Deployments set CCR_ANON_TTL_HOURS=24.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import logging
|
| 19 |
+
import threading
|
| 20 |
+
from datetime import datetime, timedelta, timezone
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
from sqlalchemy.orm import Session
|
| 24 |
+
|
| 25 |
+
from . import auth, storage
|
| 26 |
+
from .db import DATA_DIR, SessionLocal
|
| 27 |
+
from .models import Corpus, Job, Project
|
| 28 |
+
|
| 29 |
+
logger = logging.getLogger("ccr.retention")
|
| 30 |
+
|
| 31 |
+
EMB_CACHE_DIR = DATA_DIR / "emb_cache"
|
| 32 |
+
EMB_CACHE_DIR.mkdir(exist_ok=True)
|
| 33 |
+
|
| 34 |
+
_stop = threading.Event()
|
| 35 |
+
_thread: threading.Thread | None = None
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def remove_corpus_files(corpus: Corpus) -> None:
|
| 39 |
+
"""Delete the uploaded file (whatever backend holds it) and any cached
|
| 40 |
+
embeddings (always local - caches are derived data)."""
|
| 41 |
+
storage.delete(corpus.path)
|
| 42 |
+
for cached in EMB_CACHE_DIR.glob(f"{corpus.id}_*.npy"):
|
| 43 |
+
cached.unlink(missing_ok=True)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def delete_project_cascade(db: Session, project: Project) -> dict:
|
| 47 |
+
"""Shared cascade used by the DELETE endpoint and the anonymous purge.
|
| 48 |
+
Removes DB rows plus uploaded, result, and embedding-cache files. Logs
|
| 49 |
+
counts only - never any uploaded text (design doc §9)."""
|
| 50 |
+
corpora = db.query(Corpus).filter_by(project_id=project.id).all()
|
| 51 |
+
jobs = db.query(Job).filter_by(project_id=project.id).all()
|
| 52 |
+
|
| 53 |
+
for corpus in corpora:
|
| 54 |
+
remove_corpus_files(corpus)
|
| 55 |
+
for job in jobs:
|
| 56 |
+
storage.delete(job.result_path)
|
| 57 |
+
|
| 58 |
+
for job in jobs:
|
| 59 |
+
db.delete(job)
|
| 60 |
+
for corpus in corpora:
|
| 61 |
+
db.delete(corpus)
|
| 62 |
+
db.delete(project)
|
| 63 |
+
db.commit()
|
| 64 |
+
return {"corpora": len(corpora), "runs": len(jobs)}
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def purge_expired_anonymous(db: Session) -> int:
|
| 68 |
+
"""Delete anonymous projects whose last activity is older than the TTL."""
|
| 69 |
+
ttl = auth.anon_ttl_hours()
|
| 70 |
+
if ttl <= 0:
|
| 71 |
+
return 0
|
| 72 |
+
cutoff = (datetime.now(timezone.utc) - timedelta(hours=ttl)).isoformat(timespec="seconds")
|
| 73 |
+
|
| 74 |
+
purged = 0
|
| 75 |
+
candidates = db.query(Project).filter(Project.owner_user_id == "").all()
|
| 76 |
+
for project in candidates:
|
| 77 |
+
latest_job = (
|
| 78 |
+
db.query(Job.created_at)
|
| 79 |
+
.filter_by(project_id=project.id)
|
| 80 |
+
.order_by(Job.created_at.desc())
|
| 81 |
+
.first()
|
| 82 |
+
)
|
| 83 |
+
last_activity = max(project.created_at, latest_job[0]) if latest_job else project.created_at
|
| 84 |
+
if last_activity < cutoff:
|
| 85 |
+
counts = delete_project_cascade(db, project)
|
| 86 |
+
purged += 1
|
| 87 |
+
logger.info(
|
| 88 |
+
"purged expired anonymous project id=%s (corpora=%d runs=%d, ttl=%dh)",
|
| 89 |
+
project.id, counts["corpora"], counts["runs"], ttl,
|
| 90 |
+
)
|
| 91 |
+
return purged
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _loop(interval_seconds: int) -> None:
|
| 95 |
+
while not _stop.wait(interval_seconds):
|
| 96 |
+
db = SessionLocal()
|
| 97 |
+
try:
|
| 98 |
+
purge_expired_anonymous(db)
|
| 99 |
+
except Exception:
|
| 100 |
+
logger.exception("anonymous purge failed; will retry next cycle")
|
| 101 |
+
finally:
|
| 102 |
+
db.close()
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def start_cleanup(interval_seconds: int = 3600) -> None:
|
| 106 |
+
"""Run one purge now, then hourly in a daemon thread. No-op if TTL is 0."""
|
| 107 |
+
global _thread
|
| 108 |
+
db = SessionLocal()
|
| 109 |
+
try:
|
| 110 |
+
purge_expired_anonymous(db)
|
| 111 |
+
except Exception:
|
| 112 |
+
logger.exception("startup anonymous purge failed")
|
| 113 |
+
finally:
|
| 114 |
+
db.close()
|
| 115 |
+
if auth.anon_ttl_hours() > 0 and (_thread is None or not _thread.is_alive()):
|
| 116 |
+
_stop.clear()
|
| 117 |
+
_thread = threading.Thread(target=_loop, args=(interval_seconds,), daemon=True,
|
| 118 |
+
name="ccr-retention")
|
| 119 |
+
_thread.start()
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def stop_cleanup() -> None:
|
| 123 |
+
_stop.set()
|
backend/app/schemas.py
CHANGED
|
@@ -13,6 +13,24 @@ class ProjectOut(BaseModel):
|
|
| 13 |
name: str
|
| 14 |
description: str
|
| 15 |
created_at: str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
|
| 18 |
class CorpusOut(BaseModel):
|
|
@@ -32,6 +50,8 @@ class ConstructCreate(BaseModel):
|
|
| 32 |
description: str = ""
|
| 33 |
reference: str = ""
|
| 34 |
items: list[str] = Field(min_length=1)
|
|
|
|
|
|
|
| 35 |
|
| 36 |
|
| 37 |
class ConstructOut(BaseModel):
|
|
@@ -40,7 +60,13 @@ class ConstructOut(BaseModel):
|
|
| 40 |
description: str
|
| 41 |
reference: str
|
| 42 |
items: list[str]
|
|
|
|
| 43 |
is_seed: bool
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
|
| 46 |
class JobCreate(BaseModel):
|
|
@@ -48,7 +74,8 @@ class JobCreate(BaseModel):
|
|
| 48 |
corpus_id: str
|
| 49 |
construct_id: str
|
| 50 |
text_column: str
|
| 51 |
-
model_name: str = "
|
|
|
|
| 52 |
|
| 53 |
|
| 54 |
class JobOut(BaseModel):
|
|
@@ -60,6 +87,7 @@ class JobOut(BaseModel):
|
|
| 60 |
corpus_filename: str = ""
|
| 61 |
text_column: str
|
| 62 |
model_name: str
|
|
|
|
| 63 |
status: str
|
| 64 |
progress: float
|
| 65 |
error: str
|
|
|
|
| 13 |
name: str
|
| 14 |
description: str
|
| 15 |
created_at: str
|
| 16 |
+
last_activity_at: str = "" # latest run creation, else project creation
|
| 17 |
+
n_runs: int = 0
|
| 18 |
+
archived: bool = False
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ProjectPatch(BaseModel):
|
| 22 |
+
archived: bool | None = None
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class RegisterIn(BaseModel):
|
| 26 |
+
email: str = Field(min_length=3, max_length=255)
|
| 27 |
+
password: str = Field(min_length=8, max_length=200)
|
| 28 |
+
name: str = Field(min_length=1, max_length=120)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class LoginIn(BaseModel):
|
| 32 |
+
email: str = Field(min_length=3, max_length=255)
|
| 33 |
+
password: str = Field(min_length=1, max_length=200)
|
| 34 |
|
| 35 |
|
| 36 |
class CorpusOut(BaseModel):
|
|
|
|
| 50 |
description: str = ""
|
| 51 |
reference: str = ""
|
| 52 |
items: list[str] = Field(min_length=1)
|
| 53 |
+
reverse_scored: list[bool] | None = None # parallel to items; defaults to all False
|
| 54 |
+
language: str = "en"
|
| 55 |
|
| 56 |
|
| 57 |
class ConstructOut(BaseModel):
|
|
|
|
| 60 |
description: str
|
| 61 |
reference: str
|
| 62 |
items: list[str]
|
| 63 |
+
reverse_scored: list[bool] = []
|
| 64 |
is_seed: bool
|
| 65 |
+
version: int = 1
|
| 66 |
+
verification_status: str = "draft"
|
| 67 |
+
language: str = "en"
|
| 68 |
+
category: str = ""
|
| 69 |
+
item_hash: str = "" # first 16 hex chars for display
|
| 70 |
|
| 71 |
|
| 72 |
class JobCreate(BaseModel):
|
|
|
|
| 74 |
corpus_id: str
|
| 75 |
construct_id: str
|
| 76 |
text_column: str
|
| 77 |
+
model_name: str = "all-minilm-l6-v2" # registry id (spec 0003)
|
| 78 |
+
language: str = "en"
|
| 79 |
|
| 80 |
|
| 81 |
class JobOut(BaseModel):
|
|
|
|
| 87 |
corpus_filename: str = ""
|
| 88 |
text_column: str
|
| 89 |
model_name: str
|
| 90 |
+
language: str = "en"
|
| 91 |
status: str
|
| 92 |
progress: float
|
| 93 |
error: str
|
backend/app/seed_constructs.py
DELETED
|
@@ -1,66 +0,0 @@
|
|
| 1 |
-
"""Seed construct library — psychometrically validated scales.
|
| 2 |
-
|
| 3 |
-
IMPORTANT: item wordings below are seeded for demonstration. Before any
|
| 4 |
-
research use, verify each item verbatim against the cited original
|
| 5 |
-
publication (CCR's validity depends on using the validated instrument
|
| 6 |
-
as published).
|
| 7 |
-
"""
|
| 8 |
-
|
| 9 |
-
SEED_CONSTRUCTS = [
|
| 10 |
-
{
|
| 11 |
-
"name": "Satisfaction with Life",
|
| 12 |
-
"description": "Global cognitive judgment of one's life satisfaction (SWLS).",
|
| 13 |
-
"reference": "Diener, E., Emmons, R. A., Larsen, R. J., & Griffin, S. (1985). The Satisfaction with Life Scale. Journal of Personality Assessment, 49(1).",
|
| 14 |
-
"items": [
|
| 15 |
-
"In most ways my life is close to my ideal.",
|
| 16 |
-
"The conditions of my life are excellent.",
|
| 17 |
-
"I am satisfied with my life.",
|
| 18 |
-
"So far I have gotten the important things I want in life.",
|
| 19 |
-
"If I could live my life over, I would change almost nothing.",
|
| 20 |
-
],
|
| 21 |
-
},
|
| 22 |
-
{
|
| 23 |
-
"name": "Moral Foundations — Care",
|
| 24 |
-
"description": "Concern with suffering, compassion, and protection of the vulnerable (MFQ Care/Harm foundation).",
|
| 25 |
-
"reference": "Graham, J., Nosek, B. A., Haidt, J., Iyer, R., Koleva, S., & Ditto, P. H. (2011). Mapping the moral domain. JPSP, 101(2). Verify items against the published MFQ.",
|
| 26 |
-
"items": [
|
| 27 |
-
"Compassion for those who are suffering is the most crucial virtue.",
|
| 28 |
-
"One of the worst things a person could do is hurt a defenseless animal.",
|
| 29 |
-
"Whether or not someone suffered emotionally.",
|
| 30 |
-
"Whether or not someone cared for someone weak or vulnerable.",
|
| 31 |
-
],
|
| 32 |
-
},
|
| 33 |
-
{
|
| 34 |
-
"name": "Moral Foundations — Fairness",
|
| 35 |
-
"description": "Concern with justice, rights, and equal treatment (MFQ Fairness/Cheating foundation).",
|
| 36 |
-
"reference": "Graham, J., Nosek, B. A., Haidt, J., Iyer, R., Koleva, S., & Ditto, P. H. (2011). Mapping the moral domain. JPSP, 101(2). Verify items against the published MFQ.",
|
| 37 |
-
"items": [
|
| 38 |
-
"Justice is the most important requirement for a society.",
|
| 39 |
-
"When the government makes laws, the number one principle should be ensuring that everyone is treated fairly.",
|
| 40 |
-
"Whether or not some people were treated differently than others.",
|
| 41 |
-
"Whether or not someone acted unfairly.",
|
| 42 |
-
],
|
| 43 |
-
},
|
| 44 |
-
{
|
| 45 |
-
"name": "Individualism (Horizontal)",
|
| 46 |
-
"description": "Self-reliance and independence from in-groups (Triandis & Gelfand horizontal individualism).",
|
| 47 |
-
"reference": "Triandis, H. C., & Gelfand, M. J. (1998). Converging measurement of horizontal and vertical individualism and collectivism. JPSP, 74(1). Verify items against the published scale.",
|
| 48 |
-
"items": [
|
| 49 |
-
"I'd rather depend on myself than others.",
|
| 50 |
-
"I rely on myself most of the time; I rarely rely on others.",
|
| 51 |
-
"I often do my own thing.",
|
| 52 |
-
"My personal identity, independent of others, is very important to me.",
|
| 53 |
-
],
|
| 54 |
-
},
|
| 55 |
-
{
|
| 56 |
-
"name": "Collectivism (Horizontal)",
|
| 57 |
-
"description": "Interdependence, cooperation, and in-group well-being (Triandis & Gelfand horizontal collectivism).",
|
| 58 |
-
"reference": "Triandis, H. C., & Gelfand, M. J. (1998). Converging measurement of horizontal and vertical individualism and collectivism. JPSP, 74(1). Verify items against the published scale.",
|
| 59 |
-
"items": [
|
| 60 |
-
"If a coworker gets a prize, I would feel proud.",
|
| 61 |
-
"The well-being of my coworkers is important to me.",
|
| 62 |
-
"To me, pleasure is spending time with others.",
|
| 63 |
-
"I feel good when I cooperate with others.",
|
| 64 |
-
],
|
| 65 |
-
},
|
| 66 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
backend/app/storage.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""File storage behind one interface: local disk (default) or S3-compatible.
|
| 2 |
+
|
| 3 |
+
Production-ready now, enabled by configuration at deploy time (Deva,
|
| 4 |
+
2026-07-13): the R2/S3 code path ships with the codebase so flipping a
|
| 5 |
+
production instance to object storage is an env change, never a development
|
| 6 |
+
task. Local dev keeps writing plain files under CCR_DATA_DIR.
|
| 7 |
+
|
| 8 |
+
Locator scheme (stored in the DB's existing path columns - no migration):
|
| 9 |
+
* local backend: an absolute filesystem path (exactly as before);
|
| 10 |
+
* s3 backend: "s3://{key}" inside the configured bucket.
|
| 11 |
+
Old rows with absolute paths keep working even on an s3-configured instance.
|
| 12 |
+
|
| 13 |
+
Config (s3 backend): CCR_STORAGE=s3, CCR_S3_ENDPOINT (R2: the account
|
| 14 |
+
endpoint URL), CCR_S3_BUCKET, CCR_S3_ACCESS_KEY_ID, CCR_S3_SECRET_ACCESS_KEY.
|
| 15 |
+
The bucket stays private; downloads stream through the API, so no public
|
| 16 |
+
access or presigned-URL exposure is required.
|
| 17 |
+
|
| 18 |
+
Embedding caches deliberately stay on local disk: they are derived data,
|
| 19 |
+
cheap to recompute, and read with numpy - a cache does not need durability.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import os
|
| 25 |
+
import tempfile
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
|
| 28 |
+
from .db import DATA_DIR
|
| 29 |
+
|
| 30 |
+
S3_PREFIX = "s3://"
|
| 31 |
+
|
| 32 |
+
_client = None # injectable for tests
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def backend() -> str:
|
| 36 |
+
return os.environ.get("CCR_STORAGE", "local").lower()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _s3():
|
| 40 |
+
global _client
|
| 41 |
+
if _client is None:
|
| 42 |
+
import boto3 # lazy: only s3-configured deployments need it
|
| 43 |
+
|
| 44 |
+
_client = boto3.client(
|
| 45 |
+
"s3",
|
| 46 |
+
endpoint_url=os.environ["CCR_S3_ENDPOINT"],
|
| 47 |
+
aws_access_key_id=os.environ["CCR_S3_ACCESS_KEY_ID"],
|
| 48 |
+
aws_secret_access_key=os.environ["CCR_S3_SECRET_ACCESS_KEY"],
|
| 49 |
+
region_name=os.environ.get("CCR_S3_REGION", "auto"),
|
| 50 |
+
)
|
| 51 |
+
return _client
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _bucket() -> str:
|
| 55 |
+
return os.environ["CCR_S3_BUCKET"]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def is_s3(locator: str) -> bool:
|
| 59 |
+
return locator.startswith(S3_PREFIX)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
# ------------------------------------------------------------------ write
|
| 63 |
+
def store_bytes(category: str, name: str, data: bytes) -> str:
|
| 64 |
+
"""Persist bytes under category/name; return the locator to store in the DB."""
|
| 65 |
+
key = f"{category}/{name}"
|
| 66 |
+
if backend() == "s3":
|
| 67 |
+
_s3().put_object(Bucket=_bucket(), Key=key, Body=data)
|
| 68 |
+
return S3_PREFIX + key
|
| 69 |
+
dest = DATA_DIR / category / name
|
| 70 |
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
| 71 |
+
dest.write_bytes(data)
|
| 72 |
+
return str(dest)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def store_file(category: str, name: str, src: Path) -> str:
|
| 76 |
+
return store_bytes(category, name, Path(src).read_bytes())
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
# ------------------------------------------------------------------- read
|
| 80 |
+
def exists(locator: str) -> bool:
|
| 81 |
+
if not locator:
|
| 82 |
+
return False
|
| 83 |
+
if is_s3(locator):
|
| 84 |
+
try:
|
| 85 |
+
_s3().head_object(Bucket=_bucket(), Key=locator[len(S3_PREFIX):])
|
| 86 |
+
return True
|
| 87 |
+
except Exception:
|
| 88 |
+
return False
|
| 89 |
+
return Path(locator).exists()
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def fetch_to_local(locator: str) -> tuple[Path, bool]:
|
| 93 |
+
"""Return (local_path, is_temporary). Caller unlinks temporary files
|
| 94 |
+
after use; local-backend paths are returned as-is."""
|
| 95 |
+
if is_s3(locator):
|
| 96 |
+
key = locator[len(S3_PREFIX):]
|
| 97 |
+
suffix = Path(key).suffix or ".bin"
|
| 98 |
+
fd, tmp = tempfile.mkstemp(suffix=suffix, prefix="ccr_s3_")
|
| 99 |
+
os.close(fd)
|
| 100 |
+
_s3().download_file(_bucket(), key, tmp)
|
| 101 |
+
return Path(tmp), True
|
| 102 |
+
return Path(locator), False
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def open_stream(locator: str):
|
| 106 |
+
"""Iterator of byte chunks, for streaming downloads through the API."""
|
| 107 |
+
if is_s3(locator):
|
| 108 |
+
body = _s3().get_object(Bucket=_bucket(), Key=locator[len(S3_PREFIX):])["Body"]
|
| 109 |
+
return iter(lambda: body.read(64 * 1024), b"")
|
| 110 |
+
fh = open(locator, "rb")
|
| 111 |
+
|
| 112 |
+
def gen():
|
| 113 |
+
with fh:
|
| 114 |
+
while chunk := fh.read(64 * 1024):
|
| 115 |
+
yield chunk
|
| 116 |
+
|
| 117 |
+
return gen()
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
# ------------------------------------------------------------------ delete
|
| 121 |
+
def delete(locator: str) -> None:
|
| 122 |
+
if not locator:
|
| 123 |
+
return
|
| 124 |
+
if is_s3(locator):
|
| 125 |
+
try:
|
| 126 |
+
_s3().delete_object(Bucket=_bucket(), Key=locator[len(S3_PREFIX):])
|
| 127 |
+
except Exception:
|
| 128 |
+
pass # deletion is best-effort; the TTL sweep retries implicitly
|
| 129 |
+
return
|
| 130 |
+
Path(locator).unlink(missing_ok=True)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def move_local_into_storage(category: str, name: str, local_path: Path) -> str:
|
| 134 |
+
"""Store a locally produced file; the source copy is removed unless it IS
|
| 135 |
+
the stored destination (local backend writing in place)."""
|
| 136 |
+
locator = store_file(category, name, local_path)
|
| 137 |
+
src = Path(local_path)
|
| 138 |
+
if is_s3(locator) or (src.exists() and str(src.resolve()) != str(Path(locator).resolve())):
|
| 139 |
+
src.unlink(missing_ok=True)
|
| 140 |
+
return locator
|
backend/app/warnings_engine.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Structured data-quality warnings (spec 0001, design doc §12).
|
| 2 |
+
|
| 3 |
+
Every warning is an object - {code, severity, message, count?, affected_rows_sample?} -
|
| 4 |
+
never a bare string. Codes are UPPER_SNAKE and stable: downstream notebooks and the UI
|
| 5 |
+
key off them. Severity: "info" (status, not a problem) | "warning" (proceed with care).
|
| 6 |
+
Language detection is corpus-level only; short texts are exactly where detection is
|
| 7 |
+
unreliable, so uncertainty is reported instead of guessed away.
|
| 8 |
+
|
| 9 |
+
Deviation from spec 0001 (recorded there): langdetect instead of lingua - pure-Python,
|
| 10 |
+
~1 MB vs ~100 MB wheels; seeded for determinism. Upgrade path preserved by recording
|
| 11 |
+
detector + version in metadata.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
from dataclasses import dataclass
|
| 17 |
+
|
| 18 |
+
MIN_TOKENS_STABLE = 4 # texts below this are flagged TEXT_TOO_SHORT
|
| 19 |
+
DETECT_MIN_TOKENS = 5 # rows shorter than this are skipped for detection
|
| 20 |
+
DETECT_SAMPLE_MAX = 200 # rows sampled for corpus-level detection
|
| 21 |
+
DETECT_MIN_ROWS = 20 # fewer detectable rows -> LANGUAGE_UNCERTAIN
|
| 22 |
+
DETECT_CONFIDENCE = 0.70 # majority share below this -> LANGUAGE_UNCERTAIN
|
| 23 |
+
SAMPLE_ROWS_SHOWN = 5
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def warning(code: str, severity: str, message: str, **extra) -> dict:
|
| 27 |
+
return {"code": code, "severity": severity, "message": message, **extra}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
# ---------------------------------------------------------------- text QA
|
| 31 |
+
def short_text_warning(texts: list[str]) -> dict | None:
|
| 32 |
+
idx = [i for i, t in enumerate(texts) if len(t.split()) < MIN_TOKENS_STABLE]
|
| 33 |
+
if not idx:
|
| 34 |
+
return None
|
| 35 |
+
return warning(
|
| 36 |
+
"TEXT_TOO_SHORT",
|
| 37 |
+
"warning",
|
| 38 |
+
f"{len(idx)} text(s) contain fewer than {MIN_TOKENS_STABLE} words; "
|
| 39 |
+
"CCR scores may be unstable for very short texts.",
|
| 40 |
+
count=len(idx),
|
| 41 |
+
affected_rows_sample=idx[:SAMPLE_ROWS_SHOWN],
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
# ---------------------------------------------------------- language checks
|
| 46 |
+
@dataclass
|
| 47 |
+
class LanguageResult:
|
| 48 |
+
selected: str
|
| 49 |
+
detected: str | None
|
| 50 |
+
confidence: float | None
|
| 51 |
+
n_rows_sampled: int
|
| 52 |
+
detector: str
|
| 53 |
+
detector_version: str
|
| 54 |
+
|
| 55 |
+
def as_metadata(self) -> dict:
|
| 56 |
+
return {
|
| 57 |
+
"selected": self.selected,
|
| 58 |
+
"detected": self.detected,
|
| 59 |
+
"confidence": self.confidence,
|
| 60 |
+
"n_rows_sampled": self.n_rows_sampled,
|
| 61 |
+
"detector": self.detector,
|
| 62 |
+
"detector_version": self.detector_version,
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def detect_corpus_language(texts: list[str], selected: str) -> tuple[LanguageResult, list[dict]]:
|
| 67 |
+
"""Corpus-level majority-vote detection on a sample of detectable rows."""
|
| 68 |
+
from langdetect import DetectorFactory, detect # lazy import
|
| 69 |
+
from langdetect.lang_detect_exception import LangDetectException
|
| 70 |
+
|
| 71 |
+
try:
|
| 72 |
+
from importlib.metadata import version as _v
|
| 73 |
+
|
| 74 |
+
detector_version = _v("langdetect")
|
| 75 |
+
except Exception:
|
| 76 |
+
detector_version = "unknown"
|
| 77 |
+
|
| 78 |
+
DetectorFactory.seed = 0 # determinism - same corpus, same result, every run
|
| 79 |
+
|
| 80 |
+
detectable = [t for t in texts if len(t.split()) >= DETECT_MIN_TOKENS][:DETECT_SAMPLE_MAX]
|
| 81 |
+
warnings: list[dict] = []
|
| 82 |
+
|
| 83 |
+
if len(detectable) < DETECT_MIN_ROWS:
|
| 84 |
+
result = LanguageResult(selected, None, None, len(detectable), "langdetect", detector_version)
|
| 85 |
+
warnings.append(
|
| 86 |
+
warning(
|
| 87 |
+
"LANGUAGE_UNCERTAIN",
|
| 88 |
+
"info",
|
| 89 |
+
f"Language could not be determined confidently ({len(detectable)} detectable "
|
| 90 |
+
f"row(s), need {DETECT_MIN_ROWS}); language checks were skipped.",
|
| 91 |
+
)
|
| 92 |
+
)
|
| 93 |
+
return result, warnings
|
| 94 |
+
|
| 95 |
+
votes: dict[str, int] = {}
|
| 96 |
+
for t in detectable:
|
| 97 |
+
try:
|
| 98 |
+
lang = detect(t) # one detection per row (detect() is the expensive call)
|
| 99 |
+
except LangDetectException:
|
| 100 |
+
continue
|
| 101 |
+
votes[lang] = votes.get(lang, 0) + 1
|
| 102 |
+
|
| 103 |
+
if not votes:
|
| 104 |
+
result = LanguageResult(selected, None, None, len(detectable), "langdetect", detector_version)
|
| 105 |
+
warnings.append(
|
| 106 |
+
warning("LANGUAGE_UNCERTAIN", "info",
|
| 107 |
+
"Language detection produced no result; language checks were skipped.")
|
| 108 |
+
)
|
| 109 |
+
return result, warnings
|
| 110 |
+
|
| 111 |
+
top_lang, top_count = max(votes.items(), key=lambda kv: kv[1])
|
| 112 |
+
confidence = round(top_count / sum(votes.values()), 3)
|
| 113 |
+
result = LanguageResult(selected, top_lang, confidence, len(detectable), "langdetect", detector_version)
|
| 114 |
+
|
| 115 |
+
if confidence < DETECT_CONFIDENCE:
|
| 116 |
+
warnings.append(
|
| 117 |
+
warning(
|
| 118 |
+
"LANGUAGE_UNCERTAIN",
|
| 119 |
+
"info",
|
| 120 |
+
f"Detected language is uncertain (top candidate '{top_lang}' at "
|
| 121 |
+
f"{confidence:.0%} of sampled rows); interpret language checks with care.",
|
| 122 |
+
)
|
| 123 |
+
)
|
| 124 |
+
elif top_lang != selected.lower():
|
| 125 |
+
warnings.append(
|
| 126 |
+
warning(
|
| 127 |
+
"LANGUAGE_MISMATCH",
|
| 128 |
+
"warning",
|
| 129 |
+
f"You selected '{selected}', but the corpus appears to be '{top_lang}' "
|
| 130 |
+
f"({confidence:.0%} of {len(detectable)} sampled rows).",
|
| 131 |
+
detected_language=top_lang,
|
| 132 |
+
selected_language=selected,
|
| 133 |
+
)
|
| 134 |
+
)
|
| 135 |
+
return result, warnings
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def model_language_warning(selected: str, model_id: str, supported: frozenset[str],
|
| 139 |
+
language_set_name: str | None) -> dict | None:
|
| 140 |
+
if not supported or selected.lower() in supported:
|
| 141 |
+
return None
|
| 142 |
+
label = f"the '{language_set_name}' language set" if language_set_name else \
|
| 143 |
+
f"{sorted(supported)}"
|
| 144 |
+
return warning(
|
| 145 |
+
"MODEL_LANGUAGE_UNSUPPORTED",
|
| 146 |
+
"warning",
|
| 147 |
+
f"The selected model supports {label}, but you selected '{selected}'. "
|
| 148 |
+
"Switch to a multilingual model or proceed with caution.",
|
| 149 |
+
selected_language=selected,
|
| 150 |
+
model_id=model_id,
|
| 151 |
+
)
|
backend/requirements.txt
CHANGED
|
@@ -7,3 +7,6 @@ openpyxl>=3.1
|
|
| 7 |
python-multipart>=0.0.9
|
| 8 |
numpy>=1.26
|
| 9 |
sentence-transformers>=2.6
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
python-multipart>=0.0.9
|
| 8 |
numpy>=1.26
|
| 9 |
sentence-transformers>=2.6
|
| 10 |
+
pyyaml>=6.0
|
| 11 |
+
langdetect>=1.0.9
|
| 12 |
+
boto3>=1.34 # used only when CCR_STORAGE=s3 (Cloudflare R2 / any S3-compatible store)
|
backend/static/assets/index-BUzS6usZ.js
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
backend/static/assets/index-C356-0ZT.css
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
:root{--maroon: #7a1f3d;--maroon-dark: #5e1730;--ink: #1d2129;--muted: #667085;--line: #e5e7eb;--bg: #f7f7f8;--card: #ffffff;--ok: #157f3d;--err: #b42318;--accent-soft: #f6ebef}*{box-sizing:border-box}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,sans-serif;color:var(--ink);background:var(--bg);font-size:14.5px;line-height:1.5}.app{display:flex;flex-direction:column;min-height:100vh}.header{background:var(--maroon);color:#fff;padding:14px 28px;display:flex;align-items:baseline;gap:14px}.header h1{font-size:17px;margin:0;font-weight:650;letter-spacing:.2px}.header .sub{font-size:12.5px;opacity:.85}.layout{display:flex;flex:1;min-height:0}.sidebar{width:250px;background:var(--card);border-right:1px solid var(--line);padding:18px 14px;flex-shrink:0}.sidebar h2{font-size:11.5px;text-transform:uppercase;letter-spacing:.7px;color:var(--muted);margin:0 0 10px 4px}.project-item{display:block;width:100%;text-align:left;padding:9px 12px;margin-bottom:4px;border:1px solid transparent;border-radius:8px;background:none;cursor:pointer;font:inherit;color:var(--ink)}.project-item:hover{background:var(--bg)}.project-item.active{background:var(--accent-soft);border-color:var(--maroon);font-weight:600}.project-item .date{display:block;font-size:11.5px;color:var(--muted);font-weight:400}.main{flex:1;padding:22px 28px;overflow-y:auto;min-width:0}.card{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:18px 20px;margin-bottom:16px}.card h3{margin:0 0 4px;font-size:15px}.card .hint{color:var(--muted);font-size:12.5px;margin:0 0 12px}.step-badge{display:inline-flex;align-items:center;justify-content:center;width:21px;height:21px;border-radius:50%;background:var(--maroon);color:#fff;font-size:12px;font-weight:700;margin-right:8px;vertical-align:-3px}button.primary{background:var(--maroon);color:#fff;border:none;padding:9px 18px;border-radius:8px;font:inherit;font-weight:600;cursor:pointer}button.primary:hover{background:var(--maroon-dark)}button.primary:disabled{background:#c9ccd1;cursor:not-allowed}button.ghost{background:none;border:1px solid var(--line);color:var(--ink);padding:8px 14px;border-radius:8px;font:inherit;cursor:pointer}button.ghost:hover{border-color:var(--maroon);color:var(--maroon)}button.linkish{background:none;border:none;color:var(--maroon);font:inherit;cursor:pointer;padding:0;text-decoration:underline}input[type=text],textarea,select{width:100%;padding:8px 10px;border:1px solid var(--line);border-radius:8px;font:inherit;background:#fff;color:var(--ink)}textarea{resize:vertical}label.field{display:block;margin-bottom:10px;font-size:13px;font-weight:600}label.field>*{margin-top:4px;font-weight:400}.row{display:flex;gap:14px;flex-wrap:wrap}.row>.grow{flex:1;min-width:220px}.pill{display:inline-block;padding:2px 10px;border-radius:999px;font-size:11.5px;font-weight:600}.pill.completed{background:#e6f4ea;color:var(--ok)}.pill.running{background:#fff3e0;color:#b45309}.pill.queued{background:#eef2f7;color:var(--muted)}.pill.failed{background:#fdecea;color:var(--err)}.progress-track{background:var(--line);border-radius:999px;height:7px;overflow:hidden}.progress-fill{background:var(--maroon);height:100%;transition:width .4s ease}.warnings{background:#fff8e6;border:1px solid #f2dfa8;color:#7a5b00;border-radius:8px;padding:10px 14px}.error-banner{background:#fdecea;color:var(--err);border:1px solid #f5c6c0;padding:10px 14px;border-radius:8px;margin-bottom:14px;font-size:13px}table.docs{width:100%;border-collapse:collapse;font-size:13px}table.docs th{text-align:left;color:var(--muted);font-size:11.5px;text-transform:uppercase;letter-spacing:.5px;padding:6px 8px;border-bottom:1px solid var(--line)}table.docs td{padding:7px 8px;border-bottom:1px solid var(--bg);vertical-align:top}table.docs td.score{font-variant-numeric:tabular-nums;font-weight:600;white-space:nowrap}.stat-grid{display:flex;gap:12px;flex-wrap:wrap;margin-bottom:4px}.stat{flex:1;min-width:110px;background:var(--bg);border-radius:8px;padding:10px 14px}.stat .v{font-size:20px;font-weight:700;font-variant-numeric:tabular-nums}.stat .k{font-size:11.5px;color:var(--muted);text-transform:uppercase;letter-spacing:.5px}.item-bar-row{display:flex;align-items:center;gap:10px;margin-bottom:7px}.item-bar-label{flex:1;font-size:12.5px;min-width:0}.item-bar-track{flex:1.2;background:var(--bg);border-radius:999px;height:10px}.item-bar-fill{background:var(--maroon);opacity:.85;height:100%;border-radius:999px}.item-bar-val{width:52px;text-align:right;font-variant-numeric:tabular-nums;font-size:12.5px;font-weight:600}.construct-items{margin:8px 0 0;padding-left:20px;color:var(--muted);font-size:12.5px}.construct-items li{margin-bottom:2px}.meta-footer{font-size:12px;color:var(--muted);background:var(--bg);border-radius:8px;padding:10px 14px;margin-top:14px;font-variant-numeric:tabular-nums}.meta-footer code{font-size:11.5px}.muted{color:var(--muted)}.small{font-size:12.5px}.mt{margin-top:12px}
|
|
|
|
|
|
backend/static/assets/index-CN_FzJfm.css
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
:root{--maroon: #7a1f3d;--maroon-dark: #5e1730;--ink: #1d2129;--muted: #667085;--line: #e5e7eb;--bg: #f7f7f8;--card: #ffffff;--ok: #157f3d;--err: #b42318;--accent-soft: #f6ebef;--control-height: 40px}*{box-sizing:border-box}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,sans-serif;color:var(--ink);background:var(--bg);font-size:14.5px;line-height:1.5}.app{display:flex;flex-direction:column;min-height:100vh}.header{background:var(--maroon);color:#fff;padding:14px 28px;display:flex;align-items:baseline;flex-wrap:wrap;gap:14px}.header h1{flex:0 0 auto;font-size:17px;margin:0;font-weight:650;letter-spacing:.2px;white-space:nowrap}.header .sub{flex:1 1 280px;min-width:0;font-size:12.5px;opacity:.85}.layout{display:flex;flex:1;min-width:0;min-height:0}.sidebar{width:250px;background:var(--card);border-right:1px solid var(--line);padding:18px 14px;flex-shrink:0;display:flex;flex-direction:column;min-height:0}.sidebar h2{font-size:11.5px;text-transform:uppercase;letter-spacing:.7px;color:var(--muted);margin:0 0 10px 4px;display:flex;align-items:center;gap:8px}.sidebar h2 .count{background:var(--bg);border:1px solid var(--line);border-radius:999px;padding:0 8px;font-size:10.5px;letter-spacing:0;color:var(--muted)}.sidebar-filter{width:100%;padding:7px 10px;margin-bottom:10px;border:1px solid var(--line);border-radius:8px;font:inherit;font-size:13px;background:#fff;color:var(--ink)}.sidebar-filter:focus{outline:none;border-color:var(--maroon)}.project-list{flex:1;min-height:0;overflow-y:auto;margin:0 -4px;padding:0 4px 4px}.group-label{font-size:10.5px;text-transform:uppercase;letter-spacing:.6px;color:var(--muted);margin:10px 4px 5px}div:first-child>.group-label{margin-top:2px}.project-item{display:block;width:100%;text-align:left;padding:9px 12px;margin-bottom:5px;border:1px solid transparent;border-radius:8px;background:none;cursor:pointer;font:inherit;color:var(--ink)}.project-item:hover{background:var(--bg)}.project-item.active{background:var(--accent-soft);border-color:var(--maroon);font-weight:600}.project-item .project-name{display:block;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.project-item .date{display:block;font-size:11.5px;color:var(--muted);font-weight:400}.project-create{margin-top:12px;padding-top:12px;border-top:1px solid var(--line)}.project-create>button{width:100%}.main{flex:1;padding:22px 28px;overflow-y:auto;min-width:0}.card{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:18px 20px;margin-bottom:16px}.card h3{margin:0 0 4px;font-size:15px}.card .hint{color:var(--muted);font-size:12.5px;margin:0 0 12px}.step-badge{display:inline-flex;align-items:center;justify-content:center;width:21px;height:21px;border-radius:50%;background:var(--maroon);color:#fff;font-size:12px;font-weight:700;margin-right:8px;vertical-align:-3px}button{white-space:nowrap}button.primary{background:var(--maroon);color:#fff;border:none;min-height:var(--control-height);display:inline-flex;align-items:center;justify-content:center;padding:0 18px;border-radius:8px;font:inherit;font-weight:600;cursor:pointer;line-height:1.2}button.primary:hover{background:var(--maroon-dark)}button.primary:disabled{background:#c9ccd1;cursor:not-allowed}a.google-btn{background:var(--maroon);color:#fff;border:none;min-height:var(--control-height);display:flex;align-items:center;justify-content:center;padding:0 18px;border-radius:8px;font:inherit;font-weight:600;cursor:pointer;line-height:1.2;text-decoration:none;width:100%;box-sizing:border-box}a.google-btn:hover{background:var(--maroon-dark)}button.ghost{background:none;border:1px solid var(--line);color:var(--ink);min-height:var(--control-height);display:inline-flex;align-items:center;justify-content:center;padding:0 14px;border-radius:8px;font:inherit;cursor:pointer;line-height:1.2}button.ghost:hover{border-color:var(--maroon);color:var(--maroon)}button.linkish{background:none;border:none;color:var(--maroon);font:inherit;cursor:pointer;padding:0;text-decoration:underline}input[type=text],input[type=email],input[type=password],textarea,select{width:100%;padding:8px 10px;border:1px solid var(--line);border-radius:8px;font:inherit;background:#fff;color:var(--ink)}input[type=text],input[type=email],input[type=password],select{height:var(--control-height)}button:focus-visible,input:focus-visible,textarea:focus-visible,select:focus-visible{outline:2px solid rgba(122,31,61,.38);outline-offset:2px}input[type=file]{display:block;max-width:100%;margin-top:6px;font-size:13px;color:var(--muted)}input[type=file]::file-selector-button{background:#fff;border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;font:inherit;font-size:13px;cursor:pointer;margin-right:10px}input[type=file]::file-selector-button:hover{border-color:var(--maroon);color:var(--maroon)}.row>button{align-self:flex-end;margin-bottom:1px}textarea{resize:vertical}label.field{display:block;margin-bottom:10px;font-size:13px;font-weight:600}label.field>*{margin-top:4px;font-weight:400}.field-hint{margin-top:0;font-weight:400;color:var(--muted);font-size:12px}.row{display:flex;gap:14px;flex-wrap:wrap}.row>*{min-width:0}.row>.grow{flex:1;min-width:min(220px,100%)}.language-control{min-width:170px}.model-control{min-width:260px}.run-settings{display:grid;grid-template-columns:minmax(160px,230px) minmax(320px,720px) max-content;justify-content:start;gap:14px;align-items:end}.run-settings .field{margin-bottom:0}.run-button{min-width:180px;height:var(--control-height)}.construct-row{display:grid;grid-template-columns:minmax(320px,1120px) max-content;justify-content:start;align-items:end;gap:14px}.results-toolbar{display:flex;align-items:center;justify-content:space-between;gap:14px;margin-bottom:14px;flex-wrap:wrap}.result-actions{justify-content:flex-end}.result-actions a{display:inline-flex;text-decoration:none}.pill{display:inline-block;padding:2px 10px;border-radius:999px;font-size:11.5px;font-weight:600}.pill.completed{background:#e6f4ea;color:var(--ok)}.pill.running{background:#fff3e0;color:#b45309}.pill.queued{background:#eef2f7;color:var(--muted)}.pill.failed{background:#fdecea;color:var(--err)}.progress-track{background:var(--line);border-radius:999px;height:7px;overflow:hidden}.progress-fill{background:var(--maroon);height:100%;transition:width .4s ease}.warnings{background:#fff8e6;border:1px solid #f2dfa8;color:#7a5b00;border-radius:8px;padding:10px 14px}.error-banner{background:#fdecea;color:var(--err);border:1px solid #f5c6c0;padding:10px 14px;border-radius:8px;margin-bottom:14px;font-size:13px}.table-wrap{width:100%;overflow-x:auto}table.docs{width:100%;border-collapse:collapse;font-size:13px}table.docs th{text-align:left;color:var(--muted);font-size:11.5px;text-transform:uppercase;letter-spacing:.5px;padding:6px 8px;border-bottom:1px solid var(--line)}table.docs td{padding:7px 8px;border-bottom:1px solid var(--bg);vertical-align:top;overflow-wrap:anywhere}table.docs td.score{font-variant-numeric:tabular-nums;font-weight:600;white-space:nowrap}.stat-grid{display:flex;gap:12px;flex-wrap:wrap;margin-bottom:4px}.stat{flex:1;min-width:110px;background:var(--bg);border-radius:8px;padding:10px 14px}.stat .v{font-size:20px;font-weight:700;font-variant-numeric:tabular-nums}.stat .k{font-size:11.5px;color:var(--muted);text-transform:uppercase;letter-spacing:.5px}.item-bar-row{display:flex;align-items:center;gap:10px;margin-bottom:7px}.item-bar-label{flex:1;font-size:12.5px;min-width:0;overflow-wrap:anywhere}.item-bar-track{flex:1.2;background:var(--bg);border-radius:999px;height:10px}.item-bar-fill{background:var(--maroon);opacity:.85;height:100%;border-radius:999px}.item-bar-val{width:52px;text-align:right;font-variant-numeric:tabular-nums;font-size:12.5px;font-weight:600}.construct-items{margin:8px 0 0;padding-left:20px;color:var(--muted);font-size:12.5px}.construct-items li{margin-bottom:2px}.meta-footer{font-size:12px;color:var(--muted);background:var(--bg);border-radius:8px;padding:10px 14px;margin-top:14px;font-variant-numeric:tabular-nums}.meta-footer code{font-size:11.5px}.muted{color:var(--muted)}.small{font-size:12.5px}.mt{margin-top:12px}.header-auth{margin-left:auto;display:flex;align-items:center;gap:10px;color:#fff}.header-btn{background:#ffffff1f;color:#fff;border:1px solid rgba(255,255,255,.45);padding:5px 14px;border-radius:7px;font:inherit;font-size:13px;cursor:pointer}.header-btn:hover{background:#ffffff38}.project-header{display:flex;align-items:center;justify-content:space-between;gap:12px;margin-bottom:14px;flex-wrap:wrap}.project-title{font-size:17px;font-weight:650;margin-right:10px}button.danger{color:var(--err);border-color:#f0c4be}button.danger:hover{color:var(--err);border-color:var(--err)}button.danger-solid{background:var(--err)}button.danger-solid:hover{background:#93261b}button.danger-solid:disabled{background:#c9ccd1}.picker{position:relative}.picker-display{width:100%;display:flex;align-items:center;gap:10px;padding:8px 12px;border:1px solid var(--line);border-radius:8px;background:#fff;font:inherit;color:var(--ink);cursor:pointer;text-align:left}.picker-display:hover{border-color:var(--maroon)}.picker-display .picker-caret{margin-left:auto;color:var(--muted);font-size:11px}.picker-search{width:100%;padding:8px 12px;border:1px solid var(--maroon);border-radius:8px;font:inherit;background:#fff}.picker-search:focus{outline:none;box-shadow:0 0 0 3px #7a1f3d1f}.picker-panel{position:absolute;top:calc(100% + 6px);left:0;z-index:50;width:100%;max-width:640px;background:var(--card);border:1px solid var(--line);border-radius:10px;box-shadow:0 14px 40px #0000002e;max-height:340px;overflow-y:auto;padding:4px 0 6px}.picker-group{font-size:10.5px;text-transform:uppercase;letter-spacing:.6px;color:var(--muted);padding:8px 12px 3px;position:sticky;top:0;background:var(--card)}.picker-option{display:flex;align-items:baseline;justify-content:space-between;gap:12px;padding:6px 12px;cursor:pointer;font-size:13.5px}.picker-option:hover,.picker-option.active{background:var(--accent-soft)}.picker-option.selected .picker-name{font-weight:650;color:var(--maroon)}.picker-name{min-width:0}.picker-meta{flex-shrink:0;font-size:11.5px;color:var(--muted);white-space:nowrap}.picker-empty{padding:12px;margin:0}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;background:#14161a73;display:flex;align-items:center;justify-content:center;z-index:40;padding:16px}.modal{background:var(--card);border-radius:12px;padding:26px 28px 24px;width:100%;max-width:460px;box-shadow:0 12px 40px #0000002e}.modal h3{margin:0 0 8px;font-size:18px}.modal .hint{color:var(--muted);font-size:13px;line-height:1.5;margin:0 0 4px}.modal form.mt{margin-top:18px}.modal label.field{margin-bottom:16px}.modal label.field:last-of-type{margin-bottom:20px}.modal .row{gap:10px}.modal .row>.primary{flex:1}.modal p.small.muted.mt{margin-top:18px;padding-top:16px;border-top:1px solid var(--line);font-size:12.5px;line-height:1.6}@media (max-width: 820px){body{font-size:14px}.header{padding:12px 16px;align-items:flex-start;gap:2px 12px}.header h1{font-size:16px}.header .sub{flex:1 1 210px;font-size:12px;line-height:1.35}.layout{display:block}.sidebar{width:100%;border-right:0;border-bottom:1px solid var(--line);padding:14px}.project-list{max-height:220px;margin-right:0;flex:none}.project-create{border-top:0}.main{width:100%;padding:16px 14px 28px;overflow:visible}.card{padding:16px;margin-bottom:14px}.row,.run-settings,.construct-row,.results-toolbar,.result-actions{gap:10px}.row,.results-toolbar,.result-actions{flex-direction:column;align-items:stretch}.run-settings,.construct-row{grid-template-columns:1fr}.row>.grow,.construct-row>.grow,.language-control,.model-control{width:100%;min-width:0}.main .row>button,.main .row>a,.main .row>a>button,.results-toolbar>button{align-self:stretch;margin-bottom:0;width:100%}.run-button{width:100%;min-width:0}.stat-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr))}.stat{min-width:0}.item-bar-row{display:grid;grid-template-columns:minmax(0,1fr) 56px;gap:6px 10px}.item-bar-track{grid-column:1 / -1;width:100%}.item-bar-val{width:auto}table.docs{min-width:520px}}@media (max-width: 460px){.header .sub{flex-basis:100%}.stat-grid{grid-template-columns:1fr}}
|
backend/static/assets/index-DrFcy6bH.js
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
backend/static/index.html
CHANGED
|
@@ -3,9 +3,9 @@
|
|
| 3 |
<head>
|
| 4 |
<meta charset="UTF-8" />
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
| 6 |
-
<title>CCR Platform
|
| 7 |
-
<script type="module" crossorigin src="/assets/index-
|
| 8 |
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
| 9 |
</head>
|
| 10 |
<body>
|
| 11 |
<div id="root"></div>
|
|
|
|
| 3 |
<head>
|
| 4 |
<meta charset="UTF-8" />
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
| 6 |
+
<title>CCR Platform - Contextualized Construct Representations</title>
|
| 7 |
+
<script type="module" crossorigin src="/assets/index-DrFcy6bH.js"></script>
|
| 8 |
+
<link rel="stylesheet" crossorigin href="/assets/index-CN_FzJfm.css">
|
| 9 |
</head>
|
| 10 |
<body>
|
| 11 |
<div id="root"></div>
|
backend/tests/conftest.py
CHANGED
|
@@ -9,6 +9,9 @@ import tempfile
|
|
| 9 |
from pathlib import Path
|
| 10 |
|
| 11 |
os.environ["CCR_DATA_DIR"] = tempfile.mkdtemp(prefix="ccr_test_")
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
# Make `app` importable regardless of pytest invocation directory.
|
| 14 |
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
| 9 |
from pathlib import Path
|
| 10 |
|
| 11 |
os.environ["CCR_DATA_DIR"] = tempfile.mkdtemp(prefix="ccr_test_")
|
| 12 |
+
# Tests that exercise the anonymous run limit set this themselves; everything
|
| 13 |
+
# else should not trip over it while running multiple jobs per test.
|
| 14 |
+
os.environ.setdefault("CCR_ANON_MAX_RUNS_PER_DAY", "1000")
|
| 15 |
|
| 16 |
# Make `app` importable regardless of pytest invocation directory.
|
| 17 |
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
backend/tests/test_accounts_limits_retention.py
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Accounts (register/login), anonymous run limits, retention (delete-after-
|
| 2 |
+
analysis + TTL purge), saved-run cap, ownership, and construct file upload."""
|
| 3 |
+
|
| 4 |
+
import io
|
| 5 |
+
import time
|
| 6 |
+
|
| 7 |
+
import pytest
|
| 8 |
+
from fastapi.testclient import TestClient
|
| 9 |
+
|
| 10 |
+
from app.main import app
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@pytest.fixture()
|
| 14 |
+
def client():
|
| 15 |
+
with TestClient(app) as c:
|
| 16 |
+
yield c
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def csv_rows(n: int) -> bytes:
|
| 20 |
+
return ("text\n" + "\n".join(f"sample sentence number {i} here" for i in range(n))).encode()
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def upload(client, project_id, name, payload: bytes):
|
| 24 |
+
return client.post(
|
| 25 |
+
f"/api/projects/{project_id}/corpora",
|
| 26 |
+
files={"file": (name, io.BytesIO(payload), "text/csv")},
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def register(client, email="user@test.edu", name="Test User"):
|
| 31 |
+
resp = client.post(
|
| 32 |
+
"/api/auth/register", json={"email": email, "password": "password123", "name": name}
|
| 33 |
+
)
|
| 34 |
+
assert resp.status_code == 201, resp.json()
|
| 35 |
+
return resp.json()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def run_job(client, project_id, corpus_id, construct_id):
|
| 39 |
+
return client.post(
|
| 40 |
+
"/api/jobs",
|
| 41 |
+
json={
|
| 42 |
+
"project_id": project_id,
|
| 43 |
+
"corpus_id": corpus_id,
|
| 44 |
+
"construct_id": construct_id,
|
| 45 |
+
"text_column": "text",
|
| 46 |
+
"model_name": "fake-deterministic",
|
| 47 |
+
},
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def wait_for_job(client, job_id, timeout=10.0):
|
| 52 |
+
deadline = time.time() + timeout
|
| 53 |
+
while time.time() < deadline:
|
| 54 |
+
job = client.get(f"/api/jobs/{job_id}").json()
|
| 55 |
+
if job["status"] in ("completed", "failed"):
|
| 56 |
+
return job
|
| 57 |
+
time.sleep(0.05)
|
| 58 |
+
raise TimeoutError(job_id)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def any_construct(client):
|
| 62 |
+
return client.get("/api/constructs").json()[0]
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
# ---------------------------------------------------------------- accounts
|
| 66 |
+
def test_register_login_logout_roundtrip(client):
|
| 67 |
+
register(client, "roundtrip@test.edu", "Rounder")
|
| 68 |
+
me = client.get("/api/auth/me").json()
|
| 69 |
+
assert me["signed_in"] and me["email"] == "roundtrip@test.edu"
|
| 70 |
+
assert me["usage"]["max_saved_runs"] > 0
|
| 71 |
+
|
| 72 |
+
client.post("/api/auth/logout")
|
| 73 |
+
assert client.get("/api/auth/me").json()["signed_in"] is False
|
| 74 |
+
|
| 75 |
+
resp = client.post(
|
| 76 |
+
"/api/auth/login", json={"email": "ROUNDTRIP@test.edu", "password": "password123"}
|
| 77 |
+
)
|
| 78 |
+
assert resp.status_code == 200 # email is case-insensitive
|
| 79 |
+
assert client.get("/api/auth/me").json()["signed_in"] is True
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def test_wrong_password_and_duplicate_email(client):
|
| 83 |
+
register(client, "dupe@test.edu")
|
| 84 |
+
client.post("/api/auth/logout")
|
| 85 |
+
resp = client.post("/api/auth/login", json={"email": "dupe@test.edu", "password": "wrongpass1"})
|
| 86 |
+
assert resp.status_code == 401
|
| 87 |
+
resp = client.post(
|
| 88 |
+
"/api/auth/register", json={"email": "dupe@test.edu", "password": "password123", "name": "X"}
|
| 89 |
+
)
|
| 90 |
+
assert resp.status_code == 409
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def test_register_validation(client):
|
| 94 |
+
resp = client.post(
|
| 95 |
+
"/api/auth/register", json={"email": "not-an-email", "password": "password123", "name": "X"}
|
| 96 |
+
)
|
| 97 |
+
assert resp.status_code == 400
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ------------------------------------------------------ anonymous run limit
|
| 101 |
+
def test_anonymous_daily_run_limit(client, monkeypatch):
|
| 102 |
+
monkeypatch.setenv("CCR_ANON_MAX_RUNS_PER_DAY", "2")
|
| 103 |
+
project = client.post("/api/projects", json={"name": "Limited"}).json()
|
| 104 |
+
construct = any_construct(client)
|
| 105 |
+
|
| 106 |
+
for i in range(2):
|
| 107 |
+
corpus = upload(client, project["id"], f"c{i}.csv", csv_rows(5)).json()
|
| 108 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 109 |
+
assert resp.status_code == 201, resp.json()
|
| 110 |
+
wait_for_job(client, resp.json()["id"])
|
| 111 |
+
|
| 112 |
+
me = client.get("/api/auth/me").json()
|
| 113 |
+
assert me["usage"]["runs_used_today"] == 2
|
| 114 |
+
|
| 115 |
+
corpus = upload(client, project["id"], "c3.csv", csv_rows(5)).json()
|
| 116 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 117 |
+
assert resp.status_code == 429
|
| 118 |
+
assert "Sign in" in resp.json()["detail"]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def test_signed_in_users_bypass_run_limit(client, monkeypatch):
|
| 122 |
+
monkeypatch.setenv("CCR_ANON_MAX_RUNS_PER_DAY", "1")
|
| 123 |
+
register(client, "runner@test.edu")
|
| 124 |
+
project = client.post("/api/projects", json={"name": "Unlimited"}).json()
|
| 125 |
+
corpus = upload(client, project["id"], "c.csv", csv_rows(5)).json()
|
| 126 |
+
construct = any_construct(client)
|
| 127 |
+
for _ in range(3):
|
| 128 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 129 |
+
assert resp.status_code == 201, resp.json()
|
| 130 |
+
wait_for_job(client, resp.json()["id"])
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
# ----------------------------------------------------------------- retention
|
| 134 |
+
def test_anonymous_corpus_removed_after_run_and_rerun_gets_410(client):
|
| 135 |
+
from pathlib import Path
|
| 136 |
+
|
| 137 |
+
project = client.post("/api/projects", json={"name": "Ephemeral"}).json()
|
| 138 |
+
corpus = upload(client, project["id"], "c.csv", csv_rows(5)).json()
|
| 139 |
+
construct = any_construct(client)
|
| 140 |
+
|
| 141 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 142 |
+
job = wait_for_job(client, resp.json()["id"])
|
| 143 |
+
assert job["status"] == "completed"
|
| 144 |
+
|
| 145 |
+
results = client.get(f"/api/jobs/{job['id']}/results").json()
|
| 146 |
+
codes = [w["code"] for w in results["summary"]["warnings"]]
|
| 147 |
+
assert "ANONYMOUS_DATA_REMOVED" in codes
|
| 148 |
+
assert results["metadata"]["anonymous_corpus_removed"] is True
|
| 149 |
+
# results are still downloadable; the raw upload is gone
|
| 150 |
+
assert client.get(f"/api/jobs/{job['id']}/export").status_code == 200
|
| 151 |
+
|
| 152 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 153 |
+
assert resp.status_code == 410
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def test_signed_in_corpus_survives_run(client):
|
| 157 |
+
register(client, "keeper@test.edu")
|
| 158 |
+
project = client.post("/api/projects", json={"name": "Kept"}).json()
|
| 159 |
+
corpus = upload(client, project["id"], "c.csv", csv_rows(5)).json()
|
| 160 |
+
construct = any_construct(client)
|
| 161 |
+
|
| 162 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 163 |
+
job = wait_for_job(client, resp.json()["id"])
|
| 164 |
+
assert job["status"] == "completed"
|
| 165 |
+
codes = [w["code"] for w in client.get(f"/api/jobs/{job['id']}/results").json()["summary"]["warnings"]]
|
| 166 |
+
assert "ANONYMOUS_DATA_REMOVED" not in codes
|
| 167 |
+
|
| 168 |
+
# re-running the same corpus works: the file is still there
|
| 169 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 170 |
+
assert resp.status_code == 201
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def test_ttl_purge_removes_only_expired_anonymous_projects(client, monkeypatch):
|
| 174 |
+
from app.db import SessionLocal
|
| 175 |
+
from app.models import Project
|
| 176 |
+
from app.retention import purge_expired_anonymous
|
| 177 |
+
|
| 178 |
+
monkeypatch.setenv("CCR_ANON_TTL_HOURS", "24")
|
| 179 |
+
old_anon = client.post("/api/projects", json={"name": "OldAnon"}).json()
|
| 180 |
+
fresh_anon = client.post("/api/projects", json={"name": "FreshAnon"}).json()
|
| 181 |
+
register(client, "owner@test.edu")
|
| 182 |
+
owned = client.post("/api/projects", json={"name": "OwnedOld"}).json()
|
| 183 |
+
|
| 184 |
+
db = SessionLocal()
|
| 185 |
+
try:
|
| 186 |
+
db.get(Project, old_anon["id"]).created_at = "2020-01-01T00:00:00+00:00"
|
| 187 |
+
db.get(Project, owned["id"]).created_at = "2020-01-01T00:00:00+00:00"
|
| 188 |
+
db.commit()
|
| 189 |
+
purged = purge_expired_anonymous(db)
|
| 190 |
+
assert purged == 1
|
| 191 |
+
assert db.get(Project, old_anon["id"]) is None
|
| 192 |
+
assert db.get(Project, fresh_anon["id"]) is not None
|
| 193 |
+
assert db.get(Project, owned["id"]) is not None # owned data never TTL-purged
|
| 194 |
+
finally:
|
| 195 |
+
db.close()
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
# ------------------------------------------------------------ saved-run cap
|
| 199 |
+
def test_saved_run_cap_for_signed_in_users(client, monkeypatch):
|
| 200 |
+
monkeypatch.setenv("CCR_USER_MAX_SAVED_RUNS", "2")
|
| 201 |
+
register(client, "capped@test.edu")
|
| 202 |
+
project = client.post("/api/projects", json={"name": "Capped"}).json()
|
| 203 |
+
corpus = upload(client, project["id"], "c.csv", csv_rows(5)).json()
|
| 204 |
+
construct = any_construct(client)
|
| 205 |
+
|
| 206 |
+
for _ in range(2):
|
| 207 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 208 |
+
assert resp.status_code == 201
|
| 209 |
+
wait_for_job(client, resp.json()["id"])
|
| 210 |
+
|
| 211 |
+
resp = run_job(client, project["id"], corpus["id"], construct["id"])
|
| 212 |
+
assert resp.status_code == 409
|
| 213 |
+
assert "saved runs" in resp.json()["detail"]
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
# ---------------------------------------------------------------- ownership
|
| 217 |
+
def test_owned_projects_invisible_and_untouchable_to_others(client):
|
| 218 |
+
register(client, "alice@test.edu", "Alice")
|
| 219 |
+
owned = client.post("/api/projects", json={"name": "AlicePrivate"}).json()
|
| 220 |
+
client.post("/api/auth/logout")
|
| 221 |
+
|
| 222 |
+
ids = [p["id"] for p in client.get("/api/projects").json()]
|
| 223 |
+
assert owned["id"] not in ids # invisible to anonymous viewers
|
| 224 |
+
assert client.patch(f"/api/projects/{owned['id']}", json={"archived": True}).status_code == 403
|
| 225 |
+
assert client.delete(f"/api/projects/{owned['id']}").status_code == 403
|
| 226 |
+
|
| 227 |
+
register(client, "bob@test.edu", "Bob")
|
| 228 |
+
assert client.patch(f"/api/projects/{owned['id']}", json={"archived": True}).status_code == 403
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# ------------------------------------------------- construct file upload
|
| 232 |
+
def test_parse_construct_file_with_reverse_column(client):
|
| 233 |
+
csv = "item,reverse\nI am satisfied with my life.,0\nI rarely feel content. ,1\n,\nI am satisfied with my life.,0\n"
|
| 234 |
+
resp = client.post(
|
| 235 |
+
"/api/constructs/parse-file",
|
| 236 |
+
files={"file": ("swls_short.csv", io.BytesIO(csv.encode()), "text/csv")},
|
| 237 |
+
)
|
| 238 |
+
assert resp.status_code == 200, resp.json()
|
| 239 |
+
body = resp.json()
|
| 240 |
+
assert body["items"] == [
|
| 241 |
+
{"text": "I am satisfied with my life.", "reverse_scored": False},
|
| 242 |
+
{"text": "I rarely feel content.", "reverse_scored": True},
|
| 243 |
+
]
|
| 244 |
+
assert any("blank" in w for w in body["warnings"])
|
| 245 |
+
assert any("duplicate" in w for w in body["warnings"])
|
| 246 |
+
assert body["suggested_name"] == "Swls Short"
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def test_parse_construct_file_with_r_marker_single_column(client):
|
| 250 |
+
csv = "text\nLife feels meaningful to me\nNothing I do matters (R)\n"
|
| 251 |
+
resp = client.post(
|
| 252 |
+
"/api/constructs/parse-file",
|
| 253 |
+
files={"file": ("meaning.csv", io.BytesIO(csv.encode()), "text/csv")},
|
| 254 |
+
)
|
| 255 |
+
body = resp.json()
|
| 256 |
+
assert body["items"][1] == {"text": "Nothing I do matters", "reverse_scored": True}
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def test_parse_construct_file_rejects_bad_type_and_empty(client):
|
| 260 |
+
resp = client.post(
|
| 261 |
+
"/api/constructs/parse-file",
|
| 262 |
+
files={"file": ("items.pdf", io.BytesIO(b"x"), "application/pdf")},
|
| 263 |
+
)
|
| 264 |
+
assert resp.status_code == 400
|
| 265 |
+
resp = client.post(
|
| 266 |
+
"/api/constructs/parse-file",
|
| 267 |
+
files={"file": ("empty.csv", io.BytesIO(b"item\n"), "text/csv")},
|
| 268 |
+
)
|
| 269 |
+
assert resp.status_code == 400
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
# ------------------------------------------------------- perf: dedup encode
|
| 273 |
+
def test_encode_unique_matches_full_encode_and_saves_calls():
|
| 274 |
+
import numpy as np
|
| 275 |
+
|
| 276 |
+
from app.ccr import HashEmbeddingBackend, encode_unique
|
| 277 |
+
|
| 278 |
+
class Counting(HashEmbeddingBackend):
|
| 279 |
+
def __init__(self):
|
| 280 |
+
super().__init__()
|
| 281 |
+
self.n_encoded = 0
|
| 282 |
+
|
| 283 |
+
def encode(self, texts, progress_cb=None):
|
| 284 |
+
self.n_encoded += len(texts)
|
| 285 |
+
return super().encode(texts, progress_cb)
|
| 286 |
+
|
| 287 |
+
texts = ["alpha beta", "gamma delta", "alpha beta", "alpha beta", "gamma delta"]
|
| 288 |
+
counting = Counting()
|
| 289 |
+
deduped = encode_unique(counting, texts)
|
| 290 |
+
assert counting.n_encoded == 2 # only unique texts hit the encoder
|
| 291 |
+
full = HashEmbeddingBackend().encode(texts)
|
| 292 |
+
assert np.allclose(deduped, full) # bit-identical expansion
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def test_created_construct_carries_reverse_flags_into_run_metadata(client):
|
| 296 |
+
created = client.post(
|
| 297 |
+
"/api/constructs",
|
| 298 |
+
json={
|
| 299 |
+
"name": "Flagged Scale",
|
| 300 |
+
"items": ["good item", "bad item"],
|
| 301 |
+
"reverse_scored": [False, True],
|
| 302 |
+
},
|
| 303 |
+
).json()
|
| 304 |
+
assert created["reverse_scored"] == [False, True]
|
| 305 |
+
|
| 306 |
+
project = client.post("/api/projects", json={"name": "FlagRun"}).json()
|
| 307 |
+
corpus = upload(client, project["id"], "c.csv", csv_rows(5)).json()
|
| 308 |
+
resp = run_job(client, project["id"], corpus["id"], created["id"])
|
| 309 |
+
job = wait_for_job(client, resp.json()["id"])
|
| 310 |
+
snapshot = client.get(f"/api/jobs/{job['id']}/results").json()["metadata"]["construct_snapshot"]
|
| 311 |
+
assert snapshot["items"][1]["reverse_scored"] is True
|
backend/tests/test_api.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
"""End-to-end API tests: project -> upload -> job -> results -> export.
|
| 2 |
|
| 3 |
Run against the fake embedding backend (model_name = 'fake-deterministic'),
|
| 4 |
-
exercising the full pipeline
|
| 5 |
-
(worker thread + polling), results summary, warnings, and export shape
|
| 6 |
without ML dependencies.
|
| 7 |
"""
|
| 8 |
|
|
@@ -81,7 +81,7 @@ def test_health(client):
|
|
| 81 |
def test_seed_constructs_present(client):
|
| 82 |
names = {c["name"] for c in client.get("/api/constructs").json()}
|
| 83 |
assert "Satisfaction with Life" in names
|
| 84 |
-
assert "Moral Foundations
|
| 85 |
|
| 86 |
|
| 87 |
def test_corpus_upload_parses_columns(flow):
|
|
@@ -103,14 +103,24 @@ def test_results_summary_and_warnings(client, flow):
|
|
| 103 |
assert summary["n_docs"] == 5 # empty row dropped
|
| 104 |
assert summary["n_dropped_empty"] == 1
|
| 105 |
assert len(summary["item_means"]) == 5 # SWLS has 5 items
|
| 106 |
-
|
| 107 |
-
assert
|
|
|
|
|
|
|
| 108 |
# satisfaction-flavored texts should outrank the bus/printer rows
|
| 109 |
top_texts = " ".join(d["text"] for d in summary["top_docs"][:2])
|
| 110 |
assert "satisfied" in top_texts or "ideal" in top_texts
|
| 111 |
assert metadata["construct"] == "Satisfaction with Life"
|
| 112 |
assert metadata["model"] == "fake-deterministic"
|
| 113 |
assert metadata["corpus_parse_info"]["format"] == "csv"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
|
| 115 |
|
| 116 |
def test_export_csv_shape(client, flow):
|
|
@@ -163,3 +173,99 @@ def test_validation_errors(client, flow):
|
|
| 163 |
# bad file type
|
| 164 |
resp = upload(client, flow["project"]["id"], "evil.exe", b"x")
|
| 165 |
assert resp.status_code == 400
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""End-to-end API tests: project -> upload -> job -> results -> export.
|
| 2 |
|
| 3 |
Run against the fake embedding backend (model_name = 'fake-deterministic'),
|
| 4 |
+
exercising the full pipeline - tolerant upload parsing, the real job queue
|
| 5 |
+
(worker thread + polling), results summary, warnings, and export shape -
|
| 6 |
without ML dependencies.
|
| 7 |
"""
|
| 8 |
|
|
|
|
| 81 |
def test_seed_constructs_present(client):
|
| 82 |
names = {c["name"] for c in client.get("/api/constructs").json()}
|
| 83 |
assert "Satisfaction with Life" in names
|
| 84 |
+
assert "Moral Foundations - Care" in names
|
| 85 |
|
| 86 |
|
| 87 |
def test_corpus_upload_parses_columns(flow):
|
|
|
|
| 103 |
assert summary["n_docs"] == 5 # empty row dropped
|
| 104 |
assert summary["n_dropped_empty"] == 1
|
| 105 |
assert len(summary["item_means"]) == 5 # SWLS has 5 items
|
| 106 |
+
codes = {w["code"] for w in summary["warnings"]} # structured objects (spec 0001)
|
| 107 |
+
assert "DUPLICATE_TEXTS" in codes
|
| 108 |
+
assert "EMPTY_ROWS_DROPPED" in codes
|
| 109 |
+
assert all({"code", "severity", "message"} <= set(w) for w in summary["warnings"])
|
| 110 |
# satisfaction-flavored texts should outrank the bus/printer rows
|
| 111 |
top_texts = " ".join(d["text"] for d in summary["top_docs"][:2])
|
| 112 |
assert "satisfied" in top_texts or "ideal" in top_texts
|
| 113 |
assert metadata["construct"] == "Satisfaction with Life"
|
| 114 |
assert metadata["model"] == "fake-deterministic"
|
| 115 |
assert metadata["corpus_parse_info"]["format"] == "csv"
|
| 116 |
+
# spec 0001/0004 metadata additions
|
| 117 |
+
assert metadata["output_schema_version"] == "1.0"
|
| 118 |
+
assert metadata["scoring"]["adjustment_strategy"] == "none"
|
| 119 |
+
assert metadata["language"]["selected"] == "en"
|
| 120 |
+
snapshot = metadata["construct_snapshot"]
|
| 121 |
+
assert snapshot["construct_id"] == "satisfaction_with_life"
|
| 122 |
+
assert snapshot["item_hash"] and len(snapshot["items"]) == 5
|
| 123 |
+
assert "ccr_score" in metadata["output_schema"]
|
| 124 |
|
| 125 |
|
| 126 |
def test_export_csv_shape(client, flow):
|
|
|
|
| 173 |
# bad file type
|
| 174 |
resp = upload(client, flow["project"]["id"], "evil.exe", b"x")
|
| 175 |
assert resp.status_code == 400
|
| 176 |
+
# unknown model id (registry-validated now)
|
| 177 |
+
resp = client.post(
|
| 178 |
+
"/api/jobs",
|
| 179 |
+
json={
|
| 180 |
+
"project_id": flow["project"]["id"],
|
| 181 |
+
"corpus_id": flow["corpus"]["id"],
|
| 182 |
+
"construct_id": flow["construct"]["id"],
|
| 183 |
+
"text_column": "text",
|
| 184 |
+
"model_name": "sentence-transformers/all-MiniLM-L6-v2", # provider id, not registry id
|
| 185 |
+
},
|
| 186 |
+
)
|
| 187 |
+
assert resp.status_code == 400
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def test_projects_carry_activity_and_sort_by_it(client, flow):
|
| 191 |
+
projects = client.get("/api/projects").json()
|
| 192 |
+
demo = next(p for p in projects if p["id"] == flow["project"]["id"])
|
| 193 |
+
assert demo["n_runs"] >= 1
|
| 194 |
+
assert demo["last_activity_at"] >= demo["created_at"]
|
| 195 |
+
# the project with runs sorts above a freshly created empty one from earlier tests
|
| 196 |
+
order = [p["last_activity_at"] for p in projects]
|
| 197 |
+
assert order == sorted(order, reverse=True)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
# --------------------------------------------------- spec 0003: models API
|
| 201 |
+
def test_models_endpoint_from_registry(client):
|
| 202 |
+
models = client.get("/api/models").json()
|
| 203 |
+
ids = [m["id"] for m in models]
|
| 204 |
+
assert "all-minilm-l6-v2" in ids and "e5-large-v2" in ids and "multilingual-e5-base" in ids
|
| 205 |
+
defaults = [m for m in models if m["default"]]
|
| 206 |
+
assert len(defaults) == 1 and defaults[0]["id"] == "all-minilm-l6-v2"
|
| 207 |
+
assert client.get("/api/languages").json()[0] == "en"
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
# --------------------------------------------------- spec 0004: construct library
|
| 211 |
+
def test_constructs_carry_library_fields(client):
|
| 212 |
+
swls = next(
|
| 213 |
+
c for c in client.get("/api/constructs").json()
|
| 214 |
+
if c["name"] == "Satisfaction with Life"
|
| 215 |
+
)
|
| 216 |
+
assert swls["verification_status"] == "needs_verification"
|
| 217 |
+
assert swls["version"] == 1
|
| 218 |
+
assert len(swls["item_hash"]) == 16
|
| 219 |
+
assert swls["reverse_scored"] == [False] * 5
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
# --------------------------------------------------- spec 0001: language warnings
|
| 223 |
+
def test_language_uncertain_on_tiny_corpus(client, flow):
|
| 224 |
+
body = client.get(f"/api/jobs/{flow['job']['id']}/results").json()
|
| 225 |
+
codes = {w["code"] for w in body["summary"]["warnings"]}
|
| 226 |
+
# 5-row corpus is far below the 20 detectable-row minimum
|
| 227 |
+
assert "LANGUAGE_UNCERTAIN" in codes
|
| 228 |
+
assert body["metadata"]["language"]["detected"] is None
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def test_short_text_and_model_language_warnings(client, flow):
|
| 232 |
+
csv = "text\n" + "\n".join(
|
| 233 |
+
[f"esta es una frase de prueba número {i} para el corpus" for i in range(25)]
|
| 234 |
+
+ ["si", "no"] # two very short rows
|
| 235 |
+
)
|
| 236 |
+
corpus = upload(client, flow["project"]["id"], "spanish.csv", csv.encode()).json()
|
| 237 |
+
job = client.post(
|
| 238 |
+
"/api/jobs",
|
| 239 |
+
json={
|
| 240 |
+
"project_id": flow["project"]["id"],
|
| 241 |
+
"corpus_id": corpus["id"],
|
| 242 |
+
"construct_id": flow["construct"]["id"],
|
| 243 |
+
"text_column": "text",
|
| 244 |
+
"model_name": "fake-deterministic",
|
| 245 |
+
"language": "es",
|
| 246 |
+
},
|
| 247 |
+
).json()
|
| 248 |
+
job = wait_for_job(client, job["id"])
|
| 249 |
+
assert job["status"] == "completed", job["error"]
|
| 250 |
+
body = client.get(f"/api/jobs/{job['id']}/results").json()
|
| 251 |
+
warnings = {w["code"]: w for w in body["summary"]["warnings"]}
|
| 252 |
+
assert warnings["TEXT_TOO_SHORT"]["count"] == 2
|
| 253 |
+
assert warnings["TEXT_TOO_SHORT"]["affected_rows_sample"]
|
| 254 |
+
assert body["metadata"]["language"]["selected"] == "es"
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
# --------------------------------------------------- spec 0002: script export
|
| 258 |
+
def test_script_export_is_valid_offline_python(client, flow):
|
| 259 |
+
job_id = flow["job"]["id"]
|
| 260 |
+
resp = client.get(f"/api/jobs/{job_id}/script")
|
| 261 |
+
assert resp.status_code == 200
|
| 262 |
+
source = resp.text
|
| 263 |
+
compile(source, "reproduce_analysis.py", "exec") # must be valid Python
|
| 264 |
+
# embeds the construct items verbatim and never references the platform
|
| 265 |
+
assert "In most ways my life is close to my ideal." in source
|
| 266 |
+
assert "sim_item_" in source and "ccr_score" in source
|
| 267 |
+
assert "127.0.0.1" not in source and "/api/" not in source
|
| 268 |
+
|
| 269 |
+
reqs = client.get(f"/api/jobs/{job_id}/script-requirements")
|
| 270 |
+
assert reqs.status_code == 200
|
| 271 |
+
assert "==" in reqs.text # pinned versions, not ranges
|
backend/tests/test_auth_tiers_and_lifecycle.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sign-in tiers (anonymous caps lift on sign-in) and project lifecycle
|
| 2 |
+
(archive is reversible; delete cascades to files and rows)."""
|
| 3 |
+
|
| 4 |
+
import io
|
| 5 |
+
import time
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import pytest
|
| 9 |
+
from fastapi.testclient import TestClient
|
| 10 |
+
|
| 11 |
+
from app.main import app
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def upload(client, project_id, name, payload: bytes):
|
| 15 |
+
return client.post(
|
| 16 |
+
f"/api/projects/{project_id}/corpora",
|
| 17 |
+
files={"file": (name, io.BytesIO(payload), "text/csv")},
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def wait_for_job(client, job_id, timeout=10.0):
|
| 22 |
+
deadline = time.time() + timeout
|
| 23 |
+
while time.time() < deadline:
|
| 24 |
+
job = client.get(f"/api/jobs/{job_id}").json()
|
| 25 |
+
if job["status"] in ("completed", "failed"):
|
| 26 |
+
return job
|
| 27 |
+
time.sleep(0.05)
|
| 28 |
+
raise TimeoutError(job_id)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@pytest.fixture()
|
| 32 |
+
def client():
|
| 33 |
+
with TestClient(app) as c:
|
| 34 |
+
yield c
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def csv_rows(n: int) -> bytes:
|
| 38 |
+
return ("text\n" + "\n".join(f"sample sentence number {i} here" for i in range(n))).encode()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ------------------------------------------------------------------- tiers
|
| 42 |
+
def test_anonymous_row_cap_and_signin_lifts_it(client, monkeypatch):
|
| 43 |
+
monkeypatch.setenv("CCR_ANON_MAX_ROWS", "5")
|
| 44 |
+
project = client.post("/api/projects", json={"name": "Tiers"}).json()
|
| 45 |
+
|
| 46 |
+
me = client.get("/api/auth/me").json()
|
| 47 |
+
assert me["signed_in"] is False and me["limits"]["max_rows"] == 5
|
| 48 |
+
|
| 49 |
+
resp = upload(client, project["id"], "big.csv", csv_rows(10))
|
| 50 |
+
assert resp.status_code == 400
|
| 51 |
+
assert "Sign in" in resp.json()["detail"]
|
| 52 |
+
|
| 53 |
+
resp = client.post(
|
| 54 |
+
"/api/auth/register",
|
| 55 |
+
json={"email": "deva@test.edu", "password": "password123", "name": "Deva"},
|
| 56 |
+
)
|
| 57 |
+
assert resp.status_code == 201
|
| 58 |
+
me = client.get("/api/auth/me").json()
|
| 59 |
+
assert me["signed_in"] is True and me["name"] == "Deva"
|
| 60 |
+
|
| 61 |
+
resp = upload(client, project["id"], "big.csv", csv_rows(10))
|
| 62 |
+
assert resp.status_code == 201, resp.json()
|
| 63 |
+
|
| 64 |
+
client.post("/api/auth/logout")
|
| 65 |
+
assert client.get("/api/auth/me").json()["signed_in"] is False
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def test_anonymous_size_cap(client, monkeypatch):
|
| 69 |
+
monkeypatch.setenv("CCR_ANON_MAX_BYTES", "200")
|
| 70 |
+
project = client.post("/api/projects", json={"name": "SizeCap"}).json()
|
| 71 |
+
resp = upload(client, project["id"], "big.csv", csv_rows(50))
|
| 72 |
+
assert resp.status_code == 413
|
| 73 |
+
assert "Sign in" in resp.json()["detail"]
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def test_tampered_session_cookie_is_anonymous(client):
|
| 77 |
+
client.cookies.set("ccr_session", "aGFja2Vy.badsignature")
|
| 78 |
+
assert client.get("/api/auth/me").json()["signed_in"] is False
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
# --------------------------------------------------------------- lifecycle
|
| 82 |
+
def test_archive_toggle_is_reversible(client):
|
| 83 |
+
project = client.post("/api/projects", json={"name": "Archivable"}).json()
|
| 84 |
+
assert project["archived"] is False
|
| 85 |
+
|
| 86 |
+
patched = client.patch(f"/api/projects/{project['id']}", json={"archived": True}).json()
|
| 87 |
+
assert patched["archived"] is True
|
| 88 |
+
listed = next(p for p in client.get("/api/projects").json() if p["id"] == project["id"])
|
| 89 |
+
assert listed["archived"] is True
|
| 90 |
+
|
| 91 |
+
patched = client.patch(f"/api/projects/{project['id']}", json={"archived": False}).json()
|
| 92 |
+
assert patched["archived"] is False
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def test_delete_cascades_rows_and_files(client):
|
| 96 |
+
project = client.post("/api/projects", json={"name": "Doomed"}).json()
|
| 97 |
+
corpus = upload(client, project["id"], "corpus.csv", csv_rows(6)).json()
|
| 98 |
+
|
| 99 |
+
constructs = client.get("/api/constructs").json()
|
| 100 |
+
swls = next(c for c in constructs if c["name"] == "Satisfaction with Life")
|
| 101 |
+
job = client.post(
|
| 102 |
+
"/api/jobs",
|
| 103 |
+
json={
|
| 104 |
+
"project_id": project["id"],
|
| 105 |
+
"corpus_id": corpus["id"],
|
| 106 |
+
"construct_id": swls["id"],
|
| 107 |
+
"text_column": "text",
|
| 108 |
+
"model_name": "fake-deterministic",
|
| 109 |
+
},
|
| 110 |
+
).json()
|
| 111 |
+
job = wait_for_job(client, job["id"])
|
| 112 |
+
assert job["status"] == "completed"
|
| 113 |
+
|
| 114 |
+
# capture file paths before deletion
|
| 115 |
+
results = client.get(f"/api/jobs/{job['id']}/results")
|
| 116 |
+
assert results.status_code == 200
|
| 117 |
+
|
| 118 |
+
resp = client.delete(f"/api/projects/{project['id']}")
|
| 119 |
+
assert resp.status_code == 204
|
| 120 |
+
|
| 121 |
+
assert client.get(f"/api/jobs/{job['id']}").status_code == 404
|
| 122 |
+
assert all(p["id"] != project["id"] for p in client.get("/api/projects").json())
|
| 123 |
+
# corpora listing for the deleted project 404s
|
| 124 |
+
assert client.get(f"/api/projects/{project['id']}/corpora").status_code == 404
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def test_delete_removes_files_on_disk(client, tmp_path):
|
| 128 |
+
import os
|
| 129 |
+
|
| 130 |
+
data_dir = Path(os.environ["CCR_DATA_DIR"])
|
| 131 |
+
project = client.post("/api/projects", json={"name": "FileCheck"}).json()
|
| 132 |
+
before = set((data_dir / "corpora").glob("*"))
|
| 133 |
+
upload(client, project["id"], "corpus.csv", csv_rows(6))
|
| 134 |
+
created = set((data_dir / "corpora").glob("*")) - before
|
| 135 |
+
assert len(created) == 1
|
| 136 |
+
|
| 137 |
+
client.delete(f"/api/projects/{project['id']}")
|
| 138 |
+
assert not created.pop().exists()
|
backend/tests/test_auto_migration.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Additive SQLite auto-migration: an old-schema DB gains new ORM columns
|
| 2 |
+
at startup instead of 500ing (the 'no such column: projects.archived' bug)."""
|
| 3 |
+
|
| 4 |
+
import sqlite3
|
| 5 |
+
|
| 6 |
+
from sqlalchemy import create_engine, inspect
|
| 7 |
+
|
| 8 |
+
from app.db import Base, auto_migrate_sqlite
|
| 9 |
+
from app.models import Project # noqa: F401 - registers tables on Base.metadata
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def test_old_schema_gains_missing_columns(tmp_path):
|
| 13 |
+
db_path = tmp_path / "old.db"
|
| 14 |
+
|
| 15 |
+
# Simulate a dev DB created before archived/language/etc. existed.
|
| 16 |
+
conn = sqlite3.connect(db_path)
|
| 17 |
+
conn.execute(
|
| 18 |
+
"CREATE TABLE projects (id VARCHAR(32) PRIMARY KEY, name VARCHAR(200), "
|
| 19 |
+
"description TEXT, created_at VARCHAR(32))"
|
| 20 |
+
)
|
| 21 |
+
conn.execute(
|
| 22 |
+
"INSERT INTO projects VALUES ('abc123', 'Old Project', '', '2026-07-01T00:00:00')"
|
| 23 |
+
)
|
| 24 |
+
conn.commit()
|
| 25 |
+
conn.close()
|
| 26 |
+
|
| 27 |
+
engine = create_engine(f"sqlite:///{db_path}")
|
| 28 |
+
Base.metadata.create_all(engine) # creates the other, brand-new tables
|
| 29 |
+
added = auto_migrate_sqlite(engine, Base.metadata)
|
| 30 |
+
|
| 31 |
+
assert "projects.archived" in added
|
| 32 |
+
cols = {c["name"] for c in inspect(engine).get_columns("projects")}
|
| 33 |
+
assert "archived" in cols
|
| 34 |
+
|
| 35 |
+
# Existing row survives with the default applied, and queries work.
|
| 36 |
+
conn = sqlite3.connect(db_path)
|
| 37 |
+
row = conn.execute("SELECT name, archived FROM projects WHERE id='abc123'").fetchone()
|
| 38 |
+
conn.close()
|
| 39 |
+
assert row == ("Old Project", 0)
|
| 40 |
+
|
| 41 |
+
# Second run is a no-op (idempotent).
|
| 42 |
+
assert auto_migrate_sqlite(engine, Base.metadata) == []
|
backend/tests/test_ccr.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Unit tests for the CCR engine (deterministic fake backend
|
| 2 |
|
| 3 |
import numpy as np
|
| 4 |
import pytest
|
|
|
|
| 1 |
+
"""Unit tests for the CCR engine (deterministic fake backend - no torch)."""
|
| 2 |
|
| 3 |
import numpy as np
|
| 4 |
import pytest
|
backend/tests/test_google_auth.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Google sign-in (Supabase PKCE): feature flag, redirect flow, callback
|
| 2 |
+
find-or-create, and password-login guard for Google-only accounts. The
|
| 3 |
+
Supabase exchange itself is mocked - no network in tests."""
|
| 4 |
+
|
| 5 |
+
import pytest
|
| 6 |
+
from fastapi.testclient import TestClient
|
| 7 |
+
|
| 8 |
+
from app import auth, auth_google
|
| 9 |
+
from app.main import app
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@pytest.fixture()
|
| 13 |
+
def client():
|
| 14 |
+
with TestClient(app) as c:
|
| 15 |
+
yield c
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@pytest.fixture()
|
| 19 |
+
def google_env(monkeypatch):
|
| 20 |
+
monkeypatch.setenv("SUPABASE_URL", "https://fakeproj.supabase.co")
|
| 21 |
+
monkeypatch.setenv("SUPABASE_ANON_KEY", "fake-anon-key")
|
| 22 |
+
monkeypatch.setenv("CCR_APP_URL", "http://testserver")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def test_unconfigured_instance_hides_and_refuses_google(client):
|
| 26 |
+
assert client.get("/api/auth/me").json().get("google_available") is False
|
| 27 |
+
assert client.get("/api/auth/google/login", follow_redirects=False).status_code == 503
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def test_login_redirects_to_supabase_with_pkce(client, google_env):
|
| 31 |
+
me = client.get("/api/auth/me").json()
|
| 32 |
+
assert me["google_available"] is True
|
| 33 |
+
|
| 34 |
+
resp = client.get("/api/auth/google/login", follow_redirects=False)
|
| 35 |
+
assert resp.status_code == 307
|
| 36 |
+
loc = resp.headers["location"]
|
| 37 |
+
assert loc.startswith("https://fakeproj.supabase.co/auth/v1/authorize?")
|
| 38 |
+
assert "provider=google" in loc
|
| 39 |
+
assert "code_challenge=" in loc and "code_challenge_method=s256" in loc
|
| 40 |
+
assert "redirect_to=http%3A%2F%2Ftestserver%2Fapi%2Fauth%2Fgoogle%2Fcallback" in loc
|
| 41 |
+
assert auth_google.VERIFIER_COOKIE in resp.cookies
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def test_callback_creates_user_and_signs_in(client, google_env, monkeypatch):
|
| 45 |
+
monkeypatch.setattr(
|
| 46 |
+
auth_google, "exchange",
|
| 47 |
+
lambda code, verifier: {"email": "pi@lab.edu", "name": "The PI"},
|
| 48 |
+
)
|
| 49 |
+
client.cookies.set(auth_google.VERIFIER_COOKIE, auth.sign_payload({"v": "verifier123"}))
|
| 50 |
+
|
| 51 |
+
resp = client.get("/api/auth/google/callback?code=abc", follow_redirects=False)
|
| 52 |
+
assert resp.status_code == 307 and resp.headers["location"] == "/"
|
| 53 |
+
|
| 54 |
+
me = client.get("/api/auth/me").json()
|
| 55 |
+
assert me["signed_in"] is True and me["email"] == "pi@lab.edu" and me["name"] == "The PI"
|
| 56 |
+
|
| 57 |
+
# second sign-in reuses the same account (no duplicate users)
|
| 58 |
+
client.cookies.set(auth_google.VERIFIER_COOKIE, auth.sign_payload({"v": "verifier456"}))
|
| 59 |
+
client.get("/api/auth/google/callback?code=def", follow_redirects=False)
|
| 60 |
+
from app.db import SessionLocal
|
| 61 |
+
from app.models import User
|
| 62 |
+
|
| 63 |
+
db = SessionLocal()
|
| 64 |
+
try:
|
| 65 |
+
assert db.query(User).filter_by(email="pi@lab.edu").count() == 1
|
| 66 |
+
finally:
|
| 67 |
+
db.close()
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def test_callback_without_verifier_fails_safely(client, google_env):
|
| 71 |
+
resp = client.get("/api/auth/google/callback?code=abc", follow_redirects=False)
|
| 72 |
+
assert resp.status_code == 307
|
| 73 |
+
assert "auth_error=" in resp.headers["location"]
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def test_google_only_account_cannot_password_login(client, google_env, monkeypatch):
|
| 77 |
+
monkeypatch.setattr(
|
| 78 |
+
auth_google, "exchange",
|
| 79 |
+
lambda code, verifier: {"email": "gonly@lab.edu", "name": "G Only"},
|
| 80 |
+
)
|
| 81 |
+
client.cookies.set(auth_google.VERIFIER_COOKIE, auth.sign_payload({"v": "v1"}))
|
| 82 |
+
client.get("/api/auth/google/callback?code=abc", follow_redirects=False)
|
| 83 |
+
client.post("/api/auth/logout")
|
| 84 |
+
|
| 85 |
+
resp = client.post(
|
| 86 |
+
"/api/auth/login", json={"email": "gonly@lab.edu", "password": "password123"}
|
| 87 |
+
)
|
| 88 |
+
assert resp.status_code == 401
|
| 89 |
+
assert "Google" in resp.json()["detail"]
|
backend/tests/test_registry_and_prefixes.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Registry loader + E5 prefix application (spec 0003) and warnings engine units (spec 0001)."""
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import pytest
|
| 5 |
+
|
| 6 |
+
from app import registry
|
| 7 |
+
from app.ccr import run_ccr
|
| 8 |
+
from app.warnings_engine import (
|
| 9 |
+
detect_corpus_language,
|
| 10 |
+
model_language_warning,
|
| 11 |
+
short_text_warning,
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class RecordingBackend:
|
| 16 |
+
"""Captures exactly what the engine sends to the encoder."""
|
| 17 |
+
|
| 18 |
+
name = "recording"
|
| 19 |
+
|
| 20 |
+
def __init__(self):
|
| 21 |
+
self.calls: list[list[str]] = []
|
| 22 |
+
|
| 23 |
+
def encode(self, texts, progress_cb=None):
|
| 24 |
+
self.calls.append(list(texts))
|
| 25 |
+
rng = np.random.default_rng(42)
|
| 26 |
+
emb = rng.normal(size=(len(texts), 8))
|
| 27 |
+
return emb / np.linalg.norm(emb, axis=1, keepdims=True)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
# ------------------------------------------------------------------ registry
|
| 31 |
+
def test_registry_loads_three_models_with_minilm_default():
|
| 32 |
+
models = registry.list_models()
|
| 33 |
+
ids = [m.id for m in models]
|
| 34 |
+
assert ids[0] == "all-minilm-l6-v2" # default sorts first
|
| 35 |
+
assert registry.default_model().id == "all-minilm-l6-v2"
|
| 36 |
+
assert {"e5-large-v2", "multilingual-e5-base"} <= set(ids)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def test_e5_config_requires_prefix_on_both_sides():
|
| 40 |
+
cfg = registry.get_model("e5-large-v2")
|
| 41 |
+
assert cfg.requires_prefix
|
| 42 |
+
assert cfg.item_prefix == "query: " and cfg.text_prefix == "query: "
|
| 43 |
+
assert cfg.max_seq_length == 512 and cfg.embedding_dimension == 1024
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def test_multilingual_language_set_resolves_to_real_codes():
|
| 47 |
+
cfg = registry.get_model("multilingual-e5-base")
|
| 48 |
+
assert cfg.language_set_name == "xlm_roberta_100"
|
| 49 |
+
assert "es" in cfg.supported_languages and "sw" in cfg.supported_languages
|
| 50 |
+
assert cfg.supports_language("hi") and not cfg.supports_language("xx")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def test_unknown_model_raises():
|
| 54 |
+
with pytest.raises(KeyError):
|
| 55 |
+
registry.get_model("gpt-9000")
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
# ------------------------------------------------------------------ prefixes
|
| 59 |
+
def test_run_ccr_applies_prefixes_to_encoder_input_only():
|
| 60 |
+
backend = RecordingBackend()
|
| 61 |
+
result = run_ccr(
|
| 62 |
+
["first text here", "second text here"],
|
| 63 |
+
["an item statement"],
|
| 64 |
+
backend,
|
| 65 |
+
item_prefix="query: ",
|
| 66 |
+
text_prefix="query: ",
|
| 67 |
+
)
|
| 68 |
+
item_call, text_call = backend.calls
|
| 69 |
+
assert item_call == ["query: an item statement"]
|
| 70 |
+
assert text_call == ["query: first text here", "query: second text here"]
|
| 71 |
+
# metadata records the prefixes for the reproducibility bundle
|
| 72 |
+
assert result.metadata["item_prefix"] == "query: "
|
| 73 |
+
assert result.metadata["text_prefix"] == "query: "
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def test_run_ccr_without_prefixes_passes_raw_strings():
|
| 77 |
+
backend = RecordingBackend()
|
| 78 |
+
run_ccr(["a text"], ["an item"], backend)
|
| 79 |
+
assert backend.calls[0] == ["an item"] and backend.calls[1] == ["a text"]
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
# ------------------------------------------------------------- warnings units
|
| 83 |
+
def test_short_text_warning_boundary():
|
| 84 |
+
w = short_text_warning(["one two three", "one two three four", "x"])
|
| 85 |
+
assert w["count"] == 2 # 3-token and 1-token rows flagged; 4-token row not
|
| 86 |
+
assert w["affected_rows_sample"] == [0, 2]
|
| 87 |
+
assert short_text_warning(["four token sentence here"] * 3) is None
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def test_language_detection_uncertain_below_min_rows():
|
| 91 |
+
result, warnings = detect_corpus_language(["hello there my good friend"] * 5, "en")
|
| 92 |
+
assert result.detected is None
|
| 93 |
+
assert warnings[0]["code"] == "LANGUAGE_UNCERTAIN"
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def test_language_mismatch_detected_deterministically():
|
| 97 |
+
spanish = [f"esta es una frase de prueba número {i} sobre la vida cotidiana" for i in range(30)]
|
| 98 |
+
result, warnings = detect_corpus_language(spanish, "en")
|
| 99 |
+
assert result.detected == "es"
|
| 100 |
+
assert any(w["code"] == "LANGUAGE_MISMATCH" for w in warnings)
|
| 101 |
+
# determinism: same corpus, same outcome
|
| 102 |
+
result2, _ = detect_corpus_language(spanish, "en")
|
| 103 |
+
assert result2.detected == result.detected and result2.confidence == result.confidence
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def test_model_language_unsupported_warning():
|
| 107 |
+
w = model_language_warning("xx", "multilingual-e5-base", frozenset({"en", "es"}), "demo_set")
|
| 108 |
+
assert w["code"] == "MODEL_LANGUAGE_UNSUPPORTED"
|
| 109 |
+
assert model_language_warning("en", "m", frozenset({"en"}), None) is None
|
| 110 |
+
assert model_language_warning("zz", "m", frozenset(), None) is None # unknown coverage: no warning
|
backend/tests/test_storage_backends.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Storage interface: local backend (default) and S3/R2 backend via a fake
|
| 2 |
+
client - proves production object storage works end to end (upload -> run ->
|
| 3 |
+
export -> retention delete) without any network or boto3 dependency."""
|
| 4 |
+
|
| 5 |
+
import io
|
| 6 |
+
import time
|
| 7 |
+
|
| 8 |
+
import pytest
|
| 9 |
+
from fastapi.testclient import TestClient
|
| 10 |
+
|
| 11 |
+
from app import storage
|
| 12 |
+
from app.main import app
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class FakeS3:
|
| 16 |
+
"""Minimal S3 client: just what storage.py calls."""
|
| 17 |
+
|
| 18 |
+
def __init__(self):
|
| 19 |
+
self.objects: dict[str, bytes] = {}
|
| 20 |
+
|
| 21 |
+
def put_object(self, Bucket, Key, Body):
|
| 22 |
+
self.objects[Key] = Body if isinstance(Body, bytes) else Body.read()
|
| 23 |
+
|
| 24 |
+
def head_object(self, Bucket, Key):
|
| 25 |
+
if Key not in self.objects:
|
| 26 |
+
raise KeyError(Key)
|
| 27 |
+
return {"ContentLength": len(self.objects[Key])}
|
| 28 |
+
|
| 29 |
+
def download_file(self, Bucket, Key, Filename):
|
| 30 |
+
with open(Filename, "wb") as f:
|
| 31 |
+
f.write(self.objects[Key])
|
| 32 |
+
|
| 33 |
+
def get_object(self, Bucket, Key):
|
| 34 |
+
return {"Body": io.BytesIO(self.objects[Key])}
|
| 35 |
+
|
| 36 |
+
def delete_object(self, Bucket, Key):
|
| 37 |
+
self.objects.pop(Key, None)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@pytest.fixture()
|
| 41 |
+
def s3(monkeypatch):
|
| 42 |
+
fake = FakeS3()
|
| 43 |
+
monkeypatch.setenv("CCR_STORAGE", "s3")
|
| 44 |
+
monkeypatch.setenv("CCR_S3_BUCKET", "ccr-test")
|
| 45 |
+
monkeypatch.setattr(storage, "_client", fake)
|
| 46 |
+
yield fake
|
| 47 |
+
monkeypatch.setattr(storage, "_client", None)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@pytest.fixture()
|
| 51 |
+
def client():
|
| 52 |
+
with TestClient(app) as c:
|
| 53 |
+
yield c
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def csv_rows(n: int) -> bytes:
|
| 57 |
+
return ("text\n" + "\n".join(f"sample sentence number {i} here" for i in range(n))).encode()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def wait_for_job(client, job_id, timeout=10.0):
|
| 61 |
+
deadline = time.time() + timeout
|
| 62 |
+
while time.time() < deadline:
|
| 63 |
+
job = client.get(f"/api/jobs/{job_id}").json()
|
| 64 |
+
if job["status"] in ("completed", "failed"):
|
| 65 |
+
return job
|
| 66 |
+
time.sleep(0.05)
|
| 67 |
+
raise TimeoutError(job_id)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# ------------------------------------------------------------------ unit
|
| 71 |
+
def test_s3_roundtrip(s3, tmp_path):
|
| 72 |
+
locator = storage.store_bytes("corpora", "abc.csv", b"text\nhello world row\n")
|
| 73 |
+
assert locator == "s3://corpora/abc.csv"
|
| 74 |
+
assert storage.exists(locator)
|
| 75 |
+
|
| 76 |
+
local, is_temp = storage.fetch_to_local(locator)
|
| 77 |
+
assert is_temp and local.read_bytes().startswith(b"text")
|
| 78 |
+
local.unlink()
|
| 79 |
+
|
| 80 |
+
assert b"".join(storage.open_stream(locator)) == b"text\nhello world row\n"
|
| 81 |
+
storage.delete(locator)
|
| 82 |
+
assert not storage.exists(locator)
|
| 83 |
+
assert s3.objects == {}
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def test_local_backend_unchanged(tmp_path):
|
| 87 |
+
locator = storage.store_bytes("corpora", "local_check.csv", b"data")
|
| 88 |
+
assert not storage.is_s3(locator)
|
| 89 |
+
local, is_temp = storage.fetch_to_local(locator)
|
| 90 |
+
assert not is_temp and local.read_bytes() == b"data"
|
| 91 |
+
storage.delete(locator)
|
| 92 |
+
assert not storage.exists(locator)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
# ------------------------------------------------------------ end to end
|
| 96 |
+
def test_full_flow_on_s3_backend(client, s3):
|
| 97 |
+
"""Signed-in upload -> corpus lands in the bucket -> run materializes a
|
| 98 |
+
temp copy -> result CSV lands in the bucket -> export streams it ->
|
| 99 |
+
project delete empties the bucket."""
|
| 100 |
+
client.post(
|
| 101 |
+
"/api/auth/register",
|
| 102 |
+
json={"email": "s3user@test.edu", "password": "password123", "name": "S3"},
|
| 103 |
+
)
|
| 104 |
+
project = client.post("/api/projects", json={"name": "S3Flow"}).json()
|
| 105 |
+
corpus = client.post(
|
| 106 |
+
f"/api/projects/{project['id']}/corpora",
|
| 107 |
+
files={"file": ("c.csv", io.BytesIO(csv_rows(6)), "text/csv")},
|
| 108 |
+
).json()
|
| 109 |
+
assert any(k.startswith("corpora/") for k in s3.objects)
|
| 110 |
+
|
| 111 |
+
construct = client.get("/api/constructs").json()[0]
|
| 112 |
+
job = client.post(
|
| 113 |
+
"/api/jobs",
|
| 114 |
+
json={
|
| 115 |
+
"project_id": project["id"],
|
| 116 |
+
"corpus_id": corpus["id"],
|
| 117 |
+
"construct_id": construct["id"],
|
| 118 |
+
"text_column": "text",
|
| 119 |
+
"model_name": "fake-deterministic",
|
| 120 |
+
},
|
| 121 |
+
).json()
|
| 122 |
+
job = wait_for_job(client, job["id"])
|
| 123 |
+
assert job["status"] == "completed"
|
| 124 |
+
assert any(k.startswith("results/") for k in s3.objects)
|
| 125 |
+
|
| 126 |
+
export = client.get(f"/api/jobs/{job['id']}/export")
|
| 127 |
+
assert export.status_code == 200
|
| 128 |
+
assert b"ccr_score" in export.content
|
| 129 |
+
|
| 130 |
+
client.delete(f"/api/projects/{project['id']}")
|
| 131 |
+
assert s3.objects == {} # cascade emptied the bucket
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def test_anonymous_run_deletes_s3_corpus(client, s3):
|
| 135 |
+
project = client.post("/api/projects", json={"name": "S3Anon"}).json()
|
| 136 |
+
corpus = client.post(
|
| 137 |
+
f"/api/projects/{project['id']}/corpora",
|
| 138 |
+
files={"file": ("c.csv", io.BytesIO(csv_rows(5)), "text/csv")},
|
| 139 |
+
).json()
|
| 140 |
+
construct = client.get("/api/constructs").json()[0]
|
| 141 |
+
job = client.post(
|
| 142 |
+
"/api/jobs",
|
| 143 |
+
json={
|
| 144 |
+
"project_id": project["id"],
|
| 145 |
+
"corpus_id": corpus["id"],
|
| 146 |
+
"construct_id": construct["id"],
|
| 147 |
+
"text_column": "text",
|
| 148 |
+
"model_name": "fake-deterministic",
|
| 149 |
+
},
|
| 150 |
+
).json()
|
| 151 |
+
job = wait_for_job(client, job["id"])
|
| 152 |
+
assert job["status"] == "completed"
|
| 153 |
+
assert not any(k.startswith("corpora/") for k in s3.objects) # upload gone
|
| 154 |
+
assert any(k.startswith("results/") for k in s3.objects) # results kept for TTL
|
frontend/src/App.jsx
CHANGED
|
@@ -2,20 +2,114 @@ import { useEffect, useState } from "react";
|
|
| 2 |
import { api } from "./api.js";
|
| 3 |
import Workspace from "./Workspace.jsx";
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
export default function App() {
|
| 6 |
const [projects, setProjects] = useState([]);
|
| 7 |
const [selectedId, setSelectedId] = useState(null);
|
| 8 |
const [creating, setCreating] = useState(false);
|
| 9 |
const [newName, setNewName] = useState("");
|
|
|
|
| 10 |
const [error, setError] = useState("");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
const loadProjects = () =>
|
| 13 |
api.listProjects().then(setProjects).catch((e) => setError(e.message));
|
|
|
|
| 14 |
|
| 15 |
useEffect(() => {
|
| 16 |
loadProjects();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
}, []);
|
| 18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
async function createProject(e) {
|
| 20 |
e.preventDefault();
|
| 21 |
if (!newName.trim()) return;
|
|
@@ -31,6 +125,10 @@ export default function App() {
|
|
| 31 |
}
|
| 32 |
|
| 33 |
const selected = projects.find((p) => p.id === selectedId) || null;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
return (
|
| 36 |
<div className="app">
|
|
@@ -39,45 +137,183 @@ export default function App() {
|
|
| 39 |
<span className="sub">
|
| 40 |
Contextualized Construct Representations · theory-driven psychological text analysis
|
| 41 |
</span>
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
{p.name}
|
| 54 |
-
<span className="date">{p.created_at.slice(0, 10)}</span>
|
| 55 |
</button>
|
| 56 |
-
)
|
|
|
|
|
|
|
| 57 |
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
</button>
|
| 71 |
-
<button className="ghost" type="button" onClick={() =>
|
| 72 |
Cancel
|
| 73 |
</button>
|
| 74 |
</div>
|
| 75 |
</form>
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
</aside>
|
| 82 |
|
| 83 |
<main className="main">
|
|
@@ -87,7 +323,17 @@ export default function App() {
|
|
| 87 |
</div>
|
| 88 |
)}
|
| 89 |
{selected ? (
|
| 90 |
-
<Workspace
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
) : (
|
| 92 |
<div className="card">
|
| 93 |
<h3>Welcome</h3>
|
|
@@ -97,7 +343,7 @@ export default function App() {
|
|
| 97 |
score distributions, and a reproducibility record for every run.
|
| 98 |
</p>
|
| 99 |
<p className="small muted">
|
| 100 |
-
Self-contained by design: embeddings run on this server itself
|
| 101 |
third-party AI APIs. Demo instance: storage is ephemeral and may reset;
|
| 102 |
please don't upload sensitive or identifiable data.
|
| 103 |
</p>
|
|
|
|
| 2 |
import { api } from "./api.js";
|
| 3 |
import Workspace from "./Workspace.jsx";
|
| 4 |
|
| 5 |
+
function relativeTime(iso) {
|
| 6 |
+
if (!iso) return "";
|
| 7 |
+
const then = new Date(iso.endsWith("Z") || iso.includes("+") ? iso : iso + "Z");
|
| 8 |
+
const mins = Math.max(0, Math.floor((Date.now() - then.getTime()) / 60000));
|
| 9 |
+
if (mins < 1) return "just now";
|
| 10 |
+
if (mins < 60) return `${mins}m ago`;
|
| 11 |
+
const hours = Math.floor(mins / 60);
|
| 12 |
+
if (hours < 24) return `${hours}h ago`;
|
| 13 |
+
const days = Math.floor(hours / 24);
|
| 14 |
+
if (days < 7) return `${days}d ago`;
|
| 15 |
+
return then.toISOString().slice(0, 10);
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
function groupProjects(projects) {
|
| 19 |
+
// Buckets by last activity: Today / This week / Earlier, with archived
|
| 20 |
+
// projects collapsed into their own group at the bottom. Projects arrive
|
| 21 |
+
// sorted by last activity (backend), so group order falls out naturally.
|
| 22 |
+
const now = Date.now();
|
| 23 |
+
const DAY = 86400000;
|
| 24 |
+
const groups = { Today: [], "This week": [], Earlier: [], Archived: [] };
|
| 25 |
+
for (const p of projects) {
|
| 26 |
+
if (p.archived) {
|
| 27 |
+
groups.Archived.push(p);
|
| 28 |
+
continue;
|
| 29 |
+
}
|
| 30 |
+
const iso = p.last_activity_at || p.created_at;
|
| 31 |
+
const t = new Date(iso.endsWith("Z") || iso.includes("+") ? iso : iso + "Z").getTime();
|
| 32 |
+
const age = now - t;
|
| 33 |
+
if (age < DAY) groups.Today.push(p);
|
| 34 |
+
else if (age < 7 * DAY) groups["This week"].push(p);
|
| 35 |
+
else groups.Earlier.push(p);
|
| 36 |
+
}
|
| 37 |
+
return Object.entries(groups).filter(([, items]) => items.length > 0);
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
export default function App() {
|
| 41 |
const [projects, setProjects] = useState([]);
|
| 42 |
const [selectedId, setSelectedId] = useState(null);
|
| 43 |
const [creating, setCreating] = useState(false);
|
| 44 |
const [newName, setNewName] = useState("");
|
| 45 |
+
const [filter, setFilter] = useState("");
|
| 46 |
const [error, setError] = useState("");
|
| 47 |
+
const [auth, setAuth] = useState(null);
|
| 48 |
+
const [showLogin, setShowLogin] = useState(false);
|
| 49 |
+
const [authMode, setAuthMode] = useState("signin"); // signin | register
|
| 50 |
+
const [authEmail, setAuthEmail] = useState("");
|
| 51 |
+
const [authPassword, setAuthPassword] = useState("");
|
| 52 |
+
const [authName, setAuthName] = useState("");
|
| 53 |
+
const [authError, setAuthError] = useState("");
|
| 54 |
+
const [authBusy, setAuthBusy] = useState(false);
|
| 55 |
|
| 56 |
const loadProjects = () =>
|
| 57 |
api.listProjects().then(setProjects).catch((e) => setError(e.message));
|
| 58 |
+
const loadAuth = () => api.authMe().then(setAuth).catch(() => {});
|
| 59 |
|
| 60 |
useEffect(() => {
|
| 61 |
loadProjects();
|
| 62 |
+
loadAuth();
|
| 63 |
+
// Surface Google sign-in failures passed back via redirect.
|
| 64 |
+
const params = new URLSearchParams(window.location.search);
|
| 65 |
+
const authFail = params.get("auth_error");
|
| 66 |
+
if (authFail) {
|
| 67 |
+
setError(`Sign-in problem: ${authFail.replaceAll("-", " ")}.`);
|
| 68 |
+
window.history.replaceState({}, "", "/");
|
| 69 |
+
}
|
| 70 |
}, []);
|
| 71 |
|
| 72 |
+
async function handleAuthSubmit(e) {
|
| 73 |
+
e.preventDefault();
|
| 74 |
+
setAuthError("");
|
| 75 |
+
setAuthBusy(true);
|
| 76 |
+
try {
|
| 77 |
+
if (authMode === "register") {
|
| 78 |
+
await api.register({ email: authEmail.trim(), password: authPassword, name: authName.trim() });
|
| 79 |
+
} else {
|
| 80 |
+
await api.login({ email: authEmail.trim(), password: authPassword });
|
| 81 |
+
}
|
| 82 |
+
setShowLogin(false);
|
| 83 |
+
setAuthEmail("");
|
| 84 |
+
setAuthPassword("");
|
| 85 |
+
setAuthName("");
|
| 86 |
+
await Promise.all([loadAuth(), loadProjects()]); // owned projects appear on sign-in
|
| 87 |
+
} catch (err) {
|
| 88 |
+
setAuthError(err.message);
|
| 89 |
+
} finally {
|
| 90 |
+
setAuthBusy(false);
|
| 91 |
+
}
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
async function handleLogout() {
|
| 95 |
+
try {
|
| 96 |
+
await api.logout();
|
| 97 |
+
await Promise.all([loadAuth(), loadProjects()]);
|
| 98 |
+
} catch (err) {
|
| 99 |
+
setError(err.message);
|
| 100 |
+
}
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
useEffect(() => {
|
| 104 |
+
if (projects.length === 0) {
|
| 105 |
+
setSelectedId(null);
|
| 106 |
+
return;
|
| 107 |
+
}
|
| 108 |
+
if (!selectedId || !projects.some((p) => p.id === selectedId)) {
|
| 109 |
+
setSelectedId(projects[0].id);
|
| 110 |
+
}
|
| 111 |
+
}, [projects, selectedId]);
|
| 112 |
+
|
| 113 |
async function createProject(e) {
|
| 114 |
e.preventDefault();
|
| 115 |
if (!newName.trim()) return;
|
|
|
|
| 125 |
}
|
| 126 |
|
| 127 |
const selected = projects.find((p) => p.id === selectedId) || null;
|
| 128 |
+
const normalizedFilter = filter.trim().toLowerCase();
|
| 129 |
+
const visibleProjects = projects.filter((p) =>
|
| 130 |
+
p.name.toLowerCase().includes(normalizedFilter)
|
| 131 |
+
);
|
| 132 |
|
| 133 |
return (
|
| 134 |
<div className="app">
|
|
|
|
| 137 |
<span className="sub">
|
| 138 |
Contextualized Construct Representations · theory-driven psychological text analysis
|
| 139 |
</span>
|
| 140 |
+
<span className="header-auth">
|
| 141 |
+
{auth?.signed_in ? (
|
| 142 |
+
<>
|
| 143 |
+
<span className="small">Hi, {auth.name}</span>
|
| 144 |
+
<button className="header-btn" onClick={handleLogout}>
|
| 145 |
+
Sign out
|
| 146 |
+
</button>
|
| 147 |
+
</>
|
| 148 |
+
) : (
|
| 149 |
+
<button className="header-btn" onClick={() => setShowLogin(true)}>
|
| 150 |
+
Sign in
|
|
|
|
|
|
|
| 151 |
</button>
|
| 152 |
+
)}
|
| 153 |
+
</span>
|
| 154 |
+
</header>
|
| 155 |
|
| 156 |
+
{showLogin && (
|
| 157 |
+
<div className="modal-backdrop" onClick={() => setShowLogin(false)}>
|
| 158 |
+
<div className="modal" onClick={(e) => e.stopPropagation()}>
|
| 159 |
+
<h3>{authMode === "register" ? "Create an account" : "Sign in"}</h3>
|
| 160 |
+
<p className="hint">
|
| 161 |
+
Accounts are free. Signing in lifts the anonymous limits
|
| 162 |
+
{auth?.limits?.max_rows
|
| 163 |
+
? ` (${Math.round(auth.limits.max_bytes / 1048576)} MB / ${auth.limits.max_rows.toLocaleString()} rows per file, ${auth?.usage?.max_runs_per_day ?? 3} runs/day)`
|
| 164 |
+
: ""}{" "}
|
| 165 |
+
and keeps your datasets and runs instead of deleting them after analysis.
|
| 166 |
+
</p>
|
| 167 |
+
{authError && <p className="small" style={{ color: "var(--danger, #b3261e)" }}>{authError}</p>}
|
| 168 |
+
{auth?.google_available && (
|
| 169 |
+
<>
|
| 170 |
+
<a className="primary google-btn" href="/api/auth/google/login">
|
| 171 |
+
Continue with Google
|
| 172 |
+
</a>
|
| 173 |
+
<p className="small muted" style={{ textAlign: "center", margin: "8px 0" }}>
|
| 174 |
+
or use email and password
|
| 175 |
+
</p>
|
| 176 |
+
</>
|
| 177 |
+
)}
|
| 178 |
+
<form onSubmit={handleAuthSubmit} className="mt">
|
| 179 |
+
{authMode === "register" && (
|
| 180 |
+
<label className="field">
|
| 181 |
+
Name
|
| 182 |
+
<input
|
| 183 |
+
type="text"
|
| 184 |
+
autoFocus
|
| 185 |
+
value={authName}
|
| 186 |
+
onChange={(e) => setAuthName(e.target.value)}
|
| 187 |
+
placeholder="e.g. Mohammad"
|
| 188 |
+
/>
|
| 189 |
+
</label>
|
| 190 |
+
)}
|
| 191 |
+
<label className="field">
|
| 192 |
+
Email
|
| 193 |
+
<input
|
| 194 |
+
type="email"
|
| 195 |
+
autoFocus={authMode === "signin"}
|
| 196 |
+
value={authEmail}
|
| 197 |
+
onChange={(e) => setAuthEmail(e.target.value)}
|
| 198 |
+
placeholder="you@example.com"
|
| 199 |
+
/>
|
| 200 |
+
</label>
|
| 201 |
+
<label className="field">
|
| 202 |
+
Password
|
| 203 |
+
{authMode === "register" && (
|
| 204 |
+
<span className="field-hint"> at least 8 characters</span>
|
| 205 |
+
)}
|
| 206 |
+
<input
|
| 207 |
+
type="password"
|
| 208 |
+
value={authPassword}
|
| 209 |
+
onChange={(e) => setAuthPassword(e.target.value)}
|
| 210 |
+
/>
|
| 211 |
+
</label>
|
| 212 |
+
<div className="row">
|
| 213 |
+
<button
|
| 214 |
+
className="primary"
|
| 215 |
+
type="submit"
|
| 216 |
+
disabled={
|
| 217 |
+
authBusy ||
|
| 218 |
+
!authEmail.trim() ||
|
| 219 |
+
!authPassword ||
|
| 220 |
+
(authMode === "register" && !authName.trim())
|
| 221 |
+
}
|
| 222 |
+
>
|
| 223 |
+
{authBusy ? "…" : authMode === "register" ? "Create account" : "Sign in"}
|
| 224 |
</button>
|
| 225 |
+
<button className="ghost" type="button" onClick={() => setShowLogin(false)}>
|
| 226 |
Cancel
|
| 227 |
</button>
|
| 228 |
</div>
|
| 229 |
</form>
|
| 230 |
+
<p className="small muted mt">
|
| 231 |
+
{authMode === "register" ? (
|
| 232 |
+
<>
|
| 233 |
+
Already have an account?{" "}
|
| 234 |
+
<button className="linkish" onClick={() => { setAuthMode("signin"); setAuthError(""); }}>
|
| 235 |
+
Sign in
|
| 236 |
+
</button>
|
| 237 |
+
</>
|
| 238 |
+
) : (
|
| 239 |
+
<>
|
| 240 |
+
New here?{" "}
|
| 241 |
+
<button className="linkish" onClick={() => { setAuthMode("register"); setAuthError(""); }}>
|
| 242 |
+
Create a free account
|
| 243 |
+
</button>
|
| 244 |
+
</>
|
| 245 |
+
)}
|
| 246 |
+
{auth?.google_available
|
| 247 |
+
? " · Forgot your password? Contact the lab admin, or use Google."
|
| 248 |
+
: " · Google sign-in arrives with lab accounts. Forgot your password? Contact the lab admin."}
|
| 249 |
+
</p>
|
| 250 |
+
</div>
|
| 251 |
+
</div>
|
| 252 |
+
)}
|
| 253 |
+
|
| 254 |
+
<div className="layout">
|
| 255 |
+
<aside className="sidebar">
|
| 256 |
+
<h2>
|
| 257 |
+
Projects
|
| 258 |
+
{projects.length > 0 && <span className="count">{projects.length}</span>}
|
| 259 |
+
</h2>
|
| 260 |
+
<input
|
| 261 |
+
type="text"
|
| 262 |
+
className="sidebar-filter"
|
| 263 |
+
placeholder="Search projects..."
|
| 264 |
+
value={filter}
|
| 265 |
+
onChange={(e) => setFilter(e.target.value)}
|
| 266 |
+
/>
|
| 267 |
+
<div className="project-list">
|
| 268 |
+
{groupProjects(visibleProjects).map(([groupLabel, items]) => (
|
| 269 |
+
<div key={groupLabel}>
|
| 270 |
+
<div className="group-label">{groupLabel}</div>
|
| 271 |
+
{items.map((p) => (
|
| 272 |
+
<button
|
| 273 |
+
key={p.id}
|
| 274 |
+
className={"project-item" + (p.id === selectedId ? " active" : "")}
|
| 275 |
+
onClick={() => setSelectedId(p.id)}
|
| 276 |
+
title={p.name}
|
| 277 |
+
>
|
| 278 |
+
<span className="project-name">{p.name}</span>
|
| 279 |
+
<span className="date">
|
| 280 |
+
{p.n_runs > 0 ? `${p.n_runs} run${p.n_runs === 1 ? "" : "s"} · ` : ""}
|
| 281 |
+
{relativeTime(p.last_activity_at || p.created_at)}
|
| 282 |
+
</span>
|
| 283 |
+
</button>
|
| 284 |
+
))}
|
| 285 |
+
</div>
|
| 286 |
+
))}
|
| 287 |
+
{filter && visibleProjects.length === 0 && (
|
| 288 |
+
<p className="small muted">No projects match "{filter}".</p>
|
| 289 |
+
)}
|
| 290 |
+
</div>
|
| 291 |
+
|
| 292 |
+
<div className="project-create">
|
| 293 |
+
{creating ? (
|
| 294 |
+
<form onSubmit={createProject}>
|
| 295 |
+
<input
|
| 296 |
+
type="text"
|
| 297 |
+
autoFocus
|
| 298 |
+
placeholder="Project name"
|
| 299 |
+
value={newName}
|
| 300 |
+
onChange={(e) => setNewName(e.target.value)}
|
| 301 |
+
/>
|
| 302 |
+
<div className="row mt">
|
| 303 |
+
<button className="primary" type="submit">
|
| 304 |
+
Create
|
| 305 |
+
</button>
|
| 306 |
+
<button className="ghost" type="button" onClick={() => setCreating(false)}>
|
| 307 |
+
Cancel
|
| 308 |
+
</button>
|
| 309 |
+
</div>
|
| 310 |
+
</form>
|
| 311 |
+
) : (
|
| 312 |
+
<button className="ghost" onClick={() => setCreating(true)}>
|
| 313 |
+
+ New project
|
| 314 |
+
</button>
|
| 315 |
+
)}
|
| 316 |
+
</div>
|
| 317 |
</aside>
|
| 318 |
|
| 319 |
<main className="main">
|
|
|
|
| 323 |
</div>
|
| 324 |
)}
|
| 325 |
{selected ? (
|
| 326 |
+
<Workspace
|
| 327 |
+
key={selected.id}
|
| 328 |
+
project={selected}
|
| 329 |
+
auth={auth}
|
| 330 |
+
onAuthRefresh={loadAuth}
|
| 331 |
+
onProjectChanged={loadProjects}
|
| 332 |
+
onProjectDeleted={() => {
|
| 333 |
+
setSelectedId(null);
|
| 334 |
+
loadProjects();
|
| 335 |
+
}}
|
| 336 |
+
/>
|
| 337 |
) : (
|
| 338 |
<div className="card">
|
| 339 |
<h3>Welcome</h3>
|
|
|
|
| 343 |
score distributions, and a reproducibility record for every run.
|
| 344 |
</p>
|
| 345 |
<p className="small muted">
|
| 346 |
+
Self-contained by design: embeddings run on this server itself - no
|
| 347 |
third-party AI APIs. Demo instance: storage is ephemeral and may reset;
|
| 348 |
please don't upload sensitive or identifiable data.
|
| 349 |
</p>
|
frontend/src/ConstructPicker.jsx
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { useEffect, useMemo, useRef, useState } from "react";
|
| 2 |
+
|
| 3 |
+
// Searchable, grouped construct picker. A flat dropdown stops working past
|
| 4 |
+
// ~15 options; the library is at 99 and growing. Researchers either know the
|
| 5 |
+
// scale ("GAD-7") or the family ("empathy"), so search matches name, category,
|
| 6 |
+
// and questionnaire, results group by category, and the last 5 used constructs
|
| 7 |
+
// stay on top (researchers re-run the same scales constantly).
|
| 8 |
+
// Keyboard: ArrowUp/Down move, Enter selects, Escape closes.
|
| 9 |
+
|
| 10 |
+
const RECENT_KEY = "ccr_recent_constructs";
|
| 11 |
+
const RECENT_MAX = 5;
|
| 12 |
+
|
| 13 |
+
function readRecent() {
|
| 14 |
+
try {
|
| 15 |
+
return JSON.parse(localStorage.getItem(RECENT_KEY) || "[]");
|
| 16 |
+
} catch {
|
| 17 |
+
return [];
|
| 18 |
+
}
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
export function rememberRecent(id) {
|
| 22 |
+
const next = [id, ...readRecent().filter((x) => x !== id)].slice(0, RECENT_MAX);
|
| 23 |
+
try {
|
| 24 |
+
localStorage.setItem(RECENT_KEY, JSON.stringify(next));
|
| 25 |
+
} catch {
|
| 26 |
+
/* storage unavailable: recents simply don't persist */
|
| 27 |
+
}
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
export default function ConstructPicker({ constructs, value, onChange }) {
|
| 31 |
+
const [open, setOpen] = useState(false);
|
| 32 |
+
const [query, setQuery] = useState("");
|
| 33 |
+
const [active, setActive] = useState(0);
|
| 34 |
+
const rootRef = useRef(null);
|
| 35 |
+
const inputRef = useRef(null);
|
| 36 |
+
const listRef = useRef(null);
|
| 37 |
+
|
| 38 |
+
const selected = constructs.find((c) => c.id === value) || null;
|
| 39 |
+
|
| 40 |
+
const groups = useMemo(() => {
|
| 41 |
+
const q = query.trim().toLowerCase();
|
| 42 |
+
const match = (c) =>
|
| 43 |
+
!q ||
|
| 44 |
+
c.name.toLowerCase().includes(q) ||
|
| 45 |
+
(c.category || "").toLowerCase().includes(q);
|
| 46 |
+
const filtered = constructs.filter(match);
|
| 47 |
+
|
| 48 |
+
const out = [];
|
| 49 |
+
const used = new Set();
|
| 50 |
+
|
| 51 |
+
const recentIds = readRecent();
|
| 52 |
+
const recent = recentIds
|
| 53 |
+
.map((id) => filtered.find((c) => c.id === id))
|
| 54 |
+
.filter(Boolean);
|
| 55 |
+
if (recent.length) {
|
| 56 |
+
out.push(["Recently used", recent]);
|
| 57 |
+
recent.forEach((c) => used.add(c.id));
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
const custom = filtered.filter((c) => !c.is_seed && !used.has(c.id));
|
| 61 |
+
if (custom.length) {
|
| 62 |
+
out.push(["My custom constructs", custom]);
|
| 63 |
+
custom.forEach((c) => used.add(c.id));
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
const byCategory = new Map();
|
| 67 |
+
for (const c of filtered) {
|
| 68 |
+
if (used.has(c.id)) continue;
|
| 69 |
+
const cat = c.category || "Other";
|
| 70 |
+
if (!byCategory.has(cat)) byCategory.set(cat, []);
|
| 71 |
+
byCategory.get(cat).push(c);
|
| 72 |
+
}
|
| 73 |
+
for (const cat of [...byCategory.keys()].sort((a, b) => a.localeCompare(b))) {
|
| 74 |
+
out.push([cat, byCategory.get(cat).sort((a, b) => a.name.localeCompare(b.name))]);
|
| 75 |
+
}
|
| 76 |
+
return out;
|
| 77 |
+
}, [constructs, query]);
|
| 78 |
+
|
| 79 |
+
const flat = useMemo(() => groups.flatMap(([, items]) => items), [groups]);
|
| 80 |
+
|
| 81 |
+
useEffect(() => setActive(0), [query, open]);
|
| 82 |
+
|
| 83 |
+
// Close on outside click.
|
| 84 |
+
useEffect(() => {
|
| 85 |
+
if (!open) return undefined;
|
| 86 |
+
const onDown = (e) => {
|
| 87 |
+
if (rootRef.current && !rootRef.current.contains(e.target)) setOpen(false);
|
| 88 |
+
};
|
| 89 |
+
document.addEventListener("mousedown", onDown);
|
| 90 |
+
return () => document.removeEventListener("mousedown", onDown);
|
| 91 |
+
}, [open]);
|
| 92 |
+
|
| 93 |
+
// Keep the active option scrolled into view.
|
| 94 |
+
useEffect(() => {
|
| 95 |
+
const el = listRef.current?.querySelector('[data-active="true"]');
|
| 96 |
+
el?.scrollIntoView({ block: "nearest" });
|
| 97 |
+
}, [active, open]);
|
| 98 |
+
|
| 99 |
+
function choose(c) {
|
| 100 |
+
onChange(c.id);
|
| 101 |
+
rememberRecent(c.id);
|
| 102 |
+
setOpen(false);
|
| 103 |
+
setQuery("");
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
function onKeyDown(e) {
|
| 107 |
+
if (e.key === "ArrowDown") {
|
| 108 |
+
e.preventDefault();
|
| 109 |
+
setActive((a) => Math.min(a + 1, flat.length - 1));
|
| 110 |
+
} else if (e.key === "ArrowUp") {
|
| 111 |
+
e.preventDefault();
|
| 112 |
+
setActive((a) => Math.max(a - 1, 0));
|
| 113 |
+
} else if (e.key === "Enter") {
|
| 114 |
+
e.preventDefault();
|
| 115 |
+
if (flat[active]) choose(flat[active]);
|
| 116 |
+
} else if (e.key === "Escape") {
|
| 117 |
+
setOpen(false);
|
| 118 |
+
}
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
let index = -1; // running index across groups for keyboard highlight
|
| 122 |
+
|
| 123 |
+
return (
|
| 124 |
+
<div className="picker" ref={rootRef}>
|
| 125 |
+
{!open ? (
|
| 126 |
+
<button
|
| 127 |
+
type="button"
|
| 128 |
+
className="picker-display"
|
| 129 |
+
aria-haspopup="listbox"
|
| 130 |
+
aria-expanded="false"
|
| 131 |
+
onClick={() => {
|
| 132 |
+
setOpen(true);
|
| 133 |
+
setTimeout(() => inputRef.current?.focus(), 0);
|
| 134 |
+
}}
|
| 135 |
+
>
|
| 136 |
+
{selected ? (
|
| 137 |
+
<>
|
| 138 |
+
<span>{selected.name}</span>
|
| 139 |
+
<span className="picker-meta">
|
| 140 |
+
{selected.items.length} item{selected.items.length === 1 ? "" : "s"}
|
| 141 |
+
</span>
|
| 142 |
+
</>
|
| 143 |
+
) : (
|
| 144 |
+
<span className="muted">Select a construct ({constructs.length} in library)</span>
|
| 145 |
+
)}
|
| 146 |
+
<span className="picker-caret" aria-hidden="true">▾</span>
|
| 147 |
+
</button>
|
| 148 |
+
) : (
|
| 149 |
+
<>
|
| 150 |
+
<input
|
| 151 |
+
ref={inputRef}
|
| 152 |
+
type="text"
|
| 153 |
+
role="combobox"
|
| 154 |
+
aria-expanded="true"
|
| 155 |
+
aria-autocomplete="list"
|
| 156 |
+
className="picker-search"
|
| 157 |
+
placeholder="Search by scale, construct, or category (e.g. empathy, GAD-7)"
|
| 158 |
+
value={query}
|
| 159 |
+
onChange={(e) => setQuery(e.target.value)}
|
| 160 |
+
onKeyDown={onKeyDown}
|
| 161 |
+
/>
|
| 162 |
+
<div className="picker-panel" role="listbox" ref={listRef}>
|
| 163 |
+
{flat.length === 0 && (
|
| 164 |
+
<p className="small muted picker-empty">
|
| 165 |
+
No constructs match "{query}". Try a scale abbreviation or use + Custom construct.
|
| 166 |
+
</p>
|
| 167 |
+
)}
|
| 168 |
+
{groups.map(([label, items]) => (
|
| 169 |
+
<div key={label}>
|
| 170 |
+
<div className="picker-group">{label}</div>
|
| 171 |
+
{items.map((c) => {
|
| 172 |
+
index += 1;
|
| 173 |
+
const isActive = index === active;
|
| 174 |
+
return (
|
| 175 |
+
<div
|
| 176 |
+
key={c.id}
|
| 177 |
+
role="option"
|
| 178 |
+
aria-selected={c.id === value}
|
| 179 |
+
data-active={isActive || undefined}
|
| 180 |
+
className={
|
| 181 |
+
"picker-option" +
|
| 182 |
+
(isActive ? " active" : "") +
|
| 183 |
+
(c.id === value ? " selected" : "")
|
| 184 |
+
}
|
| 185 |
+
onMouseDown={(e) => {
|
| 186 |
+
e.preventDefault();
|
| 187 |
+
choose(c);
|
| 188 |
+
}}
|
| 189 |
+
>
|
| 190 |
+
<span className="picker-name">{c.name}</span>
|
| 191 |
+
<span className="picker-meta">
|
| 192 |
+
{c.category ? `${c.category} · ` : ""}
|
| 193 |
+
{c.items.length} item{c.items.length === 1 ? "" : "s"}
|
| 194 |
+
{c.verification_status !== "verified" ? " · unverified" : ""}
|
| 195 |
+
</span>
|
| 196 |
+
</div>
|
| 197 |
+
);
|
| 198 |
+
})}
|
| 199 |
+
</div>
|
| 200 |
+
))}
|
| 201 |
+
</div>
|
| 202 |
+
</>
|
| 203 |
+
)}
|
| 204 |
+
</div>
|
| 205 |
+
);
|
| 206 |
+
}
|
frontend/src/ResultsView.jsx
CHANGED
|
@@ -25,14 +25,20 @@ export default function ResultsView({ jobId, onBack }) {
|
|
| 25 |
|
| 26 |
return (
|
| 27 |
<>
|
| 28 |
-
<div className="
|
| 29 |
<button className="ghost" onClick={onBack}>
|
| 30 |
← Back to workspace
|
| 31 |
</button>
|
| 32 |
-
<div className="row">
|
| 33 |
<a href={api.exportUrl(jobId)}>
|
| 34 |
<button className="primary">Export results CSV</button>
|
| 35 |
</a>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
<a href={api.metadataUrl(jobId)}>
|
| 37 |
<button className="ghost">Run metadata (JSON)</button>
|
| 38 |
</a>
|
|
@@ -64,7 +70,13 @@ export default function ResultsView({ jobId, onBack }) {
|
|
| 64 |
<strong className="small">Data-quality notes</strong>
|
| 65 |
<ul className="small" style={{ margin: "4px 0 0", paddingLeft: 20 }}>
|
| 66 |
{summary.warnings.map((w, i) => (
|
| 67 |
-
<li key={i}>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
))}
|
| 69 |
</ul>
|
| 70 |
</div>
|
|
@@ -79,7 +91,7 @@ export default function ResultsView({ jobId, onBack }) {
|
|
| 79 |
<div className="card">
|
| 80 |
<h3>Per-item mean loadings</h3>
|
| 81 |
<p className="hint">
|
| 82 |
-
Mean similarity of the corpus to each scale item
|
| 83 |
items drive the construct signal.
|
| 84 |
</p>
|
| 85 |
{summary.item_means.map((m, i) => (
|
|
@@ -110,7 +122,7 @@ export default function ResultsView({ jobId, onBack }) {
|
|
| 110 |
</div>
|
| 111 |
|
| 112 |
<div className="meta-footer">
|
| 113 |
-
<strong>Reproducibility record</strong>
|
| 114 |
{metadata.embedding_dim}) · items hash: <code>{metadata.items_sha256_16}</code> ·
|
| 115 |
text column: <code>{metadata.text_column}</code> · run:{" "}
|
| 116 |
{metadata.started_at} → {metadata.finished_at} ({metadata.duration_seconds}s) ·
|
|
@@ -118,7 +130,7 @@ export default function ResultsView({ jobId, onBack }) {
|
|
| 118 |
{metadata.sentence_transformers &&
|
| 119 |
` · sentence-transformers ${metadata.sentence_transformers}`}
|
| 120 |
<div className="mt small">
|
| 121 |
-
Construct reference: {metadata.construct_reference || "
|
| 122 |
</div>
|
| 123 |
</div>
|
| 124 |
</>
|
|
@@ -136,22 +148,24 @@ function Stat({ k, v }) {
|
|
| 136 |
|
| 137 |
function DocTable({ docs }) {
|
| 138 |
return (
|
| 139 |
-
<
|
| 140 |
-
<
|
| 141 |
-
<
|
| 142 |
-
<
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
</thead>
|
| 146 |
-
<tbody>
|
| 147 |
-
{docs.map((d) => (
|
| 148 |
-
<tr key={d.row}>
|
| 149 |
-
<td className="score">{d.score.toFixed(3)}</td>
|
| 150 |
-
<td>{d.text}</td>
|
| 151 |
</tr>
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 155 |
);
|
| 156 |
}
|
| 157 |
|
|
|
|
| 25 |
|
| 26 |
return (
|
| 27 |
<>
|
| 28 |
+
<div className="results-toolbar">
|
| 29 |
<button className="ghost" onClick={onBack}>
|
| 30 |
← Back to workspace
|
| 31 |
</button>
|
| 32 |
+
<div className="row result-actions">
|
| 33 |
<a href={api.exportUrl(jobId)}>
|
| 34 |
<button className="primary">Export results CSV</button>
|
| 35 |
</a>
|
| 36 |
+
<a href={api.scriptUrl(jobId)}>
|
| 37 |
+
<button className="ghost">Python script</button>
|
| 38 |
+
</a>
|
| 39 |
+
<a href={api.scriptRequirementsUrl(jobId)}>
|
| 40 |
+
<button className="ghost">requirements.txt</button>
|
| 41 |
+
</a>
|
| 42 |
<a href={api.metadataUrl(jobId)}>
|
| 43 |
<button className="ghost">Run metadata (JSON)</button>
|
| 44 |
</a>
|
|
|
|
| 70 |
<strong className="small">Data-quality notes</strong>
|
| 71 |
<ul className="small" style={{ margin: "4px 0 0", paddingLeft: 20 }}>
|
| 72 |
{summary.warnings.map((w, i) => (
|
| 73 |
+
<li key={i}>
|
| 74 |
+
{typeof w === "string" ? w : (
|
| 75 |
+
<>
|
| 76 |
+
<code style={{ fontSize: 11 }}>{w.code}</code> - {w.message}
|
| 77 |
+
</>
|
| 78 |
+
)}
|
| 79 |
+
</li>
|
| 80 |
))}
|
| 81 |
</ul>
|
| 82 |
</div>
|
|
|
|
| 91 |
<div className="card">
|
| 92 |
<h3>Per-item mean loadings</h3>
|
| 93 |
<p className="hint">
|
| 94 |
+
Mean similarity of the corpus to each scale item - a face-validity check on which
|
| 95 |
items drive the construct signal.
|
| 96 |
</p>
|
| 97 |
{summary.item_means.map((m, i) => (
|
|
|
|
| 122 |
</div>
|
| 123 |
|
| 124 |
<div className="meta-footer">
|
| 125 |
+
<strong>Reproducibility record</strong> - model: <code>{metadata.model}</code> (dim{" "}
|
| 126 |
{metadata.embedding_dim}) · items hash: <code>{metadata.items_sha256_16}</code> ·
|
| 127 |
text column: <code>{metadata.text_column}</code> · run:{" "}
|
| 128 |
{metadata.started_at} → {metadata.finished_at} ({metadata.duration_seconds}s) ·
|
|
|
|
| 130 |
{metadata.sentence_transformers &&
|
| 131 |
` · sentence-transformers ${metadata.sentence_transformers}`}
|
| 132 |
<div className="mt small">
|
| 133 |
+
Construct reference: {metadata.construct_reference || "-"}
|
| 134 |
</div>
|
| 135 |
</div>
|
| 136 |
</>
|
|
|
|
| 148 |
|
| 149 |
function DocTable({ docs }) {
|
| 150 |
return (
|
| 151 |
+
<div className="table-wrap">
|
| 152 |
+
<table className="docs">
|
| 153 |
+
<thead>
|
| 154 |
+
<tr>
|
| 155 |
+
<th style={{ width: 60 }}>Score</th>
|
| 156 |
+
<th>Text</th>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
</tr>
|
| 158 |
+
</thead>
|
| 159 |
+
<tbody>
|
| 160 |
+
{docs.map((d) => (
|
| 161 |
+
<tr key={d.row}>
|
| 162 |
+
<td className="score">{d.score.toFixed(3)}</td>
|
| 163 |
+
<td>{d.text}</td>
|
| 164 |
+
</tr>
|
| 165 |
+
))}
|
| 166 |
+
</tbody>
|
| 167 |
+
</table>
|
| 168 |
+
</div>
|
| 169 |
);
|
| 170 |
}
|
| 171 |
|
frontend/src/Workspace.jsx
CHANGED
|
@@ -1,8 +1,9 @@
|
|
| 1 |
import { useCallback, useEffect, useRef, useState } from "react";
|
| 2 |
import { api } from "./api.js";
|
|
|
|
| 3 |
import ResultsView from "./ResultsView.jsx";
|
| 4 |
|
| 5 |
-
export default function Workspace({ project }) {
|
| 6 |
const [corpora, setCorpora] = useState([]);
|
| 7 |
const [constructs, setConstructs] = useState([]);
|
| 8 |
const [models, setModels] = useState([]);
|
|
@@ -12,13 +13,36 @@ export default function Workspace({ project }) {
|
|
| 12 |
const [textColumn, setTextColumn] = useState("");
|
| 13 |
const [constructId, setConstructId] = useState("");
|
| 14 |
const [modelName, setModelName] = useState("");
|
|
|
|
|
|
|
| 15 |
const [uploading, setUploading] = useState(false);
|
| 16 |
const [running, setRunning] = useState(false);
|
| 17 |
const [error, setError] = useState("");
|
| 18 |
const [showNewConstruct, setShowNewConstruct] = useState(false);
|
| 19 |
const [viewJobId, setViewJobId] = useState(null);
|
|
|
|
|
|
|
| 20 |
const fileRef = useRef(null);
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
const refreshJobs = useCallback(
|
| 23 |
() => api.listJobs(project.id).then(setJobs).catch(() => {}),
|
| 24 |
[project.id]
|
|
@@ -31,9 +55,11 @@ export default function Workspace({ project }) {
|
|
| 31 |
.models()
|
| 32 |
.then((m) => {
|
| 33 |
setModels(m);
|
| 34 |
-
|
|
|
|
| 35 |
})
|
| 36 |
.catch((e) => setError(e.message));
|
|
|
|
| 37 |
refreshJobs();
|
| 38 |
}, [project.id, refreshJobs]);
|
| 39 |
|
|
@@ -77,8 +103,10 @@ export default function Workspace({ project }) {
|
|
| 77 |
construct_id: constructId,
|
| 78 |
text_column: textColumn,
|
| 79 |
model_name: modelName,
|
|
|
|
| 80 |
});
|
| 81 |
await refreshJobs();
|
|
|
|
| 82 |
} catch (err) {
|
| 83 |
setError(err.message);
|
| 84 |
} finally {
|
|
@@ -108,13 +136,78 @@ export default function Workspace({ project }) {
|
|
| 108 |
</div>
|
| 109 |
)}
|
| 110 |
|
| 111 |
-
{/*
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
<div className="card">
|
| 113 |
<h3>
|
| 114 |
<span className="step-badge">1</span>Corpus
|
| 115 |
</h3>
|
| 116 |
<p className="hint">
|
| 117 |
Upload a CSV or XLSX file, then choose the column containing the text to analyze.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
</p>
|
| 119 |
<div className="row">
|
| 120 |
<div className="grow">
|
|
@@ -134,7 +227,7 @@ export default function Workspace({ project }) {
|
|
| 134 |
<label className="field">
|
| 135 |
Corpus
|
| 136 |
<select value={corpusId} onChange={(e) => setCorpusId(e.target.value)}>
|
| 137 |
-
<option value="">
|
| 138 |
{corpora.map((c) => (
|
| 139 |
<option key={c.id} value={c.id}>
|
| 140 |
{c.filename} ({c.n_rows.toLocaleString()} rows)
|
|
@@ -151,7 +244,7 @@ export default function Workspace({ project }) {
|
|
| 151 |
onChange={(e) => setTextColumn(e.target.value)}
|
| 152 |
disabled={!corpus}
|
| 153 |
>
|
| 154 |
-
<option value="">
|
| 155 |
{corpus?.columns.map((col) => (
|
| 156 |
<option key={col} value={col}>
|
| 157 |
{col}
|
|
@@ -167,7 +260,7 @@ export default function Workspace({ project }) {
|
|
| 167 |
)}
|
| 168 |
</div>
|
| 169 |
|
| 170 |
-
{/* Step 2
|
| 171 |
<div className="card">
|
| 172 |
<h3>
|
| 173 |
<span className="step-badge">2</span>Construct
|
|
@@ -176,16 +269,13 @@ export default function Workspace({ project }) {
|
|
| 176 |
Pick a validated scale from the library, or define custom items. CCR scores each
|
| 177 |
text by its similarity to these items.
|
| 178 |
</p>
|
| 179 |
-
<div className="row">
|
| 180 |
<div className="grow">
|
| 181 |
-
<
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
</option>
|
| 187 |
-
))}
|
| 188 |
-
</select>
|
| 189 |
</div>
|
| 190 |
<button className="ghost" onClick={() => setShowNewConstruct((s) => !s)}>
|
| 191 |
{showNewConstruct ? "Close" : "+ Custom construct"}
|
|
@@ -196,12 +286,21 @@ export default function Workspace({ project }) {
|
|
| 196 |
<>
|
| 197 |
<ul className="construct-items">
|
| 198 |
{construct.items.map((item, i) => (
|
| 199 |
-
<li key={i}>
|
|
|
|
|
|
|
|
|
|
| 200 |
))}
|
| 201 |
</ul>
|
| 202 |
{construct.reference && (
|
| 203 |
<p className="small muted mt">Reference: {construct.reference}</p>
|
| 204 |
)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
</>
|
| 206 |
)}
|
| 207 |
|
|
@@ -218,104 +317,185 @@ export default function Workspace({ project }) {
|
|
| 218 |
)}
|
| 219 |
</div>
|
| 220 |
|
| 221 |
-
{/* Step 3
|
| 222 |
<div className="card">
|
| 223 |
<h3>
|
| 224 |
-
<span className="step-badge">3</span>
|
| 225 |
</h3>
|
| 226 |
<p className="hint">
|
| 227 |
-
Embeddings run locally via sentence-transformers;
|
| 228 |
-
in the run metadata
|
|
|
|
| 229 |
</p>
|
| 230 |
-
<div className="
|
| 231 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
<select value={modelName} onChange={(e) => setModelName(e.target.value)}>
|
| 233 |
{models.map((m) => (
|
| 234 |
-
<option key={m.
|
| 235 |
{m.label}
|
| 236 |
</option>
|
| 237 |
))}
|
| 238 |
</select>
|
| 239 |
-
</
|
| 240 |
-
<button className="primary" disabled={!canRun} onClick={handleRun}>
|
| 241 |
{running ? "Starting…" : "Run CCR analysis"}
|
| 242 |
</button>
|
| 243 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
</div>
|
| 245 |
|
| 246 |
{/* Jobs */}
|
| 247 |
{jobs.length > 0 && (
|
| 248 |
<div className="card">
|
| 249 |
<h3>Runs</h3>
|
| 250 |
-
<
|
| 251 |
-
<
|
| 252 |
-
<
|
| 253 |
-
<
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
{jobs.map((j) => (
|
| 262 |
-
<tr key={j.id}>
|
| 263 |
-
<td className="muted">{(j.started_at || j.created_at).replace("T", " ").slice(0, 16)}</td>
|
| 264 |
-
<td>{j.corpus_filename}</td>
|
| 265 |
-
<td>{j.construct_name}</td>
|
| 266 |
-
<td>
|
| 267 |
-
{j.status === "running" ? (
|
| 268 |
-
<div className="progress-track" title={`${Math.round(j.progress * 100)}%`}>
|
| 269 |
-
<div
|
| 270 |
-
className="progress-fill"
|
| 271 |
-
style={{ width: `${Math.max(3, j.progress * 100)}%` }}
|
| 272 |
-
/>
|
| 273 |
-
</div>
|
| 274 |
-
) : (
|
| 275 |
-
<span className={`pill ${j.status}`}>{j.status}</span>
|
| 276 |
-
)}
|
| 277 |
-
{j.status === "failed" && (
|
| 278 |
-
<div className="small muted" title={j.error}>
|
| 279 |
-
{j.error.split("\n").pop()}
|
| 280 |
-
</div>
|
| 281 |
-
)}
|
| 282 |
-
</td>
|
| 283 |
-
<td>
|
| 284 |
-
{j.status === "completed" && (
|
| 285 |
-
<button className="linkish" onClick={() => setViewJobId(j.id)}>
|
| 286 |
-
View results
|
| 287 |
-
</button>
|
| 288 |
-
)}
|
| 289 |
-
</td>
|
| 290 |
</tr>
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
</div>
|
| 295 |
)}
|
| 296 |
</>
|
| 297 |
);
|
| 298 |
}
|
| 299 |
|
|
|
|
|
|
|
| 300 |
function NewConstructForm({ onCreated, onError }) {
|
| 301 |
const [name, setName] = useState("");
|
| 302 |
const [reference, setReference] = useState("");
|
| 303 |
const [itemsText, setItemsText] = useState("");
|
| 304 |
const [saving, setSaving] = useState(false);
|
|
|
|
|
|
|
|
|
|
| 305 |
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
|
|
|
| 309 |
.split("\n")
|
| 310 |
.map((s) => s.trim())
|
| 311 |
-
.filter(Boolean)
|
| 312 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 313 |
onError("A custom construct needs a name and at least one item (one per line).");
|
| 314 |
return;
|
| 315 |
}
|
| 316 |
setSaving(true);
|
| 317 |
try {
|
| 318 |
-
const created = await api.createConstruct({
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
onCreated(created);
|
| 320 |
} catch (err) {
|
| 321 |
onError(err.message);
|
|
@@ -324,6 +504,8 @@ function NewConstructForm({ onCreated, onError }) {
|
|
| 324 |
}
|
| 325 |
}
|
| 326 |
|
|
|
|
|
|
|
| 327 |
return (
|
| 328 |
<form onSubmit={save} className="mt">
|
| 329 |
<div className="row">
|
|
@@ -345,10 +527,29 @@ function NewConstructForm({ onCreated, onError }) {
|
|
| 345 |
</div>
|
| 346 |
</div>
|
| 347 |
<label className="field">
|
| 348 |
-
|
| 349 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
</label>
|
| 351 |
-
|
|
|
|
|
|
|
|
|
|
| 352 |
{saving ? "Saving…" : "Save construct"}
|
| 353 |
</button>
|
| 354 |
</form>
|
|
|
|
| 1 |
import { useCallback, useEffect, useRef, useState } from "react";
|
| 2 |
import { api } from "./api.js";
|
| 3 |
+
import ConstructPicker from "./ConstructPicker.jsx";
|
| 4 |
import ResultsView from "./ResultsView.jsx";
|
| 5 |
|
| 6 |
+
export default function Workspace({ project, auth, onAuthRefresh, onProjectChanged, onProjectDeleted }) {
|
| 7 |
const [corpora, setCorpora] = useState([]);
|
| 8 |
const [constructs, setConstructs] = useState([]);
|
| 9 |
const [models, setModels] = useState([]);
|
|
|
|
| 13 |
const [textColumn, setTextColumn] = useState("");
|
| 14 |
const [constructId, setConstructId] = useState("");
|
| 15 |
const [modelName, setModelName] = useState("");
|
| 16 |
+
const [languages, setLanguages] = useState(["en"]);
|
| 17 |
+
const [language, setLanguage] = useState("en");
|
| 18 |
const [uploading, setUploading] = useState(false);
|
| 19 |
const [running, setRunning] = useState(false);
|
| 20 |
const [error, setError] = useState("");
|
| 21 |
const [showNewConstruct, setShowNewConstruct] = useState(false);
|
| 22 |
const [viewJobId, setViewJobId] = useState(null);
|
| 23 |
+
const [confirmDelete, setConfirmDelete] = useState(false);
|
| 24 |
+
const [deleteText, setDeleteText] = useState("");
|
| 25 |
const fileRef = useRef(null);
|
| 26 |
|
| 27 |
+
async function toggleArchive() {
|
| 28 |
+
try {
|
| 29 |
+
await api.patchProject(project.id, { archived: !project.archived });
|
| 30 |
+
onProjectChanged?.();
|
| 31 |
+
} catch (err) {
|
| 32 |
+
setError(err.message);
|
| 33 |
+
}
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
async function handleDelete() {
|
| 37 |
+
try {
|
| 38 |
+
await api.deleteProject(project.id);
|
| 39 |
+
setConfirmDelete(false);
|
| 40 |
+
onProjectDeleted?.();
|
| 41 |
+
} catch (err) {
|
| 42 |
+
setError(err.message);
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
const refreshJobs = useCallback(
|
| 47 |
() => api.listJobs(project.id).then(setJobs).catch(() => {}),
|
| 48 |
[project.id]
|
|
|
|
| 55 |
.models()
|
| 56 |
.then((m) => {
|
| 57 |
setModels(m);
|
| 58 |
+
const def = m.find((x) => x.default) || m[0];
|
| 59 |
+
if (def) setModelName(def.id);
|
| 60 |
})
|
| 61 |
.catch((e) => setError(e.message));
|
| 62 |
+
api.languages().then(setLanguages).catch(() => {});
|
| 63 |
refreshJobs();
|
| 64 |
}, [project.id, refreshJobs]);
|
| 65 |
|
|
|
|
| 103 |
construct_id: constructId,
|
| 104 |
text_column: textColumn,
|
| 105 |
model_name: modelName,
|
| 106 |
+
language,
|
| 107 |
});
|
| 108 |
await refreshJobs();
|
| 109 |
+
onAuthRefresh?.(); // anonymous run counter changed
|
| 110 |
} catch (err) {
|
| 111 |
setError(err.message);
|
| 112 |
} finally {
|
|
|
|
| 136 |
</div>
|
| 137 |
)}
|
| 138 |
|
| 139 |
+
{/* Project header + actions */}
|
| 140 |
+
<div className="project-header">
|
| 141 |
+
<div>
|
| 142 |
+
<span className="project-title">{project.name}</span>
|
| 143 |
+
{project.archived && <span className="pill queued">archived</span>}
|
| 144 |
+
</div>
|
| 145 |
+
<div className="row">
|
| 146 |
+
<button className="ghost" onClick={toggleArchive}>
|
| 147 |
+
{project.archived ? "Unarchive" : "Archive"}
|
| 148 |
+
</button>
|
| 149 |
+
<button className="ghost danger" onClick={() => setConfirmDelete(true)}>
|
| 150 |
+
Delete
|
| 151 |
+
</button>
|
| 152 |
+
</div>
|
| 153 |
+
</div>
|
| 154 |
+
|
| 155 |
+
{confirmDelete && (
|
| 156 |
+
<div className="modal-backdrop" onClick={() => setConfirmDelete(false)}>
|
| 157 |
+
<div className="modal" onClick={(e) => e.stopPropagation()}>
|
| 158 |
+
<h3>Delete "{project.name}"?</h3>
|
| 159 |
+
<p className="hint">
|
| 160 |
+
This permanently deletes {corpora.length} dataset{corpora.length === 1 ? "" : "s"},{" "}
|
| 161 |
+
{jobs.length} run{jobs.length === 1 ? "" : "s"}, and all uploaded and result files.
|
| 162 |
+
This cannot be undone. If you might need it later, use Archive instead.
|
| 163 |
+
</p>
|
| 164 |
+
<label className="field">
|
| 165 |
+
Type the project name to confirm
|
| 166 |
+
<input
|
| 167 |
+
type="text"
|
| 168 |
+
autoFocus
|
| 169 |
+
value={deleteText}
|
| 170 |
+
onChange={(e) => setDeleteText(e.target.value)}
|
| 171 |
+
placeholder={project.name}
|
| 172 |
+
/>
|
| 173 |
+
</label>
|
| 174 |
+
<div className="row">
|
| 175 |
+
<button
|
| 176 |
+
className="primary danger-solid"
|
| 177 |
+
disabled={deleteText !== project.name}
|
| 178 |
+
onClick={handleDelete}
|
| 179 |
+
>
|
| 180 |
+
Delete permanently
|
| 181 |
+
</button>
|
| 182 |
+
<button
|
| 183 |
+
className="ghost"
|
| 184 |
+
onClick={() => {
|
| 185 |
+
setConfirmDelete(false);
|
| 186 |
+
setDeleteText("");
|
| 187 |
+
}}
|
| 188 |
+
>
|
| 189 |
+
Cancel
|
| 190 |
+
</button>
|
| 191 |
+
</div>
|
| 192 |
+
</div>
|
| 193 |
+
</div>
|
| 194 |
+
)}
|
| 195 |
+
|
| 196 |
+
{/* Step 1 - corpus */}
|
| 197 |
<div className="card">
|
| 198 |
<h3>
|
| 199 |
<span className="step-badge">1</span>Corpus
|
| 200 |
</h3>
|
| 201 |
<p className="hint">
|
| 202 |
Upload a CSV or XLSX file, then choose the column containing the text to analyze.
|
| 203 |
+
{auth && !auth.signed_in && auth.limits?.max_rows && (
|
| 204 |
+
<>
|
| 205 |
+
{" "}
|
| 206 |
+
Anonymous limit: {Math.round(auth.limits.max_bytes / 1048576)} MB /{" "}
|
| 207 |
+
{auth.limits.max_rows.toLocaleString()} rows per file; uploads are deleted
|
| 208 |
+
after analysis. Sign in (top right) for larger uploads and to keep your data.
|
| 209 |
+
</>
|
| 210 |
+
)}
|
| 211 |
</p>
|
| 212 |
<div className="row">
|
| 213 |
<div className="grow">
|
|
|
|
| 227 |
<label className="field">
|
| 228 |
Corpus
|
| 229 |
<select value={corpusId} onChange={(e) => setCorpusId(e.target.value)}>
|
| 230 |
+
<option value="">- select -</option>
|
| 231 |
{corpora.map((c) => (
|
| 232 |
<option key={c.id} value={c.id}>
|
| 233 |
{c.filename} ({c.n_rows.toLocaleString()} rows)
|
|
|
|
| 244 |
onChange={(e) => setTextColumn(e.target.value)}
|
| 245 |
disabled={!corpus}
|
| 246 |
>
|
| 247 |
+
<option value="">- select -</option>
|
| 248 |
{corpus?.columns.map((col) => (
|
| 249 |
<option key={col} value={col}>
|
| 250 |
{col}
|
|
|
|
| 260 |
)}
|
| 261 |
</div>
|
| 262 |
|
| 263 |
+
{/* Step 2 - construct */}
|
| 264 |
<div className="card">
|
| 265 |
<h3>
|
| 266 |
<span className="step-badge">2</span>Construct
|
|
|
|
| 269 |
Pick a validated scale from the library, or define custom items. CCR scores each
|
| 270 |
text by its similarity to these items.
|
| 271 |
</p>
|
| 272 |
+
<div className="construct-row">
|
| 273 |
<div className="grow">
|
| 274 |
+
<ConstructPicker
|
| 275 |
+
constructs={constructs}
|
| 276 |
+
value={constructId}
|
| 277 |
+
onChange={setConstructId}
|
| 278 |
+
/>
|
|
|
|
|
|
|
|
|
|
| 279 |
</div>
|
| 280 |
<button className="ghost" onClick={() => setShowNewConstruct((s) => !s)}>
|
| 281 |
{showNewConstruct ? "Close" : "+ Custom construct"}
|
|
|
|
| 286 |
<>
|
| 287 |
<ul className="construct-items">
|
| 288 |
{construct.items.map((item, i) => (
|
| 289 |
+
<li key={i}>
|
| 290 |
+
{item}
|
| 291 |
+
{construct.reverse_scored?.[i] ? " (reverse-scored)" : ""}
|
| 292 |
+
</li>
|
| 293 |
))}
|
| 294 |
</ul>
|
| 295 |
{construct.reference && (
|
| 296 |
<p className="small muted mt">Reference: {construct.reference}</p>
|
| 297 |
)}
|
| 298 |
+
{construct.verification_status !== "verified" && (
|
| 299 |
+
<p className="small muted">
|
| 300 |
+
⚠ Item wording not yet verified verbatim against the original publication
|
| 301 |
+
(status: {construct.verification_status.replace("_", " ")}).
|
| 302 |
+
</p>
|
| 303 |
+
)}
|
| 304 |
</>
|
| 305 |
)}
|
| 306 |
|
|
|
|
| 317 |
)}
|
| 318 |
</div>
|
| 319 |
|
| 320 |
+
{/* Step 3 - language, model + run */}
|
| 321 |
<div className="card">
|
| 322 |
<h3>
|
| 323 |
+
<span className="step-badge">3</span>Language, model & run
|
| 324 |
</h3>
|
| 325 |
<p className="hint">
|
| 326 |
+
Embeddings run locally via sentence-transformers; model and language are recorded
|
| 327 |
+
in the run metadata. If the corpus doesn't match the selected language or the
|
| 328 |
+
model doesn't support it, you'll get a warning - never a silent result.
|
| 329 |
</p>
|
| 330 |
+
<div className="run-settings">
|
| 331 |
+
<label className="field language-control">
|
| 332 |
+
Text language
|
| 333 |
+
<select value={language} onChange={(e) => setLanguage(e.target.value)}>
|
| 334 |
+
{languages.map((l) => (
|
| 335 |
+
<option key={l} value={l}>
|
| 336 |
+
{l}
|
| 337 |
+
</option>
|
| 338 |
+
))}
|
| 339 |
+
</select>
|
| 340 |
+
</label>
|
| 341 |
+
<label className="field model-control">
|
| 342 |
+
Embedding model
|
| 343 |
<select value={modelName} onChange={(e) => setModelName(e.target.value)}>
|
| 344 |
{models.map((m) => (
|
| 345 |
+
<option key={m.id} value={m.id}>
|
| 346 |
{m.label}
|
| 347 |
</option>
|
| 348 |
))}
|
| 349 |
</select>
|
| 350 |
+
</label>
|
| 351 |
+
<button className="primary run-button" disabled={!canRun} onClick={handleRun}>
|
| 352 |
{running ? "Starting…" : "Run CCR analysis"}
|
| 353 |
</button>
|
| 354 |
</div>
|
| 355 |
+
{auth && !auth.signed_in && auth.usage?.max_runs_per_day != null && (
|
| 356 |
+
<p className="small muted">
|
| 357 |
+
{Math.min(auth.usage.runs_used_today, auth.usage.max_runs_per_day)} of{" "}
|
| 358 |
+
{auth.usage.max_runs_per_day} free runs used today
|
| 359 |
+
{auth.usage.runs_used_today >= auth.usage.max_runs_per_day
|
| 360 |
+
? " - sign in (top right) to keep running."
|
| 361 |
+
: "."}
|
| 362 |
+
</p>
|
| 363 |
+
)}
|
| 364 |
+
{auth?.signed_in && auth.usage?.max_saved_runs != null && (
|
| 365 |
+
<p className="small muted">
|
| 366 |
+
{auth.usage.saved_runs} of {auth.usage.max_saved_runs} saved runs used.
|
| 367 |
+
</p>
|
| 368 |
+
)}
|
| 369 |
+
{models.find((m) => m.id === modelName)?.warnings?.map((w, i) => (
|
| 370 |
+
<p key={i} className="small muted">
|
| 371 |
+
⚠ {w}
|
| 372 |
+
</p>
|
| 373 |
+
))}
|
| 374 |
</div>
|
| 375 |
|
| 376 |
{/* Jobs */}
|
| 377 |
{jobs.length > 0 && (
|
| 378 |
<div className="card">
|
| 379 |
<h3>Runs</h3>
|
| 380 |
+
<div className="table-wrap">
|
| 381 |
+
<table className="docs">
|
| 382 |
+
<thead>
|
| 383 |
+
<tr>
|
| 384 |
+
<th>Started</th>
|
| 385 |
+
<th>Corpus</th>
|
| 386 |
+
<th>Construct</th>
|
| 387 |
+
<th>Model</th>
|
| 388 |
+
<th>Lang</th>
|
| 389 |
+
<th style={{ width: "20%" }}>Status</th>
|
| 390 |
+
<th />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
</tr>
|
| 392 |
+
</thead>
|
| 393 |
+
<tbody>
|
| 394 |
+
{jobs.map((j) => (
|
| 395 |
+
<tr key={j.id}>
|
| 396 |
+
<td className="muted">
|
| 397 |
+
{(j.started_at || j.created_at).replace("T", " ").slice(0, 16)}
|
| 398 |
+
</td>
|
| 399 |
+
<td>{j.corpus_filename}</td>
|
| 400 |
+
<td>{j.construct_name}</td>
|
| 401 |
+
<td className="muted small">{j.model_name}</td>
|
| 402 |
+
<td className="muted small">{j.language}</td>
|
| 403 |
+
<td>
|
| 404 |
+
{j.status === "running" ? (
|
| 405 |
+
<div className="progress-track" title={`${Math.round(j.progress * 100)}%`}>
|
| 406 |
+
<div
|
| 407 |
+
className="progress-fill"
|
| 408 |
+
style={{ width: `${Math.max(3, j.progress * 100)}%` }}
|
| 409 |
+
/>
|
| 410 |
+
</div>
|
| 411 |
+
) : (
|
| 412 |
+
<span className={`pill ${j.status}`}>{j.status}</span>
|
| 413 |
+
)}
|
| 414 |
+
{j.status === "failed" && (
|
| 415 |
+
<div className="small muted" title={j.error}>
|
| 416 |
+
{j.error.split("\n").pop()}
|
| 417 |
+
</div>
|
| 418 |
+
)}
|
| 419 |
+
</td>
|
| 420 |
+
<td>
|
| 421 |
+
{j.status === "completed" && (
|
| 422 |
+
<button className="linkish" onClick={() => setViewJobId(j.id)}>
|
| 423 |
+
View results
|
| 424 |
+
</button>
|
| 425 |
+
)}
|
| 426 |
+
</td>
|
| 427 |
+
</tr>
|
| 428 |
+
))}
|
| 429 |
+
</tbody>
|
| 430 |
+
</table>
|
| 431 |
+
</div>
|
| 432 |
</div>
|
| 433 |
)}
|
| 434 |
</>
|
| 435 |
);
|
| 436 |
}
|
| 437 |
|
| 438 |
+
const REVERSE_SUFFIX = /\s*\((r|rev|reversed)\)\s*$/i;
|
| 439 |
+
|
| 440 |
function NewConstructForm({ onCreated, onError }) {
|
| 441 |
const [name, setName] = useState("");
|
| 442 |
const [reference, setReference] = useState("");
|
| 443 |
const [itemsText, setItemsText] = useState("");
|
| 444 |
const [saving, setSaving] = useState(false);
|
| 445 |
+
const [parsing, setParsing] = useState(false);
|
| 446 |
+
const [parseNotes, setParseNotes] = useState([]);
|
| 447 |
+
const itemFileRef = useRef(null);
|
| 448 |
|
| 449 |
+
// Convention shared with the file parser and the lab's own spreadsheets:
|
| 450 |
+
// a trailing (R) marks a reverse-scored item.
|
| 451 |
+
function parseLines() {
|
| 452 |
+
return itemsText
|
| 453 |
.split("\n")
|
| 454 |
.map((s) => s.trim())
|
| 455 |
+
.filter(Boolean)
|
| 456 |
+
.map((line) => ({
|
| 457 |
+
text: line.replace(REVERSE_SUFFIX, "").trim(),
|
| 458 |
+
reverse: REVERSE_SUFFIX.test(line),
|
| 459 |
+
}));
|
| 460 |
+
}
|
| 461 |
+
|
| 462 |
+
async function handleItemFile(e) {
|
| 463 |
+
const file = e.target.files?.[0];
|
| 464 |
+
if (!file) return;
|
| 465 |
+
setParsing(true);
|
| 466 |
+
setParseNotes([]);
|
| 467 |
+
try {
|
| 468 |
+
const parsed = await api.parseConstructFile(file);
|
| 469 |
+
setItemsText(
|
| 470 |
+
parsed.items
|
| 471 |
+
.map((i) => (i.reverse_scored ? `${i.text} (R)` : i.text))
|
| 472 |
+
.join("\n")
|
| 473 |
+
);
|
| 474 |
+
if (!name.trim() && parsed.suggested_name) setName(parsed.suggested_name);
|
| 475 |
+
setParseNotes(parsed.warnings || []);
|
| 476 |
+
} catch (err) {
|
| 477 |
+
onError(err.message);
|
| 478 |
+
} finally {
|
| 479 |
+
setParsing(false);
|
| 480 |
+
if (itemFileRef.current) itemFileRef.current.value = "";
|
| 481 |
+
}
|
| 482 |
+
}
|
| 483 |
+
|
| 484 |
+
async function save(e) {
|
| 485 |
+
e.preventDefault();
|
| 486 |
+
const parsed = parseLines();
|
| 487 |
+
if (!name.trim() || parsed.length === 0) {
|
| 488 |
onError("A custom construct needs a name and at least one item (one per line).");
|
| 489 |
return;
|
| 490 |
}
|
| 491 |
setSaving(true);
|
| 492 |
try {
|
| 493 |
+
const created = await api.createConstruct({
|
| 494 |
+
name: name.trim(),
|
| 495 |
+
reference,
|
| 496 |
+
items: parsed.map((i) => i.text),
|
| 497 |
+
reverse_scored: parsed.map((i) => i.reverse),
|
| 498 |
+
});
|
| 499 |
onCreated(created);
|
| 500 |
} catch (err) {
|
| 501 |
onError(err.message);
|
|
|
|
| 504 |
}
|
| 505 |
}
|
| 506 |
|
| 507 |
+
const nReverse = parseLines().filter((i) => i.reverse).length;
|
| 508 |
+
|
| 509 |
return (
|
| 510 |
<form onSubmit={save} className="mt">
|
| 511 |
<div className="row">
|
|
|
|
| 527 |
</div>
|
| 528 |
</div>
|
| 529 |
<label className="field">
|
| 530 |
+
Upload items from CSV/XLSX (optional) - an "item" column, or one item per row;
|
| 531 |
+
reverse-scored via a "reverse" column or a trailing (R)
|
| 532 |
+
<input
|
| 533 |
+
ref={itemFileRef}
|
| 534 |
+
type="file"
|
| 535 |
+
accept=".csv,.xlsx,.xls"
|
| 536 |
+
onChange={handleItemFile}
|
| 537 |
+
disabled={parsing}
|
| 538 |
+
/>
|
| 539 |
+
</label>
|
| 540 |
+
{parsing && <p className="small muted">Parsing…</p>}
|
| 541 |
+
{parseNotes.map((w, i) => (
|
| 542 |
+
<p key={i} className="small muted">⚠ {w}</p>
|
| 543 |
+
))}
|
| 544 |
+
<label className="field">
|
| 545 |
+
Scale items - one per line, verbatim from the validated instrument; append (R) to
|
| 546 |
+
mark a reverse-scored item
|
| 547 |
+
<textarea rows={6} value={itemsText} onChange={(e) => setItemsText(e.target.value)} />
|
| 548 |
</label>
|
| 549 |
+
{nReverse > 0 && (
|
| 550 |
+
<p className="small muted">{nReverse} item(s) marked reverse-scored.</p>
|
| 551 |
+
)}
|
| 552 |
+
<button className="primary" type="submit" disabled={saving || parsing}>
|
| 553 |
{saving ? "Saving…" : "Save construct"}
|
| 554 |
</button>
|
| 555 |
</form>
|
frontend/src/api.js
CHANGED
|
@@ -25,9 +25,17 @@ const json = (method, body) => ({
|
|
| 25 |
export const api = {
|
| 26 |
health: () => request("/api/health"),
|
| 27 |
models: () => request("/api/models"),
|
|
|
|
| 28 |
|
| 29 |
listProjects: () => request("/api/projects"),
|
| 30 |
createProject: (body) => request("/api/projects", json("POST", body)),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
listCorpora: (projectId) => request(`/api/projects/${projectId}/corpora`),
|
| 33 |
uploadCorpus: (projectId, file) => {
|
|
@@ -38,6 +46,11 @@ export const api = {
|
|
| 38 |
|
| 39 |
listConstructs: () => request("/api/constructs"),
|
| 40 |
createConstruct: (body) => request("/api/constructs", json("POST", body)),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
|
| 42 |
createJob: (body) => request("/api/jobs", json("POST", body)),
|
| 43 |
listJobs: (projectId) => request(`/api/jobs?project_id=${projectId}`),
|
|
@@ -46,4 +59,6 @@ export const api = {
|
|
| 46 |
|
| 47 |
exportUrl: (jobId) => `/api/jobs/${jobId}/export`,
|
| 48 |
metadataUrl: (jobId) => `/api/jobs/${jobId}/metadata`,
|
|
|
|
|
|
|
| 49 |
};
|
|
|
|
| 25 |
export const api = {
|
| 26 |
health: () => request("/api/health"),
|
| 27 |
models: () => request("/api/models"),
|
| 28 |
+
languages: () => request("/api/languages"),
|
| 29 |
|
| 30 |
listProjects: () => request("/api/projects"),
|
| 31 |
createProject: (body) => request("/api/projects", json("POST", body)),
|
| 32 |
+
patchProject: (projectId, body) => request(`/api/projects/${projectId}`, json("PATCH", body)),
|
| 33 |
+
deleteProject: (projectId) => fetch(`/api/projects/${projectId}`, { method: "DELETE" }),
|
| 34 |
+
|
| 35 |
+
authMe: () => request("/api/auth/me"),
|
| 36 |
+
register: (body) => request("/api/auth/register", json("POST", body)),
|
| 37 |
+
login: (body) => request("/api/auth/login", json("POST", body)),
|
| 38 |
+
logout: () => request("/api/auth/logout", { method: "POST" }),
|
| 39 |
|
| 40 |
listCorpora: (projectId) => request(`/api/projects/${projectId}/corpora`),
|
| 41 |
uploadCorpus: (projectId, file) => {
|
|
|
|
| 46 |
|
| 47 |
listConstructs: () => request("/api/constructs"),
|
| 48 |
createConstruct: (body) => request("/api/constructs", json("POST", body)),
|
| 49 |
+
parseConstructFile: (file) => {
|
| 50 |
+
const form = new FormData();
|
| 51 |
+
form.append("file", file);
|
| 52 |
+
return request("/api/constructs/parse-file", { method: "POST", body: form });
|
| 53 |
+
},
|
| 54 |
|
| 55 |
createJob: (body) => request("/api/jobs", json("POST", body)),
|
| 56 |
listJobs: (projectId) => request(`/api/jobs?project_id=${projectId}`),
|
|
|
|
| 59 |
|
| 60 |
exportUrl: (jobId) => `/api/jobs/${jobId}/export`,
|
| 61 |
metadataUrl: (jobId) => `/api/jobs/${jobId}/metadata`,
|
| 62 |
+
scriptUrl: (jobId) => `/api/jobs/${jobId}/script`,
|
| 63 |
+
scriptRequirementsUrl: (jobId) => `/api/jobs/${jobId}/script-requirements`,
|
| 64 |
};
|
frontend/src/styles.css
CHANGED
|
@@ -9,6 +9,7 @@
|
|
| 9 |
--ok: #157f3d;
|
| 10 |
--err: #b42318;
|
| 11 |
--accent-soft: #f6ebef;
|
|
|
|
| 12 |
}
|
| 13 |
|
| 14 |
* { box-sizing: border-box; }
|
|
@@ -31,12 +32,20 @@ body {
|
|
| 31 |
padding: 14px 28px;
|
| 32 |
display: flex;
|
| 33 |
align-items: baseline;
|
|
|
|
| 34 |
gap: 14px;
|
| 35 |
}
|
| 36 |
-
.header h1 {
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
-
.layout { display: flex; flex: 1; min-height: 0; }
|
| 40 |
|
| 41 |
/* ---------- sidebar ---------- */
|
| 42 |
.sidebar {
|
|
@@ -45,27 +54,65 @@ body {
|
|
| 45 |
border-right: 1px solid var(--line);
|
| 46 |
padding: 18px 14px;
|
| 47 |
flex-shrink: 0;
|
|
|
|
|
|
|
|
|
|
| 48 |
}
|
| 49 |
.sidebar h2 {
|
| 50 |
font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.7px;
|
| 51 |
color: var(--muted); margin: 0 0 10px 4px;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
}
|
|
|
|
| 53 |
.project-item {
|
| 54 |
display: block; width: 100%; text-align: left;
|
| 55 |
-
padding: 9px 12px; margin-bottom:
|
| 56 |
border: 1px solid transparent; border-radius: 8px;
|
| 57 |
background: none; cursor: pointer; font: inherit; color: var(--ink);
|
| 58 |
}
|
| 59 |
.project-item:hover { background: var(--bg); }
|
| 60 |
.project-item.active { background: var(--accent-soft); border-color: var(--maroon); font-weight: 600; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
.project-item .date { display: block; font-size: 11.5px; color: var(--muted); font-weight: 400; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
|
| 63 |
/* ---------- main ---------- */
|
| 64 |
.main { flex: 1; padding: 22px 28px; overflow-y: auto; min-width: 0; }
|
| 65 |
|
| 66 |
.card {
|
| 67 |
background: var(--card); border: 1px solid var(--line);
|
| 68 |
-
border-radius:
|
| 69 |
}
|
| 70 |
.card h3 { margin: 0 0 4px; font-size: 15px; }
|
| 71 |
.card .hint { color: var(--muted); font-size: 12.5px; margin: 0 0 12px; }
|
|
@@ -78,15 +125,36 @@ body {
|
|
| 78 |
}
|
| 79 |
|
| 80 |
/* ---------- controls ---------- */
|
|
|
|
|
|
|
| 81 |
button.primary {
|
| 82 |
background: var(--maroon); color: #fff; border: none;
|
| 83 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
}
|
| 85 |
button.primary:hover { background: var(--maroon-dark); }
|
| 86 |
button.primary:disabled { background: #c9ccd1; cursor: not-allowed; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
button.ghost {
|
| 88 |
background: none; border: 1px solid var(--line); color: var(--ink);
|
| 89 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
}
|
| 91 |
button.ghost:hover { border-color: var(--maroon); color: var(--maroon); }
|
| 92 |
button.linkish {
|
|
@@ -94,16 +162,76 @@ button.linkish {
|
|
| 94 |
font: inherit; cursor: pointer; padding: 0; text-decoration: underline;
|
| 95 |
}
|
| 96 |
|
| 97 |
-
input[type="text"], textarea, select {
|
| 98 |
width: 100%; padding: 8px 10px; border: 1px solid var(--line);
|
| 99 |
border-radius: 8px; font: inherit; background: #fff; color: var(--ink);
|
| 100 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
textarea { resize: vertical; }
|
| 102 |
label.field { display: block; margin-bottom: 10px; font-size: 13px; font-weight: 600; }
|
| 103 |
label.field > * { margin-top: 4px; font-weight: 400; }
|
|
|
|
| 104 |
|
| 105 |
.row { display: flex; gap: 14px; flex-wrap: wrap; }
|
| 106 |
-
.row >
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
|
| 108 |
/* ---------- misc ---------- */
|
| 109 |
.pill {
|
|
@@ -128,12 +256,16 @@ label.field > * { margin-top: 4px; font-weight: 400; }
|
|
| 128 |
padding: 10px 14px; border-radius: 8px; margin-bottom: 14px; font-size: 13px;
|
| 129 |
}
|
| 130 |
|
|
|
|
| 131 |
table.docs { width: 100%; border-collapse: collapse; font-size: 13px; }
|
| 132 |
table.docs th {
|
| 133 |
text-align: left; color: var(--muted); font-size: 11.5px; text-transform: uppercase;
|
| 134 |
letter-spacing: 0.5px; padding: 6px 8px; border-bottom: 1px solid var(--line);
|
| 135 |
}
|
| 136 |
-
table.docs td {
|
|
|
|
|
|
|
|
|
|
| 137 |
table.docs td.score { font-variant-numeric: tabular-nums; font-weight: 600; white-space: nowrap; }
|
| 138 |
|
| 139 |
.stat-grid { display: flex; gap: 12px; flex-wrap: wrap; margin-bottom: 4px; }
|
|
@@ -145,7 +277,7 @@ table.docs td.score { font-variant-numeric: tabular-nums; font-weight: 600; whit
|
|
| 145 |
.stat .k { font-size: 11.5px; color: var(--muted); text-transform: uppercase; letter-spacing: 0.5px; }
|
| 146 |
|
| 147 |
.item-bar-row { display: flex; align-items: center; gap: 10px; margin-bottom: 7px; }
|
| 148 |
-
.item-bar-label { flex: 1; font-size: 12.5px; min-width: 0; }
|
| 149 |
.item-bar-track { flex: 1.2; background: var(--bg); border-radius: 999px; height: 10px; }
|
| 150 |
.item-bar-fill { background: var(--maroon); opacity: 0.85; height: 100%; border-radius: 999px; }
|
| 151 |
.item-bar-val { width: 52px; text-align: right; font-variant-numeric: tabular-nums; font-size: 12.5px; font-weight: 600; }
|
|
@@ -163,3 +295,177 @@ table.docs td.score { font-variant-numeric: tabular-nums; font-weight: 600; whit
|
|
| 163 |
.muted { color: var(--muted); }
|
| 164 |
.small { font-size: 12.5px; }
|
| 165 |
.mt { margin-top: 12px; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
--ok: #157f3d;
|
| 10 |
--err: #b42318;
|
| 11 |
--accent-soft: #f6ebef;
|
| 12 |
+
--control-height: 40px;
|
| 13 |
}
|
| 14 |
|
| 15 |
* { box-sizing: border-box; }
|
|
|
|
| 32 |
padding: 14px 28px;
|
| 33 |
display: flex;
|
| 34 |
align-items: baseline;
|
| 35 |
+
flex-wrap: wrap;
|
| 36 |
gap: 14px;
|
| 37 |
}
|
| 38 |
+
.header h1 {
|
| 39 |
+
flex: 0 0 auto;
|
| 40 |
+
font-size: 17px;
|
| 41 |
+
margin: 0;
|
| 42 |
+
font-weight: 650;
|
| 43 |
+
letter-spacing: 0.2px;
|
| 44 |
+
white-space: nowrap;
|
| 45 |
+
}
|
| 46 |
+
.header .sub { flex: 1 1 280px; min-width: 0; font-size: 12.5px; opacity: 0.85; }
|
| 47 |
|
| 48 |
+
.layout { display: flex; flex: 1; min-width: 0; min-height: 0; }
|
| 49 |
|
| 50 |
/* ---------- sidebar ---------- */
|
| 51 |
.sidebar {
|
|
|
|
| 54 |
border-right: 1px solid var(--line);
|
| 55 |
padding: 18px 14px;
|
| 56 |
flex-shrink: 0;
|
| 57 |
+
display: flex;
|
| 58 |
+
flex-direction: column;
|
| 59 |
+
min-height: 0;
|
| 60 |
}
|
| 61 |
.sidebar h2 {
|
| 62 |
font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.7px;
|
| 63 |
color: var(--muted); margin: 0 0 10px 4px;
|
| 64 |
+
display: flex; align-items: center; gap: 8px;
|
| 65 |
+
}
|
| 66 |
+
.sidebar h2 .count {
|
| 67 |
+
background: var(--bg); border: 1px solid var(--line); border-radius: 999px;
|
| 68 |
+
padding: 0 8px; font-size: 10.5px; letter-spacing: 0; color: var(--muted);
|
| 69 |
+
}
|
| 70 |
+
.sidebar-filter {
|
| 71 |
+
width: 100%; padding: 7px 10px; margin-bottom: 10px;
|
| 72 |
+
border: 1px solid var(--line); border-radius: 8px; font: inherit; font-size: 13px;
|
| 73 |
+
background: #fff; color: var(--ink);
|
| 74 |
+
}
|
| 75 |
+
.sidebar-filter:focus { outline: none; border-color: var(--maroon); }
|
| 76 |
+
.project-list {
|
| 77 |
+
flex: 1;
|
| 78 |
+
min-height: 0;
|
| 79 |
+
overflow-y: auto;
|
| 80 |
+
margin: 0 -4px;
|
| 81 |
+
padding: 0 4px 4px;
|
| 82 |
+
}
|
| 83 |
+
.group-label {
|
| 84 |
+
font-size: 10.5px; text-transform: uppercase; letter-spacing: 0.6px;
|
| 85 |
+
color: var(--muted); margin: 10px 4px 5px;
|
| 86 |
}
|
| 87 |
+
div:first-child > .group-label { margin-top: 2px; }
|
| 88 |
.project-item {
|
| 89 |
display: block; width: 100%; text-align: left;
|
| 90 |
+
padding: 9px 12px; margin-bottom: 5px;
|
| 91 |
border: 1px solid transparent; border-radius: 8px;
|
| 92 |
background: none; cursor: pointer; font: inherit; color: var(--ink);
|
| 93 |
}
|
| 94 |
.project-item:hover { background: var(--bg); }
|
| 95 |
.project-item.active { background: var(--accent-soft); border-color: var(--maroon); font-weight: 600; }
|
| 96 |
+
.project-item .project-name {
|
| 97 |
+
display: block;
|
| 98 |
+
overflow: hidden;
|
| 99 |
+
text-overflow: ellipsis;
|
| 100 |
+
white-space: nowrap;
|
| 101 |
+
}
|
| 102 |
.project-item .date { display: block; font-size: 11.5px; color: var(--muted); font-weight: 400; }
|
| 103 |
+
.project-create {
|
| 104 |
+
margin-top: 12px;
|
| 105 |
+
padding-top: 12px;
|
| 106 |
+
border-top: 1px solid var(--line);
|
| 107 |
+
}
|
| 108 |
+
.project-create > button { width: 100%; }
|
| 109 |
|
| 110 |
/* ---------- main ---------- */
|
| 111 |
.main { flex: 1; padding: 22px 28px; overflow-y: auto; min-width: 0; }
|
| 112 |
|
| 113 |
.card {
|
| 114 |
background: var(--card); border: 1px solid var(--line);
|
| 115 |
+
border-radius: 8px; padding: 18px 20px; margin-bottom: 16px;
|
| 116 |
}
|
| 117 |
.card h3 { margin: 0 0 4px; font-size: 15px; }
|
| 118 |
.card .hint { color: var(--muted); font-size: 12.5px; margin: 0 0 12px; }
|
|
|
|
| 125 |
}
|
| 126 |
|
| 127 |
/* ---------- controls ---------- */
|
| 128 |
+
button { white-space: nowrap; }
|
| 129 |
+
|
| 130 |
button.primary {
|
| 131 |
background: var(--maroon); color: #fff; border: none;
|
| 132 |
+
min-height: var(--control-height);
|
| 133 |
+
display: inline-flex;
|
| 134 |
+
align-items: center;
|
| 135 |
+
justify-content: center;
|
| 136 |
+
padding: 0 18px; border-radius: 8px; font: inherit; font-weight: 600; cursor: pointer;
|
| 137 |
+
line-height: 1.2;
|
| 138 |
}
|
| 139 |
button.primary:hover { background: var(--maroon-dark); }
|
| 140 |
button.primary:disabled { background: #c9ccd1; cursor: not-allowed; }
|
| 141 |
+
a.google-btn {
|
| 142 |
+
background: var(--maroon); color: #fff; border: none;
|
| 143 |
+
min-height: var(--control-height);
|
| 144 |
+
display: flex; align-items: center; justify-content: center;
|
| 145 |
+
padding: 0 18px; border-radius: 8px; font: inherit; font-weight: 600;
|
| 146 |
+
cursor: pointer; line-height: 1.2; text-decoration: none; width: 100%;
|
| 147 |
+
box-sizing: border-box;
|
| 148 |
+
}
|
| 149 |
+
a.google-btn:hover { background: var(--maroon-dark); }
|
| 150 |
button.ghost {
|
| 151 |
background: none; border: 1px solid var(--line); color: var(--ink);
|
| 152 |
+
min-height: var(--control-height);
|
| 153 |
+
display: inline-flex;
|
| 154 |
+
align-items: center;
|
| 155 |
+
justify-content: center;
|
| 156 |
+
padding: 0 14px; border-radius: 8px; font: inherit; cursor: pointer;
|
| 157 |
+
line-height: 1.2;
|
| 158 |
}
|
| 159 |
button.ghost:hover { border-color: var(--maroon); color: var(--maroon); }
|
| 160 |
button.linkish {
|
|
|
|
| 162 |
font: inherit; cursor: pointer; padding: 0; text-decoration: underline;
|
| 163 |
}
|
| 164 |
|
| 165 |
+
input[type="text"], input[type="email"], input[type="password"], textarea, select {
|
| 166 |
width: 100%; padding: 8px 10px; border: 1px solid var(--line);
|
| 167 |
border-radius: 8px; font: inherit; background: #fff; color: var(--ink);
|
| 168 |
}
|
| 169 |
+
input[type="text"], input[type="email"], input[type="password"], select { height: var(--control-height); }
|
| 170 |
+
|
| 171 |
+
button:focus-visible,
|
| 172 |
+
input:focus-visible,
|
| 173 |
+
textarea:focus-visible,
|
| 174 |
+
select:focus-visible {
|
| 175 |
+
outline: 2px solid rgba(122, 31, 61, 0.38);
|
| 176 |
+
outline-offset: 2px;
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
/* File input: block-level with breathing room under its label, and a styled
|
| 180 |
+
picker button so it matches the rest of the controls. */
|
| 181 |
+
input[type="file"] {
|
| 182 |
+
display: block; max-width: 100%; margin-top: 6px; font-size: 13px; color: var(--muted);
|
| 183 |
+
}
|
| 184 |
+
input[type="file"]::file-selector-button {
|
| 185 |
+
background: #fff; border: 1px solid var(--line); color: var(--ink);
|
| 186 |
+
padding: 7px 14px; border-radius: 8px; font: inherit; font-size: 13px;
|
| 187 |
+
cursor: pointer; margin-right: 10px;
|
| 188 |
+
}
|
| 189 |
+
input[type="file"]::file-selector-button:hover {
|
| 190 |
+
border-color: var(--maroon); color: var(--maroon);
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
/* Buttons inside a row of labeled fields align to the controls' baseline
|
| 194 |
+
instead of stretching to the row's full height. */
|
| 195 |
+
.row > button { align-self: flex-end; margin-bottom: 1px; }
|
| 196 |
textarea { resize: vertical; }
|
| 197 |
label.field { display: block; margin-bottom: 10px; font-size: 13px; font-weight: 600; }
|
| 198 |
label.field > * { margin-top: 4px; font-weight: 400; }
|
| 199 |
+
.field-hint { margin-top: 0; font-weight: 400; color: var(--muted); font-size: 12px; }
|
| 200 |
|
| 201 |
.row { display: flex; gap: 14px; flex-wrap: wrap; }
|
| 202 |
+
.row > * { min-width: 0; }
|
| 203 |
+
.row > .grow { flex: 1; min-width: min(220px, 100%); }
|
| 204 |
+
.language-control { min-width: 170px; }
|
| 205 |
+
.model-control { min-width: 260px; }
|
| 206 |
+
.run-settings {
|
| 207 |
+
display: grid;
|
| 208 |
+
grid-template-columns: minmax(160px, 230px) minmax(320px, 720px) max-content;
|
| 209 |
+
justify-content: start;
|
| 210 |
+
gap: 14px;
|
| 211 |
+
align-items: end;
|
| 212 |
+
}
|
| 213 |
+
.run-settings .field { margin-bottom: 0; }
|
| 214 |
+
.run-button {
|
| 215 |
+
min-width: 180px;
|
| 216 |
+
height: var(--control-height);
|
| 217 |
+
}
|
| 218 |
+
.construct-row {
|
| 219 |
+
display: grid;
|
| 220 |
+
grid-template-columns: minmax(320px, 1120px) max-content;
|
| 221 |
+
justify-content: start;
|
| 222 |
+
align-items: end;
|
| 223 |
+
gap: 14px;
|
| 224 |
+
}
|
| 225 |
+
.results-toolbar {
|
| 226 |
+
display: flex;
|
| 227 |
+
align-items: center;
|
| 228 |
+
justify-content: space-between;
|
| 229 |
+
gap: 14px;
|
| 230 |
+
margin-bottom: 14px;
|
| 231 |
+
flex-wrap: wrap;
|
| 232 |
+
}
|
| 233 |
+
.result-actions { justify-content: flex-end; }
|
| 234 |
+
.result-actions a { display: inline-flex; text-decoration: none; }
|
| 235 |
|
| 236 |
/* ---------- misc ---------- */
|
| 237 |
.pill {
|
|
|
|
| 256 |
padding: 10px 14px; border-radius: 8px; margin-bottom: 14px; font-size: 13px;
|
| 257 |
}
|
| 258 |
|
| 259 |
+
.table-wrap { width: 100%; overflow-x: auto; }
|
| 260 |
table.docs { width: 100%; border-collapse: collapse; font-size: 13px; }
|
| 261 |
table.docs th {
|
| 262 |
text-align: left; color: var(--muted); font-size: 11.5px; text-transform: uppercase;
|
| 263 |
letter-spacing: 0.5px; padding: 6px 8px; border-bottom: 1px solid var(--line);
|
| 264 |
}
|
| 265 |
+
table.docs td {
|
| 266 |
+
padding: 7px 8px; border-bottom: 1px solid var(--bg); vertical-align: top;
|
| 267 |
+
overflow-wrap: anywhere;
|
| 268 |
+
}
|
| 269 |
table.docs td.score { font-variant-numeric: tabular-nums; font-weight: 600; white-space: nowrap; }
|
| 270 |
|
| 271 |
.stat-grid { display: flex; gap: 12px; flex-wrap: wrap; margin-bottom: 4px; }
|
|
|
|
| 277 |
.stat .k { font-size: 11.5px; color: var(--muted); text-transform: uppercase; letter-spacing: 0.5px; }
|
| 278 |
|
| 279 |
.item-bar-row { display: flex; align-items: center; gap: 10px; margin-bottom: 7px; }
|
| 280 |
+
.item-bar-label { flex: 1; font-size: 12.5px; min-width: 0; overflow-wrap: anywhere; }
|
| 281 |
.item-bar-track { flex: 1.2; background: var(--bg); border-radius: 999px; height: 10px; }
|
| 282 |
.item-bar-fill { background: var(--maroon); opacity: 0.85; height: 100%; border-radius: 999px; }
|
| 283 |
.item-bar-val { width: 52px; text-align: right; font-variant-numeric: tabular-nums; font-size: 12.5px; font-weight: 600; }
|
|
|
|
| 295 |
.muted { color: var(--muted); }
|
| 296 |
.small { font-size: 12.5px; }
|
| 297 |
.mt { margin-top: 12px; }
|
| 298 |
+
|
| 299 |
+
/* ---------- header auth ---------- */
|
| 300 |
+
.header-auth {
|
| 301 |
+
margin-left: auto; display: flex; align-items: center; gap: 10px; color: #fff;
|
| 302 |
+
}
|
| 303 |
+
.header-btn {
|
| 304 |
+
background: rgba(255, 255, 255, 0.12); color: #fff;
|
| 305 |
+
border: 1px solid rgba(255, 255, 255, 0.45);
|
| 306 |
+
padding: 5px 14px; border-radius: 7px; font: inherit; font-size: 13px; cursor: pointer;
|
| 307 |
+
}
|
| 308 |
+
.header-btn:hover { background: rgba(255, 255, 255, 0.22); }
|
| 309 |
+
|
| 310 |
+
/* ---------- project header + actions ---------- */
|
| 311 |
+
.project-header {
|
| 312 |
+
display: flex; align-items: center; justify-content: space-between;
|
| 313 |
+
gap: 12px; margin-bottom: 14px; flex-wrap: wrap;
|
| 314 |
+
}
|
| 315 |
+
.project-title { font-size: 17px; font-weight: 650; margin-right: 10px; }
|
| 316 |
+
button.danger { color: var(--err); border-color: #f0c4be; }
|
| 317 |
+
button.danger:hover { color: var(--err); border-color: var(--err); }
|
| 318 |
+
button.danger-solid { background: var(--err); }
|
| 319 |
+
button.danger-solid:hover { background: #93261b; }
|
| 320 |
+
button.danger-solid:disabled { background: #c9ccd1; }
|
| 321 |
+
|
| 322 |
+
/* ---------- construct picker (searchable, grouped) ---------- */
|
| 323 |
+
.picker { position: relative; }
|
| 324 |
+
.picker-display {
|
| 325 |
+
width: 100%; display: flex; align-items: center; gap: 10px;
|
| 326 |
+
padding: 8px 12px; border: 1px solid var(--line); border-radius: 8px;
|
| 327 |
+
background: #fff; font: inherit; color: var(--ink); cursor: pointer; text-align: left;
|
| 328 |
+
}
|
| 329 |
+
.picker-display:hover { border-color: var(--maroon); }
|
| 330 |
+
.picker-display .picker-caret { margin-left: auto; color: var(--muted); font-size: 11px; }
|
| 331 |
+
.picker-search {
|
| 332 |
+
width: 100%; padding: 8px 12px; border: 1px solid var(--maroon);
|
| 333 |
+
border-radius: 8px; font: inherit; background: #fff;
|
| 334 |
+
}
|
| 335 |
+
.picker-search:focus { outline: none; box-shadow: 0 0 0 3px rgba(122, 31, 61, 0.12); }
|
| 336 |
+
.picker-panel {
|
| 337 |
+
position: absolute; top: calc(100% + 6px); left: 0; z-index: 50;
|
| 338 |
+
width: 100%; max-width: 640px;
|
| 339 |
+
background: var(--card); border: 1px solid var(--line); border-radius: 10px;
|
| 340 |
+
box-shadow: 0 14px 40px rgba(0, 0, 0, 0.18);
|
| 341 |
+
max-height: 340px; overflow-y: auto; padding: 4px 0 6px;
|
| 342 |
+
}
|
| 343 |
+
.picker-group {
|
| 344 |
+
font-size: 10.5px; text-transform: uppercase; letter-spacing: 0.6px;
|
| 345 |
+
color: var(--muted); padding: 8px 12px 3px; position: sticky; top: 0;
|
| 346 |
+
background: var(--card);
|
| 347 |
+
}
|
| 348 |
+
.picker-option {
|
| 349 |
+
display: flex; align-items: baseline; justify-content: space-between; gap: 12px;
|
| 350 |
+
padding: 6px 12px; cursor: pointer; font-size: 13.5px;
|
| 351 |
+
}
|
| 352 |
+
.picker-option:hover, .picker-option.active { background: var(--accent-soft); }
|
| 353 |
+
.picker-option.selected .picker-name { font-weight: 650; color: var(--maroon); }
|
| 354 |
+
.picker-name { min-width: 0; }
|
| 355 |
+
.picker-meta { flex-shrink: 0; font-size: 11.5px; color: var(--muted); white-space: nowrap; }
|
| 356 |
+
.picker-empty { padding: 12px; margin: 0; }
|
| 357 |
+
|
| 358 |
+
/* ---------- modal ---------- */
|
| 359 |
+
.modal-backdrop {
|
| 360 |
+
position: fixed; inset: 0; background: rgba(20, 22, 26, 0.45);
|
| 361 |
+
display: flex; align-items: center; justify-content: center; z-index: 40; padding: 16px;
|
| 362 |
+
}
|
| 363 |
+
.modal {
|
| 364 |
+
background: var(--card); border-radius: 12px; padding: 26px 28px 24px;
|
| 365 |
+
width: 100%; max-width: 460px; box-shadow: 0 12px 40px rgba(0, 0, 0, 0.18);
|
| 366 |
+
}
|
| 367 |
+
.modal h3 { margin: 0 0 8px; font-size: 18px; }
|
| 368 |
+
|
| 369 |
+
/* Auth modal: roomier fields and clearer separation between sections. */
|
| 370 |
+
.modal .hint { color: var(--muted); font-size: 13px; line-height: 1.5; margin: 0 0 4px; }
|
| 371 |
+
.modal form.mt { margin-top: 18px; }
|
| 372 |
+
.modal label.field { margin-bottom: 16px; }
|
| 373 |
+
.modal label.field:last-of-type { margin-bottom: 20px; }
|
| 374 |
+
.modal .row { gap: 10px; }
|
| 375 |
+
.modal .row > .primary { flex: 1; }
|
| 376 |
+
.modal p.small.muted.mt {
|
| 377 |
+
margin-top: 18px; padding-top: 16px; border-top: 1px solid var(--line);
|
| 378 |
+
font-size: 12.5px; line-height: 1.6;
|
| 379 |
+
}
|
| 380 |
+
|
| 381 |
+
@media (max-width: 820px) {
|
| 382 |
+
body { font-size: 14px; }
|
| 383 |
+
|
| 384 |
+
.header {
|
| 385 |
+
padding: 12px 16px;
|
| 386 |
+
align-items: flex-start;
|
| 387 |
+
gap: 2px 12px;
|
| 388 |
+
}
|
| 389 |
+
.header h1 { font-size: 16px; }
|
| 390 |
+
.header .sub {
|
| 391 |
+
flex: 1 1 210px;
|
| 392 |
+
font-size: 12px;
|
| 393 |
+
line-height: 1.35;
|
| 394 |
+
}
|
| 395 |
+
|
| 396 |
+
.layout { display: block; }
|
| 397 |
+
.sidebar {
|
| 398 |
+
width: 100%;
|
| 399 |
+
border-right: 0;
|
| 400 |
+
border-bottom: 1px solid var(--line);
|
| 401 |
+
padding: 14px;
|
| 402 |
+
}
|
| 403 |
+
.project-list {
|
| 404 |
+
max-height: 220px;
|
| 405 |
+
margin-right: 0;
|
| 406 |
+
flex: none;
|
| 407 |
+
}
|
| 408 |
+
.project-create { border-top: 0; }
|
| 409 |
+
.main {
|
| 410 |
+
width: 100%;
|
| 411 |
+
padding: 16px 14px 28px;
|
| 412 |
+
overflow: visible;
|
| 413 |
+
}
|
| 414 |
+
.card {
|
| 415 |
+
padding: 16px;
|
| 416 |
+
margin-bottom: 14px;
|
| 417 |
+
}
|
| 418 |
+
|
| 419 |
+
.row,
|
| 420 |
+
.run-settings,
|
| 421 |
+
.construct-row,
|
| 422 |
+
.results-toolbar,
|
| 423 |
+
.result-actions {
|
| 424 |
+
gap: 10px;
|
| 425 |
+
}
|
| 426 |
+
.row,
|
| 427 |
+
.results-toolbar,
|
| 428 |
+
.result-actions {
|
| 429 |
+
flex-direction: column;
|
| 430 |
+
align-items: stretch;
|
| 431 |
+
}
|
| 432 |
+
.run-settings,
|
| 433 |
+
.construct-row {
|
| 434 |
+
grid-template-columns: 1fr;
|
| 435 |
+
}
|
| 436 |
+
.row > .grow,
|
| 437 |
+
.construct-row > .grow,
|
| 438 |
+
.language-control,
|
| 439 |
+
.model-control {
|
| 440 |
+
width: 100%;
|
| 441 |
+
min-width: 0;
|
| 442 |
+
}
|
| 443 |
+
.main .row > button,
|
| 444 |
+
.main .row > a,
|
| 445 |
+
.main .row > a > button,
|
| 446 |
+
.results-toolbar > button {
|
| 447 |
+
align-self: stretch;
|
| 448 |
+
margin-bottom: 0;
|
| 449 |
+
width: 100%;
|
| 450 |
+
}
|
| 451 |
+
.run-button {
|
| 452 |
+
width: 100%;
|
| 453 |
+
min-width: 0;
|
| 454 |
+
}
|
| 455 |
+
|
| 456 |
+
.stat-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
| 457 |
+
.stat { min-width: 0; }
|
| 458 |
+
.item-bar-row {
|
| 459 |
+
display: grid;
|
| 460 |
+
grid-template-columns: minmax(0, 1fr) 56px;
|
| 461 |
+
gap: 6px 10px;
|
| 462 |
+
}
|
| 463 |
+
.item-bar-track { grid-column: 1 / -1; width: 100%; }
|
| 464 |
+
.item-bar-val { width: auto; }
|
| 465 |
+
table.docs { min-width: 520px; }
|
| 466 |
+
}
|
| 467 |
+
|
| 468 |
+
@media (max-width: 460px) {
|
| 469 |
+
.header .sub { flex-basis: 100%; }
|
| 470 |
+
.stat-grid { grid-template-columns: 1fr; }
|
| 471 |
+
}
|
packages/ccr_engine/README.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ccr_engine (Phase 1 target - spec 0005)
|
| 2 |
+
|
| 3 |
+
This package will hold the pure CCR analysis engine: preprocessing, embeddings (via the model
|
| 4 |
+
registry), similarity, scoring/aggregation, warnings, stats, reproducibility artifacts -
|
| 5 |
+
callable from the web app, worker, tests, CLI, and generated scripts.
|
| 6 |
+
|
| 7 |
+
Today the engine logic lives in `backend/app/ccr.py` + `backend/app/jobs.py`. Do NOT start the
|
| 8 |
+
extraction ad hoc: it begins by freezing current behavior with golden evals (strangler step 1,
|
| 9 |
+
design §7), then moves code behind the `run_ccr_analysis(...)` interface (design §10).
|
| 10 |
+
|
| 11 |
+
Boundary rule (enforced in review): nothing in this package may import from `backend/app`.
|
packages/construct_library/constructs/collectivism_horizontal.yaml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
construct_id: collectivism_horizontal
|
| 2 |
+
version: 1
|
| 3 |
+
name: Collectivism (Horizontal)
|
| 4 |
+
language: en
|
| 5 |
+
category: cultural-orientation
|
| 6 |
+
description: Interdependence, cooperation, and in-group well-being (Triandis & Gelfand horizontal collectivism).
|
| 7 |
+
citation: "Triandis, H. C., & Gelfand, M. J. (1998). Converging measurement of horizontal and vertical individualism and collectivism. JPSP, 74(1)."
|
| 8 |
+
verification_status: needs_verification
|
| 9 |
+
items:
|
| 10 |
+
- item_id: hc_1
|
| 11 |
+
text: "If a coworker gets a prize, I would feel proud."
|
| 12 |
+
reverse_scored: false
|
| 13 |
+
- item_id: hc_2
|
| 14 |
+
text: "The well-being of my coworkers is important to me."
|
| 15 |
+
reverse_scored: false
|
| 16 |
+
- item_id: hc_3
|
| 17 |
+
text: "To me, pleasure is spending time with others."
|
| 18 |
+
reverse_scored: false
|
| 19 |
+
- item_id: hc_4
|
| 20 |
+
text: "I feel good when I cooperate with others."
|
| 21 |
+
reverse_scored: false
|
packages/construct_library/constructs/individualism_horizontal.yaml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
construct_id: individualism_horizontal
|
| 2 |
+
version: 1
|
| 3 |
+
name: Individualism (Horizontal)
|
| 4 |
+
language: en
|
| 5 |
+
category: cultural-orientation
|
| 6 |
+
description: Self-reliance and independence from in-groups (Triandis & Gelfand horizontal individualism).
|
| 7 |
+
citation: "Triandis, H. C., & Gelfand, M. J. (1998). Converging measurement of horizontal and vertical individualism and collectivism. JPSP, 74(1)."
|
| 8 |
+
verification_status: needs_verification
|
| 9 |
+
items:
|
| 10 |
+
- item_id: hi_1
|
| 11 |
+
text: "I'd rather depend on myself than others."
|
| 12 |
+
reverse_scored: false
|
| 13 |
+
- item_id: hi_2
|
| 14 |
+
text: "I rely on myself most of the time; I rarely rely on others."
|
| 15 |
+
reverse_scored: false
|
| 16 |
+
- item_id: hi_3
|
| 17 |
+
text: "I often do my own thing."
|
| 18 |
+
reverse_scored: false
|
| 19 |
+
- item_id: hi_4
|
| 20 |
+
text: "My personal identity, independent of others, is very important to me."
|
| 21 |
+
reverse_scored: false
|
packages/construct_library/constructs/mfq_care.yaml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
construct_id: mfq_care
|
| 2 |
+
version: 1
|
| 3 |
+
name: Moral Foundations - Care
|
| 4 |
+
language: en
|
| 5 |
+
category: moral-foundations
|
| 6 |
+
description: Concern with suffering, compassion, and protection of the vulnerable (MFQ Care/Harm).
|
| 7 |
+
citation: "Graham, J., Nosek, B. A., Haidt, J., Iyer, R., Koleva, S., & Ditto, P. H. (2011). Mapping the moral domain. JPSP, 101(2)."
|
| 8 |
+
verification_status: needs_verification
|
| 9 |
+
items:
|
| 10 |
+
- item_id: care_1
|
| 11 |
+
text: "Compassion for those who are suffering is the most crucial virtue."
|
| 12 |
+
reverse_scored: false
|
| 13 |
+
- item_id: care_2
|
| 14 |
+
text: "One of the worst things a person could do is hurt a defenseless animal."
|
| 15 |
+
reverse_scored: false
|
| 16 |
+
- item_id: care_3
|
| 17 |
+
text: "Whether or not someone suffered emotionally."
|
| 18 |
+
reverse_scored: false
|
| 19 |
+
- item_id: care_4
|
| 20 |
+
text: "Whether or not someone cared for someone weak or vulnerable."
|
| 21 |
+
reverse_scored: false
|
packages/construct_library/constructs/mfq_fairness.yaml
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
construct_id: mfq_fairness
|
| 2 |
+
version: 1
|
| 3 |
+
name: Moral Foundations - Fairness
|
| 4 |
+
language: en
|
| 5 |
+
category: moral-foundations
|
| 6 |
+
description: Concern with justice, rights, and equal treatment (MFQ Fairness/Cheating).
|
| 7 |
+
citation: "Graham, J., Nosek, B. A., Haidt, J., Iyer, R., Koleva, S., & Ditto, P. H. (2011). Mapping the moral domain. JPSP, 101(2)."
|
| 8 |
+
verification_status: needs_verification
|
| 9 |
+
items:
|
| 10 |
+
- item_id: fair_1
|
| 11 |
+
text: "Justice is the most important requirement for a society."
|
| 12 |
+
reverse_scored: false
|
| 13 |
+
- item_id: fair_2
|
| 14 |
+
text: "When the government makes laws, the number one principle should be ensuring that everyone is treated fairly."
|
| 15 |
+
reverse_scored: false
|
| 16 |
+
- item_id: fair_3
|
| 17 |
+
text: "Whether or not some people were treated differently than others."
|
| 18 |
+
reverse_scored: false
|
| 19 |
+
- item_id: fair_4
|
| 20 |
+
text: "Whether or not someone acted unfairly."
|
| 21 |
+
reverse_scored: false
|
packages/construct_library/constructs/satisfaction_with_life.yaml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Construct: versioned, append-only. Edits create a NEW version (see registries rule).
|
| 2 |
+
construct_id: satisfaction_with_life
|
| 3 |
+
version: 1
|
| 4 |
+
name: Satisfaction with Life
|
| 5 |
+
language: en
|
| 6 |
+
category: well-being
|
| 7 |
+
description: Global cognitive judgment of one's life satisfaction (SWLS).
|
| 8 |
+
citation: "Diener, E., Emmons, R. A., Larsen, R. J., & Griffin, S. (1985). The Satisfaction with Life Scale. Journal of Personality Assessment, 49(1)."
|
| 9 |
+
verification_status: needs_verification # verify item wording verbatim against the publication
|
| 10 |
+
items:
|
| 11 |
+
- item_id: swl_1
|
| 12 |
+
text: "In most ways my life is close to my ideal."
|
| 13 |
+
reverse_scored: false
|
| 14 |
+
- item_id: swl_2
|
| 15 |
+
text: "The conditions of my life are excellent."
|
| 16 |
+
reverse_scored: false
|
| 17 |
+
- item_id: swl_3
|
| 18 |
+
text: "I am satisfied with my life."
|
| 19 |
+
reverse_scored: false
|
| 20 |
+
- item_id: swl_4
|
| 21 |
+
text: "So far I have gotten the important things I want in life."
|
| 22 |
+
reverse_scored: false
|
| 23 |
+
- item_id: swl_5
|
| 24 |
+
text: "If I could live my life over, I would change almost nothing."
|
| 25 |
+
reverse_scored: false
|