diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..20da00395834479e182ec8895385234436533b62 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,39 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_012.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_014.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_015.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_016.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_017.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_020.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_021.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_024.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_025.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_028.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_029.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_030.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_031.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_034.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_035.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_038.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_039.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_042.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_043.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_046.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_047.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_050.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_051.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_052.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_053.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_056.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_057.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_060.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_061.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_064.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_065.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_068.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_069.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_072.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_073.jpg filter=lfs diff=lfs merge=lfs -text +app/deploy/book1_data/book1/forster1/B1_P_074.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..b6b4d95bec856ab90a20e3919674b20f6fa0ccd0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,59 @@ +# Inkference — Hugging Face Docker Space (CPU) with LIVE UPLOAD. +# +# Ships the full pipeline (Kraken + TrOCR + confidence + Groq correction + Gemini RAG) +# and bakes the 36-page Book 1 corpus (app/deploy/book1_data), seeded on boot. +# +# HF builds the repo-ROOT Dockerfile, so before pushing to the Space: +# cp app/deploy/Dockerfile Dockerfile +# Build context = repo root. +FROM python:3.11-slim + +# System libs for the Pillow / torch / kraken image stack. +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgl1 libglib2.0-0 && \ + rm -rf /var/lib/apt/lists/* + +# HF Spaces expect a non-root uid 1000 with a writable home. +RUN useradd -m -u 1000 user +WORKDIR /app + +# Python deps first (cached layer). Includes kraken (torch 2.10) + TrOCR + RAG + API. +COPY app/deploy/requirements-space.txt . +RUN pip install --no-cache-dir -r requirements-space.txt + +# App code + frontend + baked Book 1 data (36 pages: images + confidence + corrected). +COPY app/pyproject.toml ./ +COPY app/src/ ./src/ +COPY app/frontend/ ./frontend/ +COPY app/deploy/book1_data/ ./book1_data/ +RUN pip install --no-cache-dir -e . --no-deps && \ + mkdir -p /data && chown -R user:user /app /data + +ENV INKFERENCE_DATA_ROOT=/data \ + HF_HOME=/home/user/.cache/huggingface \ + PORT=7860 \ + # --- recognition (live upload) --- + TROCR_MODEL_ID=microsoft/trocr-base-handwritten \ + HTR_MAX_LONG_EDGE=1600 \ + HTR_NUM_BEAMS=1 \ + # --- post-correction via Groq (GROQ_API_KEY is a Space secret) --- + CORRECTION_ENABLED=true \ + CORRECTION_BACKEND=api \ + CORRECTION_API_BASE=https://api.groq.com/openai/v1 \ + CORRECTION_API_MODEL=qwen/qwen3-32b \ + # --- Ask-the-Archive via Gemini (GEMINI_API_KEY is a Space secret) --- + LLM_PROVIDER=gemini \ + LLM_MODEL=gemini-2.5-flash +# Set as Space secrets (Settings -> Variables and secrets): GEMINI_API_KEY, GROQ_API_KEY. +# To use your fine-tuned recognizer, push it to the Hub and set TROCR_MODEL_ID=/inkference-trocr. + +USER user + +# Pre-download models so the first request is fast (runs as 'user' -> HF_HOME matches runtime). +RUN python -c "from sentence_transformers import SentenceTransformer as S; S('sentence-transformers/all-MiniLM-L6-v2')" +RUN python -c "from transformers import TrOCRProcessor, VisionEncoderDecoderModel as M; TrOCRProcessor.from_pretrained('microsoft/trocr-base-handwritten'); M.from_pretrained('microsoft/trocr-base-handwritten')" + +EXPOSE 7860 + +# Seed Book 1 into the (ephemeral) data root on boot, then serve API + frontend. +CMD ["sh", "-c", "python -m inkference.store.seed_book1 --alex /app/book1_data || true; uvicorn inkference.api.main:app --host 0.0.0.0 --port ${PORT:-7860}"] diff --git a/README.md b/README.md index 0794197982ad11c91aab556f1fde4ee24dc5c660..0e13eb9529fb28fa5ce95dd6502a3ae381d6742d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,90 @@ --- title: Inkference -emoji: ⚡ -colorFrom: pink +emoji: 🪶 +colorFrom: red colorTo: yellow sdk: docker +app_port: 7860 pinned: false --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# Inkference — HTR + RAG for historical handwriting + +Reader · Ask the Archive · Upload. One FastAPI container serves the Inkference UI + +API: the 36-page **Book 1** corpus is baked in and seeded on boot, and **live upload** +runs the full pipeline (Kraken → TrOCR → confidence → Groq correction). "Ask the +Archive" answers over the corrected text with Gemini, citing source pages. + +## What the Space runs +- **Frontend + API**: same URL (the app serves `frontend/` at `/`). +- **Preseeded Book 1**: baked from `app/deploy/book1_data/` (images + confidence + + corrected/green), re-seeded into the ephemeral `/data` on every boot. +- **Live upload**: Kraken segmentation + TrOCR recognition + per-word confidence + + Qwen post-correction (Groq). Works on the free 16 GB Space, but CPU-slow + (~minutes/page) and uploaded pages are lost on restart (ephemeral `/data`). +- **Ask the Archive**: MiniLM + FAISS retrieval → Gemini answer + page citations, + with an extractive fallback if the LLM is unavailable. + +## Deploy steps + +1. **Log in to Hugging Face**: `hf auth login` (token from + https://huggingface.co/settings/tokens), or use the web UI. + +2. *(Recommended)* **Push the fine-tuned recognizer to the Hub** so uploads get good OCR + (otherwise the base model is used): + ```bash + hf upload /inkference-trocr models/trocr_best_from_bentham + ``` + Then set the Space variable `TROCR_MODEL_ID=/inkference-trocr`. + +3. **Create a Space** → SDK **Docker** (free CPU, 16 GB). + +4. **Populate the Space repo** with ONLY what the image needs (do NOT push `data/`, + `models/`, or `notebooks/` — they're huge). From a clean checkout: + ```bash + cp app/deploy/Dockerfile Dockerfile # HF builds the ROOT Dockerfile + # keep: Dockerfile, app/ (src, frontend, pyproject.toml, deploy/), app/deploy/book1_data/ + git add Dockerfile app/ && git commit -m "Inkference Space" && git push main + ``` + +5. **Secrets** (Space → Settings → Variables and secrets): + - `GEMINI_API_KEY` — Ask-the-Archive answers + - `GROQ_API_KEY` — post-correction + - *(optional)* `TROCR_MODEL_ID` — your Hub recognizer + +6. HF builds the image (~4–5 GB; a few minutes) and boots: it seeds Book 1, then serves. + +Without the keys the app still runs — correction and answers degrade to their fallbacks +(raw OCR / extractive retrieval), still $0. + +## Config (env vars / Space variables) + +| Var | Default | Purpose | +|---|---|---| +| `TROCR_MODEL_ID` | `microsoft/trocr-base-handwritten` | recognizer (set to your Hub model) | +| `HTR_MAX_LONG_EDGE` | `1600` | downscale cap (speed vs accuracy) | +| `CORRECTION_ENABLED` / `CORRECTION_BACKEND` | `true` / `api` | Qwen correction via Groq | +| `CORRECTION_API_MODEL` | `qwen/qwen3-32b` | Groq model | +| `GROQ_API_KEY` | – (secret) | correction key | +| `LLM_PROVIDER` / `LLM_MODEL` | `gemini` / `gemini-2.5-flash` | Ask-the-Archive | +| `GEMINI_API_KEY` | – (secret) | RAG answer key | +| `INKFERENCE_DATA_ROOT` | `/data` | ephemeral corpus store | + +## Caveats (free tier) +- **Ephemeral storage**: `/data` resets on restart → Book 1 re-seeds automatically, but + uploaded pages are lost. For persistence, attach paid persistent storage or move the + store to a managed DB. +- **CPU speed**: live upload is minutes/page (design assumed a GPU). For production, run + HTR on a serverless GPU (Modal/Replicate) via a `remote` executor. +- **Sleep**: free Spaces sleep on inactivity (cold start ~30–60 s). + +## Local run + +```bash +pip install -r requirements.txt && pip install -e ./app +python -m inkference.store.seed_book1 --alex ~/Downloads/AlexFiles # or store.seed for the demo +uvicorn inkference.api.main:app --port 8000 +``` + +See [../projectNotes/running_and_seeds.md](../projectNotes/running_and_seeds.md) for seeds/data-roots +and [../projectNotes/inkference_platform_plan.md](../projectNotes/inkference_platform_plan.md) for the plan. diff --git a/app/deploy/book1_data/book1/forster1/B1_P_012.jpg b/app/deploy/book1_data/book1/forster1/B1_P_012.jpg new file mode 100644 index 0000000000000000000000000000000000000000..23906e5f17bd20d2285e858c579a089be4765907 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_012.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f88df87aebd14f8deb611a651968ff13a18e5bd5a427410f111f168c27a27a2f +size 569492 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_014.jpg b/app/deploy/book1_data/book1/forster1/B1_P_014.jpg new file mode 100644 index 0000000000000000000000000000000000000000..12fce0dada609c573f198b3f8a17ca2fd48f36d1 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_014.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d0153ddeb87e62308e6b3efb43b8ac794ddacd52bd2ebbe2c9458c715c029ae +size 670034 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_015.jpg b/app/deploy/book1_data/book1/forster1/B1_P_015.jpg new file mode 100644 index 0000000000000000000000000000000000000000..941c8e43f0001fb90a72d3f31cf19f85410df9fc --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_015.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08cd19fb12fa725c871ba952b5e796e37fba1d331b97dd0f0fcde4584025f667 +size 646311 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_016.jpg b/app/deploy/book1_data/book1/forster1/B1_P_016.jpg new file mode 100644 index 0000000000000000000000000000000000000000..422afc3cfdc91f5011ea30c093c013c3340c0fa9 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_016.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f630bf53365a43e41d69a01f9140128c8291965ddd30fa6f740f0d003dd272e +size 612170 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_017.jpg b/app/deploy/book1_data/book1/forster1/B1_P_017.jpg new file mode 100644 index 0000000000000000000000000000000000000000..41e7dd2bd1a1ef6277da235e75cda8e4367cd526 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_017.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d45278640fd1f9e0e11dae28d5f0fcd3e13c6c86e2c7b08db97ef78ef660c759 +size 622617 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_020.jpg b/app/deploy/book1_data/book1/forster1/B1_P_020.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1dd09961e004d02c3ec3eac5eba1f13b2c9251cb --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_020.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6e7ba48cf6319d0423535cf57ad8e7c25e8b8de3d96ae41a71ab7aa9386ee46 +size 625325 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_021.jpg b/app/deploy/book1_data/book1/forster1/B1_P_021.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0c0474e951560a2f4fb9ca63e5b39111ad1329a2 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_021.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:832b37a70972b507df0cc409e6151a52514edd36223dca220e0ae1c8834920d8 +size 642566 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_024.jpg b/app/deploy/book1_data/book1/forster1/B1_P_024.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c0cc89d7f8a17dc53545b1deb747bbca5ec630b0 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_024.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:279972530d0c0bb19b3e251e6ed769f85abddde7c4f9d9eeb38a1901deed84f1 +size 656447 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_025.jpg b/app/deploy/book1_data/book1/forster1/B1_P_025.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d65e6fea46781739142467db87c95d151ef020dd --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_025.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cda87f39fd4a54d426cab0e82e21a31f855bdae29b22f4639ca46b8d57e8156 +size 686792 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_028.jpg b/app/deploy/book1_data/book1/forster1/B1_P_028.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d93858bb4a4b4833c954dcab72b8fa21dc1e0ec4 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_028.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fbe9a82685a6c814ec3da916df3d8d23d7b4dc3a7d32f03e1657e00cb98cb8 +size 703754 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_029.jpg b/app/deploy/book1_data/book1/forster1/B1_P_029.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3fcd233015d7bc315bb06164453993aaff19e199 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_029.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07e90f70e79b6f71d61b629b713c31ce5c7c82d42d91361a492851c1c4c23abe +size 792564 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_030.jpg b/app/deploy/book1_data/book1/forster1/B1_P_030.jpg new file mode 100644 index 0000000000000000000000000000000000000000..196709e17ef035fc5771710046cb23d850c31c85 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_030.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cacd31dd23b303f59749537821689327c3a401f6307800e7d4434896b7009e5e +size 770937 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_031.jpg b/app/deploy/book1_data/book1/forster1/B1_P_031.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8c19d9681adf137e2bb60a38539cff0a1b6b2f61 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_031.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b5aa739470385039eb791094fd0e57199e81b8f34bc640b2173a06608033446 +size 770589 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_034.jpg b/app/deploy/book1_data/book1/forster1/B1_P_034.jpg new file mode 100644 index 0000000000000000000000000000000000000000..86d5f7629e03adc735f4576a85f21f9b020a5515 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_034.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35683c2e0e633251b5a2f25ddcb00349a25e9bf27c8fc2b53d12790dfc75e8f5 +size 736656 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_035.jpg b/app/deploy/book1_data/book1/forster1/B1_P_035.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fa64e386521d5f20adefb4bb9f2eaef28bf0bd03 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_035.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf441da9a2ccdf4773506e61ac49f05aa83a512b8edb8788108255ef6c20a584 +size 803027 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_038.jpg b/app/deploy/book1_data/book1/forster1/B1_P_038.jpg new file mode 100644 index 0000000000000000000000000000000000000000..145982eccf3bb6c75e6b4bf7cb6cdab93f99a473 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_038.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11fde58b1aa2a123293d743e494a5ab5e27abde162ab969e63500a36ee0c7a2c +size 684652 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_039.jpg b/app/deploy/book1_data/book1/forster1/B1_P_039.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c9feb28b80a32d67f5efcf62ce2aa56215bdcae8 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_039.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ed9d50103735db286ed8b22231f5f514a0ed166cd016d6d1e5a5b78684bb36 +size 761606 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_042.jpg b/app/deploy/book1_data/book1/forster1/B1_P_042.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cc3f40388009e86903bdcdc9b6d24fa22b387c2d --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_042.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a676bc036a9e7cae76d5cc564f620ae981ae22e42b9f8b69bab81951e858b12 +size 728401 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_043.jpg b/app/deploy/book1_data/book1/forster1/B1_P_043.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2c85f8853e0ac7e6ca699f0229309c5b182db1a9 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_043.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9089cac04e910de8c886e0ad2cb1d75a2368284b1568368894b943e69c0fec4 +size 757214 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_046.jpg b/app/deploy/book1_data/book1/forster1/B1_P_046.jpg new file mode 100644 index 0000000000000000000000000000000000000000..030abb39aa5ec015a6d44bf163739fb58113835d --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_046.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb1c8a4f661f669ac560f72aa72aa2664b5cac1293c5d7a5d682545c31fa6f72 +size 721037 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_047.jpg b/app/deploy/book1_data/book1/forster1/B1_P_047.jpg new file mode 100644 index 0000000000000000000000000000000000000000..013bdd8bb4805ae0a1695d7f8cfb64d136b3d5af --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_047.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80c5da692de754ab752f9dcdbc84be4827a1941fa58e56908e36164a4499cb45 +size 722830 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_050.jpg b/app/deploy/book1_data/book1/forster1/B1_P_050.jpg new file mode 100644 index 0000000000000000000000000000000000000000..53992854b8623ccf488b6b6501c9a3e105b20f28 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_050.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f34d0f1b1a6b02d6c7be23858ab2c1a302bbe15a355734ad9280bed0dc5cb05 +size 763164 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_051.jpg b/app/deploy/book1_data/book1/forster1/B1_P_051.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dd3eafb22f9b23adae6f03d4425aff8479ec2e7f --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_051.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d68dbbc21b790a28018ac870d254d099da50c810a6391a3be046f660cca58a0 +size 751624 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_052.jpg b/app/deploy/book1_data/book1/forster1/B1_P_052.jpg new file mode 100644 index 0000000000000000000000000000000000000000..633c2f79a8d5855dbddf0aa114d2ba6018c5b058 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_052.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c5cd004215d3205d2ec18d6d6eb3a50ea844a5a04a7decb57d24bf039e17b62 +size 781992 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_053.jpg b/app/deploy/book1_data/book1/forster1/B1_P_053.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9efa9fbc4b9b532e16aa4b733ffd47554a09de9c --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_053.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6db50c41c1e541194aa2d85807dfa76f47e8b1cc54c98f7f4a2762f7cbb42b +size 709626 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_056.jpg b/app/deploy/book1_data/book1/forster1/B1_P_056.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f08c913ef2a608faa02132eb926af9ecfe984ae3 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_056.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eadd4cb45a3bf68b4e90b70fd49537070fddfe9a60996747b664fe4467d5e48c +size 760448 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_057.jpg b/app/deploy/book1_data/book1/forster1/B1_P_057.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8c0c2f439010ad849fb9b18928a6b4b9d81fb5a8 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_057.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23634cdfcadbbeab93db38c28985ac523631722860b05ea2cad1a47d469094de +size 766518 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_060.jpg b/app/deploy/book1_data/book1/forster1/B1_P_060.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e3c2e26f2336310906c0278ffdbd04b2d05cf8e2 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_060.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38a5540d04c078225dcc9269c88625d20b2457903b9ce1f022378cadb8a56d66 +size 761765 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_061.jpg b/app/deploy/book1_data/book1/forster1/B1_P_061.jpg new file mode 100644 index 0000000000000000000000000000000000000000..06d44b85ae28c508cd19c84a3a61ad451614845a --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_061.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9a62d7759ebb26417b6ad0b42473f714cf5d7c08ca1f1f73a5c497b516ddaae +size 700467 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_064.jpg b/app/deploy/book1_data/book1/forster1/B1_P_064.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b0203508ff209ff732624cc6b9847ceb94d2012a --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_064.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a57b9c6cdfd6f86d91ac204e219c03de87992e607f76cbba45661165abbe2b4 +size 759461 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_065.jpg b/app/deploy/book1_data/book1/forster1/B1_P_065.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7879ec7833483a54ed2bfc622e6b01dec2c91541 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_065.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e384258dbaee45261891cd739de5abcefc4387e77c66708eff50da3f7bbdba84 +size 694503 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_068.jpg b/app/deploy/book1_data/book1/forster1/B1_P_068.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3444ced841453b74859ea360a6016a59fc1674c5 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_068.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09819e1bcf91a53a4cb421ad523d8e616cca50796fb2fea0a3c70bd5ac732e11 +size 709860 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_069.jpg b/app/deploy/book1_data/book1/forster1/B1_P_069.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cada8290e17ab9e4673e0c34e5ae3374d2e437da --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_069.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b1b242810af5d8a86d650ad48b60275a8df197cbc5177e584cbd90cbe595150 +size 707964 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_072.jpg b/app/deploy/book1_data/book1/forster1/B1_P_072.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e1aae0516d43e3f0b0a766b2c2722ebae3d2418b --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_072.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d38776ffcc00bd6e0b09a66434cff39378196af09881029243c7c700e37670 +size 765090 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_073.jpg b/app/deploy/book1_data/book1/forster1/B1_P_073.jpg new file mode 100644 index 0000000000000000000000000000000000000000..69f88ca02e5976544be0e61acc85598a50674be9 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_073.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a55e84e51464ec9362ae0d164e8d976796b9d1716419bc0faaffc3d2caeef23f +size 745750 diff --git a/app/deploy/book1_data/book1/forster1/B1_P_074.jpg b/app/deploy/book1_data/book1/forster1/B1_P_074.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6faf46c297cb50891005a21ec137e1d27ca59232 --- /dev/null +++ b/app/deploy/book1_data/book1/forster1/B1_P_074.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0a0a5839f9506c297386d170c4c0eeb8e350402d6562d195e371074a661bd16 +size 758676 diff --git a/app/deploy/book1_data/moredata/confidence/B1_P012.txt b/app/deploy/book1_data/moredata/confidence/B1_P012.txt new file mode 100644 index 0000000000000000000000000000000000000000..d4c49109f429e75bc9fee85923bc121b00896fe5 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P012.txt @@ -0,0 +1,17 @@ +=== Buch 1, Seite 012 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 7 von 43 + +[L01] ms.(0.99) germ.(0.83) <>(0.55) 222(0.96) +[L02] Journal(0.93) of(1.00) a(1.00) journey(0.99) +[L03] from(1.00) London(0.95) to(1.00) Plymouth(0.99) +[L04] <<&.>>(0.53) <>(0.53) Voyage(1.00) +[L05] on(1.00) board(1.00) his(1.00) Majesty's(1.00) Ship(1.00) +[L06] the(1.00) <>(0.33) +[L07] Captain(1.00) Cook(0.99) Commander(0.97) +[L08] from(1.00) <>(0.49) <>(0.45) +[L09] to(0.98) the(1.00) Cape(1.00) of(1.00) good(0.98) Hope.(0.99) +[L10] from(1.00) July(1.00) y.(0.75) <<13°November>>(0.60) +[L11] Ex(0.98) +[L12] ((0.88) Biblinth.(0.72) Regie(0.99) +[L13] Berclines(0.78) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P014.txt b/app/deploy/book1_data/moredata/confidence/B1_P014.txt new file mode 100644 index 0000000000000000000000000000000000000000..d1c3e4091ffd8fbceb5fda0516076c4d52f47488 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P014.txt @@ -0,0 +1,34 @@ +=== Buch 1, Seite 014 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 28 von 193 + +[L01] <>(0.51) +[L02] Journal(0.99) of(1.00) all(1.00) the(1.00) <>(0.46) previous(0.99) +[L03] to(1.00) my.(0.92) Appointment.(0.92) &(0.86) after(1.00) <>(0.48) +[L04] departure(0.86) from(1.00) Plymouth(0.99) +[L05] <<12.>>(0.65) <>(0.61) y(0.75) +[L06] May(0.93) +[L07] <>(0.58) <>(0.59) who(1.00) had(1.00) not(1.00) seen(1.00) me(1.00) +[L08] even(0.93) <>(0.67) I(0.98) left(1.00) <>(0.61) in(1.00) the(1.00) year(1.00) 1769,(0.73) came(1.00) +[L09] rate(0.79) in(1.00) the(1.00) <>(0.24) &(0.96) in(1.00) a(1.00) very(1.00) mysterious(1.00) +[L10] manner(0.99) told(0.87) me,(1.00) that(1.00) <>(0.53) Banks(0.99) did(1.00) not(1.00) +[L11] go,(0.99) the(1.00) <>(0.58) to(0.97) the(1.00) <>(0.49) Seas,(0.98) &(1.00) ask(0.98) +[L12] whether(1.00) I(1.00) would(0.94) go;(0.99) I(1.00) told(1.00) if(1.00) a(1.00) proper(1.00) <>(0.69) +[L13] <>(0.64) made(1.00) for(1.00) me(0.99) <>(0.53) my(1.00) Family,(1.00) I(1.00) would(1.00) not(1.00) +[L14] georye(0.98) +[L15] hesitate(0.95) one(1.00) moment,(0.99) but(1.00) my(1.00) Son,(0.89) must(0.99) +[L16] in(1.00) that(1.00) case(0.98) go(0.93) with(1.00) me,(0.99) being(1.00) both(1.00) as(1.00) +[L17] a(0.99) Nationalist(0.77) &(1.00) a(1.00) tolerable(1.00) Draughtoman(0.78) +[L18] well(0.98) qualified(1.00) for(1.00) to(1.00) assist,(0.99) no(0.90) Mr.(0.74) <>(0.12) +[L19] seimes(0.89) to(0.94) like(0.99) it(1.00) desired(0.94) me(1.00) to(1.00) keep(1.00) upon(1.00) +[L20] this(1.00) application(0.90) the(1.00) <>(0.21) secrecy,(0.98) &(0.73) <>(0.38) +[L21] that(0.95) he(1.00) had(1.00) it(1.00) in(1.00) his(1.00) power(1.00) to(1.00) recom-(0.90) +[L22] many(0.82) one(0.86) in(1.00) such(1.00) a(1.00) manner(0.99) that(1.00) +[L23] I(0.77) would(1.00) be(0.96) certainly(1.00) employed(1.00) in(1.00) the(1.00) +[L24] <>(0.69) &(1.00) <<542>>(0.42) he(1.00) was(1.00) to(1.00) break(0.99) <>(0.24) +[L25] with(0.96) Mr(0.84) Stephens(0.96) he(0.99) would(0.99) speak(1.00) to(1.00) +[L26] <>(0.12) about(1.00) <>(0.54) next(1.00) day(1.00) <>(0.37) <>(0.46) +[L27] ner(0.91) he(0.94) came(1.00) to(1.00) me(0.99) again,(1.00) &(1.00) told(0.99) me(1.00) +[L28] <<0>>(0.07) <<0>>(0.42) +[L29] that(1.00) Mr(0.99) Stephens(0.88) had(0.99) taken(1.00) down(0.97) +[L30] my(1.00) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P015.txt b/app/deploy/book1_data/moredata/confidence/B1_P015.txt new file mode 100644 index 0000000000000000000000000000000000000000..d2b4f0fb4ae83ec4b0c0a5ba555a32b022bee4ac --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P015.txt @@ -0,0 +1,30 @@ +=== Buch 1, Seite 015 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 20 von 166 + +[L01] <>(0.33) +[L02] +[L03] my(1.00) name,(1.00) qualification(0.99) &(0.99) place(0.96) of(1.00) <>(0.59) <>(0.42) &(0.97) +[L04] added(0.86) he(0.90) thought(1.00) this(1.00) were(1.00) a(1.00) great(1.00) step(0.99) &(1.00) +[L05] that(1.00) I(1.00) should(1.00) continue(1.00) to(1.00) keep(1.00) the(1.00) affair(0.98) +[L06] very(1.00) secret.(0.99) +[L07] May(1.00) <<4.>>(0.64) <<24.>>(0.60) <>(0.63) saw(1.00) Mr.(0.83) Baines(0.83) <>(0.53) &(1.00) told(0.99) him(1.00) +[L08] that(0.97) Mr(0.97) Banks(0.94) or(0.92) not(1.00) go,(1.00) &(0.84) as(0.99) he(1.00) was(1.00) as-(1.00) +[L09] <>(0.47) whether(1.00) it(0.93) was(1.00) <>(0.20) <>(0.63) it;(0.80) I(0.89) told,(0.98) I(0.97) +[L10] had(1.00) heart(0.72) that(0.90) he(1.00) had(1.00) already(0.99) <>(0.14) +[L11] his(1.00) Equipage(0.95) &(1.00) cloth,(0.92) books(0.99) &(0.88) Instru-(0.71) +[L12] ments(1.00) to(1.00) be(1.00) taken(0.98) out(1.00) of(1.00) the(1.00) Resolution(0.96) +[L13] &(0.99) that(1.00) she(1.00) was(1.00) repairing(0.95) at(1.00) Sheerness,(0.92) +[L14] having(1.00) heard(0.95) this(1.00) from(1.00) an(1.00) acquaintance,(0.98) +[L15] who(1.00) was(0.99) well(1.00) <>(0.46) with(1.00) Mr.(0.98) She-(0.90) +[L16] phens.(0.77) <>(0.41) Barrington.(0.73) hereupon(1.00) <>(0.61) <>(0.59) +[L17] In(0.83) case(0.96) provision(1.00) was(1.00) made(1.00) for(1.00) me(0.95) +[L18] &(0.98) +[L19] family,(1.00) whether(1.00) I(1.00) would(0.99) <>(0.44) I(1.00) told,(0.98) +[L20] I(0.74) would,(1.00) provided(0.96) my(1.00) Son(0.91) Geomo(0.70) was(1.00) +[L21] likewise(1.00) appointed.(0.90) He(1.00) approved(0.98) of(1.00) his(0.86) +[L22] condition(1.00) &(1.00) desired(1.00) <>(0.53) to(1.00) carry(0.99) a(1.00) letter(0.99) +[L23] to(0.99) <>(0.69) Butler(0.80) on(0.96) the(1.00) <>(0.33) to(0.99) the(1.00) +[L24] +[L25] admiralty.(0.94) I(1.00) brought(1.00) it(0.95) to(1.00) his(1.00) house(1.00) +[L26] but(1.00) found(1.00) him(1.00) not(1.00) at(1.00) <>(0.58) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P016.txt b/app/deploy/book1_data/moredata/confidence/B1_P016.txt new file mode 100644 index 0000000000000000000000000000000000000000..41f8749f99370fb3a5c8bdcadaeffa049660cdd6 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P016.txt @@ -0,0 +1,30 @@ +=== Buch 1, Seite 016 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 22 von 173 + +[L01] May(0.88) <<4.>>(0.35) 27(0.87) I(0.95) heard(1.00) that(1.00) Mr.(0.93) Divington's(0.77) letter(0.94) had(0.82) been(1.00) +[L02] read(0.98) by(0.95) Mr(0.95) <>(0.57) at(1.00) the(1.00) board(1.00) &(1.00) that(1.00) +[L03] Lord(1.00) <>(0.61) had(1.00) added,(0.99) I(1.00) have(1.00) hear(0.75) +[L04] <<4.30°>>(0.57) +[L05] June(1.00) +[L06] of(1.00) that(1.00) Man(1.00) a(1.00) very(1.00) good(1.00) character(1.00) &(0.90) +[L07] I(0.99) believe(1.00) he(1.00) could(1.00) do(1.00) in(1.00) that(1.00) branch,(0.78) +[L08] he(1.00) is(1.00) intended(0.98) for,(1.00) as(1.00) much(1.00) as(1.00) any(1.00) +[L09] man(1.00) in(0.95) the(1.00) <>(0.46) out(0.88) as(1.00) no(1.00) other(0.83) +[L10] resolution(0.97) was(1.00) taken(1.00) <>(0.69) Barrington(0.86) <>(0.54) +[L11] <>(0.67) should(0.99) give(1.00) up(1.00) all(1.00) thoughts(0.97) of(1.00) going.(1.00) +[L12] <>(0.31) Bavington(0.75) took(1.00) the(1.00) resolution(1.00) to(1.00) +[L13] write(0.98) one(1.00) some(0.93) more(1.00) to(1.00) Lord(1.00) Sandwich(0.99) +[L14] to(1.00) Hinchinbrooke(0.85) &(1.00) to(1.00) join(1.00) to(1.00) his(1.00) letter(0.86) +[L15] the(1.00) Resolution(1.00) of(1.00) the(1.00) House(1.00) of(1.00) Commons(0.83) +[L16] <>(0.54) <<400>>(0.63) <>(0.22) for(1.00) <>(0.64) on(1.00) disco(0.91) +[L17] veries(0.78) to(0.99) the(1.00) Southwards(0.97) not(1.00) <>(0.64) +[L18] ning(0.98) any(1.00) particular(0.76) person(0.98) to(0.94) whom(1.00) +[L19] it(0.99) might(1.00) be(1.00) given;(0.81) though(1.00) Dr(0.98) Lind's(0.81) n(0.85) +[L20] had(1.00) been(1.00) mentioned(0.98) in(1.00) the(1.00) house.(0.98) +[L21] and(0.98) the(1.00) letter(0.97) was(1.00) written(1.00) accordingly(0.98) +[L22] a(0.95) letter(0.99) of(1.00) a(0.82) Sandwich(0.98) came,(1.00) <>(0.12) +[L23] <>(0.27) that(1.00) he(0.98) wished(1.00) to(1.00) <>(0.36) Mr.(0.97) <>(0.58) +[L24] Tuesday(0.92) <>(0.19) <<32.>>(0.64) &(0.81) especially(0.71) did(1.00) he(0.95) <>(0.61) +[L25] whether(1.00) I(1.00) could(1.00) go(1.00) at(1.00) a(1.00) moment's(0.74) +[L26] <>(0.60) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P017.txt b/app/deploy/book1_data/moredata/confidence/B1_P017.txt new file mode 100644 index 0000000000000000000000000000000000000000..b1942558483fa1fd5cdbd18173fef565c1631fce --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P017.txt @@ -0,0 +1,28 @@ +=== Buch 1, Seite 017 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 31 von 165 + +[L01] warning:(0.75) for(0.99) which(1.00) purpose,(0.83) <>(0.58) not(0.78) +[L02] me(1.00) the(1.00) Question,(0.97) whether(1.00) he(1.00) could(0.97) <>(0.16) +[L03] this(0.99) to(1.00) B(0.85) Sandwich.(0.86) I(0.73) said(1.00) I(1.00) would(1.00) en-(0.99) +[L04] deavour(0.99) to(1.00) be(1.00) ready(1.00) in(1.00) a(1.00) few(1.00) days.(1.00) <>(0.65) +[L05] Juney(0.93) <>(0.10) <>(0.60) Barington(0.71) spoke(1.00) to(1.00) <>(0.53) Sandwich,(0.84) who(1.00) +[L06] June(1.00) +[L07] seemed(0.99) <>(0.55) with(1.00) the(1.00) plan(1.00) &(1.00) my(1.00) cho-(0.93) +[L08] <>(0.56) &(1.00) as(1.00) he(1.00) heard(1.00) of(1.00) my(1.00) <>(0.35) +[L09] <>(0.52) was(1.00) still(1.00) more(1.00) released.(0.73) +[L10] <>(0.31) <>(0.52) told(1.00) me,(1.00) that(1.00) <>(0.57) +[L11] it(1.00) was(1.00) not(1.00) certain,(1.00) whether(1.00) Mr.(0.78) Banks(0.77) +[L12] would(0.89) go(1.00) or(1.00) not;(0.98) but(1.00) that(1.00) it(1.00) would(1.00) be(1.00) +[L13] decided,(1.00) next(0.99) <>(0.41) <>(0.67) <>(0.64) 11th(0.78) +[L14] or(0.86) then(0.71) Mr(0.71) Banks(0.86) would(0.90) have(1.00) a(1.00) +[L15] <>(0.48) Audience(0.99) of(0.91) his(1.00) Majesty(0.81) +[L16] I(0.78) <>(0.64) my(0.73) <>(0.57) at(0.94) the(1.00) Club(0.73) <>(0.22) Banks(0.91) past(0.90) <>(0.43) that(0.81) in(0.76) their(0.73) was(0.91) made(1.00) +[L17] in(1.00) <>(0.24) <>(0.46) <>(0.43) +[L18] I(0.84) <>(0.62) <>(0.66) <>(0.50) Mr(0.98) <>(0.53) +[L19] would(0.99) endeavour(0.73) to(1.00) obtain(1.00) <>(0.63) new(0.81) Ship,(0.99) +[L20] he(0.99) went(1.00) to(1.00) His(1.00) Majesty,(0.99) &(1.00) said(0.98) the(1.00) plan(1.00) +[L21] of(1.00) sending(0.82) me,(0.97) on(1.00) the(1.00) Expedition,(0.99) with(1.00) +[L22] the(1.00) Character(0.99) given(0.99) <>(0.48) by(1.00) Mr.(0.73) Barrington(0.74) +[L23] before(1.00) the(1.00) ring,(0.98) who(1.00) was(1.00) <>(0.70) +[L24] pleased(0.99) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P020.txt b/app/deploy/book1_data/moredata/confidence/B1_P020.txt new file mode 100644 index 0000000000000000000000000000000000000000..8a5575279839720b2b166e8e55a89fc285756128 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P020.txt @@ -0,0 +1,28 @@ +=== Buch 1, Seite 020 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 37 von 163 + +[L01] pleased(0.83) to(1.00) approve(0.96) of(1.00) my(1.00) going(1.00) on(1.00) the(1.00) +[L02] <<0>>(0.06) <<0>>(0.47) +[L03] Canadian(0.88) &(1.00) by(1.00) this(1.00) 2(0.85) Sandwich(0.99) <>(0.36) +[L04] any(0.91) further(0.99) application(0.98) from(1.00) Mr(0.99) Banks(0.80) +[L05] <>(0.63) +[L06] June(1.00) <<16th>>(0.47) The(1.00) <>(0.61) Members(0.83) of(1.00) the(1.00) Council(1.00) of(1.00) +[L07] <>(0.57) Society(0.99) Signed(0.72) a(1.00) <>(0.43) in(1.00) +[L08] my(1.00) favour,(0.98) recommending(0.80) me(1.00) as(1.00) a(1.00) +[L09] <>(0.40) <>(0.39) for(1.00) going(1.00) on(1.00) the(1.00) Expedition(0.87) +[L10] <>(0.28) <>(0.47) 7th(0.73) <>(0.21) Hudson(0.86) spoke(1.00) to(1.00) me(1.00) about(1.00) the(1.00) affair(1.00) +[L11] June(0.91) +[L12] &(0.96) told(0.99) me(1.00) that(1.00) he(1.00) had(0.99) recommended(0.99) +[L13] me(0.99) to(1.00) <>(0.69) <>(0.37) <>(0.65) &(1.00) to(0.95) Sir(1.00) Gilbert(0.99) <>(0.60) +[L14] &(1.00) said,(1.00) that(1.00) as(0.99) there(0.95) had(0.99) been(1.00) an(1.00) <>(0.61) +[L15] tion(0.99) to(1.00) make(1.00) a(1.00) motion(1.00) in(1.00) the(1.00) house(1.00) +[L16] of(1.00) Commons(1.00) Friday(0.93) last(1.00) in(1.00) my(1.00) favour(0.81) +[L17] Mr.(0.88) <>(0.44) &(0.98) Mr(1.00) <>(0.70) <>(0.62) been(1.00) <>(0.17) +[L18] ling(0.84) to(1.00) oppose(0.96) it;(0.98) &(1.00) that(1.00) it(1.00) would(1.00) <>(0.52) +[L19] great(0.98) difficulties(1.00) in(0.97) regard(0.99) to(1.00) the(1.00) <>(0.25) +[L20] <>(0.20) <>(0.61) Burke(0.72) <<&>>(0.55) <>(0.37) wanted(1.00) to(1.00) +[L21] have(1.00) an(1.00) <>(0.55) with(1.00) Mr(1.00) <>(0.26) Palliser(0.94) +[L22] <<&>>(0.26) Mr(0.96) Stephens(1.00) at(1.00) the(1.00) Speakers,(0.99) but(1.00) <>(0.16) +[L23] <>(0.52) <>(0.59) already(0.99) saw(0.84) the(1.00) <>(0.67) +[L24] <>(0.25) <>(0.29) them,(0.99) not(1.00) to(1.00) go(0.96) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P021.txt b/app/deploy/book1_data/moredata/confidence/B1_P021.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a1c2e6a58da86bab24363e2b4587f7c11d15a95 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P021.txt @@ -0,0 +1,26 @@ +=== Buch 1, Seite 021 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 18 von 124 + +[L01] June(1.00) <<4.0°>>(0.54) +[L02] <>(0.20) +[L03] <>(0.52) by(0.94) which(1.00) means(1.00) they(1.00) <>(0.51) of(0.98) +[L04] <>(0.59) <<&>>(0.68) Sandwich(0.97) +[L05] The(1.00) House(1.00) of(1.00) Commons(1.00) <>(0.62) <>(0.52) up(0.96) &(1.00) +[L06] on(0.99) act(0.70) of(1.00) the(1.00) <>(0.61) of(1.00) the(1.00) +[L07] Members,(1.00) my(1.00) name(1.00) could(1.00) not(1.00) be(1.00) +[L08] mentioned(0.98) in(1.00) the(1.00) house.(1.00) +[L09] <>(0.36) spoke(0.99) to(1.00) Mr(0.80) Barrington,(0.90) who(1.00) +[L10] argued(0.87) the(1.00) case,(0.88) &(1.00) <>(0.51) that(1.00) as(0.98) the(1.00) +[L11] resolution(0.94) of(1.00) the(1.00) house(1.00) had(1.00) not(0.98) d(0.72) +[L12] lind's(0.78) name(0.99) mentioned,(0.99) the(1.00) many(0.98) +[L13] could(0.99) be(1.00) applied(0.89) to(1.00) any(1.00) other(1.00) par-(0.95) +[L14] son(0.98) <>(0.68) on(1.00) the(1.00) same(1.00) Expedition(1.00) +[L15] <>(0.68) having(0.99) mentioned(0.98) the(1.00) case(1.00) +[L16] to(1.00) His(0.98) Majesty,(1.00) the(1.00) money(1.00) was(1.00) or-(1.00) +[L17] dered(0.96) to(1.00) be(0.91) <>(0.18) to(0.97) me(0.99) from(1.00) +[L18] His(0.99) Majesty's(0.86) civil(1.00) List(0.95) &(0.99) that(1.00) +[L19] at(1.00) the(1.00) <>(0.60) of(1.00) the(1.00) House(1.00) +[L20] of(1.00) Commons(1.00) at(0.92) <>(0.70) be(0.99) decided(1.00) +[L21] whether(0.93) the(1.00) money(0.99) should(1.00) go(1.00) to(1.00) +[L22] <>(0.54) to(0.95) <<&>>(0.37) Lind(0.89) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P024.txt b/app/deploy/book1_data/moredata/confidence/B1_P024.txt new file mode 100644 index 0000000000000000000000000000000000000000..bd0a9059b46dcaa86c31f8343411fea10cff3baf --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P024.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 024 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 12 von 160 + +[L01] s(0.79) +[L02] From(1.00) this(1.00) day(0.80) I(1.00) could(1.00) first(1.00) consider(1.00) my-(1.00) +[L03] self(0.90) as(0.98) <>(0.48) appointed(0.98) by(1.00) <>(0.49) Majesty(1.00) +[L04] on(1.00) the(1.00) Expedition.(0.99) The(1.00) Order(0.99) signed(1.00) by(1.00) the(0.99) +[L05] <>(0.47) imported(0.93) that(1.00) 1795.(0.89) I(1.00) should(1.00) be(1.00) paid(1.00) +[L06] to(1.00) me(1.00) by(1.00) the(1.00) Exchequer,(1.00) which(1.00) I(0.98) <>(0.53) +[L07] <>(0.67) some(1.00) says(1.00) afterwards.(1.00) +[L08] the(0.72) same(1.00) day(0.99) I(1.00) went(1.00) to(1.00) <>(0.67) Banks(0.85) &(0.98) not(1.00) +[L09] finding(0.73) him(1.00) at(1.00) home(1.00) I(1.00) wrote(1.00) a(1.00) note(0.98) to(1.00) +[L10] him(0.98) desiring(0.77) him(1.00) to(1.00) appoint(0.99) me(1.00) a(1.00) +[L11] day,(1.00) when(0.98) I(1.00) could(1.00) have(1.00) the(1.00) favour(0.96) to(1.00) +[L12] speak(0.92) to(1.00) him;(0.96) <>(0.70) being(1.00) appointed(0.75) by(0.99) +[L13] his(1.00) Majesty(0.99) to(0.96) go(1.00) on(1.00) the(1.00) Expedition,(0.99) +[L14] I(0.94) wished(1.00) to(1.00) be(1.00) informed(0.99) by(1.00) him(1.00) <>(0.16) me(0.91) +[L15] particulars(0.97) relative(1.00) to(1.00) my(1.00) <>(0.60) +[L16] &(0.99) having(1.00) always(1.00) been(1.00) favoured(0.97) with(1.00) +[L17] his(1.00) friendship(1.00) I(0.93) hoped,(0.99) he(1.00) would(1.00) add(0.97) +[L18] this(0.98) to(0.98) the(1.00) many(1.00) obligations(0.83) he(0.99) +[L19] had(1.00) already,(0.98) laid(1.00) me(0.98) under(1.00) +[L20] <>(0.53) it(0.95) was(1.00) just(1.00) going(0.90) away(1.00) Dr(0.92) <>(0.44) +[L21] came(1.00) in,(0.99) I(0.94) told(0.99) of(1.00) my(1.00) going(1.00) &(0.99) <>(0.29) +[L22] likewise(0.99) his(1.00) advice,(0.96) He(1.00) congratulated(0.77) +[L23] are(0.86) coolly(0.97) &(1.00) was(1.00) civil;(0.98) but(1.00) the(1.00) moment(1.00) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P025.txt b/app/deploy/book1_data/moredata/confidence/B1_P025.txt new file mode 100644 index 0000000000000000000000000000000000000000..f304909e912b5a565260987c24ad3f77de507b1c --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P025.txt @@ -0,0 +1,31 @@ +=== Buch 1, Seite 025 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 21 von 171 + +[L01] June(0.96) y(0.98) +[L02] June(1.00) 4(0.92) +[L03] I(0.93) on(0.79) purpose(1.00) told(1.00) him(1.00) &(1.00) <>(0.58) Lind,(0.95) that(0.98) +[L04] the(1.00) money(1.00) for(1.00) Daino(0.70) was(1.00) out(0.98) of(1.00) the(0.97) +[L05] Question(0.78) &(1.00) that(1.00) I(1.00) should(1.00) have(1.00) mine(0.88) +[L06] <>(0.68) from(1.00) the(1.00) <>(0.69) <>(0.63) of(1.00) His(0.78) +[L07] &(0.99) <>(0.08) dress(0.97) +[L08] <>(0.61) the(1.00) coolness(0.77) <>(0.68) off,(1.00) &(1.00) all(1.00) +[L09] gloom(0.97) was(1.00) at(1.00) once(1.00) carries(0.87) off(1.00) <>(0.50) +[L10] their(0.99) <>(0.57) <>(0.37) the(0.97) really(0.72) gave(1.00) me(1.00) some(1.00) +[L11] good(1.00) information(1.00) relative(1.00) to(1.00) many(1.00) +[L12] particulars.(0.89) +[L13] <>(0.25) +[L14] <>(0.51) <>(0.56) sent(1.00) me(1.00) word(0.99) that(1.00) I(1.00) might(0.97) +[L15] see(0.99) him(1.00) the(1.00) next(1.00) day.(0.73) +[L16] <>(0.16) to(1.00) see(1.00) <>(0.57) Banks(0.88) &(1.00) begged(0.84) of(1.00) him(0.97) the(1.00) favour(0.93) +[L17] to(1.00) give(1.00) me(1.00) some(1.00) informations(0.89) relative(0.99) to(1.00) my(1.00) +[L18] <>(0.62) adding(1.00) that(1.00) I(1.00) knew(0.91) very(1.00) well,(0.94) the(1.00) +[L19] necessity(0.83) of(1.00) common(1.00) articles,(0.96) but(0.99) I(1.00) <>(0.18) that(0.98) +[L20] by(1.00) his(1.00) experience(1.00) he(1.00) must(1.00) have(1.00) found(1.00) out,(1.00) +[L21] the(1.00) <>(0.28) or(1.00) inconvenience(0.82) of(1.00) many(1.00) +[L22] things,(1.00) which(1.00) could(1.00) not(1.00) be(1.00) <>(0.54) at(1.00) such(1.00) +[L23] a(1.00) distance,(0.99) or(0.83) did(1.00) not(1.00) occur(1.00) to(1.00) one(1.00) who(1.00) had(1.00) +[L24] not(1.00) been(1.00) on(1.00) the(1.00) Voyage:(0.99) N.Banks(0.72) declared(1.00) +[L25] he(1.00) would(1.00) give(1.00) me(1.00) any(1.00) information(0.96) &(0.97) I(0.95) +[L26] should(0.96) ask(0.95) him(1.00) <>(0.39) &(1.00) he(1.00) would(1.00) <>(0.32) +[L27] me(1.00) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P028.txt b/app/deploy/book1_data/moredata/confidence/B1_P028.txt new file mode 100644 index 0000000000000000000000000000000000000000..edabdd036900fbe1b5d54cfd853d2300dc2769de --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P028.txt @@ -0,0 +1,29 @@ +=== Buch 1, Seite 028 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 28 von 208 + +[L01] me.(1.00) I(1.00) told(1.00) that(1.00) as(1.00) I(1.00) was(1.00) unacquainted,(0.74) <>(0.23) the(1.00) +[L02] peculiar(0.97) &(1.00) <>(0.32) particulars(0.98) I(0.91) only(1.00) wished(1.00) +[L03] to(1.00) be(1.00) by(1.00) him(1.00) <>(0.43) in(0.99) regard(0.99) to(0.99) them,(0.99) but(1.00) +[L04] Mr.(0.75) <>(0.39) declined,(0.93) this(1.00) &(0.89) <>(0.41) I(0.99) should(0.95) ask(1.00) +[L05] +[L06] him(1.00) questions(0.79) &(1.00) he(1.00) <>(0.66) answer.(0.94) a(0.87) great(1.00) many(1.00) +[L07] more(1.00) company(0.92) coming(1.00) in(1.00) I(0.74) on(1.00) purpose(1.00) asked(0.78) +[L08] only(0.98) a(1.00) few(1.00) trifling(0.74) questions(0.71) &(0.83) then(1.00) applied(1.00) +[L09] to(1.00) him(1.00) for(1.00) to(1.00) let(0.99) me(0.99) see(1.00) the(1.00) drawings(0.99) of(1.00) <>(0.60) +[L10] plants.(0.99) He(1.00) replied(1.00) this(1.00) would(1.00) be(1.00) of(1.00) little(1.00) use(0.99) +[L11] to(1.00) me,(1.00) &(0.99) excused(0.77) himself(1.00) with(1.00) want(0.93) at(1.00) time(0.97) +[L12] &(0.95) then(0.99) said,(0.98) since(1.00) I(0.99) saw,(1.00) he(1.00) declined(1.00) to(1.00) do(1.00) it(1.00) +[L13] I(1.00) would(1.00) not(1.00) insist(0.87) upon(0.99) it(1.00) hereupon(1.00) Mr(0.77) <<13am>>(0.61) +[L14] asked(0.90) me(1.00) when(1.00) she(0.89) Ship(1.00) would(0.98) <>(0.45) I(1.00) told(1.00) +[L15] I(0.99) did(1.00) not(1.00) know,(1.00) but(1.00) <>(0.63) it(1.00) would(0.99) +[L16] <>(0.59) Sunday(0.93) sex(0.74) <>(0.28) +[L17] happen(0.99) on(1.00) Saturday(0.98) next,(0.99) then(1.00) he(1.00) desired(0.78) +[L18] me(1.00) to(1.00) come(0.99) Saturday(0.99) next.(1.00) I(1.00) met(0.96) Capt.(0.78) +[L19] that(0.99) day(1.00) at(1.00) <>(0.47) <>(0.34) &(0.99) went(1.00) with(1.00) nine(0.80) to(1.00) the(0.95) +[L20] Admiralty(0.99) <>(0.70) a(1.00) board(1.00) of(1.00) longitude(1.00) was(1.00) +[L21] to(1.00) be(1.00) held,(1.00) in(1.00) order(1.00) to(1.00) see(1.00) there(1.00) <<.>>(0.21) <>(0.65) Hormby.(0.72) +[L22] then(1.00) went(1.00) with(1.00) <>(0.37) Cook(0.84) home,(0.98) o(0.74) <>(0.65) him(1.00) +[L23] to(0.88) dinner.(1.00) +[L24] <>(0.50) to(1.00) the(1.00) Treasury(1.00) &(1.00) Exchequer(0.92) &(1.00) <>(0.60) <<179>>(0.67) <<5.>>(0.57) <>(0.20) pei(0.80) +[L25] <>(0.60) 95(0.96) of(1.00) which(1.00) were(0.99) to(1.00) defray(0.97) <>(0.67) <>(0.61) <>(0.39) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P029.txt b/app/deploy/book1_data/moredata/confidence/B1_P029.txt new file mode 100644 index 0000000000000000000000000000000000000000..1e90d6b4a325ff364f27fedd4fead940daf8dc1f --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P029.txt @@ -0,0 +1,29 @@ +=== Buch 1, Seite 029 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 32 von 186 + +[L01] <>(0.56) 20th(0.98) all(0.94) my(1.00) baggage(0.90) was(1.00) sent(1.00) on(1.00) board(0.99) the(0.99) +[L02] Resolution(0.93) by(1.00) the(1.00) Sheerness.(0.89) <>(0.61) +[L03] June(0.93) <<4th>>(0.52) <>(0.18) <>(0.41) my(0.99) Equipment.(1.00) I(1.00) took(1.00) leave(0.99) +[L04] <>(0.24) <>(0.69) +[L05] of(1.00) my(1.00) Family,(0.87) &(0.95) took(0.99) a(1.00) <>(0.57) for(0.98) Plymouth,(0.98) +[L06] The(1.00) Resolution(1.00) having(1.00) left(1.00) <>(0.49) <>(0.34) <>(0.22) <<22°.>>(0.62) +[L07] <>(0.51) passed(0.99) through(1.00) Brenton(0.99) a(1.00) dirty(1.00) town,(0.99) came(1.00) to(1.00) +[L08] Aounslow,(0.96) all(1.00) a(1.00) fine(1.00) &(1.00) cultivated(1.00) country,(1.00) then(0.82) +[L09] passed(0.99) Avaslow(0.91) <>(0.62) filled(0.99) with(1.00) <>(0.61) &(0.98) <>(0.47) +[L10] both(1.00) a(1.00) disgrace(0.81) to(1.00) the(1.00) Country.(1.00) Dined(0.92) at(1.00) Stain-(0.74) +[L11] bridge:(0.95) passed(1.00) over(1.00) the(1.00) <>(0.65) hills(1.00) on(1.00) +[L12] <>(0.43) common,(0.99) which(1.00) <>(0.40) London(0.96) <>(0.62) +[L13] <>(0.69) with(1.00) the(1.00) finest(0.98) mutton,(0.73) but(1.00) in(1.00) my(1.00) o-(1.00) +[L14] pinion(0.75) would(1.00) be(1.00) better(1.00) employed(1.00) in(1.00) <>(0.54) +[L15] tations(0.99) chiefly(1.00) of(1.00) oak.(0.99) The(0.88) same(1.00) country(1.00) con-(0.89) +[L16] tinues(0.98) to(1.00) <>(0.46) green,(0.91) where(0.95) we(0.98) went(0.99) to(1.00) +[L17] Basingstock(0.99) an(0.99) ancient(1.00) town;(0.88) <>(0.20) +[L18] we(1.00) passed(1.00) Sutton(0.97) &(1.00) Stockbridge,(0.99) &(1.00) having(0.99) <>(0.36) +[L19] the(1.00) <>(0.36) &(1.00) <>(0.19) <>(0.39) of(0.96) the(1.00) plains(0.97) +[L20] of(1.00) salisbury,(0.85) we(0.98) reached(1.00) that(1.00) ancient(1.00) city.(0.94) +[L21] the(1.00) Spire(0.98) &(0.71) <>(0.58) on(1.00) account(0.99) of(1.00) +[L22] <<&>>(0.64) has(0.99) walls(0.99) from(0.91) the(1.00) city(0.99) +[L23] its'(0.82) height(0.99) &(0.99) pointed(0.99) figure(0.99) &(0.96) Our(1.00) <>(0.23) law(0.89) +[L24] still(1.00) on(1.00) the(1.00) same(1.00) <>(0.39) Heath,(0.80) +[L25] The(1.00) bottom(1.00) of(1.00) all(1.00) the(1.00) country(0.95) is(1.00) chalk.(1.00) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P030.txt b/app/deploy/book1_data/moredata/confidence/B1_P030.txt new file mode 100644 index 0000000000000000000000000000000000000000..63661e7ef86232666b4a26dfaee611b8ebc2a0ca --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P030.txt @@ -0,0 +1,34 @@ +=== Buch 1, Seite 030 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 32 von 219 + +[L01] <>(0.38) +[L02] Pame(0.73) <>(0.46) <<27.>>(0.62) <>(0.16) reached(1.00) Barnet-down(0.76) a(1.00) single(0.99) house(1.00) &(1.00) then(1.00) +[L03] <>(0.56) to(1.00) Blandford.(0.99) The(1.00) country(1.00) begins(1.00) to(0.75) grow(1.00) hill(0.85) +[L04] between(0.98) the(1.00) latter(1.00) place(1.00) &(0.79) Doschester,(0.86) a(1.00) tolerable(0.99) +[L05] good(0.99) place.(0.99) All(0.97) <>(0.56) is(1.00) well(1.00) cultivated,(0.86) &(0.84) +[L06] from(1.00) the(1.00) summits(0.90) of(0.88) its(1.00) hills,(0.89) it(1.00) bears(1.00) a(0.99) <<1>>(0.30) <>(0.36) +[L07] charming(0.72) aspect.(1.00) At(0.93) Bridport(0.72) the(1.00) Sea(1.00) comes(1.00) up(1.00) +[L08] is(0.98) in(1.00) +[L09] within(0.94) two(0.95) miles,(1.00) <>(0.68) town,(0.98) <>(0.17) a(1.00) most(0.91) thriving(0.86) +[L10] condition(1.00) &(1.00) has(1.00) a(1.00) manufacture(0.89) of(1.00) <>(0.55) +[L11] which(1.00) employs(0.77) a(1.00) good(1.00) number(1.00) of(1.00) hand.(0.70) +[L12] On(0.82) account(1.00) of(1.00) the(1.00) vicinity(1.00) of(1.00) the(1.00) Sea,(1.00) the(1.00) top(0.99) +[L13] of(1.00) the(1.00) hills(1.00) were(1.00) this(1.00) day(1.00) <>(0.68) with(1.00) clouds(0.72) +[L14] which(1.00) soon(1.00) were(0.97) <>(0.37) into(1.00) rain.(0.91) We(1.00) +[L15] in(0.95) <>(0.49) +[L16] reached(0.73) <>(0.69) for(0.99) <>(0.52) carpets(0.99) are(1.00) many(0.98) +[L17] factured;(0.77) from(0.85) this(1.00) place(0.81) we(0.86) <>(0.69) <>(0.64) for(1.00) <>(0.62) +[L18] a(1.00) populous(0.94) borough(0.99) at(1.00) the(1.00) bottom(1.00) of(1.00) a(1.00) <>(0.35) +[L19] Its(0.91) chief(1.00) <>(0.12) commodity(0.94) are(1.00) course(0.93) cloaths.(0.77) +[L20] Exeter(0.99) the(1.00) next(1.00) city,(0.82) is(1.00) a(1.00) very(1.00) <>(0.36) <>(0.48) <>(0.44) +[L21] ty(1.00) <>(0.64) surrounded(0.99) by(1.00) old(1.00) decaying(0.99) walls,(1.00) +[L22] its(0.93) streets(0.99) are(1.00) extremely(1.00) narrow,(0.99) go(0.95) up(1.00) +[L23] &(0.98) down(1.00) hill,(0.71) &(0.98) make(1.00) the(1.00) city(1.00) to(1.00) a(1.00) <>(0.09) all(0.94) +[L24] plare.(0.80) The(1.00) front(0.77) of(1.00) its(1.00) <>(0.52) has(0.98) a(1.00) great(1.00) +[L25] <<7>>(0.13) +[L26] number(1.00) of(1.00) figures(0.97) in(1.00) the(1.00) gothic(0.98) taste(0.88) +[L27] but(1.00) the(1.00) villages(0.77) are(1.00) now(0.77) quite(1.00) <>(0.51) +[L28] The(1.00) castle(0.99) is(1.00) surrounded(0.99) by(1.00) a(1.00) <>(0.47) <
>(0.31) <>(0.42) +[L29] trees,(0.93) part(1.00) of(1.00) which(1.00) is(1.00) clanted(0.72) on(0.97) the(0.96) slope(0.96) +[L30] the(0.98) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P031.txt b/app/deploy/book1_data/moredata/confidence/B1_P031.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0cf10778a91fa382f2e66c7d94910a14eb8e7fb --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P031.txt @@ -0,0 +1,28 @@ +=== Buch 1, Seite 031 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 24 von 165 + +[L01] <<12>>(0.24) <>(0.64) with(1.00) a(1.00) few(1.00) openings(0.91) that(1.00) afford(1.00) a(1.00) <>(0.48) +[L02] romantic(0.94) prospect,(0.99) beyond(1.00) a(1.00) <>(0.46) <>(0.64) the(1.00) <>(0.54) +[L03] bouring(0.90) <>(0.64) cultivated(0.97) hills.(1.00) This(0.91) walk(0.90) is(1.00) +[L04] calledy(0.82) <>(0.50) walk,(0.97) &(0.99) is(1.00) much(1.00) <>(0.10) <>(0.54) +[L05] by(1.00) the(1.00) neutnphs(0.72) of(1.00) the(1.00) town,(1.00) especially(1.00) +[L06] <>(0.30) +[L07] at(1.00) <>(0.65) hours.(0.99) The(1.00) <>(0.61) cloaths,(0.95) calli-(0.93) +[L08] mancoes(0.98) &(1.00) other(1.00) <>(0.51) &(0.92) stuffs,(0.83) which(0.99) are(0.99) +[L09] here(1.00) manufactured(1.00) <>(0.68) a(1.00) good(1.00) number(1.00) +[L10] at(1.00) poor(0.82) people(1.00) &(1.00) <>(0.67) the(1.00) wool(0.87) of(0.74) +[L11] the(0.99) neighbourhood.(0.77) +[L12] June(0.99) <<1928.>>(0.33) We(1.00) passed(1.00) some(1.00) very(0.98) high(1.00) hills(0.99) in(1.00) <>(0.53) +[L13] to(1.00) Chudeigh(0.83) a(1.00) small(1.00) <>(0.63) town(1.00) +[L14] The(1.00) same(1.00) aspect(1.00) of(1.00) the(1.00) country(1.00) is(1.00) between(1.00) +[L15] the(1.00) latter(0.97) place(0.84) &(0.99) Ashburton(0.99) another(1.00) +[L16] small(0.96) town.(0.98) The(1.00) Inn(1.00) we(1.00) put(1.00) up,(1.00) It(0.78) was(1.00) <>(0.35) +[L17] pied(0.98) by(0.99) a(1.00) Scotsman(0.81) called(0.93) Kenneth(1.00) <>(0.58) +[L18] who(1.00) overcharged(0.97) every(1.00) article(1.00) of(1.00) our(1.00) +[L19] bad(0.82) fare,(0.95) &(1.00) gave(1.00) us(0.92) most(1.00) sorry(0.99) horses.(0.92) +[L20] &(1.00) a(1.00) surely,(0.78) impudent(0.77) driver.(0.97) At(1.00) <>(0.47) +[L21] bridge,(1.00) are(0.92) a(1.00) few(1.00) houses(1.00) &(1.00) two(0.97) inns,(0.71) one(0.99) +[L22] on(1.00) each(1.00) side(0.95) the(1.00) bridge.(1.00) That(1.00) which(1.00) lie(0.81) +[L23] on(1.00) the(1.00) <>(0.46) has(1.00) the(1.00) better(0.97) appearance(0.99) +[L24] <>(0.49) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P034.txt b/app/deploy/book1_data/moredata/confidence/B1_P034.txt new file mode 100644 index 0000000000000000000000000000000000000000..a418854e504c34b68cc29fcd9275914995100ea5 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P034.txt @@ -0,0 +1,31 @@ +=== Buch 1, Seite 034 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 27 von 184 + +[L01] but(1.00) the(1.00) landlady(0.95) on(1.00) the(1.00) western(0.99) side(1.00) was(1.00) a(1.00) +[L02] sensible(0.96) old(0.99) woman.(0.99) In(1.00) the(1.00) Afternoon(0.98) +[L03] <>(0.23) reached(1.00) Plymouth.(1.00) All(0.81) the(1.00) <>(0.50) aty(0.84) of(0.92) +[L04] <>(0.53) is(1.00) pretty(1.00) well(1.00) cultivated(1.00) in(0.92) +[L05] its(0.99) <>(0.64) as(0.96) far(1.00) at(0.98) the(1.00) high(1.00) <>(0.58) <>(0.23) +[L06] of(1.00) the(1.00) numerous(0.93) hills(1.00) will(1.00) permit.(1.00) The(1.00) +[L07] <>(0.58) seems(1.00) however(1.00) to(1.00) be(1.00) rather(1.00) +[L08] neglected:(0.86) for(1.00) all(1.00) that,(1.00) it(1.00) bears(0.72) a(1.00) most(1.00) roma-(0.85) +[L09] Fio(0.92) aspect,(0.99) from(1.00) the(0.99) top(0.98) of(1.00) the(1.00) higher(1.00) hill.(0.78) +[L10] We(1.00) <>(0.56) immediately(0.93) for(1.00) the(1.00) Resolution(1.00) &(0.98) +[L11] heard(1.00) She(0.84) was(1.00) not(1.00) yet(1.00) arrived.(1.00) Plymouth(1.00) <>(0.49) +[L12] irveyula(0.81) +[L13] a(1.00) place(0.95) with(1.00) narrow(1.00) winding(0.89) trees,(0.85) which(1.00) +[L14] <>(0.42) I(1.00) +[L15] have(1.00) a(1.00) Channel(0.76) with(1.00) running(0.99) water(1.00) in(1.00) +[L16] their(1.00) <>(0.57) &(1.00) though(1.00) this(1.00) is(1.00) a(1.00) great(1.00) <>(0.43) +[L17] oenience(0.82) to(1.00) carriages,(0.97) which(1.00) must(0.99) always(1.00) +[L18] have(0.99) +[L19] one(0.97) wheel(1.00) in(1.00) the(1.00) channel,(0.99) it(0.98) is(0.81) <<&>>(0.55) to(1.00) <>(0.12) +[L20] ple(0.96) walking(0.94) in(0.99) the(1.00) streets(0.76) who(0.87) are(1.00) <>(0.29) +[L21] ly(1.00) <>(0.22) by(1.00) the(1.00) houses,(0.77) it(1.00) has(1.00) on(0.88) the(1.00) +[L22] hand(0.99) the(1.00) <>(0.34) of(1.00) carrying(0.95) off(0.93) all(1.00) +[L23] <>(0.64) <>(0.46) &(1.00) <>(0.44) <>(0.53) me(0.92) <>(0.53) especially(0.92) +[L24] at(1.00) night,(1.00) when(1.00) the(1.00) contents(1.00) of(1.00) a(1.00) great(1.00) me(0.84) +[L25] foots(0.97) with(1.00) <>(0.64) <>(0.68) <>(0.09) are(1.00) <>(0.62) +[L26] <>(0.58) it(1.00) as(0.97) few(1.00) people(1.00) have(1.00) regular(1.00) little(1.00) +[L27] houses(0.94) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P035.txt b/app/deploy/book1_data/moredata/confidence/B1_P035.txt new file mode 100644 index 0000000000000000000000000000000000000000..4fb136787f5418dbb816f1a481d11ca438c2eadc --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P035.txt @@ -0,0 +1,32 @@ +=== Buch 1, Seite 035 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 25 von 183 + +[L01] houses(0.96) in(1.00) this(1.00) town.(1.00) The(0.99) <>(0.49) of(1.00) the(1.00) Sea(0.80) +[L02] at(1.00) <>(0.51) leaves,(0.89) a(1.00) great(1.00) part(1.00) of(1.00) the(1.00) in-(1.00) +[L03] terior(0.83) harbour(0.99) dry(0.95) &(1.00) causes(1.00) an(0.99) <>(0.55) +[L04] <>(0.54) stench.(0.79) Monday(1.00) June(1.00) <>(0.68) 20th.(0.81) We(1.00) went(1.00) +[L05] of(1.00) one(1.00) <>(0.30) +[L06] to(1.00) the(1.00) Dock(0.97) &(1.00) passed(1.00) over(1.00) a(1.00) fine(1.00) bridge(0.98) +[L07] built(0.99) of(0.83) Portland(0.73) Stone,(0.99) over(0.99) an(1.00) arm(1.00) of(0.88) +[L08] the(1.00) Sen.(0.76) On(0.72) the(1.00) other(1.00) side(0.98) we(1.00) passed(1.00) by(1.00) +[L09] the(1.00) Hospital(1.00) &(1.00) came(1.00) to(1.00) Plymouth(0.99) <>(0.49) +[L10] a(1.00) <>(0.54) of(1.00) new(1.00) town(0.73) 2(1.00) miles(0.94) from(1.00) Plymouth(1.00) +[L11] poth(0.70) a(1.00) number(1.00) of(1.00) good(1.00) houses(0.88) regular(0.98) +[L12] Streets,(0.77) which(1.00) are(1.00) improving(1.00) every(0.99) day.(1.00) +[L13] She(0.89) <>(0.60) is(1.00) beyond(1.00) this(1.00) <>(0.57) in(1.00) high(1.00) +[L14] <>(0.38) +[L15] walls,(0.97) &(1.00) none(1.00) but(1.00) <<7>>(0.27) <>(0.45) are(1.00) belonging(1.00) <>(0.57) +[L16] <>(0.24) <>(0.51) &(0.84) +[L17] if(0.94) are(1.00) admitted,(0.75) we(1.00) wanted(0.99) <>(0.52) <>(0.39) +[L18] andwich(0.78) who(1.00) was(1.00) gone(1.00) up(1.00) their(0.95) 10-(0.87) +[L19] or(0.83) 12(0.89) miles,(1.00) in(1.00) the(1.00) afternoon(0.77) I(0.94) went(0.90) on(1.00) +[L20] board(0.99) the(1.00) Adventure,(1.00) who(0.99) lay(1.00) near(0.88) <>(0.65) +[L21] Edgecombe(0.81) in(0.99) +[L22] <>(0.49) but(1.00) found(1.00) neither(0.91) +[L23] Capt(0.98) Turneaux,(0.90) nor(1.00) Lieut.(0.97) Kempe(0.93) on(1.00) <>(0.41) +[L24] for(0.99) whom(1.00) I(1.00) had(1.00) letters(0.97) from(1.00) the(1.00) hostile(0.85) <>(0.60) +[L25] <>(0.68) <>(0.66) Tuesday(1.00) I(1.00) went(1.00) to(1.00) Drukes(0.73) or(0.91) St(1.00) Nicholas(1.00) +[L26] Island(0.91) which(1.00) lies(1.00) in(1.00) the(1.00) month(0.92) +[L27] e(0.97) Plymouth(0.79) +[L28] <>(0.61) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P038.txt b/app/deploy/book1_data/moredata/confidence/B1_P038.txt new file mode 100644 index 0000000000000000000000000000000000000000..29b4b09173d90ca0bff1cf5d6a52fe1dc5c41622 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P038.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 038 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 23 von 160 + +[L01] &(1.00) from(1.00) which(1.00) towards(0.91) mount(0.87) Edgecombe(0.82) a(1.00) Ledge(0.82) +[L02] of(1.00) rocks(0.96) extends.(0.94) This(0.98) Isle(1.00) is(1.00) <>(0.41) <>(0.63) &(0.94) is(0.99) a(1.00) mere(0.98) +[L03] rock(0.90) <>(0.60) a(1.00) few(1.00) houses,(0.94) in(1.00) which(1.00) a(1.00) gunner(0.83) +[L04] &(1.00) a(1.00) few(1.00) <>(0.64) <>(0.65) Here(1.00) <>(0.27) <>(0.41) <>(0.69) <>(0.44) +[L05] norner,(0.70) who(1.00) was(1.00) intended(0.99) to(0.97) <>(0.49) on(0.97) the(0.98) Expe(0.92) +[L06] dition,(0.86) in(1.00) the(1.00) Adventure,(0.99) had(0.96) erected(0.97) his(1.00) <>(0.49) +[L07] &(0.99) made(1.00) observations(0.97) of(1.00) equal(0.99) Altitudes,(0.89) &(1.00) +[L08] afterwards(0.90) when(0.71) the(1.00) Resolution(1.00) arrived(1.00) +[L09] <>(0.56) Wales(0.88) joined(1.00) him(1.00) with(1.00) the(1.00) Transit-in-(1.00) +[L10] strument,(0.89) &(1.00) they(1.00) settled(0.83) both(1.00) the(1.00) Latitude(1.00) &(0.98) +[L11] Longitude(0.98) of(1.00) this(1.00) place-vig.(0.73) +[L12] July(1.00) <>(0.31) <>(0.37) <<1st>>(0.50) +[L13] went(1.00) on(1.00) board(1.00) the(1.00) <>(0.65) to(1.00) was(0.95) +[L14] upon(1.00) Lord(1.00) Sandwich,(0.95) who(1.00) had(1.00) desired(0.99) <>(0.50) +[L15] the(0.94) day(1.00) <>(0.68) +[L16] neaux,(0.71) to(0.93) let(0.83) me(1.00) know(0.99) he(1.00) wanted(1.00) to(0.99) +[L17] speak(0.80) to(1.00) me.(1.00) His(1.00) Lordship(0.95) received(1.00) me(1.00) +[L18] in(1.00) the(1.00) most(1.00) obliging(0.99) manner;(0.91) after(1.00) +[L19] breakfast,(0.96) he(0.85) took(1.00) me(0.99) on(1.00) the(1.00) <>(0.55) of(1.00) +[L20] the,(0.85) Vacht,(0.83) &(1.00) asked(1.00) me(1.00) <>(0.69) whether(0.96) +[L21] <>(0.37) could(1.00) do(1.00) any(1.00) this(1.00) <>(0.65) for(1.00) me,(0.72) +[L22] <<&>>(0.20) thanked(0.98) his(0.98) Lordship(0.74) for(0.97) his(1.00) kind-(0.98) +[L23] ness(0.99) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P039.txt b/app/deploy/book1_data/moredata/confidence/B1_P039.txt new file mode 100644 index 0000000000000000000000000000000000000000..20bde460da52630794382ddd6633e4b0f14a7dd4 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P039.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 039 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 19 von 152 + +[L01] <<6>>(0.36) +[L02] ness(1.00) &(1.00) told,(1.00) that(1.00) I(1.00) thought(1.00) my(1.00) present(0.78) +[L03] situation(0.77) so(0.84) happy,(1.00) that(1.00) no(1.00) other(1.00) wish(0.99) +[L04] was(1.00) left(1.00) to(1.00) me(1.00) than,(0.99) that(0.85) <>(0.54) <>(0.28) +[L05] would(0.92) be(1.00) pleased(0.93) to(1.00) continue(1.00) to(1.00) me(1.00) his(1.00) +[L06] protection(0.93) &(1.00) favour.(0.99) He(1.00) said(1.00) he(1.00) <>(0.45) +[L07] the(1.00) Resolution(1.00) that(1.00) night(1.00) in(1.00) Plymouth(1.00) +[L08] sound(0.84) &(1.00) in(1.00) that(1.00) case,(1.00) wished(1.00) to(1.00) see(1.00) +[L09] me(0.95) on(1.00) board(1.00) of(1.00) her,(1.00) between(0.96) five(0.99) +[L10] &(0.97) Sir.(0.79) After(1.00) having(1.00) <>(0.69) with(1.00) <>(0.44) +[L11] <>(0.57) for(0.99) about(1.00) <<4>>(0.34) of(1.00) <<&>>(0.42) hour.(0.99) he(1.00) +[L12] went(1.00) in(0.98) his(1.00) <>(0.39) to(1.00) the(1.00) dock-yard(0.82) +[L13] &(0.90) took(1.00) me(1.00) along(1.00) with(1.00) him,(1.00) &(1.00) went(1.00) +[L14] through(1.00) the(1.00) whole(1.00) <>(0.56) with(1.00) me(0.95) &(1.00) +[L15] <>(0.28) he(0.96) thought(1.00) an(1.00) explication(0.97) +[L16] of(0.98) the(1.00) various(1.00) objects(0.98) to(0.75) works(0.72) might(0.75) +[L17] <>(0.27) necessary,(1.00) he(1.00) was(1.00) pleased(0.93) to(1.00) give(0.90) +[L18] <>(0.63) the(1.00) fullest(0.76) Information(0.98) of(1.00) every(0.98) +[L19] thing.(0.94) after(0.77) one(1.00) o'clock(0.98) His(0.71) <>(0.44) +[L20] went(0.99) on(1.00) board(1.00) the(1.00) <>(0.69) &(1.00) <>(0.31) +[L21] me(1.00) to(1.00) be(1.00) at(1.00) six(1.00) on(1.00) board(1.00) the(0.87) Resolution(1.00) +[L22] in(1.00) <>(0.60) she(1.00) should(1.00) arrive(1.00) before(1.00) that(1.00) +[L23] <>(0.39) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P042.txt b/app/deploy/book1_data/moredata/confidence/B1_P042.txt new file mode 100644 index 0000000000000000000000000000000000000000..77e7c6f17c1b854ea10180221406509829b0a306 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P042.txt @@ -0,0 +1,29 @@ +=== Buch 1, Seite 042 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 20 von 180 + +[L01] time.(1.00) His(0.94) Lordship(0.97) went(1.00) to(1.00) dine(0.74) at(1.00) Mount(0.99) Edge-(0.79) +[L02] combe(0.98) with(0.98) Lord(1.00) Edgecombe,(0.82) The(1.00) Resolution(1.00) +[L03] <>(0.58) not(1.00) come.(0.99) +[L04] July(0.97) <>(0.55) 2d.(0.85) Lord(0.99) <>(0.67) left(1.00) Plymouth(0.99) with(1.00) <>(0.55) +[L05] Yacht(0.94) at(0.75) six(1.00) oilock(0.75) in(1.00) the(1.00) morning.(1.00) +[L06] We(1.00) saw(1.00) the(1.00) Long(0.96) Room(0.77) &(1.00) bathing(0.97) place(1.00) kept(0.99) by(1.00) +[L07] a(1.00) few,(0.77) who(1.00) has(1.00) fitted(1.00) the(1.00) same(1.00) very(1.00) <>(0.36) +[L08] up;(0.97) but(1.00) 2(0.92) believe(1.00) there(1.00) are(1.00) so(0.94) few(1.00) people(0.98) +[L09] to(1.00) make(1.00) use(0.99) of(1.00) it,(0.99) that(1.00) he(1.00) <>(0.66) will(1.00) +[L10] <>(0.33) his(1.00) act.(0.91) +[L11] <>(0.52) <<3.>>(0.31) The(1.00) Resolution(1.00) was(1.00) arrived(1.00) in(1.00) the(1.00) night(1.00) +[L12] we(1.00) saw(1.00) her(1.00) from(1.00) the(1.00) Garrison,(0.88) a(0.94) fort(0.78) +[L13] <>(0.20) of(1.00) Stone(0.98) on(1.00) the(1.00) top(1.00) of(1.00) <>(0.44) <>(0.15) +[L14] hill,(0.98) which,(0.99) commands(0.96) the(1.00) entrance(0.99) +[L15] of(1.00) the(1.00) harbour;(0.84) &(0.78) is(1.00) <>(0.44) by(1.00) a(1.00) +[L16] <>(0.64) of(1.00) Soldiers.(0.98) We(0.99) went(1.00) on(1.00) board(1.00) +[L17] the(1.00) Resolution(1.00) Capt.(0.92) Cook(1.00) was(1.00) gone(1.00) +[L18] to(1.00) the(1.00) <>(0.67) &(0.95) we(1.00) followed(1.00) him(1.00) there(1.00) +[L19] &(1.00) I(1.00) went(1.00) again(1.00) on(1.00) board(1.00) the(1.00) Ship(1.00) with(1.00) the(1.00) +[L20] Carpenters(0.82) &(1.00) <>(0.34) in(1.00) order(1.00) to(1.00) have(1.00) all(1.00) +[L21] &(0.96) convenicacies(0.77) +[L22] The(0.75) Alteratiozes(0.72) <>(0.64) which(1.00) I(1.00) should(0.83) +[L23] in(1.00) the(1.00) little(1.00) Cabin(0.98) allotted(0.70) to(1.00) me(1.00) <<4>>(0.54) +[L24] wish(0.75) for.(0.88) After(0.97) having(1.00) given(1.00) proper(0.98) +[L25] directions,(0.99) I(1.00) returned(0.99) to(0.88) <>(0.18) <<&>>(0.67) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P043.txt b/app/deploy/book1_data/moredata/confidence/B1_P043.txt new file mode 100644 index 0000000000000000000000000000000000000000..71982c67725fcea0e177950bc32c46022afc31d2 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P043.txt @@ -0,0 +1,30 @@ +=== Buch 1, Seite 043 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 22 von 185 + +[L01] 18(0.85) )(0.84) dined(0.99) after(1.00) 5(1.00) o'clock,(0.95) &(1.00) as(0.78) I(1.00) could(1.00) do(1.00) nothing(1.00) +[L02] on(0.96) board(1.00) till(1.00) Tuesday(1.00) or(1.00) Wednesday,(0.98) when(0.99) the(1.00) +[L03] afterations(0.88) should(1.00) be(1.00) done,(1.00) I(1.00) took(1.00) a(1.00) +[L04] <>(0.63) into(1.00) Cornwall(0.94) in(1.00) order(1.00) to(1.00) see(1.00) the(1.00) +[L05] <>(0.51) there.(0.99) +[L06] July(0.99) <>(0.59) +[L07] <<&>>(0.14) Went(0.77) with(1.00) Mr(0.93) Arnold(0.99) &(1.00) my(1.00) son(0.92) +[L08] to(1.00) Saltash,(0.98) having(1.00) passed(1.00) the(1.00) <>(0.23) of(1.00) +[L09] the(1.00) Sea(0.83) in(0.99) a(1.00) Ferry.(0.96) The(1.00) borough(0.94) of(1.00) Saltoph(0.77) +[L10] lies(0.98) on(1.00) the(1.00) Slope(0.97) of(1.00) a(1.00) hill,(1.00) &(1.00) is(1.00) upon(1.00) the(1.00) +[L11] whole(0.96) but(1.00) a(1.00) sorry(0.93) place,(0.74) though(1.00) it(0.98) sends(0.80) <>(0.61) +[L12] Members(1.00) to(0.95) Parliament.(1.00) Passed(0.76) <>(0.52) +[L13] &(0.99) went(1.00) to(1.00) Bodmyn(0.78) which(0.96) formerly(1.00) was(1.00) a(1.00) +[L14] <>(0.46) of(1.00) greater(1.00) note,(0.98) when(1.00) the(1.00) See(0.71) of(1.00) +[L15] a(1.00) <>(0.66) was(1.00) there,(0.99) so(1.00) as(1.00) I(1.00) was(1.00) told,(0.76) though(0.97) +[L16] I(1.00) saw(1.00) no(1.00) remains(1.00) of(1.00) its(1.00) ancient(1.00) <>(0.53) +[L17] the(1.00) ruins(0.99) <>(0.56) +[L18] dor,(0.87) except(0.99) <>(0.31) old(1.00) Church(1.00) whose(0.97) tower(0.99) +[L19] still(1.00) exists,(0.94) &(0.99) afford(0.92) an(0.99) habitation(0.93) to(0.99) owls(0.82) &(1.00) <>(0.65) +[L20] <<#th>>(0.39) or(0.99) jack(0.79) <>(0.45) +[L21] <>(0.43) <>(0.50) We(0.95) came(1.00) to(1.00) <>(0.58) in(0.99) order(0.99) to(1.00) see(1.00) +[L22] there(0.99) some(1.00) mines,(0.99) which(1.00) upon(1.00) <>(0.42) +[L23] nation,(0.99) were(1.00) only(1.00) some(1.00) wretched(0.98) +[L24] Streamworks.(0.75) +[L25] July(0.98) <<4th>>(0.54) Resolved(0.87) to(1.00) go(1.00) to(1.00) <>(0.63) the(1.00) only(1.00) place(1.00) +[L26] where(1.00) I(0.98) could(1.00) expect(1.00) <>(0.59) passed(1.00) through(0.98) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P046.txt b/app/deploy/book1_data/moredata/confidence/B1_P046.txt new file mode 100644 index 0000000000000000000000000000000000000000..5991575aff681a33c3463f0cafc9beb21c76faa4 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P046.txt @@ -0,0 +1,29 @@ +=== Buch 1, Seite 046 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 25 von 179 + +[L01] 14(0.78) +[L02] A(0.79) Michael(0.97) a(1.00) poor(1.00) little(1.00) town:(0.98) arrived(1.00) at(1.00) 12(0.71) +[L03] <>(0.69) at(1.00) Souro.(0.97) Which(0.97) is(1.00) a(1.00) plan(0.73) in(1.00) every(0.95) +[L04] <>(0.62) condition.(0.91) The(1.00) Bank,(0.98) the(1.00) Custom(0.99) +[L05] house,(1.00) the(1.00) <>(0.68) office,(0.94) &(1.00) the(1.00) houses(1.00) +[L06] of(0.99) some(1.00) rich(1.00) Merchants(0.71) are(1.00) elegant(1.00) +[L07] buildings.(0.92) Here(1.00) the(0.91) white(0.99) Tin(0.78) of(1.00) the(1.00) <>(0.70) +[L08] <>(0.37) <<&>>(0.51) +[L09] <>(0.32) Cornwall(0.86) is(1.00) stamped(0.90) Each(0.99) block(1.00) of(1.00) +[L10] <>(0.31) contains(1.00) about(1.00) 300(1.00) weight.(0.98) I(0.97) <>(0.31) +[L11] one(0.89) Mr(0.90) <>(0.67) a(0.85) man(1.00) concerned(1.00) in(1.00) some(1.00) +[L12] tinmines(0.95) to(1.00) give(1.00) me(0.96) directions(1.00) &(0.89) ic-(0.88) +[L13] commandation(0.88) to(1.00) the(1.00) nearest(0.98) of(1.00) them.(1.00) +[L14] He(1.00) sent(1.00) his(1.00) clerk(1.00) one(0.98) <>(0.11) Marton(0.70) with(0.99) +[L15] who(1.00) gave(1.00) me(1.00) a(1.00) letter(1.00) to(1.00) Mrichard(0.74) +[L16] williams(0.79) at(1.00) <>(0.54) in(1.00) Gwenap(0.89) parish(1.00) +[L17] Cornwall.(0.96) It(0.81) is(1.00) about(1.00) 6(1.00) miles(1.00) west(0.97) of(1.00) <>(0.61) +[L18] <>(0.46) people(1.00) brought(0.97) <>(0.64) to(1.00) the(1.00) wrong(1.00) man(1.00) +[L19] for(0.99) there(1.00) are(1.00) <<#>>(0.30) Richard(1.00) Williams,(0.75) a(1.00) <>(0.41) +[L20] of(1.00) Mines,(0.99) or(1.00) Supervisors.(0.96) the(0.78) first(1.00) I(0.99) saw,(1.00) was(1.00) +[L21] at(1.00) Kenwyn,(0.97) in(1.00) a(1.00) new(0.99) Tinwork.(0.93) The(1.00) +[L22] Tier(0.81) is(0.98) <>(0.62) into(1.00) a(1.00) greyish(0.89) kind(0.89) of(1.00) <>(0.39) +[L23] called(0.98) there(1.00) Growan,(0.86) &(1.00) is(1.00) a(1.00) <>(0.55) of(1.00) Saxum(0.80) +[L24] <>(0.64) Moorstone(0.79) or(1.00) course(0.86) Granite.(0.98) The(1.00) country(1.00) or(1.00) +[L25] <>(0.68) <>(0.67) in(1.00) which(1.00) the(1.00) vein(0.94) <>(0.59) <>(0.47) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P047.txt b/app/deploy/book1_data/moredata/confidence/B1_P047.txt new file mode 100644 index 0000000000000000000000000000000000000000..9fd1934a1ba3ac365c8ad60b0343173eda8b85bc --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P047.txt @@ -0,0 +1,31 @@ +=== Buch 1, Seite 047 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 35 von 195 + +[L01] <>(0.70) +[L02] &(0.99) west(0.86) is(1.00) chiefly(0.99) Kellas(0.79) or(0.93) a(1.00) kind(1.00) of(1.00) fibrous(0.79) +[L03] lamellated(0.99) grey,(1.00) greenish,(1.00) or(1.00) <>(0.31) Stone(0.80) of(0.72) the(1.00) <>(0.64) +[L04] <>(0.61) kind,(1.00) <>(0.60) <>(0.63) into(1.00) Singles,(0.72) af-(1.00) +[L05] ter(0.92) having(1.00) been(1.00) exposed(0.99) to(1.00) the(1.00) air(0.99) for(1.00) a(1.00) <>(0.69) +[L06] while.(1.00) The(1.00) <>(0.60) is(1.00) from(1.00) B(0.91) inches(0.91) to(0.98) 3(1.00) <>(0.42) +[L07] wide(0.92) &(1.00) about(1.00) 40(1.00) or(0.99) 50(1.00) <>(0.37) deep.(0.96) A(1.00) great(1.00) <>(0.42) +[L08] of(0.99) Mundysk(0.82) or(1.00) <>(0.50) is(1.00) in(0.92) the(1.00) <>(0.33) +[L09] <>(0.40) road.(0.99) +[L10] Avale(0.98) with(1.00) a(1.00) <>(0.56) <>(0.63) <>(0.70) from(1.00) +[L11] Poddyte,(0.76) where(1.00) the(1.00) <>(0.20) is(1.00) more(1.00) <>(0.60) +[L12] being(1.00) a(1.00) fine(1.00) white(0.85) <>(0.60) or(0.91) <>(0.51) the(1.00) mi-(0.96) +[L13] <>(0.46) <>(0.43) +[L14] miscall(0.91) it(1.00) <>(0.57) with(0.94) fine(1.00) tin(0.78) <>(0.56) +[L15] of(0.71) the(1.00) +[L16] in(0.78) it.(0.78) but(1.00) <>(0.56) ore(0.83) is(1.00) in(1.00) this(0.99) +[L17] kind:(0.93) in(1.00) some(1.00) places(0.97) of(1.00) the(1.00) <>(0.61) they(0.98) +[L18] find(0.91) first(0.93) Gozzum(0.89) or(0.96) a(1.00) kind(1.00) of(1.00) an(0.98) Iron(0.95) +[L19] <>(0.29) pot(0.81) in(1.00) such(0.95) case(0.99) they(1.00) are(1.00) always(1.00) sure(1.00) of(1.00) +[L20] finding(0.73) Copper(0.88) below(1.00) it(0.99) the(0.93) Copper(0.92) <>(0.30) is(1.00) a(1.00) +[L21] yellow(1.00) Pyrites,(0.86) which(1.00) sometimes(0.99) on(1.00) account(0.99) +[L22] of(1.00) the(1.00) chinks(0.99) &(0.94) cracks(0.96) of(1.00) the(1.00) <>(0.40) <>(0.60) +[L23] admitted(0.99) the(1.00) external(0.78) air,(0.98) &(1.00) thus(0.70) +[L24] the(0.99) ore(0.91) <<#>>(0.33) <>(0.39) decaying(0.99) &(1.00) forms(1.00) those(1.00) +[L25] beautiful(0.71) colours,(0.95) which(1.00) are(1.00) like(1.00) the(1.00) pigeons(0.94) +[L26] neck(0.98) &(1.00) perfectly(0.99) similar(1.00) to(1.00) the(1.00) effects(0.99) of(0.96) +[L27] an(1.00) electrical(0.86) <>(0.55) upon(1.00) <>(0.68) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P050.txt b/app/deploy/book1_data/moredata/confidence/B1_P050.txt new file mode 100644 index 0000000000000000000000000000000000000000..90504f6f114d601a41e80019a78c2c2bf4619361 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P050.txt @@ -0,0 +1,30 @@ +=== Buch 1, Seite 050 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 28 von 190 + +[L01] The(1.00) vein(0.95) at(1.00) Polyce(0.87) is(1.00) from(1.00) 8.0(0.74) to(1.00) 105(0.91) <>(0.24) deep,(0.98) +[L02] is(1.00) much(1.00) richer,(0.99) &(1.00) increases(0.85) in(1.00) <>(0.57) the(1.00) +[L03] <>(0.69) it(1.00) goes,(0.99) but(1.00) cannot(1.00) be(1.00) raised(0.99) without(1.00) +[L04] loss,(1.00) on(1.00) account(1.00) of(1.00) the(1.00) immense(1.00) expense(0.94) +[L05] in(1.00) raising(1.00) the(1.00) <>(0.59) by(1.00) the(1.00) fire(1.00) engined,(0.83) +[L06] which(1.00) are(1.00) still(1.00) contrived(0.99) as(0.83) the(1.00) same(1.00) +[L07] effect(1.00) might(1.00) be(1.00) produced(0.99) by(1.00) half(1.00) the(1.00) coats,(0.91) +[L08] if(0.92) better(1.00) <>(0.58) She(0.76) veins(0.93) in(1.00) Cornwall(1.00) +[L09] for(0.99) the(1.00) greater(1.00) part(1.00) underlay(0.99) North(0.93) or(1.00) <>(0.59) +[L10] to(1.00) the(1.00) South;(0.99) the(1.00) <>(0.47) at(1.00) <>(0.41) is(1.00) just(0.93) +[L11] the(1.00) reverse,(0.73) for(1.00) it(1.00) underlays(0.75) South.(1.00) +[L12] In(1.00) all(1.00) vales(0.75) leading(1.00) toward(1.00) <>(0.42) <>(0.66) &(1.00) +[L13] in(1.00) places,(1.00) where(1.00) formerly(0.86) the(1.00) <>(0.66) +[L14] worked(1.00) for(1.00) <>(0.66) <>(0.66) pieces(0.98) of(1.00) <>(0.45) +[L15] from(1.00) the(1.00) Size(1.00) of(1.00) a(1.00) <>(0.56) to(1.00) that(1.00) of(1.00) a(1.00) +[L16] wallput(0.73) &(1.00) even(1.00) bigger(0.99) are(0.89) to(1.00) be(1.00) met(1.00) with.(1.00) +[L17] &(0.98) these(1.00) the(1.00) <>(0.38) Separate(0.89) from(1.00) the(1.00) unyiel(0.85) +[L18] ding(1.00) stuff(0.83) by(1.00) washing,(0.99) the(1.00) light(0.96) stuff(0.97) going(1.00) +[L19] away,(1.00) when(1.00) exposed(1.00) to(1.00) the(1.00) current(1.00) of(1.00) a(1.00) <>(0.12) +[L20] <>(0.52) for(1.00) which(1.00) reason(0.94) all(1.00) the(1.00) <>(0.67) got(1.00) +[L21] in(1.00) this(0.99) manner(0.97) is(1.00) supposed(0.99) to(1.00) be(1.00) +[L22] <>(0.12) +[L23] at(0.76) <>(0.56) of(0.97) a(0.97) <>(0.13) <>(0.46) +[L24] a(1.00) Streamwork.(0.95) This(0.98) <>(0.17) <>(0.29) as(0.95) <>(0.67) <>(0.64) +[L25] picked(0.99) from(1.00) the(1.00) <>(0.58) Stone,(0.92) is(0.98) brought(0.99) +[L26] 70(0.90) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P051.txt b/app/deploy/book1_data/moredata/confidence/B1_P051.txt new file mode 100644 index 0000000000000000000000000000000000000000..8f25498aebef38b36c1cd413a4bb118314f6c719 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P051.txt @@ -0,0 +1,35 @@ +=== Buch 1, Seite 051 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 33 von 193 + +[L01] <>(0.17) +[L02] +[L03] <>(0.04) +[L04] to(1.00) a(1.00) stampingmill(0.91) &(1.00) then(1.00) washed(1.00) &(0.99) <>(0.58) <>(0.33) +[L05] in(0.98) lots(0.96) to(1.00) those,(1.00) who(1.00) offer(1.00) most(1.00) for(1.00) at,(0.98) after(1.00) +[L06] having(1.00) <>(0.39) <<&>>(0.34) the(1.00) <>(0.63) as(1.00) they(1.00) call(1.00) it.(1.00) +[L07] This(1.00) blacktin(0.94) they(1.00) burn,(0.97) if(1.00) a(1.00) great(1.00) +[L08] <>(0.19) <>(0.49) of(1.00) <>(0.60) is(1.00) contained(0.99) in(1.00) the(1.00) +[L09] dresod(0.73) black(1.00) tin,(0.99) in(1.00) order(1.00) to(1.00) expel(0.80) the(0.99) +[L10] <>(0.56) &(1.00) sulphureous(0.90) particles,(0.92) <>(0.68) it,(0.89) +[L11] &(0.86) lastly(0.76) they(1.00) <>(0.54) it(1.00) with(1.00) <>(0.66) &(1.00) <>(0.68) <<1st>>(0.54) it(1.00) +[L12] in(0.79) a(1.00) furnace,(0.99) to(1.00) white(0.96) tin;(0.71) which(1.00) is(1.00) made(1.00) into(0.98) +[L13] <>(0.46) of(0.99) 300(0.98) weight,(0.97) <>(0.69) &(1.00) stamped.(0.78) +[L14] In(1.00) the(1.00) great(1.00) Gauze-Moor,(0.76) between(1.00) Bodwyn(0.71) +[L15] <>(0.57) +[L16] &(0.99) Michael(0.96) is(1.00) a(1.00) great(1.00) number(1.00) of(1.00) <>(0.63) +[L17] to(0.94) the(1.00) amount(1.00) of(1.00) <<58,>>(0.66) now(1.00) actually(0.82) working(1.00) +[L18] The(0.92) minors,(0.75) contrive(0.95) several(1.00) method,(0.99) for(1.00) to(0.92) +[L19] <>(0.24) +[L20] use(1.00) <>(0.64) of(1.00) this(1.00) <>(0.51) as(1.00) streams(0.96) +[L21] <>(0.46) washing(0.99) their(1.00) Tenstuff,(0.71) &(0.95) likewise(0.99) to(1.00) +[L22] work(0.89) with(1.00) it(1.00) some(1.00) pumps(0.97) to(1.00) clear(1.00) the(1.00) +[L23] places(0.92) under(1.00) ground(1.00) from(1.00) water,(1.00) where(1.00) +[L24] They(0.97) are(1.00) at(1.00) work.(1.00) In(1.00) these(1.00) Streamworks(0.76) +[L25] a(1.00) kind(1.00) of(1.00) white(0.99) Tinspar(0.88) is(1.00) found(1.00) which(1.00) +[L26] the(1.00) miners(0.99) call(1.00) <>(0.60) &(1.00) which(1.00) <>(0.38) +[L27] pure(0.97) +[L28] a(1.00) King(0.80) of(1.00) Jin-(0.74) +[L29] In(1.00) the(1.00) common(0.99) load(0.79) of.(0.72) Tin,(0.86) there(1.00) are(1.00) sometime(0.92) +[L30] <>(0.46) <>(0.58) <>(0.46) +[L31] brown,(0.92) blueish(0.99) black(1.00) <>(0.51) <>(0.43) <>(0.49) by(1.00) the(1.00) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P052.txt b/app/deploy/book1_data/moredata/confidence/B1_P052.txt new file mode 100644 index 0000000000000000000000000000000000000000..babad0332565b43dfb6cfe9d6279cbe2d58e6028 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P052.txt @@ -0,0 +1,35 @@ +=== Buch 1, Seite 052 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 27 von 207 + +[L01] Cornish(0.99) <>(0.69) Black(0.96) Jack,(0.98) &(0.99) is(1.00) nothing(1.00) but(1.00) what(1.00) +[L02] the(0.99) German(1.00) mineralogists(0.89) call(0.99) Wolfram(0.87) or(1.00) <>(0.30) +[L03] lupi(0.79) <<#>>(0.65) or(0.80) a(1.00) mixture(0.95) of(1.00) Iron,(0.95) <>(0.64) &(1.00) manganese;(0.97) +[L04] &(0.97) it(1.00) must(1.00) therefore(0.99) carefully(0.95) be(1.00) distinguished(0.85) from(1.00) +[L05] the(1.00) <>(0.65) called(1.00) BlackJack(0.83) by(1.00) the(1.00) Shropshire(0.82) +[L06] Welsh(0.93) &(0.99) Derbyshire(0.82) miners,(0.99) which(1.00) is(1.00) a(1.00) <>(0.54) +[L07] blend.(1.00) Having(1.00) examined(0.90) these(1.00) mines,(0.99) I(0.97) thought(0.98) +[L08] myself(0.99) pretty(0.98) well(1.00) instructed(1.00) of(0.97) the(1.00) <>(0.67) <<&>>(0.51) +[L09] returned(1.00) first(0.78) to(1.00) <>(0.62) &(1.00) then(1.00) to(0.99) Bodmyn.(0.93) +[L10] July(1.00) <>(0.55) <<7.74>>(0.65) +[L11] We(1.00) saw(1.00) the(1.00) races(0.98) at(1.00) 13odmyn,(0.87) four(1.00) horses(0.96) <>(0.18) +[L12] ted,(0.99) once(0.95) was(1.00) distanceed;(0.80) &(1.00) the(1.00) second(0.98) heat(0.97) <>(0.47) +[L13] prize(1.00) was(1.00) got(1.00) by(1.00) <>(0.52) +[L14] July,(0.94) 8th(0.77) <>(0.53) took(1.00) leave(1.00) at(0.71) <>(0.61) &(1.00) Cornwall,(0.95) <>(0.67) +[L15] ned(0.97) to(1.00) Plymouth(0.99) &(1.00) found(1.00) all(1.00) things(1.00) on(1.00) board(1.00) in(0.99) +[L16] <>(0.69) for(0.84) me(1.00) done,(0.99) +[L17] <<.>>(0.12) <<4.91.>>(0.52) +[L18] 4e(0.85) +[L19] &(0.82) came(1.00) on(1.00) board,(1.00) but(1.00) the(1.00) Ship(1.00) was(0.80) all(0.99) in(1.00) +[L20] a(1.00) confusion,(0.98) &(0.99) I(1.00) could(1.00) range(1.00) nothing(1.00) in(1.00) +[L21] my(1.00) &(0.96) my(1.00) Sons(0.97) Cabin.(0.76) +[L22] <<4th>>(0.56) +[L23] The(0.98) Ships(0.95) company(1.00) was(1.00) paid(1.00) <<&>>(0.70) the(1.00) +[L24] whole(0.99) Ship(0.97) a(1.00) whole(0.99) scene(0.98) of(1.00) drunk(0.93) in(0.99) <>(0.30) +[L25] confusing,(0.75) &(1.00) a(1.00) regular(1.00) fair(1.00) was(1.00) held(1.00) on(1.00) +[L26] <>(0.63) of(1.00) it,(1.00) no(1.00) <>(0.35) being(1.00) more(1.00) permit(1.00) +[L27] Fed(0.78) to(1.00) go(1.00) on(1.00) Shore.(0.97) +[L28] <<->>(0.08) ye(0.94) <<12.>>(0.63) +[L29] 16(0.81) +[L30] I(0.99) came(1.00) on(1.00) board(1.00) &(1.00) began(1.00) to(1.00) settle(1.00) my(1.00) +[L31] things,(1.00) at(1.00) least(1.00) so(1.00) much,(1.00) that(1.00) I(1.00) could(1.00) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P053.txt b/app/deploy/book1_data/moredata/confidence/B1_P053.txt new file mode 100644 index 0000000000000000000000000000000000000000..c891e91d3b9af2b7e79ad43fe0954448a39a4531 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P053.txt @@ -0,0 +1,30 @@ +=== Buch 1, Seite 053 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 15 von 176 + +[L01] sleep(0.99) on(1.00) board(1.00) the(1.00) Ship:(0.99) which(1.00) however,(1.00) was(1.00) +[L02] very(1.00) difficult(1.00) on(1.00) <>(0.51) of(1.00) the(1.00) same(1.00) <>(0.39) +[L03] <>(0.58) <>(0.64) +[L04] July(1.00) <<4/2°>>(0.59) when(0.73) I(1.00) was(1.00) walking(1.00) on(1.00) the(1.00) Quarter-(0.93) +[L05] <>(0.50) I(1.00) found,(0.75) that(1.00) the(1.00) Ship(1.00) altered(0.98) it.(0.72) +[L06] <>(0.05) +[L07] Situation(0.85) in(1.00) regard(1.00) to(1.00) the(1.00) Adventure(1.00) &(0.98) +[L08] a(1.00) <>(0.55) Ship,(0.94) that(1.00) were(1.00) at(1.00) anchor(0.88) in(1.00) +[L09] the(1.00) Sound,(0.98) &(1.00) that(1.00) we(1.00) came(1.00) nearer(0.99) &(1.00) nearly(0.82) +[L10] to(1.00) the(1.00) Garrison(0.85) or(1.00) castle,(0.97) The(1.00) Master(1.00) who(1.00) +[L11] walked(0.83) along(1.00) with(1.00) me(1.00) on(1.00) the(1.00) Quarterdeck,(0.86) +[L12] began,(1.00) to(1.00) be(1.00) allarmed.(0.94) Our(1.00) Ship(1.00) was(1.00) faste-(0.98) +[L13] ned(0.99) to(0.99) one(1.00) of(1.00) the(0.99) boys(0.99) in(1.00) the(1.00) harbour;(0.83) +[L14] We(0.79) found(1.00) nozo,(0.87) that(1.00) the(1.00) <>(0.55) by(1.00) which(0.99) +[L15] the(0.98) boy(0.98) was(1.00) +[L16] fastened(0.77) was(1.00) broke(1.00) &(0.99) that(1.00) the(1.00) Ship(1.00) +[L17] was(1.00) adrift(0.76) along(1.00) with(1.00) the(1.00) way.(0.71) In(1.00) a(0.99) +[L18] <<&>>(0.35) moment(0.99) all(1.00) lands(0.81) were(1.00) on(1.00) deck,(0.98) the(1.00) +[L19] Sails(0.94) spread,(0.95) the(1.00) <>(0.57) cleared(0.99) &(0.99) +[L20] kept(0.74) ready(1.00) to(1.00) +[L21] <>(0.64) us(0.95) from(1.00) being(1.00) +[L22] <>(0.50) by(1.00) the(1.00) tide(0.94) against(1.00) the(1.00) <>(0.18) +[L23] rocks(0.85) of(1.00) the(1.00) castle,(0.97) &(1.00) stove(0.92) to(1.00) pieces.(0.94) +[L24] Our.(1.00) Captain(1.00) &(1.00) all(1.00) the(1.00) Officers(0.83) were(0.84) ex-(0.99) +[L25] tremely(0.76) busy(0.72) &(0.99) gave(1.00) such(0.98) orders,(1.00) that(1.00) +[L26] just,(0.94) when(1.00) the(1.00) Ship(1.00) turned,(0.89) <>(0.66) happily(1.00) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P056.txt b/app/deploy/book1_data/moredata/confidence/B1_P056.txt new file mode 100644 index 0000000000000000000000000000000000000000..5e1e0ebea3193b0169733c9d9a327f14efaf064f --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P056.txt @@ -0,0 +1,37 @@ +=== Buch 1, Seite 056 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 36 von 139 + +[L01] <<25>>(0.58) +[L02] passed(0.98) between(1.00) two(1.00) rocks(0.98) under(1.00) water,(1.00) &(1.00) +[L03] by(1.00) the(1.00) same(1.00) manoeuvre(0.74) the(1.00) Ship(0.90) <>(0.68) +[L04] the(1.00) Adventure,(1.00) the(1.00) Mast(0.94) Ship,(1.00) &(1.00) the(1.00) Isle(1.00) of(1.00) +[L05] Nicholas;(0.80) &(1.00) there(1.00) are(0.80) dropt(0.91) our(0.98) <>(0.59) he-(0.99) +[L06] ving(0.99) narrowly(1.00) escaped,(0.99) the(1.00) most(1.00) <>(0.44) +[L07] danger.(1.00) +[L08] Journal(0.99) of(1.00) a(1.00) Voyage(1.00) on(1.00) +[L09] board(0.99) her(0.74) Majesty's(0.86) Ship(1.00) the(0.91) Resolution(0.87) +[L10] <>(0.41) +[L11] Captain(1.00) <>(0.59) Commander(1.00) +[L12] at(0.85) Palliser(0.98) +[L13] <>(0.27) Mr.(0.72) Cooper(0.83) +[L14] <<23>>(0.29) <>(0.52) Mr(0.73) Charles(0.97) Clench.(0.83) +[L15] <>(0.57) +[L16] <<3°>>(0.36) <>(0.70) Mr(0.99) <>(0.67) +[L17] <>(0.47) +[L18] John(0.96) +[L19] <>(0.61) +[L20] John(0.99) +[L21] Master.(1.00) <>(0.38) Gilbert.(0.92) <>(0.54) <>(0.46) Whitehouse,(0.88) <>(0.35) &(0.96) +[L22] <>(0.45) Smith.(0.97) +[L23] <>(0.64) Messom.(0.77) <>(0.68) <>(0.70) <>(0.60) Logie,(0.74) Symed,(0.91) +[L24] Roberts,(0.99) Elliot,(0.99) Mitchell,(0.80) Maxwell,(0.99) Van(0.90) <>(0.67) Harvey(0.94) +[L25] <>(0.42) Coughlan,(0.91) <>(0.53) Burney.(0.97) +[L26] Lieut.(0.97) of(1.00) the(1.00) Marines.(0.96) Mr(0.85) <>(0.64) +[L27] Surgeon(0.84) M.(0.97) Patton.(0.98) +[L28] ronomer(0.74) Mr(0.78) Wales.(0.95) +[L29] Painter(0.96) Mr.(0.98) <>(0.59) +[L30] For(0.98) the(1.00) Natural(1.00) History(1.00) Department,(0.95) <<&>>(0.32) my(0.99) Son.(0.79) +[L31] <>(0.63) +[L32] <>(0.32) <>(0.49) Dawson(0.91) Capt.(0.90) <>(0.50) Mr.(0.96) Church(0.82) +[L33] Surgeons(0.78) <>(0.47) M.(0.95) Anderson(0.99) &(1.00) <>(0.60) Drawwater.(0.74) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P057.txt b/app/deploy/book1_data/moredata/confidence/B1_P057.txt new file mode 100644 index 0000000000000000000000000000000000000000..7ed76041c59d301de28e6199349222171dc8ac7a --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P057.txt @@ -0,0 +1,30 @@ +=== Buch 1, Seite 057 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 0 von 175 + +[L01] 26(1.00) +[L02] 13th(0.99) +[L03] July(1.00) ye(0.98) 13(0.99) +[L04] We(1.00) weighed(1.00) anchor(1.00) &(1.00) left(0.99) Plymouth(1.00) +[L05] Pound(0.97) in(0.98) company(0.99) with(1.00) the(0.99) Adventure(1.00) at(1.00) 4(1.00) o'(0.99) +[L06] clock(1.00) in(1.00) the(1.00) morning.(1.00) The(1.00) Adventure(1.00) Sloop(0.99) +[L07] was(1.00) of(1.00) 380(0.99) Tuns,(0.99) Capt.(1.00) Turneaux,&(0.96) under(1.00) him(1.00) +[L08] Lieut.(0.99) Shanks(0.98) &(1.00) Kempe.(0.99) The(1.00) Astronomer(0.97) who(1.00) +[L09] went(1.00) on(0.99) board(1.00) of(1.00) her(1.00) was(1.00) Mr(1.00) Bailey.(1.00) Our(1.00) Ship(1.00) +[L10] was(1.00) of(1.00) 469(0.97) Tuns(0.99) both(0.99) well(1.00) rigged,(1.00) stored(0.98) &(0.99) man-(0.98) +[L11] ned(0.99) &(1.00) in(1.00) all(1.00) appearance(1.00) the(1.00) best(1.00) matched(0.99) +[L12] Ships(1.00) for(0.99) going(1.00) on(1.00) an(1.00) Expedition,(1.00) which(1.00) +[L13] must(1.00) of(1.00) course(1.00) require(1.00) the(1.00) greatest(0.99) attention(0.99) +[L14] as(1.00) their(1.00) success(1.00) depended(0.98) greatly(1.00) on(1.00) it,(1.00) +[L15] &(1.00) by(0.99) all(1.00) probability(0.99) would(1.00) become(1.00) of(1.00) great(1.00) +[L16] importance,(1.00) to(0.99) Navigation,(1.00) Geography(1.00) +[L17] astronomy,(0.99) &(1.00) all(1.00) the(1.00) other(1.00) branches(0.99) +[L18] of(1.00) Science.(0.98) The(1.00) Lords(1.00) of(1.00) the(1.00) Admiralty(1.00) &(1.00) +[L19] Lord(0.99) Sandwich(0.99) +[L20] especially(1.00) ,(0.85) had(0.94) paid(0.99) a(1.00) particular(1.00) attention(1.00) +[L21] to(1.00) the(1.00) equipment(1.00) of(1.00) these(1.00) ships(0.98) &(1.00) especially(1.00) +[L22] to(1.00) everything(1.00) that(1.00) bade(0.99) fair(0.98) to(1.00) promote(0.99) +[L23] the(1.00) success(0.97) of(1.00) the(1.00) Voyage(1.00) &(1.00) make(1.00) the(1.00) +[L24] whole(0.99) Expedition(0.99) as(1.00) compleat(0.99) as(1.00) it(1.00) could(0.99) +[L25] be(1.00) wished.(1.00) As(1.00) Mr(0.99) Banks(0.98) &(1.00) Dr(0.97) Solander(0.99) had(1.00) decli-(0.99) +[L26] ned(0.99) going(1.00) on(1.00) the(1.00) Expedition,(1.00) &(1.00) the(1.00) branches(0.99) of(1.00) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P060.txt b/app/deploy/book1_data/moredata/confidence/B1_P060.txt new file mode 100644 index 0000000000000000000000000000000000000000..963233afebac5476a7cf6ae1be52850baee271f8 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P060.txt @@ -0,0 +1,31 @@ +=== Buch 1, Seite 060 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 20 von 183 + +[L01] Natural(0.97) History(1.00) &(1.00) the(1.00) drawings(0.99) of(1.00) the(1.00) various(1.00) +[L02] new(1.00) objects(1.00) which(0.95) might(0.96) offer(1.00) on(1.00) the(1.00) course(1.00) +[L03] of(0.99) this(1.00) Voyage(0.88) remained(1.00) unoccupied(0.96) Lord(1.00) Sandwich(0.98) +[L04] appointed(0.91) <>(0.59) for(1.00) the(1.00) one(1.00) with(1.00) my(0.98) Son;(0.95) &(0.99) +[L05] a(1.00) few(1.00) days(1.00) before(1.00) the(1.00) Ship(1.00) <>(0.69) Mr.(0.74) Hodges(0.92) an(0.98) +[L06] ingenious,(0.75) Artilt(0.86) was(1.00) sent(1.00) to(1.00) make(1.00) up(1.00) the(1.00) +[L07] <>(0.43) +[L08] deficiency(0.97) in(1.00) the(1.00) branch.(0.93) <>(0.66) +[L09] When(0.96) we(1.00) just(0.79) had(1.00) left(0.97) Plymouth-Sound,(0.84) I(1.00) +[L10] <>(0.65) &(1.00) his(1.00) friend(1.00) 100(0.93) Jones(0.85) followed(1.00) us(0.98) in(1.00) +[L11] a(1.00) small(1.00) boat.(0.98) They(1.00) were(0.98) making(1.00) a(1.00) li(0.89) +[L12] four(0.92) into(1.00) <>(0.50) as(1.00) far(1.00) as(1.00) <>(0.61) <<&.S.>>(0.54) Aust.(0.71) +[L13] &(1.00) were(1.00) going(1.00) to(1.00) the(1.00) first(1.00) place(1.00) by(1.00) Sea,(1.00) &(1.00) +[L14] wanted(1.00) to(1.00) take(1.00) leave(1.00) of(1.00) us:(0.76) they(0.99) came(1.00) on(1.00) +[L15] board(0.98) of(0.99) our(1.00) Ship(1.00) &(1.00) after(1.00) having(1.00) <>(0.64) +[L16] <>(0.32) +[L17] <>(0.51) +[L18] with(1.00) us,(1.00) they(1.00) took(1.00) leave(1.00) of(1.00) us(0.99) &(1.00) +[L19] <>(0.36) <>(0.18) from,(0.77) &(1.00) we(1.00) directed(0.99) our(1.00) +[L20] under(1.00) &(0.91) <>(0.66) W.(0.96) <>(0.65) +[L21] course(0.95) into(1.00) the(1.00) Shanhel.(0.71) At(1.00) 8(1.00) o'clock(0.93) +[L22] the(1.00) Ramhead(0.97) bore(0.78) North,(1.00) distant(0.99) 2(0.98) miles.(1.00) +[L23] at(0.90) 12(0.99) o'clock(1.00) at(1.00) noon(1.00) the(1.00) Latitude(0.99) was(0.96) +[L24] observed(1.00) 50°(0.96) <>(0.66) the(0.88) Ramhead(0.94) bore(0.79) <>(0.66) <<34>>(0.41) +[L25] distant(0.80) +[L26] leagues.(0.96) We(1.00) passed(1.00) <>(0.49) Stone(0.79) light(0.99) +[L27] house,(1.00) it(1.00) was(1.00) about(1.00) a(1.00) <>(0.23) to(1.00) reward(0.87) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P061.txt b/app/deploy/book1_data/moredata/confidence/B1_P061.txt new file mode 100644 index 0000000000000000000000000000000000000000..55faeee15cad7a9ad64c0c206d1801b44f068ac0 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P061.txt @@ -0,0 +1,28 @@ +=== Buch 1, Seite 061 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 17 von 187 + +[L01] 28(0.98) +[L02] July(0.99) <<14°>>(0.47) <<)>>(0.68) The(1.00) Ships(0.96) remained(0.98) in(1.00) <>(0.69) +[L03] the(1.00) weather(0.85) continued(0.93) fair,(0.99) the(1.00) wind(1.00) changed(1.00) +[L04] to(0.96) W.H.W.(0.90) &(1.00) sometimes(1.00) W.(1.00) 6.(0.82) N.N.(0.81) <>(0.58) course(0.99) was(1.00) <>(0.50) <<4>>(0.39) +[L05] S.W.b.S.(0.87) In(0.99) the(1.00) Evening(0.99) at(1.00) 9(0.83) o'clock(1.00) we(1.00) bore(0.91) +[L06] down(0.99) to(1.00) the(1.00) Adventure.(0.97) In(1.00) the(1.00) night(1.00) the(1.00) +[L07] weather(1.00) became(0.99) cloudy(0.99) &(0.99) towards(1.00) morning(1.00) +[L08] <>(0.50) with(1.00) some(1.00) rain(1.00) at(0.98) 8(1.00) in(1.00) the(1.00) morning(1.00) +[L09] we(1.00) fired(0.96) a(1.00) gun(1.00) for(1.00) the(1.00) Adventure,(1.00) after(1.00) +[L10] having(1.00) racked,(0.75) directing(0.71) our(1.00) Course(0.96) West(0.82) +[L11] S.S.(0.89) W.(0.79) SWW.(0.70) <>(0.52) towards(1.00) noon(0.98) it(1.00) grew(0.99) foggy(0.91) +[L12] &(1.00) only(1.00) light(1.00) airs.(0.86) +[L13] July(0.99) of(0.89) 15th(0.99) We(0.97) met(1.00) with(1.00) a(1.00) <>(0.63) Swell,(0.94) from(0.96) +[L14] the(1.00) N.W.(0.92) &(0.99) towards(1.00) night(1.00) a(0.99) thick(0.99) fog(0.97) <>(0.53) me(0.94) on(1.00) +[L15] we(0.99) soon(1.00) found(1.00) that(1.00) we(1.00) were(1.00) approaching(0.94) +[L16] the(1.00) Bay(0.88) of(1.00) Biscay,(0.96) by(1.00) the(1.00) great(1.00) waves.(0.97) I(1.00) had(1.00) +[L17] been(1.00) seasick(0.81) ever(0.99) since(1.00) the(1.00) first(1.00) day(1.00) of(0.99) one(0.86) +[L18] Voyage,(0.88) could(1.00) eat(1.00) very(1.00) little,(1.00) was(1.00) obliged(1.00) to(1.00) +[L19] <>(0.61) up(1.00) <>(0.52) Virtuals,(0.80) had(1.00) no(1.00) head.each(0.82) +[L20] I(0.98) found(1.00) <>(0.22) much(1.00) better(1.00) &(1.00) greatly(0.97) +[L21] relieved(0.81) by(1.00) some(1.00) mylled(0.95) <>(0.67) +[L22] We(1.00) tached(0.87) on(1.00) <<&>>(0.51) off(1.00) till(0.96) +[L23] July(0.76) <<18,>>(0.66) when(0.99) we(1.00) began(1.00) to(1.00) make(1.00) our(0.99) course(1.00) directly(0.98) +[L24] W.b.(0.85) W.(0.99) <>(0.58) some(1.00) squalls(0.74) &(1.00) now(1.00) &(1.00) then(1.00) rain(0.98) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P064.txt b/app/deploy/book1_data/moredata/confidence/B1_P064.txt new file mode 100644 index 0000000000000000000000000000000000000000..aed418afe5a58e68e79a763b78dae85e960df983 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P064.txt @@ -0,0 +1,32 @@ +=== Buch 1, Seite 064 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 32 von 193 + +[L01] July(0.78) +[L02] <>(0.57) +[L03] 20(1.00) at(1.00) half(1.00) past(0.99) 8(0.81) o'clock(0.92) <>(0.66) Cape(1.00) <>(0.64) or(1.00) as(0.99) +[L04] it(0.97) should(1.00) be(1.00) called(1.00) Ortequera(0.75) bearing(0.78) South(1.00) +[L05] For(0.97) 6(1.00) Leagues(0.73) distant,(0.71) at(1.00) no(0.73) on(1.00) the(1.00) <>(0.49) <>(0.68) Lope(0.88) +[L06] bore(0.98) South(1.00) about(1.00) 2(1.00) or(1.00) 3(1.00) leagues(0.93) distant(0.99) +[L07] The(1.00) latitude(0.92) was(1.00) 43°(0.98) <<55.>>(0.49) No.(0.91) The(1.00) Country(0.99) +[L08] was(1.00) very(1.00) hilly,(0.71) <>(0.67) with(1.00) small(0.99) wood(0.98) &(1.00) +[L09] I(0.92) believe,(0.95) <>(0.60) &(1.00) we(1.00) saw(1.00) several(1.00) pieces(1.00) +[L10] of(1.00) corn,(1.00) in(1.00) all(1.00) appearance(1.00) <>(0.36) +[L11] 21.(0.82) We(1.00) saw(1.00) Cape(1.00) Prior(0.86) at(1.00) 2(0.94) or(0.99) 3(0.97) miles(0.84) distance(1.00) +[L12] bearing(0.92) S.(0.82) W.(0.88) <>(0.65) weather(1.00) <>(0.16) &(0.99) sometimes(0.99) +[L13] rainly.(0.88) at(0.99) noon(1.00) it(1.00) was(1.00) cloudy,(0.94) however(1.00) had(1.00) +[L14] an(1.00) Observation(0.97) of(1.00) the(1.00) <>(0.56) 43°(0.87) <>(0.58) +[L15] Groyn(0.82) or(1.00) Corunna(0.97) SSE.(0.96) &(0.88) <<2>>(0.43) <>(0.48) 3(1.00) Leagues(0.77) +[L16] <>(0.58) Ortegal(0.77) East.(1.00) 12(0.98) or(1.00) 14(1.00) leagues(0.75) distant(0.99) +[L17] <>(0.26) <>(0.66) +[L18] We(1.00) saw(0.99) Groyn(0.86) lighthouse.(0.79) S.E.b.S.(0.94) at(1.00) 5(0.94) tea-(0.80) +[L19] <>(0.33) distance,(1.00) sounded(0.97) 87°.(0.80) <>(0.58) We(0.95) went(0.84) +[L20] at(1.00) I(0.90) o'clock(0.84) <>(0.69) board(1.00) a(1.00) small(1.00) Tartane(0.94) +[L21] which(1.00) we(1.00) believed(1.00) to(1.00) be(1.00) a(1.00) Spanish(1.00) +[L22] fishing(0.96) Vessel(0.79) in(1.00) hopes(1.00) to(1.00) get(1.00) fish,(1.00) <>(0.27) +[L23] <>(0.59) it(0.93) to(1.00) be(1.00) a(1.00) Turtan(0.73) from(1.00) Marseille(0.99) +[L24] of(1.00) about(1.00) <<100>>(0.53) Tuns,(0.94) freighted(0.78) for(0.76) <>(0.39) +[L25] by(0.74) Corunna(0.89) with(1.00) <>(0.69) We(1.00) bought(0.92) +[L26] some(1.00) Frontignac,(0.86) at(0.98) <<1/2°>>(0.63) <>(0.35) +[L27] <>(0.14) +[L28] <>(0.65) of(1.00) 50(1.00) bottles,(0.94) <<&>>(0.21) <>(0.34) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P065.txt b/app/deploy/book1_data/moredata/confidence/B1_P065.txt new file mode 100644 index 0000000000000000000000000000000000000000..c90330b452cddf2167e7281089cf3f48801173bf --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P065.txt @@ -0,0 +1,29 @@ +=== Buch 1, Seite 065 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 23 von 169 + +[L01] of(0.91) Syrup(0.93) of(1.00) Orgeat(0.94) at(1.00) the(1.00) same(1.00) price(1.00) +[L02] reuseved(0.82) the(1.00) Tartan(0.74) with(1.00) water,(0.88) which(1.00) they(1.00) +[L03] stood(0.98) in(1.00) great(1.00) need(1.00) of,(0.96) having(1.00) been(0.98) <>(0.62) +[L04] <>(0.27) for(1.00) 2(1.00) months(1.00) to(0.99) beat(0.93) to(1.00) <>(0.63) +[L05] The(1.00) Variation(0.91) from(1.00) Azimull(0.97) 23°(0.76) <<58°>>(0.56) <>(0.37) <>(0.67) +[L06] 4(0.84) or(1.00) 5(1.00) <>(0.66) distant.(1.00) The(1.00) Country(0.99) appeared(0.99) +[L07] <>(0.56) billy,(0.74) The(1.00) tops(0.84) of(1.00) the(1.00) hills(0.99) were,(0.85) white(1.00) +[L08] from.(0.72) Sand(0.94) or(1.00) Limestone(0.98) &(0.99) had(1.00) a(1.00) barren(0.99) +[L09] <>(0.62) appearance;(0.98) in(1.00) the(1.00) lower(0.99) part(0.99) +[L10] July(1.00) 4(0.85) +[L11] we(1.00) saw(1.00) cornfields(0.97) &(0.83) inclosures(0.76) &(1.00) some(1.00) +[L12] small(1.00) hamlets,(1.00) together(1.00) with(1.00) a(1.00) well(1.00) +[L13] looking(0.99) house(1.00) &(1.00) some(0.72) <>(0.67) +[L14] like(1.00) a(1.00) gentleman's(0.97) seat.(0.89) +[L15] <<9>>(0.37) +[L16] .(0.83) <>(0.08) three(1.00) Spanish(1.00) Men(1.00) of(1.00) war,(1.00) the(1.00) +[L17] first(1.00) a(1.00) 60(0.73) <>(0.13) the(1.00) second(0.99) a(1.00) 74(0.98) gun(0.82) Ship(0.72) +[L18] the(1.00) last(1.00) which(1.00) was(1.00) the(1.00) <>(0.41) +[L19] had(0.99) 60(0.91) guns,(0.90) &(1.00) hoisted(0.99) English(1.00) colours(0.88) +[L20] we(0.99) hoisted(0.97) ours,(0.96) the(1.00) <>(0.40) &(1.00) the(1.00) +[L21] Jack;(0.77) The(1.00) Spaniard(0.98) having(1.00) <>(0.50) +[L22] us(0.96) <>(0.59) a(1.00) gun(0.99) to(1.00) leeward,(0.97) <<">>(0.28) a(0.91) +[L23] little(1.00) while(0.75) after(1.00) hoisted(0.98) Spanish(0.99) +[L24] <>(0.67) &(1.00) pen(0.78) <>(0.45) <>(0.66) with(1.00) a(1.00) <>(0.36) white(0.94) +[L25] <>(0.68) Jack(0.93) on(1.00) the(1.00) top(1.00) of(1.00) the(1.00) fore(0.97) mast(0.89) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P068.txt b/app/deploy/book1_data/moredata/confidence/B1_P068.txt new file mode 100644 index 0000000000000000000000000000000000000000..788f64c2cc5090efee85b65c2e4ba6007020177d --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P068.txt @@ -0,0 +1,34 @@ +=== Buch 1, Seite 068 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 17 von 187 + +[L01] after(1.00) +[L02] a(1.00) little(0.99) while(0.98) <<,>>(0.41) he(1.00) fired(0.94) a(1.00) Gun(0.98) at(1.00) the(1.00) Adventure(1.00) +[L03] whose,(0.87) Shot,(0.87) dropt(0.87) some(1.00) where(1.00) near(1.00) the(1.00) Ship(1.00) +[L04] <>(0.53) we(1.00) went(1.00) on(1.00) our(1.00) Course,(0.99) the(1.00) Spanish(1.00) Ship(0.99) +[L05] went(1.00) about(1.00) &(1.00) fired(0.92) another(1.00) Shot(0.84) at(1.00) the(1.00) +[L06] Adventure:(0.97) we(1.00) brought(0.98) to(0.92) &(0.97) the(1.00) Adventure(1.00) +[L07] followed(1.00) our(1.00) example,(0.71) till(1.00) the(1.00) Spaniard(0.99) +[L08] came(1.00) abreast(0.99) of(1.00) her,(1.00) about(1.00) a(1.00) mile(0.98) or(1.00) +[L09] &(0.84) <>(0.66) distance,(1.00) &(1.00) probably(1.00) spoke(0.99) to(1.00) +[L10] her,(1.00) after(1.00) which(1.00) he(1.00) again(0.99) went(1.00) about(1.00) +[L11] m(0.99) +[L12] &(1.00) we(0.99) made(1.00) <>(0.61) At(1.00) 6(1.00) o'clock(0.99) the(1.00) Variation(0.98) +[L13] by(1.00) Ajimuth(0.92) was(1.00) 24(1.00) 50(0.84) +[L14] by(1.00) amplitude-24°(0.78) 39.(0.80) +[L15] West.(0.91) +[L16] at(1.00) 6'(0.97) o'clock(0.77) a.(0.95) m.(0.88) Variation(1.00) <>(0.23) 19(0.75) <<21.>>(0.38) +[L17] July(1.00) 4(0.96) +[L18] <<&>>(0.27) at(0.93) Observed(0.83) 42°(0.99) 18°(0.74) +[L19] Having(0.87) passed(0.98) Cape(1.00) <>(0.67) the(0.99) night(0.99) +[L20] before,(1.00) we(1.00) got(1.00) a(1.00) fair(0.99) N.E.(0.85) wind(1.00) &(0.98) fair(0.99) wea-(0.97) +[L21] ther(0.97) &(1.00) <>(0.37) at(1.00) 6.(0.99) <<&>>(0.45) 7(0.99) knots(0.74) per(1.00) hour.(0.98) +[L22] Latitude(0.94) 40°.(0.88) <<3'>>(0.55) N.(0.93) +[L23] 25(0.99) +[L24] The(1.00) same(1.00) weather(1.00) continued,(0.97) &(1.00) we(1.00) +[L25] <>(0.40) a(1.00) good(1.00) many(1.00) <>(0.41) passing(0.97) +[L26] by(1.00) again(1.00) &(0.92) the(1.00) wind.(1.00) Ever(1.00) since(1.00) we(1.00) left(1.00) +[L27] <>(0.69) Finisterra(0.98) the(1.00) Sea(1.00) <>(0.68) in(1.00) the(1.00) +[L28] night,(1.00) the(1.00) foam(0.99) was(0.99) luminous(0.72) &(0.97) +[L29] in(1.00) it(1.00) appeared(1.00) some(1.00) <>(0.53) amazing(0.97) +[L30] <>(0.57) bright,(1.00) I(1.00) suppose(0.99) this(1.00) not(1.00) to(1.00) be(1.00) caused(0.99) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P069.txt b/app/deploy/book1_data/moredata/confidence/B1_P069.txt new file mode 100644 index 0000000000000000000000000000000000000000..bd7c3e28e86c574c30bc392064a7befc50fd154e --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P069.txt @@ -0,0 +1,33 @@ +=== Buch 1, Seite 069 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 28 von 205 + +[L01] 32(0.96) +[L02] by(1.00) any(1.00) insects(0.84) or(1.00) other(1.00) phosphorescent(0.84) Sub(0.82) +[L03] Stanie,(0.72) from(1.00) dead(0.99) animals(1.00) in(1.00) the(1.00) Seawater(0.94) +[L04] but(1.00) rather(1.00) to(1.00) be(1.00) Electrical,(0.89) from(1.00) the(1.00) +[L05] cutting(0.93) of(1.00) the(1.00) tarred(0.78) Sides(0.99) of(1.00) the(1.00) Ship.(1.00) +[L06] &(1.00) causing(0.95) a(1.00) violent(1.00) motion(1.00) in(1.00) the(1.00) +[L07] water;(0.99) for(1.00) I(0.99) could(1.00) see(1.00) this(1.00) luminous(0.99) foam(0.73) +[L08] &(1.00) the(1.00) <>(0.60) in(1.00) it,(1.00) only(1.00) <>(0.37) the(1.00) Ship(1.00) +[L09] &(1.00) not(1.00) at(1.00) some(1.00) distance;(0.95) but(1.00) I(1.00) will(1.00) <>(0.41) +[L10] tainly(0.95) take(1.00) some(1.00) method(1.00) or(1.00) other(1.00) to(1.00) exa-(0.82) +[L11] mine(0.91) this(1.00) phenomenon(0.98) more(1.00) <>(0.37) +[L12] rely.(0.82) The(1.00) Latitude(0.95) was(1.00) observed(1.00) 37°(0.99) 41(0.98) North(0.94) +[L13] <>(0.25) <>(0.47) <<16>>(0.52) +[L14] <<27>>(0.51) +[L15] The(1.00) same(1.00) weather,(0.91) but(1.00) somewhat(0.99) <>(0.15) +[L16] heavy,(0.86) &(1.00) our(0.94) course(1.00) was(1.00) S.V.(0.72) as(1.00) <>(0.58) +[L17] b(0.76) +[L18] The(1.00) Latitude(0.99) observed(0.99) was(1.00) 35°(0.97) 31(0.95) N.(0.96) +[L19] The(1.00) same(0.99) weather,(1.00) the(1.00) Course(0.94) S.(1.00) W.(0.98) by(0.95) S.(1.00) +[L20] Cat.(0.89) <<&>>(0.43) Observation(0.99) 33°(0.97) 43°.(0.80) +[L21] <<-28th>>(0.55) Course(0.92) <>(0.62) W.(0.85) then(0.98) W.by.(0.97) <<&>>(0.62) then(0.86) W.(1.00) &(0.98) W.S.S.(0.72) <>(0.52) 4.d-(0.75) +[L22] clock.(0.85) the(0.92) Variation(1.00) was(1.00) taken(1.00) by(0.96) the(1.00) <>(0.62) +[L23] <<200>>(0.40) <<23.W.>>(0.46) At(0.96) 6.(1.00) The(1.00) Land(0.97) was(1.00) <>(0.65) ahead(1.00) from(1.00) +[L24] the(1.00) Masthead.(0.81) At(1.00) 8(0.98) o'clock(0.99) A.M.(0.96) <>(0.62) Body(0.97) of(1.00) the(1.00) <>(0.57) +[L25] land(0.99) Portosanto(0.85) <>(0.57) N.(0.87) W.(0.97) by(0.91) N.(0.98) at(1.00) 4(1.00) miles(1.00) distance(1.00) +[L26] <>(0.37) <>(0.23) End,(0.89) North(0.98) <>(0.54) 2(1.00) Congress.(0.73) &(1.00) the(1.00) Deserters.(0.90) +[L27] <>(0.68) +[L28] Porta(0.92) Santo(0.99) is(1.00) a(1.00) <>(0.48) Island(1.00) about(1.00) 6(0.93) leagues(0.77) in(1.00) +[L29] length,(1.00) barren(0.83) &(1.00) very(1.00) thinly(0.96) inhabited,(0.94) <>(0.58) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P072.txt b/app/deploy/book1_data/moredata/confidence/B1_P072.txt new file mode 100644 index 0000000000000000000000000000000000000000..ea461785ff2d8a32fef71d50930de91abc68beb0 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P072.txt @@ -0,0 +1,36 @@ +=== Buch 1, Seite 072 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 31 von 202 + +[L01] belongs(0.91) under(1.00) the(1.00) Government(0.99) of(1.00) Madeira.(1.00) There(0.99) +[L02] is(1.00) a(1.00) small(1.00) town(0.94) in(1.00) this(0.83) Island,(0.82) on(0.84) <>(0.45) SE(0.89) part.(0.98) Round(0.93) +[L03] she(0.94) +[L04] the(1.00) <>(0.63) &(1.00) up(0.74) towards(1.00) North,(1.00) <>(0.62) is(0.97) scale,(0.95) which(1.00) seems(0.85) +[L05] to(1.00) be(1.00) inhabited(0.94) <>(0.55) cultivated(1.00) chiefly(0.99) with(1.00) vineyards,(0.99) +[L06] the(1.00) <>(0.35) being(1.00) all(1.00) a(1.00) barren,(0.97) <>(0.60) Rock.(0.84) +[L07] at(1.00) <>(0.56) the(1.00) <>(0.64) of(1.00) the(1.00) Isle'(0.74) <>(0.38) N.e.e.(0.77) 2.(0.83) The(1.00) +[L08] Brazenhead,(0.92) a(1.00) Landpoint(0.99) in(1.00) Madeira(1.00) inclu-(0.99) +[L09] ding(0.77) the(1.00) road(0.77) of(1.00) Funchal(0.71) on(1.00) its(1.00) East(0.94) side(0.99) +[L10] bore(0.97) W.S(0.94) <>(0.64) &(0.96) leagues(0.76) distance.(1.00) The(1.00) body(1.00) of(1.00) +[L11] the(1.00) <>(0.51) <>(0.47) S.S.M.(0.85) +[L12] <<52'>>(0.36) 49'(0.89) IV.(0.83) +[L13] 29(0.97) +[L14] W.(0.91) The(1.00) <>(0.48) +[L15] <>(0.30) +[L16] We(1.00) had(1.00) very(1.00) Sight(0.90) Ains.(0.88) Saw(0.96) Madeira(1.00) a(1.00) <>(0.21) +[L17] &(0.98) the(1.00) town(1.00) of(1.00) Ste(0.84) <>(0.27) about(0.92) 6(0.91) o'clock.(0.98) +[L18] The(1.00) Isle(1.00) with(0.97) its(1.00) <>(0.52) hills,(0.98) bore(0.98) a(1.00) very(1.00) fine(0.99) +[L19] prospect.(1.00) The(1.00) little(1.00) <>(0.43) which(1.00) <>(0.52) the(1.00) +[L20] hills(1.00) caused(1.00) a(1.00) variety(0.99) which(1.00) beggars(0.80) all(1.00) +[L21] description.(0.77) <>(0.55) Cruz(0.99) lies(1.00) at(1.00) the(1.00) bottom(0.99) of(1.00) a(1.00) +[L22] small(0.98) bay,(1.00) at(1.00) the(1.00) east(0.99) of(1.00) which(1.00) we(0.99) <>(0.37) a(0.98) +[L23] <>(0.63) +[L24] castle.(1.00) The(1.00) vineyards(0.99) with(1.00) some(1.00) +[L25] countryhouses,(0.87) &(1.00) here(1.00) and(0.90) there(0.99) a(1.00) grove(0.90) of(1.00) +[L26] <>(0.57) &(1.00) in(1.00) one(1.00) <>(0.48) some(1.00) <>(0.63) +[L27] trees(0.96) gave(1.00) the(1.00) whole(1.00) +[L28] which(0.93) +[L29] romantic(0.96) appears(0.78) +[L30] We(1.00) <>(0.10) 2(1.00) boats(1.00) out,(1.00) took(1.00) the(1.00) Ship(1.00) <>(0.54) <>(0.68) +[L31] <>(0.61) 10(0.96) o'clock(0.99) at(1.00) night(0.99) to(1.00) an(1.00) <>(0.18) +[L32] win(0.97) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P073.txt b/app/deploy/book1_data/moredata/confidence/B1_P073.txt new file mode 100644 index 0000000000000000000000000000000000000000..14ce384f3ec6076b258179c7c05701d5e04ba957 --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P073.txt @@ -0,0 +1,30 @@ +=== Buch 1, Seite 073 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 33 von 224 + +[L01] <>(0.64) with(1.00) the(1.00) small(1.00) <>(0.64) in(1.00) 25(0.99) <>(0.20) <>(0.67) +[L02] vered(0.87) to(1.00) half(0.98) a(1.00) cable.(1.00) The(1.00) Boazen(0.84) head(0.94) bone(0.95) +[L03] <>(0.34) E.(0.98) The(1.00) Lao(0.80) Rock(0.99) <>(0.49) W.(0.95) by(0.90) W.S.(0.83) <>(0.52) &(0.89) the(1.00) +[L04] westernmost(0.93) point(1.00) of(0.99) the(1.00) <>(0.63) <>(0.59) N.(0.99) The(0.81) Adven(0.88) +[L05] ture(0.76) came(1.00) likewise(0.99) to(0.93) an(0.95) Anchor.(0.95) +[L06] Early(0.92) in(0.91) the(1.00) morning,(1.00) I(1.00) was(1.00) <>(0.41) by(1.00) the(0.99) mor-(0.99) +[L07] ning(1.00) <>(0.48) &,(0.78) <>(0.66) I(1.00) knew(0.98) the(1.00) Ship(1.00) was(1.00) going(0.97) to(0.99) +[L08] salute(0.86) the(1.00) castle,(0.91) I(1.00) got(1.00) up,(1.00) in(1.00) order(1.00) to(1.00) be(1.00) out(0.79) +[L09] of(1.00) the(1.00) way,(1.00) when(1.00) the(1.00) <>(0.67) on(1.00) both(1.00) sides(0.90) of(0.98) +[L10] my(1.00) Cabin(1.00) should(1.00) be(1.00) fired.(0.93) <>(0.47) on(1.00) deck(1.00) +[L11] <<&>>(0.28) <>(0.67) the(1.00) finest(0.99) sight,(0.97) I(1.00) ever(1.00) met(1.00) with(1.00) in(1.00) +[L12] the(1.00) course(1.00) of(1.00) my(1.00) various(1.00) travel.(0.88) The(1.00) town(0.76) +[L13] <>(0.59) <>(0.65) is(0.79) built(1.00) round(1.00) the(1.00) bay(0.71) in(0.99) form(1.00) +[L14] of(1.00) an(1.00) Amphiteatre(0.77) on(1.00) the(1.00) slope(0.72) of(1.00) the(1.00) +[L15] hills,(1.00) <>(0.19) figure(1.00) sets(0.99) all(1.00) the(1.00) buildings(0.96) of(1.00) +[L16] the(1.00) <>(0.50) off(1.00) to(1.00) great(1.00) advantage;(0.86) which(1.00) +[L17] are(1.00) all(1.00) white,(1.00) with(1.00) low(1.00) roofs,(0.95) &(1.00) by(1.00) this(1.00) <>(0.56) +[L18] <>(0.59) a(1.00) kind(1.00) of(1.00) neatness(0.86) &(1.00) elegant,(0.99) far(0.97) +[L19] Beyond(0.80) <>(0.09) +[L20] <>(0.57) any(1.00) description.(1.00) <>(0.62) town(1.00) +[L21] the(1.00) hills(0.99) rise(0.98) bolder(0.76) &(1.00) bolder,(0.92) till(1.00) at(1.00) last(1.00) +[L22] some(0.99) are(1.00) <>(0.37) by(1.00) cloud.(0.80) The(1.00) <>(0.36) <>(0.67) +[L23] are(1.00) clad(0.95) in(1.00) a(1.00) most(0.98) <>(0.33) <>(0.67) caused(0.99) +[L24] by(1.00) the(1.00) wines,(0.79) that(1.00) occupy(0.98) all(1.00) the(1.00) space(0.98) of(1.00) ground(0.98) +[L25] <>(0.60) to(1.00) the(1.00) very(1.00) summits,(0.95) that(1.00) are(1.00) involved(0.99) +[L26] in(1.00) clouds,(0.97) the(1.00) hedges(1.00) are(1.00) of(1.00) <>(0.60) <>(0.55) green(1.00) &(1.00) here(0.85) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/confidence/B1_P074.txt b/app/deploy/book1_data/moredata/confidence/B1_P074.txt new file mode 100644 index 0000000000000000000000000000000000000000..56dd5cc45d7171fccea2693f5ded0552dee54b2d --- /dev/null +++ b/app/deploy/book1_data/moredata/confidence/B1_P074.txt @@ -0,0 +1,36 @@ +=== Buch 1, Seite 074 === +SCHWELLE: 0.7 +UNSICHERE_WÖRTER: 23 von 228 + +[L01] appearance(1.00) +[L02] <<35>>(0.70) +[L03] there(1.00) a(1.00) cluster(0.96) of(1.00) trees(0.98) &(0.99) the(1.00) <>(0.48) a(1.00) white(1.00) +[L04] <>(0.46) <>(0.53) of(1.00) a(1.00) church(0.99) between(1.00) these(0.99) <>(0.43) +[L05] yards,(0.85) gave(0.98) to(0.99) the(1.00) whole(1.00) a(1.00) sight(1.00) similar(1.00) to(1.00) +[L06] what(1.00) we(1.00) often(1.00) are(1.00) told,(0.99) of(1.00) the(1.00) gardens(0.86) of(1.00) fai-(0.92) +[L07] ries(1.00) &(1.00) the(1.00) hanging(1.00) Gardens(0.88) of(1.00) Queen(1.00) Semirame(0.72) +[L08] The(1.00) westernmost(0.82) point(0.96) of(0.99) the(1.00) harbour(1.00) has(1.00) on(0.99) +[L09] a(1.00) rock(0.98) surrounded(0.99) by(1.00) the(1.00) Sea,(0.92) Sea(0.98) a(1.00) Castle,(0.95) which(1.00) +[L10] had(1.00) +[L11] returned(0.93) the(1.00) Salute,(0.98) we(1.00) given(1.00) with(1.00) <<14>>(0.36) <>(0.65) +[L12] All(0.98) the(1.00) part(1.00) of(1.00) the(1.00) town(1.00) along(1.00) the(1.00) Shore(0.99) <>(0.56) for-(1.00) +[L13] tified(0.99) &(1.00) surrounded(1.00) with(1.00) guns.(1.00) Besides(0.99) the(1.00) <>(0.24) +[L14] already(0.78) mentioned(0.80) there(1.00) is(1.00) another(1.00) which(0.99) +[L15] commands(0.88) the(1.00) town(0.99) &(1.00) is(1.00) called(1.00) St(0.95) <>(0.57) +[L16] <>(0.69) +[L17] About(0.97) 7(1.00) o'clock(0.97) there(1.00) came(1.00) a(1.00) boat(1.00) on(1.00) +[L18] board,(0.91) with(1.00) a(1.00) Captain(1.00) of(0.89) the(1.00) Hall(1.00) &(1.00) an(1.00) Inter(1.00) +[L19] preter,(1.00) to(1.00) ensure(0.90) into(1.00) the(1.00) state(0.72) of(1.00) the(1.00) +[L20] health(1.00) of(1.00) our(1.00) Inips-company(0.75) &(1.00) whether(1.00) we(1.00) +[L21] came(1.00) from(0.96) any(1.00) part(1.00) of(1.00) <>(0.58) the(1.00) Arches(0.75) +[L22] &(1.00) other(0.87) places,(1.00) that(1.00) are(1.00) suspected(0.80) to(1.00) be(1.00) in-(0.99) +[L23] <>(0.57) with(1.00) the(0.87) plague.(0.99) They(1.00) call(0.99) this(1.00) the(0.96) +[L24] Pratiour-boat,(0.77) &(1.00) belong(0.98) to(1.00) a(1.00) board(1.00) of(1.00) health(1.00) +[L25] composed(0.87) at(0.79) <>(0.22) <>(0.50) +[L26] <>(0.51) are(1.00) commonly(1.00) <>(0.68) Guarda(0.85) mores(0.83) &(0.96) re-(0.75) +[L27] gulate(0.86) the(1.00) <>(0.16) of(1.00) such(1.00) Ships,(0.96) as(1.00) come(0.91) +[L28] from(1.00) respected(0.80) places.(0.71) After(0.83) they(1.00) left(1.00) us(1.00) <>(0.66) +[L29] went(1.00) on(1.00) board(1.00) <>(0.49) the(1.00) Adventure.(0.99) Capt.(1.00) <>(0.60) +[L30] breakfasted(1.00) with(1.00) us,(0.99) &(1.00) then(1.00) we(1.00) went(1.00) altogether(0.89) +[L31] in(1.00) the(1.00) <>(0.48) ashore,(0.99) but(1.00) we(1.00) could(1.00) not(1.00) <>(0.40) +[L32] close(0.88) \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P012.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P012.txt new file mode 100644 index 0000000000000000000000000000000000000000..916b230b7eea69feb21933be5b44cc8578fee2ca --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P012.txt @@ -0,0 +1,14 @@ +=== Buch 1, Seite 012 === +ms. germ. quart. 222 +Journal of a journey +from London to Plymouth +&. A Voyage +on board his Majesty's Ship +the Resolution +Captain Cook Commander +from Portsmouth +to the Cape of good Hope. +from July y. 13°November +Ex +( Biblinth. Regie +Berclines \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P014.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P014.txt new file mode 100644 index 0000000000000000000000000000000000000000..1ea58efe42b246bbb53b54f0671c0ba7ceddebf6 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P014.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 014 === +Journal of all the incidents previous to my Appointment. & after departure from Plymouth +12. May +May +Mr Irwin, who had not seen me +even since. I left London, in the year 1769, came +rate in the morning & in a very mysterious +manner told me, that M. Banks did not +go, the Resolution to the both Seas, & ask +whether I would go; I told if a proper provision +made for me of my Family, I would not +hesitate one moment, but my Son, must +in that case go with me, being both as +a Nationalist & a tolerable Draughtoman +well qualified for to assist, no Mr. Irwin +seimes to like it desired me to keep upon +this application the strongest secrecy, & told +that he had it in his power to recom- +many one in such a manner that +I would be certainly employed in the +Expedition; & 542 he was to break left +with Mr Stephens he would speak to +thing about the next day up six +ner he came to me again, & told me +0 0 +that Mr Stephens had taken down +my \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P015.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P015.txt new file mode 100644 index 0000000000000000000000000000000000000000..6f2a1158dfba8eeeaacd0bf14de23fb020fe8915 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P015.txt @@ -0,0 +1,23 @@ +=== Buch 1, Seite 015 === +my name, qualification & place of side & +added he thought this were a great step & +that I should continue to keep the affair +very secret. +May 4. 24. I saw Mr. Baines Barrington & told him +that Mr Banks or not go, & as he was as- +kine whether it was two of it; I told, I +had heart that he had already scared +his Equipage & cloth, books & Instru- +ments to be taken out of the Resolution +& that she was repairing at Sheerness, +having heard this from an acquaintance, +who was well acquainted with Mr. She- +phens. Mr Barrington. hereupon as become: +In case provision was made for me +& family, whether I would go, I told, +I would, provided my Son Geomo was +likewise appointed. He approved of his +condition & desired me to carry a letter +to III. Butler on the Subject to the +admiralty. I brought it to his house +but found him not at home. \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P016.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P016.txt new file mode 100644 index 0000000000000000000000000000000000000000..40c15796328668f289d360e4b744ee36cb4c32d8 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P016.txt @@ -0,0 +1,26 @@ +=== Buch 1, Seite 016 === +May 27 I heard that Mr. Divington's letter had been +read by Mr Butler at the board & that +Lord Palmerstone had added, I have hear +4.30° +June +of that Man a very good character & +I believe he could do in that branch, +he is intended for, as much as any +man in the Kingworn. out as no other +resolution was taken Mr. Barrington the +I should give up all thoughts of going. +W. Bavington took the resolution to +write one some more to Lord Sandwich +to Hinchinbrooke & to join to his letter +the Resolution of the House of Commons +quanting 400 lb for carrying on disco +veries to the Southwards not mentioning any particular person to whom +it might be given; though Dr Lind's n +had been mentioned in the house. +and the letter was written accordingly +a letter of a Sandwich came, utmost +ting that he wished to see Mr. Barringt. +Tuesday ye 32. & especially did he ask, +whether I could go at a moment's +warning \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P017.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P017.txt new file mode 100644 index 0000000000000000000000000000000000000000..0d549f9ecd4cf1a2f23798465cf574e56cb61bb1 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P017.txt @@ -0,0 +1,25 @@ +=== Buch 1, Seite 017 === +warning: for which purpose, still, not +me the Question, whether he could brought +this to B Sandwich. I said I would en- +deavour to be ready in a few days. I +Juney ? Mr Barington spoke to L. Sandwich, who +June +seemed bleased with the plan & my cho +raiter, & as he heard of my reassness +hisLordship was still more released. +IT Solander told me, that he +it was not certain, whether Mr. Banks +would go or not; but that it would be +decided, next Wiednesday Irene ye 11th +or then Mr Banks would have a +mixate Audience of his Majesty +I hat my day at the Club of Banks past men's that in their was made +in into my recovery +I Sandwin suspecting that Mr Banks +would endeavour to obtain a new Ship, +he went to His Majesty, & said the plan +of sending me, on the Expedition, with +the Character given me by Mr. Barrington +before the ring, who was graciously +pleased \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P020.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P020.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b7bc81414199970eed695caf5d63b15a74f9ac4 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P020.txt @@ -0,0 +1,24 @@ +=== Buch 1, Seite 020 === +pleased to approve of my going on the +0 0 +Canadian & by this 2 Sandwich Islands +any further application from Mr Banks +side. +June 16th The Chief Members of the Council of +Rural Society Signed a Certificate in +my favour, recommending me as a +poroner person for going on the Expedition +ane ye 7th Hudson spoke to me about the affair +June +& told me that he had recommended +me to met Dy son & to Sir Gilbert Elliot +& said, that as there had been an inter +tion to make a motion in the house of Commons Friday last in my favour +Mr. Thips & Mr Burke has been oil +ling to oppose it; & that it would cease +great difficulties in regard to the mono +MR Burks, Burke & Philips wanted to +have an interview with Mr Hughes Palliser +& Mr Stephens at the Speakers, but LP +Sandwick had already saw the streetess +ins of them, not to go \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P021.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P021.txt new file mode 100644 index 0000000000000000000000000000000000000000..e3a56fda3704ee2ef2460fe8bb50a59171fa46f0 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P021.txt @@ -0,0 +1,23 @@ +=== Buch 1, Seite 021 === +June 4th +de +there; by which means they desquired of +reconfiling & Sandwich +The House of Commons oro ke up & +on act of the Infrequently of the +Members, my name could not be +mentioned in the house. +North spoke to Mr Barrington, who +argued the case, & showed that as the +resolution of the house had not d +lind's name mentioned, the many +could be applied to any other par- +son giving on the same Expedition +North having mentioned the case +to His Majesty, the money was or- +dered to be issued to me from +His Majesty's civil List & that +at the meeting of the House +of Commons at shows be decided +whether the money should go to +mer & Lind \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P024.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P024.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab7650798f88d6927778563ca07b720e6eb18834 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P024.txt @@ -0,0 +1,3 @@ +=== Buch 1, Seite 024 === +From this day I could first consider myself as appointed by his Majesty on the Expedition. The Order signed by the King imported that 1795. I should be paid to me by the Exchequer, which I receive accordingly some says afterwards. +the same day I went to Mr. Banks & not finding him at home I wrote a note to him desiring him to appoint me a day, when I could have the favour to speak to him; for being appointed by his Majesty to go on the Expedition, I wished to be informed by him to me particulars relative to my equipment & having always been favoured with his friendship I hoped, he would add this to the many obligations he had already, laid me under as it was just going away Dr Scharter came in, I told of my going & began likewise his advice, He congratulated are coolly & was civil; but the moment \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P025.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P025.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a829cdb4a509559d3f5b5648da3ea71d591598b --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P025.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 025 === +June 4 +I on purpose told him & Dr Lind, that +the money for Daino was out of the +Question & that I should have mine +isoued from the civil List of His +& reserve dress +Majorty; the coolness wore off, & all +gloom was at once carries off upon. +their tases, or the really gave me some +good information relative to many +particulars. +s +Mr Banks sent me word that I might +see him the next day. +Name to see W. Banks & begged of him the favour +to give me some informations relative to my +Equipment, adding that I knew very well, the +necessity of common articles, but I shift that +by his experience he must have found out, +the conscience or inconvenience of many +things, which could not be frozen at such +a distance, or did not occur to one who had +not been on the Voyage: NBanks declared +he would give me any information & I +should ask him Questions & he would must +me \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P028.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P028.txt new file mode 100644 index 0000000000000000000000000000000000000000..00550c362ac92b9a38bdd6c23790e3ce94fb6170 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P028.txt @@ -0,0 +1,2 @@ +=== Buch 1, Seite 028 === +me. I told that as I was unacquainted, with the peculiar & unfamiliar particulars I only wished to be by him instructed in regard to them, but Mr. Touk declined, this & repeated I should ask him questions & he would answer. a great many more company coming in I on purpose asked only a few trifling questions & then applied to him for to let me see the drawings of his plants. He replied this would be of little use to me, & excused himself with want at time & then said, since I saw, he declined to do it I would not insist upon it hereupon Mr 13am asked me when she Ship would sell. I told I did not know, but suspected it would happen on Saturday next, then he desired me to come Saturday next. I met Capt. that day at Mr Banks's & went with nine to the Admiralty where a board of longitude was to be held, in order to see there Mr Hormby. then went with Cape Cook home, o wave him to dinner. went to the Treasury & Exchequer & got 179 5. E pei me: 95 of which were to defray the Expences N. \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P029.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P029.txt new file mode 100644 index 0000000000000000000000000000000000000000..b3d0931ef330b18bc7dba12c094340b8da390caf --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P029.txt @@ -0,0 +1,26 @@ +=== Buch 1, Seite 029 === +20th all my baggage was sent on board the +Resolution by the Sheerness. payout. +June 4th String completed my Equipment. I took leave +by friends +of my Family, & took a post-things for Plymouth, +The Resolution having left to Spearness y 22°. +We passed through Brenton a dirty town, came to +Aounslow, all a fine & cultivated country, then +passed Avaslow Heath, filled with furge & gallous +both a disgrace to the Country. Dined at Stain- +bridge: passed over the uncultivated hills on +Baghd common, which furnishy London Mac-> +rets with the finest mutton, but in my o- +pinion would be better employed in plm-> +tations chiefly of oak. The same country con- +tinues to Marcel green, where we went to +Basingstock an ancient town; Immunology +we passed Sutton & Stockbridge, & having closed +the Sleak & crazy health of the plains +of salisbury, we reached that ancient city. +the Spire & remarkable on account of +& has walls from the city +its' height & pointed figure & Our mild' law +still on the same Warren Heath, +The bottom of all the country is chalk. \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P030.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P030.txt new file mode 100644 index 0000000000000000000000000000000000000000..757c65dd675b0fa63a58a51b754a2ee723597ffc --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P030.txt @@ -0,0 +1,30 @@ +=== Buch 1, Seite 030 === +U Pame ye 27 We reached Barnet-down a single house & then +came to Blandford. The country begins to grow hill +between the latter place & Doschester, a tolerable +good place. All Dosetshire is well cultivated, & +from the summits of its hills, it bears a 1 mg. +charming aspect. At Bridport the Sea comes up +is in +within two miles, The town, let a most thriving +condition & has a manufacture of Sailists +which employs a good number of hand. +On account of the vicinity of the Sea, the top +of the hills were this day codered with clouds +which soon were resolved into rain. We +in Deponshire +reached Axmin for white carpets are many +factured; from this place we sit out for Honito +a populous borough at the bottom of a pale +Its chief house commodity are course cloaths. +Exeter the next city, is a very busy cable dinner +ty place, surrounded by old decaying walls, +its streets are extremely narrow, go up +& down hill, & make the city to a firm all +plare. The front of its cathidra has a great +7 +number of figures in the gothic taste +but the villages are now quite disagree +The castle is surrounded by a few main offi +trees, part of which is clanted on the slope +the \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P031.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P031.txt new file mode 100644 index 0000000000000000000000000000000000000000..7edd242bc620ff067a2eb746211aeb16791dd253 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P031.txt @@ -0,0 +1,25 @@ +=== Buch 1, Seite 031 === +12 hull, with a few openings that afford a most +romantic prospect, beyond a pale on the nei +bouring well cultivated hills. This walk is +called Naxon's walk, & is much Army ended +by the neutnphs of the town, especially +de +at late hours. The coarser cloaths, calli- +mancoes & other Sergeo & stuffs, which are +here manufactured employ a good number +at poor people & consume the wool of +the neighbourhood. +June 1928. We passed some very high hills in gay +to Chudeigh a small incomprehensible town +The same aspect of the country is between +the latter place & Ashburton another +small town. The Inn we put up, It was a +pied by a Scotsman called Kenneth A +who overcharged every article of our +bad fare, & gave us most sorry horses. +& a surely, impudent driver. At toy +bridge, are a few houses & two inns, one +on each side the bridge. That which lie +on the casside has the better appearance +but \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P034.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P034.txt new file mode 100644 index 0000000000000000000000000000000000000000..f3457a9848d1e613b59b5a875628537e682787fa --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P034.txt @@ -0,0 +1,28 @@ +=== Buch 1, Seite 034 === +but the landlady on the western side was a +sensible old woman. In the Afternoon +over reached Plymouth. All the coast of +Poonshir is pretty well cultivated in +its dalei, as far at the high swing +of the numerous hills will permit. The +Harbandry seems however to be rather +neglected: for all that, it bears a most roma- +Fio aspect, from the top of the higher hill. +We enquired immediately for the Resolution & +heard She was not yet arrived. Plymouth J. +irveyula +a place with narrow winding trees, which +Breh I +have a Channel with running water in +their middle; & though this is a great incon +venience to carriages, which must always +have +one wheel in the channel, it is & to persuade +ple walking in the streets who are construed +ly splendid by the houses, it has on the +hand the Advance of carrying off all +the fifth & imputaries of me born, especially +at night, when the contents of a great me +foots with humane extra points are emptied +into it as few people have regular little +houses \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P035.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P035.txt new file mode 100644 index 0000000000000000000000000000000000000000..33b8f19d7340bd1886191020f9a4e6b75d2f4ac0 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P035.txt @@ -0,0 +1,26 @@ +=== Buch 1, Seite 035 === +houses in this town. The reception of the Sea at ebb tide leaves, a great part of the in- +terior harbour dry & causes an untolerable stench. Monday June ye 20th. We went +of one such +to the Dock & passed over a fine bridge +built of Portland Stone, over an arm of +the Sen. On the other side we passed by +the Hospital & came to Plymouth cook +a Kind of new town 2 miles from Plymouth +poth a number of good houses regular +Streets, which are improving every day. +She dock-card is beyond this inclosed in high +those walls, & none but 7 who are belonging to +un itupons & +if are admitted, we wanted on Lord +Wichit who was gone up their 10- +or 12 miles, in the afternoon I went on +board the Adventure, who lay near mount +Edgecombe in +soon, but found neither +Capt Turneaux, nor Lieut. Kempe on beef, +for whom I had letters from the hostile cap +wards Byron. Tuesday I went to Drukes or St Nicholas +Island which lies in the month +e Plymouth +harbour \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P038.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P038.txt new file mode 100644 index 0000000000000000000000000000000000000000..7ae0c0b41d680999904f29e0a22f2e395e7ddc26 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P038.txt @@ -0,0 +1,2 @@ +=== Buch 1, Seite 038 === +& from which towards mount Edgecombe a Ledge of rocks extends. This Isle is so & is a mere rock with a few houses, in which a gunner & a few soldiers vice. Here Mr Deiley that stop. norner, who was intended to do on the Expe dition, in the Adventure, had erected his ten & made observations of equal Altitudes, & after wards when the Resolution arrived Mr Wales joined him with the Transit-instrument, & they settled both the Latitude & Longitude of this place-vig. July ye ye 1st went on board the Augusta-Shackt, to was upon Lord Sandwich, who had desired Capi-sim-> the day before neaux, to let me know he wanted to speak to me. His Lordship received me in the most obliging manner; after breakfast, he took me on the poop of the Vacht, & asked me respectively, whether he could do any this more for me, & thanked his Lordship for his kindness \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P039.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P039.txt new file mode 100644 index 0000000000000000000000000000000000000000..f2cf5604ee01bea5e69602422b121b22750b1fa2 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P039.txt @@ -0,0 +1,23 @@ +=== Buch 1, Seite 039 === +6 +ness & told, that I thought my present +situation so happy, that no other wish +was left to me than, that this +worship would be pleased to continue to me his +protection & favour. He said he expected +the Resolution that night in Plymouth +sound & in that case, wished to see +me on board of her, between five +& Sir. After having conversed with his +aordship for about 4 of & hour. he +went in his large to the dock-yard +& took me along with him, & went +through the whole yard with me & whether +he thought an explication of the various objects to works might +A necessary, he was pleased to give +me the fullest Information of every +thing. after one o'clock His Lochip +went on board the barge & recommend +me to be at six on board the Resolution +in ease she should arrive before that +Jim \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P042.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P042.txt new file mode 100644 index 0000000000000000000000000000000000000000..05337476e165f2024ec4aaa23c484f12e7ce4af6 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P042.txt @@ -0,0 +1,6 @@ +=== Buch 1, Seite 042 === +time. His Lordship went to dine at Mount Edgecombe with Lord Edgecombe, The Resolution did not come. +July the 2d. Lord Sandwich left Plymouth with her Yacht at six o'clock in the morning. +We saw the Long Room & bathing place kept by a few, who has fitted the same very neatly up; but 2 believe there are so few people to make use of it, that he scarcely will find his act. +July, 3. The Resolution was arrived in the night we saw her from the Garrison, a fort built of Stone on the top of La Rock hill, which commands the entrance of the harbour; & is garrisoned by a regiment of Soldiers. We went on board the Resolution Capt. Cook was gone to the Yard: & we followed him there & I went again on board the Ship with the Carpenters & joiners in order to have all & convenicacies +The Alterations made, which I should in the little Cabin allotted to me 4 wish for. After having given proper directions, I returned to smooth & \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P043.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P043.txt new file mode 100644 index 0000000000000000000000000000000000000000..d58f36ca878e42dd1187ab7ddb174f37afa757c5 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P043.txt @@ -0,0 +1,26 @@ +=== Buch 1, Seite 043 === +18 ) dined after 5 o'clock, & as I could do nothing +on board till Tuesday or Wednesday, when the +afterations should be done, I took a +journey into Cornwall in order to see the +Finns there. +July 25th +& Went with Mr Arnold & my son +to Saltash, having passed the +Arm of the Sea in a Ferry. The borough of Saltash +lies on the Slope of a hill, & is upon the +whole but a sorry place, though it sends two +Members to Parliament. Passed Leskean & went to Bodmyn which formerly was a +Town of greater note, when the See of a +Bishopric was there, so as I was told, though +I saw no remains of its ancient plea- +the ruins of +dor, except an old Church whose tower +still exists, & afford an habitation to owls & chapighs +#th or jack days +July y.e We came to Lainivet in order to see +there some mines, which upon exasiation +nation, were only some wretched +Streamworks. +July 4th Resolved to go to Touro the only place +where I could expect mines: passed through \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P046.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P046.txt new file mode 100644 index 0000000000000000000000000000000000000000..5df0dddc96ec5a17e7706abefee419b682856410 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P046.txt @@ -0,0 +1,25 @@ +=== Buch 1, Seite 046 === +14 +A Michael a poor little town: arrived at 12 +docked at Souro. Which is a plan in every +thick condition. The Bank, the Custom +house, the Stamp office, & the houses +of some rich Merchants are elegant +buildings. Here the white Tin of the country +& Cornwall is stamped Each block of +Fix contains about 300 weight. I began +one Mr Beniele a man concerned in some +tinmines to give me directions & ic- +commandation to the nearest of them. +He sent his clerk one UN Marton with +who gave me a letter to Mrichard +williams at Poldice in Gwenap parish +Cornwall. It is about 6 miles west of Trare +The people brought one to the wrong man +for there are # Richard Williams, a Captain +of Mines, or Supervisors. the first I saw, was +at Kenwyn, in a new Tinwork. The +Tier is specheed into a greyish kind of Stone +called there Growan, & is a Mind of Saxum +or Moorstone or course Granite. The country or +wild Stone in which the vein runo case \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P047.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P047.txt new file mode 100644 index 0000000000000000000000000000000000000000..ae7f394f4d054d0ae502caa55e971c0bdcdc1e2c --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P047.txt @@ -0,0 +1,28 @@ +=== Buch 1, Seite 047 === +vs +& west is chiefly Kellas or a kind of fibrous +lamellated grey, greenish, or rare Stone of the As +best kind, which, combler into Singles, af- +ter having been exposed to the air for a good +while. The vein is from B inches to 3 fattycoms +wide & about 40 or 50 fathom deep. A great deal +of Mundysk or Arseniculogite is in the via +for road. +Avale with a ribulet davider Kenwyn from +Poddyte, where the drain is more valuable, +being a fine white Quartz or as the mi- +r Curasione +miscall it bear with fine tin greains +of the +in it. but few/arsenical ore is in this +kind: in some places of the vein, they +find first Gozzum or a kind of an Iron +ochre pot in such case they are always sure of +finding Copper below it the Copper ore is a +yellow Pyrites, which sometimes on account +of the chinks & cracks of the ore had +admitted the external air, & thus +the ore begins decaying & forms those +beautiful colours, which are like the pigeons +neck & perfectly similar to the effects of +an electrical Explosion upon metalo. \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P050.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P050.txt new file mode 100644 index 0000000000000000000000000000000000000000..b2ebb8fce5829dfa24c3c07bd52a1a1fd928554c --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P050.txt @@ -0,0 +1,26 @@ +=== Buch 1, Seite 050 === +The vein at Polyce is from 8.0 to 105 deep, +is much richer, & increases in richness the +deeper it goes, but cannot be raised without +loss, on account of the immense expense +in raising the winter by the fire engined, +which are still contrived as the same +effect might be produced by half the coats, +if better contrive. She veins in Cornwall +for the greater part underlay North or dip +to the South; the load at Podque is just +the reverse, for it underlays South. +In all vales leading toward Tin veins, & +in places, where formerly the ancients +worked for Sin, small pieces of tingingus +from the Size of a centil to that of a +wallput & even bigger are to be met with. +& these the minars Separate from the unyiel +ding stuff by washing, the light stuff going +away, when exposed to the current of a small +born, for which reason all the Tirquff got +in this manner is supposed to be +from +at well, of a wood when +a Streamwork. This Jury is as the ore +picked from the unagielding Stone, is brought \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P051.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P051.txt new file mode 100644 index 0000000000000000000000000000000000000000..2bad418dd793b93e6971a01b82d3defff90247eb --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P051.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 051 === +to a stampingmill & then washed & lastly sold in lots to those, who offer most for at, after +having assay & the blacktin, as they call it. +This blacktin they burn, if a great +Quantity of Mumbick is contained in the +dresod black tin, in order to expel the +arsenical & sulphureous particles, from it, +& lastly they miss it with fleux & op 1st it +in a furnace, to white tin; which is made into +sloccks. of 300 weight, asayed & stamped. +In the great Gauze-Moor, between Bodwyn +s & Michael is a great number of Streamworkers +to the amount of 58, now actually working +The minors, contrive several method, for to +edge +use trestirulets of this mood as streams +For washing their Tenstuff, & likewise to +work with it some pumps to clear the +places under ground from water, where +They are at work. In these Streamworks +a kind of white Tinspar is found which +the miners call Pryan, & which yields +pure +a King of Jin- +In the common load of. Tin, there are sometime +somewah at fibrous +brown, blueish black your moagoes, called by the \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P052.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P052.txt new file mode 100644 index 0000000000000000000000000000000000000000..047921b8502d258eec8d923241eed635924fedcd --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P052.txt @@ -0,0 +1,32 @@ +=== Buch 1, Seite 052 === +Cornish Stannaries Black Jack, & is nothing but what +the German mineralogists call Wolfram or Sigma +lupi or a mixture of Iron, Tin & manganese; +& it must therefore carefully be distinguished from +the Substance called BlackJack by the Shropshire +Welsh & Derbyshire miners, which is a calam +blend. Having examined these mines, I thought +myself pretty well instructed of the Subject: & +returned first to Toura, & then to Bodmyn. +July ye 7.74 +We saw the races at 13odmyn, four horses far +ted, once was distanceed; & the second heat tho +prize was got by S. +July, 8th I took leave at Dodnyn & Cornwall, return +ed to Plymouth & found all things on board in +Ship for me done, +. 4.91. +4e +& came on board, but the Ship was all in +a confusion, & I could range nothing in +my & my Sons Cabin. +4th +The Ships company was paid & the +whole Ship a whole scene of drunk in nest. +confusing, & a regular fair was held on +board of it, no mean being more permit +Fed to go on Shore. + ye 12. +16 +I came on board & began to settle my +things, at least so much, that I could \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P053.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P053.txt new file mode 100644 index 0000000000000000000000000000000000000000..919d4bd3ee979458b38f359aef91e4c32bec6b11 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P053.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 053 === +sleep on board the Ship: which however, was +very difficult on aft of the same disorder +still prevailing. +July 4/2° when I was walking on the Quarter- +beck I found, that the Ship altered it. +lot +Situation in regard to the Adventure & +a Mast Ship, that were at anchor in +the Sound, & that we came nearer & nearly +to the Garrison or castle, The Master who +walked along with me on the Quarterdeck, +began, to be allarmed. Our Ship was faste- +ned to one of the boys in the harbour; +We found nozo, that the cham by which +the boy was +fastened was broke & that the Ship +was adrift along with the way. In a +& moment all lands were on deck, the +Sails spread, the authors cleared & +kept ready to +grevent us from being +caryied by the tide against the All +rocks of the castle, & stove to pieces. +Our. Captain & all the Officers were ex- +tremely busy & gave such orders, that +just, when the Ship turned, She happily \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P056.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P056.txt new file mode 100644 index 0000000000000000000000000000000000000000..870290cfcbf81869e536f26ec046790538291467 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P056.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 056 === +25 +passed between two rocks under water, & by the same manoeuvre the Ship nosed the Adventure, the Mast Ship, & the Isle of Nicholas; & there are dropt our anchor, heaving narrowly escaped, the most least danger. +Journal of a Voyage on +board her Majesty's Ship the Resolution +father +Captain cook Commander +at Palliser +patient. Mr. Cooper +23 Kent. Mr Charles Clench. +Rigars +3° Lieut. Mr Tathersgill. +Joseph +John +Dava +John +Master. N. Gilbert. Mated Map. Whitehouse, Bever & Isaac Smith. +Misshipmen Messom. Price, Willis Brindal, Logie, Symed, Roberts, Elliot, Mitchell, Maxwell, Van Kover, Harvey +collect Coughlan, Blood, Burney. +Lieut. of the Marines. Mr Edgecombe +Surgeon M. Patton. +ronomer Mr Wales. +Painter Mr Hodyes: +For the Natural History Department, & my Son. +Ship +clock Mr. Dawson Capt. Clerk, Mr. Church +Surgeons Mates M. Anderson & N. Drawwater. \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P057.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P057.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d626266596cde9f40def218ae889faadd782387 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P057.txt @@ -0,0 +1,27 @@ +=== Buch 1, Seite 057 === +26 +13th +July ye 13 +We weighed anchor & left Plymouth +Pound in company with the Adventure at 4 o' +clock in the morning. The Adventure Sloop +was of 380 Tuns, Capt. Turneaux,& under him +Lieut. Shanks & Kempe. The Astronomer who +went on board of her was Mr Bailey. Our Ship +was of 469 Tuns both well rigged, stored & man- +ned & in all appearance the best matched +Ships for going on an Expedition, which +must of course require the greatest attention +as their success depended greatly on it, +& by all probability would become of great +importance, to Navigation, Geography +astronomy, & all the other branches +of Science. The Lords of the Admiralty & +Lord Sandwich +especially , had paid a particular attention +to the equipment of these ships & especially +to everything that bade fair to promote +the success of the Voyage & make the +whole Expedition as compleat as it could +be wished. As Mr Banks & Dr Solander had decli- +ned going on the Expedition, & the branches of diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P060.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P060.txt new file mode 100644 index 0000000000000000000000000000000000000000..e512d297466c61b1660ba2675b73ee8fbd78095f --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P060.txt @@ -0,0 +1,10 @@ +=== Buch 1, Seite 060 === +Natural History & the drawings of the various new objects which might offer on the course of this Voyage remained unoccupied Lord Sandwich appointed me for the one with my Son; & a few days before the Ship Sailed, Mr. Hodges an ingenious, Artiller was sent to make up the things deficiency in the branch. In +When we just had left Plymouth-Sound, I Souvia & his friend 100 Jones followed us in a small boat. They were making a li four into C.nicell as far as Fovey &.S. Aust. & were going to the first place by Sea, & wanted to take leave of us: they came on board of our Ship & after having breakfastes good tts with us, they took leave of us to date from, & we directed our under & N.W.b. W. wind +course into the Shanhel. At 8 o'clock +the Ramhead bore North, distant 2 miles. +at 12 o'clock at noon the Latitude was +observed 50° N. the Ramhead bore N.E 34 +distant +leagues. We passed Early Stone light +house, it was about a leagues to reward \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P061.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P061.txt new file mode 100644 index 0000000000000000000000000000000000000000..92dc56adf20163bfa9cbda4f91811009505b68d4 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P061.txt @@ -0,0 +1,25 @@ +=== Buch 1, Seite 061 === +28 +July 14° ) The Ships remained in company +the weather continued fair, the wind changed +to W.H.W. & sometimes W. 6. N.N. nor course was SM. 4 +S.W.b.S. In the Evening at 9 o'clock we bore +down to the Adventure. In the night the +weather became cloudy & towards morning +heavy with some rain at 8 in the morning +we fired a gun for the Adventure, after +having racked, directing our Course West +S.S. W. SWW. b.W towards noon it grew foggy +& only light airs. +July of 15th We met with a larvae Swell, from +the N.W. & towards night a thick fog co me on +we soon found that we were approaching +the Bay of Biscay, by the great waves. I had +been seasick ever since the first day of one +Voyage, could eat very little, was obliged to +throw up any Virtuals, had no head.each +I found squash much better & greatly +relieved by some mylled Portwine, +We tached on & off till +July 18, when we began to make our course directly +W.b. W. had some squalls & now & then rain \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P064.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P064.txt new file mode 100644 index 0000000000000000000000000000000000000000..c94e67462d48e8a9af0c7aa3911a167ed26134d8 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P064.txt @@ -0,0 +1,25 @@ +=== Buch 1, Seite 064 === +July +S 20 at half past 8 o'clock Saw Cape Ortegal or as it should be called Ortequera bearing South +For 6 Leagues distant, at no on the ba me Lope +bore South about 2 or 3 leagues distant +The latitude was 43° 55. No. The Country +was very hilly, rovered with small wood & I believe, health, & we saw several pieces +of corn, in all appearance ripe: +21. We saw Cape Prior at 2 or 3 miles distance +bearing S. W. the weather happy & sometimes +rainly. at noon it was cloudy, however had +an Observation of the air 43° N. Groyn or Corunna SSE. & 2 Distant 3 Leagues +Cape Ortegal East. 12 or 14 leagues distant +ly no +We saw Groyn lighthouse. S.E.b.S. at 5 tea- +ques distance, sounded 87°. from. We went +at I o'clock on board a small Tartane +which we believed to be a Spanish +fishing Vessel in hopes to get fish, but +foods it to be a Turtan from Marseille +of about 100 Tuns, freighted for school +by Corunna with Plower. We bought +some Frontignac, at 1/2° a +Small +Chest of 50 bottles, & bottles \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P065.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P065.txt new file mode 100644 index 0000000000000000000000000000000000000000..35ae2288a762f8ae3f9feaefeab962481ca87068 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P065.txt @@ -0,0 +1,25 @@ +=== Buch 1, Seite 065 === +of Syrup of Orgeat at the same price +reuseved the Tartan with water, which they +stood in great need of, having been able for 2 months to beat to windward. +The Variation from Azimull 23° 58° Gwyn S.D +4 or 5 leageces distant. The Country appeared +overw billy, The tops of the hills were, white +from. Sand or Limestone & had a barren +bleak appearance; in the lower part +July 4 +we saw cornfields & inclosures & some +small hamlets, together with a well +looking house & some outbildings +like a gentleman's seat. +9 +. Passed three Spanish Men of war, the +first a 60 amusing the second a 74 gun Ship +the last which was the Commodore +had 60 guns, & hoisted English colours +we hoisted ours, the Ventiant & the +Jack; The Spaniard having nasod +us fired a gun to leeward, "> a +little while after hoisted Spanish +colours & pen na net; with a bolster, white +striped Jack on the top of the fore mast \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P068.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P068.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b4e88c09bdde2bb2d326da2723b1cdbe2d81593 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P068.txt @@ -0,0 +1,31 @@ +=== Buch 1, Seite 068 === +after +a little while he fired a Gun at the Adventure +whose Shot dropt some where near the Ship +As we went on our Course the Spanish Ship +went about & fired another Shot at the +Adventure: we brought to & the Adventure +followed our example till the Spaniard +came abreast of her about a mile or +mile distance & probably spoke to +her after which he again went about +m +& we made Sail. At 6 o'clock the Variation +by Ajimuth was 24 50 +by amplitude-24° 39. +West. +at 6' o'clock a. m. Variation 19 21. +July 4 +& at Observed 42° 18° +Having passed Cape Finisterre the night +before we got a fair N.E. wind & fair wea- +ther & sailed at 6. & 7 knots per hour. +Latitude 40°. 3' N. +25 +The same weather continued & we +saw a good many purposes passing +by again & the wind. Ever since we left +Cape Finisterra the Sea parked +in the night the foam was luminous & +in it appeared some parks amazing +hy bright I suppose this not to be caused \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P069.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P069.txt new file mode 100644 index 0000000000000000000000000000000000000000..73e6057e6a588f9f506fb5074aaf5033f37df5f8 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P069.txt @@ -0,0 +1,28 @@ +=== Buch 1, Seite 069 === +32 +by any insects or other phosphorescent Substance +Stain, from dead animals in the Seawater +but rather to be Electrical, from the +cutting of the tarred Sides of the Ship. +& causing a violent motion in the +water; for I could see this luminous foam +& the sparks in it, only near the Ship +& not at some distance; but I will certainly take some method or other to exa- +mine this phenomenon more reliably. The Latitude was observed 37° 41 North +July ye 16 +27 +The same weather, but somewhat clearly +heavy, & our course was S.V. as before. +b +The Latitude observed was 35° 31 N. +The same weather, the Course S. W. by S. +Cat. & Observation 33° 43°. +-28th Course W.S. W. then W.by. & then W. & W.S.S. at 4.d- +clock. the Variation was taken by the Ayimuth +200 23.W. At 6. The Land was seen ahead from +the Masthead. At 8 o'clock A.M. The Body of the St. +land Portosanto Borc N. W. by N. at 4 miles distance +Its Ear End, North at 2 Congress. & the Deserters. +S.W. +Porta Santo is a small Island about 6 leagues in +length, barren & very thinly inhabited, belong \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P072.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P072.txt new file mode 100644 index 0000000000000000000000000000000000000000..06fd4d90165faf4549746fd6e666ad9891acc6af --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P072.txt @@ -0,0 +1,31 @@ +=== Buch 1, Seite 072 === +belongs under the Government of Madeira. There +is a small town in this Island, on its SE part. Round +she the town & up towards North, is scale, which seems +to be inhabited or cultivated chiefly with vineyards, +the rest being all a barren, clean Rock. +at Noon the End of the Isle' bore N.e.e. 2. The +Brazenhead, a Landpoint in Madeira inclu- +ding the road of Funchal on its East side +bore W.S about & leagues distance. The body of +the Southwest Deserters. S.S.M. 52' 49' IV. +29 +W. The Satetu +is +We had very Sight Ains. Saw Madeira a break +& the town of Ste Cruz about 6 o'clock. +The Isle with its lofty hills, bore a very fine +prospect. The little rules which interested +the hills caused a variety which beggars all +description. Sta Cruz lies at the bottom of a +small bay, at the east of which we saw a +plasant +castle. The vineyards with some +countryhouses, & here and there a grove of +Chesticultrees & in one place some cyprusous +trees gave the whole +which +romantic appears +We stopped 2 boats out, took the Ship in low, +cament 10 o'clock at night to an anchor +win \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P073.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P073.txt new file mode 100644 index 0000000000000000000000000000000000000000..6eec0eee2c432e088fd5d65b3c90ed18ad7c0f4b --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P073.txt @@ -0,0 +1,26 @@ +=== Buch 1, Seite 073 === +be with the small bower in 25 from water +veered to half a cable. The Bouzen head bone +ES E. The Lao Rock SN. W. by W.S. DW. & the +westernmost point of the bay W.e N. The Adventure came likewise to an Anchor. +Early in the morning, I was roused by the mor- +ning Jun, & as I knew the Ship was going to +salute the castle, I got up, in order to be out +of the way, when the guns on both sides of +my Cabin should be fired. Coming on deck +& beheld the finest sight, I ever met with in +the course of my various travel. The town +of Funchal is built round the bay in form +of an Amphitheatre on the slope of the +hills, other figure sets all the buildings of +the touch off to great advantage; which +are all white, with low roofs, & by this ke- +quire a kind of neatness & elegant, far +Beyond Beacon +Beyond any description. the town +the hills rise bolder & bolder, till at last +some are tapet by cloud. The sleeping +ground are clad in a most agreeable veroiri, caused +by the wines, that occupy all the space of ground +till to the very summits, that are involved +in clouds, the hedges are of a darker green & here \ No newline at end of file diff --git a/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P074.txt b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P074.txt new file mode 100644 index 0000000000000000000000000000000000000000..9285786efceb695fc33ebf66bc98a72b6edebb60 --- /dev/null +++ b/app/deploy/book1_data/moredata/corrected_transcriptions/B1_P074.txt @@ -0,0 +1,29 @@ +=== Buch 1, Seite 074 === +appearance 35 there a cluster of trees & the of a white +overstombouse or of a church between these vines +yards, gave to the whole a sight similar to +what we often are told, of the gardens of fairies & the hanging Gardens of Queen Semirame +The westernmost point of the harbour has on +a rock surrounded by the Sea, Sea a Castle, which +had returned the Salute, we given with 14 guns. +All the part of the town along the Shore is +fortified & surrounded with guns. Besides the +cash already mentioned there is another which +commands the town & is called St Johns +cafale +About 7 o'clock there came a boat on +board, with a Captain of the Hall & an Inter +preter, to ensure into the state of the +health of our Inips-company & whether we +came from any part of Barbery, the Arches +& other places, that are suspected to be in- +seoted with the plague. They call this the +Pratiour-boat, & belong to a board of health +composed at your mothers +Who are commonly called Guarda mores & re- +gulate the Guarantine of such Ships, as come +from respected places. After they left us tho-> +went on board of the Adventure. Capt. Furtheraux +breakfasted with us, & then we went altogether +in the pinnacle ashore, but we could not were +close \ No newline at end of file diff --git a/app/deploy/requirements-space.txt b/app/deploy/requirements-space.txt new file mode 100644 index 0000000000000000000000000000000000000000..a6449678265a90c47b267b44e9b55b5d00a67bc0 --- /dev/null +++ b/app/deploy/requirements-space.txt @@ -0,0 +1,21 @@ +# Runtime deps for the Hugging Face Docker Space. +# Kraken is REQUIRED (it is now the only segmenter). Kraken pins torch 2.10 and +# pulls its own (CUDA) torch build, so we do NOT force the CPU wheel here — the +# CUDA build runs fine on a CPU Space, it's just a larger image (~4-5 GB). If image +# size becomes a problem, build a slim CPU-only variant in a dedicated stage. +torch==2.10.0 +torchvision==0.25.0 +kraken==7.0.2 +transformers==5.10.2 +accelerate==1.13.0 +sentencepiece==0.2.1 +protobuf==7.35.0 +Pillow==11.3.0 +numpy==2.4.4 +pandas==3.0.3 +sentence-transformers==5.6.0 +faiss-cpu==1.14.3 +fastapi==0.138.2 +uvicorn[standard]==0.49.0 +python-multipart==0.0.32 +requests==2.34.2 diff --git a/app/frontend/app.js b/app/frontend/app.js new file mode 100644 index 0000000000000000000000000000000000000000..99c2eda0e0fca03da0928ed1ed28783410f7dbff --- /dev/null +++ b/app/frontend/app.js @@ -0,0 +1,303 @@ +/* Inkference frontend logic. Talks to the FastAPI backend at /api + (override with window.INKFERENCE_API for a separately-hosted backend). */ +const API = (window.INKFERENCE_API || "") + "/api"; + +const state = { doc: null, page: 1, totalPages: 0, pageData: null, readerView: null }; + +/* ---------- helpers ---------- */ +const $ = (sel) => document.querySelector(sel); +const el = (tag, cls) => { const e = document.createElement(tag); if (cls) e.className = cls; return e; }; + +async function api(path, opts) { + const r = await fetch(API + path, opts); + if (!r.ok) throw new Error((await r.text()) || r.status); + return r.json(); +} + +/* confidence 1.0 -> dark ink, 0.0 -> faded; matches the legend gradient */ +function confColor(c) { + const dark = [43, 33, 26], light = [205, 191, 169]; + const t = Math.max(0, Math.min(1, c)); + const mix = dark.map((d, i) => Math.round(light[i] + (d - light[i]) * t)); + return `rgb(${mix[0]},${mix[1]},${mix[2]})`; +} + +/* ---------- tabs ---------- */ +document.querySelectorAll(".tab").forEach((t) => { + t.addEventListener("click", () => showView(t.dataset.view)); +}); +function showView(name) { + if (!["reader", "ask", "upload"].includes(name)) name = "reader"; + document.querySelectorAll(".tab").forEach((t) => t.classList.toggle("active", t.dataset.view === name)); + document.querySelectorAll(".view").forEach((v) => v.classList.add("hidden")); + $("#view-" + name).classList.remove("hidden"); + if (location.hash.slice(1) !== name) history.replaceState(null, "", "#" + name); +} +window.addEventListener("hashchange", () => showView(location.hash.slice(1))); + +/* ---------- init ---------- */ +async function init() { + let docs = []; + try { docs = await api("/documents"); } catch (e) { /* backend down */ } + if (!docs.length) { + $("#doc-subtitle").textContent = "No documents — upload pages to begin"; + showView("upload"); + return; + } + state.doc = docs[0]; + state.totalPages = state.doc.page_count; + $("#doc-subtitle").textContent = `${state.doc.title} · ${state.doc.subtitle || ""}`; + $("#ask-banner").textContent = + `Answers are drawn from all ${state.totalPages} transcribed pages of this document`; + await loadPage(1); + if (location.hash) showView(location.hash.slice(1)); +} + +/* ---------- Reader ---------- */ +async function loadPage(n) { + if (!state.doc || n < 1 || n > state.totalPages) return; + state.page = n; + $("#page-cur").textContent = n; + $("#page-total").textContent = "/ " + state.totalPages; + + let page; + try { page = await api(`/documents/${state.doc.id}/pages/${n}`); } + catch (e) { return; } + + // scan + const img = $("#scan-img"); + img.src = `${API}/documents/${state.doc.id}/pages/${n}/image`; + img.onerror = () => { img.style.display = "none"; $("#scan-empty").style.display = "block"; }; + img.onload = () => { img.style.display = "block"; $("#scan-empty").style.display = "none"; }; + $("#scan-name").textContent = `page ${n}`; + $("#scan-dims").textContent = page.width ? `${page.width} × ${page.height}` : ""; + + // readouts + $("#trans-sub").textContent = `Machine reading · page ${n}`; + $("#avg-conf").textContent = page.avg_confidence != null ? Math.round(page.avg_confidence * 100) : "–"; + const low = page.low_conf_words || 0; + $("#lowconf-count").textContent = low ? `${low} word${low > 1 ? "s" : ""} below 60%` : ""; + + // remember page; show the Raw/Corrected toggle only when correction exists + state.pageData = page; + // correction present if page-level corrected_lines OR any per-line corrected_words + const hasCorrection = (page.corrected_lines && page.corrected_lines.length) || + page.lines.some((l) => l.corrected_words && l.corrected_words.length); + $("#view-toggle").style.display = hasCorrection ? "inline-flex" : "none"; + if (hasCorrection && state.readerView == null) state.readerView = "corrected"; + renderTranscription(); +} + +/* build a line
of confidence-tinted words (green when Qwen-corrected) */ +function renderWordLine(words, flagReview) { + const div = el("div", "t-line" + (flagReview ? " review" : "")); + words.forEach((w) => { + const span = el("w"); + span.textContent = w.text; + if (w.qwen_replaced) { + span.className = "qwen"; + span.title = `Qwen correction (orig conf ${Math.round(w.confidence * 100)}%)`; + } else { + span.style.color = confColor(w.confidence); + if (w.needs_review) span.title = `low confidence (${Math.round(w.confidence * 100)}%)`; + } + div.appendChild(span); + div.appendChild(document.createTextNode(" ")); + }); + return div; +} + +function renderTranscription() { + const page = state.pageData; + if (!page) return; + const corrected = state.readerView === "corrected"; + document.querySelectorAll(".toggle-opt").forEach((o) => + o.classList.toggle("active", o.dataset.view === (corrected ? "corrected" : "raw"))); + + const box = $("#transcription"); + box.innerHTML = ""; + + // Corrected view: prefer page-level corrected_lines (preseed), else per-line corrected_words. + if (corrected && page.corrected_lines && page.corrected_lines.length) { + page.corrected_lines.forEach((words) => box.appendChild(renderWordLine(words, false))); + return; + } + if (!page.lines.length) { box.innerHTML = '
No transcription for this page.
'; return; } + for (const line of page.lines) { + const words = corrected && line.corrected_words && line.corrected_words.length + ? line.corrected_words : line.words; + if (words && words.length) { + box.appendChild(renderWordLine(words, line.needs_review && !corrected)); + } else { + const div = el("div", "t-line"); + div.textContent = corrected && line.corrected_text != null ? line.corrected_text : line.text; + box.appendChild(div); + } + } +} +$("#prev-page").addEventListener("click", () => loadPage(state.page - 1)); +$("#next-page").addEventListener("click", () => loadPage(state.page + 1)); +$("#view-toggle").addEventListener("click", (e) => { + if (!e.target.dataset.view) return; + state.readerView = e.target.dataset.view; + renderTranscription(); +}); + +/* ---------- Ask the Archive ---------- */ +async function ask(question) { + if (!question.trim() || !state.doc) return; + const thread = $("#thread"); + const q = el("div", "bubble-q"); q.textContent = question; thread.appendChild(q); + + const ans = el("div", "answer"); + ans.innerHTML = `
I
+ Inkference
+
…thinking…
`; + thread.appendChild(ans); + thread.scrollTop = thread.scrollHeight; + + try { + const res = await api(`/documents/${state.doc.id}/ask`, { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ question }), + }); + ans.querySelector(".answer-body").textContent = res.answer; + if (res.source_pages && res.source_pages.length) { + const src = el("div", "sources"); + src.innerHTML = 'Sources'; + res.source_pages.forEach((p) => { + const chip = el("span", "chip-page"); chip.textContent = "Page " + p; + chip.addEventListener("click", () => { showView("reader"); loadPage(p); }); + src.appendChild(chip); + }); + ans.appendChild(src); + } + } catch (e) { + ans.querySelector(".answer-body").textContent = "Error: " + e.message; + } + thread.scrollTop = thread.scrollHeight; +} +$("#ask-send").addEventListener("click", () => { const i = $("#ask-input"); ask(i.value); i.value = ""; }); +$("#ask-input").addEventListener("keydown", (e) => { if (e.key === "Enter") { ask(e.target.value); e.target.value = ""; } }); +$("#suggestions").addEventListener("click", (e) => { if (e.target.dataset.q) ask(e.target.dataset.q); }); + +/* ---------- Upload & Process ---------- */ +const dz = $("#dropzone"); +["dragover", "dragenter"].forEach((ev) => dz.addEventListener(ev, (e) => { e.preventDefault(); dz.classList.add("drag"); })); +["dragleave", "drop"].forEach((ev) => dz.addEventListener(ev, (e) => { e.preventDefault(); dz.classList.remove("drag"); })); +dz.addEventListener("drop", (e) => handleFiles(e.dataTransfer.files)); +$("#file-input").addEventListener("change", (e) => handleFiles(e.target.files)); + +function setStep(stage) { + const order = ["segmentation", "recognition", "confidence", "correction"]; + const idx = order.indexOf(stage); + document.querySelectorAll(".step").forEach((s) => { + const i = order.indexOf(s.dataset.stage); + s.classList.toggle("active", i === idx); + s.classList.toggle("done", i < idx); + }); +} + +async function handleFiles(fileList) { + const files = Array.from(fileList).filter((f) => f.type.startsWith("image/")); + if (!files.length) return; + + // ensure a document exists to ingest into + if (!state.doc) { + const d = await api("/documents", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ title: "My Manuscript" }), + }); + state.doc = { id: d.id, slug: d.slug, title: "My Manuscript", subtitle: "", page_count: 0 }; + } + + // preview first file + const reader = new FileReader(); + reader.onload = () => { $("#ingest-img").src = reader.result; }; + reader.readAsDataURL(files[0]); + + // build queue rows + const queue = $("#queue"); queue.innerHTML = ""; + const rows = files.map((f, i) => { + const row = el("div", "q-row" + (i === 0 ? " active" : "")); + row.innerHTML = `
+
${f.name}
+
+
Queued
`; + queue.appendChild(row); + return row; + }); + + const fd = new FormData(); + files.forEach((f) => fd.append("files", f)); + $("#ingest-status").textContent = "Uploading…"; + const { job_id } = await api(`/documents/${state.doc.id}/pages`, { method: "POST", body: fd }); + pollJob(job_id, rows, files.length); +} + +async function pollJob(jobId, rows, total) { + const timer = setInterval(async () => { + let job; + try { job = await api(`/jobs/${jobId}`); } catch (e) { return; } + const done = job.done_pages || 0; + const stage = job.stage || "segmentation"; + setStep(stage); + $("#ingest-status").textContent = job.message || job.status; + + rows.forEach((row, i) => { + const bar = row.querySelector(".q-bar > div"); + const st = row.querySelector(".q-status"); + row.classList.toggle("active", i === done && job.status !== "complete"); + if (i < done || job.status === "complete") { + bar.style.width = "100%"; st.textContent = "Complete"; st.className = "q-status done"; + } else if (i === done) { + const frac = ((job.progress || 0) * total) - done; + bar.style.width = Math.max(5, Math.min(100, frac * 100)) + "%"; + st.textContent = job.status === "recognizing" ? "Recognizing" : + job.status === "correcting" ? "Correcting" : + job.status === "scoring" ? "Scoring" : "Segmenting"; + } + }); + + if (job.status === "complete" || job.status === "failed") { + clearInterval(timer); + if (job.status === "complete") { + state.doc.page_count = (state.doc.page_count || 0) + total; + state.totalPages = state.doc.page_count; + await drawSegmentation(state.totalPages - total + 1); + } else { + $("#ingest-status").textContent = "Failed: " + (job.message || "error"); + } + } + }, 1000); +} + +/* overlay detected line boxes on the processed page preview */ +async function drawSegmentation(pageNumber) { + try { + const page = await api(`/documents/${state.doc.id}/pages/${pageNumber}`); + const img = $("#ingest-img"); + img.src = `${API}/documents/${state.doc.id}/pages/${pageNumber}/image`; + img.onload = () => { + const overlay = $("#seg-overlay"); + overlay.innerHTML = ""; + const r = img.getBoundingClientRect(); + const pr = overlay.getBoundingClientRect(); + const offX = r.left - pr.left, offY = r.top - pr.top; + const sx = r.width / page.width, sy = r.height / page.height; + page.lines.forEach((ln) => { + const [x0, y0, x1, y1] = ln.bbox; + const b = el("div", "seg-box" + (ln.needs_review ? " review" : "")); + b.style.left = offX + x0 * sx + "px"; + b.style.top = offY + y0 * sy + "px"; + b.style.width = (x1 - x0) * sx + "px"; + b.style.height = (y1 - y0) * sy + "px"; + overlay.appendChild(b); + }); + $("#ingest-status").textContent = `Done · ${page.lines.length} lines`; + }; + } catch (e) { /* ignore */ } +} +$("#open-reader").addEventListener("click", () => { showView("reader"); loadPage(state.totalPages); }); + +init(); diff --git a/app/frontend/index.html b/app/frontend/index.html new file mode 100644 index 0000000000000000000000000000000000000000..299f93775d94899623416695351dbc5d063474c4 --- /dev/null +++ b/app/frontend/index.html @@ -0,0 +1,151 @@ + + + + + + Inkference + + + + + + + +
+
+
I
+
Inkference
+
+
+
—
+
+ +
EH
+
+ + +
+
+
+
+ + Page – / – + +
+
manuscript scan
+
+
+ page scan +
No scan for this page
+
+
+ — + +
+
+ +
+
+
+
Transcription
+
Machine reading
+
+
+ +
+
–%
+
avg. confidence
+
+
+
+
+ Reading confidence + high +
+ low +
+ +
+
+
+
+ + +
+
+
+
Answers are drawn from the transcribed pages of this document
+
+
+
+ What happened at Plymouth? + Describe the weather and the ship + Summarise this voyage +
+
+ + +
+
Inkference can make mistakes. Verify answers against the cited pages.
+
+
+
+ + +
+
+
+
⤓
+
+
Drop scanned pages to ingest
+
JPEG, PNG · processed live on free CPU (slow)
+
+ +
+
+
Awaiting upload
+
+ processing page +
+
+
+
+ +
+
+
+
Batch ingestion
+
Live CPU pipeline
+
+
+
Fine-tuned TrOCR
+
Segmentation → Recognition → Confidence → Correction
+
+
+
+
1 Segmentation
+
+
2 Recognition
+
+
3 Confidence
+
+
4 Correction
+
+
+
+ Low-confidence lines are flagged for review. + +
+
+
+ + + + diff --git a/app/frontend/styles.css b/app/frontend/styles.css new file mode 100644 index 0000000000000000000000000000000000000000..304769758a658cbd16e0ed4078b240fb4ee432d2 --- /dev/null +++ b/app/frontend/styles.css @@ -0,0 +1,252 @@ +/* Inkference — palette & type from the design (design/Inkference.html) */ +:root { + --ink: #1f1810; /* darkest background */ + --ink-2: #16110b; /* scan viewer */ + --paper: #faf6ed; /* transcription paper */ + --paper-2: #f4eee2; /* app body */ + --accent: #8a3a2f; /* maroon */ + --accent-2: #c46a4f; /* active tab underline */ + --text: #2b211a; + --muted: #9a8a70; + --muted-2: #b6a78d; + --border: #ece1cd; + --serif: 'Spectral', Georgia, serif; + --sans: 'IBM Plex Sans', -apple-system, BlinkMacSystemFont, sans-serif; +} +* { margin: 0; padding: 0; box-sizing: border-box; } +html, body { height: 100%; } +body { + font-family: var(--sans); + background: var(--paper-2); + color: var(--text); + display: flex; + flex-direction: column; + height: 100vh; + overflow: hidden; +} +.muted { color: var(--muted); } +.small { font-size: 12px; } +.spacer { flex: 1; } +.hidden { display: none !important; } + +/* ===== Header ===== */ +.app-header { + height: 64px; flex: 0 0 64px; + background: var(--ink); + display: flex; align-items: center; gap: 18px; + padding: 0 24px; border-bottom: 1px solid #000; +} +.brand { display: flex; align-items: center; gap: 11px; } +.brand-mark { + width: 30px; height: 30px; border-radius: 8px; background: var(--accent); + display: flex; align-items: center; justify-content: center; + font-family: var(--serif); font-weight: 600; font-size: 18px; color: #f6ead8; +} +.brand-name { font-family: var(--serif); font-size: 20px; font-weight: 500; color: #f4ead8; letter-spacing: .02em; } +.divider { width: 1px; height: 26px; background: rgba(255,255,255,.13); } +.doc-subtitle { font-size: 13px; color: var(--muted-2); } +.tabs { display: flex; gap: 26px; margin-right: 12px; height: 100%; } +.tab { + background: none; border: none; cursor: pointer; + font-family: var(--sans); font-size: 14px; font-weight: 500; + color: #9a8a70; height: 100%; border-bottom: 2px solid transparent; +} +.tab.active { color: #f4ead8; border-bottom-color: var(--accent-2); } +.avatar { + width: 30px; height: 30px; border-radius: 50%; background: #3a2e20; + border: 1px solid rgba(255,255,255,.1); + display: flex; align-items: center; justify-content: center; + font-family: var(--serif); font-size: 13px; color: #d9c8a8; +} + +/* ===== Views (split layout shared by Reader + Upload) ===== */ +.view { flex: 1; display: flex; min-height: 0; } +.scan-pane { + width: 47%; flex: 0 0 47%; background: var(--ink-2); + display: flex; flex-direction: column; min-width: 0; +} +.paper-pane { + flex: 1; background: var(--paper); + display: flex; flex-direction: column; min-width: 0; + box-shadow: inset 14px 0 24px -18px rgba(40,28,12,.28); +} + +/* ===== Reader: scan viewer ===== */ +.scan-toolbar { + height: 48px; flex: 0 0 48px; display: flex; align-items: center; + justify-content: space-between; padding: 0 18px; + border-bottom: 1px solid rgba(255,255,255,.07); color: #d8ccb4; +} +.pager { display: flex; align-items: center; gap: 10px; font-size: 13px; } +.pager b { color: #f3ead8; } +.pager .muted { color: #7c6f58; } +.icon-btn { + background: none; border: none; color: #d8ccb4; cursor: pointer; + font-size: 20px; line-height: 1; padding: 2px 6px; border-radius: 6px; +} +.icon-btn:hover { background: rgba(255,255,255,.06); } +.zoom { color: #a99c84; font-size: 13px; } +.scan-stage { + flex: 1; display: flex; align-items: center; justify-content: center; + padding: 28px; min-height: 0; position: relative; +} +.scan-stage img { + max-width: 100%; max-height: 100%; object-fit: contain; + box-shadow: 0 18px 50px rgba(0,0,0,.55); + border: 1px solid rgba(220,200,160,.18); background: var(--paper); +} +.scan-empty { position: absolute; } +.scan-caption { + flex: 0 0 auto; padding: 11px 20px; display: flex; justify-content: space-between; + border-top: 1px solid rgba(255,255,255,.07); color: #8a7c64; font-size: 12px; +} + +/* ===== Reader: transcription paper ===== */ +.paper-header { + flex: 0 0 auto; padding: 20px 30px 15px; border-bottom: 1px solid var(--border); + display: flex; align-items: flex-start; justify-content: space-between; +} +.paper-title { font-family: var(--serif); font-size: 23px; font-weight: 500; } +.conf-readout { text-align: right; } +.conf-pct { font-family: var(--serif); font-size: 26px; font-weight: 500; line-height: 1; } +.conf-pct .pct { font-size: 16px; color: #8d7c61; } +.conf-label { font-size: 11px; letter-spacing: .06em; text-transform: uppercase; color: var(--muted); margin-top: 3px; } +.legend { + flex: 0 0 auto; padding: 13px 30px; display: flex; align-items: center; gap: 14px; + border-bottom: 1px solid #f0e7d6; +} +.legend-cap { font-size: 11px; letter-spacing: .08em; text-transform: uppercase; color: var(--muted); } +.legend-bar { + width: 170px; height: 8px; border-radius: 5px; + background: linear-gradient(90deg,#2b211a 0%,#5d4c3b 35%,#8a7765 62%,#b3a48f 82%,#cdbfa9 100%); +} +.transcription { + flex: 1; overflow: auto; padding: 22px 30px 40px; min-height: 0; + font-family: var(--serif); font-size: 19px; line-height: 1.92; max-width: 620px; +} +.transcription .t-line { margin-bottom: 4px; } +.transcription .t-line.review { background: rgba(255,206,120,.13); border-radius: 3px; } +.transcription w { display: inline; } /* confidence-tinted word, color set inline */ +.transcription w.qwen { color: #1a8c1a; } /* Qwen-corrected word (matches v2 notebook) */ +.transcription .empty { color: var(--muted); font-style: italic; font-size: 16px; } + +/* Raw / Corrected toggle in the Reader header */ +.header-right { display: flex; align-items: center; gap: 18px; } +.view-toggle { display: inline-flex; border: 1px solid #ddd0b8; border-radius: 8px; overflow: hidden; } +.toggle-opt { font-size: 12px; padding: 5px 11px; cursor: pointer; color: #8a7765; background: #fff; } +.toggle-opt.active { background: var(--accent); color: #f6ead8; } + +/* ===== Ask the Archive ===== */ +#view-ask { justify-content: center; } +.chat-wrap { + width: 100%; max-width: 720px; display: flex; flex-direction: column; + min-height: 0; padding: 0 24px; +} +.thread { flex: 1; overflow: auto; padding: 28px 0 18px; display: flex; flex-direction: column; gap: 22px; } +.intro { text-align: center; } +.intro span { + display: inline-block; padding: 5px 12px; border-radius: 999px; + background: #ece2cf; color: #7a6a51; font-size: 13px; +} +.bubble-q { + align-self: flex-end; max-width: 78%; background: #e7dcc7; color: #3a2f22; + padding: 12px 16px; border-radius: 16px 16px 4px 16px; font-size: 15px; line-height: 1.5; +} +.answer { align-self: stretch; } +.answer-head { display: flex; align-items: center; gap: 8px; margin-bottom: 8px; } +.answer-mark { + width: 20px; height: 20px; border-radius: 5px; background: var(--accent); + display: flex; align-items: center; justify-content: center; + font-family: var(--serif); font-size: 12px; color: #f6ead8; +} +.answer-who { font-size: 11px; letter-spacing: .08em; text-transform: uppercase; color: var(--muted); } +.answer-body { font-family: var(--serif); font-size: 17px; line-height: 1.72; color: var(--text); white-space: pre-wrap; } +.sources { display: flex; align-items: center; gap: 8px; margin-top: 12px; flex-wrap: wrap; } +.sources-cap { font-size: 11px; letter-spacing: .06em; text-transform: uppercase; color: #a99a82; } +.chip-page { + font-size: 12px; color: var(--accent); background: rgba(138,58,47,.09); + border: 1px solid rgba(138,58,47,.2); border-radius: 999px; padding: 3px 10px; cursor: pointer; +} +.chip-page:hover { background: rgba(138,58,47,.16); } +.dock-area { flex: 0 0 auto; padding-bottom: 20px; } +.suggestions { display: flex; gap: 8px; margin-bottom: 12px; flex-wrap: wrap; } +.chip-suggest { + font-size: 13px; color: #6f5d49; background: #fff; border: 1px solid #e2d6bf; + border-radius: 999px; padding: 7px 14px; cursor: pointer; +} +.chip-suggest:hover { border-color: var(--accent); } +.dock { + display: flex; align-items: center; gap: 10px; background: #fff; + border: 1px solid #ddd0b8; border-radius: 14px; padding: 8px 8px 8px 18px; + box-shadow: 0 6px 20px rgba(60,42,18,.07); +} +.dock input { flex: 1; border: none; outline: none; font-size: 16px; font-family: var(--sans); background: none; color: var(--text); } +.send-btn { + width: 40px; height: 40px; border: none; border-radius: 11px; background: var(--accent); + color: #fff; font-size: 18px; cursor: pointer; +} +.disclaimer { text-align: center; font-size: 11.5px; color: #a99a82; margin-top: 10px; } + +/* ===== Upload & Process ===== */ +.dropzone { + margin: 20px; border: 1.5px dashed rgba(220,200,160,.32); border-radius: 12px; + padding: 22px; display: flex; align-items: center; gap: 16px; + background: rgba(255,255,255,.02); color: #f3ead8; +} +.dropzone.drag { border-color: var(--accent-2); background: rgba(196,106,79,.08); } +.drop-icon { + width: 46px; height: 46px; flex: 0 0 46px; border-radius: 11px; + background: rgba(138,58,47,.22); border: 1px solid rgba(196,106,79,.35); + display: flex; align-items: center; justify-content: center; font-size: 22px; color: #e7b9a5; +} +.drop-title { font-size: 14.5px; font-weight: 500; } +.browse-btn { + font-size: 13px; font-weight: 500; color: var(--ink); background: #d8c3a0; + border-radius: 9px; padding: 8px 14px; cursor: pointer; +} +.active-detail { flex: 1; padding: 0 20px 14px; display: flex; flex-direction: column; min-height: 0; } +.active-detail .muted { color: #8a7c64; margin-bottom: 10px; } +.active-preview { + position: relative; flex: 1; display: flex; align-items: center; justify-content: center; min-height: 0; +} +.active-preview img { max-width: 100%; max-height: 100%; object-fit: contain; border-radius: 3px; box-shadow: 0 16px 44px rgba(0,0,0,.5); } +#ingest-img:not([src]), #scan-img:not([src]) { display: none; } +.seg-overlay { position: absolute; inset: 0; pointer-events: none; } +.seg-box { position: absolute; border: 1.5px solid rgba(124,224,176,.85); background: rgba(124,224,176,.10); border-radius: 2px; } +.seg-box.review { border-color: rgba(255,206,120,.9); background: rgba(255,206,120,.12); } + +.model-card { text-align: right; } +.model-name { display: flex; align-items: center; gap: 7px; justify-content: flex-end; font-family: var(--serif); font-size: 14px; } +.model-name .dot { width: 7px; height: 7px; border-radius: 50%; background: #5f7355; } +.stepper { display: flex; align-items: center; padding: 16px 28px; border-bottom: 1px solid var(--border); } +.step { display: flex; align-items: center; gap: 9px; font-size: 13px; font-weight: 500; color: #a3937a; } +.step-dot { + width: 24px; height: 24px; border-radius: 50%; background: #e7dcc7; + display: flex; align-items: center; justify-content: center; font-size: 12px; font-weight: 600; color: #a3937a; +} +.step.active { color: var(--text); } +.step.active .step-dot { background: var(--accent); color: #f6ead8; } +.step.done .step-dot { background: #5f7355; color: #fff; } +.step.done { color: var(--text); } +.step-line { flex: 1; height: 2px; background: #ddd0b8; margin: 0 12px; } +.queue { flex: 1; overflow: auto; padding: 8px 16px 16px; min-height: 0; } +.q-row { display: flex; align-items: center; gap: 14px; padding: 14px 12px; border-bottom: 1px solid #f0e7d6; } +.q-row.active { background: #fff; border-radius: 10px; box-shadow: 0 3px 14px rgba(60,42,18,.06); } +.q-thumb { width: 38px; height: 50px; flex: 0 0 38px; border-radius: 3px; background: #e7dcc7; border: 1px solid #d8c9ad; overflow: hidden; } +.q-thumb img { width: 100%; height: 100%; object-fit: cover; } +.q-main { flex: 1; min-width: 0; } +.q-name { font-size: 14px; font-weight: 500; } +.q-bar { height: 5px; border-radius: 3px; background: #ece1cd; overflow: hidden; margin-top: 6px; max-width: 220px; } +.q-bar > div { height: 100%; background: var(--accent); border-radius: 3px; width: 0; transition: width .3s; } +.q-status { font-size: 12.5px; font-weight: 500; } +.q-status.done { color: #5f7355; } +.upload-footer { + flex: 0 0 auto; padding: 14px 24px; border-top: 1px solid var(--border); + display: flex; align-items: center; justify-content: space-between; +} +.primary-btn { font-size: 13px; font-weight: 500; color: #f6ead8; background: var(--accent); border: none; border-radius: 9px; padding: 9px 16px; cursor: pointer; } + +@media (max-width: 860px) { + .view { flex-direction: column; } + .scan-pane { width: 100%; flex: 0 0 42%; } +} diff --git a/app/pyproject.toml b/app/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..5257b3bdde4793b507cf35f1e2eb18c5ddd7d1db --- /dev/null +++ b/app/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "inkference" +version = "0.1.0" +description = "HTR + RAG platform for historical handwritten documents (Inkference)" +readme = "README.md" +requires-python = ">=3.11" +# Runtime deps are pinned in requirements.txt (single source of truth for the +# Docker Space build). This list mirrors the import-level requirements so the +# package is installable with `pip install -e .` for local dev. +dependencies = [ + "torch", + "torchvision", + "transformers", + "Pillow", + "numpy", + "pandas", +] + +[project.optional-dependencies] +htr = ["kraken", "doclayout-yolo"] +rag = ["sentence-transformers", "faiss-cpu"] +api = ["fastapi", "uvicorn[standard]", "python-multipart"] +eval = ["jiwer", "evaluate"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +inkference = ["py.typed"] diff --git a/app/src/inkference.egg-info/PKG-INFO b/app/src/inkference.egg-info/PKG-INFO new file mode 100644 index 0000000000000000000000000000000000000000..9fc37530b262503e2e0790cdb5ae58606e57b134 --- /dev/null +++ b/app/src/inkference.egg-info/PKG-INFO @@ -0,0 +1,63 @@ +Metadata-Version: 2.4 +Name: inkference +Version: 0.1.0 +Summary: HTR + RAG platform for historical handwritten documents (Inkference) +Requires-Python: >=3.11 +Description-Content-Type: text/markdown +Requires-Dist: torch +Requires-Dist: torchvision +Requires-Dist: transformers +Requires-Dist: Pillow +Requires-Dist: numpy +Requires-Dist: pandas +Provides-Extra: htr +Requires-Dist: kraken; extra == "htr" +Requires-Dist: doclayout-yolo; extra == "htr" +Provides-Extra: rag +Requires-Dist: sentence-transformers; extra == "rag" +Requires-Dist: faiss-cpu; extra == "rag" +Provides-Extra: api +Requires-Dist: fastapi; extra == "api" +Requires-Dist: uvicorn[standard]; extra == "api" +Requires-Dist: python-multipart; extra == "api" +Provides-Extra: eval +Requires-Dist: jiwer; extra == "eval" +Requires-Dist: evaluate; extra == "eval" + +# Inkference — application + +Everything in this `app/` folder was built **on top of** the HTR research repo to +turn it into the Inkference platform (Reader · Ask the Archive · Upload). The +original research assets (notebooks, `data/`, `models/`, `transcriptions/`) remain +at the repo root, untouched. + +``` +app/ +├── pyproject.toml installable package metadata +├── src/inkference/ +│ ├── config.py env-driven config + path layout +│ ├── schemas.py domain types (Word / Line / PageResult, enums) +│ ├── htr/ segmentation → recognition (+confidence) → pipeline +│ ├── rag/ chunk + embed + FAISS retrieval + grounded answers +│ ├── store/ SQLite store + seed loader (from ../transcriptions) +│ └── api/ FastAPI app + background ingestion jobs +├── frontend/ Inkference UI (index.html / styles.css / app.js) +├── scripts/ prepare_model.py (assemble inference model) +└── deploy/ Dockerfile + Space README + lean CPU requirements +``` + +## Run locally (from the repo root) + +```bash +pip install -r requirements.txt && pip install -e ./app +python -m inkference.store.seed # loads the demo corpus +uvicorn inkference.api.main:app --reload # http://127.0.0.1:8000 +``` + +See [deploy/README.md](deploy/README.md) for free Hugging Face Space deployment and +the full environment-variable reference. The build plan and progress log live in +[../projectNotes/inkference_platform_plan.md](../projectNotes/inkference_platform_plan.md). + +## For empty data or seed +INKFERENCE_DATA_ROOT=$(pwd)/app/.inkference_data_fresh \ + uvicorn inkference.api.main:app --port 8000 --log-level debug 2>&1 | tee app/server.log diff --git a/app/src/inkference.egg-info/SOURCES.txt b/app/src/inkference.egg-info/SOURCES.txt new file mode 100644 index 0000000000000000000000000000000000000000..b69e08b7a4485c756712c93a4d1f5b329bde313f --- /dev/null +++ b/app/src/inkference.egg-info/SOURCES.txt @@ -0,0 +1,26 @@ +README.md +pyproject.toml +src/inkference/__init__.py +src/inkference/config.py +src/inkference/schemas.py +src/inkference.egg-info/PKG-INFO +src/inkference.egg-info/SOURCES.txt +src/inkference.egg-info/dependency_links.txt +src/inkference.egg-info/requires.txt +src/inkference.egg-info/top_level.txt +src/inkference/api/__init__.py +src/inkference/api/main.py +src/inkference/api/services.py +src/inkference/htr/__init__.py +src/inkference/htr/align.py +src/inkference/htr/correction.py +src/inkference/htr/pipeline.py +src/inkference/htr/recognition.py +src/inkference/htr/segmentation.py +src/inkference/rag/__init__.py +src/inkference/rag/answer.py +src/inkference/rag/index.py +src/inkference/rag/llm.py +src/inkference/store/__init__.py +src/inkference/store/seed.py +src/inkference/store/store.py \ No newline at end of file diff --git a/app/src/inkference.egg-info/dependency_links.txt b/app/src/inkference.egg-info/dependency_links.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/app/src/inkference.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/app/src/inkference.egg-info/requires.txt b/app/src/inkference.egg-info/requires.txt new file mode 100644 index 0000000000000000000000000000000000000000..dbbc2b3f9e4aaf66aa43dc7b1014ae60e0d1c0ef --- /dev/null +++ b/app/src/inkference.egg-info/requires.txt @@ -0,0 +1,23 @@ +torch +torchvision +transformers +Pillow +numpy +pandas + +[api] +fastapi +uvicorn[standard] +python-multipart + +[eval] +jiwer +evaluate + +[htr] +kraken +doclayout-yolo + +[rag] +sentence-transformers +faiss-cpu diff --git a/app/src/inkference.egg-info/top_level.txt b/app/src/inkference.egg-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..44b4244dedcb43e5d630755154548ba9b44ad0b1 --- /dev/null +++ b/app/src/inkference.egg-info/top_level.txt @@ -0,0 +1 @@ +inkference diff --git a/app/src/inkference/__init__.py b/app/src/inkference/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dee802eca84edf6c77748de4ae6e66ac952a1521 --- /dev/null +++ b/app/src/inkference/__init__.py @@ -0,0 +1,10 @@ +"""Inkference — HTR + RAG platform for historical handwritten documents. + +Subpackages: + htr line segmentation + TrOCR recognition + confidence scoring + rag embeddings + FAISS retrieval + grounded answer generation + store SQLite document/page/line/word store + filesystem assets + api FastAPI backend +""" + +__version__ = "0.1.0" diff --git a/app/src/inkference/__pycache__/__init__.cpython-311.pyc b/app/src/inkference/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e4ffb2037be955fa02cb34990132c8258742b5ce Binary files /dev/null and b/app/src/inkference/__pycache__/__init__.cpython-311.pyc differ diff --git a/app/src/inkference/__pycache__/config.cpython-311.pyc b/app/src/inkference/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbc441c48f8c3a034678c1e831ae9b104a0e1404 Binary files /dev/null and b/app/src/inkference/__pycache__/config.cpython-311.pyc differ diff --git a/app/src/inkference/__pycache__/schemas.cpython-311.pyc b/app/src/inkference/__pycache__/schemas.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f8e18c792aa5503709d2bf90bf56ef9872e78d6 Binary files /dev/null and b/app/src/inkference/__pycache__/schemas.cpython-311.pyc differ diff --git a/app/src/inkference/api/__init__.py b/app/src/inkference/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d0ff3ff49b7c7af4679800a64a422df1cc6ddc2 --- /dev/null +++ b/app/src/inkference/api/__init__.py @@ -0,0 +1 @@ +"""FastAPI backend for the Inkference platform.""" diff --git a/app/src/inkference/api/__pycache__/__init__.cpython-311.pyc b/app/src/inkference/api/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..447e15544651ebca9fb2e9238087c15c25d4ddc1 Binary files /dev/null and b/app/src/inkference/api/__pycache__/__init__.cpython-311.pyc differ diff --git a/app/src/inkference/api/__pycache__/main.cpython-311.pyc b/app/src/inkference/api/__pycache__/main.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b96dd67c2e42cfb0694c9ed00bbf49fef9f51421 Binary files /dev/null and b/app/src/inkference/api/__pycache__/main.cpython-311.pyc differ diff --git a/app/src/inkference/api/__pycache__/services.cpython-311.pyc b/app/src/inkference/api/__pycache__/services.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f31177c79b8b158efb5648c52b9e22d5558a3048 Binary files /dev/null and b/app/src/inkference/api/__pycache__/services.cpython-311.pyc differ diff --git a/app/src/inkference/api/main.py b/app/src/inkference/api/main.py new file mode 100644 index 0000000000000000000000000000000000000000..d0e6674988d42fa28dbb8484ea57876732ec06ea --- /dev/null +++ b/app/src/inkference/api/main.py @@ -0,0 +1,177 @@ +"""Inkference FastAPI app. + +Endpoints (see projectNotes/inkference_platform_plan.md): + GET /api/health + GET /api/documents + POST /api/documents + GET /api/documents/{id} + POST /api/documents/{id}/pages upload scans -> background ingest job + GET /api/documents/{id}/pages/{n} transcription (lines + words + confidence) + GET /api/documents/{id}/pages/{n}/image + POST /api/documents/{id}/ask RAG answer + source pages + GET /api/jobs/{id} ingestion progress +""" +from __future__ import annotations + +import os +import re +import unicodedata +from pathlib import Path + +from fastapi import FastAPI, File, HTTPException, UploadFile +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel + +from ..config import ( + FRONTEND_DIR, + correction as correction_cfg, + htr as htr_cfg, + rag as rag_cfg, +) +from ..rag.answer import answer_question +from . import services + +app = FastAPI(title="Inkference", version="0.1.0") + +app.add_middleware( + CORSMiddleware, + allow_origins=os.getenv("CORS_ORIGINS", "*").split(","), + allow_methods=["*"], + allow_headers=["*"], +) + + +# --------------------------------------------------------------------------- # +# request models +# --------------------------------------------------------------------------- # +class CreateDocument(BaseModel): + title: str + slug: str | None = None + subtitle: str | None = None + + +class AskRequest(BaseModel): + question: str + top_k: int | None = None + + +def _slugify(text: str) -> str: + text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode() + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") or "document" + + +# --------------------------------------------------------------------------- # +# documents +# --------------------------------------------------------------------------- # +@app.get("/api/health") +def health() -> dict: + return {"status": "ok", "trocr_model": htr_cfg.trocr_model_id, + "llm_provider": rag_cfg.llm_provider, + "llm_configured": bool(rag_cfg.llm_api_key), + "correction_enabled": correction_cfg.enabled, + "correction_backend": correction_cfg.backend, + "correction_model": (correction_cfg.api_model if correction_cfg.backend == "api" + else correction_cfg.model_id)} + + +@app.get("/api/documents") +def list_documents() -> list[dict]: + return services.get_store().list_documents() + + +@app.post("/api/documents") +def create_document(body: CreateDocument) -> dict: + store = services.get_store() + slug = body.slug or _slugify(body.title) + if store.get_document_by_slug(slug): + raise HTTPException(409, f"document with slug '{slug}' already exists") + doc_id = store.create_document(title=body.title, slug=slug, subtitle=body.subtitle) + return {"id": doc_id, "slug": slug} + + +@app.get("/api/documents/{doc_id}") +def get_document(doc_id: int) -> dict: + doc = services.get_store().get_document(doc_id) + if not doc: + raise HTTPException(404, "document not found") + return doc + + +# --------------------------------------------------------------------------- # +# pages: upload + ingest +# --------------------------------------------------------------------------- # +@app.post("/api/documents/{doc_id}/pages") +async def upload_pages(doc_id: int, files: list[UploadFile] = File(...)) -> dict: + store = services.get_store() + doc = store.get_document(doc_id) + if not doc: + raise HTTPException(404, "document not found") + + assets = store.cfg.assets_dir / doc["slug"] + assets.mkdir(parents=True, exist_ok=True) + start = doc["page_count"] + 1 + + specs: list[tuple[int, int, str]] = [] + for offset, upload in enumerate(files): + page_number = start + offset + ext = Path(upload.filename or "").suffix.lower() or ".png" + dest = assets / f"page_{page_number:04d}{ext}" + dest.write_bytes(await upload.read()) + page_id = store.add_page(doc_id, page_number, image_path=str(dest)) + specs.append((page_id, page_number, str(dest))) + + job_id = store.create_job(doc_id, total_pages=len(specs)) + services.submit_ingest(doc_id, specs, job_id) + return {"job_id": job_id, "pages": [s[1] for s in specs]} + + +@app.get("/api/jobs/{job_id}") +def get_job(job_id: int) -> dict: + job = services.get_store().get_job(job_id) + if not job: + raise HTTPException(404, "job not found") + return job + + +# --------------------------------------------------------------------------- # +# pages: read +# --------------------------------------------------------------------------- # +@app.get("/api/documents/{doc_id}/pages/{page_number}") +def get_page(doc_id: int, page_number: int) -> dict: + page = services.get_store().get_page(doc_id, page_number) + if not page: + raise HTTPException(404, "page not found") + page.pop("image_path", None) # internal path; image served via dedicated route + return page + + +@app.get("/api/documents/{doc_id}/pages/{page_number}/image") +def get_page_image(doc_id: int, page_number: int): + path = services.get_store().get_page_image_path(doc_id, page_number) + if not path or not Path(path).exists(): + raise HTTPException(404, "page image not found") + return FileResponse(path) + + +# --------------------------------------------------------------------------- # +# ask the archive (RAG) +# --------------------------------------------------------------------------- # +@app.post("/api/documents/{doc_id}/ask") +def ask(doc_id: int, body: AskRequest) -> dict: + store = services.get_store() + if not store.get_document(doc_id): + raise HTTPException(404, "document not found") + index = services.get_index() + if not index.exists(doc_id): + index.build_from_store(doc_id, store) + ans = answer_question(doc_id, body.question, index, top_k=body.top_k) + return ans.to_dict() + + +# --------------------------------------------------------------------------- # +# static frontend (mounted last so /api/* wins) +# --------------------------------------------------------------------------- # +if FRONTEND_DIR.is_dir(): + app.mount("/", StaticFiles(directory=str(FRONTEND_DIR), html=True), name="frontend") diff --git a/app/src/inkference/api/services.py b/app/src/inkference/api/services.py new file mode 100644 index 0000000000000000000000000000000000000000..7eb25115d748f6b9842ba69896d218df57028425 --- /dev/null +++ b/app/src/inkference/api/services.py @@ -0,0 +1,93 @@ +"""Shared singletons and the background ingestion job runner. + +Heavy objects (TrOCR pipeline, embedder) load lazily on first use. Ingestion runs +on a single-worker thread pool so CPU-bound OCR never blocks the event loop and +free-CPU pages process serially; the frontend polls GET /jobs/{id} for progress. +""" +from __future__ import annotations + +import threading +import traceback +from concurrent.futures import ThreadPoolExecutor + +from ..config import htr as htr_cfg +from ..htr.pipeline import HTRPipeline +from ..rag.index import RagIndex +from ..schemas import JobStatus, Stage +from ..store import DocumentStore + +# Map a pipeline Stage -> the job status shown in the queue UI. +_STAGE_STATUS = { + Stage.SEGMENTATION: JobStatus.SEGMENTING, + Stage.RECOGNITION: JobStatus.RECOGNIZING, + Stage.CONFIDENCE: JobStatus.SCORING, + Stage.CORRECTION: JobStatus.CORRECTING, +} + +_store: DocumentStore | None = None +_pipeline: HTRPipeline | None = None +_index: RagIndex | None = None +_pipeline_lock = threading.Lock() +_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="ingest") + + +def get_store() -> DocumentStore: + global _store + if _store is None: + _store = DocumentStore() + return _store + + +def get_index() -> RagIndex: + global _index + if _index is None: + _index = RagIndex() + return _index + + +def get_pipeline() -> HTRPipeline: + global _pipeline + if _pipeline is None: + with _pipeline_lock: + if _pipeline is None: + _pipeline = HTRPipeline(htr_cfg) + return _pipeline + + +def submit_ingest(doc_id: int, page_specs: list[tuple[int, int, str]], job_id: int) -> None: + """page_specs = [(page_id, page_number, image_path), ...]""" + _executor.submit(_run_ingest, doc_id, page_specs, job_id) + + +def _run_ingest(doc_id: int, page_specs: list[tuple[int, int, str]], job_id: int) -> None: + store = get_store() + total = len(page_specs) + store.update_job(job_id, status=JobStatus.QUEUED, total_pages=total, done_pages=0) + try: + pipeline = get_pipeline() + for done, (page_id, page_number, image_path) in enumerate(page_specs): + def progress(stage: Stage, frac: float, msg: str, _done=done) -> None: + overall = (_done + frac) / total + store.update_job( + job_id, + status=_STAGE_STATUS.get(stage, JobStatus.RECOGNIZING), + stage=stage, + progress=round(overall, 4), + message=f"Page {page_number} — {msg}", + ) + + store.set_page_status(page_id, "processing") + result = pipeline.process_path(image_path, page_number, progress) + store.save_page_result(page_id, result) + store.update_job(job_id, done_pages=done + 1) + + # Rebuild the retrieval index now that new pages exist. + get_index().build_from_store(doc_id, store) + store.update_job( + job_id, status=JobStatus.COMPLETE, progress=1.0, message="Complete" + ) + except Exception as exc: # surface failure to the job poller + store.update_job( + job_id, status=JobStatus.FAILED, + error=f"{exc}\n{traceback.format_exc()}", message=str(exc), + ) diff --git a/app/src/inkference/config.py b/app/src/inkference/config.py new file mode 100644 index 0000000000000000000000000000000000000000..b77a14d8a8ffd0f25a58b622a5aac3e841233917 --- /dev/null +++ b/app/src/inkference/config.py @@ -0,0 +1,187 @@ +"""Central configuration. All values overridable via environment variables so the +same code runs on a laptop, a free HF Docker Space, or a GPU prod host.""" +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path + + +def _env_bool(key: str, default: bool) -> bool: + val = os.getenv(key) + if val is None: + return default + return val.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_float(key: str, default: float) -> float: + val = os.getenv(key) + return float(val) if val else default + + +def _env_int(key: str, default: int) -> int: + val = os.getenv(key) + return int(val) if val else default + + +# Path layout: this file is app/src/inkference/config.py +# parents[2] = app/ (APP_ROOT — holds frontend/, deploy/, scripts/) +# parents[3] = repo root (PROJECT_ROOT — holds transcriptions/, models/, data/) +APP_ROOT = Path(__file__).resolve().parents[2] +PROJECT_ROOT = Path(os.getenv("INKFERENCE_PROJECT_ROOT", APP_ROOT.parent)) +# Back-compat alias (older modules referenced REPO_ROOT). +REPO_ROOT = PROJECT_ROOT + +# Load .env for local dev (app/.env takes precedence over repo/.env). Real +# environment variables always win — load_dotenv does not override them. +try: + from dotenv import load_dotenv + + load_dotenv(APP_ROOT / ".env") + load_dotenv(PROJECT_ROOT / ".env") +except Exception: + pass + +DATA_ROOT = Path(os.getenv("INKFERENCE_DATA_ROOT", APP_ROOT / ".inkference_data")) +FRONTEND_DIR = Path(os.getenv("INKFERENCE_FRONTEND_DIR", APP_ROOT / "frontend")) +TRANSCRIPTIONS_ROOT = Path( + os.getenv("INKFERENCE_TRANSCRIPTIONS_ROOT", PROJECT_ROOT / "transcriptions") +) + + +@dataclass +class HTRConfig: + """Knobs ported from info_files/line_segmentation_output.txt plus runtime ones.""" + + # Recognition model: a HF Hub id (prod) or a local checkpoint path (dev). + trocr_model_id: str = field( + default_factory=lambda: os.getenv( + "TROCR_MODEL_ID", "microsoft/trocr-base-handwritten" + ) + ) + # local | remote (remote = serverless GPU endpoint in production) + executor: str = field(default_factory=lambda: os.getenv("HTR_EXECUTOR", "local")) + device: str = field(default_factory=lambda: os.getenv("HTR_DEVICE", "auto")) + + # Segmentation knobs (from the notebook config cell). + pad_x: int = 6 + pad_y: int = 2 + mask_to_polygon: bool = True + poly_pad: int = 2 + min_w: int = 40 + min_h: int = 12 + use_layout_filter: bool = field( + default_factory=lambda: _env_bool("HTR_USE_LAYOUT_FILTER", False) + ) + + # Free-CPU survival: cap the long edge before segmentation/recognition. + max_page_long_edge: int = field( + default_factory=lambda: _env_int("HTR_MAX_LONG_EDGE", 2000) + ) + num_beams: int = field(default_factory=lambda: _env_int("HTR_NUM_BEAMS", 1)) + max_target_length: int = 128 + recognition_batch_size: int = field( + default_factory=lambda: _env_int("HTR_BATCH_SIZE", 8) + ) + + # Confidence: words below this are "needs review" (design's <60% flag). + low_confidence_threshold: float = field( + default_factory=lambda: _env_float("HTR_LOW_CONF", 0.60) + ) + + def __post_init__(self) -> None: + # If trocr_model_id names a local folder (absolute, CWD-relative, or + # PROJECT_ROOT-relative), resolve it to an absolute path so the model + # loads regardless of working directory. Otherwise leave it as a Hub id. + mid = self.trocr_model_id + for candidate in (Path(mid), PROJECT_ROOT / mid): + if candidate.exists(): + self.trocr_model_id = str(candidate.resolve()) + break + + +@dataclass +class RAGConfig: + embed_model_id: str = field( + default_factory=lambda: os.getenv( + "EMBED_MODEL_ID", "sentence-transformers/all-MiniLM-L6-v2" + ) + ) + top_k: int = field(default_factory=lambda: _env_int("RAG_TOP_K", 5)) + # Provider for the written answer: gemini | groq | claude | openai + llm_provider: str = field(default_factory=lambda: os.getenv("LLM_PROVIDER", "gemini")) + llm_model: str = field(default_factory=lambda: os.getenv("LLM_MODEL", "")) + llm_api_key: str = field(default_factory=lambda: os.getenv("LLM_API_KEY", "")) + + _PROVIDER_KEYS = { + "gemini": ("GEMINI_API_KEY", "GOOGLE_API_KEY"), + "groq": ("GROQ_API_KEY",), + "openai": ("OPENAI_API_KEY",), + "claude": ("ANTHROPIC_API_KEY",), + } + + def __post_init__(self) -> None: + # If no explicit LLM_API_KEY, pull the key for the SELECTED provider so a + # provider switch (e.g. gemini -> groq) uses the right key automatically. + if not self.llm_api_key: + for env in self._PROVIDER_KEYS.get((self.llm_provider or "").lower(), ()): + if os.getenv(env): + self.llm_api_key = os.getenv(env) + break + + +@dataclass +class StoreConfig: + db_path: Path = field(default_factory=lambda: DATA_ROOT / "inkference.db") + assets_dir: Path = field(default_factory=lambda: DATA_ROOT / "assets") + index_dir: Path = field(default_factory=lambda: DATA_ROOT / "index") + + +@dataclass +class CorrectionConfig: + """Qwen few-shot page-level post-correction (runs after recognition).""" + + enabled: bool = field(default_factory=lambda: _env_bool("CORRECTION_ENABLED", True)) + # backend: local (transformers) | api (OpenAI-compatible hosted Qwen) + backend: str = field(default_factory=lambda: os.getenv("CORRECTION_BACKEND", "local")) + # local: a CPU-friendly Qwen3 by default; use Qwen/Qwen3-4B on a GPU. + model_id: str = field( + default_factory=lambda: os.getenv("CORRECTION_MODEL_ID", "Qwen/Qwen3-1.7B") + ) + device: str = field(default_factory=lambda: os.getenv("CORRECTION_DEVICE", "auto")) + # api backend (hosted Qwen via Groq/OpenRouter/DashScope/etc.) + api_base: str = field(default_factory=lambda: os.getenv("CORRECTION_API_BASE", "")) + # falls back to GROQ_API_KEY / OPENROUTER_API_KEY so an existing key just works + api_key: str = field( + default_factory=lambda: ( + os.getenv("CORRECTION_API_KEY") + or os.getenv("GROQ_API_KEY") + or os.getenv("OPENROUTER_API_KEY") + or "" + ) + ) + api_model: str = field( + default_factory=lambda: os.getenv("CORRECTION_API_MODEL", "qwen/qwen3-32b") + ) + # few-shot + num_shots: int = field(default_factory=lambda: _env_int("CORRECTION_NUM_SHOTS", 2)) + examples_path: Path = field( + default_factory=lambda: Path( + os.getenv( + "CORRECTION_EXAMPLES", + str(Path(__file__).resolve().parent / "htr" / "few_shot_examples.json"), + ) + ) + ) + max_new_tokens: int = field( + default_factory=lambda: _env_int("CORRECTION_MAX_NEW_TOKENS", 2048) + ) + temperature: float = field( + default_factory=lambda: _env_float("CORRECTION_TEMPERATURE", 0.2) + ) + + +htr = HTRConfig() +rag = RAGConfig() +store = StoreConfig() +correction = CorrectionConfig() diff --git a/app/src/inkference/htr/__init__.py b/app/src/inkference/htr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7e0abb221eaff6650ff7bc702c2ae3c789b27980 --- /dev/null +++ b/app/src/inkference/htr/__init__.py @@ -0,0 +1 @@ +"""HTR core: segmentation -> recognition -> confidence.""" diff --git a/app/src/inkference/htr/__pycache__/__init__.cpython-311.pyc b/app/src/inkference/htr/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce3acc90e774d8e0a565efc149776dfe61a37d53 Binary files /dev/null and b/app/src/inkference/htr/__pycache__/__init__.cpython-311.pyc differ diff --git a/app/src/inkference/htr/__pycache__/align.cpython-311.pyc b/app/src/inkference/htr/__pycache__/align.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..765dd811aa6837037e5d97fe4a0fdb14e870cdef Binary files /dev/null and b/app/src/inkference/htr/__pycache__/align.cpython-311.pyc differ diff --git a/app/src/inkference/htr/__pycache__/correction.cpython-311.pyc b/app/src/inkference/htr/__pycache__/correction.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32f31241cc3db0e334e43fbdcee4d8fefd363482 Binary files /dev/null and b/app/src/inkference/htr/__pycache__/correction.cpython-311.pyc differ diff --git a/app/src/inkference/htr/__pycache__/pipeline.cpython-311.pyc b/app/src/inkference/htr/__pycache__/pipeline.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..42a6c15e7e7eeb56d99793bff36dc631671350e8 Binary files /dev/null and b/app/src/inkference/htr/__pycache__/pipeline.cpython-311.pyc differ diff --git a/app/src/inkference/htr/__pycache__/recognition.cpython-311.pyc b/app/src/inkference/htr/__pycache__/recognition.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c598f3118f8944cd0b1ca2a95c425dca0152151 Binary files /dev/null and b/app/src/inkference/htr/__pycache__/recognition.cpython-311.pyc differ diff --git a/app/src/inkference/htr/__pycache__/segmentation.cpython-311.pyc b/app/src/inkference/htr/__pycache__/segmentation.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..14f544fe16d096a5ab27594cea8afbab53b291f8 Binary files /dev/null and b/app/src/inkference/htr/__pycache__/segmentation.cpython-311.pyc differ diff --git a/app/src/inkference/htr/align.py b/app/src/inkference/htr/align.py new file mode 100644 index 0000000000000000000000000000000000000000..a62aea4d47a5a387e8fcf9a91f28a43fd6f3e3a1 --- /dev/null +++ b/app/src/inkference/htr/align.py @@ -0,0 +1,123 @@ +"""Align corrected words back to the original TrOCR words' confidence. + +Ported from post_correction_full_test_v2.ipynb (`align_words`/`_normalise`). Page-level +correction rewrites text, so to keep the per-word confidence tint in the Reader we map +each corrected word to the original word(s) it came from via difflib.SequenceMatcher: + + equal -> keep the original word's confidence + replace -> average confidence of the replaced originals, flagged qwen_replaced + insert -> a word Qwen added with no original -> default confidence, qwen_replaced + delete -> original dropped -> skipped + +Returns a fresh list[Word] for the corrected line. +""" +from __future__ import annotations + +import difflib +import re + +from ..schemas import Word + +# Confidence for words Qwen inserted that have no original counterpart. +DEFAULT_INSERT_CONF = 0.5 + + +def _normalise(word: str) -> str: + return re.sub(r"[^\w]", "", word.lower()) + + +def align_line( + original_words: list[Word], + corrected_text: str, + low_conf_threshold: float = 0.60, +) -> list[Word]: + """Map the corrected line's words onto the original words' confidences.""" + corr_tokens = corrected_text.split() + if not corr_tokens: + return [] + if not original_words: + return [ + Word(text=t, confidence=DEFAULT_INSERT_CONF, + needs_review=DEFAULT_INSERT_CONF < low_conf_threshold, qwen_replaced=True) + for t in corr_tokens + ] + + orig_keys = [_normalise(w.text) for w in original_words] + corr_keys = [_normalise(t) for t in corr_tokens] + matcher = difflib.SequenceMatcher(None, orig_keys, corr_keys, autojunk=False) + + out: list[Word] = [] + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if tag == "equal": + for i, j in zip(range(i1, i2), range(j1, j2)): + o = original_words[i] + out.append(Word(text=corr_tokens[j], confidence=o.confidence, + needs_review=o.needs_review, qwen_replaced=False)) + elif tag == "replace": + confs = [original_words[i].confidence for i in range(i1, i2)] + avg = sum(confs) / len(confs) if confs else DEFAULT_INSERT_CONF + for j in range(j1, j2): + out.append(Word(text=corr_tokens[j], confidence=round(avg, 4), + needs_review=avg < low_conf_threshold, qwen_replaced=True)) + elif tag == "insert": + for j in range(j1, j2): + out.append(Word(text=corr_tokens[j], confidence=DEFAULT_INSERT_CONF, + needs_review=DEFAULT_INSERT_CONF < low_conf_threshold, + qwen_replaced=True)) + # tag == "delete": original word dropped by the corrector -> nothing emitted + return out + + +def align_page( + conf_words: list[tuple[str, float]], + corrected_lines: list[str], + low_conf_threshold: float = 0.60, +) -> list[list[Word]]: + """Page-level alignment (faithful to post_correction_full_test_v2.ipynb). + + Aligns the flat stream of confidence-scored raw words against the flat stream + of corrected words (which carry their corrected-line index), then groups the + resulting words back by corrected line. Robust when the corrected text has a + different line structure than the raw OCR (common here). + + Opcodes: equal -> original confidence; replace -> avg confidence of the + replaced originals + qwen_replaced; insert -> default confidence (grey, not + flagged, matching the v2 HTML); delete -> dropped. + Returns one list[Word] per corrected line. + """ + n_lines = len(corrected_lines) + grouped: list[list[Word]] = [[] for _ in range(n_lines)] + # flat corrected words tagged with their line index + corr_wl: list[tuple[int, str]] = [] + for li, line in enumerate(corrected_lines): + for w in line.split(): + corr_wl.append((li, w)) + if not corr_wl: + return grouped + + orig_keys = [_normalise(w) for w, _ in conf_words] + corr_keys = [_normalise(w) for _, w in corr_wl] + matcher = difflib.SequenceMatcher(None, orig_keys, corr_keys, autojunk=False) + + def emit(li: int, text: str, conf: float, replaced: bool) -> None: + grouped[li].append(Word(text=text, confidence=round(conf, 4), + needs_review=conf < low_conf_threshold, + qwen_replaced=replaced)) + + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if tag == "equal": + for i, j in zip(range(i1, i2), range(j1, j2)): + li, w = corr_wl[j] + emit(li, w, conf_words[i][1], False) + elif tag == "replace": + confs = [conf_words[i][1] for i in range(i1, i2)] + avg = sum(confs) / len(confs) if confs else DEFAULT_INSERT_CONF + for j in range(j1, j2): + li, w = corr_wl[j] + emit(li, w, avg, True) + elif tag == "insert": + for j in range(j1, j2): + li, w = corr_wl[j] + emit(li, w, DEFAULT_INSERT_CONF, False) + # delete: raw word dropped -> nothing + return grouped diff --git a/app/src/inkference/htr/correction.py b/app/src/inkference/htr/correction.py new file mode 100644 index 0000000000000000000000000000000000000000..28c5435e36fd3d20e1bdb05ce44616ddabf078f1 --- /dev/null +++ b/app/src/inkference/htr/correction.py @@ -0,0 +1,205 @@ +"""Qwen few-shot page-level post-correction. + +Runs after recognition: takes a page's raw TrOCR lines (low-confidence words wrapped +`<>`), primes Qwen with a few in-context (raw -> corrected) exemplars, and asks it +to fix OCR errors while preserving archaic spelling and the line count. + +Backends (config.CorrectionConfig.backend): + local Qwen3 via transformers (notebook `run_qwen` pattern: /no_think, strip ) + api hosted Qwen over an OpenAI-compatible endpoint (OpenRouter/DashScope/…) + +Heavy imports (torch/transformers/requests) are lazy. +""" +from __future__ import annotations + +import json +import re +import time +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from ..config import CorrectionConfig +from ..config import correction as default_correction + +if TYPE_CHECKING: # pragma: no cover + from ..schemas import Line + +_SYSTEM = """\ +You are an expert transcriber correcting OCR output of an 18th-century handwritten \ +manuscript journal (Johann Reinhold Forster's journal of Captain Cook's second voyage, \ +1772–1774: nautical, botanical and zoological terminology). + +Your job: fix obvious OCR recognition errors — misread letters, split or merged words, \ +stray characters and spacing. + +Each input line is prefixed with a number and a pipe, like "3| some text". + +Strict rules: +- PRESERVE the historical spelling, capitalisation, abbreviations and punctuation. Do NOT \ +modernise (keep forms like "ye", "Tuns", "&", "o'clock"). +- Do NOT add, remove, translate, summarise or reorder content. +- Words wrapped in << >> are low-confidence — focus your corrections there, but you may \ +also fix clear errors elsewhere. +- Return EVERY input line, each prefixed with its SAME number and a pipe: "3| corrected \ +text". Keep the same count and order. Remove the << >> markers. No commentary.\ +""" + + +def _number(lines: list[str]) -> str: + return "\n".join(f"{i + 1}| {ln}" for i, ln in enumerate(lines)) + + +_NUM_RE = re.compile(r"^\s*(\d+)\s*\|\s?(.*)$") + + +@dataclass +class CorrectionResult: + # per-line corrected text aligned 1:1 with the input lines, or None if the model + # returned a different number of lines (caller falls back to page_text) + lines: list[str] | None + page_text: str # full corrected block as returned + raw_prompt_lines: list[str] = field(default_factory=list) + + +def _marked_line(line: "Line") -> str: + """Rebuild a line's text with low-confidence words wrapped <>.""" + if line.words: + return " ".join( + f"<<{w.text}>>" if w.needs_review else w.text for w in line.words + ) + return line.text + + +def _strip_think(text: str) -> str: + return re.sub(r".*?", "", text, flags=re.DOTALL).strip() + + +class PostCorrector: + def __init__(self, cfg: CorrectionConfig = default_correction) -> None: + self.cfg = cfg + self._examples: list[dict] | None = None + self._tokenizer = None + self._model = None + + # -- few-shot exemplars ------------------------------------------------- # + def _load_examples(self) -> list[dict]: + if self._examples is None: + try: + data = json.loads(self.cfg.examples_path.read_text(encoding="utf-8")) + self._examples = data.get("examples", []) if isinstance(data, dict) else data + except Exception: + self._examples = [] + return self._examples[: max(0, self.cfg.num_shots)] + + def build_messages(self, marked_lines: list[str]) -> list[dict]: + """system + few-shot (raw->corrected) turns + the actual page. + + Exemplars and the target are numbered ("N| text") on the fly so the model + echoes line numbers — that makes the reply robust to line drift.""" + messages = [{"role": "system", "content": _SYSTEM}] + for ex in self._load_examples(): + messages.append({"role": "user", "content": _number(ex["raw"].split("\n"))}) + messages.append({"role": "assistant", "content": _number(ex["corrected"].split("\n"))}) + messages.append({"role": "user", "content": _number(marked_lines)}) + return messages + + # -- backends ----------------------------------------------------------- # + def _ensure_local(self): + if self._model is not None: + return + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + + device = self.cfg.device + if device == "auto": + device = "cuda" if torch.cuda.is_available() else "cpu" + self._tokenizer = AutoTokenizer.from_pretrained(self.cfg.model_id) + dtype = torch.float16 if device == "cuda" else torch.float32 + model = AutoModelForCausalLM.from_pretrained(self.cfg.model_id, dtype=dtype) + model.to(device) + model.eval() + self._model = model + self._device = device + self._torch = torch + + def _run_local(self, messages: list[dict]) -> str: + self._ensure_local() + torch, tok, model = self._torch, self._tokenizer, self._model + # Qwen3: disable thinking (append /no_think, like the notebook). + msgs = list(messages) + msgs[-1] = {**msgs[-1], "content": msgs[-1]["content"] + "\n/no_think"} + prompt = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True) + inputs = tok(prompt, return_tensors="pt").to(self._device) + gen_kwargs = dict(max_new_tokens=self.cfg.max_new_tokens, + pad_token_id=tok.eos_token_id) + if self.cfg.temperature > 0.05: + gen_kwargs.update(do_sample=True, temperature=self.cfg.temperature) + else: + gen_kwargs.update(do_sample=False) + with torch.no_grad(): + out = model.generate(**inputs, **gen_kwargs) + new_tokens = out[0][inputs["input_ids"].shape[1]:] + return _strip_think(tok.decode(new_tokens, skip_special_tokens=True)) + + def _run_api(self, messages: list[dict]) -> str: + import requests + + # No silent default: fail loudly so the corrector never posts to an + # unexpected host. Set CORRECTION_API_BASE (e.g. Groq's + # https://api.groq.com/openai/v1) and a key in the environment/.env. + base = (self.cfg.api_base or "").rstrip("/") + if not base: + raise ValueError( + "CORRECTION_BACKEND=api but CORRECTION_API_BASE is not set " + "(e.g. https://api.groq.com/openai/v1)." + ) + if not self.cfg.api_key: + raise ValueError( + "CORRECTION_BACKEND=api but no API key found " + "(set CORRECTION_API_KEY or GROQ_API_KEY)." + ) + # Disable Qwen3 "thinking" (soft switch) so reasoning tokens don't eat the + # completion budget and truncate long pages. Also cap output generously. + msgs = list(messages) + msgs[-1] = {**msgs[-1], "content": msgs[-1]["content"] + "\n/no_think"} + payload = {"model": self.cfg.api_model, "messages": msgs, + "temperature": self.cfg.temperature, + "max_tokens": self.cfg.max_new_tokens} + headers = {"Authorization": f"Bearer {self.cfg.api_key}"} + # Retry on 429 (free-tier rate limits) honouring Retry-After, with backoff. + for attempt in range(4): + r = requests.post(f"{base}/chat/completions", headers=headers, + json=payload, timeout=120) + if r.status_code == 429 and attempt < 3: + wait = float(r.headers.get("retry-after", 2 ** attempt * 2)) + time.sleep(min(wait, 30)) + continue + r.raise_for_status() + return _strip_think(r.json()["choices"][0]["message"]["content"].strip()) + r.raise_for_status() # exhausted retries + return "" + + # -- public API --------------------------------------------------------- # + def correct_page(self, lines: list["Line"]) -> CorrectionResult: + marked = [_marked_line(ln) for ln in lines] + if not marked: + return CorrectionResult(lines=[], page_text="", raw_prompt_lines=[]) + messages = self.build_messages(marked) + reply = self._run_api(messages) if self.cfg.backend == "api" else self._run_local(messages) + + # Parse "N| text" replies -> {line_number: corrected_text}. Robust to the + # model dropping/merging lines: we map by number and fill any gaps from raw. + parsed: dict[int, str] = {} + for row in reply.split("\n"): + m = _NUM_RE.match(row) + if m: + parsed[int(m.group(1))] = m.group(2).strip() + + n = len(lines) + hits = sum(1 for i in range(1, n + 1) if i in parsed) + if hits < max(1, n // 2): # too few numbered lines came back — treat as failure + return CorrectionResult(lines=None, page_text=reply.strip(), raw_prompt_lines=marked) + + # Fill gaps with the original line text (markers stripped) so output is 1:1. + aligned = [parsed.get(i + 1, lines[i].text) for i in range(n)] + return CorrectionResult(lines=aligned, page_text="\n".join(aligned), raw_prompt_lines=marked) diff --git a/app/src/inkference/htr/few_shot_examples.json b/app/src/inkference/htr/few_shot_examples.json new file mode 100644 index 0000000000000000000000000000000000000000..8c8fbf6fc180217c800c9da8ea7f7f3b69c72fe6 --- /dev/null +++ b/app/src/inkference/htr/few_shot_examples.json @@ -0,0 +1,17 @@ +{ + "_note": "Few-shot exemplars for Qwen page-level OCR post-correction. 'raw' = real fine-tuned-TrOCR output (low-confidence words wrapped <<...>>), 'corrected' = verified ground-truth for the same lines. Bootstrapped from transcriptions/ line crops. Same #lines in raw/corrected.", + "examples": [ + { + "raw": "Journal of the\nNov. ye 23� <<.>> After having eaten our dinner\nat Mr Brand's, we took leave of his Family &\nwent on board the Resolution; We immediately", + "corrected": "Journal of the\nNov. ye 23⁹ 1772 . After having eaten our dinner\nat Mr Brand's, we took leave of his Family &\nwent on board the Resolution; We immediately" + }, + { + "raw": "8)From the Cape towards the South. Nov.-y� 30\nbut the cold air in the morning together with\nthe dampness of my bed & of the whole body\ncaused some stifness in my limbs; which", + "corrected": "8)From the Cape towards the South. Nov.-y⁰ 30\nbut the cold air in the morning together with\nthe dampness of my bed & of the whole body\ncaused some stifness in my limbs; which" + }, + { + "raw": "I had been obliged to climb up the freepest\nprecipices, often up to the Ancles in\nmud, &-through briars & thorns. My little\nbottle was broke from two hard falls I", + "corrected": "I had been obliged to climb up the freepest\nprecipices, often up to the Ancles in\nmud, &-through briars & thorns. My little\nbottle was broke from two hard falls I" + } + ] +} \ No newline at end of file diff --git a/app/src/inkference/htr/pipeline.py b/app/src/inkference/htr/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..c136dd0b7ee78f571bc6b2ef449690ed14b7a23c --- /dev/null +++ b/app/src/inkference/htr/pipeline.py @@ -0,0 +1,133 @@ +"""HTR pipeline orchestrator: page image -> segment -> recognize -> confidence. + +Produces a PageResult and emits stage/progress events so the API can drive the +design's three-step stepper (Segmentation -> Recognition -> Confidence) and the +per-page progress bar (e.g. "Recognizing 64%"). +""" +from __future__ import annotations + +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Callable + +logger = logging.getLogger("inkference.correction") + +from ..config import CorrectionConfig, HTRConfig +from ..config import correction as default_correction +from ..config import htr as default_htr +from .align import align_line +from ..schemas import Line, PageResult, Stage +from .correction import PostCorrector +from .recognition import TrOCRRecognizer +from .segmentation import LineCrop, Segmenter, downscale, get_segmenter + +if TYPE_CHECKING: # pragma: no cover + from PIL import Image + +# progress(stage, fraction_0_to_1, message) +ProgressCB = Callable[[Stage, float, str], None] + + +def _noop(stage: Stage, frac: float, msg: str) -> None: # pragma: no cover + pass + + +class HTRPipeline: + def __init__( + self, + cfg: HTRConfig = default_htr, + segmenter: Segmenter | None = None, + recognizer: TrOCRRecognizer | None = None, + corrector: PostCorrector | None = None, + correction_cfg: CorrectionConfig = default_correction, + ) -> None: + self.cfg = cfg + self.correction_cfg = correction_cfg + self.segmenter = segmenter or get_segmenter("auto", cfg) + self.recognizer = recognizer or TrOCRRecognizer(cfg) + # Lazily created on first use unless injected; skipped when disabled. + self._corrector = corrector + + def process_image( + self, + page: "Image.Image", + page_number: int = 1, + progress: ProgressCB = _noop, + ) -> PageResult: + # --- Stage 1: segmentation ---------------------------------------- # + progress(Stage.SEGMENTATION, 0.0, "Segmenting…") + scaled, scale = downscale(page, self.cfg.max_page_long_edge) + crops: list[LineCrop] = self.segmenter.segment(scaled) + progress(Stage.SEGMENTATION, 1.0, f"{len(crops)} lines detected") + + w, h = scaled.size + if not crops: + return PageResult(page_number, w, h, lines=[], scale=scale) + + # --- Stage 2: recognition ----------------------------------------- # + progress(Stage.RECOGNITION, 0.0, "Recognizing…") + recognitions = self.recognizer.recognize([c.image for c in crops]) + progress(Stage.RECOGNITION, 1.0, "Recognized") + + # --- Stage 3: confidence flagging --------------------------------- # + progress(Stage.CONFIDENCE, 0.0, "Scoring confidence…") + thr = self.cfg.low_confidence_threshold + lines: list[Line] = [] + for crop, rec in zip(crops, recognitions): + for word in rec.words: + word.needs_review = word.confidence < thr + line_needs_review = any(wd.needs_review for wd in rec.words) + lines.append( + Line( + index=crop.index, + bbox=crop.bbox, + text=rec.text, + confidence=rec.confidence, + words=rec.words, + needs_review=line_needs_review, + ) + ) + progress(Stage.CONFIDENCE, 1.0, "Complete") + + # --- Stage 4: Qwen post-correction (optional) --------------------- # + if self.correction_cfg.enabled: + self._correct(lines, progress) + + return PageResult(page_number, w, h, lines=lines, scale=scale) + + def _corrector_instance(self) -> PostCorrector: + if self._corrector is None: + self._corrector = PostCorrector(self.correction_cfg) + return self._corrector + + def _correct(self, lines: list[Line], progress: ProgressCB) -> None: + """Run the corrector over the page and align results back onto each Line.""" + progress(Stage.CORRECTION, 0.0, "Proofreading…") + try: + result = self._corrector_instance().correct_page(lines) + except Exception as exc: # never fail ingest because of correction + # Surface it: log to the server console AND put it in the stage message + # so a failed correction is diagnosable instead of silently absent. + logger.warning("post-correction failed, skipping page: %s", exc) + progress(Stage.CORRECTION, 1.0, f"Correction skipped ({exc})") + return + if result.lines is None: + logger.warning( + "post-correction returned %d lines for a %d-line page — keeping raw", + len(result.page_text.splitlines()), len(lines), + ) + progress(Stage.CORRECTION, 1.0, "Complete (unaligned)") + return + thr = self.cfg.low_confidence_threshold + for line, corrected in zip(lines, result.lines): + line.corrected_text = corrected + line.corrected_words = align_line(line.words, corrected, thr) + progress(Stage.CORRECTION, 1.0, "Complete") + + def process_path( + self, image_path: str | Path, page_number: int = 1, progress: ProgressCB = _noop + ) -> PageResult: + from PIL import Image + + with Image.open(image_path) as im: + return self.process_image(im.copy(), page_number, progress) diff --git a/app/src/inkference/htr/recognition.py b/app/src/inkference/htr/recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..50377b33f4f59c6bdb3abe94ffcdf625ab222820 --- /dev/null +++ b/app/src/inkference/htr/recognition.py @@ -0,0 +1,145 @@ +"""TrOCR recognition with per-word and per-line confidence. + +Confidence comes from the decoder's own token probabilities: we generate with +`output_scores=True` and use `compute_transition_scores(..., normalize_logits=True)` +to get a calibrated log-prob per generated token, exp() it, group tokens into words, +and average. This drives the design's word-tinting, avg-confidence %, and the +"needs review" / "N words below 60%" flags. + +Heavy imports (torch/transformers) are lazy so the module imports with nothing installed. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from ..config import HTRConfig +from ..config import htr as default_htr +from ..schemas import Word + +if TYPE_CHECKING: # pragma: no cover + from PIL import Image + +# BPE/sentencepiece markers that indicate the start of a new word. +_WORD_START_MARKERS = ("Ġ", "▁") # 'Ġ' (RoBERTa/GPT2), '▁' (sentencepiece) + + +@dataclass +class LineRecognition: + text: str + confidence: float + words: list[Word] + + +def _resolve_device(device: str) -> str: + if device != "auto": + return device + try: + import torch + + return "cuda" if torch.cuda.is_available() else "cpu" + except Exception: # pragma: no cover + return "cpu" + + +class TrOCRRecognizer: + def __init__(self, cfg: HTRConfig = default_htr) -> None: + self.cfg = cfg + self.device = _resolve_device(cfg.device) + self._processor = None + self._model = None + + # -- lazy model load ---------------------------------------------------- # + def _ensure_loaded(self): + if self._model is not None: + return + import torch + from transformers import TrOCRProcessor, VisionEncoderDecoderModel + + self._processor = TrOCRProcessor.from_pretrained(self.cfg.trocr_model_id) + model = VisionEncoderDecoderModel.from_pretrained(self.cfg.trocr_model_id) + model.to(self.device) + model.eval() + self._model = model + self._torch = torch + + # -- public API --------------------------------------------------------- # + def recognize(self, images: list["Image.Image"]) -> list[LineRecognition]: + """Recognize a list of line crops, batching per recognition_batch_size.""" + if not images: + return [] + self._ensure_loaded() + out: list[LineRecognition] = [] + bs = max(1, self.cfg.recognition_batch_size) + for i in range(0, len(images), bs): + out.extend(self._recognize_batch(images[i : i + bs])) + return out + + def _recognize_batch(self, images: list["Image.Image"]) -> list[LineRecognition]: + torch = self._torch + proc = self._processor + model = self._model + + pixel_values = proc( + images=[im.convert("RGB") for im in images], return_tensors="pt" + ).pixel_values.to(self.device) + + with torch.no_grad(): + gen = model.generate( + pixel_values, + num_beams=self.cfg.num_beams, + max_new_tokens=self.cfg.max_target_length, + output_scores=True, + return_dict_in_generate=True, + ) + + beam_indices = getattr(gen, "beam_indices", None) + transition = model.compute_transition_scores( + gen.sequences, gen.scores, beam_indices=beam_indices, normalize_logits=True + ) + probs = transition.exp().cpu() # [batch, gen_len-1] + sequences = gen.sequences.cpu() # [batch, gen_len] + + tokenizer = proc.tokenizer + special_ids = set(tokenizer.all_special_ids) + + results: list[LineRecognition] = [] + for b in range(sequences.size(0)): + # sequences[b,0] is the decoder start token; transition aligns to [1:]. + tok_ids = sequences[b, 1:].tolist() + tok_probs = probs[b].tolist() + results.append(self._aggregate(tok_ids, tok_probs, tokenizer, special_ids)) + return results + + def _aggregate(self, tok_ids, tok_probs, tokenizer, special_ids) -> LineRecognition: + """Group sub-word tokens into words with averaged confidence.""" + words: list[Word] = [] + cur_tokens: list[str] = [] + cur_probs: list[float] = [] + + def flush(): + if not cur_tokens: + return + text = tokenizer.convert_tokens_to_string(cur_tokens).strip() + if text: + conf = sum(cur_probs) / len(cur_probs) + words.append(Word(text=text, confidence=round(conf, 4))) + cur_tokens.clear() + cur_probs.clear() + + for tid, prob in zip(tok_ids, tok_probs): + if tid in special_ids: + continue + token = tokenizer.convert_ids_to_tokens(tid) + starts_word = token.startswith(_WORD_START_MARKERS) + if starts_word and cur_tokens: + flush() + cur_tokens.append(token) + cur_probs.append(float(prob)) + flush() + + text = " ".join(w.text for w in words) + line_conf = ( + sum(w.confidence for w in words) / len(words) if words else 0.0 + ) + return LineRecognition(text=text, confidence=round(line_conf, 4), words=words) diff --git a/app/src/inkference/htr/segmentation.py b/app/src/inkference/htr/segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..3b475b4da548dbb9234e8122a4013832ff48b4f8 --- /dev/null +++ b/app/src/inkference/htr/segmentation.py @@ -0,0 +1,241 @@ +"""Line segmentation — Kraken baseline segmentation (`blla`). + +Faithful port of notebooks/segmentManualTranscriptionsIntoLinesWithKraken*.ipynb: +Kraken's bundled baseline model predicts, per line, a **baseline + boundary +polygon**; we take the polygon, derive a bbox, drop noise boxes (MIN_W/MIN_H), +sort top-to-bottom by vertical centre, and crop each line **masked to its polygon** +(dilated by POLY_PAD) so neighbouring ascenders/descenders don't bleed in. + +The old dependency-light ProjectionSegmenter is disabled — see the commented block +at the bottom. Kraken is now the only backend. + +Heavy imports (kraken/torch/PIL) are lazy so this module imports with nothing installed. +""" +from __future__ import annotations + +import importlib.util +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Protocol + +from ..config import HTRConfig +from ..config import htr as default_htr +from ..schemas import BBox + +if TYPE_CHECKING: # pragma: no cover + from PIL import Image + + +@dataclass +class LineCrop: + """One detected line: its polygon-masked crop plus where it sits on the page.""" + + index: int + bbox: BBox + image: "Image.Image" + polygon: list[tuple[float, float]] = field(default_factory=list) + + +class Segmenter(Protocol): + def segment(self, page: "Image.Image") -> list[LineCrop]: ... + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # +def downscale(page: "Image.Image", max_long_edge: int) -> tuple["Image.Image", float]: + """Cap the long edge to keep free-CPU work bounded. Returns (image, scale).""" + w, h = page.size + long_edge = max(w, h) + if long_edge <= max_long_edge: + return page, 1.0 + scale = max_long_edge / long_edge + return page.resize((round(w * scale), round(h * scale))), scale + + +def _resolve_device(device: str) -> str: + if device != "auto": + return device + try: + import torch + + return "cuda" if torch.cuda.is_available() else "cpu" + except Exception: # pragma: no cover + return "cpu" + + +def _boundary_of(line): + """A line's boundary polygon as a list of points, across kraken API versions.""" + b = getattr(line, "boundary", None) + if b is None and isinstance(line, dict): + b = line.get("boundary") + return b + + +# --------------------------------------------------------------------------- # +# Kraken backend +# --------------------------------------------------------------------------- # +class KrakenSegmenter: + """Kraken baseline segmentation with polygon-masked line crops.""" + + def __init__(self, cfg: HTRConfig = default_htr) -> None: + self.cfg = cfg + self.device = _resolve_device(cfg.device) + self._seg_model = None + self._model_loaded = False + + def _ensure_model(self): + """Load Kraken's bundled default baseline model (blla.mlmodel) once. + + Falls back to None, in which case blla.segment uses its own default.""" + if self._model_loaded: + return self._seg_model + import os + + import kraken + from kraken.lib import vgsl + + cand = os.path.join(os.path.dirname(kraken.__file__), "blla.mlmodel") + if os.path.exists(cand): + self._seg_model = vgsl.TorchVGSLModel.load_model(cand) + else: + self._seg_model = None + self._model_loaded = True + return self._seg_model + + def _run_blla(self, im): + from kraken import blla + + model = self._ensure_model() + # the `device` kwarg exists on newer kraken; fall back gracefully. + try: + return blla.segment(im, model=model, device=self.device) + except TypeError: + return blla.segment(im, model=model) if model is not None else blla.segment(im) + + def segment(self, page: "Image.Image") -> list[LineCrop]: + from PIL import Image, ImageDraw, ImageFilter + + rgb = page.convert("RGB") + seg = self._run_blla(rgb) + lines = getattr(seg, "lines", None) + if lines is None and isinstance(seg, dict): + lines = seg.get("lines", []) + lines = lines or [] + + W, H = rgb.size + cfg = self.cfg + + # 1. polygon -> bbox record, dropping noise boxes (MIN_W / MIN_H). + records: list[tuple[BBox, list[tuple[float, float]]]] = [] + for ln in lines: + poly = _boundary_of(ln) + if not poly: + continue + pts = [(float(p[0]), float(p[1])) for p in poly] + xs = [p[0] for p in pts] + ys = [p[1] for p in pts] + bbox = (min(xs), min(ys), max(xs), max(ys)) + if (bbox[2] - bbox[0]) < cfg.min_w or (bbox[3] - bbox[1]) < cfg.min_h: + continue + records.append((bbox, pts)) + + # 2. reading order: top-to-bottom by vertical centre. + records.sort(key=lambda r: (r[0][1] + r[0][3]) / 2) + + # 3. polygon-masked crop per line (PAD_X/PAD_Y + MaxFilter(POLY_PAD) dilation). + crops: list[LineCrop] = [] + for idx, (bbox, pts) in enumerate(records): + x0, y0, x1, y1 = bbox + cx0 = max(0, int(x0) - cfg.pad_x) + cy0 = max(0, int(y0) - cfg.pad_y) + cx1 = min(W, int(x1) + cfg.pad_x) + cy1 = min(H, int(y1) + cfg.pad_y) + crop = rgb.crop((cx0, cy0, cx1, cy1)) + + if cfg.mask_to_polygon: + mask = Image.new("L", crop.size, 0) + ImageDraw.Draw(mask).polygon( + [(px - cx0, py - cy0) for px, py in pts], fill=255 + ) + if cfg.poly_pad > 0: # dilate so the line's own strokes aren't clipped + mask = mask.filter(ImageFilter.MaxFilter(cfg.poly_pad * 2 + 1)) + white = Image.new("RGB", crop.size, (255, 255, 255)) + crop = Image.composite(crop, white, mask) + + crops.append( + LineCrop(index=idx, bbox=(cx0, cy0, cx1, cy1), image=crop, polygon=pts) + ) + return crops + + +def kraken_available() -> bool: + return importlib.util.find_spec("kraken") is not None + + +def get_segmenter(name: str = "kraken", cfg: HTRConfig = default_htr) -> Segmenter: + """Return the Kraken segmenter. The projection fallback is disabled.""" + if name in ("kraken", "auto"): + if not kraken_available(): + raise ImportError( + "Kraken is not installed. Install it (see the note in requirements.txt — " + "Kraken pins its own torch, so a dedicated venv is recommended)." + ) + return KrakenSegmenter(cfg) + if name == "projection": + raise ValueError("ProjectionSegmenter is disabled; use the Kraken segmenter.") + raise ValueError(f"unknown segmenter: {name!r}") + + +# --------------------------------------------------------------------------- # +# DISABLED — dependency-light projection-profile fallback (kept for reference). +# Re-enable in get_segmenter() only if you deliberately want a Kraken-free path. +# --------------------------------------------------------------------------- # +# class ProjectionSegmenter: +# """Find line bands from the horizontal ink-projection profile. Weak on dense +# or multi-column layouts — replaced by Kraken baseline segmentation.""" +# +# def __init__(self, cfg: HTRConfig = default_htr) -> None: +# self.cfg = cfg +# +# def segment(self, page: "Image.Image") -> list[LineCrop]: +# import numpy as np +# from PIL import ImageOps +# +# gray = ImageOps.grayscale(page) +# arr = np.asarray(gray, dtype=np.float32) +# h, w = arr.shape +# thresh = float(arr.mean()) - 0.4 * float(arr.std()) +# ink = (arr < thresh).astype(np.float32) +# row_ink = ink.sum(axis=1) +# if row_ink.max() <= 0: +# return [] +# active = row_ink > (0.04 * row_ink.max()) +# bands: list[tuple[int, int]] = [] +# start = None +# for y in range(h): +# if active[y] and start is None: +# start = y +# elif not active[y] and start is not None: +# bands.append((start, y)) +# start = None +# if start is not None: +# bands.append((start, h)) +# rgb = page.convert("RGB") +# crops: list[LineCrop] = [] +# idx = 0 +# for (y0, y1) in bands: +# if (y1 - y0) < self.cfg.min_h: +# continue +# band_ink = ink[y0:y1].sum(axis=0) +# cols = np.where(band_ink > 0)[0] +# if cols.size == 0: +# continue +# x0, x1 = int(cols[0]), int(cols[-1]) + 1 +# if (x1 - x0) < self.cfg.min_w: +# continue +# x0p = max(0, x0 - self.cfg.pad_x); y0p = max(0, y0 - self.cfg.pad_y) +# x1p = min(w, x1 + self.cfg.pad_x); y1p = min(h, y1 + self.cfg.pad_y) +# crops.append(LineCrop(index=idx, bbox=(x0p, y0p, x1p, y1p), +# image=rgb.crop((x0p, y0p, x1p, y1p)))) +# idx += 1 +# return crops diff --git a/app/src/inkference/rag/__init__.py b/app/src/inkference/rag/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..605731615fa40439d710936225ece4956e8c499d --- /dev/null +++ b/app/src/inkference/rag/__init__.py @@ -0,0 +1,5 @@ +"""RAG: chunk + embed + FAISS retrieval + grounded answer generation.""" +from .index import RagIndex, Chunk +from .answer import answer_question, Answer + +__all__ = ["RagIndex", "Chunk", "answer_question", "Answer"] diff --git a/app/src/inkference/rag/__pycache__/__init__.cpython-311.pyc b/app/src/inkference/rag/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0cdad2f159a860d1e23c031d6a9821201c42243e Binary files /dev/null and b/app/src/inkference/rag/__pycache__/__init__.cpython-311.pyc differ diff --git a/app/src/inkference/rag/__pycache__/answer.cpython-311.pyc b/app/src/inkference/rag/__pycache__/answer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf695b91b693d7032dd10b2209fa94e253e35cae Binary files /dev/null and b/app/src/inkference/rag/__pycache__/answer.cpython-311.pyc differ diff --git a/app/src/inkference/rag/__pycache__/index.cpython-311.pyc b/app/src/inkference/rag/__pycache__/index.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bbd95797d822aa9b8a3b68f7bc7ad9177697dae7 Binary files /dev/null and b/app/src/inkference/rag/__pycache__/index.cpython-311.pyc differ diff --git a/app/src/inkference/rag/__pycache__/llm.cpython-311.pyc b/app/src/inkference/rag/__pycache__/llm.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5ccbcca6001bb41618636c8ff2a67692f9e236a8 Binary files /dev/null and b/app/src/inkference/rag/__pycache__/llm.cpython-311.pyc differ diff --git a/app/src/inkference/rag/answer.py b/app/src/inkference/rag/answer.py new file mode 100644 index 0000000000000000000000000000000000000000..b06df54097a792a5a24834dc920387a977411ad9 --- /dev/null +++ b/app/src/inkference/rag/answer.py @@ -0,0 +1,57 @@ +"""Orchestrate one RAG turn: retrieve -> generate -> answer + source pages.""" +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..config import RAGConfig +from ..config import rag as default_rag +from .index import RagIndex +from .llm import generate_answer + + +@dataclass +class Answer: + question: str + answer: str + source_pages: list[int] = field(default_factory=list) + # retrieved evidence, for transparency / debugging + contexts: list[dict] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "question": self.question, + "answer": self.answer, + "source_pages": self.source_pages, + "contexts": self.contexts, + } + + +def answer_question( + doc_id: int, + question: str, + index: RagIndex, + cfg: RAGConfig = default_rag, + top_k: int | None = None, +) -> Answer: + retrieved = index.query(doc_id, question, top_k=top_k) + if not retrieved: + return Answer(question, "No transcribed text is available for this document yet.") + + contexts = [(r.page_number, r.text) for r in retrieved] + text = generate_answer(question, contexts, cfg) + + # Distinct source pages in retrieval order -> the design's "Sources" chips. + seen: set[int] = set() + source_pages: list[int] = [] + for r in retrieved: + if r.page_number not in seen: + seen.add(r.page_number) + source_pages.append(r.page_number) + + return Answer( + question=question, + answer=text, + source_pages=source_pages, + contexts=[{"page_number": r.page_number, "score": round(r.score, 4), + "text": r.text} for r in retrieved], + ) diff --git a/app/src/inkference/rag/index.py b/app/src/inkference/rag/index.py new file mode 100644 index 0000000000000000000000000000000000000000..ac064696f59bfbf8f1cb15136783e47242419620 --- /dev/null +++ b/app/src/inkference/rag/index.py @@ -0,0 +1,162 @@ +"""Per-document retrieval index: chunk page text, embed, store in FAISS. + +Chunks carry their source `page_number` so answers can cite pages (the design's +"Sources: Page 24" chips). Index + chunk metadata persist under +StoreConfig.index_dir/doc_.{faiss,json} so a free Space can rebuild lazily. + +sentence-transformers / faiss imports are lazy. +""" +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Iterable + +from ..config import RAGConfig, StoreConfig +from ..config import rag as default_rag +from ..config import store as default_store + +if TYPE_CHECKING: # pragma: no cover + from ..store import DocumentStore + +# Chunk target size in characters. Diary pages are short; one chunk per page is +# common, but long pages split on line boundaries so retrieval stays focused. +_CHUNK_CHARS = 600 +_CHUNK_OVERLAP_LINES = 1 + + +@dataclass +class Chunk: + page_number: int + text: str + + +@dataclass +class Retrieved: + page_number: int + text: str + score: float + + +def chunk_page(page_number: int, text: str) -> list[Chunk]: + """Split a page into <=_CHUNK_CHARS chunks on line boundaries.""" + lines = [ln for ln in text.splitlines() if ln.strip()] + if not lines: + return [] + chunks: list[Chunk] = [] + buf: list[str] = [] + size = 0 + for ln in lines: + if size + len(ln) > _CHUNK_CHARS and buf: + chunks.append(Chunk(page_number, "\n".join(buf))) + buf = buf[-_CHUNK_OVERLAP_LINES:] if _CHUNK_OVERLAP_LINES else [] + size = sum(len(x) for x in buf) + buf.append(ln) + size += len(ln) + if buf: + chunks.append(Chunk(page_number, "\n".join(buf))) + return chunks + + +class RagIndex: + def __init__( + self, + cfg: RAGConfig = default_rag, + store_cfg: StoreConfig = default_store, + ) -> None: + self.cfg = cfg + self.store_cfg = store_cfg + self._embedder = None + # in-memory state for the currently loaded document + self._doc_id: int | None = None + self._index = None + self._chunks: list[Chunk] = [] + + # -- embedder ----------------------------------------------------------- # + def _ensure_embedder(self): + if self._embedder is None: + from sentence_transformers import SentenceTransformer + + self._embedder = SentenceTransformer(self.cfg.embed_model_id) + return self._embedder + + def _embed(self, texts: list[str]): + import numpy as np + + emb = self._ensure_embedder().encode( + texts, normalize_embeddings=True, convert_to_numpy=True + ) + return emb.astype(np.float32) + + # -- paths -------------------------------------------------------------- # + def _paths(self, doc_id: int) -> tuple[Path, Path]: + base = self.store_cfg.index_dir + return base / f"doc_{doc_id}.faiss", base / f"doc_{doc_id}.json" + + # -- build / persist ---------------------------------------------------- # + def build(self, doc_id: int, pages: Iterable[tuple[int, str]]) -> int: + """Build (or rebuild) the index for a document from (page_no, text) pairs.""" + import faiss + + chunks: list[Chunk] = [] + for page_number, text in pages: + chunks.extend(chunk_page(page_number, text)) + if not chunks: + self._doc_id, self._index, self._chunks = doc_id, None, [] + return 0 + + vectors = self._embed([c.text for c in chunks]) + index = faiss.IndexFlatIP(vectors.shape[1]) # cosine via normalized vectors + index.add(vectors) + + self._doc_id, self._index, self._chunks = doc_id, index, chunks + self._persist(doc_id) + return len(chunks) + + def build_from_store(self, doc_id: int, store: "DocumentStore") -> int: + return self.build(doc_id, store.iter_pages_text(doc_id)) + + def _persist(self, doc_id: int) -> None: + import faiss + + idx_path, meta_path = self._paths(doc_id) + idx_path.parent.mkdir(parents=True, exist_ok=True) + if self._index is not None: + faiss.write_index(self._index, str(idx_path)) + meta_path.write_text( + json.dumps([asdict(c) for c in self._chunks]), encoding="utf-8" + ) + + # -- load / query ------------------------------------------------------- # + def _ensure_loaded(self, doc_id: int) -> bool: + if self._doc_id == doc_id and self._index is not None: + return True + import faiss + + idx_path, meta_path = self._paths(doc_id) + if not idx_path.exists() or not meta_path.exists(): + return False + self._index = faiss.read_index(str(idx_path)) + self._chunks = [Chunk(**c) for c in json.loads(meta_path.read_text("utf-8"))] + self._doc_id = doc_id + return True + + def query(self, doc_id: int, question: str, top_k: int | None = None) -> list[Retrieved]: + k = top_k or self.cfg.top_k + if not self._ensure_loaded(doc_id) or self._index is None: + return [] + qv = self._embed([question]) + k = min(k, len(self._chunks)) + scores, idxs = self._index.search(qv, k) + out: list[Retrieved] = [] + for score, i in zip(scores[0], idxs[0]): + if i < 0: + continue + c = self._chunks[int(i)] + out.append(Retrieved(page_number=c.page_number, text=c.text, score=float(score))) + return out + + def exists(self, doc_id: int) -> bool: + idx_path, meta_path = self._paths(doc_id) + return idx_path.exists() and meta_path.exists() diff --git a/app/src/inkference/rag/llm.py b/app/src/inkference/rag/llm.py new file mode 100644 index 0000000000000000000000000000000000000000..5e9d161af43120793f6d9400b33683dacb9326d4 --- /dev/null +++ b/app/src/inkference/rag/llm.py @@ -0,0 +1,142 @@ +"""LLM answer generation behind a provider-agnostic REST call. + +Providers are called directly over HTTP with `requests` (already a dependency) so +no provider SDK needs installing. Supported: gemini, groq, openai, claude. With no +API key configured, falls back to an extractive answer (stitched top snippets) so +the app still works at $0 and offline. +""" +from __future__ import annotations + +import time + +from ..config import RAGConfig +from ..config import rag as default_rag + + +def _post_retry(url: str, retries: int = 3, **kwargs): + """POST that retries transient 429/503 (rate limit / overload) with backoff.""" + import requests + + last = None + for attempt in range(retries): + last = requests.post(url, **kwargs) + if last.status_code in (429, 503) and attempt < retries - 1: + wait = float(last.headers.get("retry-after", 2 ** attempt)) + time.sleep(min(wait, 20)) + continue + return last + return last + +_DEFAULT_MODELS = { + "gemini": "gemini-2.5-flash", + "groq": "llama-3.3-70b-versatile", + "openai": "gpt-4o-mini", + "claude": "claude-haiku-4-5-20251001", +} + +_SYSTEM = ( + "You are Inkference, an assistant answering questions about a historical " + "handwritten document using ONLY the provided transcribed page excerpts. " + "Ground every claim in the excerpts. If the answer is not present, say so. " + "Do not invent facts. Write 2–4 sentences in a clear, scholarly tone." +) + + +def _build_prompt(question: str, contexts: list[tuple[int, str]]) -> str: + blocks = "\n\n".join(f"[Page {pn}]\n{txt}" for pn, txt in contexts) + return ( + f"Transcribed excerpts:\n\n{blocks}\n\n" + f"Question: {question}\n\n" + "Answer using only the excerpts above." + ) + + +def _extractive_fallback(question: str, contexts: list[tuple[int, str]]) -> str: + if not contexts: + return "No transcribed text is available to answer this question yet." + top = contexts[0][1].replace("\n", " ").strip() + pages = ", ".join(str(pn) for pn, _ in contexts) + return ( + f"(No language model configured — showing the most relevant transcribed " + f"passage from page{'s' if ',' in pages else ''} {pages}.)\n\n“{top}”" + ) + + +def generate_answer( + question: str, contexts: list[tuple[int, str]], cfg: RAGConfig = default_rag +) -> str: + """contexts = [(page_number, text), ...] in relevance order.""" + provider = (cfg.llm_provider or "").lower() + if not cfg.llm_api_key or provider not in _DEFAULT_MODELS: + return _extractive_fallback(question, contexts) + + model = cfg.llm_model or _DEFAULT_MODELS[provider] + prompt = _build_prompt(question, contexts) + try: + if provider == "gemini": + return _call_gemini(model, prompt, cfg.llm_api_key) + if provider in ("groq", "openai"): + return _call_openai_compatible(provider, model, prompt, cfg.llm_api_key) + if provider == "claude": + return _call_claude(model, prompt, cfg.llm_api_key) + except Exception as exc: # network/quota/etc — degrade gracefully + return _extractive_fallback(question, contexts) + f"\n\n[generation error: {exc}]" + return _extractive_fallback(question, contexts) + + +# --------------------------------------------------------------------------- # +# provider calls +# --------------------------------------------------------------------------- # +def _call_gemini(model: str, prompt: str, api_key: str) -> str: + url = ( + f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent" + f"?key={api_key}" + ) + body = { + "system_instruction": {"parts": [{"text": _SYSTEM}]}, + "contents": [{"role": "user", "parts": [{"text": prompt}]}], + } + r = _post_retry(url, json=body, timeout=60) + r.raise_for_status() + return r.json()["candidates"][0]["content"]["parts"][0]["text"].strip() + + +def _call_openai_compatible(provider: str, model: str, prompt: str, api_key: str) -> str: + base = "https://api.groq.com/openai/v1" if provider == "groq" else "https://api.openai.com/v1" + r = _post_retry( + f"{base}/chat/completions", + headers={"Authorization": f"Bearer {api_key}"}, + json={ + "model": model, + "messages": [ + {"role": "system", "content": _SYSTEM}, + {"role": "user", "content": prompt}, + ], + "temperature": 0.2, + }, + timeout=60, + ) + r.raise_for_status() + return r.json()["choices"][0]["message"]["content"].strip() + + +def _call_claude(model: str, prompt: str, api_key: str) -> str: + import requests + + r = requests.post( + "https://api.anthropic.com/v1/messages", + headers={ + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + json={ + "model": model, + "max_tokens": 600, + "system": _SYSTEM, + "messages": [{"role": "user", "content": prompt}], + }, + timeout=60, + ) + r.raise_for_status() + return r.json()["content"][0]["text"].strip() diff --git a/app/src/inkference/schemas.py b/app/src/inkference/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..e24876ae77b03331c2504daf577647c013d2ff70 --- /dev/null +++ b/app/src/inkference/schemas.py @@ -0,0 +1,100 @@ +"""Dependency-free domain types shared across htr / store / rag. + +Plain dataclasses (no pydantic) so the HTR core imports without the API stack. +The API layer mirrors these as pydantic response models.""" +from __future__ import annotations + +from dataclasses import dataclass, field, asdict +from enum import Enum +from typing import Any + + +# Bounding box in pixel coordinates on the (possibly downscaled) page image. +BBox = tuple[int, int, int, int] # (x0, y0, x1, y1) + + +class Stage(str, Enum): + """The pipeline stages shown in the design's stepper.""" + + SEGMENTATION = "segmentation" + RECOGNITION = "recognition" + CONFIDENCE = "confidence" + CORRECTION = "correction" # optional Qwen post-correction stage + + +class JobStatus(str, Enum): + QUEUED = "queued" + SEGMENTING = "segmenting" + RECOGNIZING = "recognizing" + SCORING = "scoring" + CORRECTING = "correcting" + COMPLETE = "complete" + FAILED = "failed" + + +@dataclass +class Word: + text: str + confidence: float # 0..1 mean token probability for this word + needs_review: bool = False # set by the pipeline using the low-conf threshold + qwen_replaced: bool = False # set on corrected words the LLM changed + + +@dataclass +class Line: + index: int # 0-based, top-to-bottom reading order + bbox: BBox + text: str + confidence: float # mean token probability across the line + words: list[Word] = field(default_factory=list) + needs_review: bool = False + # Populated by the post-correction stage (None/empty when correction is off). + corrected_text: str | None = None + corrected_words: list[Word] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class PageResult: + page_number: int + image_width: int + image_height: int + lines: list[Line] = field(default_factory=list) + # scale applied to the original scan before processing (for overlay mapping) + scale: float = 1.0 + + @property + def text(self) -> str: + return "\n".join(line.text for line in self.lines) + + @property + def corrected(self) -> str: + """Corrected page text (per-line corrected where available, else raw).""" + return "\n".join( + (line.corrected_text if line.corrected_text is not None else line.text) + for line in self.lines + ) + + @property + def has_correction(self) -> bool: + return any(line.corrected_text is not None for line in self.lines) + + @property + def avg_confidence(self) -> float: + confs = [w.confidence for line in self.lines for w in line.words] + return sum(confs) / len(confs) if confs else 0.0 + + @property + def low_confidence_word_count(self) -> int: + return sum(1 for line in self.lines for w in line.words if w.needs_review) + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["text"] = self.text + d["corrected"] = self.corrected + d["has_correction"] = self.has_correction + d["avg_confidence"] = self.avg_confidence + d["low_confidence_word_count"] = self.low_confidence_word_count + return d diff --git a/app/src/inkference/store/__init__.py b/app/src/inkference/store/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4ed189cdb226c1ece8e4df0673ada50f28fe87d8 --- /dev/null +++ b/app/src/inkference/store/__init__.py @@ -0,0 +1,4 @@ +"""SQLite document store + filesystem assets.""" +from .store import DocumentStore + +__all__ = ["DocumentStore"] diff --git a/app/src/inkference/store/__pycache__/__init__.cpython-311.pyc b/app/src/inkference/store/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c9e6169cf1ebb9bbe5e41c09bc9b0180306d1f7 Binary files /dev/null and b/app/src/inkference/store/__pycache__/__init__.cpython-311.pyc differ diff --git a/app/src/inkference/store/__pycache__/seed.cpython-311.pyc b/app/src/inkference/store/__pycache__/seed.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..416b5c3174455f5f39b12196c80630c0d774bf90 Binary files /dev/null and b/app/src/inkference/store/__pycache__/seed.cpython-311.pyc differ diff --git a/app/src/inkference/store/__pycache__/seed_book1.cpython-311.pyc b/app/src/inkference/store/__pycache__/seed_book1.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13eb0a82a9fc3eca9b3651c8a9eaccb6344f5ed0 Binary files /dev/null and b/app/src/inkference/store/__pycache__/seed_book1.cpython-311.pyc differ diff --git a/app/src/inkference/store/__pycache__/store.cpython-311.pyc b/app/src/inkference/store/__pycache__/store.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a402ae618d3941efba1115b48c3f7a5f7298bedb Binary files /dev/null and b/app/src/inkference/store/__pycache__/store.cpython-311.pyc differ diff --git a/app/src/inkference/store/seed.py b/app/src/inkference/store/seed.py new file mode 100644 index 0000000000000000000000000000000000000000..222e0d43e9a2e65f4302f55e4f50a4190c60cdd1 --- /dev/null +++ b/app/src/inkference/store/seed.py @@ -0,0 +1,140 @@ +"""Seed the store from existing line-crop transcriptions. + +We already have (line_crop.jpg, ground_truth.txt) pairs under transcriptions/. +Rather than re-OCR them, this loads them directly so the app ships with a populated +document (Reader + Ask work on first run, before any upload). Per page folder +(e.g. B1_P057) the crops are stitched vertically into a single page image for the +Reader's left pane, and each line's bbox is recorded against that stitched image. + +Ground-truth lines get confidence 1.0 (manual transcription, not model output). + +Usage: + python -m inkference.store.seed # default Captain Cook corpus + python -m inkference.store.seed --limit-pages 8 # smaller demo +""" +from __future__ import annotations + +import argparse +import re +from pathlib import Path + +from ..config import TRANSCRIPTIONS_ROOT, StoreConfig +from ..config import store as default_store +from ..schemas import Line, PageResult, Word +from .store import DocumentStore + +_MARGIN, _GAP = 40, 26 +_LINE_RE = re.compile(r"_L(\d+)\.(jpg|jpeg|png)$", re.IGNORECASE) + + +def _is_junk(path: Path) -> bool: + """Skip macOS archive artefacts (__MACOSX dirs, AppleDouble ._ files).""" + return path.name.startswith("._") or "__MACOSX" in path.parts + + +def _find_page_dirs(root: Path) -> list[Path]: + """Directories named like B_P that contain *_L##.jpg crops.""" + dirs: set[Path] = set() + for img in root.rglob("*_L*.jpg"): + if _LINE_RE.search(img.name) and not _is_junk(img): + dirs.add(img.parent) + return sorted(dirs, key=lambda p: p.name) + + +def _line_files(page_dir: Path) -> list[tuple[int, Path, Path]]: + """Sorted (line_no, image, text) triples for a page; only pairs with text.""" + out = [] + for img in page_dir.glob("*_L*.*"): + m = _LINE_RE.search(img.name) + if not m or _is_junk(img): + continue + txt = img.with_suffix(".txt") + if txt.exists(): + out.append((int(m.group(1)), img, txt)) + return sorted(out, key=lambda t: t[0]) + + +def _build_page(page_dir: Path, page_number: int, assets_dir: Path) -> PageResult | None: + from PIL import Image + + triples = _line_files(page_dir) + if not triples: + return None + + crops = [(Image.open(img).convert("RGB"), txt.read_text(encoding="utf-8").strip()) + for _, img, txt in triples] + crops = [(im, t) for im, t in crops if t] # drop empty GT + if not crops: + return None + + width = max(im.width for im, _ in crops) + 2 * _MARGIN + height = sum(im.height for im, _ in crops) + _GAP * (len(crops) + 1) + page_img = Image.new("RGB", (width, height), (250, 246, 237)) # paper tone + + lines: list[Line] = [] + y = _GAP + for idx, (im, text) in enumerate(crops): + page_img.paste(im, (_MARGIN, y)) + bbox = (_MARGIN, y, _MARGIN + im.width, y + im.height) + words = [Word(text=w, confidence=1.0, needs_review=False) for w in text.split()] + lines.append(Line(index=idx, bbox=bbox, text=text, confidence=1.0, + words=words, needs_review=False)) + y += im.height + _GAP + + assets_dir.mkdir(parents=True, exist_ok=True) + out_path = assets_dir / f"{page_dir.name}.png" + page_img.save(out_path) + + result = PageResult(page_number=page_number, image_width=width, + image_height=height, lines=lines, scale=1.0) + result._image_path = str(out_path) # type: ignore[attr-defined] + return result + + +def seed_corpus( + store: DocumentStore | None = None, + transcriptions_root: Path | None = None, + title: str = "Diary of Captain Cook Voyage", + slug: str = "captain-cook-voyage", + subtitle: str = "1772–1774", + limit_pages: int | None = None, +) -> int: + store = store or DocumentStore() + root = transcriptions_root or TRANSCRIPTIONS_ROOT + + if store.get_document_by_slug(slug): + print(f"document '{slug}' already seeded — skipping") + return store.get_document_by_slug(slug)["id"] + + page_dirs = _find_page_dirs(root) + if limit_pages: + page_dirs = page_dirs[:limit_pages] + if not page_dirs: + raise SystemExit(f"no page folders with *_L##.jpg + .txt found under {root}") + + doc_id = store.create_document(title=title, slug=slug, subtitle=subtitle) + assets = store.cfg.assets_dir / slug + seeded = 0 + for i, pd in enumerate(page_dirs, start=1): + result = _build_page(pd, page_number=i, assets_dir=assets) + if result is None: + continue + page_id = store.add_page(doc_id, i, image_path=getattr(result, "_image_path", None)) + store.save_page_result(page_id, result) + seeded += 1 + print(f" seeded page {i}: {pd.name} ({len(result.lines)} lines)") + + print(f"\nseeded document '{slug}' (id={doc_id}) with {seeded} pages") + return doc_id + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--limit-pages", type=int, default=None) + ap.add_argument("--slug", default="captain-cook-voyage") + args = ap.parse_args() + seed_corpus(slug=args.slug, limit_pages=args.limit_pages) + + +if __name__ == "__main__": + main() diff --git a/app/src/inkference/store/seed_book1.py b/app/src/inkference/store/seed_book1.py new file mode 100644 index 0000000000000000000000000000000000000000..bb07437398276874a6500fabc3573de224c9d6c4 --- /dev/null +++ b/app/src/inkference/store/seed_book1.py @@ -0,0 +1,158 @@ +"""Preseed the store from Alex's real Book-1 data (36 pages). + +Sources (default ~/Downloads/AlexFiles): + book1/forster1/B1_P_.jpg page scans + moredata/confidence/B1_P.txt raw words + per-word confidence ([L##] word(conf), <>(conf)) + moredata/corrected_transcriptions/ verified corrected transcription (the 36 pages) + +Raw and corrected have different line structures, so the corrected "green" view is +built with page-level word alignment (align_page, per post_correction_full_test_v2). + +Each page stores: raw lines (confidence view) + corrected display lines (green) + a +page image, then the RAG index is (re)built over the corrected text. + +Usage (from repo root, into a fresh data dir): + INKFERENCE_DATA_ROOT=$(pwd)/app/.inkference_data_book1 \ + python app/scripts/seed_book1.py --alex ~/Downloads/AlexFiles +""" +from __future__ import annotations + +import argparse +import re +import shutil +from pathlib import Path + +from ..htr.align import align_page +from ..schemas import Line, PageResult, Word +from ..store import DocumentStore + +_TOKEN_RE = re.compile(r"<<([^>]+)>>\(([\d.]+)\)|([^\s(]+)\(([\d.]+)\)") +_LINE_RE = re.compile(r"^\[L\d+\]\s?(.*)$") + + +def parse_confidence(path: Path) -> tuple[float, list[list[tuple[str, float]]]]: + """Return (threshold, lines) where each line is a list of (word, confidence).""" + threshold = 0.7 + lines: list[list[tuple[str, float]]] = [] + for row in path.read_text(encoding="utf-8").splitlines(): + if row.startswith("SCHWELLE:"): + try: + threshold = float(row.split(":", 1)[1]) + except ValueError: + pass + continue + m = _LINE_RE.match(row) + if not m: + continue + words: list[tuple[str, float]] = [] + for mt in _TOKEN_RE.finditer(m.group(1)): + if mt.group(1): + words.append((mt.group(1).strip(), float(mt.group(2)))) + else: + words.append((mt.group(3).strip(), float(mt.group(4)))) + lines.append(words) + return threshold, lines + + +def parse_corrected(path: Path) -> list[str]: + out = [] + for row in path.read_text(encoding="utf-8").splitlines(): + row = row.rstrip() + if row.startswith("===") or not row.strip(): + continue + out.append(row.strip()) + return out + + +def _image_for(stem: str, alex: Path) -> Path | None: + # data stem B1_P012 -> image B1_P_012.jpg + num = stem[len("B1_P"):] + cand = alex / "book1" / "forster1" / f"B1_P_{num}.jpg" + return cand if cand.exists() else None + + +def build_page(stem: str, alex: Path, assets: Path, page_number: int): + from PIL import Image + + conf_path = alex / "moredata" / "confidence" / f"{stem}.txt" + corr_path = alex / "moredata" / "corrected_transcriptions" / f"{stem}.txt" + if not conf_path.exists() or not corr_path.exists(): + return None + threshold, conf_lines = parse_confidence(conf_path) + corrected_lines = parse_corrected(corr_path) + flat_conf = [wc for line in conf_lines for wc in line] + + img_src = _image_for(stem, alex) + if img_src is None: + return None + with Image.open(img_src) as im: + W, H = im.size + assets.mkdir(parents=True, exist_ok=True) + dest = assets / f"{stem}.jpg" + shutil.copy2(img_src, dest) + + # raw lines (Raw view) with synthetic vertical bands + rowh = H / max(1, len(conf_lines)) + raw_lines: list[Line] = [] + for i, words in enumerate(conf_lines): + ws = [Word(w, c, c < threshold) for w, c in words] + raw_lines.append(Line(index=i, bbox=(0, int(i * rowh), W, int((i + 1) * rowh)), + text=" ".join(w for w, _ in words), + confidence=(sum(c for _, c in words) / len(words) if words else 0.0), + words=ws)) + + # corrected display lines (green) via page-level alignment + corrected_words = align_page(flat_conf, corrected_lines, threshold) + + return { + "page_number": page_number, "width": W, "height": H, "image_path": str(dest), + "raw_lines": raw_lines, "corrected_words": corrected_words, + "corrected_text": "\n".join(corrected_lines), + } + + +def seed_book1(alex_root: Path, store: DocumentStore | None = None, + title: str = "Forster — Voyage Round the World (Book 1)", + slug: str = "forster-book1", + subtitle: str = "Captain Cook's 2nd voyage · 1772–1775") -> int: + store = store or DocumentStore() + if store.get_document_by_slug(slug): + print(f"'{slug}' already seeded") + return store.get_document_by_slug(slug)["id"] + + corr_dir = alex_root / "moredata" / "corrected_transcriptions" + stems = sorted(p.stem for p in corr_dir.glob("B1_P*.txt")) + if not stems: + raise SystemExit(f"no corrected_transcriptions found under {corr_dir}") + + doc_id = store.create_document(title=title, slug=slug, subtitle=subtitle) + assets = store.cfg.assets_dir / slug + n = 0 + for i, stem in enumerate(stems, start=1): + page = build_page(stem, alex_root, assets, page_number=i) + if page is None: + print(f" {stem}: missing data/image, skipped") + continue + page_id = store.add_page(doc_id, i, image_path=page["image_path"]) + store.save_full_page(page_id, page["width"], page["height"], + page["raw_lines"], page["corrected_words"], page["corrected_text"]) + greens = sum(1 for ln in page["corrected_words"] for w in ln if w.qwen_replaced) + print(f" page {i:2d} ({stem}): {len(page['raw_lines'])} raw lines, {greens} corrections") + n += 1 + + # build RAG index over the corrected text + from ..rag.index import RagIndex + chunks = RagIndex().build_from_store(doc_id, store) + print(f"\nseeded '{slug}' (id={doc_id}) with {n} pages; RAG index: {chunks} chunks") + return doc_id + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--alex", default=str(Path.home() / "Downloads" / "AlexFiles")) + args = ap.parse_args() + seed_book1(Path(args.alex).expanduser()) + + +if __name__ == "__main__": + main() diff --git a/app/src/inkference/store/store.py b/app/src/inkference/store/store.py new file mode 100644 index 0000000000000000000000000000000000000000..fd7067fea5b1921e1524ada64c7c3194af68bf3d --- /dev/null +++ b/app/src/inkference/store/store.py @@ -0,0 +1,353 @@ +"""SQLite-backed store for documents, pages, lines, words, and ingest jobs. + +Uses only the stdlib `sqlite3` (no extra dependency). WAL mode + a fresh +connection per operation keeps it safe to use from FastAPI background threads. +Page scans and line crops live on the filesystem under StoreConfig.assets_dir; +the DB holds paths and structured transcription data. +""" +from __future__ import annotations + +import datetime as _dt +import json +import sqlite3 +from pathlib import Path +from typing import Any, Iterator + +from ..config import StoreConfig +from ..config import store as default_store +from ..schemas import JobStatus, Line, PageResult, Word + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS documents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + slug TEXT UNIQUE NOT NULL, + title TEXT NOT NULL, + subtitle TEXT, + created_at TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS pages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + page_number INTEGER NOT NULL, + image_path TEXT, + width INTEGER, + height INTEGER, + scale REAL DEFAULT 1.0, + status TEXT NOT NULL DEFAULT 'queued', + avg_confidence REAL, + low_conf_words INTEGER, + corrected_text TEXT, + corrected_lines TEXT, -- JSON: page-level corrected display lines (words w/ qwen_replaced) + UNIQUE(document_id, page_number) +); +CREATE TABLE IF NOT EXISTS lines ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, + idx INTEGER NOT NULL, + x0 INTEGER, y0 INTEGER, x1 INTEGER, y1 INTEGER, + text TEXT, + confidence REAL, + needs_review INTEGER DEFAULT 0, + corrected_text TEXT, + corrected_words TEXT -- JSON: [{text,confidence,needs_review,qwen_replaced}] +); +CREATE TABLE IF NOT EXISTS words ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + line_id INTEGER NOT NULL REFERENCES lines(id) ON DELETE CASCADE, + idx INTEGER NOT NULL, + text TEXT, + confidence REAL, + needs_review INTEGER DEFAULT 0 +); +CREATE TABLE IF NOT EXISTS jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + document_id INTEGER REFERENCES documents(id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'queued', + stage TEXT, + progress REAL DEFAULT 0.0, + message TEXT, + total_pages INTEGER DEFAULT 0, + done_pages INTEGER DEFAULT 0, + error TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_pages_doc ON pages(document_id); +CREATE INDEX IF NOT EXISTS idx_lines_page ON lines(page_id); +CREATE INDEX IF NOT EXISTS idx_words_line ON words(line_id); +""" + + +def _now() -> str: + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + + +class DocumentStore: + def __init__(self, cfg: StoreConfig = default_store) -> None: + self.cfg = cfg + self.cfg.assets_dir.mkdir(parents=True, exist_ok=True) + self.cfg.index_dir.mkdir(parents=True, exist_ok=True) + self.cfg.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init_schema() + + # -- connection --------------------------------------------------------- # + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.cfg.db_path, timeout=30) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + return conn + + def _init_schema(self) -> None: + with self._connect() as conn: + conn.executescript(_SCHEMA) + self._migrate(conn) + + def _migrate(self, conn: sqlite3.Connection) -> None: + """Add post-correction columns to DBs created before the feature existed.""" + def cols(table: str) -> set[str]: + return {r["name"] for r in conn.execute(f"PRAGMA table_info({table})")} + + page_cols = cols("pages") + if "corrected_text" not in page_cols: + conn.execute("ALTER TABLE pages ADD COLUMN corrected_text TEXT") + if "corrected_lines" not in page_cols: + conn.execute("ALTER TABLE pages ADD COLUMN corrected_lines TEXT") + line_cols = cols("lines") + if "corrected_text" not in line_cols: + conn.execute("ALTER TABLE lines ADD COLUMN corrected_text TEXT") + if "corrected_words" not in line_cols: + conn.execute("ALTER TABLE lines ADD COLUMN corrected_words TEXT") + + # -- documents ---------------------------------------------------------- # + def create_document(self, title: str, slug: str, subtitle: str | None = None) -> int: + with self._connect() as conn: + cur = conn.execute( + "INSERT INTO documents(slug,title,subtitle,created_at) VALUES(?,?,?,?)", + (slug, title, subtitle, _now()), + ) + return int(cur.lastrowid) + + def get_document_by_slug(self, slug: str) -> dict[str, Any] | None: + with self._connect() as conn: + row = conn.execute("SELECT * FROM documents WHERE slug=?", (slug,)).fetchone() + return dict(row) if row else None + + def list_documents(self) -> list[dict[str, Any]]: + with self._connect() as conn: + rows = conn.execute( + """SELECT d.*, + COUNT(p.id) AS page_count, + SUM(CASE WHEN p.status='complete' THEN 1 ELSE 0 END) AS pages_done, + AVG(p.avg_confidence) AS avg_confidence + FROM documents d LEFT JOIN pages p ON p.document_id=d.id + GROUP BY d.id ORDER BY d.created_at DESC""" + ).fetchall() + return [dict(r) for r in rows] + + def get_document(self, doc_id: int) -> dict[str, Any] | None: + with self._connect() as conn: + row = conn.execute("SELECT * FROM documents WHERE id=?", (doc_id,)).fetchone() + if not row: + return None + doc = dict(row) + pages = conn.execute( + """SELECT page_number,status,avg_confidence,low_conf_words,width,height + FROM pages WHERE document_id=? ORDER BY page_number""", + (doc_id,), + ).fetchall() + doc["pages"] = [dict(p) for p in pages] + doc["page_count"] = len(doc["pages"]) + return doc + + # -- pages -------------------------------------------------------------- # + def add_page( + self, doc_id: int, page_number: int, image_path: str | Path | None = None + ) -> int: + with self._connect() as conn: + cur = conn.execute( + """INSERT INTO pages(document_id,page_number,image_path,status) + VALUES(?,?,?, 'queued') + ON CONFLICT(document_id,page_number) + DO UPDATE SET image_path=excluded.image_path, status='queued'""", + (doc_id, page_number, str(image_path) if image_path else None), + ) + if cur.lastrowid: + return int(cur.lastrowid) + row = conn.execute( + "SELECT id FROM pages WHERE document_id=? AND page_number=?", + (doc_id, page_number), + ).fetchone() + return int(row["id"]) + + def set_page_status(self, page_id: int, status: str) -> None: + with self._connect() as conn: + conn.execute("UPDATE pages SET status=? WHERE id=?", (status, page_id)) + + def save_page_result(self, page_id: int, result: PageResult) -> None: + """Persist a completed PageResult: page metadata + lines + words.""" + with self._connect() as conn: + conn.execute( + """UPDATE pages SET width=?,height=?,scale=?,status='complete', + avg_confidence=?,low_conf_words=?,corrected_text=? WHERE id=?""", + ( + result.image_width, + result.image_height, + result.scale, + result.avg_confidence, + result.low_confidence_word_count, + result.corrected if result.has_correction else None, + page_id, + ), + ) + # Replace any prior lines/words for idempotent re-ingest. + conn.execute("DELETE FROM lines WHERE page_id=?", (page_id,)) + for line in result.lines: + x0, y0, x1, y1 = line.bbox + corrected_words_json = ( + json.dumps([ + {"text": w.text, "confidence": w.confidence, + "needs_review": w.needs_review, "qwen_replaced": w.qwen_replaced} + for w in line.corrected_words + ]) if line.corrected_words else None + ) + cur = conn.execute( + """INSERT INTO lines(page_id,idx,x0,y0,x1,y1,text,confidence,needs_review, + corrected_text,corrected_words) + VALUES(?,?,?,?,?,?,?,?,?,?,?)""", + (page_id, line.index, x0, y0, x1, y1, line.text, + line.confidence, int(line.needs_review), + line.corrected_text, corrected_words_json), + ) + line_id = cur.lastrowid + conn.executemany( + """INSERT INTO words(line_id,idx,text,confidence,needs_review) + VALUES(?,?,?,?,?)""", + [(line_id, i, w.text, w.confidence, int(w.needs_review)) + for i, w in enumerate(line.words)], + ) + + def save_full_page(self, page_id: int, width: int, height: int, + raw_lines: list[Line], corrected_lines: list[list["Word"]], + corrected_text: str, scale: float = 1.0) -> None: + """Persist a preseeded page whose raw and corrected texts have DIFFERENT + line structures: raw lines go in the lines table (Raw view); the corrected + display lines (page-level word alignment) are stored as page JSON.""" + confs = [w.confidence for ln in raw_lines for w in ln.words] + avg = sum(confs) / len(confs) if confs else 0.0 + low = sum(1 for ln in raw_lines for w in ln.words if w.needs_review) + cl_json = json.dumps([ + [{"text": w.text, "confidence": w.confidence, + "needs_review": w.needs_review, "qwen_replaced": w.qwen_replaced} for w in line] + for line in corrected_lines + ]) + with self._connect() as conn: + conn.execute( + """UPDATE pages SET width=?,height=?,scale=?,status='complete', + avg_confidence=?,low_conf_words=?,corrected_text=?,corrected_lines=? + WHERE id=?""", + (width, height, scale, avg, low, corrected_text, cl_json, page_id), + ) + conn.execute("DELETE FROM lines WHERE page_id=?", (page_id,)) + for line in raw_lines: + x0, y0, x1, y1 = line.bbox + cur = conn.execute( + """INSERT INTO lines(page_id,idx,x0,y0,x1,y1,text,confidence,needs_review) + VALUES(?,?,?,?,?,?,?,?,?)""", + (page_id, line.index, x0, y0, x1, y1, line.text, + line.confidence, int(line.needs_review)), + ) + line_id = cur.lastrowid + conn.executemany( + "INSERT INTO words(line_id,idx,text,confidence,needs_review) VALUES(?,?,?,?,?)", + [(line_id, i, w.text, w.confidence, int(w.needs_review)) + for i, w in enumerate(line.words)], + ) + + def get_page(self, doc_id: int, page_number: int) -> dict[str, Any] | None: + """Full page payload for the Reader: metadata + lines (with words).""" + with self._connect() as conn: + page = conn.execute( + "SELECT * FROM pages WHERE document_id=? AND page_number=?", + (doc_id, page_number), + ).fetchone() + if not page: + return None + page = dict(page) + lines = conn.execute( + "SELECT * FROM lines WHERE page_id=? ORDER BY idx", (page["id"],) + ).fetchall() + out_lines = [] + for ln in lines: + ln = dict(ln) + words = conn.execute( + "SELECT text,confidence,needs_review FROM words WHERE line_id=? ORDER BY idx", + (ln["id"],), + ).fetchall() + ln["words"] = [dict(w) for w in words] + ln["corrected_words"] = ( + json.loads(ln["corrected_words"]) if ln.get("corrected_words") else [] + ) + ln["bbox"] = [ln.pop("x0"), ln.pop("y0"), ln.pop("x1"), ln.pop("y1")] + out_lines.append(ln) + page["lines"] = out_lines + page["corrected_lines"] = ( + json.loads(page["corrected_lines"]) if page.get("corrected_lines") else [] + ) + return page + + def get_page_image_path(self, doc_id: int, page_number: int) -> str | None: + with self._connect() as conn: + row = conn.execute( + "SELECT image_path FROM pages WHERE document_id=? AND page_number=?", + (doc_id, page_number), + ).fetchone() + return row["image_path"] if row and row["image_path"] else None + + def iter_pages_text( + self, doc_id: int, prefer_corrected: bool = True + ) -> Iterator[tuple[int, str]]: + """(page_number, full_text) for completed pages — feeds RAG indexing. + + When prefer_corrected, use the page's Qwen-corrected text if present, so + retrieval and answers run over the cleaned transcription.""" + with self._connect() as conn: + pages = conn.execute( + "SELECT id,page_number,corrected_text FROM pages " + "WHERE document_id=? AND status='complete' ORDER BY page_number", + (doc_id,), + ).fetchall() + for p in pages: + if prefer_corrected and p["corrected_text"]: + yield int(p["page_number"]), p["corrected_text"] + continue + rows = conn.execute( + "SELECT text FROM lines WHERE page_id=? ORDER BY idx", (p["id"],) + ).fetchall() + text = "\n".join(r["text"] for r in rows if r["text"]) + yield int(p["page_number"]), text + + # -- jobs --------------------------------------------------------------- # + def create_job(self, doc_id: int, total_pages: int) -> int: + with self._connect() as conn: + cur = conn.execute( + """INSERT INTO jobs(document_id,status,total_pages,created_at,updated_at) + VALUES(?, 'queued', ?, ?, ?)""", + (doc_id, total_pages, _now(), _now()), + ) + return int(cur.lastrowid) + + def update_job(self, job_id: int, **fields: Any) -> None: + if not fields: + return + fields["updated_at"] = _now() + # JobStatus/Stage enums -> their string value + cleaned = {k: (v.value if hasattr(v, "value") else v) for k, v in fields.items()} + cols = ", ".join(f"{k}=?" for k in cleaned) + with self._connect() as conn: + conn.execute(f"UPDATE jobs SET {cols} WHERE id=?", (*cleaned.values(), job_id)) + + def get_job(self, job_id: int) -> dict[str, Any] | None: + with self._connect() as conn: + row = conn.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone() + return dict(row) if row else None