0.2.2: route through the authenticated optillm proxy (fast tier via DI_MIN_MAX_TOKENS), prompt fixes (plain prose, no invented specifics, no callbacks/placeholders), two-order accuracy-first preference judge, accurate dataset-card provenance

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 dist/datainventor-0.2.0-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
 dist/datainventor-0.2.1-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
+dist/datainventor-0.2.2-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -41,7 +41,7 @@ You register Hugging Face datasets as **sources** to curate, and optionally desc
 The hosted run works on about 200 documents per source and returns the kept and created documents with a report. The runner does the same on every document:
 
 ```bash
-pip install 'datainventor[full] @ https://huggingface.co/spaces/codelion/data-inventor/resolve/main/dist/datainventor-0.2.1-py3-none-any.whl'
+pip install 'datainventor[full] @ https://huggingface.co/spaces/codelion/data-inventor/resolve/main/dist/datainventor-0.2.2-py3-none-any.whl'
 export DI_API_KEY=...                      # key for the endpoint named in the recipe
 datainventor init recipe.yaml              # or download one from the app
 datainventor preview recipe.yaml           # every step on a sample

app/curate_api.py

@@ -23,7 +23,7 @@ PER_SOURCE = int(os.environ.get("CURATE_PER_SOURCE", "200"))
 GRADE_N = int(os.environ.get("CURATE_GRADE", "48"))
 REPHRASE_N = int(os.environ.get("CURATE_REPHRASE", "8"))
 MAX_SOURCES, MAX_TARGETS = 4, 2
-INSTALL = "pip install 'datainventor[full] @ https://huggingface.co/spaces/codelion/data-inventor/resolve/main/dist/datainventor-0.2.1-py3-none-any.whl'"
+INSTALL = "pip install 'datainventor[full] @ https://huggingface.co/spaces/codelion/data-inventor/resolve/main/dist/datainventor-0.2.2-py3-none-any.whl'"
 PREVIEWS: dict[str, Preview] = {}
 TRIES: dict[str, list] = {}
 GUARD = threading.Lock()

app/engine.py

@@ -89,6 +89,10 @@ No commentary, no code fences."""
 LEAK = re.compile(r"the user (asks|wants)|write one|return only|^okay|^sure,|^here is|^here's|^as an ai", re.I)
 END = re.compile(r"[.?!)\"”’'।:。！？]$")
 MULTI = re.compile(r"\b(generate|draft|write|compile|create|give me|produce)\s+(\d+|several|multiple|a set of|a list of)\b", re.I)
+PLACEHOLDER = re.compile(r"\[[A-Za-z'’ ]{0,30}\b(name|number|email|address|phone|date|id)\]", re.I)
+CALLBACK = re.compile(r"\b(like|as) you (said|mentioned|suggested|told me)\b|\b(last time|earlier) (you|we)\b|\bthanks again\b", re.I)
+TABLE = re.compile(r"^\s*\|.*\|\s*$\n^\s*\|?\s*:?-{3,}", re.M)
+VERDICT = re.compile(r"\W*\(?([AB])\b", re.I)
 LENGTHS = ["one blunt sentence", "two sentences with one specific detail",
            "three or four sentences with concrete specifics", "a short paragraph that sounds a little rushed"]
 
@@ -156,10 +160,12 @@ def run_job(job: Job, ep: Endpoint):
 
         def ok_q(t):
             t = t.strip().strip('"')
-            return 15 <= len(t) <= 1500 and END.search(t) and not degenerate(t) and not LEAK.search(t[:150]) and not MULTI.search(t[:120])
+            return (15 <= len(t) <= 1500 and END.search(t) and not degenerate(t) and not LEAK.search(t[:150])
+                    and not MULTI.search(t[:120]) and not PLACEHOLDER.search(t) and not CALLBACK.search(t))
 
         def ok_a(t):
-            return len(t.split()) >= 12 and not degenerate(t[:600]) and not LEAK.search(t[:100])
+            return (len(t.split()) >= 12 and not degenerate(t[:600]) and not LEAK.search(t[:100])
+                    and not TABLE.search(t) and not PLACEHOLDER.search(t))
 
         kept_grams: list[set] = []
         job.state, job.message = "generating", "Writing examples"
@@ -178,6 +184,7 @@ def run_job(job: Job, ep: Endpoint):
 Write ONE realistic message that {persona} would send to {role}, for this dataset, about: {leaf} ({area}, {sub}).
 Shape: {fmt}. Difficulty: {diff}. Length: {length}. Language: {lang}.
 It should sound like a real person typing, specific and a little imperfect, not a textbook prompt. It must be ONE question or ONE task whose answer fits in a single reply: never ask for a list of many items, a set of Q&A pairs, or a dataset. Do not name the shape or the persona.
+It is the first message in the conversation: do not refer to earlier chats or to anything said before. Do not use placeholders such as [Name] or [order number].
 Return only the message.""", max_tokens=700, temperature=1.0, validate=ok_q).strip().strip('"')
                 g = grams(q)
                 with job.lock:
@@ -186,6 +193,9 @@ Return only the message.""", max_tokens=700, temperature=1.0, validate=ok_q).str
                         return
                 answer_prompt = f"""You are {role}. Reply to the message below. The reply is a training example for this dataset: "{request}".
 Language: {lang}. Be accurate, specific and complete. Start directly with the content, with no preamble about who you are. Do not invent statistics or facts you cannot know.
+Write plain prose: short paragraphs, or a simple numbered list for steps. No tables, headings or emoji.
+Never invent order numbers, dates, prices, phone numbers, emails, URLs, product or brand names, menu paths or policies. If a detail like that is needed, ask for it or speak generally. Do not use placeholders such as [Your Name].
+For physical techniques and safety, give only the standard, widely taught practice, and make sure every step agrees with the others.
 {('Requirements: ' + constraints.strip()) if constraints.strip() else ''}
 
 Message: {q}"""
@@ -195,18 +205,25 @@ Message: {q}"""
                 if kind == "preference":
                     a2 = ep.chat(answer_prompt + "\n\nWrite a different answer from the obvious one, in a different structure and with different emphasis.",
                                  max_tokens=2500, temperature=1.0, validate=ok_a)
-                    verdict = ep.chat(f"""Which answer is better for the message? Judge accuracy, completeness and clarity.
+                    def judge(first: str, second: str) -> str:
+                        v = ep.chat(f"""Which answer is better for the message? Judge accuracy first: an answer that invents details (order numbers, dates, contact details, policies) or gives unsafe advice is worse. Then completeness and clarity.
 Message: {q}
 
 Answer A:
-{a1[:3000]}
+{first[:3000]}
 
 Answer B:
-{a2[:3000]}
-
-Reply with only the single letter A or B.""", max_tokens=8, temperature=0, validate=lambda s: s.strip()[:1].upper() in ("A", "B"))
-                    a_wins = verdict.strip()[:1].upper() == "A"
-                    row["chosen"], row["rejected"] = (a1, a2) if a_wins else (a2, a1)
+{second[:3000]}
+
+Reply with only the single letter A or B.""", max_tokens=16, temperature=0, allow_cut=True,
+                                    validate=lambda s: bool(VERDICT.match(s)))
+                        return VERDICT.match(v).group(1).upper()
+                    # judge both orders so the first-shown answer gets no edge; keep the row only if they agree
+                    v1, v2 = judge(a1, a2), judge(a2, a1)
+                    if (v1 == "A") != (v2 == "B"):
+                        job.reject("judge disagreed")
+                        return
+                    row["chosen"], row["rejected"] = (a1, a2) if v1 == "A" else (a2, a1)
                 else:
                     row["completion"] = a1
                 with job.lock:

app/main.py

@@ -54,7 +54,8 @@ def make_endpoint() -> Endpoint:
     if not key:
         raise HTTPException(503, "This host has no model configured. Set DI_API_KEY (or HF_TOKEN) and restart.")
     return Endpoint(base_url=os.environ.get("DI_BASE_URL", HF_BASE).rstrip("/"), api_key=key, model=os.environ.get("DI_MODEL", HF_MODEL),
-                    disable_thinking=os.environ.get("DI_DISABLE_THINKING", "0") != "0", workers=max(1, min(int(os.environ.get("DI_WORKERS", "4")), 8)))
+                    disable_thinking=os.environ.get("DI_DISABLE_THINKING", "0") != "0", workers=max(1, min(int(os.environ.get("DI_WORKERS", "4")), 8)),
+                    min_max_tokens=int(os.environ.get("DI_MIN_MAX_TOKENS", "0")))
 
 
 @app.get("/api/defaults")

app/publish.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 
 import io
 import json
+import os
 import re
 
 from huggingface_hub import HfApi
@@ -72,7 +73,7 @@ This dataset was generated with [DataInventor]({SPACE_URL}), an open-source tool
 3. For each combination the model wrote one message, then the reply{" (two replies, then a ranking)" if pref else ""}.
 4. Rows were rejected for cut-off or degenerate text, for asking for many items at once, and for near-duplicate questions.
 
-Model: `{model}`, served through Hugging Face Inference Providers.
+Model: `{model}`{", served through Hugging Face Inference Providers" if "router.huggingface.co" in os.environ.get("DI_BASE_URL", "router.huggingface.co") else ", served through an OpenAI-compatible router that routes each request to a hosted model (mostly gpt-oss-120b and Gemini Flash)"}.
 
 ## Quick numbers
 

datainventor/llm.py

@@ -25,11 +25,16 @@ class Endpoint:
     model: str = HF_MODEL
     disable_thinking: bool = False
     workers: int = 4
+    # Floor on max_tokens for every call. Routers that send small requests to slow
+    # local boxes (the optillm proxy keeps <=2000-token calls local) can be steered
+    # to their fast hosted tiers by raising the floor, e.g. 2048.
+    min_max_tokens: int = 0
 
     def chat(self, prompt: str, max_tokens: int = 1500, temperature: float = 0.8, retries: int = 5,
-             validate: Optional[Callable[[str], bool]] = None) -> str:
+             validate: Optional[Callable[[str], bool]] = None, allow_cut: bool = False) -> str:
         reasoning = bool(_REASONING.search(self.model)) and not self.disable_thinking
-        body = {"model": self.model, "max_tokens": max_tokens + (REASONING_HEADROOM if reasoning else 0),
+        body = {"model": self.model,
+                "max_tokens": max(self.min_max_tokens, max_tokens + (REASONING_HEADROOM if reasoning else 0)),
                 "temperature": temperature, "messages": [{"role": "user", "content": prompt}]}
         if reasoning and "gpt-oss" in self.model.lower():
             body["reasoning_effort"] = "low"
@@ -44,7 +49,8 @@ class Endpoint:
                                                          "Authorization": "Bearer " + self.api_key})
                 resp = json.load(urllib.request.urlopen(req, timeout=180))
                 choice = (resp.get("choices") or [{}])[0] or {}
-                if choice.get("finish_reason") == "length":
+                # allow_cut: short answers (a verdict letter) can be read even if the model kept talking
+                if choice.get("finish_reason") == "length" and not (allow_cut and ((choice.get("message") or {}).get("content") or "").strip()):
                     last = "reply cut off at max_tokens"
                     time.sleep(1.5)
                     continue

dist/datainventor-0.2.2-py3-none-any.whl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1263ef18bb7c8099a1264f0103213dd981d1e3c5075edae9736b4c611f7e5b14
+size 2414181

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "datainventor"
-version = "0.2.1"
+version = "0.2.2"
 description = "Curate a pretraining corpus on your own machines: clean, calibrate, create, compose."
 readme = "README.md"
 requires-python = ">=3.10"