| """Ollama wire schema shared with src/model/structured.rs; validators stay unchanged.""" |
| import json |
| import re |
| import uuid |
| from pathlib import Path |
| import prompt |
|
|
| ROOT = Path(__file__).resolve().parents[1]/'src'/'model' |
| WIRE_INSTRUCTION = (ROOT/'paired_sources.txt').read_text().rstrip() |
| S = {'type': 'string'} |
|
|
| def obj(properties): |
| return {'type':'object','properties':properties,'required':list(properties),'additionalProperties':False} |
|
|
| def quote_candidates(content): |
| spans = re.split(r'\n\[\.\.\. \d+ characters omitted \.\.\.\]\n', content) |
| if len(spans) > 1: |
| return list(dict.fromkeys(q for span in spans for q in quote_candidates(span))) |
| candidates = [content, *content.splitlines()] |
| start = 0 |
| previous = ' ' |
| for index, char in enumerate(content): |
| if char.isspace() and previous in '.!?': |
| candidates.append(content[start:index]); start = index |
| previous = char |
| candidates.append(content[start:].lstrip()) |
| return list(dict.fromkeys(s.lstrip() for s in candidates if s.strip() and 'characters omitted' not in s)) |
|
|
| def first_json(text): |
| start = len(text)-len(text.lstrip()) |
| value, end = json.JSONDecoder().raw_decode(text, start) |
| return value, text[end:] |
|
|
| def reflection_schema(user): |
| _, tail = first_json(user.split(' Question: ', 1)[1]) |
| boundary = '\nEvidence:\n' |
| if not tail.startswith(boundary): |
| raise ValueError('missing reflection evidence') |
| ids = [] |
| for line in tail[len(boundary):].splitlines(): |
| if not line.startswith('[') or '] ' not in line: |
| continue |
| candidate = line[1:].split('] ', 1)[0] |
| try: |
| uuid.UUID(candidate) |
| except ValueError: |
| continue |
| if candidate not in ids: |
| ids.append(candidate) |
| def branch(insufficient, minimum): |
| citations = ({'type':'array','items':{'enum':ids},'minItems':minimum,'maxItems':len(ids)} |
| if ids else {'type':'array','items':S,'maxItems':0}) |
| return obj({'answer':S,'citations':citations,'insufficient_evidence':{'const':insufficient}}) |
| return {'anyOf':[branch(False, 1),branch(True, 0)]} if ids else branch(True, 0) |
|
|
| def prepare(user): |
| if user.startswith('You are the extraction worker'): |
| existing, tail = first_json(user.split('\nExisting facts: ',1)[1]) |
| tail = tail.lstrip() |
| if not tail.startswith('Events: '): raise ValueError('missing events') |
| events, _ = first_json(tail[len('Events: '):]) |
| branches = [] |
| for entry in events: |
| quotes = quote_candidates(entry['event']['content']) |
| if quotes: |
| branches.append(obj({'source_index':{'const':entry['source_index']},'quote':{'enum':quotes}})) |
| ids = [f['id'] for f in existing] |
| related = {'type':'array','items':obj({'assertion_id':{'enum':ids},'relation':{'enum':['extends','contradicts','causes']},'explanation':S})} if ids else {'type':'array','items':{'type':'object'},'maxItems':0} |
| schema = json.loads(prompt.fill((ROOT/'extract_schema.json').read_text(),{'SOURCES':json.dumps(branches),'RELATED':json.dumps(related)})) if branches else obj({'claims':{'type':'array','items':{'type':'object'},'maxItems':0}}) |
| return user+'\n'+WIRE_INSTRUCTION, schema, True |
| if user.startswith('Consolidate evidence into'): |
| facts, _ = first_json(user.split(' Facts: ',1)[1]) |
| def ids(key): |
| values = [f[key] for f in facts] |
| return {'enum':values} if values else S |
| schema=json.loads(prompt.fill((ROOT/'consolidate_schema.json').read_text(),{'SUBJECTS':json.dumps(ids('subject_id')),'IDS':json.dumps(ids('id'))})) |
| return user, schema, False |
| if user.startswith('Answer only from the following cited evidence.'): |
| return user, reflection_schema(user), False |
| return user, 'json', False |
|
|
| def canonicalize(value, paired): |
| if paired: |
| for claim in value['claims']: |
| if 'source_indices' in claim or 'quotes' in claim: raise ValueError('mixed source formats') |
| pairs = claim.pop('source_quotes') |
| if not isinstance(pairs,list): raise ValueError('source pairs must be array') |
| if any(type(p['source_index']) is not int or p['source_index'] < 0 or not isinstance(p['quote'],str) for p in pairs): raise ValueError('invalid source pair') |
| claim['source_indices']=[p['source_index'] for p in pairs] |
| claim['quotes']=[p['quote'] for p in pairs] |
| return value |
|
|
| def training_row(row): |
| user, _, paired = prepare(row['prompt']) |
| target = json.loads(row['target']) |
| if paired: |
| for i,c in enumerate(target['claims']): |
| refs=[{'source_index':idx,'quote':q} for idx,q in zip(c.pop('source_indices'),c.pop('quotes'),strict=True)] |
| target['claims'][i]={'source_quotes':refs,**c} |
| return {**row,'prompt':user,'target':json.dumps(target,ensure_ascii=False)} |
|
|
