Blame for engine.js

/* ResearchRAG Browser Edition | Deterministic local retrieval, no model API. */
(function (root) {
  'use strict';
  const STOP = new Set(('a al algo algunas algunos ante como con contra cual cuando de del desde donde dos el ella ellos en entre era es esa ese eso esta estar este esto estos fue ha haber hacia hasta hay la las le les lo los mas mi mis muy no nos o para pero por porque que quien se ser si sin sobre son su sus tambien te tiene todo todos tu un una unas uno unos y ya ' +
    'the this that these those with from there which what when where why how does do are was were have has had can could should would for and but not yes into over under between after before both each all any some than then your you our their its out per via into about ' +
    'um uma uns umas do da dos das em para pela pelo pelos pelas aos nas nos por com nao no na e ou mas sao foi esta este isso esses como que quais onde quando ser ter sua suas seus seus ao').split(/\s+/));
  const ACCENT = /[\u0300-\u036f]/g;
  const norm = s => String(s || '').normalize('NFD').replace(ACCENT,'').toLowerCase().replace(/[^a-z0-9\s]/g,' ').replace(/\s+/g,' ').trim();
  const tokens = text => norm(text).split(' ').filter(x => x.length >= 3 && !STOP.has(x)).slice(0,50000);
  const trigrams = text => {
    const s = norm(text).replace(/\s+/g,' '), out = new Set();
    for(let i=0;i<s.length-2;i++) { const tri=s.slice(i,i+3); if(tri.trim().length>=2) out.add(tri); }
    return out;
  };
  function segments(text, limit=650, overlap=1) {
    const cleaned=String(text||'').replace(/\r/g,'').trim();
    if(!cleaned) return [];
    const paragraphs=cleaned.split(/\n\s*\n/).flatMap(p=> {
      const p2=p.replace(/[ \t]+/g,' ').trim();
      if(p2.length<=limit) return p2?[p2]:[];
      // Sentence boundary first; split long sentences only at words.
      const ss=p2.split(/(?<=[.!?])\s+(?=[^\s])/u), units=[];
      for(const sentence of ss) {
        if(sentence.length<=limit) { units.push(sentence); continue; }
        let part='';
        for(const word of sentence.split(/\s+/)) {
          if(part.length+word.length+1>limit && part) {units.push(part);part='';}
          if(word.length>limit) { if(part) {units.push(part);part='';} for(let k=0;k<word.length;k+=limit) units.push(word.slice(k,k+limit)); }
          else part+=(part?' ':'')+word;
        }
        if(part)units.push(part);
      }
      const out=[];let curr='';
      for(const unit of units) {
        if(curr.length+unit.length+1>limit && curr){ out.push(curr);curr=''; }
        curr+=(curr?' ':'')+unit;
      }
      if(curr)out.push(curr);
      return out;
    });
    const result=[];
    for(let i=0;i<paragraphs.length;i++) {
      const val=paragraphs[i]; if(!val.trim())continue;
      const prev=(overlap && i>0?paragraphs[i-1].slice(-120)+' ':'');
      result.push((prev+val).slice(0,limit+130));
    }
    return result;
  }
  const DEMO=[
    {name:'01_Research_Methods.txt',text:'A transparent research workflow begins with a clearly defined question, explicit inclusion criteria, and traceable evidence. Reproducibility improves when data sources, transformations, assumptions, and limitations are documented. Researchers should retain methodological notes and verify every citation against the original source.\n\nFor multilingual research, lexical retrieval identifies matching words and phrases across the indexed documents. Semantic similarity may help surface conceptually related evidence, but semantic similarity is not a truth score. Researchers must inspect the retrieved passages, preserve citations, and report uncertainty whenever the evidence is incomplete.\n\nA review should distinguish reported findings from interpretation. When a result cannot be linked to a source passage, it should not be presented as a verified conclusion.'},
    {name:'02_Latin_America_Policy.txt',text:'Public policy research in Latin America often combines economic, social, educational, and institutional indicators. Cross-country comparisons require caution because definitions, coverage, and reporting years may differ. A responsible data dashboard must expose missing data, sources, and methodological limitations.\n\nEconomic indicators such as GDP per capita are useful for longitudinal comparisons, but they cannot on their own measure social wellbeing. Education indicators may reveal disparities in schooling access and completion. Reliable analyses compare like-for-like data and disclose uncertainty.\n\nOpen public data can improve reproducibility when the extraction date, indicator code, and source are recorded. Data revisions mean that historical series can change over time.'},
    {name:'03_AI_in_Education_ES.txt',text:'La inteligencia artificial en educación puede facilitar la revisión de literatura y el análisis de documentos académicos. Sin embargo, las respuestas automáticas no reemplazan la validación humana de las fuentes.\n\nLa privacidad es especialmente importante cuando se procesan trabajos estudiantiles o investigaciones inéditas. Los documentos sensibles deben mantenerse bajo control de sus propietarios. Un sistema de recuperación documental debe mostrar fragmentos verificables, identificar cada documento y reconocer cuando no encuentra evidencia suficiente.\n\nLa transparencia metodológica exige explicar cómo se seleccionan los fragmentos relevantes y evitar presentar coincidencias de palabras como pruebas definitivas.'}
  ];
  function buildIndex(documents) {
    const chunks=[], docs=[];
    documents.forEach((doc,d)=>{
      const name=String(doc.name||`Document ${d+1}`).slice(0,130), text=String(doc.text||'').trim();
      if(!text) return;
      const slices=segments(text); let idx=0;
      for(const slice of slices){idx++;chunks.push({id:`D${docs.length+1}:C${idx}`,docIndex:docs.length,doc:name,order:idx,text:slice,terms:tokens(slice),grams:trigrams(slice)});}
      docs.push({name,characters:text.length,chunks:idx});
    });
    const N=chunks.length, dfs=new Map();let total=0;
    for(const c of chunks){total+=c.terms.length;for(const t of new Set(c.terms))dfs.set(t,(dfs.get(t)||0)+1);}
    return {docs,chunks,dfs,N,avgdl:N?total/N:1};
  }
  function scoreQuery(index, query) {
    const qTerms=[...new Set(tokens(query))], qTrigrams=trigrams(query); const {N,dfs,avgdl}=index;
    return index.chunks.map(c=>{
      const counts=new Map();for(const t of c.terms)counts.set(t,(counts.get(t)||0)+1);
      let bm=0;
      for(const word of qTerms){ const tf=counts.get(word)||0;if(tf<=0)continue;
        const idf=Math.log(1+(N-(dfs.get(word)||0)+.5)/((dfs.get(word)||0)+.5));
        const k1=1.2,b=.75;
        bm+=idf*(tf*(k1+1))/(tf+k1*(1-b+b*c.terms.length/Math.max(1,avgdl)));
      }
      let intersect=0; for(const tri of qTrigrams)if(c.grams.has(tri))intersect++;
      const jac=qTrigrams.size?intersect/(qTrigrams.size+c.grams.size-intersect):0;
      const wordCoverage=qTerms.length?qTerms.filter(t=>counts.has(t)).length/qTerms.length:0;
      const lexical=bm/(bm+2.2);
      const score=0.63*lexical+0.27*wordCoverage+0.10*jac;
      return {...c,score,bm25:bm,wordCoverage,matchCount:qTerms.filter(t=>counts.has(t)).length};
    }).filter(x=>x.score>=0.075 && (x.bm25>0 || x.wordCoverage>0 || x.score>=0.095)).sort((a,b)=>b.score-a.score);
  }
  function snippetSentences(text) {
    const ss=String(text).split(/(?<=[.!?])\s+|\n+/u).map(x=>x.trim()).filter(x=>x.length>=25);
    return ss.length?ss:[String(text).slice(0,400)];
  }
  function query(index,q,k=5) {
    if(!index || !index.N)return {status:'no_documents',hits:[],answer:[],message:'Add a document before searching.'};
    if(!tokens(q).length)return {status:'empty_question',hits:[],answer:[],message:'Enter a question with meaningful words.'};
    const candidates=scoreQuery(index,q), hits=[], selectedIds=new Set();
    // Diversity: prefer new documents or nonadjacent chunks for additional sources.
    for(let i=0;i<Math.min(k,candidates.length);i++){
      let best,bs=-100;
      for(const c of candidates){if(selectedIds.has(c.id))continue;
        const redundancy=hits.some(h=>h.docIndex===c.docIndex)?0.035:0;
        const near=hits.some(h=>h.docIndex===c.docIndex && Math.abs(h.order-c.order)<=1)?0.025:0;
        const v=c.score-redundancy-near;
        if(v>bs){bs=v;best=c;}
      }
      if(!best)break;selectedIds.add(best.id);hits.push(best);
    }
    if(!hits.length)return {status:'no_evidence',hits:[],answer:[],message:'No sufficiently relevant passages found. Try different terms or add more source text.'};
    const qTerms=new Set(tokens(q)); const answerCandidates=[];
    hits.forEach(h=>{
      for(const s of snippetSentences(h.text)){
        const st=new Set(tokens(s)); const m=[...qTerms].filter(t=>st.has(t)).length;
        const density=m/Math.max(1,qTerms.size);
        const importance=0.65*h.score+0.35*density;
        answerCandidates.push({text:s,chunkId:h.id,doc:h.doc,score:importance,match:m});
      }
    });
    answerCandidates.sort((a,b)=>b.score-a.score);
    const answer=[], seen=new Set();
    for(const a of answerCandidates){
      const key=norm(a.text).slice(0,120);if(seen.has(key))continue;
      if(a.match===0 && answer.length) continue;
      answer.push(a);seen.add(key);
      if(answer.length>=3)break;
    }
    return {status:'ok',hits,answer,message:'Evidence-based excerpt selection. Verify against the original documents; this is not a generative answer.'};
  }
  root.ResearchRAG={buildIndex,query,segments,tokens,norm,DEMO};
})(typeof window!=='undefined'?window:globalThis);