Spaces:

ethicalabs
/

echo-dsrn-openaire-clf-docs

Running

App Files Files Community

mrs83 commited on 8 days ago

Commit

12fb91f

verified ·

1 Parent(s): dc57d80

Upload folder using huggingface_hub

Browse files

Files changed (12) hide show

404.html +12 -12
architecture.html +759 -0
deployment.html +890 -0
index.html +14 -14
inference.html +783 -0
license.html +746 -0
llm-as-judge.html +1028 -0
mcp.html +889 -0
oauth.html +796 -0
quickstart.html +827 -0
search/search_index.json +1 -1
story.html +982 -0

404.html CHANGED Viewed

@@ -81,7 +81,7 @@
 <header class="md-header md-header--shadow" data-md-component="header">
   <nav class="md-header__inner md-grid" aria-label="Header">
-    <a href="/." title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
@@ -191,7 +191,7 @@
 <nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
   <label class="md-nav__title" for="__drawer">
-    <a href="/." title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
@@ -221,7 +221,7 @@
     <li class="md-nav__item">
-      <a href="/." class="md-nav__link">
@@ -248,7 +248,7 @@
     <li class="md-nav__item">
-      <a href="/quickstart/" class="md-nav__link">
@@ -275,7 +275,7 @@
     <li class="md-nav__item">
-      <a href="/architecture/" class="md-nav__link">
@@ -302,7 +302,7 @@
     <li class="md-nav__item">
-      <a href="/llm-as-judge/" class="md-nav__link">
@@ -329,7 +329,7 @@
     <li class="md-nav__item">
-      <a href="/mcp/" class="md-nav__link">
@@ -356,7 +356,7 @@
     <li class="md-nav__item">
-      <a href="/oauth/" class="md-nav__link">
@@ -383,7 +383,7 @@
     <li class="md-nav__item">
-      <a href="/inference/" class="md-nav__link">
@@ -410,7 +410,7 @@
     <li class="md-nav__item">
-      <a href="/story/" class="md-nav__link">
@@ -437,7 +437,7 @@
     <li class="md-nav__item">
-      <a href="/deployment/" class="md-nav__link">
@@ -464,7 +464,7 @@
     <li class="md-nav__item">
-      <a href="/license/" class="md-nav__link">

 <header class="md-header md-header--shadow" data-md-component="header">
   <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="/index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
 <nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
   <label class="md-nav__title" for="__drawer">
+    <a href="/index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
     <li class="md-nav__item">
+      <a href="/index.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/quickstart.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/architecture.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/llm-as-judge.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/mcp.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/oauth.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/inference.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/story.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/deployment.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="/license.html" class="md-nav__link">

architecture.html ADDED Viewed

	@@ -0,0 +1,759 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="quickstart.html">
+        <link rel="next" href="llm-as-judge.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>Architecture - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#architecture" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              Architecture
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="architecture.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#intent-labels" class="md-nav__link">
+    <span class="md-ellipsis">
+        Intent Labels
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#data-flow" class="md-nav__link">
+    <span class="md-ellipsis">
+        Data Flow
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+    <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#intent-labels" class="md-nav__link">
+    <span class="md-ellipsis">
+        Intent Labels
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#data-flow" class="md-nav__link">
+    <span class="md-ellipsis">
+        Data Flow
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="architecture">Architecture</h1>
+<table>
+<thead>
+<tr>
+<th>Component</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong>Frontend</strong></td>
+<td>Vue 3 / Vite dashboard — tabs: Model Card, Collab Hub, OpenAIRE Stream, Free Text, Saved History</td>
+</tr>
+<tr>
+<td><strong>Backend</strong></td>
+<td>FastAPI — inference, arXiv/OpenAIRE proxy, LLM judge orchestration, OAuth</td>
+</tr>
+<tr>
+<td><strong>Classifier</strong></td>
+<td>Echo-DSRN-98M (<code>EchoForSequenceClassification</code>) — fast CPU inference</td>
+</tr>
+<tr>
+<td><strong>Annotation DB</strong></td>
+<td>SQLite (<code>data/collaborative.db</code>) — user classifications and LLM judge labels</td>
+</tr>
+<tr>
+<td><strong>Worker</strong></td>
+<td>Celery + Redis — async classification and LLM-as-Judge queue</td>
+</tr>
+</tbody>
+</table>
+<h2 id="intent-labels">Intent Labels</h2>
+<table>
+<thead>
+<tr>
+<th>Label</th>
+<th>Meaning</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>Methodology</code></td>
+<td>Introduces a new method, model, or algorithm</td>
+</tr>
+<tr>
+<td><code>Dataset</code></td>
+<td>Introduces or documents a dataset or benchmark</td>
+</tr>
+<tr>
+<td><code>Review</code></td>
+<td>Surveys or synthesises existing work</td>
+</tr>
+<tr>
+<td><code>Applied</code></td>
+<td>Applies existing methods to a domain problem</td>
+</tr>
+<tr>
+<td><code>Theoretical</code></td>
+<td>Mathematical or formal analysis without empirical evaluation</td>
+</tr>
+</tbody>
+</table>
+<h2 id="data-flow">Data Flow</h2>
+<ol>
+<li><strong>OpenAIRE Graph API</strong> streams paper metadata (title, abstract, DOI, language)</li>
+<li><strong>Echo-DSRN-98M</strong> classifies research intent in &lt;1ms on CPU</li>
+<li>Results displayed in the Vue 3 dashboard with probability bars</li>
+<li>Optional: <strong>LLM-as-Judge</strong> enlists multiple LLMs to validate Echo's predictions</li>
+<li>Annotations stored in SQLite, forming a golden consensus dataset</li>
+</ol>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>

deployment.html ADDED Viewed

	@@ -0,0 +1,890 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="story.html">
+        <link rel="next" href="license.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>Deployment - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#kubernetes-deployment-k3s" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              Deployment
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="deployment.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#prerequisites" class="md-nav__link">
+    <span class="md-ellipsis">
+        Prerequisites
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#1-generate-postgresql-dump" class="md-nav__link">
+    <span class="md-ellipsis">
+        1. Generate PostgreSQL dump
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#2-install" class="md-nav__link">
+    <span class="md-ellipsis">
+        2. Install
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#3-load-golden-data" class="md-nav__link">
+    <span class="md-ellipsis">
+        3. Load golden data
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#4-access" class="md-nav__link">
+    <span class="md-ellipsis">
+        4. Access
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#components" class="md-nav__link">
+    <span class="md-ellipsis">
+        Components
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#troubleshooting" class="md-nav__link">
+    <span class="md-ellipsis">
+        Troubleshooting
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+    <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#prerequisites" class="md-nav__link">
+    <span class="md-ellipsis">
+        Prerequisites
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#1-generate-postgresql-dump" class="md-nav__link">
+    <span class="md-ellipsis">
+        1. Generate PostgreSQL dump
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#2-install" class="md-nav__link">
+    <span class="md-ellipsis">
+        2. Install
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#3-load-golden-data" class="md-nav__link">
+    <span class="md-ellipsis">
+        3. Load golden data
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#4-access" class="md-nav__link">
+    <span class="md-ellipsis">
+        4. Access
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#components" class="md-nav__link">
+    <span class="md-ellipsis">
+        Components
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#troubleshooting" class="md-nav__link">
+    <span class="md-ellipsis">
+        Troubleshooting
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="kubernetes-deployment-k3s">Kubernetes Deployment (k3s)</h1>
+<p>Deploy the OpenAIRE-AI-Research-Evaluator with Helm on k3s.</p>
+<h2 id="prerequisites">Prerequisites</h2>
+<ul>
+<li>k3s cluster running</li>
+<li><code>helm</code> installed</li>
+<li>Access to the Docker image <code>ethicalabs/openaire-research-evaluator:latest</code></li>
+<li><a href="https://huggingface.co/ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF">Request access</a> to the gated Echo-DSRN model</li>
+</ul>
+<h2 id="1-generate-postgresql-dump">1. Generate PostgreSQL dump</h2>
+<p>The SQLite golden database must be migrated to PostgreSQL:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>./scripts/sqlite_to_postgres.sh
+</code></pre></div>
+<p>Generates <code>data/collaborative_pg.sql</code> — ready for PostgreSQL import.</p>
+<h2 id="2-install">2. Install</h2>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>helm<span class="w"> </span>install<span class="w"> </span>evaluator<span class="w"> </span>./helm/openaire-research-evaluator<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="w">  </span>--set<span class="w"> </span>secrets.hfToken<span class="o">=</span>hf_...<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="w">  </span>--set<span class="w"> </span>secrets.secretKey<span class="o">=</span><span class="k">$(</span>openssl<span class="w"> </span>rand<span class="w"> </span>-hex<span class="w"> </span><span class="m">32</span><span class="k">)</span><span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="w">  </span>--set<span class="w"> </span>ingress.host<span class="o">=</span>openaire-eval.local<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="w">  </span>--set<span class="w"> </span>config.llmBaseUrl<span class="o">=</span>http://192.168.1.66:13305/v1
+</code></pre></div>
+<p>Or create a <code>values.local.yaml</code>:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="nt">secrets</span><span class="p">:</span>
+<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="w">  </span><span class="nt">hfToken</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">hf_...</span>
+<a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="w">  </span><span class="nt">secretKey</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">abc123...</span>
+<a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="w">  </span><span class="nt">hfClientId</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
+<a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="w">  </span><span class="nt">hfClientSecret</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">...</span>
+<a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a><span class="nt">ingress</span><span class="p">:</span>
+<a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a><span class="w">  </span><span class="nt">host</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">openaire-eval.local</span>
+<a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="nt">config</span><span class="p">:</span>
+<a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a><span class="w">  </span><span class="nt">llmBaseUrl</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">http://192.168.1.66:13305/v1</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a>helm<span class="w"> </span>install<span class="w"> </span>evaluator<span class="w"> </span>./helm/openaire-research-evaluator<span class="w"> </span>-f<span class="w"> </span>values.local.yaml
+</code></pre></div>
+<h2 id="3-load-golden-data">3. Load golden data</h2>
+<p>Wait for PostgreSQL to be ready, then import:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a>kubectl<span class="w"> </span>cp<span class="w"> </span>data/collaborative_pg.sql<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a><span class="w">  </span><span class="k">$(</span>kubectl<span class="w"> </span>get<span class="w"> </span>pod<span class="w"> </span>-l<span class="w"> </span><span class="nv">component</span><span class="o">=</span>postgres<span class="w"> </span>-o<span class="w"> </span><span class="nv">jsonpath</span><span class="o">=</span><span class="s1">&#39;{.items[0].metadata.name}&#39;</span><span class="k">)</span>:/tmp/collaborative_pg.sql
+<a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a>
+<a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a>kubectl<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>-it<span class="w"> </span>deployment/evaluator-openaire-research-evaluator-postgres<span class="w"> </span>--<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-4-5" name="__codelineno-4-5" href="#__codelineno-4-5"></a><span class="w">  </span>psql<span class="w"> </span>-U<span class="w"> </span>postgres<span class="w"> </span>-d<span class="w"> </span>echo_dsrn<span class="w"> </span>-f<span class="w"> </span>/tmp/collaborative_pg.sql
+<a id="__codelineno-4-6" name="__codelineno-4-6" href="#__codelineno-4-6"></a>
+<a id="__codelineno-4-7" name="__codelineno-4-7" href="#__codelineno-4-7"></a><span class="c1"># Reset auto-increment sequences after import (IDs were explicit in the dump)</span>
+<a id="__codelineno-4-8" name="__codelineno-4-8" href="#__codelineno-4-8"></a>kubectl<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>deployment/evaluator-openaire-research-evaluator-postgres<span class="w"> </span>--<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-4-9" name="__codelineno-4-9" href="#__codelineno-4-9"></a><span class="w">  </span>psql<span class="w"> </span>-U<span class="w"> </span>postgres<span class="w"> </span>-d<span class="w"> </span>echo_dsrn<span class="w"> </span>-c<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-4-10" name="__codelineno-4-10" href="#__codelineno-4-10"></a><span class="w">  </span><span class="s2">&quot;SELECT setval(&#39;annotations_id_seq&#39;, COALESCE((SELECT MAX(id) FROM annotations), 1))&quot;</span>
+</code></pre></div>
+<h2 id="4-access">4. Access</h2>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a>kubectl<span class="w"> </span>port-forward<span class="w"> </span>svc/evaluator-openaire-research-evaluator-web<span class="w"> </span><span class="m">7860</span>:7860
+</code></pre></div>
+<p>App at <code>http://localhost:7860</code>. With Ingress configured, at <code>http://openaire-eval.local</code>.</p>
+<p><strong>Local DNS setup:</strong> add to <code>/etc/hosts</code>:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;127.0.0.1 openaire-eval.local&quot;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>sudo<span class="w"> </span>tee<span class="w"> </span>-a<span class="w"> </span>/etc/hosts
+</code></pre></div>
+<p>Then access at <code>http://openaire-eval.local</code> (k3s Traefik listens on port 80).</p>
+<h2 id="components">Components</h2>
+<table>
+<thead>
+<tr>
+<th>Component</th>
+<th>Service</th>
+<th>Port</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>Web (FastAPI + Vue)</td>
+<td><code>&lt;release&gt;-web</code></td>
+<td>7860</td>
+</tr>
+<tr>
+<td>PostgreSQL</td>
+<td><code>&lt;release&gt;-postgres</code></td>
+<td>5432</td>
+</tr>
+<tr>
+<td>Redis</td>
+<td><code>&lt;release&gt;-redis</code></td>
+<td>6379</td>
+</tr>
+<tr>
+<td>Worker (Celery)</td>
+<td>— (internal)</td>
+<td>—</td>
+</tr>
+</tbody>
+</table>
+<h2 id="troubleshooting">Troubleshooting</h2>
+<p>Check web pod logs for alembic migration status:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>deployment/evaluator-openaire-research-evaluator-web
+</code></pre></div>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>

index.html CHANGED Viewed

@@ -12,7 +12,7 @@
-        <link rel="next" href="quickstart/">
@@ -88,7 +88,7 @@
 <header class="md-header md-header--shadow" data-md-component="header">
   <nav class="md-header__inner md-grid" aria-label="Header">
-    <a href="." title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
@@ -198,7 +198,7 @@
 <nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
   <label class="md-nav__title" for="__drawer">
-    <a href="." title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
@@ -255,7 +255,7 @@
           <span class="md-nav__icon md-icon"></span>
         </label>
-      <a href="." class="md-nav__link md-nav__link--active">
@@ -324,7 +324,7 @@
     <li class="md-nav__item">
-      <a href="quickstart/" class="md-nav__link">
@@ -351,7 +351,7 @@
     <li class="md-nav__item">
-      <a href="architecture/" class="md-nav__link">
@@ -378,7 +378,7 @@
     <li class="md-nav__item">
-      <a href="llm-as-judge/" class="md-nav__link">
@@ -405,7 +405,7 @@
     <li class="md-nav__item">
-      <a href="mcp/" class="md-nav__link">
@@ -432,7 +432,7 @@
     <li class="md-nav__item">
-      <a href="oauth/" class="md-nav__link">
@@ -459,7 +459,7 @@
     <li class="md-nav__item">
-      <a href="inference/" class="md-nav__link">
@@ -486,7 +486,7 @@
     <li class="md-nav__item">
-      <a href="story/" class="md-nav__link">
@@ -513,7 +513,7 @@
     <li class="md-nav__item">
-      <a href="deployment/" class="md-nav__link">
@@ -540,7 +540,7 @@
     <li class="md-nav__item">
-      <a href="license/" class="md-nav__link">
@@ -641,7 +641,7 @@
 </ul>
 <h2 id="hackathon-deliverable">Hackathon Deliverable</h2>
 <p>This project is the artifact submission for the OpenAIRE AI Hackathon 2026.</p>
-<p>The accompanying <a href="story/">story</a> explains the question, journey, insight, and what others can reuse.</p>
 <p><a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator">Source</a> · <a href="https://graph.openaire.eu">OpenAIRE Graph</a> · <a href="https://huggingface.co/ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF">Echo-DSRN Model</a></p>

+        <link rel="next" href="quickstart.html">
 <header class="md-header md-header--shadow" data-md-component="header">
   <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
 <nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
   <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
           <span class="md-nav__icon md-icon"></span>
         </label>
+      <a href="index.html" class="md-nav__link md-nav__link--active">
     <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
     <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
 </ul>
 <h2 id="hackathon-deliverable">Hackathon Deliverable</h2>
 <p>This project is the artifact submission for the OpenAIRE AI Hackathon 2026.</p>
+<p>The accompanying <a href="story.html">story</a> explains the question, journey, insight, and what others can reuse.</p>
 <p><a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator">Source</a> · <a href="https://graph.openaire.eu">OpenAIRE Graph</a> · <a href="https://huggingface.co/ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF">Echo-DSRN Model</a></p>

inference.html ADDED Viewed

	@@ -0,0 +1,783 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="oauth.html">
+        <link rel="next" href="story.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>Inference - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#inference" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              Inference
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="inference.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#install" class="md-nav__link">
+    <span class="md-ellipsis">
+        Install
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#classify" class="md-nav__link">
+    <span class="md-ellipsis">
+        Classify
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#model-specs" class="md-nav__link">
+    <span class="md-ellipsis">
+        Model Specs
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+    <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#install" class="md-nav__link">
+    <span class="md-ellipsis">
+        Install
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#classify" class="md-nav__link">
+    <span class="md-ellipsis">
+        Classify
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#model-specs" class="md-nav__link">
+    <span class="md-ellipsis">
+        Model Specs
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="inference">Inference</h1>
+<blockquote>
+<p><strong>Model:</strong> <a href="https://huggingface.co/ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF"><code>ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF</code></a> — gated repo, <a href="https://huggingface.co/ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF">request access</a> before use.</p>
+</blockquote>
+<h2 id="install">Install</h2>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>pip<span class="w"> </span>install<span class="w"> </span>git+https://github.com/ethicalabs-ai/Echo-DSRN.git
+</code></pre></div>
+<h2 id="classify">Classify</h2>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">echo_dsrn</span><span class="w"> </span><span class="kn">import</span> <span class="n">EchoForSequenceClassification</span>
+<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="kn">from</span><span class="w"> </span><span class="nn">transformers</span><span class="w"> </span><span class="kn">import</span> <span class="n">AutoTokenizer</span>
+<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>
+<a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="n">model</span>     <span class="o">=</span> <span class="n">EchoForSequenceClassification</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
+<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a>    <span class="s2">&quot;ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF&quot;</span><span class="p">,</span> <span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span>
+<a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a><span class="p">)</span>
+<a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a><span class="n">tokenizer</span> <span class="o">=</span> <span class="n">AutoTokenizer</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
+<a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a>    <span class="s2">&quot;ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF&quot;</span><span class="p">,</span> <span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span>
+<a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a><span class="p">)</span>
+<a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a>
+<a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a><span class="n">label</span><span class="p">,</span> <span class="n">probs</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">classify</span><span class="p">(</span>
+<a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a>    <span class="s2">&quot;Title: Attention Is All You Need</span><span class="se">\n</span><span class="s2">&quot;</span>
+<a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a>    <span class="s2">&quot;Abstract: We propose a new simple network architecture, the Transformer...&quot;</span><span class="p">,</span>
+<a id="__codelineno-1-14" name="__codelineno-1-14" href="#__codelineno-1-14"></a>    <span class="n">tokenizer</span><span class="o">=</span><span class="n">tokenizer</span><span class="p">,</span>
+<a id="__codelineno-1-15" name="__codelineno-1-15" href="#__codelineno-1-15"></a><span class="p">)</span>
+<a id="__codelineno-1-16" name="__codelineno-1-16" href="#__codelineno-1-16"></a><span class="nb">print</span><span class="p">(</span><span class="n">label</span><span class="p">,</span> <span class="n">probs</span><span class="p">)</span>
+<a id="__codelineno-1-17" name="__codelineno-1-17" href="#__codelineno-1-17"></a><span class="c1"># → Methodology  tensor([0.87, 0.03, 0.02, 0.06, 0.02])</span>
+</code></pre></div>
+<h2 id="model-specs">Model Specs</h2>
+<table>
+<thead>
+<tr>
+<th>Property</th>
+<th>Value</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>Architecture</td>
+<td>Echo-DSRN (Recurrent Neural Network)</td>
+</tr>
+<tr>
+<td>Parameters</td>
+<td>98,266,629 (~98M)</td>
+</tr>
+<tr>
+<td>Layers</td>
+<td>8 DSRN blocks</td>
+</tr>
+<tr>
+<td>Hidden dim</td>
+<td>512</td>
+</tr>
+<tr>
+<td>Attention heads</td>
+<td>4</td>
+</tr>
+<tr>
+<td>Vocab size</td>
+<td>32,017 tokens</td>
+</tr>
+<tr>
+<td>Precision</td>
+<td>bfloat16</td>
+</tr>
+<tr>
+<td>Inference</td>
+<td>Fast on CPU</td>
+</tr>
+</tbody>
+</table>
+<blockquote>
+<p>Run <code>curl /api/model/card</code> for live specs from the loaded model.</p>
+</blockquote>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>

license.html ADDED Viewed

	@@ -0,0 +1,746 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="deployment.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>License - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#license" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              License
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    License
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="license.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#code" class="md-nav__link">
+    <span class="md-ellipsis">
+        Code
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#openaire-graph" class="md-nav__link">
+    <span class="md-ellipsis">
+        OpenAIRE Graph
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#hackathon-deliverable" class="md-nav__link">
+    <span class="md-ellipsis">
+        Hackathon Deliverable
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#attribution" class="md-nav__link">
+    <span class="md-ellipsis">
+        Attribution
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#code" class="md-nav__link">
+    <span class="md-ellipsis">
+        Code
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#openaire-graph" class="md-nav__link">
+    <span class="md-ellipsis">
+        OpenAIRE Graph
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#hackathon-deliverable" class="md-nav__link">
+    <span class="md-ellipsis">
+        Hackathon Deliverable
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#attribution" class="md-nav__link">
+    <span class="md-ellipsis">
+        Attribution
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="license">License</h1>
+<h2 id="code">Code</h2>
+<p>Apache License 2.0 — see <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator/blob/main/LICENSE">LICENSE</a>.</p>
+<p>Copyright © 2026 ethicalabs.ai</p>
+<h2 id="openaire-graph">OpenAIRE Graph</h2>
+<p>This project uses the <a href="https://graph.openaire.eu">OpenAIRE Graph API</a>. All materials created by OpenAIRE are licensed under <a href="https://creativecommons.org/licenses/by/4.0/">CC BY 4.0</a>.</p>
+<h2 id="hackathon-deliverable">Hackathon Deliverable</h2>
+<p>This project is submitted to the <a href="https://innovation.openaire.eu/component/content/article/openaire-ai-hackathon.html">OpenAIRE AI Hackathon 2026</a>. Per hackathon requirements, all submitted materials are available under a CC-BY license.</p>
+<h2 id="attribution">Attribution</h2>
+<p>If you reuse this work, please cite:</p>
+<blockquote>
+<p><em>OpenAIRE-AI-Research-Evaluator — Multi-model LLM-as-Judge pipeline for research intent classification.</em>
+ethicalabs.ai, 2026. Apache-2.0 / CC-BY 4.0.
+<a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator">github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator</a></p>
+</blockquote>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>

llm-as-judge.html ADDED Viewed

	@@ -0,0 +1,1028 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="architecture.html">
+        <link rel="next" href="mcp.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>LLM-as-Judge - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#llm-as-judge" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              LLM-as-Judge
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="llm-as-judge.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#search-queries" class="md-nav__link">
+    <span class="md-ellipsis">
+        Search Queries
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#setup" class="md-nav__link">
+    <span class="md-ellipsis">
+        Setup
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#llm-server-setup" class="md-nav__link">
+    <span class="md-ellipsis">
+        LLM Server Setup
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="LLM Server Setup">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#ollama" class="md-nav__link">
+    <span class="md-ellipsis">
+        Ollama
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#lm-studio" class="md-nav__link">
+    <span class="md-ellipsis">
+        LM Studio
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#lemonade-llamacpp" class="md-nav__link">
+    <span class="md-ellipsis">
+        Lemonade / llama.cpp
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#docker-llm-on-host" class="md-nav__link">
+    <span class="md-ellipsis">
+        Docker — LLM on host
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#batch-runner-sync" class="md-nav__link">
+    <span class="md-ellipsis">
+        Batch Runner — Sync
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#batch-runner-async" class="md-nav__link">
+    <span class="md-ellipsis">
+        Batch Runner — Async
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#kubernetes-k3s" class="md-nav__link">
+    <span class="md-ellipsis">
+        Kubernetes (k3s)
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#single-run" class="md-nav__link">
+    <span class="md-ellipsis">
+        Single Run
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#pre-labeled-datasets" class="md-nav__link">
+    <span class="md-ellipsis">
+        Pre-labeled Datasets
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#evaluation" class="md-nav__link">
+    <span class="md-ellipsis">
+        Evaluation
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+    <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#search-queries" class="md-nav__link">
+    <span class="md-ellipsis">
+        Search Queries
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#setup" class="md-nav__link">
+    <span class="md-ellipsis">
+        Setup
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#llm-server-setup" class="md-nav__link">
+    <span class="md-ellipsis">
+        LLM Server Setup
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="LLM Server Setup">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#ollama" class="md-nav__link">
+    <span class="md-ellipsis">
+        Ollama
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#lm-studio" class="md-nav__link">
+    <span class="md-ellipsis">
+        LM Studio
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#lemonade-llamacpp" class="md-nav__link">
+    <span class="md-ellipsis">
+        Lemonade / llama.cpp
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#docker-llm-on-host" class="md-nav__link">
+    <span class="md-ellipsis">
+        Docker — LLM on host
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#batch-runner-sync" class="md-nav__link">
+    <span class="md-ellipsis">
+        Batch Runner — Sync
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#batch-runner-async" class="md-nav__link">
+    <span class="md-ellipsis">
+        Batch Runner — Async
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#kubernetes-k3s" class="md-nav__link">
+    <span class="md-ellipsis">
+        Kubernetes (k3s)
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#single-run" class="md-nav__link">
+    <span class="md-ellipsis">
+        Single Run
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#pre-labeled-datasets" class="md-nav__link">
+    <span class="md-ellipsis">
+        Pre-labeled Datasets
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#evaluation" class="md-nav__link">
+    <span class="md-ellipsis">
+        Evaluation
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="llm-as-judge">LLM-as-Judge</h1>
+<p>The LLM-as-Judge pipeline evaluates Echo-DSRN predictions against high-capability LLMs and produces a golden consensus dataset in <code>data/collaborative.db</code>.</p>
+<h2 id="search-queries">Search Queries</h2>
+<p>arXiv and OpenAIRE sources use keyword queries from <code>assets/topics.txt</code> — one query per line, blank lines and <code>#</code> comments ignored. Edit this file to target specific research domains.</p>
+<p>Container path: <code>/app/assets/topics.txt</code></p>
+<h2 id="setup">Setup</h2>
+<p>Point at a lemonade or llama.cpp server via <code>.env</code>:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>LLM_BASE_URL=http://192.168.1.40:13305/v1
+</code></pre></div>
+<p>Migrate the database:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>uv<span class="w"> </span>run<span class="w"> </span>alembic<span class="w"> </span>-c<span class="w"> </span>backend/alembic.ini<span class="w"> </span>upgrade<span class="w"> </span>head
+</code></pre></div>
+<h2 id="llm-server-setup">LLM Server Setup</h2>
+<p>Any OpenAI-compatible server works. Configuration via <code>.env</code>:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a>LLM_BASE_URL=&lt;server-url&gt;
+<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a>LLM_API_KEY=not-needed     # for local servers that skip auth
+</code></pre></div>
+<h3 id="ollama">Ollama</h3>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a>ollama<span class="w"> </span>pull<span class="w"> </span>qwen3:4b
+<a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>ollama<span class="w"> </span>serve
+</code></pre></div>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a>LLM_BASE_URL=http://localhost:11434/v1
+</code></pre></div>
+<h3 id="lm-studio">LM Studio</h3>
+<p>Start the local server in the UI (Developer → Local Server). Default port:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a>LLM_BASE_URL=http://localhost:1234/v1
+</code></pre></div>
+<h3 id="lemonade-llamacpp">Lemonade / llama.cpp</h3>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a>lemonade<span class="w"> </span>--host<span class="w"> </span><span class="m">0</span>.0.0.0<span class="w"> </span>load<span class="w"> </span>--pinned<span class="w"> </span>Qwen3.6-35B-A3B-GGUF
+</code></pre></div>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>LLM_BASE_URL=http://192.168.1.40:13305/v1
+</code></pre></div>
+<h3 id="docker-llm-on-host">Docker — LLM on host</h3>
+<p>When running the app in Docker and the LLM server on the host, use <code>host.docker.internal</code>:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a>LLM_BASE_URL=http://host.docker.internal:11434/v1
+</code></pre></div>
+<p>The <code>docker-compose.yaml</code> already includes <code>extra_hosts: host.docker.internal:host-gateway</code>.</p>
+<h2 id="batch-runner-sync">Batch Runner — Sync</h2>
+<p>Loads models on lemonade, judges papers inline:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a>./scripts/run_judges.sh<span class="w"> </span><span class="m">100</span>
+</code></pre></div>
+<p>Cycles through 11 GGUF models (Qwen3.6, Nemotron, Gemma, GPT-OSS, DeepSeek, GLM) against the catalog source.</p>
+<h2 id="batch-runner-async">Batch Runner — Async</h2>
+<p>Dispatches Celery tasks to the worker, no model loading:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a>./scripts/run_judges.sh<span class="w"> </span><span class="m">100</span><span class="w"> </span>--async
+</code></pre></div>
+<h2 id="kubernetes-k3s">Kubernetes (k3s)</h2>
+<p>Run the judge CLI directly in the deployed web pod:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a>kubectl<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>-it<span class="w"> </span>deployment/evaluator-openaire-research-evaluator-web<span class="w"> </span>--<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-11-2" name="__codelineno-11-2" href="#__codelineno-11-2"></a><span class="w">  </span>env<span class="w"> </span><span class="nv">LLM_BASE_URL</span><span class="o">=</span>http://192.168.1.66:13305/v1<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-11-3" name="__codelineno-11-3" href="#__codelineno-11-3"></a><span class="w">  </span>python<span class="w"> </span>backend/judge_cli.py<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-11-4" name="__codelineno-11-4" href="#__codelineno-11-4"></a><span class="w">    </span>--source<span class="w"> </span>catalog<span class="w"> </span>--n<span class="w"> </span><span class="m">100</span><span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-11-5" name="__codelineno-11-5" href="#__codelineno-11-5"></a><span class="w">    </span>--model<span class="w"> </span>gpt-oss-20b-mxfp4-GGUF<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-11-6" name="__codelineno-11-6" href="#__codelineno-11-6"></a><span class="w">    </span>--async
+</code></pre></div>
+<p>Tasks dispatch to the cluster's Celery worker, using the same Redis and PostgreSQL.</p>
+<h2 id="single-run">Single Run</h2>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a><span class="c1"># Local GGUF via llama.cpp / lemonade</span>
+<a id="__codelineno-12-2" name="__codelineno-12-2" href="#__codelineno-12-2"></a>uv<span class="w"> </span>run<span class="w"> </span>python<span class="w"> </span>backend/judge_cli.py<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-12-3" name="__codelineno-12-3" href="#__codelineno-12-3"></a><span class="w">    </span>--source<span class="w"> </span>catalog<span class="w"> </span>--n<span class="w"> </span><span class="m">100</span><span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-12-4" name="__codelineno-12-4" href="#__codelineno-12-4"></a><span class="w">    </span>--model<span class="w"> </span>Qwen3.6-35B-A3B-GGUF
+<a id="__codelineno-12-5" name="__codelineno-12-5" href="#__codelineno-12-5"></a>
+<a id="__codelineno-12-6" name="__codelineno-12-6" href="#__codelineno-12-6"></a><span class="c1"># Google Gemini (OpenAI-compatible endpoint)</span>
+<a id="__codelineno-12-7" name="__codelineno-12-7" href="#__codelineno-12-7"></a>uv<span class="w"> </span>run<span class="w"> </span>python<span class="w"> </span>backend/judge_cli.py<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-12-8" name="__codelineno-12-8" href="#__codelineno-12-8"></a><span class="w">    </span>--source<span class="w"> </span>catalog<span class="w"> </span>--n<span class="w"> </span><span class="m">100</span><span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-12-9" name="__codelineno-12-9" href="#__codelineno-12-9"></a><span class="w">    </span>--model<span class="w"> </span>gemini-2.5-flash<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-12-10" name="__codelineno-12-10" href="#__codelineno-12-10"></a><span class="w">    </span>--llm-url<span class="w"> </span>https://generativelanguage.googleapis.com/v1beta/openai/<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-12-11" name="__codelineno-12-11" href="#__codelineno-12-11"></a><span class="w">    </span>--llm-key<span class="w"> </span><span class="si">${</span><span class="nv">GEMINI_API_KEY</span><span class="si">}</span>
+</code></pre></div>
+<h2 id="pre-labeled-datasets">Pre-labeled Datasets</h2>
+<p>Convert chat-format JSONL to flat records, then judge:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-13-1" name="__codelineno-13-1" href="#__codelineno-13-1"></a>uv<span class="w"> </span>run<span class="w"> </span>python<span class="w"> </span>scripts/reformat_for_judge.py<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-13-2" name="__codelineno-13-2" href="#__codelineno-13-2"></a><span class="w">    </span>~/.ethicalabs/datasets/research-intent/train.jsonl<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-13-3" name="__codelineno-13-3" href="#__codelineno-13-3"></a><span class="w">    </span>~/.ethicalabs/datasets/research-intent/train_flat.jsonl
+<a id="__codelineno-13-4" name="__codelineno-13-4" href="#__codelineno-13-4"></a>
+<a id="__codelineno-13-5" name="__codelineno-13-5" href="#__codelineno-13-5"></a>uv<span class="w"> </span>run<span class="w"> </span>python<span class="w"> </span>backend/judge_cli.py<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-13-6" name="__codelineno-13-6" href="#__codelineno-13-6"></a><span class="w">    </span>--source<span class="w"> </span>dataset<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-13-7" name="__codelineno-13-7" href="#__codelineno-13-7"></a><span class="w">    </span>--dataset-path<span class="w"> </span>~/.ethicalabs/datasets/research-intent/train_flat.jsonl<span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-13-8" name="__codelineno-13-8" href="#__codelineno-13-8"></a><span class="w">    </span>--n<span class="w"> </span><span class="m">100</span><span class="w"> </span>--model<span class="w"> </span>Qwen3.6-35B-A3B-GGUF
+</code></pre></div>
+<h2 id="evaluation">Evaluation</h2>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-14-1" name="__codelineno-14-1" href="#__codelineno-14-1"></a>uv<span class="w"> </span>run<span class="w"> </span>python<span class="w"> </span>backend/judge_eval.py<span class="w">                </span><span class="c1"># all papers</span>
+<a id="__codelineno-14-2" name="__codelineno-14-2" href="#__codelineno-14-2"></a>uv<span class="w"> </span>run<span class="w"> </span>python<span class="w"> </span>backend/judge_eval.py<span class="w"> </span>--paper-source<span class="w"> </span>dataset<span class="w">  </span><span class="c1"># golden set only</span>
+</code></pre></div>
+<p>Computes agreement rates, class-level accuracy, and per-model alignment stats.</p>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>

mcp.html ADDED Viewed

	@@ -0,0 +1,889 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="llm-as-judge.html">
+        <link rel="next" href="oauth.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>MCP - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#mcp-agent-classification-endpoint" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              MCP
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="mcp.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#server-configuration" class="md-nav__link">
+    <span class="md-ellipsis">
+        Server Configuration
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#authentication" class="md-nav__link">
+    <span class="md-ellipsis">
+        Authentication
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#tools-exposed" class="md-nav__link">
+    <span class="md-ellipsis">
+        Tools exposed
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="Tools exposed">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#input" class="md-nav__link">
+    <span class="md-ellipsis">
+        Input
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#output" class="md-nav__link">
+    <span class="md-ellipsis">
+        Output
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#raw-sse-usage" class="md-nav__link">
+    <span class="md-ellipsis">
+        Raw SSE usage
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#architecture" class="md-nav__link">
+    <span class="md-ellipsis">
+        Architecture
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+    <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#server-configuration" class="md-nav__link">
+    <span class="md-ellipsis">
+        Server Configuration
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#authentication" class="md-nav__link">
+    <span class="md-ellipsis">
+        Authentication
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#tools-exposed" class="md-nav__link">
+    <span class="md-ellipsis">
+        Tools exposed
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="Tools exposed">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#input" class="md-nav__link">
+    <span class="md-ellipsis">
+        Input
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#output" class="md-nav__link">
+    <span class="md-ellipsis">
+        Output
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#raw-sse-usage" class="md-nav__link">
+    <span class="md-ellipsis">
+        Raw SSE usage
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#architecture" class="md-nav__link">
+    <span class="md-ellipsis">
+        Architecture
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="mcp-agent-classification-endpoint">MCP — Agent Classification Endpoint</h1>
+<p>Expose Echo-DSRN-98M research intent classification to AI agents via the Model Context Protocol (SSE transport).</p>
+<h2 id="server-configuration">Server Configuration</h2>
+<p>Add to your MCP client config (Claude Desktop, Cursor, Continue, etc.):</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="p">{</span>
+<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="w">  </span><span class="nt">&quot;mcpServers&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
+<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="w">    </span><span class="nt">&quot;echo-dsrn&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
+<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="w">      </span><span class="nt">&quot;url&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;http://localhost:7860/api/mcp/classify&quot;</span><span class="p">,</span>
+<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="w">      </span><span class="nt">&quot;transport&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;sse&quot;</span><span class="p">,</span>
+<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="w">      </span><span class="nt">&quot;headers&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
+<a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="w">        </span><span class="nt">&quot;Authorization&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;Bearer echo-dsrn-mcp-change-me-in-production&quot;</span>
+<a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a><span class="w">      </span><span class="p">}</span>
+<a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a><span class="w">    </span><span class="p">}</span>
+<a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a><span class="w">  </span><span class="p">}</span>
+<a id="__codelineno-0-11" name="__codelineno-0-11" href="#__codelineno-0-11"></a><span class="p">}</span>
+</code></pre></div>
+<p>Set <code>MCP_API_KEY</code> in <code>.env</code> to match:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>MCP_API_KEY=echo-dsrn-mcp-change-me-in-production
+</code></pre></div>
+<h2 id="authentication">Authentication</h2>
+<p>Bearer token via <code>Authorization</code> header. No query-string keys.</p>
+<h2 id="tools-exposed">Tools exposed</h2>
+<table>
+<thead>
+<tr>
+<th>Tool</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>classify_intent</code></td>
+<td>Classify a research paper by title + abstract into one of 5 intents</td>
+</tr>
+</tbody>
+</table>
+<h3 id="input">Input</h3>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="p">{</span>
+<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="w">  </span><span class="nt">&quot;title&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;Attention Is All You Need&quot;</span><span class="p">,</span>
+<a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="w">  </span><span class="nt">&quot;abstract&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;We propose a new simple network architecture, the Transformer...&quot;</span>
+<a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="p">}</span>
+</code></pre></div>
+<h3 id="output">Output</h3>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="p">{</span>
+<a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a><span class="w">  </span><span class="nt">&quot;label&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;Methodology&quot;</span><span class="p">,</span>
+<a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a><span class="w">  </span><span class="nt">&quot;probabilities&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
+<a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a><span class="w">    </span><span class="nt">&quot;Methodology&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.87</span><span class="p">,</span>
+<a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a><span class="w">    </span><span class="nt">&quot;Dataset&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.03</span><span class="p">,</span>
+<a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a><span class="w">    </span><span class="nt">&quot;Review&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.02</span><span class="p">,</span>
+<a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a><span class="w">    </span><span class="nt">&quot;Applied&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.06</span><span class="p">,</span>
+<a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a><span class="w">    </span><span class="nt">&quot;Theoretical&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.02</span>
+<a id="__codelineno-3-9" name="__codelineno-3-9" href="#__codelineno-3-9"></a><span class="w">  </span><span class="p">}</span>
+<a id="__codelineno-3-10" name="__codelineno-3-10" href="#__codelineno-3-10"></a><span class="p">}</span>
+</code></pre></div>
+<h2 id="raw-sse-usage">Raw SSE usage</h2>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a>curl<span class="w"> </span>-N<span class="w"> </span>-H<span class="w"> </span><span class="s2">&quot;Authorization: Bearer echo-dsrn-mcp-change-me-in-production&quot;</span><span class="w"> </span><span class="se">\</span>
+<a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a><span class="w">  </span><span class="s2">&quot;http://localhost:7860/api/mcp/classify?title=Attention+Is+All+You+Need&amp;abstract=We+propose+the+Transformer&quot;</span>
+</code></pre></div>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a><span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
+<a id="__codelineno-5-2" name="__codelineno-5-2" href="#__codelineno-5-2"></a>
+<a id="__codelineno-5-3" name="__codelineno-5-3" href="#__codelineno-5-3"></a><span class="n">resp</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span>
+<a id="__codelineno-5-4" name="__codelineno-5-4" href="#__codelineno-5-4"></a>    <span class="s2">&quot;http://localhost:7860/api/mcp/classify&quot;</span><span class="p">,</span>
+<a id="__codelineno-5-5" name="__codelineno-5-5" href="#__codelineno-5-5"></a>    <span class="n">params</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;title&quot;</span><span class="p">:</span> <span class="s2">&quot;Attention Is All You Need&quot;</span><span class="p">,</span> <span class="s2">&quot;abstract&quot;</span><span class="p">:</span> <span class="s2">&quot;We propose...&quot;</span><span class="p">},</span>
+<a id="__codelineno-5-6" name="__codelineno-5-6" href="#__codelineno-5-6"></a>    <span class="n">headers</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;Authorization&quot;</span><span class="p">:</span> <span class="s2">&quot;Bearer echo-dsrn-mcp-change-me-in-production&quot;</span><span class="p">},</span>
+<a id="__codelineno-5-7" name="__codelineno-5-7" href="#__codelineno-5-7"></a>    <span class="n">stream</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+<a id="__codelineno-5-8" name="__codelineno-5-8" href="#__codelineno-5-8"></a><span class="p">)</span>
+<a id="__codelineno-5-9" name="__codelineno-5-9" href="#__codelineno-5-9"></a><span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">iter_lines</span><span class="p">():</span>
+<a id="__codelineno-5-10" name="__codelineno-5-10" href="#__codelineno-5-10"></a>    <span class="k">if</span> <span class="n">line</span><span class="p">:</span>
+<a id="__codelineno-5-11" name="__codelineno-5-11" href="#__codelineno-5-11"></a>        <span class="nb">print</span><span class="p">(</span><span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span>
+</code></pre></div>
+<h2 id="architecture">Architecture</h2>
+<ol>
+<li>Request hits SSE endpoint → Bearer token validated</li>
+<li>Celery task <code>classify_mcp</code> enqueued to Redis</li>
+<li>Worker loads Echo-DSRN-98M (baked into Docker image, no HF download)</li>
+<li>Fast CPU inference → result streamed via SSE</li>
+</ol>
+<p>No papers saved. No database writes. Stateless.</p>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>

oauth.html ADDED Viewed

	@@ -0,0 +1,796 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="mcp.html">
+        <link rel="next" href="inference.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>OAuth - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#oauth-hugging-face-login" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              OAuth
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="oauth.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#setup" class="md-nav__link">
+    <span class="md-ellipsis">
+        Setup
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="Setup">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#1-create-an-oauth-app" class="md-nav__link">
+    <span class="md-ellipsis">
+        1. Create an OAuth App
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#2-configure-env" class="md-nav__link">
+    <span class="md-ellipsis">
+        2. Configure .env
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#3-production" class="md-nav__link">
+    <span class="md-ellipsis">
+        3. Production
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#how-it-works" class="md-nav__link">
+    <span class="md-ellipsis">
+        How it works
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+    <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#setup" class="md-nav__link">
+    <span class="md-ellipsis">
+        Setup
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="Setup">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#1-create-an-oauth-app" class="md-nav__link">
+    <span class="md-ellipsis">
+        1. Create an OAuth App
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#2-configure-env" class="md-nav__link">
+    <span class="md-ellipsis">
+        2. Configure .env
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#3-production" class="md-nav__link">
+    <span class="md-ellipsis">
+        3. Production
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#how-it-works" class="md-nav__link">
+    <span class="md-ellipsis">
+        How it works
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="oauth-hugging-face-login">OAuth — Hugging Face Login</h1>
+<p>The Collab Hub uses Hugging Face OAuth for authentication. Users log in with their HF account to vote on annotations and save papers.</p>
+<h2 id="setup">Setup</h2>
+<h3 id="1-create-an-oauth-app">1. Create an OAuth App</h3>
+<p>Go to <a href="https://huggingface.co/settings/applications/new">Hugging Face OAuth Apps</a> and create a new app:</p>
+<ul>
+<li><strong>Name</strong>: <code>Echo-DSRN Collab Hub</code></li>
+<li><strong>Homepage URL</strong>: <code>http://localhost:7860</code> (or your production URL)</li>
+<li><strong>Redirect URL</strong>: <code>http://localhost:7860/api/auth/callback/hf</code></li>
+</ul>
+<h3 id="2-configure-env">2. Configure .env</h3>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>APP_BASE_URL=http://localhost:7860
+<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>HF_CLIENT_ID=&lt;your-client-id&gt;
+<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a>HF_CLIENT_SECRET=&lt;your-client-secret&gt;
+</code></pre></div>
+<h3 id="3-production">3. Production</h3>
+<p>Replace <code>localhost:7860</code> with your production domain. Both the Homepage URL and Redirect URL must match.</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>APP_BASE_URL=https://echo-dsrn.example.com
+</code></pre></div>
+<p>Redirect URL: <code>https://echo-dsrn.example.com/api/auth/callback/hf</code></p>
+<h2 id="how-it-works">How it works</h2>
+<ol>
+<li>User clicks "Login with Hugging Face" in the Collab Hub</li>
+<li>Redirected to HF authorization page</li>
+<li>After consent, redirected back to <code>/api/auth/callback/hf</code></li>
+<li>Backend exchanges code for token, fetches user profile</li>
+<li>JWT session cookie set — user is authenticated</li>
+</ol>
+<p>User data stored: HF username, name, email, avatar URL, and optionally institution (from profile).</p>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>

quickstart.html ADDED Viewed

	@@ -0,0 +1,827 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="index.html">
+        <link rel="next" href="architecture.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>Quick Start - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#quick-start" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              Quick Start
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="quickstart.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#prerequisites" class="md-nav__link">
+    <span class="md-ellipsis">
+        Prerequisites
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#docker-compose" class="md-nav__link">
+    <span class="md-ellipsis">
+        Docker Compose
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#python" class="md-nav__link">
+    <span class="md-ellipsis">
+        Python
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="Python">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#uv" class="md-nav__link">
+    <span class="md-ellipsis">
+        uv
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#pip" class="md-nav__link">
+    <span class="md-ellipsis">
+        pip
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#makefile" class="md-nav__link">
+    <span class="md-ellipsis">
+        Makefile
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+    <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="story.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#prerequisites" class="md-nav__link">
+    <span class="md-ellipsis">
+        Prerequisites
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#docker-compose" class="md-nav__link">
+    <span class="md-ellipsis">
+        Docker Compose
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#python" class="md-nav__link">
+    <span class="md-ellipsis">
+        Python
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="Python">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#uv" class="md-nav__link">
+    <span class="md-ellipsis">
+        uv
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#pip" class="md-nav__link">
+    <span class="md-ellipsis">
+        pip
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#makefile" class="md-nav__link">
+    <span class="md-ellipsis">
+        Makefile
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="quick-start">Quick Start</h1>
+<h2 id="prerequisites">Prerequisites</h2>
+<ul>
+<li><strong>Python</strong> ≥ 3.12 (tested with 3.12.3)</li>
+<li><strong>Node.js</strong> ≥ 22 (tested with 22.22.3)</li>
+<li><strong>npm</strong> ≥ 10 (tested with 10.9.8)</li>
+<li><strong>Docker</strong> (optional, for containerised deployment)</li>
+</ul>
+<h2 id="docker-compose">Docker Compose</h2>
+<p>The Echo-DSRN classifier is a gated HF model. Before building:</p>
+<ol>
+<li><a href="https://huggingface.co/ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF">Request access</a> to the model repo</li>
+<li>Create an <a href="https://huggingface.co/settings/tokens">HF access token</a> with read permissions</li>
+<li>Add it to <code>.env</code>: <code>HF_TOKEN=hf_...</code></li>
+</ol>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>make<span class="w"> </span>rebuild<span class="w">      </span><span class="c1"># builds Docker image with the gated model</span>
+<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>docker<span class="w"> </span>compose<span class="w"> </span>up<span class="w"> </span>-d
+</code></pre></div>
+<p>App serves on <code>http://localhost:7860</code>. Redis, the web server, and a Celery worker start automatically.</p>
+<h2 id="python">Python</h2>
+<h3 id="uv">uv</h3>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>uv<span class="w"> </span>run<span class="w"> </span>python<span class="w"> </span>backend/server.py
+</code></pre></div>
+<h3 id="pip">pip</h3>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a>pip<span class="w"> </span>install<span class="w"> </span>-r<span class="w"> </span>requirements.txt
+<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a>python<span class="w"> </span>backend/server.py
+</code></pre></div>
+<p>Build the frontend first: <code>cd frontend &amp;&amp; npm ci &amp;&amp; npm run build</code>. App serves on <code>http://localhost:7860</code>.</p>
+<h2 id="makefile">Makefile</h2>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a>make<span class="w"> </span><span class="nb">help</span><span class="w">          </span><span class="c1"># list all targets</span>
+<a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>make<span class="w"> </span>up<span class="w">            </span><span class="c1"># docker compose up -d</span>
+<a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a>make<span class="w"> </span>build-docker<span class="w">  </span><span class="c1"># build the Docker image</span>
+<a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a>make<span class="w"> </span><span class="nb">test</span><span class="w">          </span><span class="c1"># run pytest</span>
+<a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a>make<span class="w"> </span>lint<span class="w">          </span><span class="c1"># pre-commit hooks</span>
+<a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a>make<span class="w"> </span>migrate<span class="w">       </span><span class="c1"># alembic upgrade head</span>
+<a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a>make<span class="w"> </span>judge<span class="w"> </span><span class="nv">N</span><span class="o">=</span><span class="m">100</span><span class="w">   </span><span class="c1"># run LLM-as-Judge (sync)</span>
+<a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a>make<span class="w"> </span>judge-async<span class="w"> </span><span class="nv">N</span><span class="o">=</span><span class="m">100</span><span class="w">  </span><span class="c1"># dispatch to Celery worker</span>
+</code></pre></div>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>

search/search_index.json CHANGED Viewed

@@ -1 +1 @@

- {"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"],"fields":{"title":{"boost":1000.0},"text":{"boost":1.0},"tags":{"boost":1000000.0}}},"docs":[{"location":"","title":"OpenAIRE-AI-Research-Evaluator","text":"A multi-model LLM-as-Judge pipeline that builds annotation datasets to train Echo-DSRN intent classifiers \u2014 entry for the OpenAIRE AI Hackathon 2026, co-organised by OpenAIRE and Alien Intelligence. The app streams scientific paper metadata from the OpenAIRE Graph API, classifies research intent with a 98M-parameter Echo-DSRN model (fast CPU inference), and enlists multiple LLMs as judges to build a golden consensus dataset."},{"location":"#features","title":"Features","text":"<ul> <li>OpenAIRE Stream \u2014 live publication metadata via the OpenAIRE Graph API</li> <li>Collab Hub \u2014 community annotations with LLM judge consensus</li> <li>Free Text \u2014 ad-hoc classification of custom titles and abstracts</li> <li>Saved History \u2014 local browser storage with optional server-side sync</li> <li>Model Card \u2014 architecture specs, parameter census, and consumption code</li> </ul>"},{"location":"#hackathon-deliverable","title":"Hackathon Deliverable","text":"This project is the artifact submission for the OpenAIRE AI Hackathon 2026. The accompanying story explains the question, journey, insight, and what others can reuse. Source \u00b7 OpenAIRE Graph \u00b7 Echo-DSRN Model"},{"location":"architecture/","title":"Architecture","text":"Component Description Frontend Vue 3 / Vite dashboard \u2014 tabs: Model Card, Collab Hub, OpenAIRE Stream, Free Text, Saved History Backend FastAPI \u2014 inference, arXiv/OpenAIRE proxy, LLM judge orchestration, OAuth Classifier Echo-DSRN-98M (<code>EchoForSequenceClassification</code>) \u2014 fast CPU inference Annotation DB SQLite (<code>data/collaborative.db</code>) \u2014 user classifications and LLM judge labels Worker Celery + Redis \u2014 async classification and LLM-as-Judge queue"},{"location":"architecture/#intent-labels","title":"Intent Labels","text":"Label Meaning <code>Methodology</code> Introduces a new method, model, or algorithm <code>Dataset</code> Introduces or documents a dataset or benchmark <code>Review</code> Surveys or synthesises existing work <code>Applied</code> Applies existing methods to a domain problem <code>Theoretical</code> Mathematical or formal analysis without empirical evaluation"},{"location":"architecture/#data-flow","title":"Data Flow","text":"<ol> <li>OpenAIRE Graph API streams paper metadata (title, abstract, DOI, language)</li> <li>Echo-DSRN-98M classifies research intent in <1ms on CPU</li> <li>Results displayed in the Vue 3 dashboard with probability bars</li> <li>Optional: LLM-as-Judge enlists multiple LLMs to validate Echo's predictions</li> <li>Annotations stored in SQLite, forming a golden consensus dataset</li> </ol>"},{"location":"deployment/","title":"Kubernetes Deployment (k3s)","text":"Deploy the OpenAIRE-AI-Research-Evaluator with Helm on k3s."},{"location":"deployment/#prerequisites","title":"Prerequisites","text":"<ul> <li>k3s cluster running</li> <li><code>helm</code> installed</li> <li>Access to the Docker image <code>ethicalabs/openaire-research-evaluator:latest</code></li> <li>Request access to the gated Echo-DSRN model</li> </ul>"},{"location":"deployment/#1-generate-postgresql-dump","title":"1. Generate PostgreSQL dump","text":"The SQLite golden database must be migrated to PostgreSQL: <pre><code>./scripts/sqlite_to_postgres.sh\n</code></pre> Generates <code>data/collaborative_pg.sql</code> \u2014 ready for PostgreSQL import."},{"location":"deployment/#2-install","title":"2. Install","text":"<pre><code>helm install evaluator ./helm/openaire-research-evaluator \\\n --set secrets.hfToken=hf_... \\\n --set secrets.secretKey=$(openssl rand -hex 32) \\\n --set ingress.host=openaire-eval.local \\\n --set config.llmBaseUrl=http://192.168.1.66:13305/v1\n</code></pre> Or create a <code>values.local.yaml</code>: <pre><code>secrets:\n hfToken: hf_...\n secretKey: abc123...\n hfClientId: ...\n hfClientSecret: ...\ningress:\n host: openaire-eval.local\nconfig:\n llmBaseUrl: http://192.168.1.66:13305/v1\n</code></pre> <pre><code>helm install evaluator ./helm/openaire-research-evaluator -f values.local.yaml\n</code></pre>"},{"location":"deployment/#3-load-golden-data","title":"3. Load golden data","text":"Wait for PostgreSQL to be ready, then import: <pre><code>kubectl cp data/collaborative_pg.sql \\\n $(kubectl get pod -l component=postgres -o jsonpath='{.items[0].metadata.name}'):/tmp/collaborative_pg.sql\n\nkubectl exec -it deployment/evaluator-openaire-research-evaluator-postgres -- \\\n psql -U postgres -d echo_dsrn -f /tmp/collaborative_pg.sql\n\n# Reset auto-increment sequences after import (IDs were explicit in the dump)\nkubectl exec deployment/evaluator-openaire-research-evaluator-postgres -- \\\n psql -U postgres -d echo_dsrn -c \\\n \"SELECT setval('annotations_id_seq', COALESCE((SELECT MAX(id) FROM annotations), 1))\"\n</code></pre>"},{"location":"deployment/#4-access","title":"4. Access","text":"<pre><code>kubectl port-forward svc/evaluator-openaire-research-evaluator-web 7860:7860\n</code></pre> App at <code>http://localhost:7860</code>. With Ingress configured, at <code>http://openaire-eval.local</code>. Local DNS setup: add to <code>/etc/hosts</code>: <pre><code>echo \"127.0.0.1 openaire-eval.local\" | sudo tee -a /etc/hosts\n</code></pre> Then access at <code>http://openaire-eval.local</code> (k3s Traefik listens on port 80)."},{"location":"deployment/#components","title":"Components","text":"Component Service Port Web (FastAPI + Vue) <code><release>-web</code> 7860 PostgreSQL <code><release>-postgres</code> 5432 Redis <code><release>-redis</code> 6379 Worker (Celery) \u2014 (internal) \u2014"},{"location":"deployment/#troubleshooting","title":"Troubleshooting","text":"Check web pod logs for alembic migration status: <pre><code>kubectl logs deployment/evaluator-openaire-research-evaluator-web\n</code></pre>"},{"location":"inference/","title":"Inference","text":"Model: <code>ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF</code> \u2014 gated repo, request access before use."},{"location":"inference/#install","title":"Install","text":"<pre><code>pip install git+https://github.com/ethicalabs-ai/Echo-DSRN.git\n</code></pre>"},{"location":"inference/#classify","title":"Classify","text":"<pre><code>from echo_dsrn import EchoForSequenceClassification\nfrom transformers import AutoTokenizer\n\nmodel = EchoForSequenceClassification.from_pretrained(\n \"ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF\", trust_remote_code=True\n)\ntokenizer = AutoTokenizer.from_pretrained(\n \"ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF\", trust_remote_code=True\n)\n\nlabel, probs = model.classify(\n \"Title: Attention Is All You Need\\n\"\n \"Abstract: We propose a new simple network architecture, the Transformer...\",\n tokenizer=tokenizer,\n)\nprint(label, probs)\n# \u2192 Methodology tensor([0.87, 0.03, 0.02, 0.06, 0.02])\n</code></pre>"},{"location":"inference/#model-specs","title":"Model Specs","text":"Property Value Architecture Echo-DSRN (Recurrent Neural Network) Parameters 98,266,629 (~98M) Layers 8 DSRN blocks Hidden dim 512 Attention heads 4 Vocab size 32,017 tokens Precision bfloat16 Inference Fast on CPU Run <code>curl /api/model/card</code> for live specs from the loaded model."},{"location":"license/","title":"License","text":""},{"location":"license/#code","title":"Code","text":"Apache License 2.0 \u2014 see LICENSE. Copyright \u00a9 2026 ethicalabs.ai"},{"location":"license/#openaire-graph","title":"OpenAIRE Graph","text":"This project uses the OpenAIRE Graph API. All materials created by OpenAIRE are licensed under CC BY 4.0."},{"location":"license/#hackathon-deliverable","title":"Hackathon Deliverable","text":"This project is submitted to the OpenAIRE AI Hackathon 2026. Per hackathon requirements, all submitted materials are available under a CC-BY license."},{"location":"license/#attribution","title":"Attribution","text":"If you reuse this work, please cite: OpenAIRE-AI-Research-Evaluator \u2014 Multi-model LLM-as-Judge pipeline for research intent classification. ethicalabs.ai, 2026. Apache-2.0 / CC-BY 4.0. github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator"},{"location":"llm-as-judge/","title":"LLM-as-Judge","text":"The LLM-as-Judge pipeline evaluates Echo-DSRN predictions against high-capability LLMs and produces a golden consensus dataset in <code>data/collaborative.db</code>."},{"location":"llm-as-judge/#search-queries","title":"Search Queries","text":"arXiv and OpenAIRE sources use keyword queries from <code>assets/topics.txt</code> \u2014 one query per line, blank lines and <code>#</code> comments ignored. Edit this file to target specific research domains. Container path: <code>/app/assets/topics.txt</code>"},{"location":"llm-as-judge/#setup","title":"Setup","text":"Point at a lemonade or llama.cpp server via <code>.env</code>: <pre><code>LLM_BASE_URL=http://192.168.1.40:13305/v1\n</code></pre> Migrate the database: <pre><code>uv run alembic -c backend/alembic.ini upgrade head\n</code></pre>"},{"location":"llm-as-judge/#llm-server-setup","title":"LLM Server Setup","text":"Any OpenAI-compatible server works. Configuration via <code>.env</code>: <pre><code>LLM_BASE_URL=<server-url>\nLLM_API_KEY=not-needed # for local servers that skip auth\n</code></pre>"},{"location":"llm-as-judge/#ollama","title":"Ollama","text":"<pre><code>ollama pull qwen3:4b\nollama serve\n</code></pre> <pre><code>LLM_BASE_URL=http://localhost:11434/v1\n</code></pre>"},{"location":"llm-as-judge/#lm-studio","title":"LM Studio","text":"Start the local server in the UI (Developer \u2192 Local Server). Default port: <pre><code>LLM_BASE_URL=http://localhost:1234/v1\n</code></pre>"},{"location":"llm-as-judge/#lemonade-llamacpp","title":"Lemonade / llama.cpp","text":"<pre><code>lemonade --host 0.0.0.0 load --pinned Qwen3.6-35B-A3B-GGUF\n</code></pre> <pre><code>LLM_BASE_URL=http://192.168.1.40:13305/v1\n</code></pre>"},{"location":"llm-as-judge/#docker-llm-on-host","title":"Docker \u2014 LLM on host","text":"When running the app in Docker and the LLM server on the host, use <code>host.docker.internal</code>: <pre><code>LLM_BASE_URL=http://host.docker.internal:11434/v1\n</code></pre> The <code>docker-compose.yaml</code> already includes <code>extra_hosts: host.docker.internal:host-gateway</code>."},{"location":"llm-as-judge/#batch-runner-sync","title":"Batch Runner \u2014 Sync","text":"Loads models on lemonade, judges papers inline: <pre><code>./scripts/run_judges.sh 100\n</code></pre> Cycles through 11 GGUF models (Qwen3.6, Nemotron, Gemma, GPT-OSS, DeepSeek, GLM) against the catalog source."},{"location":"llm-as-judge/#batch-runner-async","title":"Batch Runner \u2014 Async","text":"Dispatches Celery tasks to the worker, no model loading: <pre><code>./scripts/run_judges.sh 100 --async\n</code></pre>"},{"location":"llm-as-judge/#kubernetes-k3s","title":"Kubernetes (k3s)","text":"Run the judge CLI directly in the deployed web pod: <pre><code>kubectl exec -it deployment/evaluator-openaire-research-evaluator-web -- \\\n env LLM_BASE_URL=http://192.168.1.66:13305/v1 \\\n python backend/judge_cli.py \\\n --source catalog --n 100 \\\n --model gpt-oss-20b-mxfp4-GGUF \\\n --async\n</code></pre> Tasks dispatch to the cluster's Celery worker, using the same Redis and PostgreSQL."},{"location":"llm-as-judge/#single-run","title":"Single Run","text":"<pre><code># Local GGUF via llama.cpp / lemonade\nuv run python backend/judge_cli.py \\\n --source catalog --n 100 \\\n --model Qwen3.6-35B-A3B-GGUF\n\n# Google Gemini (OpenAI-compatible endpoint)\nuv run python backend/judge_cli.py \\\n --source catalog --n 100 \\\n --model gemini-2.5-flash \\\n --llm-url https://generativelanguage.googleapis.com/v1beta/openai/ \\\n --llm-key ${GEMINI_API_KEY}\n</code></pre>"},{"location":"llm-as-judge/#pre-labeled-datasets","title":"Pre-labeled Datasets","text":"Convert chat-format JSONL to flat records, then judge: <pre><code>uv run python scripts/reformat_for_judge.py \\\n ~/.ethicalabs/datasets/research-intent/train.jsonl \\\n ~/.ethicalabs/datasets/research-intent/train_flat.jsonl\n\nuv run python backend/judge_cli.py \\\n --source dataset \\\n --dataset-path ~/.ethicalabs/datasets/research-intent/train_flat.jsonl \\\n --n 100 --model Qwen3.6-35B-A3B-GGUF\n</code></pre>"},{"location":"llm-as-judge/#evaluation","title":"Evaluation","text":"<pre><code>uv run python backend/judge_eval.py # all papers\nuv run python backend/judge_eval.py --paper-source dataset # golden set only\n</code></pre> Computes agreement rates, class-level accuracy, and per-model alignment stats."},{"location":"mcp/","title":"MCP \u2014 Agent Classification Endpoint","text":"Expose Echo-DSRN-98M research intent classification to AI agents via the Model Context Protocol (SSE transport)."},{"location":"mcp/#server-configuration","title":"Server Configuration","text":"Add to your MCP client config (Claude Desktop, Cursor, Continue, etc.): <pre><code>{\n \"mcpServers\": {\n \"echo-dsrn\": {\n \"url\": \"http://localhost:7860/api/mcp/classify\",\n \"transport\": \"sse\",\n \"headers\": {\n \"Authorization\": \"Bearer echo-dsrn-mcp-change-me-in-production\"\n }\n }\n }\n}\n</code></pre> Set <code>MCP_API_KEY</code> in <code>.env</code> to match: <pre><code>MCP_API_KEY=echo-dsrn-mcp-change-me-in-production\n</code></pre>"},{"location":"mcp/#authentication","title":"Authentication","text":"Bearer token via <code>Authorization</code> header. No query-string keys."},{"location":"mcp/#tools-exposed","title":"Tools exposed","text":"Tool Description <code>classify_intent</code> Classify a research paper by title + abstract into one of 5 intents"},{"location":"mcp/#input","title":"Input","text":"<pre><code>{\n \"title\": \"Attention Is All You Need\",\n \"abstract\": \"We propose a new simple network architecture, the Transformer...\"\n}\n</code></pre>"},{"location":"mcp/#output","title":"Output","text":"<pre><code>{\n \"label\": \"Methodology\",\n \"probabilities\": {\n \"Methodology\": 0.87,\n \"Dataset\": 0.03,\n \"Review\": 0.02,\n \"Applied\": 0.06,\n \"Theoretical\": 0.02\n }\n}\n</code></pre>"},{"location":"mcp/#raw-sse-usage","title":"Raw SSE usage","text":"<pre><code>curl -N -H \"Authorization: Bearer echo-dsrn-mcp-change-me-in-production\" \\\n \"http://localhost:7860/api/mcp/classify?title=Attention+Is+All+You+Need&abstract=We+propose+the+Transformer\"\n</code></pre> <pre><code>import requests\n\nresp = requests.get(\n \"http://localhost:7860/api/mcp/classify\",\n params={\"title\": \"Attention Is All You Need\", \"abstract\": \"We propose...\"},\n headers={\"Authorization\": \"Bearer echo-dsrn-mcp-change-me-in-production\"},\n stream=True,\n)\nfor line in resp.iter_lines():\n if line:\n print(line.decode())\n</code></pre>"},{"location":"mcp/#architecture","title":"Architecture","text":"<ol> <li>Request hits SSE endpoint \u2192 Bearer token validated</li> <li>Celery task <code>classify_mcp</code> enqueued to Redis</li> <li>Worker loads Echo-DSRN-98M (baked into Docker image, no HF download)</li> <li>Fast CPU inference \u2192 result streamed via SSE</li> </ol> No papers saved. No database writes. Stateless."},{"location":"oauth/","title":"OAuth \u2014 Hugging Face Login","text":"The Collab Hub uses Hugging Face OAuth for authentication. Users log in with their HF account to vote on annotations and save papers."},{"location":"oauth/#setup","title":"Setup","text":""},{"location":"oauth/#1-create-an-oauth-app","title":"1. Create an OAuth App","text":"Go to Hugging Face OAuth Apps and create a new app: <ul> <li>Name: <code>Echo-DSRN Collab Hub</code></li> <li>Homepage URL: <code>http://localhost:7860</code> (or your production URL)</li> <li>Redirect URL: <code>http://localhost:7860/api/auth/callback/hf</code></li> </ul>"},{"location":"oauth/#2-configure-env","title":"2. Configure .env","text":"<pre><code>APP_BASE_URL=http://localhost:7860\nHF_CLIENT_ID=<your-client-id>\nHF_CLIENT_SECRET=<your-client-secret>\n</code></pre>"},{"location":"oauth/#3-production","title":"3. Production","text":"Replace <code>localhost:7860</code> with your production domain. Both the Homepage URL and Redirect URL must match. <pre><code>APP_BASE_URL=https://echo-dsrn.example.com\n</code></pre> Redirect URL: <code>https://echo-dsrn.example.com/api/auth/callback/hf</code>"},{"location":"oauth/#how-it-works","title":"How it works","text":"<ol> <li>User clicks \"Login with Hugging Face\" in the Collab Hub</li> <li>Redirected to HF authorization page</li> <li>After consent, redirected back to <code>/api/auth/callback/hf</code></li> <li>Backend exchanges code for token, fetches user profile</li> <li>JWT session cookie set \u2014 user is authenticated</li> </ol> User data stored: HF username, name, email, avatar URL, and optionally institution (from profile)."},{"location":"quickstart/","title":"Quick Start","text":""},{"location":"quickstart/#prerequisites","title":"Prerequisites","text":"<ul> <li>Python \u2265 3.12 (tested with 3.12.3)</li> <li>Node.js \u2265 22 (tested with 22.22.3)</li> <li>npm \u2265 10 (tested with 10.9.8)</li> <li>Docker (optional, for containerised deployment)</li> </ul>"},{"location":"quickstart/#docker-compose","title":"Docker Compose","text":"The Echo-DSRN classifier is a gated HF model. Before building: <ol> <li>Request access to the model repo</li> <li>Create an HF access token with read permissions</li> <li>Add it to <code>.env</code>: <code>HF_TOKEN=hf_...</code></li> </ol> <pre><code>make rebuild # builds Docker image with the gated model\ndocker compose up -d\n</code></pre> App serves on <code>http://localhost:7860</code>. Redis, the web server, and a Celery worker start automatically."},{"location":"quickstart/#python","title":"Python","text":""},{"location":"quickstart/#uv","title":"uv","text":"<pre><code>uv run python backend/server.py\n</code></pre>"},{"location":"quickstart/#pip","title":"pip","text":"<pre><code>pip install -r requirements.txt\npython backend/server.py\n</code></pre> Build the frontend first: <code>cd frontend && npm ci && npm run build</code>. App serves on <code>http://localhost:7860</code>."},{"location":"quickstart/#makefile","title":"Makefile","text":"<pre><code>make help # list all targets\nmake up # docker compose up -d\nmake build-docker # build the Docker image\nmake test # run pytest\nmake lint # pre-commit hooks\nmake migrate # alembic upgrade head\nmake judge N=100 # run LLM-as-Judge (sync)\nmake judge-async N=100 # dispatch to Celery worker\n</code></pre>"},{"location":"story/","title":"Hackathon Story","text":""},{"location":"story/#the-question","title":"The Question","text":"How can we turn the OpenAIRE Graph \u2014 a living index of millions of research outputs \u2014 into validated, high-quality training data for AI classifiers, at scale and with minimal human effort? The OpenAIRE Graph aggregates publication metadata across disciplines, languages, and repositories. It is an extraordinary resource for building AI tools that understand research, But raw metadata is noisy: titles are truncated, abstracts vary in quality, and classification labels are absent. To train a classifier that can reliably distinguish a methodology paper from a dataset paper, a review from an applied case study, you need labeled data \u2014 and labeling thousands of papers by hand is slow, expensive, and fragile. Our question was: can we automate this with a feedback loop that combines a fast, lightweight classifier with a panel of LLM judges, using the OpenAIRE Graph as both the data source and the proving ground?"},{"location":"story/#the-journey","title":"The Journey","text":""},{"location":"story/#phase-1-the-classifier","title":"Phase 1 \u2014 The Classifier","text":"We started with Echo-DSRN, a recurrent neural network developed built from scratch by us. Echo-DSRN is small (98M-114M parameters) and runs efficiently on CPU, making it suitable for real-time streaming applications where a GPU is unavailable or impractical. We fine-tuned it on a 5-class research intent taxonomy: <ul> <li>Methodology \u2014 introduces a new method, model, or algorithm</li> <li>Dataset \u2014 introduces or documents a dataset or benchmark</li> <li>Review \u2014 surveys or synthesises existing work</li> <li>Applied \u2014 applies existing methods to a domain problem</li> <li>Theoretical \u2014 mathematical or formal analysis without empirical evaluation</li> </ul> The initial training set was ~6,000 records curated from PubMed, Semantic Scholar, Papers With Code, arXiv, and the OpenAIRE Graph itself \u2014 silver-labeled with keyword heuristics and arXiv category priors. The classifier worked well, but we had no systematic way to measure how well, or to improve it."},{"location":"story/#phase-2-the-app","title":"Phase 2 \u2014 The App","text":"We built a Vue 3 + FastAPI web application that streams paper metadata from the OpenAIRE Graph API in real time. Users see papers appear with Echo-DSRN's predicted intent and a probability distribution across all five classes. They can classify custom text, save results to local history, and browse a Model Card showing the exact architecture, parameter census, and consumption code. But passive classification is only half the story. To improve the model, we needed ground truth \u2014 and that meant building a pipeline for annotation at scale."},{"location":"story/#phase-3-llm-as-judge","title":"Phase 3 \u2014 LLM-as-Judge","text":"We designed an LLM-as-Judge pipeline where multiple high-capability language models evaluate Echo-DSRN's predictions and produce structured verdicts: <ul> <li>Is the predicted label correct?</li> <li>If not, what should it be?</li> <li>Should the paper be flagged (garbled text, non-English, math-only)?</li> <li>What is the judge's confidence?</li> </ul> Each judge receives the paper title, abstract, and Echo's prediction, then returns a JSON verdict following a strict Pydantic schema enforced via OpenAI's <code>response_format</code>. We support any OpenAI-compatible inference server: llama.cpp, vLLM, LM Studio, Google Gemini, and lemonade for local GGUF model orchestration. We assembled a panel of 11 models spanning architectures and parameter counts: <code>Qwen3.6-35B</code>, <code>Qwen3.6-27B</code>, <code>Qwen3.5-35B</code>, <code>Nemotron-3-Nano-30B</code>, <code>Gemma-4-26B</code>, <code>GPT-OSS-20B</code>, <code>GPT-OSS-120B</code>, <code>DeepSeek-Qwen3-8B</code>, <code>GLM-4.7-Flash</code>, <code>Qwen3.6-35B-A3B</code>, and <code>Qwen3.5-4B</code>."},{"location":"story/#phase-4-scale-with-celery","title":"Phase 4 \u2014 Scale with Celery","text":"Running 11 models across thousands of papers synchronously is slow \u2014 each paper needs an LLM API call with its own latency. We added a Celery + Redis worker queue: the CLI dispatches fine-grained per-paper tasks, and workers pull from the queue and process them asynchronously. The same pipeline can run with a single command: <pre><code>./scripts/run_judges.sh 100 --async\n</code></pre> This dispatches 100 papers per model (1,100 total tasks) to the worker pool. The worker handles classification, LLM judging, and database persistence in one atomic unit, with idempotency guarantees \u2014 no paper is judged twice by the same model."},{"location":"story/#phase-5-the-golden-dataset","title":"Phase 5 \u2014 The Golden Dataset","text":"All annotations are stored in a SQLite database (<code>data/collaborative.db</code>). The <code>judge_eval.py</code> script computes: <ul> <li>Agreement rates between judges</li> <li>Per-class accuracy against ground truth (when available)</li> <li>Per-model alignment stats against Echo-DSRN</li> </ul> The result is a golden consensus dataset where each paper has labels from multiple LLMs alongside Echo-DSRN's original prediction. This dataset becomes the training data for the next iteration of the classifier \u2014 closing the feedback loop."},{"location":"story/#the-insight","title":"The Insight","text":"A multi-model LLM-as-Judge pipeline, fed by the OpenAIRE Graph, produces higher-quality training data than any single annotator \u2014 human or machine \u2014 at a fraction of the cost. Three findings stand out: <ol> <li> Diversity matters. Different models disagree on edge cases (applied vs. methodology, dataset vs. methodology), and consensus across architectures produces more robust labels than the strongest single model alone. </li> <li> The OpenAIRE Graph is a live data flywheel. New papers appear daily. A streaming classifier backed by LLM judges can continuously expand the training set without manual curation. </li> <li> CPU-first is viable. Echo-DSRN-98M runs fast enough on CPU to be deployable anywhere \u2014 from a cloud VM to a Raspberry Pi. The heavy lifting (LLM judging) is offloaded to a remote GPU server, but the classification itself is free. </li> </ol>"},{"location":"story/#what-others-can-reuse","title":"What Others Can Reuse","text":"This project is designed to be remixed. Here is what you can take and adapt: Component Reuse scenario LLM-as-Judge pipeline Swap the 5-class taxonomy for your own labels. Point at any OpenAI-compatible server. The judge schema, prompt, and evaluation scripts are label-agnostic. OpenAIRE Graph integration The keyword streaming + multilingual query approach works for any research domain. The fetcher handles both arXiv and OpenAIRE APIs. Async Celery worker pattern Fine-grained per-paper tasks with lazy database imports and idempotency \u2014 copy the pattern for any batch annotation workflow. Dataset preparation scripts <code>prepare_research_intent.py</code> and <code>expand_multilingual_openaire.py</code> build balanced, multilingual training sets from public APIs. Adapt the keyword banks and label taxonomy. Echo-DSRN classifier The model is on Hugging Face (gated \u2014 request access). <code>model.classify(title + abstract, tokenizer)</code> is the entire API. Swap the classification head for your own labels."},{"location":"story/#license","title":"License","text":"Code: Apache 2.0. OpenAIRE Graph: CC BY 4.0. All hackathon materials: CC-BY."}]}

+ {"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"],"fields":{"title":{"boost":1000.0},"text":{"boost":1.0},"tags":{"boost":1000000.0}}},"docs":[{"location":"index.html","title":"OpenAIRE-AI-Research-Evaluator","text":"A multi-model LLM-as-Judge pipeline that builds annotation datasets to train Echo-DSRN intent classifiers \u2014 entry for the OpenAIRE AI Hackathon 2026, co-organised by OpenAIRE and Alien Intelligence. The app streams scientific paper metadata from the OpenAIRE Graph API, classifies research intent with a 98M-parameter Echo-DSRN model (fast CPU inference), and enlists multiple LLMs as judges to build a golden consensus dataset."},{"location":"index.html#features","title":"Features","text":"<ul> <li>OpenAIRE Stream \u2014 live publication metadata via the OpenAIRE Graph API</li> <li>Collab Hub \u2014 community annotations with LLM judge consensus</li> <li>Free Text \u2014 ad-hoc classification of custom titles and abstracts</li> <li>Saved History \u2014 local browser storage with optional server-side sync</li> <li>Model Card \u2014 architecture specs, parameter census, and consumption code</li> </ul>"},{"location":"index.html#hackathon-deliverable","title":"Hackathon Deliverable","text":"This project is the artifact submission for the OpenAIRE AI Hackathon 2026. The accompanying story explains the question, journey, insight, and what others can reuse. Source \u00b7 OpenAIRE Graph \u00b7 Echo-DSRN Model"},{"location":"architecture.html","title":"Architecture","text":"Component Description Frontend Vue 3 / Vite dashboard \u2014 tabs: Model Card, Collab Hub, OpenAIRE Stream, Free Text, Saved History Backend FastAPI \u2014 inference, arXiv/OpenAIRE proxy, LLM judge orchestration, OAuth Classifier Echo-DSRN-98M (<code>EchoForSequenceClassification</code>) \u2014 fast CPU inference Annotation DB SQLite (<code>data/collaborative.db</code>) \u2014 user classifications and LLM judge labels Worker Celery + Redis \u2014 async classification and LLM-as-Judge queue"},{"location":"architecture.html#intent-labels","title":"Intent Labels","text":"Label Meaning <code>Methodology</code> Introduces a new method, model, or algorithm <code>Dataset</code> Introduces or documents a dataset or benchmark <code>Review</code> Surveys or synthesises existing work <code>Applied</code> Applies existing methods to a domain problem <code>Theoretical</code> Mathematical or formal analysis without empirical evaluation"},{"location":"architecture.html#data-flow","title":"Data Flow","text":"<ol> <li>OpenAIRE Graph API streams paper metadata (title, abstract, DOI, language)</li> <li>Echo-DSRN-98M classifies research intent in <1ms on CPU</li> <li>Results displayed in the Vue 3 dashboard with probability bars</li> <li>Optional: LLM-as-Judge enlists multiple LLMs to validate Echo's predictions</li> <li>Annotations stored in SQLite, forming a golden consensus dataset</li> </ol>"},{"location":"deployment.html","title":"Kubernetes Deployment (k3s)","text":"Deploy the OpenAIRE-AI-Research-Evaluator with Helm on k3s."},{"location":"deployment.html#prerequisites","title":"Prerequisites","text":"<ul> <li>k3s cluster running</li> <li><code>helm</code> installed</li> <li>Access to the Docker image <code>ethicalabs/openaire-research-evaluator:latest</code></li> <li>Request access to the gated Echo-DSRN model</li> </ul>"},{"location":"deployment.html#1-generate-postgresql-dump","title":"1. Generate PostgreSQL dump","text":"The SQLite golden database must be migrated to PostgreSQL: <pre><code>./scripts/sqlite_to_postgres.sh\n</code></pre> Generates <code>data/collaborative_pg.sql</code> \u2014 ready for PostgreSQL import."},{"location":"deployment.html#2-install","title":"2. Install","text":"<pre><code>helm install evaluator ./helm/openaire-research-evaluator \\\n --set secrets.hfToken=hf_... \\\n --set secrets.secretKey=$(openssl rand -hex 32) \\\n --set ingress.host=openaire-eval.local \\\n --set config.llmBaseUrl=http://192.168.1.66:13305/v1\n</code></pre> Or create a <code>values.local.yaml</code>: <pre><code>secrets:\n hfToken: hf_...\n secretKey: abc123...\n hfClientId: ...\n hfClientSecret: ...\ningress:\n host: openaire-eval.local\nconfig:\n llmBaseUrl: http://192.168.1.66:13305/v1\n</code></pre> <pre><code>helm install evaluator ./helm/openaire-research-evaluator -f values.local.yaml\n</code></pre>"},{"location":"deployment.html#3-load-golden-data","title":"3. Load golden data","text":"Wait for PostgreSQL to be ready, then import: <pre><code>kubectl cp data/collaborative_pg.sql \\\n $(kubectl get pod -l component=postgres -o jsonpath='{.items[0].metadata.name}'):/tmp/collaborative_pg.sql\n\nkubectl exec -it deployment/evaluator-openaire-research-evaluator-postgres -- \\\n psql -U postgres -d echo_dsrn -f /tmp/collaborative_pg.sql\n\n# Reset auto-increment sequences after import (IDs were explicit in the dump)\nkubectl exec deployment/evaluator-openaire-research-evaluator-postgres -- \\\n psql -U postgres -d echo_dsrn -c \\\n \"SELECT setval('annotations_id_seq', COALESCE((SELECT MAX(id) FROM annotations), 1))\"\n</code></pre>"},{"location":"deployment.html#4-access","title":"4. Access","text":"<pre><code>kubectl port-forward svc/evaluator-openaire-research-evaluator-web 7860:7860\n</code></pre> App at <code>http://localhost:7860</code>. With Ingress configured, at <code>http://openaire-eval.local</code>. Local DNS setup: add to <code>/etc/hosts</code>: <pre><code>echo \"127.0.0.1 openaire-eval.local\" | sudo tee -a /etc/hosts\n</code></pre> Then access at <code>http://openaire-eval.local</code> (k3s Traefik listens on port 80)."},{"location":"deployment.html#components","title":"Components","text":"Component Service Port Web (FastAPI + Vue) <code><release>-web</code> 7860 PostgreSQL <code><release>-postgres</code> 5432 Redis <code><release>-redis</code> 6379 Worker (Celery) \u2014 (internal) \u2014"},{"location":"deployment.html#troubleshooting","title":"Troubleshooting","text":"Check web pod logs for alembic migration status: <pre><code>kubectl logs deployment/evaluator-openaire-research-evaluator-web\n</code></pre>"},{"location":"inference.html","title":"Inference","text":"Model: <code>ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF</code> \u2014 gated repo, request access before use."},{"location":"inference.html#install","title":"Install","text":"<pre><code>pip install git+https://github.com/ethicalabs-ai/Echo-DSRN.git\n</code></pre>"},{"location":"inference.html#classify","title":"Classify","text":"<pre><code>from echo_dsrn import EchoForSequenceClassification\nfrom transformers import AutoTokenizer\n\nmodel = EchoForSequenceClassification.from_pretrained(\n \"ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF\", trust_remote_code=True\n)\ntokenizer = AutoTokenizer.from_pretrained(\n \"ethicalabs/Echo-DSRN-v0.1.3-Research-Intent-CLF\", trust_remote_code=True\n)\n\nlabel, probs = model.classify(\n \"Title: Attention Is All You Need\\n\"\n \"Abstract: We propose a new simple network architecture, the Transformer...\",\n tokenizer=tokenizer,\n)\nprint(label, probs)\n# \u2192 Methodology tensor([0.87, 0.03, 0.02, 0.06, 0.02])\n</code></pre>"},{"location":"inference.html#model-specs","title":"Model Specs","text":"Property Value Architecture Echo-DSRN (Recurrent Neural Network) Parameters 98,266,629 (~98M) Layers 8 DSRN blocks Hidden dim 512 Attention heads 4 Vocab size 32,017 tokens Precision bfloat16 Inference Fast on CPU Run <code>curl /api/model/card</code> for live specs from the loaded model."},{"location":"license.html","title":"License","text":""},{"location":"license.html#code","title":"Code","text":"Apache License 2.0 \u2014 see LICENSE. Copyright \u00a9 2026 ethicalabs.ai"},{"location":"license.html#openaire-graph","title":"OpenAIRE Graph","text":"This project uses the OpenAIRE Graph API. All materials created by OpenAIRE are licensed under CC BY 4.0."},{"location":"license.html#hackathon-deliverable","title":"Hackathon Deliverable","text":"This project is submitted to the OpenAIRE AI Hackathon 2026. Per hackathon requirements, all submitted materials are available under a CC-BY license."},{"location":"license.html#attribution","title":"Attribution","text":"If you reuse this work, please cite: OpenAIRE-AI-Research-Evaluator \u2014 Multi-model LLM-as-Judge pipeline for research intent classification. ethicalabs.ai, 2026. Apache-2.0 / CC-BY 4.0. github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator"},{"location":"llm-as-judge.html","title":"LLM-as-Judge","text":"The LLM-as-Judge pipeline evaluates Echo-DSRN predictions against high-capability LLMs and produces a golden consensus dataset in <code>data/collaborative.db</code>."},{"location":"llm-as-judge.html#search-queries","title":"Search Queries","text":"arXiv and OpenAIRE sources use keyword queries from <code>assets/topics.txt</code> \u2014 one query per line, blank lines and <code>#</code> comments ignored. Edit this file to target specific research domains. Container path: <code>/app/assets/topics.txt</code>"},{"location":"llm-as-judge.html#setup","title":"Setup","text":"Point at a lemonade or llama.cpp server via <code>.env</code>: <pre><code>LLM_BASE_URL=http://192.168.1.40:13305/v1\n</code></pre> Migrate the database: <pre><code>uv run alembic -c backend/alembic.ini upgrade head\n</code></pre>"},{"location":"llm-as-judge.html#llm-server-setup","title":"LLM Server Setup","text":"Any OpenAI-compatible server works. Configuration via <code>.env</code>: <pre><code>LLM_BASE_URL=<server-url>\nLLM_API_KEY=not-needed # for local servers that skip auth\n</code></pre>"},{"location":"llm-as-judge.html#ollama","title":"Ollama","text":"<pre><code>ollama pull qwen3:4b\nollama serve\n</code></pre> <pre><code>LLM_BASE_URL=http://localhost:11434/v1\n</code></pre>"},{"location":"llm-as-judge.html#lm-studio","title":"LM Studio","text":"Start the local server in the UI (Developer \u2192 Local Server). Default port: <pre><code>LLM_BASE_URL=http://localhost:1234/v1\n</code></pre>"},{"location":"llm-as-judge.html#lemonade-llamacpp","title":"Lemonade / llama.cpp","text":"<pre><code>lemonade --host 0.0.0.0 load --pinned Qwen3.6-35B-A3B-GGUF\n</code></pre> <pre><code>LLM_BASE_URL=http://192.168.1.40:13305/v1\n</code></pre>"},{"location":"llm-as-judge.html#docker-llm-on-host","title":"Docker \u2014 LLM on host","text":"When running the app in Docker and the LLM server on the host, use <code>host.docker.internal</code>: <pre><code>LLM_BASE_URL=http://host.docker.internal:11434/v1\n</code></pre> The <code>docker-compose.yaml</code> already includes <code>extra_hosts: host.docker.internal:host-gateway</code>."},{"location":"llm-as-judge.html#batch-runner-sync","title":"Batch Runner \u2014 Sync","text":"Loads models on lemonade, judges papers inline: <pre><code>./scripts/run_judges.sh 100\n</code></pre> Cycles through 11 GGUF models (Qwen3.6, Nemotron, Gemma, GPT-OSS, DeepSeek, GLM) against the catalog source."},{"location":"llm-as-judge.html#batch-runner-async","title":"Batch Runner \u2014 Async","text":"Dispatches Celery tasks to the worker, no model loading: <pre><code>./scripts/run_judges.sh 100 --async\n</code></pre>"},{"location":"llm-as-judge.html#kubernetes-k3s","title":"Kubernetes (k3s)","text":"Run the judge CLI directly in the deployed web pod: <pre><code>kubectl exec -it deployment/evaluator-openaire-research-evaluator-web -- \\\n env LLM_BASE_URL=http://192.168.1.66:13305/v1 \\\n python backend/judge_cli.py \\\n --source catalog --n 100 \\\n --model gpt-oss-20b-mxfp4-GGUF \\\n --async\n</code></pre> Tasks dispatch to the cluster's Celery worker, using the same Redis and PostgreSQL."},{"location":"llm-as-judge.html#single-run","title":"Single Run","text":"<pre><code># Local GGUF via llama.cpp / lemonade\nuv run python backend/judge_cli.py \\\n --source catalog --n 100 \\\n --model Qwen3.6-35B-A3B-GGUF\n\n# Google Gemini (OpenAI-compatible endpoint)\nuv run python backend/judge_cli.py \\\n --source catalog --n 100 \\\n --model gemini-2.5-flash \\\n --llm-url https://generativelanguage.googleapis.com/v1beta/openai/ \\\n --llm-key ${GEMINI_API_KEY}\n</code></pre>"},{"location":"llm-as-judge.html#pre-labeled-datasets","title":"Pre-labeled Datasets","text":"Convert chat-format JSONL to flat records, then judge: <pre><code>uv run python scripts/reformat_for_judge.py \\\n ~/.ethicalabs/datasets/research-intent/train.jsonl \\\n ~/.ethicalabs/datasets/research-intent/train_flat.jsonl\n\nuv run python backend/judge_cli.py \\\n --source dataset \\\n --dataset-path ~/.ethicalabs/datasets/research-intent/train_flat.jsonl \\\n --n 100 --model Qwen3.6-35B-A3B-GGUF\n</code></pre>"},{"location":"llm-as-judge.html#evaluation","title":"Evaluation","text":"<pre><code>uv run python backend/judge_eval.py # all papers\nuv run python backend/judge_eval.py --paper-source dataset # golden set only\n</code></pre> Computes agreement rates, class-level accuracy, and per-model alignment stats."},{"location":"mcp.html","title":"MCP \u2014 Agent Classification Endpoint","text":"Expose Echo-DSRN-98M research intent classification to AI agents via the Model Context Protocol (SSE transport)."},{"location":"mcp.html#server-configuration","title":"Server Configuration","text":"Add to your MCP client config (Claude Desktop, Cursor, Continue, etc.): <pre><code>{\n \"mcpServers\": {\n \"echo-dsrn\": {\n \"url\": \"http://localhost:7860/api/mcp/classify\",\n \"transport\": \"sse\",\n \"headers\": {\n \"Authorization\": \"Bearer echo-dsrn-mcp-change-me-in-production\"\n }\n }\n }\n}\n</code></pre> Set <code>MCP_API_KEY</code> in <code>.env</code> to match: <pre><code>MCP_API_KEY=echo-dsrn-mcp-change-me-in-production\n</code></pre>"},{"location":"mcp.html#authentication","title":"Authentication","text":"Bearer token via <code>Authorization</code> header. No query-string keys."},{"location":"mcp.html#tools-exposed","title":"Tools exposed","text":"Tool Description <code>classify_intent</code> Classify a research paper by title + abstract into one of 5 intents"},{"location":"mcp.html#input","title":"Input","text":"<pre><code>{\n \"title\": \"Attention Is All You Need\",\n \"abstract\": \"We propose a new simple network architecture, the Transformer...\"\n}\n</code></pre>"},{"location":"mcp.html#output","title":"Output","text":"<pre><code>{\n \"label\": \"Methodology\",\n \"probabilities\": {\n \"Methodology\": 0.87,\n \"Dataset\": 0.03,\n \"Review\": 0.02,\n \"Applied\": 0.06,\n \"Theoretical\": 0.02\n }\n}\n</code></pre>"},{"location":"mcp.html#raw-sse-usage","title":"Raw SSE usage","text":"<pre><code>curl -N -H \"Authorization: Bearer echo-dsrn-mcp-change-me-in-production\" \\\n \"http://localhost:7860/api/mcp/classify?title=Attention+Is+All+You+Need&abstract=We+propose+the+Transformer\"\n</code></pre> <pre><code>import requests\n\nresp = requests.get(\n \"http://localhost:7860/api/mcp/classify\",\n params={\"title\": \"Attention Is All You Need\", \"abstract\": \"We propose...\"},\n headers={\"Authorization\": \"Bearer echo-dsrn-mcp-change-me-in-production\"},\n stream=True,\n)\nfor line in resp.iter_lines():\n if line:\n print(line.decode())\n</code></pre>"},{"location":"mcp.html#architecture","title":"Architecture","text":"<ol> <li>Request hits SSE endpoint \u2192 Bearer token validated</li> <li>Celery task <code>classify_mcp</code> enqueued to Redis</li> <li>Worker loads Echo-DSRN-98M (baked into Docker image, no HF download)</li> <li>Fast CPU inference \u2192 result streamed via SSE</li> </ol> No papers saved. No database writes. Stateless."},{"location":"oauth.html","title":"OAuth \u2014 Hugging Face Login","text":"The Collab Hub uses Hugging Face OAuth for authentication. Users log in with their HF account to vote on annotations and save papers."},{"location":"oauth.html#setup","title":"Setup","text":""},{"location":"oauth.html#1-create-an-oauth-app","title":"1. Create an OAuth App","text":"Go to Hugging Face OAuth Apps and create a new app: <ul> <li>Name: <code>Echo-DSRN Collab Hub</code></li> <li>Homepage URL: <code>http://localhost:7860</code> (or your production URL)</li> <li>Redirect URL: <code>http://localhost:7860/api/auth/callback/hf</code></li> </ul>"},{"location":"oauth.html#2-configure-env","title":"2. Configure .env","text":"<pre><code>APP_BASE_URL=http://localhost:7860\nHF_CLIENT_ID=<your-client-id>\nHF_CLIENT_SECRET=<your-client-secret>\n</code></pre>"},{"location":"oauth.html#3-production","title":"3. Production","text":"Replace <code>localhost:7860</code> with your production domain. Both the Homepage URL and Redirect URL must match. <pre><code>APP_BASE_URL=https://echo-dsrn.example.com\n</code></pre> Redirect URL: <code>https://echo-dsrn.example.com/api/auth/callback/hf</code>"},{"location":"oauth.html#how-it-works","title":"How it works","text":"<ol> <li>User clicks \"Login with Hugging Face\" in the Collab Hub</li> <li>Redirected to HF authorization page</li> <li>After consent, redirected back to <code>/api/auth/callback/hf</code></li> <li>Backend exchanges code for token, fetches user profile</li> <li>JWT session cookie set \u2014 user is authenticated</li> </ol> User data stored: HF username, name, email, avatar URL, and optionally institution (from profile)."},{"location":"quickstart.html","title":"Quick Start","text":""},{"location":"quickstart.html#prerequisites","title":"Prerequisites","text":"<ul> <li>Python \u2265 3.12 (tested with 3.12.3)</li> <li>Node.js \u2265 22 (tested with 22.22.3)</li> <li>npm \u2265 10 (tested with 10.9.8)</li> <li>Docker (optional, for containerised deployment)</li> </ul>"},{"location":"quickstart.html#docker-compose","title":"Docker Compose","text":"The Echo-DSRN classifier is a gated HF model. Before building: <ol> <li>Request access to the model repo</li> <li>Create an HF access token with read permissions</li> <li>Add it to <code>.env</code>: <code>HF_TOKEN=hf_...</code></li> </ol> <pre><code>make rebuild # builds Docker image with the gated model\ndocker compose up -d\n</code></pre> App serves on <code>http://localhost:7860</code>. Redis, the web server, and a Celery worker start automatically."},{"location":"quickstart.html#python","title":"Python","text":""},{"location":"quickstart.html#uv","title":"uv","text":"<pre><code>uv run python backend/server.py\n</code></pre>"},{"location":"quickstart.html#pip","title":"pip","text":"<pre><code>pip install -r requirements.txt\npython backend/server.py\n</code></pre> Build the frontend first: <code>cd frontend && npm ci && npm run build</code>. App serves on <code>http://localhost:7860</code>."},{"location":"quickstart.html#makefile","title":"Makefile","text":"<pre><code>make help # list all targets\nmake up # docker compose up -d\nmake build-docker # build the Docker image\nmake test # run pytest\nmake lint # pre-commit hooks\nmake migrate # alembic upgrade head\nmake judge N=100 # run LLM-as-Judge (sync)\nmake judge-async N=100 # dispatch to Celery worker\n</code></pre>"},{"location":"story.html","title":"Hackathon Story","text":""},{"location":"story.html#the-question","title":"The Question","text":"How can we turn the OpenAIRE Graph \u2014 a living index of millions of research outputs \u2014 into validated, high-quality training data for AI classifiers, at scale and with minimal human effort? The OpenAIRE Graph aggregates publication metadata across disciplines, languages, and repositories. It is an extraordinary resource for building AI tools that understand research, But raw metadata is noisy: titles are truncated, abstracts vary in quality, and classification labels are absent. To train a classifier that can reliably distinguish a methodology paper from a dataset paper, a review from an applied case study, you need labeled data \u2014 and labeling thousands of papers by hand is slow, expensive, and fragile. Our question was: can we automate this with a feedback loop that combines a fast, lightweight classifier with a panel of LLM judges, using the OpenAIRE Graph as both the data source and the proving ground?"},{"location":"story.html#the-journey","title":"The Journey","text":""},{"location":"story.html#phase-1-the-classifier","title":"Phase 1 \u2014 The Classifier","text":"We started with Echo-DSRN, a recurrent neural network developed built from scratch by us. Echo-DSRN is small (98M-114M parameters) and runs efficiently on CPU, making it suitable for real-time streaming applications where a GPU is unavailable or impractical. We fine-tuned it on a 5-class research intent taxonomy: <ul> <li>Methodology \u2014 introduces a new method, model, or algorithm</li> <li>Dataset \u2014 introduces or documents a dataset or benchmark</li> <li>Review \u2014 surveys or synthesises existing work</li> <li>Applied \u2014 applies existing methods to a domain problem</li> <li>Theoretical \u2014 mathematical or formal analysis without empirical evaluation</li> </ul> The initial training set was ~6,000 records curated from PubMed, Semantic Scholar, Papers With Code, arXiv, and the OpenAIRE Graph itself \u2014 silver-labeled with keyword heuristics and arXiv category priors. The classifier worked well, but we had no systematic way to measure how well, or to improve it."},{"location":"story.html#phase-2-the-app","title":"Phase 2 \u2014 The App","text":"We built a Vue 3 + FastAPI web application that streams paper metadata from the OpenAIRE Graph API in real time. Users see papers appear with Echo-DSRN's predicted intent and a probability distribution across all five classes. They can classify custom text, save results to local history, and browse a Model Card showing the exact architecture, parameter census, and consumption code. But passive classification is only half the story. To improve the model, we needed ground truth \u2014 and that meant building a pipeline for annotation at scale."},{"location":"story.html#phase-3-llm-as-judge","title":"Phase 3 \u2014 LLM-as-Judge","text":"We designed an LLM-as-Judge pipeline where multiple high-capability language models evaluate Echo-DSRN's predictions and produce structured verdicts: <ul> <li>Is the predicted label correct?</li> <li>If not, what should it be?</li> <li>Should the paper be flagged (garbled text, non-English, math-only)?</li> <li>What is the judge's confidence?</li> </ul> Each judge receives the paper title, abstract, and Echo's prediction, then returns a JSON verdict following a strict Pydantic schema enforced via OpenAI's <code>response_format</code>. We support any OpenAI-compatible inference server: llama.cpp, vLLM, LM Studio, Google Gemini, and lemonade for local GGUF model orchestration. We assembled a panel of 11 models spanning architectures and parameter counts: <code>Qwen3.6-35B</code>, <code>Qwen3.6-27B</code>, <code>Qwen3.5-35B</code>, <code>Nemotron-3-Nano-30B</code>, <code>Gemma-4-26B</code>, <code>GPT-OSS-20B</code>, <code>GPT-OSS-120B</code>, <code>DeepSeek-Qwen3-8B</code>, <code>GLM-4.7-Flash</code>, <code>Qwen3.6-35B-A3B</code>, and <code>Qwen3.5-4B</code>."},{"location":"story.html#phase-4-scale-with-celery","title":"Phase 4 \u2014 Scale with Celery","text":"Running 11 models across thousands of papers synchronously is slow \u2014 each paper needs an LLM API call with its own latency. We added a Celery + Redis worker queue: the CLI dispatches fine-grained per-paper tasks, and workers pull from the queue and process them asynchronously. The same pipeline can run with a single command: <pre><code>./scripts/run_judges.sh 100 --async\n</code></pre> This dispatches 100 papers per model (1,100 total tasks) to the worker pool. The worker handles classification, LLM judging, and database persistence in one atomic unit, with idempotency guarantees \u2014 no paper is judged twice by the same model."},{"location":"story.html#phase-5-the-golden-dataset","title":"Phase 5 \u2014 The Golden Dataset","text":"All annotations are stored in a SQLite database (<code>data/collaborative.db</code>). The <code>judge_eval.py</code> script computes: <ul> <li>Agreement rates between judges</li> <li>Per-class accuracy against ground truth (when available)</li> <li>Per-model alignment stats against Echo-DSRN</li> </ul> The result is a golden consensus dataset where each paper has labels from multiple LLMs alongside Echo-DSRN's original prediction. This dataset becomes the training data for the next iteration of the classifier \u2014 closing the feedback loop."},{"location":"story.html#the-insight","title":"The Insight","text":"A multi-model LLM-as-Judge pipeline, fed by the OpenAIRE Graph, produces higher-quality training data than any single annotator \u2014 human or machine \u2014 at a fraction of the cost. Three findings stand out: <ol> <li> Diversity matters. Different models disagree on edge cases (applied vs. methodology, dataset vs. methodology), and consensus across architectures produces more robust labels than the strongest single model alone. </li> <li> The OpenAIRE Graph is a live data flywheel. New papers appear daily. A streaming classifier backed by LLM judges can continuously expand the training set without manual curation. </li> <li> CPU-first is viable. Echo-DSRN-98M runs fast enough on CPU to be deployable anywhere \u2014 from a cloud VM to a Raspberry Pi. The heavy lifting (LLM judging) is offloaded to a remote GPU server, but the classification itself is free. </li> </ol>"},{"location":"story.html#what-others-can-reuse","title":"What Others Can Reuse","text":"This project is designed to be remixed. Here is what you can take and adapt: Component Reuse scenario LLM-as-Judge pipeline Swap the 5-class taxonomy for your own labels. Point at any OpenAI-compatible server. The judge schema, prompt, and evaluation scripts are label-agnostic. OpenAIRE Graph integration The keyword streaming + multilingual query approach works for any research domain. The fetcher handles both arXiv and OpenAIRE APIs. Async Celery worker pattern Fine-grained per-paper tasks with lazy database imports and idempotency \u2014 copy the pattern for any batch annotation workflow. Dataset preparation scripts <code>prepare_research_intent.py</code> and <code>expand_multilingual_openaire.py</code> build balanced, multilingual training sets from public APIs. Adapt the keyword banks and label taxonomy. Echo-DSRN classifier The model is on Hugging Face (gated \u2014 request access). <code>model.classify(title + abstract, tokenizer)</code> is the entire API. Swap the classification head for your own labels."},{"location":"story.html#license","title":"License","text":"Code: Apache 2.0. OpenAIRE Graph: CC BY 4.0. All hackathon materials: CC-BY."}]}

story.html ADDED Viewed

	@@ -0,0 +1,982 @@

+<!doctype html>
+<html lang="en" class="no-js">
+  <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+        <meta name="description" content="Multi-model LLM-as-Judge pipeline for the OpenAIRE AI Hackathon 2026">
+        <link rel="prev" href="inference.html">
+        <link rel="next" href="deployment.html">
+      <link rel="icon" href="assets/images/favicon.png">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
+        <title>Hackathon Story - OpenAIRE-AI-Research-Evaluator</title>
+      <link rel="stylesheet" href="assets/stylesheets/main.484c7ddc.min.css">
+        <link rel="stylesheet" href="assets/stylesheets/palette.ab4e12ef.min.css">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
+        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
+    <script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
+  </head>
+    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
+    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
+    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
+    <label class="md-overlay" for="__drawer"></label>
+    <div data-md-component="skip">
+        <a href="#hackathon-story" class="md-skip">
+          Skip to content
+        </a>
+    </div>
+    <div data-md-component="announce">
+    </div>
+<header class="md-header md-header--shadow" data-md-component="header">
+  <nav class="md-header__inner md-grid" aria-label="Header">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-header__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    <label class="md-header__button md-icon" for="__drawer">
+      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
+    </label>
+    <div class="md-header__title" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            OpenAIRE-AI-Research-Evaluator
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+              Hackathon Story
+          </span>
+        </div>
+      </div>
+    </div>
+        <label class="md-header__button md-icon" for="__search">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        </label>
+        <div class="md-search" data-md-component="search" role="dialog">
+  <label class="md-search__overlay" for="__search"></label>
+  <div class="md-search__inner" role="search">
+    <form class="md-search__form" name="search">
+      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
+      <label class="md-search__icon md-icon" for="__search">
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
+        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
+      </label>
+      <nav class="md-search__options" aria-label="Search">
+        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
+          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
+        </button>
+      </nav>
+    </form>
+    <div class="md-search__output">
+      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
+        <div class="md-search-result" data-md-component="search-result">
+          <div class="md-search-result__meta">
+            Initializing search
+          </div>
+          <ol class="md-search-result__list" role="presentation"></ol>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+      <div class="md-header__source">
+        <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+      </div>
+  </nav>
+</header>
+    <div class="md-container" data-md-component="container">
+      <main class="md-main" data-md-component="main">
+        <div class="md-main__inner md-grid">
+              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
+  <label class="md-nav__title" for="__drawer">
+    <a href="index.html" title="OpenAIRE-AI-Research-Evaluator" class="md-nav__button md-logo" aria-label="OpenAIRE-AI-Research-Evaluator" data-md-component="logo">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
+    </a>
+    OpenAIRE-AI-Research-Evaluator
+  </label>
+    <div class="md-nav__source">
+      <a href="https://github.com/ethicalabs-ai/OpenAIRE-AI-Research-Evaluator" title="Go to repository" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
+  </div>
+  <div class="md-source__repository">
+    GitHub
+  </div>
+</a>
+    </div>
+  <ul class="md-nav__list" data-md-scrollfix>
+    <li class="md-nav__item">
+      <a href="index.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Home
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="quickstart.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Quick Start
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="architecture.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Architecture
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="llm-as-judge.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    LLM-as-Judge
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="mcp.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    MCP
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="oauth.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    OAuth
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="inference.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Inference
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item md-nav__item--active">
+      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
+        <label class="md-nav__link md-nav__link--active" for="__toc">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+          <span class="md-nav__icon md-icon"></span>
+        </label>
+      <a href="story.html" class="md-nav__link md-nav__link--active">
+  <span class="md-ellipsis">
+    Hackathon Story
+  </span>
+      </a>
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#the-question" class="md-nav__link">
+    <span class="md-ellipsis">
+        The Question
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#the-journey" class="md-nav__link">
+    <span class="md-ellipsis">
+        The Journey
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="The Journey">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#phase-1-the-classifier" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 1 — The Classifier
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#phase-2-the-app" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 2 — The App
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#phase-3-llm-as-judge" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 3 — LLM-as-Judge
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#phase-4-scale-with-celery" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 4 — Scale with Celery
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#phase-5-the-golden-dataset" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 5 — The Golden Dataset
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#the-insight" class="md-nav__link">
+    <span class="md-ellipsis">
+        The Insight
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#what-others-can-reuse" class="md-nav__link">
+    <span class="md-ellipsis">
+        What Others Can Reuse
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#license" class="md-nav__link">
+    <span class="md-ellipsis">
+        License
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+    </li>
+    <li class="md-nav__item">
+      <a href="deployment.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    Deployment
+  </span>
+      </a>
+    </li>
+    <li class="md-nav__item">
+      <a href="license.html" class="md-nav__link">
+  <span class="md-ellipsis">
+    License
+  </span>
+      </a>
+    </li>
+  </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
+                <div class="md-sidebar__scrollwrap">
+                  <div class="md-sidebar__inner">
+<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
+    <label class="md-nav__title" for="__toc">
+      <span class="md-nav__icon md-icon"></span>
+      Table of contents
+    </label>
+    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
+        <li class="md-nav__item">
+  <a href="#the-question" class="md-nav__link">
+    <span class="md-ellipsis">
+        The Question
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#the-journey" class="md-nav__link">
+    <span class="md-ellipsis">
+        The Journey
+    </span>
+  </a>
+    <nav class="md-nav" aria-label="The Journey">
+      <ul class="md-nav__list">
+          <li class="md-nav__item">
+  <a href="#phase-1-the-classifier" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 1 — The Classifier
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#phase-2-the-app" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 2 — The App
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#phase-3-llm-as-judge" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 3 — LLM-as-Judge
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#phase-4-scale-with-celery" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 4 — Scale with Celery
+    </span>
+  </a>
+</li>
+          <li class="md-nav__item">
+  <a href="#phase-5-the-golden-dataset" class="md-nav__link">
+    <span class="md-ellipsis">
+        Phase 5 — The Golden Dataset
+    </span>
+  </a>
+</li>
+      </ul>
+    </nav>
+</li>
+        <li class="md-nav__item">
+  <a href="#the-insight" class="md-nav__link">
+    <span class="md-ellipsis">
+        The Insight
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#what-others-can-reuse" class="md-nav__link">
+    <span class="md-ellipsis">
+        What Others Can Reuse
+    </span>
+  </a>
+</li>
+        <li class="md-nav__item">
+  <a href="#license" class="md-nav__link">
+    <span class="md-ellipsis">
+        License
+    </span>
+  </a>
+</li>
+    </ul>
+</nav>
+                  </div>
+                </div>
+              </div>
+            <div class="md-content" data-md-component="content">
+              <article class="md-content__inner md-typeset">
+<h1 id="hackathon-story">Hackathon Story</h1>
+<h2 id="the-question">The Question</h2>
+<blockquote>
+<p>How can we turn the OpenAIRE Graph — a living index of millions of research outputs — into validated, high-quality training data for AI classifiers, at scale and with minimal human effort?</p>
+</blockquote>
+<p>The OpenAIRE Graph aggregates publication metadata across disciplines, languages, and repositories.</p>
+<p>It is an extraordinary resource for building AI tools that understand research, But raw metadata is noisy: titles are truncated, abstracts vary in quality, and classification labels are absent.</p>
+<p>To train a classifier that can reliably distinguish a methodology paper from a dataset paper, a review from an applied case study, you need <strong>labeled data</strong> — and labeling thousands of papers by hand is slow, expensive, and fragile.</p>
+<p>Our question was: can we automate this with a feedback loop that combines a fast, lightweight classifier with a panel of LLM judges, using the OpenAIRE Graph as both the data source and the proving ground?</p>
+<hr />
+<h2 id="the-journey">The Journey</h2>
+<h3 id="phase-1-the-classifier">Phase 1 — The Classifier</h3>
+<p>We started with <a href="https://www.ethicalabs.ai/research/echo-dsrn/"><strong>Echo-DSRN</strong></a>, a recurrent neural network developed built from scratch by us.</p>
+<p>Echo-DSRN is small (98M-114M parameters) and runs efficiently on CPU, making it suitable for real-time streaming applications where a GPU is unavailable or impractical.</p>
+<p>We fine-tuned it on a 5-class research intent taxonomy:</p>
+<ul>
+<li><strong>Methodology</strong> — introduces a new method, model, or algorithm</li>
+<li><strong>Dataset</strong> — introduces or documents a dataset or benchmark</li>
+<li><strong>Review</strong> — surveys or synthesises existing work</li>
+<li><strong>Applied</strong> — applies existing methods to a domain problem</li>
+<li><strong>Theoretical</strong> — mathematical or formal analysis without empirical evaluation</li>
+</ul>
+<p>The initial training set was ~6,000 records curated from PubMed, Semantic Scholar, Papers With Code, arXiv, and the OpenAIRE Graph itself — silver-labeled with keyword heuristics and arXiv category priors.</p>
+<p>The classifier worked well, but we had no systematic way to measure how well, or to improve it.</p>
+<h3 id="phase-2-the-app">Phase 2 — The App</h3>
+<p>We built a Vue 3 + FastAPI web application that streams paper metadata from the OpenAIRE Graph API in real time.</p>
+<p>Users see papers appear with Echo-DSRN's predicted intent and a probability distribution across all five classes.</p>
+<p>They can classify custom text, save results to local history, and browse a Model Card showing the exact architecture, parameter census, and consumption code.</p>
+<p>But passive classification is only half the story.</p>
+<p>To improve the model, we needed ground truth — and that meant building a pipeline for annotation at scale.</p>
+<h3 id="phase-3-llm-as-judge">Phase 3 — LLM-as-Judge</h3>
+<p>We designed an LLM-as-Judge pipeline where multiple high-capability language models evaluate Echo-DSRN's predictions and produce structured verdicts:</p>
+<ul>
+<li>Is the predicted label correct?</li>
+<li>If not, what should it be?</li>
+<li>Should the paper be flagged (garbled text, non-English, math-only)?</li>
+<li>What is the judge's confidence?</li>
+</ul>
+<p>Each judge receives the paper title, abstract, and Echo's prediction, then returns a JSON verdict following a strict Pydantic schema enforced via OpenAI's <code>response_format</code>.</p>
+<p>We support any OpenAI-compatible inference server: llama.cpp, vLLM, LM Studio, Google Gemini, and lemonade for local GGUF model orchestration.</p>
+<p>We assembled a panel of 11 models spanning architectures and parameter counts: <code>Qwen3.6-35B</code>, <code>Qwen3.6-27B</code>, <code>Qwen3.5-35B</code>, <code>Nemotron-3-Nano-30B</code>, <code>Gemma-4-26B</code>, <code>GPT-OSS-20B</code>, <code>GPT-OSS-120B</code>, <code>DeepSeek-Qwen3-8B</code>, <code>GLM-4.7-Flash</code>, <code>Qwen3.6-35B-A3B</code>, and <code>Qwen3.5-4B</code>.</p>
+<h3 id="phase-4-scale-with-celery">Phase 4 — Scale with Celery</h3>
+<p>Running 11 models across thousands of papers synchronously is slow — each paper needs an LLM API call with its own latency.</p>
+<p>We added a Celery + Redis worker queue: the CLI dispatches fine-grained per-paper tasks, and workers pull from the queue and process them asynchronously. The same pipeline can run with a single command:</p>
+<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>./scripts/run_judges.sh<span class="w"> </span><span class="m">100</span><span class="w"> </span>--async
+</code></pre></div>
+<p>This dispatches 100 papers per model (1,100 total tasks) to the worker pool. The worker handles classification, LLM judging, and database persistence in one atomic unit, with idempotency guarantees — no paper is judged twice by the same model.</p>
+<h3 id="phase-5-the-golden-dataset">Phase 5 — The Golden Dataset</h3>
+<p>All annotations are stored in a SQLite database (<code>data/collaborative.db</code>). The <code>judge_eval.py</code> script computes:</p>
+<ul>
+<li>Agreement rates between judges</li>
+<li>Per-class accuracy against ground truth (when available)</li>
+<li>Per-model alignment stats against Echo-DSRN</li>
+</ul>
+<p>The result is a <strong>golden consensus dataset</strong> where each paper has labels from multiple LLMs alongside Echo-DSRN's original prediction. This dataset becomes the training data for the next iteration of the classifier — closing the feedback loop.</p>
+<hr />
+<h2 id="the-insight">The Insight</h2>
+<p><strong>A multi-model LLM-as-Judge pipeline, fed by the OpenAIRE Graph, produces higher-quality training data than any single annotator — human or machine — at a fraction of the cost.</strong></p>
+<p>Three findings stand out:</p>
+<ol>
+<li>
+<p><strong>Diversity matters.</strong> Different models disagree on edge cases (applied vs. methodology, dataset vs. methodology), and consensus across architectures produces more robust labels than the strongest single model alone.</p>
+</li>
+<li>
+<p><strong>The OpenAIRE Graph is a live data flywheel.</strong> New papers appear daily. A streaming classifier backed by LLM judges can continuously expand the training set without manual curation.</p>
+</li>
+<li>
+<p><strong>CPU-first is viable.</strong> Echo-DSRN-98M runs fast enough on CPU to be deployable anywhere — from a cloud VM to a Raspberry Pi. The heavy lifting (LLM judging) is offloaded to a remote GPU server, but the classification itself is free.</p>
+</li>
+</ol>
+<hr />
+<h2 id="what-others-can-reuse">What Others Can Reuse</h2>
+<p>This project is designed to be remixed. Here is what you can take and adapt:</p>
+<table>
+<thead>
+<tr>
+<th>Component</th>
+<th>Reuse scenario</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong>LLM-as-Judge pipeline</strong></td>
+<td>Swap the 5-class taxonomy for your own labels. Point at any OpenAI-compatible server. The judge schema, prompt, and evaluation scripts are label-agnostic.</td>
+</tr>
+<tr>
+<td><strong>OpenAIRE Graph integration</strong></td>
+<td>The keyword streaming + multilingual query approach works for any research domain. The fetcher handles both arXiv and OpenAIRE APIs.</td>
+</tr>
+<tr>
+<td><strong>Async Celery worker pattern</strong></td>
+<td>Fine-grained per-paper tasks with lazy database imports and idempotency — copy the pattern for any batch annotation workflow.</td>
+</tr>
+<tr>
+<td><strong>Dataset preparation scripts</strong></td>
+<td><code>prepare_research_intent.py</code> and <code>expand_multilingual_openaire.py</code> build balanced, multilingual training sets from public APIs. Adapt the keyword banks and label taxonomy.</td>
+</tr>
+<tr>
+<td><strong>Echo-DSRN classifier</strong></td>
+<td>The model is on Hugging Face (gated — request access). <code>model.classify(title + abstract, tokenizer)</code> is the entire API. Swap the classification head for your own labels.</td>
+</tr>
+</tbody>
+</table>
+<h2 id="license">License</h2>
+<p>Code: Apache 2.0. OpenAIRE Graph: CC BY 4.0. All hackathon materials: CC-BY.</p>
+              </article>
+            </div>
+<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
+        </div>
+      </main>
+        <footer class="md-footer">
+  <div class="md-footer-meta md-typeset">
+    <div class="md-footer-meta__inner md-grid">
+      <div class="md-copyright">
+    Made with
+    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
+      Material for MkDocs
+    </a>
+</div>
+    </div>
+  </div>
+</footer>
+    </div>
+    <div class="md-dialog" data-md-component="dialog">
+      <div class="md-dialog__inner md-typeset"></div>
+    </div>
+      <script id="__config" type="application/json">{"annotate": null, "base": ".", "features": [], "search": "assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
+      <script src="assets/javascripts/bundle.79ae519e.min.js"></script>
+  </body>
+</html>