init

app.py

@@ -0,0 +1,348 @@
+import gradio as gr
+from ai4data import extract_from_text
+DATA_MODEL_ID = "rafmacalaba/datause-extraction-v3-finetuned"
+DATASET_COLORS = {
+    "named": "#ff6b6b",
+    "unnamed": "#51cf66",
+    "vague": "#fcc419",
+    "invalid": "#868e96"
+}
+
+RELATION_COLORS = {
+    "acronym": "#4dabf7",                 # REL_ACR
+    "author": "#f06595",                  # REL_AUT
+    "data description": "#e599f7",        # REL_DSC
+    "data geography": "#20c997",          # REL_GEO
+    "data type": "#339af0",               # REL_TYP
+    "publication year": "#f783ac",        # REL_PY
+    "publisher": "#4dabf7",               # REL_PUB
+    "reference population": "#8e44ad",    # REL_POP
+    "reference year": "#ffd43b",          # REL_RY
+    # Removed on purpose — we are filtering these out
+    # "usage context": "#e8590c"
+}
+
+RELATION_DISPLAY = {
+    "data geography": "geography",
+    "data description": "description",
+}
+
+
+def overlaps(a, b):
+    # spans overlap if they share any character position
+    return not (a["end"] <= b["start"] or a["start"] >= b["end"])
+
+
+# def collect_spans(res):
+    base = res[0]["text"]
+
+    dataset_spans = []
+    relation_spans = []
+    usage_context_map = {}
+    # --- Collect DATASETS (always keep) ---
+    for item in res:
+        ds = item.get("datasets")
+        if ds:
+            dataset_spans.append({
+                "start": ds["start"],
+                "end": ds["end"],
+                "label": ds["label"].lower(),
+                "kind": "dataset",
+                "text": base[ds["start"]:ds["end"]]
+            })
+
+    # --- Collect RELATIONS (but skip usage context) ---
+    for item in res:
+        for r in item.get("relations", []):
+            if r["relation"].lower() == "usage context":   # remove usage context entirely
+                continue
+            if r.get("score", 0) < 0.6:
+                continue  # ignore low-confidence relations
+            relation_spans.append({
+                "start": r["start"],
+                "end": r["end"],
+                "label": r["relation"].lower(),
+                "kind": "relation",
+                "text": base[r["start"]:r["end"]],
+                "source": r['source']
+            })
+
+    # --- Priority Rule: Dataset spans win over relation overlaps ---
+    def overlaps(a, b):
+        return not (a["end"] <= b["start"] or a["start"] >= b["end"])
+
+    dataset_spans = sorted(dataset_spans, key=lambda s: s["start"])
+
+    filtered_relations = []
+    for rel in relation_spans:
+        if any(overlaps(rel, ds) for ds in dataset_spans):
+            continue
+        filtered_relations.append(rel)
+
+    # --- Deduplicate spans ---
+    uniq = {}
+    for span in dataset_spans + filtered_relations:
+        key = (span["start"], span["end"], span["label"], span["kind"])
+        uniq[key] = span  # last wins (order doesn't matter because datasets sorted first)
+
+    spans = list(uniq.values())
+    spans = sorted(spans, key=lambda s: s["start"])
+
+    return spans
+
+def collect_spans(res):
+    base = res[0]["text"]
+
+    dataset_spans = []
+    relation_spans = []
+
+    # We store usage context *indexed by item*, not by string
+    usage_context_for_item = {}
+
+    # --- First: capture usage context by item index ---
+    for idx, item in enumerate(res):
+        for r in item.get("relations", []):
+            if r["relation"].lower() == "usage context":
+                usage_context_for_item[idx] = r.get("target", "").lower()
+
+    # --- Collect DATASETS and attach usage context correctly ---
+    for idx, item in enumerate(res):
+        ds = item.get("datasets")
+        if ds:
+            usage = usage_context_for_item.get(idx, None)
+            dataset_spans.append({
+                "start": ds["start"],
+                "end": ds["end"],
+                "label": ds["label"].lower(),
+                "kind": "dataset",
+                "text": base[ds["start"]:ds["end"]],
+                "usage": usage     # ✅ always aligned — no string matching needed
+            })
+
+    # --- Collect RELATIONS (excluding usage context & low confidence) ---
+    for idx, item in enumerate(res):
+        for r in item.get("relations", []):
+            rel_label = r["relation"].lower()
+            if rel_label == "usage context":
+                continue
+            if r.get("score", 0) < 0.6:
+                continue
+            relation_spans.append({
+                "start": r["start"],
+                "end": r["end"],
+                "label": rel_label,
+                "kind": "relation",
+                "text": base[r["start"]:r["end"]],
+                "source": r['source'],
+                "source_item": idx   # align with dataset via index
+            })
+
+    # --- Priority Rule: Dataset spans win over relation overlaps ---
+    def overlaps(a, b):
+        return not (a["end"] <= b["start"] or a["start"] >= b["end"])
+
+    dataset_spans = sorted(dataset_spans, key=lambda s: s["start"])
+
+    filtered_relations = []
+    for rel in relation_spans:
+        if any(overlaps(rel, ds) for ds in dataset_spans):
+            continue
+        filtered_relations.append(rel)
+
+    # --- Deduplicate ---
+    uniq = {(s["start"], s["end"], s["label"], s["kind"]): s
+            for s in dataset_spans + filtered_relations}
+
+    spans = sorted(uniq.values(), key=lambda s: s["start"])
+    return spans
+
+
+def render_highlight(text):
+    res = extract_from_text(text, use_classifier_gate=False)
+    base = res[0]["text"]
+
+    spans = collect_spans(res)
+
+    if not spans:
+        return f"<div style='white-space:pre-wrap;'>{base}</div>"
+
+    html = """<style>
+    .hl {
+    position: relative;
+    cursor: pointer;
+    display: inline-block;
+    }
+
+    .hl:hover::after {
+    content: attr(data-tip);
+    position: absolute;
+    top: -2.2em;
+    left: 0;
+    background: #fff;           /* white tooltip */
+    color: #222;                /* dark text */
+    padding: 4px 8px;
+    font-size: 0.78em;
+    border-radius: 6px;
+    border: 1px solid #ccc;     /* subtle border */
+    box-shadow: 0px 2px 6px rgba(0,0,0,0.15);  /* soft shadow */
+    white-space: nowrap;
+    z-index: 9999;
+    opacity: 1;
+    }
+
+    .hl::after {
+    opacity: 0;
+    pointer-events: none;
+    transition: opacity 0.05s; /* still instant, but fades cleanly */
+    }
+    </style>
+    <div style='white-space:pre-wrap; font-family:monospace;'>
+    """
+
+
+    last = 0
+
+    for span in spans:
+        html += base[last:span["start"]]
+
+        if span["kind"] == "dataset":
+            color = DATASET_COLORS.get(span["label"], "#999")
+        else:
+            color = RELATION_COLORS.get(span["label"], "#74c0fc")
+
+        # if span["kind"] == "dataset":
+        #     tooltip = span["label"]
+        if span["kind"] == "dataset":
+            tooltip = span["label"] if not span.get("usage") else f"usage context: {span['usage']}"
+
+        else:
+            # Show which dataset this relation corresponds to
+            # We look up the nearest dataset with same extraction item.
+            # But since source is provided already, use it directly:
+            tooltip = f"{span['label']} for {span['source']}"
+
+        display_label = RELATION_DISPLAY.get(span["label"], span["label"])
+        html += (
+            f"<span class='hl' data-tip='{tooltip}' "
+            f"style='background:{color}22; border:1px solid {color}; "
+            f"border-radius:4px; padding:0px 3px; margin:0 2px;'>"
+            f"{span['text']}"
+            f"<span style='border:1px solid {color}; color:{color}; "
+            f"font-size:0.7em; border-radius:3px; padding:1px 3px; margin-left:5px;'>"
+            f"{display_label}</span></span>"
+        )
+        last = span["end"]
+
+    html += base[last:]
+    html += "</div>"
+
+    return html
+
+with gr.Blocks(
+    title="AI for Data Use: Dataset Extraction",
+    css="""
+    body, .gradio-container { background-color: #111 !important; color: #e6e6e6 !important; }
+
+    textarea, input, .gr-textbox {
+        background: #1a1a1a !important;
+        color: #f2f2f2 !important;
+        border: 1px solid #444 !important;
+        border-radius: 6px !important;
+    }
+    textarea:focus, input:focus {
+        border-color: #888 !important;
+        box-shadow: 0 0 0 1px #888 !important;
+    }
+
+    button {
+        background: #333 !important;
+        color: #eee !important;
+        border-radius: 6px !important;
+        border: 1px solid #444 !important;
+        padding: 8px 14px !important;
+    }
+    button:hover {
+        background: #444 !important;
+        border-color: #666 !important;
+    }
+
+    .gr-html {
+        background: #1a1a1a !important;
+        color: #fff !important;
+        border-radius: 6px !important;
+        border: 1px solid #333 !important;
+        padding: 14px !important;
+        min-height: 200px;
+    }
+    """
+) as demo:
+
+    gr.Markdown("# AI for Data Use: Dataset Extraction")
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            input_box = gr.Textbox(lines=14, label="Input Text")
+
+        with gr.Column(scale=1):
+            output_box = gr.HTML(
+                label="Highlighted Output",
+                value="<div style='min-height:200px; border:1px solid #444; border-radius:8px; padding:10px; opacity:0.6; text-align:center;'>Waiting for input…</div>"
+            )
+
+    submit_btn = gr.Button("Submit")
+
+    submit_btn.click(
+        fn=render_highlight,
+        inputs=input_box,
+        outputs=output_box
+    )
+
+    gr.Examples(
+        examples=[
+    ["We examine early childhood nutrition and health inequality across Sub-Saharan Africa by combining information from the Demographic and Health Surveys (DHS, various years) and the Multiple Indicator Cluster Surveys (MICS). Anthropometric outcomes (height-for-age, weight-for-age) are standardized using WHO Child Growth Standards. To account for environmental exposure, we align survey cluster coordinates with gridded temperature and precipitation data from the CRU TS 4.06 dataset at the nearest spatial cell."],
+    ["Introduction The mining sector in Africa is growing rapidly and is the main recipient of foreign direct investment ( World Bank 2011 ). The welfare effects of this sector are not well understood, although a literature has recently developed around this question. The main contribution of this paper is to shed light on the welfare effects of gold mining in a detailed, in-depth country study of Ghana, a country with a long tradition of gold mining and a recent, large expansion in capital - intensive and industrial-scale production. A second contribution of this paper is to show the importance of decomposing the effects with respect to distance from the mines. Given the spatial heterogeneity of the results, we explore the effects in an individual-level, difference-in-differences analysis by using spatial lag models to allow for nonlinear effects with distance from mine. We also allow for spillovers across districts, in a district-level analysis. We use two complementary geocoded household data sets to analyze outcomes in Ghana: the Demographic and Health Survey ( DHS ) and the Ghana Living Standard Survey ( GLSS ), which provide information on a wide range of welfare outcomes."],
+    ["The main mining data is a dataset from InterraRMG covering all large-scale mines in Ghana, explained in more detail in section 3. 1. This dataset is linked to survey data from the DHS and GLSS, using spatial information. Geographical coordinates of enumeration areas in GLSS are from Ghana Statistical Services ( GSS ). 2 Point coordinates ( global positioning system [ GPS ] ) for the surveyed DHS clusters3 allow us to match all individuals to one or several mineral mines. We do this in two ways. First, we calculate distance spans from an exact mine location given by its GPS coordinates, and match surveyed individuals to mines. "],
+    ["We study learning outcomes by linking standardized test score data from the Programme for International Student Assessment (PISA) and the Trends in International Mathematics and Science Study (TIMSS) to school resource indicators compiled by the UNESCO Institute for Statistics (UIS). Household socioeconomic background is proxied using parental education information extracted from the Demographic and Health Surveys (DHS) female and household datasets. To examine the role of inequality, we calculate within-school and between-school variance components and correlate these with school funding data from national Ministry of Education financial reports. The goal is to understand how unequal resource allocation contributes to learning gaps across income groups and geographic regions."],
+    ["Patterns of forced displacement are analyzed using the UNHCR Refugee Population Statistics Database, which provides annual country-to-country asylum flows and refugee stock counts. To understand displacement drivers, we integrate conflict event data from the ACLED conflict monitoring system and national political stability indices from the World Governance Indicators (WGI). We additionally incorporate bilateral migration dyads from the World Bank Global Bilateral Migration Database to capture historical migration ties. Geographic exposure to violence is assigned using region-level coordinates and spatial joining procedures. This dataset allows us to estimate the relationship between escalating conflict intensity and subsequent cross-border population movements."],
+    ],
+        inputs=input_box
+    )
+
+    gr.Markdown("""
+<hr style='border: none; border-top: 1px solid #333; margin: 20px 0;'>
+
+### AI for Data Use: Dataset Extraction
+
+This tool identifies **dataset mentions** (e.g., *Demographic and Health Survey*, *Living Standards and Measurement Survey*, etc.) and extracts **contextual metadata** such as:
+- publisher  
+- publication year  
+- reference year
+- geography  
+- acronym  
+- reference population  
+- data description
+- data type
+Usage context:
+primary, background, supporting mention - hover through each datasets to see its usage context.
+
+
+The detected spans are **highlighted inline** with labels so you can quickly review how datasets are being referenced in text.
+
+#### How to Use
+1. Paste or type text into the input box (left) or use any of the examples.
+2. Click **Submit**.
+3. Review highlighted dataset mentions and relation tags (right).
+4. Hover over highlights to see details.
+
+<hr style='border: none; border-top: 1px solid #333; margin: 20px 0;'>
+
+**Resources**
+- Model: https://huggingface.co/{DATA_MODEL_ID}
+- Paper (ArXiv): https://arxiv.org/pdf/2502.10263
+- GLiNER Repo: https://github.com/urchade/GLiNER
+- Project Docs: https://worldbank.github.io/ai4data-use/docs/introduction.html
+""")
+
+demo.launch()
+
+

requirements.txt

@@ -0,0 +1,8 @@
+gradio>=4.0.0
+datasets
+transformers
+accelerate
+sentencepiece
+gliner
+huggingface_hub
+-e ./ai4data_use