Spaces:
Sleeping
Sleeping
Melika Kheirieh
commited on
Commit
Β·
f8224ec
1
Parent(s):
454d146
chore(factory): safely load .env via dotenv (with fallback under CI)
Browse files- benchmarks/results/20251108-115229/eval.jsonl +5 -0
- benchmarks/results/20251108-115229/metrics_summary.json +12 -0
- benchmarks/results/20251108-115229/results.csv +6 -0
- benchmarks/results/20251108-115229/spider_eval.jsonl +5 -0
- benchmarks/results/20251108-115229/summary.json +12 -0
- nl2sql/pipeline_factory.py +7 -0
benchmarks/results/20251108-115229/eval.jsonl
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"source": "demo", "db_id": "demo", "query": "list all customers", "ok": true, "latency_ms": 7641, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 4165}, {"stage": "generator", "ms": 1516}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 2}, {"stage": "repair", "ms": 1141}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 807}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "pipeline", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
|
| 2 |
+
{"source": "demo", "db_id": "demo", "query": "show total invoices per country", "ok": true, "latency_ms": 12918, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 7763}, {"stage": "generator", "ms": 1787}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 1782}, {"stage": "safety", "ms": 0}, {"stage": "executor", "ms": 0}, {"stage": "repair", "ms": 1582}, {"stage": "safety", "ms": 0}, {"stage": "executor", "ms": 0}, {"stage": "pipeline", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
|
| 3 |
+
{"source": "demo", "db_id": "demo", "query": "top 3 albums by total sales", "ok": true, "latency_ms": 1, "trace": [{"stage": "detector", "ms": 0}], "error": null}
|
| 4 |
+
{"source": "demo", "db_id": "demo", "query": "artists with more than 3 albums", "ok": false, "latency_ms": 11580, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 5497}, {"stage": "generator", "ms": 2348}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 2094}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 1632}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "pipeline", "ms": 0}], "error": null}
|
| 5 |
+
{"source": "demo", "db_id": "demo", "query": "number of employees per city", "ok": true, "latency_ms": 8164, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 3488}, {"stage": "generator", "ms": 2289}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 972}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 1410}, {"stage": "safety", "ms": 0}, {"stage": "executor", "ms": 0}, {"stage": "pipeline", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
|
benchmarks/results/20251108-115229/metrics_summary.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"queries_total": 5,
|
| 3 |
+
"total": 5,
|
| 4 |
+
"pipeline_source": "adapter",
|
| 5 |
+
"success_rate": 0.8,
|
| 6 |
+
"avg_latency_ms": 8060.8,
|
| 7 |
+
"mode": "single-db",
|
| 8 |
+
"db_path": "/Users/melikakheirieh/Desktop/my/career-developement/LLM/nl2sql-copilot/demo.db",
|
| 9 |
+
"config": "/Users/melikakheirieh/Desktop/my/career-developement/LLM/nl2sql-copilot/configs/sqlite_pipeline.yaml",
|
| 10 |
+
"provider_hint": "REAL",
|
| 11 |
+
"timestamp": "2025-11-08 11:53:09"
|
| 12 |
+
}
|
benchmarks/results/20251108-115229/results.csv
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
query,ok,latency_ms
|
| 2 |
+
list all customers,β
,7641
|
| 3 |
+
show total invoices per country,β
,12918
|
| 4 |
+
top 3 albums by total sales,β
,1
|
| 5 |
+
artists with more than 3 albums,β,11580
|
| 6 |
+
number of employees per city,β
,8164
|
benchmarks/results/20251108-115229/spider_eval.jsonl
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"source": "demo", "db_id": "demo", "query": "list all customers", "ok": true, "latency_ms": 7641, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 4165}, {"stage": "generator", "ms": 1516}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 2}, {"stage": "repair", "ms": 1141}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 807}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "pipeline", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
|
| 2 |
+
{"source": "demo", "db_id": "demo", "query": "show total invoices per country", "ok": true, "latency_ms": 12918, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 7763}, {"stage": "generator", "ms": 1787}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 1782}, {"stage": "safety", "ms": 0}, {"stage": "executor", "ms": 0}, {"stage": "repair", "ms": 1582}, {"stage": "safety", "ms": 0}, {"stage": "executor", "ms": 0}, {"stage": "pipeline", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
|
| 3 |
+
{"source": "demo", "db_id": "demo", "query": "top 3 albums by total sales", "ok": true, "latency_ms": 1, "trace": [{"stage": "detector", "ms": 0}], "error": null}
|
| 4 |
+
{"source": "demo", "db_id": "demo", "query": "artists with more than 3 albums", "ok": false, "latency_ms": 11580, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 5497}, {"stage": "generator", "ms": 2348}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 2094}, {"stage": "safety", "ms": 2}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 1632}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "pipeline", "ms": 0}], "error": null}
|
| 5 |
+
{"source": "demo", "db_id": "demo", "query": "number of employees per city", "ok": true, "latency_ms": 8164, "trace": [{"stage": "detector", "ms": 0}, {"stage": "planner", "ms": 3488}, {"stage": "generator", "ms": 2289}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 972}, {"stage": "safety", "ms": 1}, {"stage": "executor", "ms": 1}, {"stage": "repair", "ms": 1410}, {"stage": "safety", "ms": 0}, {"stage": "executor", "ms": 0}, {"stage": "pipeline", "ms": 0}, {"stage": "pipeline", "ms": 0}], "error": null}
|
benchmarks/results/20251108-115229/summary.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"queries_total": 5,
|
| 3 |
+
"total": 5,
|
| 4 |
+
"pipeline_source": "adapter",
|
| 5 |
+
"success_rate": 0.8,
|
| 6 |
+
"avg_latency_ms": 8060.8,
|
| 7 |
+
"mode": "single-db",
|
| 8 |
+
"db_path": "/Users/melikakheirieh/Desktop/my/career-developement/LLM/nl2sql-copilot/demo.db",
|
| 9 |
+
"config": "/Users/melikakheirieh/Desktop/my/career-developement/LLM/nl2sql-copilot/configs/sqlite_pipeline.yaml",
|
| 10 |
+
"provider_hint": "REAL",
|
| 11 |
+
"timestamp": "2025-11-08 11:53:09"
|
| 12 |
+
}
|
nl2sql/pipeline_factory.py
CHANGED
|
@@ -4,6 +4,13 @@ import os
|
|
| 4 |
from typing import Any, Dict, Optional, cast
|
| 5 |
import yaml # type: ignore[import-untyped]
|
| 6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
from nl2sql.pipeline import Pipeline
|
| 8 |
from nl2sql.registry import (
|
| 9 |
DETECTORS,
|
|
|
|
| 4 |
from typing import Any, Dict, Optional, cast
|
| 5 |
import yaml # type: ignore[import-untyped]
|
| 6 |
|
| 7 |
+
try:
|
| 8 |
+
from dotenv import load_dotenv
|
| 9 |
+
|
| 10 |
+
load_dotenv()
|
| 11 |
+
except Exception:
|
| 12 |
+
pass
|
| 13 |
+
|
| 14 |
from nl2sql.pipeline import Pipeline
|
| 15 |
from nl2sql.registry import (
|
| 16 |
DETECTORS,
|