CreatedDBAgent

Co-authored-by: Copilot <copilot@github.com>

app.py

@@ -24,6 +24,7 @@ from extractors import (
     ExpenseUserExtractor,
     ExpenseAmountExtractor,
 )
+from sql_generator import generate_sql
 
 
 # HuggingFace Token (если нужен для моделей)
@@ -401,6 +402,12 @@ def parse_context(raw: str | None) -> dict[str, Any]:
         return {}
 
 
+def parse_json_payload() -> dict[str, Any]:
+    """Возвращает JSON payload из входящего запроса."""
+    payload = request.get_json(silent=True)
+    return payload if isinstance(payload, dict) else {}
+
+
 # ============================================================================
 # ENDPOINTS
 # ============================================================================
@@ -413,6 +420,7 @@ def index():
         "message": "Voice processing API is running",
         "endpoints": {
             "POST /process-audio": "Process audio file",
+            "POST /generate-sql": "Generate SQLite SELECT query from natural language",
             "GET /health": "Health check",
             "GET /test-data": "Run text-only extraction tests"
         }
@@ -495,5 +503,19 @@ def process_audio():
             os.unlink(temp_path)
 
 
+@app.post("/generate-sql")
+def generate_sql_endpoint():
+    """Генерирует SQL по текстовому запросу и схеме БД."""
+    payload = parse_json_payload()
+    query = payload.get("query") or payload.get("text") or ""
+    limit = payload.get("limit") or 200
+
+    try:
+        sql = generate_sql(question=query, limit=int(limit))
+        return jsonify({"sql": sql})
+    except Exception as exception:
+        return jsonify({"status": "error", "message": str(exception)}), 422
+
+
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=int(os.getenv("PORT", "7860")))

requirements.txt

@@ -9,3 +9,4 @@ python-dateutil
 iuliia
 torch
 scikit-learn
+sentencepiece

sql_generator.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+
+
+DEFAULT_DB_SCHEMA = """CREATE TABLE \"migrations\" (\"id\" integer primary key autoincrement not null, \"migration\" varchar not null, \"batch\" integer not null);
+
+CREATE TABLE \"users\" (\"id\" integer primary key autoincrement not null, \"image\" varchar, \"name\" varchar not null, \"email\" varchar not null, \"email_verified_at\" datetime, \"password\" varchar not null, \"widget_preferences\" text, \"remember_token\" varchar, \"created_at\" datetime, \"updated_at\" datetime);
+
+CREATE UNIQUE INDEX \"users_email_unique\" on \"users\" (\"email\");
+
+CREATE TABLE \"password_reset_tokens\" (\"email\" varchar not null, \"token\" varchar not null, \"created_at\" datetime, primary key (\"email\"));
+
+CREATE TABLE \"sessions\" (\"id\" varchar not null, \"user_id\" integer, \"ip_address\" varchar, \"user_agent\" text, \"payload\" text not null, \"last_activity\" integer not null, primary key (\"id\"));
+
+CREATE INDEX \"sessions_user_id_index\" on \"sessions\" (\"user_id\");
+
+CREATE INDEX \"sessions_last_activity_index\" on \"sessions\" (\"last_activity\");
+
+CREATE TABLE \"cache\" (\"key\" varchar not null, \"value\" text not null, \"expiration\" integer not null, primary key (\"key\"));
+
+CREATE TABLE \"cache_locks\" (\"key\" varchar not null, \"owner\" varchar not null, \"expiration\" integer not null, primary key (\"key\"));
+
+CREATE TABLE \"jobs\" (\"id\" integer primary key autoincrement not null, \"queue\" varchar not null, \"payload\" text not null, \"attempts\" integer not null, \"reserved_at\" integer, \"available_at\" integer not null, \"created_at\" integer not null);
+
+CREATE INDEX \"jobs_queue_index\" on \"jobs\" (\"queue\");
+
+CREATE TABLE \"job_batches\" (\"id\" varchar not null, \"name\" varchar not null, \"total_jobs\" integer not null, \"pending_jobs\" integer not null, \"failed_jobs\" integer not null, \"failed_job_ids\" text not null, \"options\" text, \"cancelled_at\" integer, \"created_at\" integer not null, \"finished_at\" integer, primary key (\"id\"));
+
+CREATE TABLE \"failed_jobs\" (\"id\" integer primary key autoincrement not null, \"uuid\" varchar not null, \"connection\" text not null, \"queue\" text not null, \"payload\" text not null, \"exception\" text not null, \"failed_at\" datetime not null default CURRENT_TIMESTAMP);
+
+CREATE UNIQUE INDEX \"failed_jobs_uuid_unique\" on \"failed_jobs\" (\"uuid\");
+
+CREATE TABLE \"categories\" (\"id\" integer primary key autoincrement not null, \"name\" varchar not null, \"slug\" varchar not null, \"image\" varchar, \"notes\" text not null, \"created_at\" datetime, \"updated_at\" datetime);
+
+CREATE UNIQUE INDEX \"categories_slug_unique\" on \"categories\" (\"slug\");
+
+CREATE TABLE \"suppliers\" (\"id\" integer primary key autoincrement not null, \"name\" varchar not null, \"slug\" varchar not null, \"image\" varchar, \"category_id\" integer not null, \"created_at\" datetime, \"updated_at\" datetime, foreign key(\"category_id\") references \"categories\"(\"id\") on delete cascade);
+
+CREATE UNIQUE INDEX \"suppliers_slug_unique\" on \"suppliers\" (\"slug\");
+
+CREATE TABLE \"expenses\" (\"id\" integer primary key autoincrement not null, \"user_id\" integer not null, \"date\" date not null, \"category_id\" integer not null, \"supplier_id\" integer not null, \"sum\" numeric not null, \"notes\" text, \"created_at\" datetime, \"updated_at\" datetime, foreign key(\"user_id\") references \"users\"(\"id\") on delete cascade, foreign key(\"category_id\") references \"categories\"(\"id\") on delete set null, foreign key(\"supplier_id\") references \"suppliers\"(\"id\") on delete set null);
+
+CREATE INDEX \"expenses_date_index\" on \"expenses\" (\"date\");
+
+CREATE INDEX \"expenses_user_id_date_index\" on \"expenses\" (\"user_id\", \"date\");
+
+CREATE TABLE \"overpayments\" (\"id\" integer primary key autoincrement not null, \"user_id\" integer not null, \"sum\" numeric not null, \"notes\" text not null, \"created_at\" datetime, \"updated_at\" datetime, foreign key(\"user_id\") references \"users\"(\"id\") on delete cascade);
+
+CREATE INDEX \"overpayments_created_at_index\" on \"overpayments\" (\"created_at\");
+
+CREATE TABLE \"debts\" (\"id\" integer primary key autoincrement not null, \"date\" date not null, \"user_id\" integer, \"debt_sum\" numeric not null default '0', \"overpayment_id\" integer, \"notes\" text not null, \"payment_status\" varchar check (\"payment_status\" in ('unpaid', 'partial', 'paid')) not null default 'unpaid', \"partial_sum\" numeric not null default '0', \"date_paid\" date, \"created_at\" datetime, \"updated_at\" datetime, foreign key(\"user_id\") references \"users\"(\"id\") on delete cascade, foreign key(\"overpayment_id\") references \"overpayments\"(\"id\") on delete cascade);
+
+CREATE INDEX \"debts_payment_status_date_index\" on \"debts\" (\"payment_status\", \"date\");
+
+CREATE UNIQUE INDEX \"debts_date_unique\" on \"debts\" (\"date\");
+
+CREATE TABLE \"notifications\" (\"id\" varchar not null, \"type\" varchar not null, \"notifiable_type\" varchar not null, \"notifiable_id\" integer not null, \"data\" text not null, \"read_at\" datetime, \"created_at\" datetime, \"updated_at\" datetime, primary key (\"id\"));
+
+CREATE INDEX \"notifications_notifiable_type_notifiable_id_index\" on \"notifications\" (\"notifiable_type\", \"notifiable_id\");
+
+CREATE TABLE \"expense_change_requests\" (\"id\" integer primary key autoincrement not null, \"expense_id\" integer, \"user_id\" integer not null, \"action_type\" varchar check (\"action_type\" in ('create', 'edit', 'delete')) not null, \"current_date\" date, \"current_user_id\" integer, \"current_category_id\" integer, \"current_supplier_id\" integer, \"current_sum\" numeric, \"current_notes\" text, \"requested_date\" date, \"requested_user_id\" integer, \"requested_category_id\" integer, \"requested_supplier_id\" integer, \"requested_sum\" numeric, \"requested_notes\" text, \"notes\" text not null, \"status\" varchar check (\"status\" in ('pending', 'rejected', 'completed')) not null default 'pending', \"applied_at\" datetime, \"created_at\" datetime, \"updated_at\" datetime, foreign key(\"expense_id\") references \"expenses\"(\"id\") on delete set null, foreign key(\"user_id\") references \"users\"(\"id\") on delete cascade, foreign key(\"current_user_id\") references \"users\"(\"id\") on delete set null, foreign key(\"current_category_id\") references \"categories\"(\"id\") on delete set null, foreign key(\"current_supplier_id\") references \"suppliers\"(\"id\") on delete set null, foreign key(\"requested_user_id\") references \"users\"(\"id\") on delete set null, foreign key(\"requested_category_id\") references \"categories\"(\"id\") on delete set null, foreign key(\"requested_supplier_id\") references \"suppliers\"(\"id\") on delete set null);
+
+CREATE INDEX \"expense_change_requests_status_created_at_index\" on \"expense_change_requests\" (\"status\", \"created_at\");
+
+CREATE INDEX \"expense_change_requests_expense_id_status_index\" on \"expense_change_requests\" (\"expense_id\", \"status\");
+
+CREATE INDEX \"expense_change_requests_user_id_index\" on \"expense_change_requests\" (\"user_id\");
+
+CREATE TABLE \"expense_change_request_votes\" (\"id\" integer primary key autoincrement not null, \"expense_change_request_id\" integer not null, \"user_id\" integer not null, \"vote\" varchar check (\"vote\" in ('approved', 'rejected')) not null, \"notes\" text, \"created_at\" datetime, \"updated_at\" datetime, foreign key(\"expense_change_request_id\") references \"expense_change_requests\"(\"id\") on delete cascade, foreign key(\"user_id\") references \"users\"(\"id\") on delete cascade);
+
+CREATE UNIQUE INDEX \"unique_vote_per_user\" on \"expense_change_request_votes\" (\"expense_change_request_id\", \"user_id\");
+
+CREATE INDEX \"expense_change_request_votes_user_id_vote_index\" on \"expense_change_request_votes\" (\"user_id\", \"vote\");
+
+CREATE TABLE \"paid_debts\" (\"id\" integer primary key autoincrement not null, \"created_at\" datetime, \"updated_at\" datetime, \"debt_id\" integer not null, \"changed_debt_date\" date not null, \"paid_by_user_id\" integer not null, \"payment_status\" varchar check (\"payment_status\" in ('partial', 'paid')) not null, \"paid_sum\" numeric not null, foreign key(\"debt_id\") references \"debts\"(\"id\") on delete cascade, foreign key(\"paid_by_user_id\") references \"users\"(\"id\") on delete cascade);
+
+CREATE INDEX \"paid_debts_debt_id_index\" on \"paid_debts\" (\"debt_id\");
+
+CREATE INDEX \"paid_debts_paid_by_user_id_index\" on \"paid_debts\" (\"paid_by_user_id\");"""
+
+_SQL_GENERATOR: Any | None = None
+
+
+@dataclass(frozen=True)
+class SqlGenerationRequest:
+    question: str
+    limit: int = 200
+
+
+def _get_sql_generator() -> Any:
+    global _SQL_GENERATOR
+
+    if _SQL_GENERATOR is None:
+        from transformers import pipeline
+
+        model_id = os.getenv("SQL_MODEL", "google/flan-t5-base")
+        _SQL_GENERATOR = pipeline(
+            task="text2text-generation",
+            model=model_id,
+            tokenizer=model_id,
+            device=-1,
+            torch_dtype=torch.float32,
+        )
+
+    return _SQL_GENERATOR
+
+
+def _build_prompt(payload: SqlGenerationRequest) -> str:
+    return (
+        "You translate user requests into SQLite SELECT queries. "
+        "Return only SQL without explanations. "
+        "Use only tables and columns from the schema. "
+        "Never generate INSERT, UPDATE, DELETE, DROP, ALTER, PRAGMA, ATTACH or CREATE. "
+        "Prefer explicit JOIN conditions using foreign keys from the schema. "
+        f"Add LIMIT {payload.limit} when the query is not an aggregate result.\n\n"
+        f"Schema:\n{DEFAULT_DB_SCHEMA}\n\n"
+        f"User request:\n{payload.question}\n\n"
+        "SQL:"
+    )
+
+
+def _normalize_sql(raw_sql: str, limit: int) -> str:
+    sql = (raw_sql or "").strip()
+    if not sql:
+        raise ValueError("SQL model returned an empty result.")
+
+    if "```" in sql:
+        parts = [part.strip() for part in sql.split("```") if part.strip()]
+        sql = parts[-1]
+
+    upper_sql = sql.upper()
+    sql_start = upper_sql.find("SELECT")
+    if sql_start == -1:
+        raise ValueError("Generated SQL is not a SELECT query.")
+
+    sql = sql[sql_start:]
+    if ";" in sql:
+        sql = sql.split(";", 1)[0].strip()
+
+    upper_sql = sql.upper()
+    forbidden = ("INSERT ", "UPDATE ", "DELETE ", "DROP ", "ALTER ", "PRAGMA ", "ATTACH ", "CREATE ", "REPLACE ")
+    if any(keyword in upper_sql for keyword in forbidden):
+        raise ValueError("Generated SQL contains forbidden statements.")
+
+    if not upper_sql.startswith("SELECT "):
+        raise ValueError("Only SELECT queries are allowed.")
+
+    aggregate_markers = ("COUNT(", "SUM(", "AVG(", "MIN(", "MAX(")
+    has_limit = " LIMIT " in upper_sql
+    if not has_limit and not any(marker in upper_sql for marker in aggregate_markers):
+        sql = f"{sql} LIMIT {limit}"
+
+    return sql
+
+
+def generate_sql(question: str, limit: int = 200) -> str:
+    clean_question = (question or "").strip()
+    if not clean_question:
+        raise ValueError("Field 'query' is required.")
+
+    payload = SqlGenerationRequest(
+        question=clean_question,
+        limit=limit,
+    )
+
+    generator = _get_sql_generator()
+    prompt = _build_prompt(payload)
+    result = generator(
+        prompt,
+        max_new_tokens=256,
+        do_sample=False,
+        truncation=True,
+    )
+
+    generated_text = result[0].get("generated_text", "") if result else ""
+    return _normalize_sql(generated_text, limit=payload.limit)