Update app.py

app.py

@@ -1,5 +1,3 @@
-# debug_app.py — 把它放到 HF Space 替换原来的 app.py
-import os, hashlib, json
 import numpy as np
 import torch
 import torch.nn as nn
@@ -8,7 +6,7 @@ import joblib
 from collections import Counter
 import gradio as gr
 
-# --- utils (同训练代码) ---
+# --- utils (from the notebook) ---
 def ngrams(sentence, n=1, lc=True):
     ngram_l = []
     sentence = sentence.lower()
@@ -32,16 +30,16 @@ def reproducible_hash(string):
     h = hashlib.md5(string.encode("utf-8"), usedforsecurity=False)
     return int.from_bytes(h.digest()[0:8], 'big', signed=True)
 
-def hash_ngrams(ngrams_list, modulos):
+def hash_ngrams(ngrams, modulos):
     hash_codes = []
-    for ngram_list, modulo in zip(ngrams_list, modulos):
+    for ngram_list, modulo in zip(ngrams, modulos):
         codes = [(reproducible_hash(x) % modulo) for x in ngram_list]
         hash_codes.append(codes)
     return hash_codes
 
 def calc_rel_freq(codes):
     cnt = Counter(codes)
-    total = sum(cnt.values()) if cnt else 1
+    total = sum(cnt.values())
     for k in cnt:
         cnt[k] /= total
     return cnt
@@ -59,131 +57,45 @@ def shift_keys(dicts, MAX_SHIFT):
 
 def build_freq_dict(sentence, MAXES=MAXES, MAX_SHIFT=MAX_SHIFT):
     hngrams = hash_ngrams(all_ngrams(sentence), MAXES)
-    fhcodes = list(map(calc_rel_freq, hngrams))
+    fhcodes = map(calc_rel_freq, hngrams)
     return shift_keys(fhcodes, MAX_SHIFT)
 
-# --- helper diagnostics ---
-def file_md5(path):
-    if not os.path.exists(path):
-        return None
-    with open(path, "rb") as f:
-        return hashlib.md5(f.read()).hexdigest()
-
-def model_param_stats(m):
-    mins, maxs, means = [], [], []
-    for p in m.parameters():
-        arr = p.detach().cpu().numpy().ravel()
-        if arr.size == 0:
-            continue
-        mins.append(float(arr.min()))
-        maxs.append(float(arr.max()))
-        means.append(float(arr.mean()))
-    if not mins:
-        return {"min": None, "max": None, "mean": None}
-    return {"min": min(mins), "max": max(maxs), "mean": float(np.mean(means))}
-
-# --- load artifacts (ensure these files exist in your repo) ---
-VEC_FN = "nld_vectorizer.joblib"
-LANG_FN = "nld_lang_codes.joblib"
-MODEL_FN = "nld.pth"
-
-vectorizer = joblib.load(VEC_FN)
-idx2lang = joblib.load(LANG_FN)
+# --- load models ---
+clf = joblib.load("nld.joblib")
+vectorizer = joblib.load("nld_vectorizer.joblib")
+idx2lang = joblib.load("nld_lang_codes.joblib")
 
 input_dim = len(vectorizer.vocabulary_)
 nbr_classes = len(idx2lang)
 
-# build model skeleton same as training
 model = nn.Sequential(
     nn.Linear(input_dim, 50),
     nn.ReLU(),
     nn.Linear(50, nbr_classes)
 )
-model.load_state_dict(torch.load(MODEL_FN, map_location="cpu"))
+model.load_state_dict(torch.load("nld.pth", map_location="cpu"))
 model.eval()
 
-# quick sanity info (will also print to logs)
-print(">>> artifact md5:", MODEL_FN, file_md5(MODEL_FN))
-print(">>> artifact md5:", VEC_FN, file_md5(VEC_FN))
-print(">>> artifact md5:", LANG_FN, file_md5(LANG_FN))
-print("vocab size:", len(vectorizer.vocabulary_))
-print("sample vocab items:", list(vectorizer.vocabulary_.items())[:10])
-print("idx2lang sample:", list(idx2lang.items())[:10])
-print("model param stats:", model_param_stats(model))
-
-# --- prediction + debug function ---
-def detect_lang_debug(src_sentence: str):
-    debug = {}
-    debug['md5_model'] = file_md5(MODEL_FN)
-    debug['md5_vectorizer'] = file_md5(VEC_FN)
-    debug['md5_idx2lang'] = file_md5(LANG_FN)
-    debug['vocab_size'] = len(vectorizer.vocabulary_)
-    debug['idx2lang_len'] = len(idx2lang)
-    debug['idx2lang_sample'] = dict(list(idx2lang.items())[:10])
-    debug['model_param_stats'] = model_param_stats(model)
-
-    feat_dict = build_freq_dict(src_sentence)
-    X_test = vectorizer.transform([feat_dict])  # ensure a single dict in list
-    if hasattr(X_test, "toarray"):
-        X_arr = X_test.toarray()
-    else:
-        X_arr = np.array(X_test)
-    debug['nonzero_features'] = int(np.count_nonzero(X_arr))
-    debug['X_shape'] = X_arr.shape
-
-    X_tensor = torch.from_numpy(X_arr.astype("float32"))
-    with torch.no_grad():
-        logits = model(X_tensor)
-        probs = torch.softmax(logits, dim=-1).cpu().numpy().ravel()
-
-    topk = list(np.argsort(probs)[::-1][:5])
-    topk_info = [(int(k), idx2lang[int(k)], float(probs[int(k)])) for k in topk]
-    pred_idx = int(topk[0])
-    pred_lang = idx2lang[pred_idx]
-
-    debug_text = json.dumps({
-        "pred_lang": pred_lang,
-        "pred_idx": pred_idx,
-        "topk": topk_info,
-        "debug": debug
-    }, ensure_ascii=False, indent=2)
-    print("DEBUG:", debug_text)  # Visible in Spaces logs
-    return pred_lang, debug_text
-
-# --- self-test example set ---
-SELF_TESTS = {
-    "eng": "Hello, how are you?",
-    "fra": "Bonjour, comment allez-vous?",
-    "cmn": "你好，你在做什么？",
-    "jpn": "こんにちは、お元気ですか？",
-    "kor": "안녕하세요. 잘 지내세요?",
-    "ara": "مرحبا كيف حالك",
-    "swe": "Hej, hur mår du?",
-    "dan": "Godmorgen, hvordan har du det?"
-}
-
-def run_self_test():
-    results = []
-    for lang, sent in SELF_TESTS.items():
-        pred, dbg = detect_lang_debug(sent)
-        ok = (pred == lang) or (pred == lang)  # best-effort equality
-        results.append(f"{lang} | sent: {sent} | pred: {pred} | ok: {ok}")
-    out = "\n".join(results)
-    print("SELF-TEST RESULTS:\n", out)
-    return out
-
+# --- prediction function ---
+# 修改预测函数为sklearn版本
+def detect_lang(src_sentence):
+    # 特征提取逻辑不变
+    X_test = vectorizer.transform([build_freq_dict(src_sentence)])
+    # 使用sklearn模型预测
+    pred_idx = clf.predict(X_test)[0]
+    return idx2lang[pred_idx]
+    
 # --- Gradio UI ---
-with gr.Blocks(title="Antons language detector (debug)") as demo:
-    gr.Markdown("# Antons language detector — debug build")
+with gr.Blocks(title="Antons language detector") as demo:
+    gr.Markdown("# Antons language detector")
     with gr.Row():
-        with gr.Column(scale=3):
-            src = gr.Textbox(label="Text", placeholder="Write your text...")
-            btn = gr.Button("Guess the language!")
-            selftest_btn = gr.Button("Run self-test")
-        with gr.Column(scale=2):
-            out_lang = gr.Textbox(label="Language", interactive=False)
-            out_debug = gr.Textbox(label="Debug info (JSON)", interactive=False, lines=20)
-    btn.click(fn=detect_lang_debug, inputs=[src], outputs=[out_lang, out_debug])
-    selftest_btn.click(fn=run_self_test, inputs=[], outputs=[out_debug])
-
-demo.launch()
+        with gr.Column():
+            src_sentence = gr.Textbox(
+                label="Text", placeholder="Write your text...")
+        with gr.Column():
+            tgt_sentence = gr.Textbox(
+                label="Language", placeholder="Language will show here...")
+    btn = gr.Button("Guess the language!")
+    btn.click(fn=detect_lang, inputs=[src_sentence], outputs=[tgt_sentence])
+
+demo.launch()