Upload jieba3k/Test_Jieba3k.py with huggingface_hub

jieba3k/Test_Jieba3k.py

@@ -0,0 +1,132 @@
+"""
+On-device verification for the jieba3k 0.35.1 wheel.
+
+jieba3k is the Python-3 port of the jieba Chinese text segmentation library.
+We verify imports, tokenizer configuration, basic cut() output for Chinese
+and English text, and search-mode segmentation, all without any I/O.
+
+Usage:
+    python Test_Jieba3k.py
+
+Exit code 0 = everything required PASSed.
+
+Generated by RIMI
+"""
+import sys
+
+RESULTS = []
+
+
+class SkipTest(Exception):
+    pass
+
+
+def test(name, fn):
+    try:
+        fn()
+        RESULTS.append((name, "PASS", None))
+        print("  [PASS] %s" % name)
+    except SkipTest as exc:
+        RESULTS.append((name, "SKIP", str(exc)))
+        print("  [SKIP] %s: %s (optional)" % (name, exc))
+    except Exception as exc:
+        RESULTS.append((name, "FAIL", "%s: %s" % (type(exc).__name__, exc)))
+        print("  [FAIL] %s: %s" % (name, exc))
+
+
+def section(title):
+    print("=" * 60)
+    print("  %s" % title)
+    print("=" * 60)
+
+
+def eq(label, a, b):
+    if a != b:
+        raise AssertionError("%s: %r != %r" % (label, a, b))
+    return True
+
+
+def t_import():
+    import jieba
+    section("Core Import")
+    print("  jieba imported from:", jieba.__file__)
+    for attr in ("cut", "cut_for_search", "add_word", "load_userdict", "setLogLevel", "initialize"):
+        eq("has %s" % attr, True, hasattr(jieba, attr))
+    jieba.setLogLevel(60)
+
+
+def t_cut_default():
+    import jieba
+    section("cut() default mode")
+    jieba.initialize()
+    words = list(jieba.cut("我来到北京清华大学"))
+    eq("returns tokens", len(words) > 0, True)
+    eq("joined text", "".join(words), "我来到北京清华大学")
+
+
+def t_cut_all():
+    import jieba
+    section("cut_all=True full mode")
+    jieba.setLogLevel(60)
+    words = list(jieba.cut("小明硕士毕业于中国科学院计算所", cut_all=True))
+    joined = "".join(words)
+    eq("returns tokens", len(words) > 0, True)
+    eq("contains 中国科学院", "中国科学院" in words, True)
+    eq("all tokens concatenate to source", joined, "小明硕士毕业于中国中国科学院科学科学院学院计算计算所")
+
+
+def t_cut_english():
+    import jieba
+    section("cut() English")
+    words = list(jieba.cut("hello world python programming"))
+    eq("contains hello", "hello" in [w.strip() for w in words], True)
+
+
+def t_cut_for_search():
+    import jieba
+    section("cut_for_search")
+    words = list(jieba.cut_for_search("小明硕士毕业于中国科学院计算所"))
+    joined = "".join(words)
+    eq("returns tokens", len(words) > 0, True)
+    eq("contains 中国科学院", "中国科学院" in words, True)
+    eq("search tokens concatenate to source", joined, "小明硕士毕业于中国科学学院科学院中国科学院计算计算所")
+
+
+def main():
+    tests = [
+        ("import + API", t_import),
+        ("cut() default mode", t_cut_default),
+        ("cut_all=True full mode", t_cut_all),
+        ("cut() English", t_cut_english),
+        ("cut_for_search", t_cut_for_search),
+    ]
+    for name, fn in tests:
+        test(name, fn)
+    print()
+    print("=" * 60)
+    print("  SUMMARY")
+    print("=" * 60)
+    total = len(RESULTS)
+    fails = sum(1 for _, s, _ in RESULTS if s == "FAIL")
+    skips = sum(1 for _, s, _ in RESULTS if s == "SKIP")
+    passed = total - fails - skips
+    print("  Total: %d | Passed: %d | Skipped: %d | Failed: %d" % (total, passed, skips, fails))
+    skipped_items = [(n, m) for n, s, m in RESULTS if s == "SKIP"]
+    if skipped_items:
+        print()
+        print("  Skipped tests (optional):")
+        for n, m in skipped_items:
+            print("    - %s: %s" % (n, m))
+    print()
+    if fails == 0:
+        print("  ALL TESTS PASSED")
+    else:
+        print("  SOME TESTS FAILED")
+        for n, s, m in RESULTS:
+            if s == "FAIL":
+                print("    FAILED: %s (%s)" % (n, m))
+    sys.exit(1 if fails else 0)
+
+
+if __name__ == "__main__":
+    main()