Spaces:
Sleeping
Sleeping
Update src/codex_extractor.py
Browse files- src/codex_extractor.py +6 -2
src/codex_extractor.py
CHANGED
|
@@ -114,15 +114,19 @@ except Exception:
|
|
| 114 |
|
| 115 |
try:
|
| 116 |
import nltk
|
|
|
|
| 117 |
_NLTK_DATA = ["punkt", "punkt_tab", "averaged_perceptron_tagger", "cmudict", "stopwords"]
|
| 118 |
for _pkg in _NLTK_DATA:
|
| 119 |
try:
|
| 120 |
nltk.data.find(f"tokenizers/{_pkg}" if "punkt" in _pkg else f"corpora/{_pkg}")
|
| 121 |
except LookupError:
|
| 122 |
try:
|
|
|
|
|
|
|
| 123 |
nltk.download(_pkg, quiet=True)
|
|
|
|
| 124 |
except Exception:
|
| 125 |
-
|
| 126 |
from nltk.tokenize import sent_tokenize, word_tokenize
|
| 127 |
from nltk.corpus import cmudict as _cmudict
|
| 128 |
CMU_DICT = _cmudict.dict()
|
|
@@ -1909,4 +1913,4 @@ if __name__ == "__main__":
|
|
| 1909 |
debug_output_dir="/tmp/codex_debug",
|
| 1910 |
)
|
| 1911 |
print(format_fingerprint_report(fp))
|
| 1912 |
-
print("\nDebug artefacts written to:", fp.get("Debug_Artefacts", {}))
|
|
|
|
| 114 |
|
| 115 |
try:
|
| 116 |
import nltk
|
| 117 |
+
import socket as _socket
|
| 118 |
_NLTK_DATA = ["punkt", "punkt_tab", "averaged_perceptron_tagger", "cmudict", "stopwords"]
|
| 119 |
for _pkg in _NLTK_DATA:
|
| 120 |
try:
|
| 121 |
nltk.data.find(f"tokenizers/{_pkg}" if "punkt" in _pkg else f"corpora/{_pkg}")
|
| 122 |
except LookupError:
|
| 123 |
try:
|
| 124 |
+
_old_timeout = _socket.getdefaulttimeout()
|
| 125 |
+
_socket.setdefaulttimeout(5)
|
| 126 |
nltk.download(_pkg, quiet=True)
|
| 127 |
+
_socket.setdefaulttimeout(_old_timeout)
|
| 128 |
except Exception:
|
| 129 |
+
_socket.setdefaulttimeout(_old_timeout)
|
| 130 |
from nltk.tokenize import sent_tokenize, word_tokenize
|
| 131 |
from nltk.corpus import cmudict as _cmudict
|
| 132 |
CMU_DICT = _cmudict.dict()
|
|
|
|
| 1913 |
debug_output_dir="/tmp/codex_debug",
|
| 1914 |
)
|
| 1915 |
print(format_fingerprint_report(fp))
|
| 1916 |
+
print("\nDebug artefacts written to:", fp.get("Debug_Artefacts", {}))
|