Pointf5ive commited on
Commit
066e2d1
·
verified ·
1 Parent(s): 7064f8d

Update src/codex_extractor.py

Browse files
Files changed (1) hide show
  1. src/codex_extractor.py +6 -2
src/codex_extractor.py CHANGED
@@ -114,15 +114,19 @@ except Exception:
114
 
115
  try:
116
  import nltk
 
117
  _NLTK_DATA = ["punkt", "punkt_tab", "averaged_perceptron_tagger", "cmudict", "stopwords"]
118
  for _pkg in _NLTK_DATA:
119
  try:
120
  nltk.data.find(f"tokenizers/{_pkg}" if "punkt" in _pkg else f"corpora/{_pkg}")
121
  except LookupError:
122
  try:
 
 
123
  nltk.download(_pkg, quiet=True)
 
124
  except Exception:
125
- pass
126
  from nltk.tokenize import sent_tokenize, word_tokenize
127
  from nltk.corpus import cmudict as _cmudict
128
  CMU_DICT = _cmudict.dict()
@@ -1909,4 +1913,4 @@ if __name__ == "__main__":
1909
  debug_output_dir="/tmp/codex_debug",
1910
  )
1911
  print(format_fingerprint_report(fp))
1912
- print("\nDebug artefacts written to:", fp.get("Debug_Artefacts", {}))
 
114
 
115
  try:
116
  import nltk
117
+ import socket as _socket
118
  _NLTK_DATA = ["punkt", "punkt_tab", "averaged_perceptron_tagger", "cmudict", "stopwords"]
119
  for _pkg in _NLTK_DATA:
120
  try:
121
  nltk.data.find(f"tokenizers/{_pkg}" if "punkt" in _pkg else f"corpora/{_pkg}")
122
  except LookupError:
123
  try:
124
+ _old_timeout = _socket.getdefaulttimeout()
125
+ _socket.setdefaulttimeout(5)
126
  nltk.download(_pkg, quiet=True)
127
+ _socket.setdefaulttimeout(_old_timeout)
128
  except Exception:
129
+ _socket.setdefaulttimeout(_old_timeout)
130
  from nltk.tokenize import sent_tokenize, word_tokenize
131
  from nltk.corpus import cmudict as _cmudict
132
  CMU_DICT = _cmudict.dict()
 
1913
  debug_output_dir="/tmp/codex_debug",
1914
  )
1915
  print(format_fingerprint_report(fp))
1916
+ print("\nDebug artefacts written to:", fp.get("Debug_Artefacts", {}))