ll-monkey commited on
Commit
b35ecbb
·
verified ·
1 Parent(s): 14a1e51

Update src/streamlit_app.py

Browse files
Files changed (1) hide show
  1. src/streamlit_app.py +19 -17
src/streamlit_app.py CHANGED
@@ -26,27 +26,29 @@ hf_token = os.getenv("HF_TOKEN")
26
  # use cache
27
  @st.cache_resource
28
  def load_tokenizer(model_path):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
 
30
- if "wangchanberta" in model_path.lower():
31
- return AutoTokenizer.from_pretrained(
32
- model_path,
33
- token=hf_token,
34
- use_fast=False,
35
- trust_remote_code=True
36
- )
37
-
38
- tokenizer = AutoTokenizer.from_pretrained(
39
- model_path,
40
- token=hf_token,
41
- trust_remote_code=True
42
- )
43
-
44
- if tokenizer.pad_token is None:
45
- tokenizer.pad_token = tokenizer.eos_token
46
  return tokenizer
47
 
48
  except Exception as e:
49
- # ส่ง Error กลับไปแสดงที่หน้า UI
50
  st.error(f"Error loading tokenizer for {model_path}: {e}")
51
  return None
52
 
 
26
  # use cache
27
  @st.cache_resource
28
  def load_tokenizer(model_path):
29
+
30
+ Try:
31
+
32
+ if "wangchanberta" in model_path.lower():
33
+ return AutoTokenizer.from_pretrained(
34
+ model_path,
35
+ token=hf_token,
36
+ use_fast=False,
37
+ trust_remote_code=True
38
+ )
39
+
40
+ tokenizer = AutoTokenizer.from_pretrained(
41
+ model_path,
42
+ token=hf_token,
43
+ trust_remote_code=True
44
+ )
45
 
46
+ if tokenizer.pad_token is None:
47
+ tokenizer.pad_token = tokenizer.eos_token
48
+
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  return tokenizer
50
 
51
  except Exception as e:
 
52
  st.error(f"Error loading tokenizer for {model_path}: {e}")
53
  return None
54