f64 commited on
Commit
5d61439
·
1 Parent(s): 84fa5b9
my_logging_print.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ class logging:
2
+ info = print
3
+ warning = print
4
+ debug = print
5
+ error = print
6
+ critical = print
my_static_tools.py CHANGED
@@ -1,5 +1,6 @@
1
  from typing import Union, NamedTuple
2
- import io,os,re,sys,math,time,uuid,ctypes,pickle,random,shutil,string,urllib,decimal,datetime,itertools,traceback,collections #,statistics
 
3
  import numpy as np, pandas as pd
4
  import plotly.express as px
5
  import huggingface_hub
@@ -47,14 +48,14 @@ def pandas_random_dataframe(n_cols:int = 15, n_rows:int = 100) -> pd.DataFrame:
47
  class HfRepo(NamedTuple):
48
  repo_id: str
49
  repo_type: str
50
- token: str
51
 
52
  class HF_tools:
53
  """ Huggingface tools """
54
 
55
- def list_models_spaces(token: str, author = 'f64k'):
56
  """ list models and spaces """
57
- api = huggingface_hub.HfApi(token=token)
58
  #spaces = api.list_spaces(author=author)
59
  models = api.list_models(author=author)
60
  datasets = api.list_datasets(author=author)
@@ -68,7 +69,7 @@ class HF_tools:
68
  local_filename = os.path.join(".temp", new_filename)
69
  #df.to_csv('compressed_data.zip', index=False, compression={'method': 'zip', 'archive_name': 'data.csv'})
70
  dfToSave.to_csv(local_filename, index=False, sep=";", encoding="utf-8") # , compression="zip"
71
- apiHF = huggingface_hub.HfApi(token=repo.token)
72
  path_in_repo = os.path.basename(local_filename)
73
  if remote_subdir:
74
  path_in_repo = f"{remote_subdir}/{path_in_repo}"
@@ -82,7 +83,7 @@ class HF_tools:
82
  #https://huggingface.co/datasets/f64k/gaziev/blob/main/TestData3_2204_noAB_gaziev.zip
83
  dict_res = {}
84
  for fl_name in lstCsvFiles:
85
- try: file_loaded = huggingface_hub.hf_hub_download(filename=fl_name, repo_id=repo.repo_id, repo_type=repo.repo_type, token=repo.token)
86
  except: file_loaded = ""
87
  if os.path.exists(file_loaded):
88
  compress = "zip" if file_loaded.lower().endswith("zip") else None
@@ -93,7 +94,7 @@ class HF_tools:
93
  def list_files_hf(repo: HfRepo) -> list[str]:
94
  """ List CSV and ZIP files in HF repo - список CSV и ZIP файлов (c уровнем вложенности) в репозитории """
95
  ### https://huggingface.co/docs/huggingface_hub/en/guides/hf_file_system
96
- fs = huggingface_hub.HfFileSystem(token=repo.token, use_listings_cache=False) # , skip_instance_cache=True
97
  path_hf = f"{repo.repo_type}s/{repo.repo_id}/"
98
  #lst = fs.ls(path_hf, detail=False)
99
  lstGlob = fs.glob(path_hf + "**") # map(os.path.basename, lstGlob)
 
1
  from typing import Union, NamedTuple
2
+ from collections.abc import Callable
3
+ import io,os,re,sys,math,time,uuid,ctypes,pickle,random,shutil,string,urllib,decimal,datetime,itertools,traceback,collections,statistics
4
  import numpy as np, pandas as pd
5
  import plotly.express as px
6
  import huggingface_hub
 
48
  class HfRepo(NamedTuple):
49
  repo_id: str
50
  repo_type: str
51
+ get_token: Callable[[], str]
52
 
53
  class HF_tools:
54
  """ Huggingface tools """
55
 
56
+ def list_models_spaces(get_token: Callable[[], str], author = 'f64k'):
57
  """ list models and spaces """
58
+ api = huggingface_hub.HfApi(token=get_token()) #
59
  #spaces = api.list_spaces(author=author)
60
  models = api.list_models(author=author)
61
  datasets = api.list_datasets(author=author)
 
69
  local_filename = os.path.join(".temp", new_filename)
70
  #df.to_csv('compressed_data.zip', index=False, compression={'method': 'zip', 'archive_name': 'data.csv'})
71
  dfToSave.to_csv(local_filename, index=False, sep=";", encoding="utf-8") # , compression="zip"
72
+ apiHF = huggingface_hub.HfApi(token=repo.get_token()) # os.getenv(repo.env_token)
73
  path_in_repo = os.path.basename(local_filename)
74
  if remote_subdir:
75
  path_in_repo = f"{remote_subdir}/{path_in_repo}"
 
83
  #https://huggingface.co/datasets/f64k/gaziev/blob/main/TestData3_2204_noAB_gaziev.zip
84
  dict_res = {}
85
  for fl_name in lstCsvFiles:
86
+ try: file_loaded = huggingface_hub.hf_hub_download(filename=fl_name, repo_id=repo.repo_id, repo_type=repo.repo_type, token=repo.get_token())
87
  except: file_loaded = ""
88
  if os.path.exists(file_loaded):
89
  compress = "zip" if file_loaded.lower().endswith("zip") else None
 
94
  def list_files_hf(repo: HfRepo) -> list[str]:
95
  """ List CSV and ZIP files in HF repo - список CSV и ZIP файлов (c уровнем вложенности) в репозитории """
96
  ### https://huggingface.co/docs/huggingface_hub/en/guides/hf_file_system
97
+ fs = huggingface_hub.HfFileSystem(token=repo.get_token(), use_listings_cache=False) # , skip_instance_cache=True
98
  path_hf = f"{repo.repo_type}s/{repo.repo_id}/"
99
  #lst = fs.ls(path_hf, detail=False)
100
  lstGlob = fs.glob(path_hf + "**") # map(os.path.basename, lstGlob)
pages/2_TECT_IDXYZ.py CHANGED
@@ -27,7 +27,7 @@ if True:
27
  #st.sidebar.write(dirParams)
28
 
29
 
30
- REPO = mst.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
31
  lstRepoFiles = hft.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
32
  dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
33
 
 
27
  #st.sidebar.write(dirParams)
28
 
29
 
30
+ REPO = mst.HfRepo("f64k/gaziev", "dataset", lambda: st.secrets["HF_WRITE"])
31
  lstRepoFiles = hft.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
32
  dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
33
 
pages/4_Загрузка Просмотр CSV.py CHANGED
@@ -9,7 +9,7 @@ from my_static_tools import XYZV_tools as xvt
9
  st.set_page_config(page_title="Загрузка + Просмотр", page_icon="💾", layout="wide", initial_sidebar_state="expanded")
10
 
11
  st.html(mst.STYLE_CORRECTION)
12
- REPO = mst.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
13
 
14
  st.sidebar.markdown("# 💾 Загрузка")
15
  st.sidebar.markdown("# 🧊 Просмотр")
 
9
  st.set_page_config(page_title="Загрузка + Просмотр", page_icon="💾", layout="wide", initial_sidebar_state="expanded")
10
 
11
  st.html(mst.STYLE_CORRECTION)
12
+ REPO = mst.HfRepo("f64k/gaziev", "dataset", lambda: st.secrets["HF_WRITE"])
13
 
14
  st.sidebar.markdown("# 💾 Загрузка")
15
  st.sidebar.markdown("# 🧊 Просмотр")
pages/6_Chat.py CHANGED
@@ -21,7 +21,7 @@ st.sidebar.markdown("# Переговоры 💬")
21
  #st.write(conn)
22
  #st.help(conn)
23
 
24
- REPO = mst.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
25
  lstRepoFiles = hft.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
26
  dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
27
 
 
21
  #st.write(conn)
22
  #st.help(conn)
23
 
24
+ REPO = mst.HfRepo("f64k/gaziev", "dataset", lambda: st.secrets["HF_WRITE"])
25
  lstRepoFiles = hft.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
26
  dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
27
 
pages/old/_просмотр_CSV.py CHANGED
@@ -10,7 +10,7 @@ st.html(mst.STYLE_CORRECTION)
10
  #col1.metric("Temperature", "70 °F", "1.2 °F") #col2.metric("Wind", "9 mph", "-8%") #col3.metric("Humidity", "86%", "4%")
11
 
12
 
13
- REPO = mst.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
14
  st.sidebar.info("🧊💽 сохраненные\n таблицы CSV")
15
 
16
  lstTestFiles = [f for f in hft.list_files_hf(REPO) if not f.upper().startswith("ID_XYZ/")]
 
10
  #col1.metric("Temperature", "70 °F", "1.2 °F") #col2.metric("Wind", "9 mph", "-8%") #col3.metric("Humidity", "86%", "4%")
11
 
12
 
13
+ REPO = mst.HfRepo("f64k/gaziev", "dataset", lambda: st.secrets["HF_WRITE"])
14
  st.sidebar.info("🧊💽 сохраненные\n таблицы CSV")
15
 
16
  lstTestFiles = [f for f in hft.list_files_hf(REPO) if not f.upper().startswith("ID_XYZ/")]
requirements.txt CHANGED
@@ -7,9 +7,10 @@
7
  streamlit # https://libraries.io/pypi/streamlit/ # HAS pandas numpy
8
  #pandas
9
  #numpy
10
- ipywidgets # https://libraries.io/pypi/ipywidgets/ # HAS ipykernel
11
  huggingface_hub
12
  scikit-learn
13
  plotly
14
  psutil
 
 
15
  faker
 
7
  streamlit # https://libraries.io/pypi/streamlit/ # HAS pandas numpy
8
  #pandas
9
  #numpy
 
10
  huggingface_hub
11
  scikit-learn
12
  plotly
13
  psutil
14
+ ipykernel # https://libraries.io/pypi/ipykernel/
15
+ ipywidgets # https://libraries.io/pypi/ipywidgets/
16
  faker