f64 commited on
Commit ·
5d61439
1
Parent(s): 84fa5b9
- my_logging_print.py +6 -0
- my_static_tools.py +8 -7
- pages/2_TECT_IDXYZ.py +1 -1
- pages/4_Загрузка Просмотр CSV.py +1 -1
- pages/6_Chat.py +1 -1
- pages/old/_просмотр_CSV.py +1 -1
- requirements.txt +2 -1
my_logging_print.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
class logging:
|
| 2 |
+
info = print
|
| 3 |
+
warning = print
|
| 4 |
+
debug = print
|
| 5 |
+
error = print
|
| 6 |
+
critical = print
|
my_static_tools.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
from typing import Union, NamedTuple
|
| 2 |
-
|
|
|
|
| 3 |
import numpy as np, pandas as pd
|
| 4 |
import plotly.express as px
|
| 5 |
import huggingface_hub
|
|
@@ -47,14 +48,14 @@ def pandas_random_dataframe(n_cols:int = 15, n_rows:int = 100) -> pd.DataFrame:
|
|
| 47 |
class HfRepo(NamedTuple):
|
| 48 |
repo_id: str
|
| 49 |
repo_type: str
|
| 50 |
-
|
| 51 |
|
| 52 |
class HF_tools:
|
| 53 |
""" Huggingface tools """
|
| 54 |
|
| 55 |
-
def list_models_spaces(
|
| 56 |
""" list models and spaces """
|
| 57 |
-
api = huggingface_hub.HfApi(token=
|
| 58 |
#spaces = api.list_spaces(author=author)
|
| 59 |
models = api.list_models(author=author)
|
| 60 |
datasets = api.list_datasets(author=author)
|
|
@@ -68,7 +69,7 @@ class HF_tools:
|
|
| 68 |
local_filename = os.path.join(".temp", new_filename)
|
| 69 |
#df.to_csv('compressed_data.zip', index=False, compression={'method': 'zip', 'archive_name': 'data.csv'})
|
| 70 |
dfToSave.to_csv(local_filename, index=False, sep=";", encoding="utf-8") # , compression="zip"
|
| 71 |
-
apiHF = huggingface_hub.HfApi(token=repo.
|
| 72 |
path_in_repo = os.path.basename(local_filename)
|
| 73 |
if remote_subdir:
|
| 74 |
path_in_repo = f"{remote_subdir}/{path_in_repo}"
|
|
@@ -82,7 +83,7 @@ class HF_tools:
|
|
| 82 |
#https://huggingface.co/datasets/f64k/gaziev/blob/main/TestData3_2204_noAB_gaziev.zip
|
| 83 |
dict_res = {}
|
| 84 |
for fl_name in lstCsvFiles:
|
| 85 |
-
try: file_loaded = huggingface_hub.hf_hub_download(filename=fl_name, repo_id=repo.repo_id, repo_type=repo.repo_type, token=repo.
|
| 86 |
except: file_loaded = ""
|
| 87 |
if os.path.exists(file_loaded):
|
| 88 |
compress = "zip" if file_loaded.lower().endswith("zip") else None
|
|
@@ -93,7 +94,7 @@ class HF_tools:
|
|
| 93 |
def list_files_hf(repo: HfRepo) -> list[str]:
|
| 94 |
""" List CSV and ZIP files in HF repo - список CSV и ZIP файлов (c уровнем вложенности) в репозитории """
|
| 95 |
### https://huggingface.co/docs/huggingface_hub/en/guides/hf_file_system
|
| 96 |
-
fs = huggingface_hub.HfFileSystem(token=repo.
|
| 97 |
path_hf = f"{repo.repo_type}s/{repo.repo_id}/"
|
| 98 |
#lst = fs.ls(path_hf, detail=False)
|
| 99 |
lstGlob = fs.glob(path_hf + "**") # map(os.path.basename, lstGlob)
|
|
|
|
| 1 |
from typing import Union, NamedTuple
|
| 2 |
+
from collections.abc import Callable
|
| 3 |
+
import io,os,re,sys,math,time,uuid,ctypes,pickle,random,shutil,string,urllib,decimal,datetime,itertools,traceback,collections,statistics
|
| 4 |
import numpy as np, pandas as pd
|
| 5 |
import plotly.express as px
|
| 6 |
import huggingface_hub
|
|
|
|
| 48 |
class HfRepo(NamedTuple):
|
| 49 |
repo_id: str
|
| 50 |
repo_type: str
|
| 51 |
+
get_token: Callable[[], str]
|
| 52 |
|
| 53 |
class HF_tools:
|
| 54 |
""" Huggingface tools """
|
| 55 |
|
| 56 |
+
def list_models_spaces(get_token: Callable[[], str], author = 'f64k'):
|
| 57 |
""" list models and spaces """
|
| 58 |
+
api = huggingface_hub.HfApi(token=get_token()) #
|
| 59 |
#spaces = api.list_spaces(author=author)
|
| 60 |
models = api.list_models(author=author)
|
| 61 |
datasets = api.list_datasets(author=author)
|
|
|
|
| 69 |
local_filename = os.path.join(".temp", new_filename)
|
| 70 |
#df.to_csv('compressed_data.zip', index=False, compression={'method': 'zip', 'archive_name': 'data.csv'})
|
| 71 |
dfToSave.to_csv(local_filename, index=False, sep=";", encoding="utf-8") # , compression="zip"
|
| 72 |
+
apiHF = huggingface_hub.HfApi(token=repo.get_token()) # os.getenv(repo.env_token)
|
| 73 |
path_in_repo = os.path.basename(local_filename)
|
| 74 |
if remote_subdir:
|
| 75 |
path_in_repo = f"{remote_subdir}/{path_in_repo}"
|
|
|
|
| 83 |
#https://huggingface.co/datasets/f64k/gaziev/blob/main/TestData3_2204_noAB_gaziev.zip
|
| 84 |
dict_res = {}
|
| 85 |
for fl_name in lstCsvFiles:
|
| 86 |
+
try: file_loaded = huggingface_hub.hf_hub_download(filename=fl_name, repo_id=repo.repo_id, repo_type=repo.repo_type, token=repo.get_token())
|
| 87 |
except: file_loaded = ""
|
| 88 |
if os.path.exists(file_loaded):
|
| 89 |
compress = "zip" if file_loaded.lower().endswith("zip") else None
|
|
|
|
| 94 |
def list_files_hf(repo: HfRepo) -> list[str]:
|
| 95 |
""" List CSV and ZIP files in HF repo - список CSV и ZIP файлов (c уровнем вложенности) в репозитории """
|
| 96 |
### https://huggingface.co/docs/huggingface_hub/en/guides/hf_file_system
|
| 97 |
+
fs = huggingface_hub.HfFileSystem(token=repo.get_token(), use_listings_cache=False) # , skip_instance_cache=True
|
| 98 |
path_hf = f"{repo.repo_type}s/{repo.repo_id}/"
|
| 99 |
#lst = fs.ls(path_hf, detail=False)
|
| 100 |
lstGlob = fs.glob(path_hf + "**") # map(os.path.basename, lstGlob)
|
pages/2_TECT_IDXYZ.py
CHANGED
|
@@ -27,7 +27,7 @@ if True:
|
|
| 27 |
#st.sidebar.write(dirParams)
|
| 28 |
|
| 29 |
|
| 30 |
-
REPO = mst.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
|
| 31 |
lstRepoFiles = hft.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
|
| 32 |
dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
|
| 33 |
|
|
|
|
| 27 |
#st.sidebar.write(dirParams)
|
| 28 |
|
| 29 |
|
| 30 |
+
REPO = mst.HfRepo("f64k/gaziev", "dataset", lambda: st.secrets["HF_WRITE"])
|
| 31 |
lstRepoFiles = hft.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
|
| 32 |
dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
|
| 33 |
|
pages/4_Загрузка Просмотр CSV.py
CHANGED
|
@@ -9,7 +9,7 @@ from my_static_tools import XYZV_tools as xvt
|
|
| 9 |
st.set_page_config(page_title="Загрузка + Просмотр", page_icon="💾", layout="wide", initial_sidebar_state="expanded")
|
| 10 |
|
| 11 |
st.html(mst.STYLE_CORRECTION)
|
| 12 |
-
REPO = mst.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
|
| 13 |
|
| 14 |
st.sidebar.markdown("# 💾 Загрузка")
|
| 15 |
st.sidebar.markdown("# 🧊 Просмотр")
|
|
|
|
| 9 |
st.set_page_config(page_title="Загрузка + Просмотр", page_icon="💾", layout="wide", initial_sidebar_state="expanded")
|
| 10 |
|
| 11 |
st.html(mst.STYLE_CORRECTION)
|
| 12 |
+
REPO = mst.HfRepo("f64k/gaziev", "dataset", lambda: st.secrets["HF_WRITE"])
|
| 13 |
|
| 14 |
st.sidebar.markdown("# 💾 Загрузка")
|
| 15 |
st.sidebar.markdown("# 🧊 Просмотр")
|
pages/6_Chat.py
CHANGED
|
@@ -21,7 +21,7 @@ st.sidebar.markdown("# Переговоры 💬")
|
|
| 21 |
#st.write(conn)
|
| 22 |
#st.help(conn)
|
| 23 |
|
| 24 |
-
REPO = mst.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
|
| 25 |
lstRepoFiles = hft.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
|
| 26 |
dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
|
| 27 |
|
|
|
|
| 21 |
#st.write(conn)
|
| 22 |
#st.help(conn)
|
| 23 |
|
| 24 |
+
REPO = mst.HfRepo("f64k/gaziev", "dataset", lambda: st.secrets["HF_WRITE"])
|
| 25 |
lstRepoFiles = hft.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
|
| 26 |
dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
|
| 27 |
|
pages/old/_просмотр_CSV.py
CHANGED
|
@@ -10,7 +10,7 @@ st.html(mst.STYLE_CORRECTION)
|
|
| 10 |
#col1.metric("Temperature", "70 °F", "1.2 °F") #col2.metric("Wind", "9 mph", "-8%") #col3.metric("Humidity", "86%", "4%")
|
| 11 |
|
| 12 |
|
| 13 |
-
REPO = mst.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
|
| 14 |
st.sidebar.info("🧊💽 сохраненные\n таблицы CSV")
|
| 15 |
|
| 16 |
lstTestFiles = [f for f in hft.list_files_hf(REPO) if not f.upper().startswith("ID_XYZ/")]
|
|
|
|
| 10 |
#col1.metric("Temperature", "70 °F", "1.2 °F") #col2.metric("Wind", "9 mph", "-8%") #col3.metric("Humidity", "86%", "4%")
|
| 11 |
|
| 12 |
|
| 13 |
+
REPO = mst.HfRepo("f64k/gaziev", "dataset", lambda: st.secrets["HF_WRITE"])
|
| 14 |
st.sidebar.info("🧊💽 сохраненные\n таблицы CSV")
|
| 15 |
|
| 16 |
lstTestFiles = [f for f in hft.list_files_hf(REPO) if not f.upper().startswith("ID_XYZ/")]
|
requirements.txt
CHANGED
|
@@ -7,9 +7,10 @@
|
|
| 7 |
streamlit # https://libraries.io/pypi/streamlit/ # HAS pandas numpy
|
| 8 |
#pandas
|
| 9 |
#numpy
|
| 10 |
-
ipywidgets # https://libraries.io/pypi/ipywidgets/ # HAS ipykernel
|
| 11 |
huggingface_hub
|
| 12 |
scikit-learn
|
| 13 |
plotly
|
| 14 |
psutil
|
|
|
|
|
|
|
| 15 |
faker
|
|
|
|
| 7 |
streamlit # https://libraries.io/pypi/streamlit/ # HAS pandas numpy
|
| 8 |
#pandas
|
| 9 |
#numpy
|
|
|
|
| 10 |
huggingface_hub
|
| 11 |
scikit-learn
|
| 12 |
plotly
|
| 13 |
psutil
|
| 14 |
+
ipykernel # https://libraries.io/pypi/ipykernel/
|
| 15 |
+
ipywidgets # https://libraries.io/pypi/ipywidgets/
|
| 16 |
faker
|