1

my_static_methods.py

@@ -78,7 +78,7 @@ def load_dataframes_from_hf(repo: HfRepo, lstCsvFiles: list[str] = []) -> {str,
         if os.path.exists(file_loaded):
             compress = "zip" if file_loaded.lower().endswith("zip") else None
             df_loaded = pd.read_csv(file_loaded, sep=";", encoding = "utf-8", compression=compress)
-            dict_res[fl_name] = df_process_v_column(df_loaded)
+            dict_res[fl_name] = df_loaded # df_Vproc = df_process_v_column(df_loaded)
     return dict_res
 
 ### список CSV и ZIP файлов (c уровнем вложенности) в репозитории

pages/4_Просмотр_CSV.py

@@ -4,7 +4,6 @@ import my_static_methods as my_stm
 st.html(my_stm.STYLE_CORRECTION)
 
 #st.sidebar.title("⚜️")
-#st.sidebar.info("три CSV")
 
 #col1.metric("Temperature", "70 °F", "1.2 °F") #col2.metric("Wind", "9 mph", "-8%") #col3.metric("Humidity", "86%", "4%")
 
@@ -12,8 +11,7 @@ st.html(my_stm.STYLE_CORRECTION)
 REPO = my_stm.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
 st.sidebar.info("🧊💽 сохраненные\n таблицы CSV")
 
-lstTestFiles = my_stm.list_files_hf(REPO)
-#dictXYZV = my_stm.load_gaziev_from_hf(REPO, lstTestFiles)
+lstTestFiles = [f for f in my_stm.list_files_hf(REPO) if not f.upper().startswith("ID_XYZ/")]
 
 cols = st.columns(2)
 
@@ -23,11 +21,12 @@ if one_file_selected:
     dict_ONE_XYZV = my_stm.load_dataframes_from_hf(REPO, [one_file_selected])
     if len(dict_ONE_XYZV) > 0:
         df_xyz = list(dict_ONE_XYZV.values())[0] #df_xyz = dictXYZV[key_xyz]
-        fig = my_stm.plotly_xyzv_scatter_gray(df_xyz)
+        df_xyz_vproc = my_stm.df_process_v_column(df_xyz)
+        fig = my_stm.plotly_xyzv_scatter_gray(df_xyz_vproc)
         cols[0].plotly_chart(fig) # st.plotly_chart(fig) #fig.show()
         lstValues = sorted(set(df_xyz.notna().sum()), reverse=True)
-        cols[1].info(f"Всего записей : {lstValues[0]};  Значений V : {lstValues[1]}")
-        cols[1].dataframe(df_xyz, height=700)
+        cols[1].info(f"Bceгo записей : {lstValues[0]};  Значений V : {lstValues[1]}")
+        cols[1].dataframe(df_xyz_vproc, height=700)
 
 
 

pages/6_TECT_7_ID.py

@@ -2,7 +2,7 @@ import os, re, sys, time, math, shutil, urllib, string, random, pickle, zipfile,
 import streamlit as st, pandas as pd, numpy as np
 import my_static_methods as my_stm
 
-st.html(my_stm.STYLE_CORRECTION)
+# st.html(my_stm.STYLE_CORRECTION)
 st.sidebar.markdown("🧊 проверка по пакетам XYZ")
 
 def ReRun():
@@ -23,7 +23,7 @@ def save_dataframe_nodialog_idxyz(new_filename, dfToSave):
 
 REPO = my_stm.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
 lstRepoFiles = my_stm.list_files_hf(REPO)
-lstTestFilesIdXyz = [f.upper().replace("ID_XYZ/","") for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")]
+dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
 
 col1, col2 = st.columns([1,1])
 with col1.container():
@@ -46,10 +46,12 @@ with col1.container():
                     st.error(uploaded_file.type)
             # dataframe ready. try to upload to HF
             if not dfLoaded is None:
-                dfToUpload = dfLoaded.query("ID!='ID'")
+                dfToUpload = dfLoaded
+                if "ID" in dfToUpload.columns:
+                    dfToUpload = dfLoaded.query("ID!='ID'")
                 #col2.dataframe(df)
                 colnames = "".join(dfToUpload.columns)
-                if colnames.lower().startswith("idxyz"):
+                if colnames.upper().startswith("IDXYZ"):
                     dgID = dfToUpload.groupby("ID")
                     dictGroupID = dict(list(dgID))
                     lstGroupIDs = list(dictGroupID.keys())
@@ -64,8 +66,20 @@ with col1.container():
                             save_dataframe_nodialog_idxyz(fileXYZ, dfToUpload)
                     else:
                         st.error(f"Разные размеры пакетов для разных ID, варианты : {lst_len}")
+                else:
+                    st.error(f"Столбцы не ID;X;Y;Z ! Наблюдаем столбцы : {colnames}")
     # список уже имеющихся файлов
-    st.write(lstTestFilesIdXyz)
-#with col1.container():
+    selectedFile = st.radio("# загруженные тестовые пакеты", dictTestFilesIdXyz.keys(), index=None)
+    if selectedFile is not None:
+        dict_ONE_IDXYZ = my_stm.load_dataframes_from_hf(REPO, [dictTestFilesIdXyz[selectedFile]])
+        if len(dict_ONE_IDXYZ) > 0:
+            df_idxyz = list(dict_ONE_IDXYZ.values())[0]
+            dfShow = df_idxyz
+            dgID = df_idxyz.groupby("ID")
+            dictGroupID = dict(list(dgID))
+            dfShow = dgID.apply(len) #.reset_index()
+            col2.dataframe(dfShow, height=700)
+
+
 
 

static/test.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -16,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -25,7 +25,7 @@
      "text": [
       "<>:1: SyntaxWarning: invalid escape sequence '\\M'\n",
       "<>:1: SyntaxWarning: invalid escape sequence '\\M'\n",
-      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_38792\\1255343956.py:1: SyntaxWarning: invalid escape sequence '\\M'\n",
+      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_14236\\1255343956.py:1: SyntaxWarning: invalid escape sequence '\\M'\n",
       "  path1 = \"N:\\Makarov\\Development\\Python\\Jupiter Notebooks\\Gaziev CSV\\TestData_1504_AB_soloV_gaziev.zip\"\n"
      ]
     },
@@ -228,7 +228,7 @@
        "[12010 rows x 10 columns]"
       ]
      },
-     "execution_count": 3,
+     "execution_count": 2,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -249,7 +249,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -270,7 +270,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -437,7 +437,7 @@
        "18  AAA011113    21  392  -205"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -456,7 +456,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -470,40 +470,167 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_38792\\1806989190.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
+      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_14236\\117446433.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
       "  print(f\"{set(df2.groupby(\"ID\").apply(len))=}\")\n",
-      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_38792\\1806989190.py:4: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
-      "  dgID.apply(len)\n"
+      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_14236\\117446433.py:4: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
+      "  dgID.apply(len).reset_index()\n"
      ]
     },
     {
      "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ID</th>\n",
+       "      <th>0</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>AAA011111</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>AAA011112</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>AAA011113</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>AAA011114</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>AAA011115</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>AAA011116</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>AAA011117</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>AAA011118</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>BBB011111</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>BBB011112</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>BBB011113</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>BBB011114</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>BBB011115</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>BBB011116</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>BBB011117</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>CCC011111</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>CCC011112</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>DDD011111</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>DDD011112</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>DDD011113</td>\n",
+       "      <td>7</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
       "text/plain": [
-       "ID\n",
-       "AAA011111    7\n",
-       "AAA011112    7\n",
-       "AAA011113    7\n",
-       "AAA011114    7\n",
-       "AAA011115    7\n",
-       "AAA011116    7\n",
-       "AAA011117    7\n",
-       "AAA011118    7\n",
-       "BBB011111    7\n",
-       "BBB011112    7\n",
-       "BBB011113    7\n",
-       "BBB011114    7\n",
-       "BBB011115    7\n",
-       "BBB011116    7\n",
-       "BBB011117    7\n",
-       "CCC011111    7\n",
-       "CCC011112    7\n",
-       "DDD011111    7\n",
-       "DDD011112    7\n",
-       "DDD011113    7\n",
-       "dtype: int64"
+       "           ID  0\n",
+       "0   AAA011111  7\n",
+       "1   AAA011112  7\n",
+       "2   AAA011113  7\n",
+       "3   AAA011114  7\n",
+       "4   AAA011115  7\n",
+       "5   AAA011116  7\n",
+       "6   AAA011117  7\n",
+       "7   AAA011118  7\n",
+       "8   BBB011111  7\n",
+       "9   BBB011112  7\n",
+       "10  BBB011113  7\n",
+       "11  BBB011114  7\n",
+       "12  BBB011115  7\n",
+       "13  BBB011116  7\n",
+       "14  BBB011117  7\n",
+       "15  CCC011111  7\n",
+       "16  CCC011112  7\n",
+       "17  DDD011111  7\n",
+       "18  DDD011112  7\n",
+       "19  DDD011113  7"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -512,12 +639,12 @@
     "print(f\"{set(df2.groupby(\"ID\").apply(len))=}\")\n",
     "dgID = df2.groupby(\"ID\")\n",
     "dictGroupID = dict(list(dgID))\n",
-    "dgID.apply(len)"
+    "dgID.apply(len).reset_index()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -620,7 +747,7 @@
        "79  BBB011112  -210  529  -943"
       ]
      },
-     "execution_count": 30,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -637,14 +764,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_38792\\572365488.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
+      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_14236\\572365488.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
       "  dgID.apply(lambda df: df[xyz].values.reshape(1,21)).reset_index()\n"
      ]
     },
@@ -802,7 +929,7 @@
        "19  DDD011113  [[-220, 487, 741, 22, 516, 433, -123, 478, 234..."
       ]
      },
-     "execution_count": 34,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }