1

my_static_methods.py

@@ -1,9 +1,13 @@
 from typing import Union, NamedTuple
-import os, io
+import io,os,re,sys,math,time,uuid,ctypes,pickle,random,shutil,string,urllib,decimal,datetime,itertools,traceback,collections,statistics
 import numpy as np, pandas as pd
 import plotly.express as px
 import huggingface_hub
 
+import sklearn #, statsmodels
+from sklearn import svm, neighbors, naive_bayes, neural_network, tree, ensemble, linear_model, discriminant_analysis, gaussian_process, manifold, cluster
+#from statsmodels.tsa import seasonal
+
 os.makedirs(".temp", exist_ok=True) # for temporary local files
 
 class HfRepo(NamedTuple):
@@ -105,11 +109,58 @@ def plotly_xyzv_scatter_gray(df3D):
     return fig
 
 
-
-
-
-
-
+# lstRepoZipFiles = ["TrainData_1504_AB_gaziev.zip","TestData_1504_AB_gaziev.zip","TestData3_2204_noAB_gaziev.zip"]
+### returns (classifier_object, df_train_with_predict, time_elapsed)
+def GetClassifier(lstDfOriginal, nHystorySteps) :
+    #lstDfOriginal = [df_9125_Train, df_12010_Test, df_9051_Test3]
+    nShift = nHystorySteps
+    nCurrShift = nHystorySteps
+    classifierName = "DecisionTreeClassifier"
+    colsVectorInp = ["X","Y","Z"]
+    fieldY = "Vis" #
+    lstDataFrames = MakeHystoryColumns(lstDfOriginal, nShift)
+    df_train = pd.concat(lstDataFrames)
+    lstColsShift = [f"{c}-{i}" for i in range(1, nCurrShift+1) for c in colsVectorInp] # для nCurrShift=0 lstColsShift=[]
+    colsVectorInpAll = colsVectorInp + lstColsShift
+    y_train = df_train[fieldY]
+    x_train_vect = df_train[colsVectorInpAll]
+    dictClassifiers = createDictClassifiers_BestForXYZ()
+    classifierObject = dictClassifiers[classifierName]
+    start2 = time.time()
+    classifierObject.fit(x_train_vect, y_train) # процесс обучения
+    time_elapsed = time.time() - start2
+    y_pred = classifierObject.predict(x_train_vect)
+    df_train[f"predict_{fieldY}"] = y_pred
+    print(f"{time_elapsed=}")
+    return (classifierObject, df_train, time_elapsed)
+
+#
+def MakeHystoryColumns(lstDfOriginal, nShift) :
+    lstDataframesShifted = [df.copy() for df in lstDfOriginal]
+    lstColsShift = []
+    for i in range(1, nShift+1):
+        #cols = ["X","Y","Z"]+["A","B"]
+        cols = ["X","Y","Z"]
+        #cols = ["A","B"]
+        for c in cols:
+            for dfShift in lstDataframesShifted:
+                dfShift[f'{c}-{i}'] = dfShift[c].shift(i).fillna(0)
+            lstColsShift.append(lstDataframesShifted[0].columns[-1])
+    print(lstColsShift)
+    return lstDataframesShifted
+
+RANDOM_STATE=11
+
+def createDictClassifiers_BestForXYZ() :
+    dictFastTree = {
+        #"RandomForestClassifier": ensemble.RandomForestClassifier(random_state=RANDOM_STATE), # совсем плохие показатели
+        #"ExtraTreeClassifier": tree.ExtraTreeClassifier(random_state=RANDOM_STATE), #
+        "DecisionTreeClassifier": tree.DecisionTreeClassifier(random_state=RANDOM_STATE), # лучший по последним баллам
+    }
+    #return {**dictFast}
+    #return {**dict_Test_MLPClassifier}
+    #return {**dictFast, **dictLongTrain}
+    return {**dictFastTree}
 
 
 

pages/6_TECT_7_ID.py

@@ -3,18 +3,28 @@ import streamlit as st, pandas as pd, numpy as np
 import my_static_methods as my_stm
 
 st.html(my_stm.STYLE_CORRECTION)
-#st.sidebar.markdown("🧊 проверка по пакетам XYZ")
-st.info("🧊 проверка предсказаний по пакетам XYZ")
 
-def ReRun():
+REPO = my_stm.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
+lstRepoFiles = my_stm.list_files_hf(REPO) # список уже имеющихся в репозитории файлов
+dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
+
+@st.cache_data
+def GetListOf_XYZV_ToTrainClassifier(repo):
+    lstRepoZipFiles = ["TrainData_1504_AB_gaziev.zip","TestData_1504_AB_gaziev.zip","TestData3_2204_noAB_gaziev.zip"]
+    dictTrainThreeDataframes = my_stm.load_dataframes_from_hf(REPO, lstRepoZipFiles)
+    lstDfOriginal = [my_stm.df_process_v_column(df) for df in dictTrainThreeDataframes.values()]
+    return lstDfOriginal
+
+def ReRun() :
     try: st.rerun()
     except: pass
 
 def DescriptionMarkdown() -> str:
     return """
         ## Описание
-        # 1) Загрузка нового файла
+        ### 1) Загрузка нового файла
         Источником данных является файл CSV
+        Первая строка - названия столбцов ID;X;Y;Z
     """
 
 def save_dataframe_nodialog_idxyz(new_filename, dfToSave):
@@ -22,66 +32,84 @@ def save_dataframe_nodialog_idxyz(new_filename, dfToSave):
     st.toast(commit_info, icon='🆕')
     ReRun()
 
+#st.sidebar.markdown("🧊 проверка по пакетам XYZ")
 
-REPO = my_stm.HfRepo("f64k/gaziev", "dataset", st.secrets["HF_WRITE"])
-lstRepoFiles = my_stm.list_files_hf(REPO)
-dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
+with st.container():
+    cols1 = st.columns([1,16]) # vertical_alignment: "center"
+    cols1[0].popover("❓", help="пояснения").markdown(DescriptionMarkdown())
+    cols1[1].info("🔮 проверка предсказаний по пакетам ID_XYZ. 📜 формат CSV. 🧊 названия столбцов ID;X;Y;Z. 📐 размер пакетов одинаковый.")
+
+#col1, col2 = st.columns([2,5])
+col1, col2 = st.columns([3,2])
 
-col1, col2 = st.columns([2,1])
-with col1.container():
-    cont_cols = st.columns([1,2])
-    cont_cols[0].popover("❓").markdown(DescriptionMarkdown())
-    with cont_cols[1].popover("🆕 добавить новый файл"):
-        uploaded_file = st.file_uploader("💾 “откройте CSV для загрузки”", ["csv"])
-        if uploaded_file is not None:
-            dfLoaded = None
-            delim = ";"
-            enc = "utf-8"
-            if uploaded_file.type == "text/csv":
-                try: dfLoaded = pd.read_csv(uploaded_file, sep=delim, encoding=enc)
+with col1.popover("🆕 добавить новый файл", use_container_width=False):
+    uploaded_file = st.file_uploader("💾 “откройте CSV для загрузки”", ["csv"])
+    if uploaded_file is not None:
+        dfLoaded = None
+        delim = ";"
+        enc = "utf-8"
+        if uploaded_file.type == "text/csv":
+            try: dfLoaded = pd.read_csv(uploaded_file, sep=delim, encoding=enc)
+            except Exception as ex: st.write(ex)
+        else:
+            if uploaded_file.type == "application/x-zip-compressed":
+                try: dfLoaded = pd.read_csv(uploaded_file, sep=delim, encoding=enc, compression="zip")
                 except Exception as ex: st.write(ex)
             else:
-                if uploaded_file.type == "application/x-zip-compressed":
-                    try: dfLoaded = pd.read_csv(uploaded_file, sep=delim, encoding=enc, compression="zip")
-                    except Exception as ex: st.write(ex)
-                else:
-                    st.error(uploaded_file.type)
-            # dataframe ready. try to upload to HF
-            if not dfLoaded is None:
-                dfToUpload = dfLoaded
-                if "ID" in dfToUpload.columns:
-                    dfToUpload = dfLoaded.query("ID!='ID'")
-                #col2.dataframe(df)
-                colnames = "".join(dfToUpload.columns)
-                if colnames.upper().startswith("IDXYZ"):
-                    dgID = dfToUpload.groupby("ID")
-                    dictGroupID = dict(list(dgID))
-                    lstGroupIDs = list(dictGroupID.keys())
-                    #col2.write(dictGroupID)
-                    lst_len = list(set(dgID.apply(len)))
-                    if len(lst_len) == 1:
-                        fileXYZ = f"{colnames}_{len(dictGroupID)}x{lst_len[0]}_{lstGroupIDs[0]}_{lstGroupIDs[-1]}.csv".upper()
-                        if fileXYZ in dictTestFilesIdXyz.keys():
-                            if st.button(f"такой файл есть! перезаписать файл '{fileXYZ}'?"):
-                                save_dataframe_nodialog_idxyz(fileXYZ, dfToUpload)
-                        else:
+                st.error(uploaded_file.type)
+        # dataframe ready. try to upload to HF
+        if not dfLoaded is None:
+            dfToUpload = dfLoaded
+            if "ID" in dfToUpload.columns:
+                dfToUpload = dfLoaded.query("ID!='ID'")
+            #col2.dataframe(df)
+            colnames = "".join(dfToUpload.columns)
+            if colnames.upper().startswith("IDXYZ"):
+                dgID = dfToUpload.groupby("ID")
+                dictGroupID = dict(list(dgID))
+                lstGroupIDs = list(dictGroupID.keys())
+                #col2.write(dictGroupID)
+                lst_len = list(set(dgID.apply(len)))
+                if len(lst_len) == 1:
+                    fileXYZ = f"{colnames}_{len(dictGroupID)}x{lst_len[0]}_{lstGroupIDs[0]}_{lstGroupIDs[-1]}.csv".upper()
+                    if fileXYZ in dictTestFilesIdXyz.keys():
+                        if st.button(f"такой файл есть! перезаписать файл '{fileXYZ}'?"):
                             save_dataframe_nodialog_idxyz(fileXYZ, dfToUpload)
                     else:
-                        st.error(f"Разные размеры пакетов для разных ID, варианты : {lst_len}")
+                        save_dataframe_nodialog_idxyz(fileXYZ, dfToUpload)
                 else:
-                    st.error(f"Столбцы не ID;X;Y;Z ! Наблюдаем столбцы : {colnames}")
-    # список уже имеющихся файлов
-    selectedFile = st.radio("📰 загруженные тестовые пакеты", dictTestFilesIdXyz.keys(), index=None)
-    if selectedFile is not None:
-        dict_ONE_IDXYZ = my_stm.load_dataframes_from_hf(REPO, [dictTestFilesIdXyz[selectedFile]])
-        if len(dict_ONE_IDXYZ) > 0:
-            df_idxyz = list(dict_ONE_IDXYZ.values())[0]
-            dfShow = df_idxyz
-            dgID = df_idxyz.groupby("ID")
-            dictGroupID = dict(list(dgID))
-            dfShow = dgID.apply(len) #.reset_index()
-            col2.dataframe(dfShow, height=700)
+                    st.error(f"Разные размеры пакетов для разных ID, варианты : {lst_len}")
+            else:
+                st.error(f"Столбцы не ID;X;Y;Z ! Наблюдаем столбцы : {colnames}")
+
+# список уже имеющихся в репозитории файлов
+lstRepoFiles = my_stm.list_files_hf(REPO)
+dictTestFilesIdXyz = {f.upper().replace("ID_XYZ/",""): f.upper() for f in lstRepoFiles if f.upper().startswith("ID_XYZ/")}
+selectedFile = col1.radio("📰 загруженные тестовые пакеты", dictTestFilesIdXyz.keys(), index=None)
 
+# выбран файл для предсказания
+if selectedFile is not None:
+    dict_ONE_IDXYZ = my_stm.load_dataframes_from_hf(REPO, [dictTestFilesIdXyz[selectedFile]])
+    if len(dict_ONE_IDXYZ) > 0:
+        df_idxyz = list(dict_ONE_IDXYZ.values())[0]
+        dfShow = df_idxyz
+        dgID = df_idxyz.groupby("ID")
+        dictGroupID = dict(list(dgID))
+        dfShow = dgID.apply(len).reset_index()
+        #col1.dataframe(dfShow, height=400)
+        pack_size = list(set(dgID.apply(len)))[0]
+        lstDfOriginal = GetListOf_XYZV_ToTrainClassifier(REPO)
+        classifier_object, df_train_with_predict, time_elapsed = my_stm.GetClassifier(lstDfOriginal, pack_size-1)
+        col2.popover(type(classifier_object).__name__).write(type(classifier_object))
+        # прогноз на обучающей выборке
+        columns_xyzv = [c for c in df_train_with_predict.columns if "Vis" in c] + [c for c in df_train_with_predict.columns if c[0] in "XYZ"]
+        #col2.dataframe(df_train_with_predict[columns_xyzv], height=650)
+        # расчет пакетов
+        xyz = ["X","Y","Z"]
+        df_packs_reshaped = dgID.apply(lambda df: pd.Series(df[xyz].values[::-1].reshape(1,-1)[0])).reset_index()
+        x_test_vect = df_packs_reshaped.iloc[:,1:]
+        df_packs_reshaped["Прогноз_V"] = classifier_object.predict(x_test_vect)
+        col2.dataframe(df_packs_reshaped[["ID","Прогноз_V"]], height=650)
 
 
 

static/test.ipynb

@@ -25,7 +25,7 @@
      "text": [
       "<>:1: SyntaxWarning: invalid escape sequence '\\M'\n",
       "<>:1: SyntaxWarning: invalid escape sequence '\\M'\n",
-      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_14236\\1255343956.py:1: SyntaxWarning: invalid escape sequence '\\M'\n",
+      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_36416\\1255343956.py:1: SyntaxWarning: invalid escape sequence '\\M'\n",
       "  path1 = \"N:\\Makarov\\Development\\Python\\Jupiter Notebooks\\Gaziev CSV\\TestData_1504_AB_soloV_gaziev.zip\"\n"
      ]
     },
@@ -456,7 +456,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -470,9 +470,9 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_14236\\117446433.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
+      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_36416\\117446433.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
       "  print(f\"{set(df2.groupby(\"ID\").apply(len))=}\")\n",
-      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_14236\\117446433.py:4: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
+      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_36416\\117446433.py:4: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
       "  dgID.apply(len).reset_index()\n"
      ]
     },
@@ -630,7 +630,7 @@
        "19  DDD011113  7"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -644,13 +644,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 26,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "[['-210' '529' '-943' '102' '578' '-830' '-221' '577' '-859' '-211' '579'\n",
+      "  '-752' '-122' '636' '-592' '-21' '602' '-558' '-200' '573' '-505']]\n",
       "[['-200' '573' '-505' '-21' '602' '-558' '-122' '636' '-592' '-211' '579'\n",
       "  '-752' '-221' '577' '-859' '102' '578' '-830' '-210' '529' '-943']]\n"
      ]
@@ -747,7 +749,7 @@
        "79  BBB011112  -210  529  -943"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 26,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -757,22 +759,24 @@
     "#dgID.apply(lambda df: type(df))\n",
     "#df2.groupby(\"ID\").apply(lambda df: list(df.columns))\n",
     "dgID.get_group(\"BBB011112\")\n",
-    "print(dictGroupID[\"BBB011112\"][xyz].values.reshape(1,21))  #.stack()\n",
+    "print(dictGroupID[\"BBB011112\"][xyz].values.reshape(1,-1))\n",
+    "print(dictGroupID[\"BBB011112\"][xyz].values[::-1].reshape(1,-1))\n",
+    "\n",
     "dictGroupID[\"BBB011112\"]\n",
     "#dgID.indices\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 31,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_14236\\572365488.py:1: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
-      "  dgID.apply(lambda df: df[xyz].values.reshape(1,21)).reset_index()\n"
+      "C:\\Users\\f64\\AppData\\Local\\Temp\\ipykernel_36416\\1960408511.py:2: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
+      "  dgID.apply(lambda df: pd.Series(df[xyz].values[::-1].reshape(1,-1)[0])).reset_index().iloc[:,1:]\n"
      ]
     },
     {
@@ -796,151 +800,576 @@
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
-       "      <th>ID</th>\n",
        "      <th>0</th>\n",
+       "      <th>1</th>\n",
+       "      <th>2</th>\n",
+       "      <th>3</th>\n",
+       "      <th>4</th>\n",
+       "      <th>5</th>\n",
+       "      <th>6</th>\n",
+       "      <th>7</th>\n",
+       "      <th>8</th>\n",
+       "      <th>9</th>\n",
+       "      <th>...</th>\n",
+       "      <th>11</th>\n",
+       "      <th>12</th>\n",
+       "      <th>13</th>\n",
+       "      <th>14</th>\n",
+       "      <th>15</th>\n",
+       "      <th>16</th>\n",
+       "      <th>17</th>\n",
+       "      <th>18</th>\n",
+       "      <th>19</th>\n",
+       "      <th>20</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
-       "      <td>AAA011111</td>\n",
-       "      <td>[[111, 702, 536, 200, 711, 556, -221, 703, 505...</td>\n",
+       "      <td>-200</td>\n",
+       "      <td>732</td>\n",
+       "      <td>257</td>\n",
+       "      <td>211</td>\n",
+       "      <td>746</td>\n",
+       "      <td>312</td>\n",
+       "      <td>-22</td>\n",
+       "      <td>714</td>\n",
+       "      <td>277</td>\n",
+       "      <td>-202</td>\n",
+       "      <td>...</td>\n",
+       "      <td>382</td>\n",
+       "      <td>-221</td>\n",
+       "      <td>703</td>\n",
+       "      <td>505</td>\n",
+       "      <td>200</td>\n",
+       "      <td>711</td>\n",
+       "      <td>556</td>\n",
+       "      <td>111</td>\n",
+       "      <td>702</td>\n",
+       "      <td>536</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
-       "      <td>AAA011112</td>\n",
-       "      <td>[[201, 584, -36, 200, 572, 50, -2, 557, 58, -1...</td>\n",
+       "      <td>202</td>\n",
+       "      <td>486</td>\n",
+       "      <td>-547</td>\n",
+       "      <td>-320</td>\n",
+       "      <td>452</td>\n",
+       "      <td>-505</td>\n",
+       "      <td>-222</td>\n",
+       "      <td>525</td>\n",
+       "      <td>-178</td>\n",
+       "      <td>-102</td>\n",
+       "      <td>...</td>\n",
+       "      <td>22</td>\n",
+       "      <td>-2</td>\n",
+       "      <td>557</td>\n",
+       "      <td>58</td>\n",
+       "      <td>200</td>\n",
+       "      <td>572</td>\n",
+       "      <td>50</td>\n",
+       "      <td>201</td>\n",
+       "      <td>584</td>\n",
+       "      <td>-36</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
-       "      <td>AAA011113</td>\n",
-       "      <td>[[-222, 412, -343, 21, 392, -205, -201, 357, -...</td>\n",
+       "      <td>-302</td>\n",
+       "      <td>336</td>\n",
+       "      <td>-443</td>\n",
+       "      <td>-202</td>\n",
+       "      <td>343</td>\n",
+       "      <td>-257</td>\n",
+       "      <td>-201</td>\n",
+       "      <td>336</td>\n",
+       "      <td>-206</td>\n",
+       "      <td>121</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-175</td>\n",
+       "      <td>-201</td>\n",
+       "      <td>357</td>\n",
+       "      <td>-235</td>\n",
+       "      <td>21</td>\n",
+       "      <td>392</td>\n",
+       "      <td>-205</td>\n",
+       "      <td>-222</td>\n",
+       "      <td>412</td>\n",
+       "      <td>-343</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
-       "      <td>AAA011114</td>\n",
-       "      <td>[[222, 643, 403, 202, 625, 446, -202, 644, 372...</td>\n",
+       "      <td>401</td>\n",
+       "      <td>681</td>\n",
+       "      <td>768</td>\n",
+       "      <td>-212</td>\n",
+       "      <td>705</td>\n",
+       "      <td>660</td>\n",
+       "      <td>220</td>\n",
+       "      <td>712</td>\n",
+       "      <td>694</td>\n",
+       "      <td>401</td>\n",
+       "      <td>...</td>\n",
+       "      <td>544</td>\n",
+       "      <td>-202</td>\n",
+       "      <td>644</td>\n",
+       "      <td>372</td>\n",
+       "      <td>202</td>\n",
+       "      <td>625</td>\n",
+       "      <td>446</td>\n",
+       "      <td>222</td>\n",
+       "      <td>643</td>\n",
+       "      <td>403</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
-       "      <td>AAA011115</td>\n",
-       "      <td>[[0, 645, 109, -202, 585, -4, 201, 584, -36, 2...</td>\n",
+       "      <td>-222</td>\n",
+       "      <td>525</td>\n",
+       "      <td>-178</td>\n",
+       "      <td>-102</td>\n",
+       "      <td>616</td>\n",
+       "      <td>22</td>\n",
+       "      <td>-2</td>\n",
+       "      <td>557</td>\n",
+       "      <td>58</td>\n",
+       "      <td>200</td>\n",
+       "      <td>...</td>\n",
+       "      <td>50</td>\n",
+       "      <td>201</td>\n",
+       "      <td>584</td>\n",
+       "      <td>-36</td>\n",
+       "      <td>-202</td>\n",
+       "      <td>585</td>\n",
+       "      <td>-4</td>\n",
+       "      <td>0</td>\n",
+       "      <td>645</td>\n",
+       "      <td>109</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
-       "      <td>AAA011116</td>\n",
-       "      <td>[[211, 335, -157, -200, 356, -136, -12, 316, -...</td>\n",
+       "      <td>21</td>\n",
+       "      <td>405</td>\n",
+       "      <td>-173</td>\n",
+       "      <td>-111</td>\n",
+       "      <td>427</td>\n",
+       "      <td>-180</td>\n",
+       "      <td>-200</td>\n",
+       "      <td>377</td>\n",
+       "      <td>-150</td>\n",
+       "      <td>-110</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-91</td>\n",
+       "      <td>-12</td>\n",
+       "      <td>316</td>\n",
+       "      <td>-98</td>\n",
+       "      <td>-200</td>\n",
+       "      <td>356</td>\n",
+       "      <td>-136</td>\n",
+       "      <td>211</td>\n",
+       "      <td>335</td>\n",
+       "      <td>-157</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
-       "      <td>AAA011117</td>\n",
-       "      <td>[[0, 600, 157, 0, 645, 109, -202, 585, -4, 201...</td>\n",
+       "      <td>-102</td>\n",
+       "      <td>616</td>\n",
+       "      <td>22</td>\n",
+       "      <td>-2</td>\n",
+       "      <td>557</td>\n",
+       "      <td>58</td>\n",
+       "      <td>200</td>\n",
+       "      <td>572</td>\n",
+       "      <td>50</td>\n",
+       "      <td>201</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-36</td>\n",
+       "      <td>-202</td>\n",
+       "      <td>585</td>\n",
+       "      <td>-4</td>\n",
+       "      <td>0</td>\n",
+       "      <td>645</td>\n",
+       "      <td>109</td>\n",
+       "      <td>0</td>\n",
+       "      <td>600</td>\n",
+       "      <td>157</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7</th>\n",
-       "      <td>AAA011118</td>\n",
-       "      <td>[[-122, 433, -670, -212, 460, -595, 512, 411, ...</td>\n",
+       "      <td>-302</td>\n",
+       "      <td>279</td>\n",
+       "      <td>-2298</td>\n",
+       "      <td>-502</td>\n",
+       "      <td>282</td>\n",
+       "      <td>-1456</td>\n",
+       "      <td>-220</td>\n",
+       "      <td>409</td>\n",
+       "      <td>-362</td>\n",
+       "      <td>-12</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-393</td>\n",
+       "      <td>512</td>\n",
+       "      <td>411</td>\n",
+       "      <td>-487</td>\n",
+       "      <td>-212</td>\n",
+       "      <td>460</td>\n",
+       "      <td>-595</td>\n",
+       "      <td>-122</td>\n",
+       "      <td>433</td>\n",
+       "      <td>-670</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
-       "      <td>BBB011111</td>\n",
-       "      <td>[[-212, 303, -1919, 420, 253, -1668, 222, 271,...</td>\n",
+       "      <td>-114</td>\n",
+       "      <td>277</td>\n",
+       "      <td>-97</td>\n",
+       "      <td>2</td>\n",
+       "      <td>247</td>\n",
+       "      <td>-329</td>\n",
+       "      <td>-220</td>\n",
+       "      <td>284</td>\n",
+       "      <td>-602</td>\n",
+       "      <td>-122</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-860</td>\n",
+       "      <td>222</td>\n",
+       "      <td>271</td>\n",
+       "      <td>-1227</td>\n",
+       "      <td>420</td>\n",
+       "      <td>253</td>\n",
+       "      <td>-1668</td>\n",
+       "      <td>-212</td>\n",
+       "      <td>303</td>\n",
+       "      <td>-1919</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9</th>\n",
-       "      <td>BBB011112</td>\n",
-       "      <td>[[-200, 573, -505, -21, 602, -558, -122, 636, ...</td>\n",
+       "      <td>-210</td>\n",
+       "      <td>529</td>\n",
+       "      <td>-943</td>\n",
+       "      <td>102</td>\n",
+       "      <td>578</td>\n",
+       "      <td>-830</td>\n",
+       "      <td>-221</td>\n",
+       "      <td>577</td>\n",
+       "      <td>-859</td>\n",
+       "      <td>-211</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-752</td>\n",
+       "      <td>-122</td>\n",
+       "      <td>636</td>\n",
+       "      <td>-592</td>\n",
+       "      <td>-21</td>\n",
+       "      <td>602</td>\n",
+       "      <td>-558</td>\n",
+       "      <td>-200</td>\n",
+       "      <td>573</td>\n",
+       "      <td>-505</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
-       "      <td>BBB011113</td>\n",
-       "      <td>[[12, 416, -340, -202, 415, -143, -302, 422, -...</td>\n",
+       "      <td>-200</td>\n",
+       "      <td>320</td>\n",
+       "      <td>82</td>\n",
+       "      <td>121</td>\n",
+       "      <td>289</td>\n",
+       "      <td>58</td>\n",
+       "      <td>22</td>\n",
+       "      <td>336</td>\n",
+       "      <td>-26</td>\n",
+       "      <td>122</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-106</td>\n",
+       "      <td>-302</td>\n",
+       "      <td>422</td>\n",
+       "      <td>-158</td>\n",
+       "      <td>-202</td>\n",
+       "      <td>415</td>\n",
+       "      <td>-143</td>\n",
+       "      <td>12</td>\n",
+       "      <td>416</td>\n",
+       "      <td>-340</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
-       "      <td>BBB011114</td>\n",
-       "      <td>[[-210, 547, 161, -212, 559, 20, 402, 616, 161...</td>\n",
+       "      <td>-201</td>\n",
+       "      <td>612</td>\n",
+       "      <td>63</td>\n",
+       "      <td>-212</td>\n",
+       "      <td>604</td>\n",
+       "      <td>201</td>\n",
+       "      <td>-103</td>\n",
+       "      <td>620</td>\n",
+       "      <td>261</td>\n",
+       "      <td>22</td>\n",
+       "      <td>...</td>\n",
+       "      <td>250</td>\n",
+       "      <td>402</td>\n",
+       "      <td>616</td>\n",
+       "      <td>161</td>\n",
+       "      <td>-212</td>\n",
+       "      <td>559</td>\n",
+       "      <td>20</td>\n",
+       "      <td>-210</td>\n",
+       "      <td>547</td>\n",
+       "      <td>161</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>12</th>\n",
-       "      <td>BBB011115</td>\n",
-       "      <td>[[222, 587, -400, -200, 573, -505, -21, 602, -...</td>\n",
+       "      <td>102</td>\n",
+       "      <td>578</td>\n",
+       "      <td>-830</td>\n",
+       "      <td>-221</td>\n",
+       "      <td>577</td>\n",
+       "      <td>-859</td>\n",
+       "      <td>-211</td>\n",
+       "      <td>579</td>\n",
+       "      <td>-752</td>\n",
+       "      <td>-122</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-592</td>\n",
+       "      <td>-21</td>\n",
+       "      <td>602</td>\n",
+       "      <td>-558</td>\n",
+       "      <td>-200</td>\n",
+       "      <td>573</td>\n",
+       "      <td>-505</td>\n",
+       "      <td>222</td>\n",
+       "      <td>587</td>\n",
+       "      <td>-400</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13</th>\n",
-       "      <td>BBB011116</td>\n",
-       "      <td>[[212, 546, 1557, -231, 563, 1342, -221, 562, ...</td>\n",
+       "      <td>-2</td>\n",
+       "      <td>550</td>\n",
+       "      <td>736</td>\n",
+       "      <td>220</td>\n",
+       "      <td>531</td>\n",
+       "      <td>814</td>\n",
+       "      <td>-112</td>\n",
+       "      <td>527</td>\n",
+       "      <td>840</td>\n",
+       "      <td>320</td>\n",
+       "      <td>...</td>\n",
+       "      <td>906</td>\n",
+       "      <td>-221</td>\n",
+       "      <td>562</td>\n",
+       "      <td>1013</td>\n",
+       "      <td>-231</td>\n",
+       "      <td>563</td>\n",
+       "      <td>1342</td>\n",
+       "      <td>212</td>\n",
+       "      <td>546</td>\n",
+       "      <td>1557</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14</th>\n",
-       "      <td>BBB011117</td>\n",
-       "      <td>[[-221, 575, 438, -102, 601, 258, -220, 561, -...</td>\n",
+       "      <td>-300</td>\n",
+       "      <td>554</td>\n",
+       "      <td>-130</td>\n",
+       "      <td>222</td>\n",
+       "      <td>598</td>\n",
+       "      <td>-85</td>\n",
+       "      <td>221</td>\n",
+       "      <td>581</td>\n",
+       "      <td>-249</td>\n",
+       "      <td>113</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-242</td>\n",
+       "      <td>-220</td>\n",
+       "      <td>561</td>\n",
+       "      <td>-13</td>\n",
+       "      <td>-102</td>\n",
+       "      <td>601</td>\n",
+       "      <td>258</td>\n",
+       "      <td>-221</td>\n",
+       "      <td>575</td>\n",
+       "      <td>438</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>15</th>\n",
-       "      <td>CCC011111</td>\n",
-       "      <td>[[11, 425, -396, -220, 465, -473, -320, 452, -...</td>\n",
+       "      <td>-322</td>\n",
+       "      <td>381</td>\n",
+       "      <td>-1133</td>\n",
+       "      <td>222</td>\n",
+       "      <td>413</td>\n",
+       "      <td>-1103</td>\n",
+       "      <td>223</td>\n",
+       "      <td>370</td>\n",
+       "      <td>-1289</td>\n",
+       "      <td>-520</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-1204</td>\n",
+       "      <td>-320</td>\n",
+       "      <td>452</td>\n",
+       "      <td>-732</td>\n",
+       "      <td>-220</td>\n",
+       "      <td>465</td>\n",
+       "      <td>-473</td>\n",
+       "      <td>11</td>\n",
+       "      <td>425</td>\n",
+       "      <td>-396</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16</th>\n",
-       "      <td>CCC011112</td>\n",
-       "      <td>[[222, 404, -1306, -220, 433, -1375, 201, 481,...</td>\n",
+       "      <td>-202</td>\n",
+       "      <td>316</td>\n",
+       "      <td>-1791</td>\n",
+       "      <td>-232</td>\n",
+       "      <td>333</td>\n",
+       "      <td>-1659</td>\n",
+       "      <td>-22</td>\n",
+       "      <td>421</td>\n",
+       "      <td>-1718</td>\n",
+       "      <td>-511</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-1547</td>\n",
+       "      <td>201</td>\n",
+       "      <td>481</td>\n",
+       "      <td>-1213</td>\n",
+       "      <td>-220</td>\n",
+       "      <td>433</td>\n",
+       "      <td>-1375</td>\n",
+       "      <td>222</td>\n",
+       "      <td>404</td>\n",
+       "      <td>-1306</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>17</th>\n",
-       "      <td>DDD011111</td>\n",
-       "      <td>[[220, 436, 318, 401, 492, 535, 211, 483, 813,...</td>\n",
+       "      <td>-203</td>\n",
+       "      <td>438</td>\n",
+       "      <td>-86</td>\n",
+       "      <td>-521</td>\n",
+       "      <td>436</td>\n",
+       "      <td>327</td>\n",
+       "      <td>-422</td>\n",
+       "      <td>453</td>\n",
+       "      <td>745</td>\n",
+       "      <td>-222</td>\n",
+       "      <td>...</td>\n",
+       "      <td>921</td>\n",
+       "      <td>211</td>\n",
+       "      <td>483</td>\n",
+       "      <td>813</td>\n",
+       "      <td>401</td>\n",
+       "      <td>492</td>\n",
+       "      <td>535</td>\n",
+       "      <td>220</td>\n",
+       "      <td>436</td>\n",
+       "      <td>318</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>18</th>\n",
-       "      <td>DDD011112</td>\n",
-       "      <td>[[-512, 357, -3812, -623, 318, -5477, 260, 301...</td>\n",
+       "      <td>724</td>\n",
+       "      <td>526</td>\n",
+       "      <td>-5020</td>\n",
+       "      <td>-412</td>\n",
+       "      <td>345</td>\n",
+       "      <td>-7138</td>\n",
+       "      <td>-532</td>\n",
+       "      <td>352</td>\n",
+       "      <td>-6875</td>\n",
+       "      <td>-453</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-6739</td>\n",
+       "      <td>260</td>\n",
+       "      <td>301</td>\n",
+       "      <td>-6366</td>\n",
+       "      <td>-623</td>\n",
+       "      <td>318</td>\n",
+       "      <td>-5477</td>\n",
+       "      <td>-512</td>\n",
+       "      <td>357</td>\n",
+       "      <td>-3812</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19</th>\n",
-       "      <td>DDD011113</td>\n",
-       "      <td>[[-220, 487, 741, 22, 516, 433, -123, 478, 234...</td>\n",
+       "      <td>400</td>\n",
+       "      <td>503</td>\n",
+       "      <td>905</td>\n",
+       "      <td>502</td>\n",
+       "      <td>490</td>\n",
+       "      <td>564</td>\n",
+       "      <td>-121</td>\n",
+       "      <td>489</td>\n",
+       "      <td>299</td>\n",
+       "      <td>320</td>\n",
+       "      <td>...</td>\n",
+       "      <td>283</td>\n",
+       "      <td>-123</td>\n",
+       "      <td>478</td>\n",
+       "      <td>234</td>\n",
+       "      <td>22</td>\n",
+       "      <td>516</td>\n",
+       "      <td>433</td>\n",
+       "      <td>-220</td>\n",
+       "      <td>487</td>\n",
+       "      <td>741</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
+       "<p>20 rows × 21 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
-       "           ID                                                  0\n",
-       "0   AAA011111  [[111, 702, 536, 200, 711, 556, -221, 703, 505...\n",
-       "1   AAA011112  [[201, 584, -36, 200, 572, 50, -2, 557, 58, -1...\n",
-       "2   AAA011113  [[-222, 412, -343, 21, 392, -205, -201, 357, -...\n",
-       "3   AAA011114  [[222, 643, 403, 202, 625, 446, -202, 644, 372...\n",
-       "4   AAA011115  [[0, 645, 109, -202, 585, -4, 201, 584, -36, 2...\n",
-       "5   AAA011116  [[211, 335, -157, -200, 356, -136, -12, 316, -...\n",
-       "6   AAA011117  [[0, 600, 157, 0, 645, 109, -202, 585, -4, 201...\n",
-       "7   AAA011118  [[-122, 433, -670, -212, 460, -595, 512, 411, ...\n",
-       "8   BBB011111  [[-212, 303, -1919, 420, 253, -1668, 222, 271,...\n",
-       "9   BBB011112  [[-200, 573, -505, -21, 602, -558, -122, 636, ...\n",
-       "10  BBB011113  [[12, 416, -340, -202, 415, -143, -302, 422, -...\n",
-       "11  BBB011114  [[-210, 547, 161, -212, 559, 20, 402, 616, 161...\n",
-       "12  BBB011115  [[222, 587, -400, -200, 573, -505, -21, 602, -...\n",
-       "13  BBB011116  [[212, 546, 1557, -231, 563, 1342, -221, 562, ...\n",
-       "14  BBB011117  [[-221, 575, 438, -102, 601, 258, -220, 561, -...\n",
-       "15  CCC011111  [[11, 425, -396, -220, 465, -473, -320, 452, -...\n",
-       "16  CCC011112  [[222, 404, -1306, -220, 433, -1375, 201, 481,...\n",
-       "17  DDD011111  [[220, 436, 318, 401, 492, 535, 211, 483, 813,...\n",
-       "18  DDD011112  [[-512, 357, -3812, -623, 318, -5477, 260, 301...\n",
-       "19  DDD011113  [[-220, 487, 741, 22, 516, 433, -123, 478, 234..."
+       "      0    1      2     3    4      5     6    7      8     9   ...     11  \\\n",
+       "0   -200  732    257   211  746    312   -22  714    277  -202  ...    382   \n",
+       "1    202  486   -547  -320  452   -505  -222  525   -178  -102  ...     22   \n",
+       "2   -302  336   -443  -202  343   -257  -201  336   -206   121  ...   -175   \n",
+       "3    401  681    768  -212  705    660   220  712    694   401  ...    544   \n",
+       "4   -222  525   -178  -102  616     22    -2  557     58   200  ...     50   \n",
+       "5     21  405   -173  -111  427   -180  -200  377   -150  -110  ...    -91   \n",
+       "6   -102  616     22    -2  557     58   200  572     50   201  ...    -36   \n",
+       "7   -302  279  -2298  -502  282  -1456  -220  409   -362   -12  ...   -393   \n",
+       "8   -114  277    -97     2  247   -329  -220  284   -602  -122  ...   -860   \n",
+       "9   -210  529   -943   102  578   -830  -221  577   -859  -211  ...   -752   \n",
+       "10  -200  320     82   121  289     58    22  336    -26   122  ...   -106   \n",
+       "11  -201  612     63  -212  604    201  -103  620    261    22  ...    250   \n",
+       "12   102  578   -830  -221  577   -859  -211  579   -752  -122  ...   -592   \n",
+       "13    -2  550    736   220  531    814  -112  527    840   320  ...    906   \n",
+       "14  -300  554   -130   222  598    -85   221  581   -249   113  ...   -242   \n",
+       "15  -322  381  -1133   222  413  -1103   223  370  -1289  -520  ...  -1204   \n",
+       "16  -202  316  -1791  -232  333  -1659   -22  421  -1718  -511  ...  -1547   \n",
+       "17  -203  438    -86  -521  436    327  -422  453    745  -222  ...    921   \n",
+       "18   724  526  -5020  -412  345  -7138  -532  352  -6875  -453  ...  -6739   \n",
+       "19   400  503    905   502  490    564  -121  489    299   320  ...    283   \n",
+       "\n",
+       "      12   13     14    15   16     17    18   19     20  \n",
+       "0   -221  703    505   200  711    556   111  702    536  \n",
+       "1     -2  557     58   200  572     50   201  584    -36  \n",
+       "2   -201  357   -235    21  392   -205  -222  412   -343  \n",
+       "3   -202  644    372   202  625    446   222  643    403  \n",
+       "4    201  584    -36  -202  585     -4     0  645    109  \n",
+       "5    -12  316    -98  -200  356   -136   211  335   -157  \n",
+       "6   -202  585     -4     0  645    109     0  600    157  \n",
+       "7    512  411   -487  -212  460   -595  -122  433   -670  \n",
+       "8    222  271  -1227   420  253  -1668  -212  303  -1919  \n",
+       "9   -122  636   -592   -21  602   -558  -200  573   -505  \n",
+       "10  -302  422   -158  -202  415   -143    12  416   -340  \n",
+       "11   402  616    161  -212  559     20  -210  547    161  \n",
+       "12   -21  602   -558  -200  573   -505   222  587   -400  \n",
+       "13  -221  562   1013  -231  563   1342   212  546   1557  \n",
+       "14  -220  561    -13  -102  601    258  -221  575    438  \n",
+       "15  -320  452   -732  -220  465   -473    11  425   -396  \n",
+       "16   201  481  -1213  -220  433  -1375   222  404  -1306  \n",
+       "17   211  483    813   401  492    535   220  436    318  \n",
+       "18   260  301  -6366  -623  318  -5477  -512  357  -3812  \n",
+       "19  -123  478    234    22  516    433  -220  487    741  \n",
+       "\n",
+       "[20 rows x 21 columns]"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 31,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "dgID.apply(lambda df: df[xyz].values.reshape(1,21)).reset_index()"
+    "#dgID.apply(lambda df: df[xyz].values.reshape(1,21)).reset_index()\n",
+    "dgID.apply(lambda df: pd.Series(df[xyz].values[::-1].reshape(1,-1)[0])).reset_index().iloc[:,1:]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {