2809

src/cherrypick.ipynb

@@ -0,0 +1,1614 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "ce9ebe41-3fb5-459e-a800-416103fb3f4b",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "52940617f54245418aaf7236b246ee03",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading dataset from disk:   0%|          | 0/329 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "26b058c989394d49aedaa4de4f6d8e7c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ba0ed28c7e204df19e83a8be534a9a94",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0ab0775fb07640588e47a991b4d8ccd1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1928e46884264fc48089c5dac0e932a8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "55ed14e31fd2439aa17b4fe374fc6f36",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "383954937ecf43bdb40d1bb00a9117ea",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e63ad6333820407c84123b55dc6c70a7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a0d67bf099544cdf94aa9b8dadbd8c69",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "19c07ed06940465386d1c330a0fb3f4a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d4fcfe324db2484a95401409e1f51d2a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3f5603691d8b41348068162e7f43e43e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e47972712554aabb38c3279372ed077",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b57f32bf1acf42db8f23db4bf5f23ce3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8a56141dc03f401c9909fa09865d7ef1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8c5ec13d5d1f4393a5efc9328c5d3d51",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f2a628d019594312a2fc7541657d6f68",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a85452b9542f4e75bc273c620c869ba8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "46b38226b34448c3abf1c29ebf50c4d6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a60d94b35f754f818fbc55238b57c527",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c92ca7d7b0f04a36899f6d23f5c61408",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f7fefeee8ec345758cf4a6a70259c722",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c36c956511d64c06bdd83438bfccafa2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b0c568e26ee94956bed7e8821af83420",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f23724b0ddb74644a9063600c173d29c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a3e0fe81365a49f7a19212bd6279b38a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3369d7a46b3a45eab0f4b4cfdbd41b20",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "951b8909b65142578e1af4cb04cd70a3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "152cfdffdac44403a019f35d431de158",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c8c1794598704e658923d8691b4c8973",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7517d76d26364743addb8c1b8cb8bf1d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1a8781fa9e854029b7ec76c550d76fa2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c209d8466a524fa09978f7b1d9d17cc9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "751a5f0dcac14ea7a516654a1e7da20f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6558b76e476f44fbb1b1fab1fac3c450",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3a6893bfb66a4592a1724da4d9ed7063",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "62eb1dcb51904e7ea9710930ddc0ee30",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b31d5a6f8c8b42a2a9e5feb6f0954c3a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5623c42b05d8499cb40340fae33e33fe",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ed1340e74ad5424c826e38e09d709214",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "022b98782a8943929097c5575a875ed8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2fc825537704406cbbbe5af31a220845",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "32f6ac57fc4547a39612e9e8b93b4d95",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d3eb079543f54bd48d3f2e0bb2d1699c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ec1e4cb94f0b453eba94e81f2263b9f1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "21cc0bc9ba62447fa0460a4bd05a4e6c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e187bb05045a4409a1eabb1731a6bd7e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6f44a7df475446f8aab1c4bea66510b4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1455330d17fe4e6f9f1db3fc14ebf281",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7e796aaa2ab04d209002746564259499",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0f6c726352124d4198ff32532353d11d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b151010578504ed8b3e524e16b580872",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6eef4c9c19b04dd8bb04970946f5adb4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "de5df2cfbbc14f30a61bb38ab931192a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dda4e690cc7e4c3fb82c9449ee40dff0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ddfa848bb4b24438ae237be7c281586a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ee24a8af80874905bdc7f1cb38ae5262",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ddcf6709805c4e938ca3cc296a562d5a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3b39ee415b1e4920b4851bbfe698eb23",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e22891bb74224b43a7f002005908667f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2683a16b44d440f7a3c62c7c3871df27",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "136010b37fc94e369c46679898dc209c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "166ba4346ce54680b4475f99bcf745ba",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5edea12e672b45b09eb3a1d0e7187e0d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0c3f65ae1d794aef990eb83ca5febad1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "cc4b08df225040368e421aac76ab8524",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "774bd02f10e24438be2d049afeb30550",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ac512bb0c0e049f19ef32a584daba960",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e3d68b81bf4d4f16a0adc7c21a4f0391",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a038981827fb496498d091423f30125d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "281a48b8882f4e58a07366e8f37377b0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "07ccca2e999a45069568c8c932281e3d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6077bc9e6d054395923da058eaac7929",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "cb5f3dac7363444f9a8170a44a8991d1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b488a5658f0847578b493ae536ebdfa6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2b608321703e4cd4a53a782e9266a6bb",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c2e89571ca324a37927dc6fd26b425d2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "de1eaaeed4714fb6b7a8e2a3c16ca8b9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f49182a8fa68464a8157138e4da202db",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8e43173527b543bb9fafbe695eb0395b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c2a00f52bfa14bf4a2f1e10a37acba31",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c7e77ec85f7749ea8a291defd21454bc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "22d05db5ceb24a6c8407996fcf538a4c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f4a1b3561fe34239ac9306d083c93b45",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a7837e9d78c5423ca1e22f077f665ef4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "52f2ca450fa940e2b2e1e6f65da39d99",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4f1d53f8d9a14473b39456cbfba80978",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ae26bf8da3dc48a9823114a2d6884f11",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7f904fb6f4f34dc49cc4d66d51ffbfc6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "506cace9c3e945578d857c35224d96fd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "10b2da3080d64a5fb46a969203e4df31",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "24ec24f5b88e4cb89a006be8f9a7609b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "face9c8c64e04b1f9b5a7902903ee1ac",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e2e63e38ce2b4bf5b3dd862e9685bc72",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0427e75f6e744461bf137e5c88e94061",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "32d5bd3560384e188fe3669e66095d23",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "909b216601b347648e3af0582665c558",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "961f79a42070460d88a1813bf18feba4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "180768311e974f318f2a1b8a6433a57b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c05cb94470974807bf854617d0ce80b6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ccd29e7358c24f358d060e16d889771b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Генерация:   0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Images generated and saved to: samples\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_from_disk\n",
+    "import torch\n",
+    "from diffusers import AutoencoderKL, DDPMScheduler, UNet2DConditionModel\n",
+    "from transformers import AutoModel, AutoTokenizer\n",
+    "from PIL import Image\n",
+    "from tqdm.auto import tqdm\n",
+    "import os\n",
+    "import random\n",
+    "\n",
+    "# Initialize models and tokenizer\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "dtype = torch.float16 if torch.cuda.is_available() else torch.float32\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\"visheratin/mexma-siglip\")\n",
+    "text_model = AutoModel.from_pretrained(\n",
+    "    \"visheratin/mexma-siglip\", torch_dtype=dtype, trust_remote_code=True\n",
+    ").to(device, dtype=dtype).eval()\n",
+    "\n",
+    "pipeid = \"AiArtLab/sdxs\"\n",
+    "variant = \"fp16\" if torch.cuda.is_available() else None\n",
+    "\n",
+    "unet = UNet2DConditionModel.from_pretrained(\"/workspace/sdxs\"#pipeid\n",
+    "                                            , subfolder=\"unet\", variant=variant).to(device, dtype=dtype).eval()\n",
+    "vae = AutoencoderKL.from_pretrained(pipeid, subfolder=\"vae\", variant=variant).to(device, dtype=dtype).eval()\n",
+    "scheduler = DDPMScheduler.from_pretrained(pipeid, subfolder=\"scheduler\")\n",
+    "\n",
+    "def get_random_samples(dataset_path, num_samples=20):\n",
+    "    \"\"\"\n",
+    "    Возвращает случайные тексты, ширину и высоту из датасета.\n",
+    "    \"\"\"\n",
+    "    # Загружаем датасет\n",
+    "    dataset = load_from_disk(dataset_path)\n",
+    "\n",
+    "    # Удаление записей с \"vectorstock\"\n",
+    "    filtered_indices = [i for i, text in enumerate(dataset['text']) if 'vector' not in text.lower()]\n",
+    "    dataset = dataset.select(filtered_indices)\n",
+    "\n",
+    "    # Выбираем случайные индексы\n",
+    "    random_indices = random.sample(range(len(dataset)), num_samples)\n",
+    "\n",
+    "    # Извлекаем тексты, ширину и высоту\n",
+    "    samples = [\n",
+    "        {\n",
+    "            \"text\": dataset[i]['text'],\n",
+    "            \"width\": dataset[i]['width'],\n",
+    "            \"height\": dataset[i]['height']\n",
+    "        }\n",
+    "        for i in random_indices\n",
+    "    ]\n",
+    "\n",
+    "    return samples\n",
+    "\n",
+    "def encode_prompt(prompt, negative_prompt, device, dtype):\n",
+    "    if negative_prompt is None:\n",
+    "        negative_prompt = \"\"\n",
+    "\n",
+    "    with torch.no_grad():\n",
+    "        positive_inputs = tokenizer(\n",
+    "            prompt,\n",
+    "            return_tensors=\"pt\",\n",
+    "            padding=\"max_length\",\n",
+    "            max_length=512,\n",
+    "            truncation=True,\n",
+    "        ).to(device)\n",
+    "        positive_embeddings = text_model.encode_texts(\n",
+    "            positive_inputs.input_ids, positive_inputs.attention_mask\n",
+    "        )\n",
+    "        if positive_embeddings.ndim == 2:\n",
+    "            positive_embeddings = positive_embeddings.unsqueeze(1)\n",
+    "        positive_embeddings = positive_embeddings.to(device, dtype=dtype)\n",
+    "\n",
+    "        negative_inputs = tokenizer(\n",
+    "            negative_prompt,\n",
+    "            return_tensors=\"pt\",\n",
+    "            padding=\"max_length\",\n",
+    "            max_length=512,\n",
+    "            truncation=True,\n",
+    "        ).to(device)\n",
+    "        negative_embeddings = text_model.encode_texts(negative_inputs.input_ids, negative_inputs.attention_mask)\n",
+    "        if negative_embeddings.ndim == 2:\n",
+    "            negative_embeddings = negative_embeddings.unsqueeze(1)\n",
+    "        negative_embeddings = negative_embeddings.to(device, dtype=dtype)\n",
+    "    return torch.cat([negative_embeddings, positive_embeddings], dim=0)\n",
+    "\n",
+    "def generate_latents(embeddings, height=576, width=576, num_inference_steps=50, guidance_scale=5.5):\n",
+    "    with torch.no_grad():\n",
+    "        device, dtype = embeddings.device, embeddings.dtype\n",
+    "        half = embeddings.shape[0] // 2\n",
+    "        latent_shape = (half, 16, height // 8, width // 8)\n",
+    "        latents = torch.randn(latent_shape, device=device, dtype=dtype)\n",
+    "        embeddings = embeddings.repeat_interleave(half, dim=0)\n",
+    "\n",
+    "        scheduler.set_timesteps(num_inference_steps)\n",
+    "\n",
+    "        for t in tqdm(scheduler.timesteps, desc=\"Генерация\"):\n",
+    "            latent_model_input = torch.cat([latents] * 2)\n",
+    "            latent_model_input = scheduler.scale_model_input(latent_model_input, t)\n",
+    "            noise_pred = unet(latent_model_input, t, embeddings).sample\n",
+    "            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)\n",
+    "            noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)\n",
+    "            latents = scheduler.step(noise_pred, t, latents).prev_sample\n",
+    "    return latents\n",
+    "\n",
+    "def decode_latents(latents, vae, output_type=\"pil\"):\n",
+    "    latents = (latents / vae.config.scaling_factor) + vae.config.shift_factor\n",
+    "    with torch.no_grad():\n",
+    "        images = vae.decode(latents).sample\n",
+    "    images = (images / 2 + 0.5).clamp(0, 1)\n",
+    "    images = images.cpu().permute(0, 2, 3, 1).float().numpy()\n",
+    "    if output_type == \"pil\":\n",
+    "        images = (images * 255).round().astype(\"uint8\")\n",
+    "        images = [Image.fromarray(image) for image in images]\n",
+    "    return images\n",
+    "\n",
+    "def generate_and_save_images(dataset_path, output_folder=\"samples\", project_name=\"sdxs\"):\n",
+    "    # Load random samples\n",
+    "    samples = get_random_samples(dataset_path,num_samples=100)\n",
+    "\n",
+    "    os.makedirs(output_folder, exist_ok=True)\n",
+    "\n",
+    "    for idx, sample in enumerate(samples):\n",
+    "        prompt = sample[\"text\"]\n",
+    "        negative_prompt = \"bad quality, low quality, low resolution\"\n",
+    "        height, width = sample[\"height\"], sample[\"width\"]\n",
+    "\n",
+    "        # Encode prompt\n",
+    "        embeddings = encode_prompt(prompt, negative_prompt, device, dtype)\n",
+    "\n",
+    "        # Generate latents\n",
+    "        latents = generate_latents(\n",
+    "            embeddings=embeddings,\n",
+    "            height=height,\n",
+    "            width=width,\n",
+    "            num_inference_steps=50\n",
+    "        )\n",
+    "\n",
+    "        # Decode latents to images\n",
+    "        images = decode_latents(latents, vae)\n",
+    "\n",
+    "        # Save the image\n",
+    "        for img_idx, image in enumerate(images):\n",
+    "            image.save(f\"{output_folder}/{project_name}_{idx}_{img_idx}.jpg\")\n",
+    "\n",
+    "    print(\"Images generated and saved to:\", output_folder)\n",
+    "\n",
+    "# Example usage\n",
+    "dataset_path = \"/workspace/sdxs/datasets/576\"\n",
+    "generate_and_save_images(dataset_path)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7b597fec-ff15-40fc-b9d5-356702cf66b8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

src/dataset_combine.py

@@ -0,0 +1,68 @@
+import os
+import shutil
+from datasets import load_from_disk, concatenate_datasets
+
+def combine_datasets(main_dataset_path, datasets_to_add):
+    """
+    Объединяет указанные датасеты с основным датасетом.
+    
+    Args:
+        main_dataset_path (str): Путь к основному датасету, в который нужно добавить данные
+        datasets_to_add (list): Список путей к датасетам, которые нужно добавить
+    
+    Returns:
+        Dataset: Объединенный датасет
+    """
+    # Загружаем основной датасет
+    try:
+        main_dataset = load_from_disk(main_dataset_path)
+        print(f"Загружен основной датасет: {main_dataset_path} ({len(main_dataset)} записей)")
+    except Exception as e:
+        print(f"Ошибка загрузки основного датасета: {e}")
+        return None
+    
+    # Список всех датасетов для объединения
+    all_datasets = [main_dataset]
+    
+    # Загружаем и добавляем все дополнительные датасеты
+    for path in datasets_to_add:
+        try:
+            ds = load_from_disk(path)
+            all_datasets.append(ds)
+            print(f"Добавлен датасет: {path} ({len(ds)} записей)")
+        except Exception as e:
+            print(f"Ошибка загрузки датасета {path}: {e}")
+    
+    # Объединяем все датасеты
+    print(f"Объединение {len(all_datasets)} датасетов...")
+    combined = concatenate_datasets(all_datasets)
+    
+    # Создаем временную директорию на основе имени основного датасета
+    temp_dir = f"{main_dataset_path}_temp"
+    
+    # Удаляем временную директорию, если она уже существует
+    if os.path.exists(temp_dir):
+        shutil.rmtree(temp_dir)
+    
+    try:
+        # Сохраняем в временную директорию
+        print(f"Сохранение во временную директорию {temp_dir}...")
+        combined.save_to_disk(temp_dir)
+        
+        # Удаляем старую директорию и перемещаем новую на ее место
+        print(f"Обновление основного датасета...")
+        #if os.path.exists(main_dataset_path):
+        #    shutil.rmtree(main_dataset_path)
+        #shutil.copytree(temp_dir, main_dataset_path)
+        
+        # Удаляем временную директорию после успешного копирования
+        #shutil.rmtree(temp_dir)
+        
+        print(f"✅ Объединенный датасет ({len(combined)} записей) успешно сохранен в: {main_dataset_path}")
+    except Exception as e:
+        print(f"Ошибка при сохранении датасета: {e}")
+        print(f"Временные данные сохранены в: {temp_dir}")
+    
+    return combined
+
+combine_datasets("/workspace/sdxs3d/datasets/mjnj", ["/workspace/animesfw384"])

src/dataset_from_folder.py

@@ -27,8 +27,8 @@ step = 64
 empty_share = 0.05
 limit = 0
 # Основная процедура обработки
-folder_path = "/workspace/mjnj" #alchemist"
-save_path = "/workspace/sdxs3d/datasets/mjnj" #"alchemist"
+folder_path = "/workspace/animesfw" #alchemist"
+save_path = "/workspace/animesfw384" #"alchemist"
 os.makedirs(save_path, exist_ok=True)
 
 # Функция для очистки CUDA памяти
@@ -42,7 +42,7 @@ def clear_cuda_memory():
 # ---------------- 2️⃣ Загрузка моделей ----------------
 def load_models():
     print("Загрузка моделей...")
-    vae = AutoencoderKL.from_pretrained("AiArtLab/simplevae",subfolder="simple_vae_nightly",torch_dtype=dtype).to(device).eval()
+    vae = AutoencoderKL.from_pretrained("AiArtLab/sdxs3d",subfolder="vae",torch_dtype=dtype).to(device).eval()
 
     tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen3-Embedding-0.6B', padding_side='left')
     model = AutoModel.from_pretrained('Qwen/Qwen3-Embedding-0.6B').to("cuda")

src/dataset_sample.ipynb

@@ -12,18 +12,18 @@
      "text": [
       "Загрузка VAE модели...\n",
       "None None\n",
-      "Загрузка датасета из /workspace/sdxs3d/datasets/mjnj...\n"
+      "Загрузка датасета из /workspace/sdxs3d/datasets/mjnj_temp...\n"
      ]
     },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b8440eeb7a96445fbc97026d43499e0c",
+       "model_id": "6439c281be894d13967507c46fd382fb",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "Loading dataset from disk:   0%|          | 0/128 [00:00<?, ?it/s]"
+       "Loading dataset from disk:   0%|          | 0/139 [00:00<?, ?it/s]"
       ]
      },
      "metadata": {},
@@ -38,7 +38,7 @@
       "\n",
       "Сортируем...\n",
       "\n",
-      "--- Батч 256x384: 415207 примеров ---\n",
+      "--- Батч 256x384: 461454 примеров ---\n",
       "3 torch.Size([16, 48, 32])\n",
       "denormalized_latent torch.Size([1, 16, 48, 32])\n",
       "reconstructed_image torch.Size([1, 3, 384, 256])\n",
@@ -66,7 +66,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "--- Батч 384x384: 318891 примеров ---\n",
+      "--- Батч 384x384: 323287 примеров ---\n",
       "3 torch.Size([16, 48, 48])\n",
       "denormalized_latent torch.Size([1, 16, 48, 48])\n",
       "reconstructed_image torch.Size([1, 3, 384, 384])\n",
@@ -98,7 +98,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "--- Батч 384x192: 160398 примеров ---\n",
+      "--- Батч 384x192: 168933 примеров ---\n",
       "3 torch.Size([16, 24, 48])\n",
       "denormalized_latent torch.Size([1, 16, 24, 48])\n",
       "reconstructed_image torch.Size([1, 3, 192, 384])\n",
@@ -126,7 +126,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "--- Батч 192x384: 91622 примеров ---\n",
+      "--- Батч 192x384: 103783 примеров ---\n",
       "3 torch.Size([16, 48, 24])\n",
       "denormalized_latent torch.Size([1, 16, 48, 24])\n",
       "reconstructed_image torch.Size([1, 3, 384, 192])\n",
@@ -154,7 +154,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "--- Батч 384x256: 47266 примеров ---\n",
+      "--- Батч 384x256: 58854 примеров ---\n",
       "3 torch.Size([16, 32, 48])\n",
       "denormalized_latent torch.Size([1, 16, 32, 48])\n",
       "reconstructed_image torch.Size([1, 3, 256, 384])\n",
@@ -182,7 +182,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "--- Батч 320x384: 3648 примеров ---\n",
+      "--- Батч 320x384: 11109 примеров ---\n",
       "3 torch.Size([16, 48, 40])\n",
       "denormalized_latent torch.Size([1, 16, 48, 40])\n",
       "reconstructed_image torch.Size([1, 3, 384, 320])\n",
@@ -210,7 +210,7 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "--- Батч 384x320: 2808 примеров ---\n",
+      "--- Батч 384x320: 7445 примеров ---\n",
       "3 torch.Size([16, 40, 48])\n",
       "denormalized_latent torch.Size([1, 16, 40, 48])\n",
       "reconstructed_image torch.Size([1, 3, 320, 384])\n",
@@ -254,7 +254,7 @@
     "    \n",
     "    # Загрузка VAE модели\n",
     "    print(\"Загрузка VAE модели...\")\n",
-    "    vae = AutoencoderKL.from_pretrained(\"AiArtLab/simplevae\",subfolder=\"simple_vae_nightly\",torch_dtype=dtype).to(device).eval()\n",
+    "    vae = AutoencoderKL.from_pretrained(\"AiArtLab/sdxs3d\",subfolder=\"vae\",torch_dtype=dtype).to(device).eval()\n",
     "    \n",
     "    shift_factor = getattr(vae.config, \"shift_factor\", 0.0)\n",
     "    if shift_factor is None:\n",
@@ -347,7 +347,7 @@
     "\n",
     "# Использование\n",
     "if __name__ == \"__main__\":\n",
-    "    save_path = \"/workspace/sdxs3d/datasets/mjnj\"\n",
+    "    save_path = \"/workspace/sdxs3d/datasets/mjnj_temp\"\n",
     "    size_groups = analyze_dataset_by_size(save_path)\n",
     "\n"
    ]

unet/config.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afc06beff07034f0ce9f671c83222e7f78eedc3b3ce93293143accdebef1b111
-size 1887
+oid sha256:0ef8fbaff98c8d479d68b566d07ef4fb8e51ac26b9e8b5a3cb2b23f9a978f6ca
+size 1874

unet/diffusion_pytorch_model.fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:072a5932b339f1c98d492fe71c74a374de6c0e5480dab5f0c393aed03818b6e1
+size 3092571208

unet/diffusion_pytorch_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dadca894650cddefea1f9251abb5948aba795370e117ace7387e3ff82385dbd
-size 3092571208
+oid sha256:958b8d4d7c0c89ddd9977f61f3d17bb2dd5027b75c0d3669c30d55cf448d5c94
+size 6184944280