new

app.py

@@ -24,7 +24,7 @@ warnings.filterwarnings("ignore")
 os.environ["PYTHONWARNINGS"] = "ignore"
 
 # =================== MODEL LOADING ===================
-
+@gr.cache_resource
 def load_model():
     """Load the TinyLlama model"""
     print("🚀 Loading Mistral_Test  model...")
@@ -34,8 +34,9 @@ def load_model():
     model = AutoModelForCausalLM.from_pretrained(
         MODEL_ID,
         torch_dtype=torch.float32,
-        device_map="auto",
-        low_cpu_mem_usage=True
+        device_map="cpu", 
+        low_cpu_mem_usage=True,
+        offload_folder="offload"
     )
     
     print("✅ Model loaded successfully!")

imdb_dataset/dataset_dict.json

@@ -0,0 +1 @@
+{"splits": ["train", "test", "unsupervised"]}

imdb_dataset/test/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c7203aaae5b948ac341d611e27d8ca54b7812d34141bf2edba8d3753c89e7dd
+size 32656552

imdb_dataset/test/dataset_info.json

@@ -0,0 +1,65 @@
+{
+  "builder_name": "parquet",
+  "citation": "",
+  "config_name": "plain_text",
+  "dataset_name": "imdb",
+  "dataset_size": 133202802,
+  "description": "",
+  "download_checksums": {
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/train-00000-of-00001.parquet": {
+      "num_bytes": 20979968,
+      "checksum": null
+    },
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/test-00000-of-00001.parquet": {
+      "num_bytes": 20470363,
+      "checksum": null
+    },
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/unsupervised-00000-of-00001.parquet": {
+      "num_bytes": 41996509,
+      "checksum": null
+    }
+  },
+  "download_size": 83446840,
+  "features": {
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "label": {
+      "names": [
+        "neg",
+        "pos"
+      ],
+      "_type": "ClassLabel"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 216649642,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 33435948,
+      "num_examples": 25000,
+      "dataset_name": "imdb"
+    },
+    "test": {
+      "name": "test",
+      "num_bytes": 32653810,
+      "num_examples": 25000,
+      "dataset_name": "imdb"
+    },
+    "unsupervised": {
+      "name": "unsupervised",
+      "num_bytes": 67113044,
+      "num_examples": 50000,
+      "dataset_name": "imdb"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

imdb_dataset/test/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "0c4517be449a88ae",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": "test"
+}

imdb_dataset/train/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f135c5a8f758597f5f96f36d9e7676a612f0aac53ae8411bfb799253eefbc7c
+size 33438688

imdb_dataset/train/dataset_info.json

@@ -0,0 +1,65 @@
+{
+  "builder_name": "parquet",
+  "citation": "",
+  "config_name": "plain_text",
+  "dataset_name": "imdb",
+  "dataset_size": 133202802,
+  "description": "",
+  "download_checksums": {
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/train-00000-of-00001.parquet": {
+      "num_bytes": 20979968,
+      "checksum": null
+    },
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/test-00000-of-00001.parquet": {
+      "num_bytes": 20470363,
+      "checksum": null
+    },
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/unsupervised-00000-of-00001.parquet": {
+      "num_bytes": 41996509,
+      "checksum": null
+    }
+  },
+  "download_size": 83446840,
+  "features": {
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "label": {
+      "names": [
+        "neg",
+        "pos"
+      ],
+      "_type": "ClassLabel"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 216649642,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 33435948,
+      "num_examples": 25000,
+      "dataset_name": "imdb"
+    },
+    "test": {
+      "name": "test",
+      "num_bytes": 32653810,
+      "num_examples": 25000,
+      "dataset_name": "imdb"
+    },
+    "unsupervised": {
+      "name": "unsupervised",
+      "num_bytes": 67113044,
+      "num_examples": 50000,
+      "dataset_name": "imdb"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

imdb_dataset/train/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "2819011aee707696",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": "train"
+}

imdb_dataset/unsupervised/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3700e4aea3020b97dabf57c511456492423935cf7df0b66eefd986f343c73a9b
+size 67118128

imdb_dataset/unsupervised/dataset_info.json

@@ -0,0 +1,65 @@
+{
+  "builder_name": "parquet",
+  "citation": "",
+  "config_name": "plain_text",
+  "dataset_name": "imdb",
+  "dataset_size": 133202802,
+  "description": "",
+  "download_checksums": {
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/train-00000-of-00001.parquet": {
+      "num_bytes": 20979968,
+      "checksum": null
+    },
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/test-00000-of-00001.parquet": {
+      "num_bytes": 20470363,
+      "checksum": null
+    },
+    "hf://datasets/imdb@e6281661ce1c48d982bc483cf8a173c1bbeb5d31/plain_text/unsupervised-00000-of-00001.parquet": {
+      "num_bytes": 41996509,
+      "checksum": null
+    }
+  },
+  "download_size": 83446840,
+  "features": {
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "label": {
+      "names": [
+        "neg",
+        "pos"
+      ],
+      "_type": "ClassLabel"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 216649642,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 33435948,
+      "num_examples": 25000,
+      "dataset_name": "imdb"
+    },
+    "test": {
+      "name": "test",
+      "num_bytes": 32653810,
+      "num_examples": 25000,
+      "dataset_name": "imdb"
+    },
+    "unsupervised": {
+      "name": "unsupervised",
+      "num_bytes": 67113044,
+      "num_examples": 50000,
+      "dataset_name": "imdb"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

imdb_dataset/unsupervised/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "df8b13b1c356ff29",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": "unsupervised"
+}

load.py

@@ -1,6 +1,11 @@
 from datasets import load_dataset
 
-# Load a dataset
-dataset = load_dataset("imdb")  # Example: IMDB reviews
-# Or your custom dataset
-# dataset = load_dataset("json", data_files="your_data.json")
+# Load dataset
+dataset = load_dataset("imdb")
+
+# Save to disk
+dataset.save_to_disk("./imdb_dataset")
+
+# Later: reload
+# from datasets import load_from_disk
+# dataset = load_from_disk("./imdb_dataset")