save cache

app.py

@@ -491,7 +491,13 @@ def prepare_compression_and_rag(combined_text, retrieval_slider_value, global_lo
         target_token_size = 0
         past_key_values = FinchCache()
         compressed_length = past_key_values.get_seq_length()
-        
+
+    cache_name = "default_cache_" + uuid.uuid4().hex[:6]
+    cache_name = "default_cache_" + uuid.uuid4().hex[:6] + ".pt"
+    save_dir = "./cache_dir"
+    os.makedirs(save_dir, exist_ok=True)
+    save_path = os.path.join(save_dir, cache_name)
+    past_key_values.save(save_path)
         
     # Use the precomputed rag_index from state.
     collection_name = state.get("rag_index", None)
@@ -505,7 +511,7 @@ def prepare_compression_and_rag(combined_text, retrieval_slider_value, global_lo
         rag_index = create_rag_index(collection_name, rag_text)
 
     state.update({
-        "compressed_cache": past_key_values,
+        "compressed_cache": save_path,
         "compressed_length": compressed_length,
         "rag_index": collection_name,
         "target_token_size": target_token_size,
@@ -526,7 +532,12 @@ def chat_response_stream(message: str, history: list, state: dict):
     Returns a simple string (not a list of message dicts) for ChatInterface.
     """
     user_message = message
-    past_key_values = state["compressed_cache"]
+    save_path = state["compressed_cache"]
+    past_key_values = FinchCache.load(save_path, device=model.device)
+    try:
+        os.remove(save_path)
+    except Exception as e:
+        print(f"Error removing cache file: {e}")
     compressed_length = past_key_values.get_seq_length()
     collection_name = state["rag_index"]
     retrieval_slider_value = state["retrieval_slider"]

chroma_db/0f6ffa1c-1cc6-4993-bdb7-4bf497c593bb/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95042e844cfb77b20e578cf65635282a99d7c4dd20e589ac062f38bc389f8e58
+size 4236000

chroma_db/0f6ffa1c-1cc6-4993-bdb7-4bf497c593bb/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcc596bc1909f7cc610d5839236c90513b4fbad06776c253fa1b21bfd712e940
+size 100

chroma_db/0f6ffa1c-1cc6-4993-bdb7-4bf497c593bb/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e902f1c1fadd234656c661646122d8f7fd9dcf9988b046a0e5b7be886e77c29
+size 4000

chroma_db/9f7c3d0f-8107-441e-9db6-b53a36071e34/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95042e844cfb77b20e578cf65635282a99d7c4dd20e589ac062f38bc389f8e58
+size 4236000

chroma_db/9f7c3d0f-8107-441e-9db6-b53a36071e34/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcc596bc1909f7cc610d5839236c90513b4fbad06776c253fa1b21bfd712e940
+size 100

chroma_db/9f7c3d0f-8107-441e-9db6-b53a36071e34/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:707925f820f4e8cba99037d1116d2601037be2b1a92a68df99cb47393c6a8616
+size 4000