download logic

app.py

@@ -29,6 +29,11 @@ cfg = load_config()
 download_file_override(cfg.get('interp_space_url'), cfg.get('interp_space_path'))
 download_file_override(cfg.get('instances_to_explain_url'), cfg.get('instances_to_explain_path'))
 download_file_override(cfg.get('gram2vec_feats_url'), cfg.get('gram2vec_feats_path'))
+download_file_override(cfg.get('embeddings_cache_url'), cfg.get('embeddings_cache_path'))
+download_file_override(cfg.get('zoom_cache_url'), cfg.get('zoom_cache_path'))
+download_file_override(cfg.get('region_cache_url'), cfg.get('region_cache_path'))
+download_file_override(cfg.get('tsne_cache_url'), cfg.get('tsne_cache_path'))
+download_file_override(cfg.get('llm_style_features_cache_url'), cfg.get('llm_style_features_cache_path'))
 
 from utils.visualizations import *
 from utils.llm_feat_utils import *

config/config.yaml

@@ -6,6 +6,17 @@ interp_space_url:    "https://huggingface.co/datasets/miladalsh/explanation_tool
 gram2vec_feats_path:      "./datasets/gram2vec_feats.csv"
 gram2vec_feats_url:      "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/gram2vec_feats.csv?download=true"
 
+embeddings_cache_url: "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/embeddings_cache.zip?download=true"
+embeddings_cache_path: "./datasets/embeddings_cache/"
+zoom_cache_url: "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/zoom_cache.zip?download=true"
+zoom_cache_path: "./datasets/zoom_cache/"
+region_cache_url: "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/region_cache.zip?download=true"
+region_cache_path: "./datasets/region_cache/"
+tsne_cache_url: "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/tsne_cache.pkl?download=true"
+tsne_cache_path: "./datasets/tsne_cache.pkl"
+llm_style_features_cache_url: "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/feature_spans_cache.zip?download=true"
+llm_style_features_cache_path: "./datasets/feature_spans_cache/"
+
 style_feat_clm:       "llm_tfidf_weights"
 top_k:                10
 only_llm_feats:       false

utils/file_download.py

@@ -46,14 +46,30 @@ def download_file_override(url: str, dest_path: str):
                 with zipfile.ZipFile(tmp_path, 'r') as z:
                     z.extractall(tmp_extract_dir)
 
-                # Move *contents* of extracted folder into dest_path
-                for item in os.listdir(tmp_extract_dir):
-                    src = os.path.join(tmp_extract_dir, item)
-                    dst = os.path.join(dest_path, item)
-                    if os.path.isdir(src):
-                        shutil.move(src, dst)
+                if "cache" not in dest_path:
+                    # Move *contents* of extracted folder into dest_path
+                    # cache folders have a different structure, so we skip this step for them
+                    for item in os.listdir(tmp_extract_dir):
+                        src = os.path.join(tmp_extract_dir, item)
+                        dst = os.path.join(dest_path, item)
+                        if os.path.isdir(src):
+                            shutil.move(src, dst)
+                        else:
+                            shutil.move(src, dst)
+                else:
+                    # processing for cache folders of structure like zoom_cache.zip -> zoom_cache/zoom_cache/*
+                    # also hold some auto generated macos metadata.
+                    # Move the entire extracted folder into dest_path
+                    contents = [x for x in os.listdir(tmp_extract_dir) if not x.startswith('__MACOSX')]
+                    if len(contents) == 1 and os.path.isdir(os.path.join(tmp_extract_dir, contents[0])):
+                        # Flatten: Only one top-level dir, move its contents
+                        only_dir = os.path.join(tmp_extract_dir, contents[0])
+                        for item in os.listdir(only_dir):
+                            shutil.move(os.path.join(only_dir, item), os.path.join(dest_path, item))
                     else:
-                        shutil.move(src, dst)
+                        # Usual: move everything as-is
+                        for item in contents:
+                            shutil.move(os.path.join(tmp_extract_dir, item), os.path.join(dest_path, item))
 
             print(f"Extracted zip contents into '{dest_path}'.")
         else: