taking files from the 02_LOCATION_PHOTOS.thumbs folder instead of non-thumbnailed PhotoLibrary.

app.ini

@@ -1,4 +1,4 @@
 [DEFAULT]
-# pkl = 02_LOCATION_PHOTOS.deduped.f16.pkl
-pkl = PhotoLibrary.854G.deduped.f16.pkl
+pkl = 02_LOCATION_PHOTOS.deduped.f16.pkl
+# pkl = PhotoLibrary.854G.deduped.f16.pkl
 url = https://static.renyi.hu/ai-shared/daniel/sameenergy/

app.py

@@ -68,10 +68,15 @@ def build_ann_index(embeddings):
     return annoy_index
 
 
-annoy_index = build_ann_index(embeddings)
+filenames = data["filenames"]
+def thumb_patch(filename):
+    prefix = "02_LOCATION_PHOTOS"
+    assert filename.startswith(prefix)
+    return prefix + ".thumbs" + filename[len(prefix): ]
 
 
-filenames = data["filenames"]
+print("patching filenames")
+filenames = [thumb_patch(filename) for filename in filenames]
 
 folders = ["/".join(filename.split("/")[:-1]) for filename in filenames]
 # to make smart indexing possible:
@@ -81,6 +86,8 @@ urls = [base_url + filename for filename in filenames]
 urls = np.array(urls)
 
 
+annoy_index = build_ann_index(embeddings)
+
 model, preprocess = clip.load('RN50', device=device)
 
 

readme.sh

@@ -95,3 +95,26 @@ wc -l 02_and_PhotoLibrary.854G.deduped_md5sums PhotoLibrary.854G.deduped_md5sums
 # doing the complete would need a re-hash, is not worth the hassle either. staying with PhotoLibrary.854G.deduped_md5sums
 
 # TODO I don't think lftp has finished successfully, because the Tünde folder has never arrived.
+
+
+
+#####
+# thumbnailing
+
+# on hexagon
+cd ~/ai-shared/daniel/sameenergy
+nohup cp -r 02_LOCATION_PHOTOS 02_LOCATION_PHOTOS.thumbs &
+nohup cp -r PhotoLibrary PhotoLibrary.thumbs &
+# -> this is slooooow, a day or so.
+
+# the following code, located at hexagon:~/ai-shared/daniel/sameenergy/downscale.sh ,
+# downscales so that the image fits into 1024x1024
+find $root -type f | grep -i "jpeg\|jpg$" | while read f ; do echo "$f" ; convert "$f" -resize "1024x1024>" "$f" ; done
+
+# it was run like this, setting root=02_LOCATION_PHOTOS.thumbs
+nohup bash downscale.sh > 02_LOCATION_PHOTOS.downscale.cout 2> 02_LOCATION_PHOTOS.downscale.cerr &
+# -> took a night or so.
+nohup bash downscale.sh > PhotoLibrary.downscale.cout 2> PhotoLibrary.downscale.cerr &
+# -> will take 4 days or so
+
+# added to app.py to patch the filenames in the pickle to change 02_LOCATION_PHOTOS to 02_LOCATION_PHOTOS.thumbs