PhotoLibrary. create_embeddings.py refactor, intermediate save.

app.ini

@@ -1,3 +1,4 @@
 [DEFAULT]
-pkl = 02_LOCATION_PHOTOS.deduped.f16.pkl
+# pkl = 02_LOCATION_PHOTOS.deduped.f16.pkl
+pkl = PhotoLibrary.854G.deduped.f16.pkl
 url = https://static.renyi.hu/ai-shared/daniel/sameenergy/

create_embeddings.py

@@ -31,11 +31,27 @@ assert output_filename.endswith("pkl"), "first argument is the output pickle"
 assert sys.argv[2] in ("thumbs", "no-thumbs"), "second argument either thumbs or no-thumbs"
 do_thumbs = sys.argv[2] == "thumbs"
 
+
+def save(output_filename, embeddings, filenames):
+    embeddings = np.array(embeddings)
+    assert len(embeddings) == len(filenames)
+    print(f"processed {len(embeddings)} images")
+
+    data = {"embeddings": embeddings, "filenames": filenames}
+    if do_thumbs:
+        assert len(embeddings) == len(thumbs)
+        data["thumbs"] = thumbs
+
+    with open(output_filename, "wb") as f:
+        pickle.dump(data, f)
+
+
 embeddings = []
 filenames = []
 thumbs = []
 print("starting processing")
 batch = []
+batch_count = 0
 for filename in sys.stdin:
     filename = filename.rstrip()
     if filename.lower().endswith("jpg") or filename.lower().endswith("jpeg"):
@@ -43,10 +59,13 @@ for filename in sys.stdin:
             rgb = Image.open(filename).convert("RGB")
             img = preprocess(rgb)
             batch.append(img)
+            filenames.append(filename)
             if len(batch) >= batch_size:
                 do_batch(batch, embeddings)
                 batch = []
-            filenames.append(filename)
+                batch_count += 1
+                if batch_count % 200 == 0:
+                    save(output_filename, embeddings, filenames)
             if do_thumbs:
                 rgb.thumbnail((128, 128))
                 thumb = np.array(rgb)
@@ -63,14 +82,4 @@ for filename in sys.stdin:
 if len(batch) > 0:
     do_batch(batch, embeddings)
 
-embeddings = np.array(embeddings)
-assert len(embeddings) == len(filenames)
-print(f"processed {len(embeddings)} images")
-
-data = {"embeddings": embeddings, "filenames": filenames}
-if do_thumbs:
-    assert len(embeddings) == len(thumbs)
-    data["thumbs"] = thumbs
-
-with open(output_filename, "wb") as f:
-    pickle.dump(data, f)
+save(output_filename, embeddings, filenames)

create_embeddings.sh

@@ -1 +1 @@
-cat jpg_files | python ~/experiments/kalman/se/create_embeddings.py 02_LOCATION_PHOTOS.pkl no-thumbs
+cat PhotoLibrary.854G.deduped_md5sums | cut -f3- -d' ' | python ~/experiments/kalman/se/create_embeddings.py PhotoLibrary.854G.deduped.pkl no-thumbs

readme.sh

@@ -68,3 +68,30 @@ python convert.py 02_LOCATION_PHOTOS.pkl
 # -> creates float16 02_LOCATION_PHOTOS.f16.pkl
 mv md5sums 02_LOCATION_PHOTOS.f16.md5sums
 python dedupe.py 02_LOCATION_PHOTOS.f16.pkl 02_LOCATION_PHOTOS.f16.md5sums 02_LOCATION_PHOTOS.deduped.f16.pkl
+
+
+# started downloading PhotoLibrary, but it's super big, 6 days of downloading and counting.
+ssh buda
+cd /data/daniel/sameenergy
+nohup rsync -r hexagon.renyi.hu:./ai-shared/daniel/sameenergy/PhotoLibrary . &
+# 30MB/sec, that's some 10 hours? don't forget that the source is still increasing.
+
+
+nohup bash hashes.sh > md5.cout 2> md5.cerr &
+# -> creates PhotoLibrary.854G.md5sums , md5.cout and md5.cerr are just logs.
+cat PhotoLibrary.854G.md5sums | awk '{ if ($1 in m) { print "skipping\t" $0 "\tbecause\t" m[$1] } else { print $0 } ; m[$1] = $0 }' | grep -v "^skipping" | wc
+
+cat PhotoLibrary.854G.md5sums | awk '{ if ($1 in m) { print "skipping\t" $0 "\tbecause\t" m[$1] } else { print $0 } ; m[$1] = $0 }' | grep -v "^skipping" > PhotoLibrary.854G.deduped_md5sums
+
+
+cat 02_LOCATION_PHOTOS PhotoLibrary.854G.md5sums | awk '{ if ($1 in m) { print "skipping\t" $0 "\tbecause\t" m[$1] } else { print $0 } ; m[$1] = $0 }' | grep -v "^skipping" > 02_and_PhotoLibrary.854G.deduped_md5sums
+
+wc -l 02_and_PhotoLibrary.854G.deduped_md5sums PhotoLibrary.854G.deduped_md5sums
+   591500 02_and_PhotoLibrary.854G.deduped_md5sums
+   514706 PhotoLibrary.854G.deduped_md5sums
+# -> is not worth the hassle merging them. let's just do PhotoLibrary.
+
+# rsync has finished, turns out i've collected PhotoLibrary.854G.raw_files right before that,
+# doing the complete would need a re-hash, is not worth the hassle either. staying with PhotoLibrary.854G.deduped_md5sums
+
+# TODO I don't think lftp has finished successfully, because the Tünde folder has never arrived.