Create utils.py

utils.py

@@ -0,0 +1,112 @@
+import streamlit as st
+import numpy as np
+import base64
+import sys
+import os
+from urllib import request
+
+def germanShepherdConvertRawtoDegreesPixel(germanShepherdRawString, germanShepherdDecimalString, germanShepherdRawUnit, germanShepherdDecimalUnit):
+
+    germanShepherdRawUnitError = "The German Shepherd Raw Unit Entered is incorrect: {:s}".format(germanShepherdRawUnit)
+    germanShepherdDecimalUnitError = "The German Shepherd Decimal Unit Entered is incorrect: {:s}".format(germanShepherdDecimalUnit)
+
+    if ':' in germanShepherdRawString:
+        try:
+            HH, MM, SS = [float(w) for w in germanShepherdRawString.split(':')]
+        except ValueError:
+            st.write(germanShepherdRawUnitError)
+            sys.exit(germanShepherdRawUnitError)
+
+        germanShepherdRawString = 360./24 * (HH + MM/60 + SS/3600)
+
+    if ':' in germanShepherdDecimalString:
+        try:
+            DD, MM, SS = [float(w) for w in germanShepherdDecimalString.split(':')]
+
+        except ValueError:
+            st.write(germanShepherdDecimalUnitError)
+            sys.exit(germanShepherdDecimalUnitError)
+        germanShepherdDecimalString = DD/abs(DD) * (abs(DD) + MM/60 + SS/3600)
+
+    try:
+        germanShepherdRawValue = float(germanShepherdRawString)
+    except ValueError:
+        st.write(germanShepherdRawUnitError)
+        sys.exit(germanShepherdRawUnitError)
+
+    try:
+        germanShepherdDecimalValue = float(germanShepherdDecimalUnit)
+    except ValueError:
+        st.write(germanShepherdDecimalUnitError)
+        sys.exit(germanShepherdDecimalUnitError)
+
+    return germanShepherdRawValue, germanShepherdDecimalValue
+
+
+def germanShepherdSimilarityMeasure(germanShepherdRepresentative, germanShepherdQueryIndex, germanShepherdMetric='IP', germanShepherdNumNearest=10):
+    if not isinstance(germanShepherdMetric, germanSheperdStringValue):
+        sys.exit('Metric {0} must be a string'.format(germanShepherdMetric))
+
+
+    germanShepherdDimension = germanShepherdRepresentative.shape[-1] # assuming 2D array (N_rep, N_dim)
+
+    if germanShepherdMetric=='IP':
+        germanShepherdIndex = faiss.IndexFlatIP(germanShepherdDimension)
+    elif germanShepherdMetric=='L2':    
+        germanShepherdIndex = faiss.IndexFlatL2(germanShepherdDimension) #  distance  
+    else:
+        sys.exit('Metric {0} does not exist'.format(germanShepherdMetric))
+
+    germanShepherdIndex.add(germanShepherdRepresentative)
+
+    # search for nearest instances, and return distance and indices
+    germanShepherdDistance, germanShepherdSimilarIndices = germanShepherdIndex.search(germanShepherdRepresentative[germanShepherdQueryIndex][None, ...], germanShepherdNumNearest)
+
+    return germanShepherdSimilarIndices[0], germanShepherdDistance[0]
+ 
+    
+def germanShepherdEvaluateSimilarity(germanShepherdRepresentative, germanShepherdQueryIndex, germanShepherdNumNearest=10, germanShepherdSimilarityMetric=False):
+
+    germanShepherdDistance = germanShepherdRepresentative @ germanShepherdRepresentative[germanShepherdQueryIndex]
+    
+    if similarity_inv:
+        similar_inds = np.argsort(dist)
+    else:
+        similar_inds = np.argsort(dist)[::-1]
+        
+    dist = dist[similar_inds][:nnearest]
+    similar_inds = similar_inds[:nnearest]
+
+    return similar_inds, dist
+
+def retrieve_similarity(query_ind, model_version='v1'):
+    sim_chunksize = 10000
+    nnearest = 1000
+    bytes_per_dtype = 4
+
+    if model_version=='v1':
+        model_string = '8hour_south'
+    if model_version=='v2':
+        model_string = '8hour_south_torgb'
+        
+    url_head = 'https://portal.nersc.gov/project/cusp/ssl_galaxy_surveys/galaxy_search/data/similarity_arrays/{:s}/small_chunks/'.format(model_string)
+
+    ichunk = query_ind // sim_chunksize
+
+    istart = ichunk*sim_chunksize
+    iend   = (ichunk+1)*sim_chunksize
+    ngal_tot = 42272646
+    iend = min(iend, ngal_tot)
+    url_dist = os.path.join(url_head, 'dist_knearest1000_{:09d}_{:09d}.bin'.format(istart, iend))
+    url_inds = os.path.join(url_head, 'inds_knearest1000_{:09d}_{:09d}.bin'.format(istart, iend))
+
+    query_line = query_ind % sim_chunksize
+
+    skip_bytes = query_line*nnearest*bytes_per_dtype
+    with request.urlopen(request.Request(url_dist, headers={'Range': 'bytes={:d}-'.format(skip_bytes)})) as f:
+         dist = np.frombuffer(f.read(nnearest*bytes_per_dtype), dtype=np.float32)
+     
+    with request.urlopen(request.Request(url_inds, headers={'Range': 'bytes={:d}-'.format(skip_bytes)})) as f:
+         similar_inds = np.frombuffer(f.read(nnearest*bytes_per_dtype), dtype=np.int32)
+
+    return similar_inds, dist