nomic login

Dockerfile

@@ -26,9 +26,9 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user
 COPY --chown=user . $HOME/app
 
-# Expose the secret NOMIC_API_KEY at buildtime and use its value
-RUN --mount=type=secret,id=NOMIC_API_KEY,mode=0444,required=true \
-    nomic login $(cat /run/secrets/NOMIC_API_KEY)
+# # Expose the secret NOMIC_API_KEY at buildtime and use its value
+# RUN --mount=type=secret,id=NOMIC_API_KEY,mode=0444,required=true \
+#     nomic login $(cat /run/secrets/NOMIC_API_KEY)
 
 # Make port 7860 available to the world outside this container
 EXPOSE 7860

build_map.py

@@ -110,6 +110,7 @@ def load_dataset_and_metadata(dataset_name,
 
 
 def upload_dataset_to_atlas(dataset_dict, 
+                            atlas_api_token: str,
                             project_name = None,
                             unique_id_field_name=None, 
                             indexed_field = None, 
@@ -117,6 +118,7 @@ def upload_dataset_to_atlas(dataset_dict,
                             organization_name=None,
                             wait_for_map=True,
                             datum_limit=30000):
+    nomic.login(atlas_api_token)
 
     if modality is None:
         modality = "text"
@@ -140,15 +142,18 @@ def upload_dataset_to_atlas(dataset_dict,
     uncategorized_fields = get_datum_fields(dataset_dict)
 
 
-    # return longest string field
+    # return longest string field from 5 samples
     if indexed_field is None:
-        ex = dataset_dict["head"].take([0])
-        longest_len = 0
+        longest_length = 0
         for field in string_fields:
-            if ex[field] and  len(ex[field]) > longest_len:
+            length = 0
+            for i in range(len(dataset_dict["head"])):
+                ex = dataset_dict["head"].take([i])
+                if ex[field]:
+                    length += len(ex[field].split())
+            if length > longest_length:
+                longest_length = length
                 indexed_field = field
-                longest_len = len(ex[field])
-
 
     topic_label_field = None
     if modality == "embedding":

main.py

@@ -12,7 +12,7 @@ from huggingface_hub import create_discussion, comment_discussion
 from build_map import load_dataset_and_metadata, upload_dataset_to_atlas
 from models import WebhookPayload
 
-WEBHOOK_SECRET = os.environ.get("WEBHOOK_SECRET")
+# WEBHOOK_SECRET = os.environ.get("WEBHOOK_SECRET")
 HUGGINGFACE_ACCESS_TOKEN = os.environ.get("HUGGINGFACE_ACCESS_TOKEN")
 
 app = FastAPI()
@@ -20,12 +20,13 @@ app = FastAPI()
 tasks = {}
 templates = Jinja2Templates(directory="templates")
 
-def upload_atlas_task(task_id, 
-                      dataset_name, 
+def upload_atlas_task(task_id: str, 
+                      dataset_name: str,
+                      atlas_api_token: str,
                       webhook_payload: WebhookPayload = None, 
                       webhook_notify: bool = False):
     dataset_dict = load_dataset_and_metadata(dataset_name)
-    map_url = upload_dataset_to_atlas(dataset_dict)
+    map_url = upload_dataset_to_atlas(dataset_dict, atlas_api_token)
     tasks[task_id]['status'] = 'done'
     tasks[task_id]['url'] = map_url
     tasks[task_id]['finish_time'] = time.time()
@@ -61,18 +62,18 @@ async def cleanup_tasks():
             del tasks[task_id]
         await asyncio.sleep(1800)  # Wait for 30 minutes
 
-# @app.get("/", response_class=HTMLResponse)
-# async def read_form(request: Request):
-#     # Render the form.html template
-#     return templates.TemplateResponse("form.html", {"request": request})
+@app.get("/", response_class=HTMLResponse)
+async def read_form(request: Request):
+    # Render the form.html template
+    return templates.TemplateResponse("form.html", {"request": request})
 
-# @app.post("/submit_form")
-# async def form_post(background_tasks: BackgroundTasks, dataset_name: str = Form(...)):
-#     task_id = str(uuid4())
-#     tasks[task_id] = {'status': 'running'}
-#     #form_data = DatasetForm(dataset_name=dataset_name)
-#     background_tasks.add_task(upload_atlas_task, task_id, dataset_name)
-#     return {'task_id': task_id}
+@app.post("/submit_form")
+async def form_post(background_tasks: BackgroundTasks, dataset_name: str = Form(...), atlas_api_token: str = Form(...)):
+    task_id = str(uuid4())
+    tasks[task_id] = {'status': 'running'}
+    #form_data = DatasetForm(dataset_name=dataset_name)
+    background_tasks.add_task(upload_atlas_task, task_id, dataset_name, atlas_api_token)
+    return {'task_id': task_id}
 
 @app.get("/status/{task_id}")
 async def read_task(task_id: str):
@@ -85,8 +86,9 @@ async def read_task(task_id: str):
 async def post_webhook(background_tasks: BackgroundTasks, payload: WebhookPayload, x_webhook_secret:  Optional[str] = Header(default=None)):
     if x_webhook_secret is None:
         raise HTTPException(401)
-    if x_webhook_secret != WEBHOOK_SECRET:
-        raise HTTPException(403)
+
+    # if x_webhook_secret != WEBHOOK_SECRET:
+    #     raise HTTPException(403)
 
     if not (
         payload.event.action == "update"
@@ -98,5 +100,5 @@ async def post_webhook(background_tasks: BackgroundTasks, payload: WebhookPayloa
         task_id = str(uuid4())
         tasks[task_id] = {'status': 'running'}
         #form_data = DatasetForm(dataset_name=dataset_name)
-        background_tasks.add_task(upload_atlas_task, task_id, payload.repo.name, payload, True)
+        background_tasks.add_task(upload_atlas_task, task_id, payload.repo.name, x_webhook_secret, payload, True)
         return {'task_id': task_id}

templates/form.html

@@ -58,6 +58,9 @@
         <div class="form-group">
             <label for="dataset_name">Dataset Name:</label>
             <input type="text" class="form-control" id="dataset_name" name="dataset_name">
+
+            <label for="atlas_api_token">Atlas API Token:</label>
+            <input type="text" class="form-control" id="atlas_api_token" name="atlas_api_token">
         </div>
         <button type="submit" class="btn btn-primary">Submit</button>
     </form>