updater

server/listing_checks.py

@@ -10,6 +10,7 @@ from dotenv import load_dotenv
 import time
 import pandas as pd
 import uuid
+from typing import Optional
 
 load_dotenv()
 
@@ -30,6 +31,8 @@ drive_service = build("drive", "v3", credentials=credentials)
 sheets_service = build("sheets", "v4", credentials=credentials)
 
 
+# ---------------- Walmart Auth ----------------
+
 def get_walmart_token(client_id: str, client_secret: str) -> str:
     auth_str = f"{client_id}:{client_secret}"
     auth_b64 = base64.b64encode(auth_str.encode()).decode()
@@ -48,88 +51,195 @@ def get_walmart_token(client_id: str, client_secret: str) -> str:
     if response.status_code != 200:
         raise Exception(f"Token request failed: {response.status_code}\n{response.text}")
 
-    # Parse the XML response
+    # XML response
     root = ET.fromstring(response.text)
     token_elem = root.find("accessToken")
-    
     if token_elem is None:
         raise Exception("accessToken not found in response.")
-
     return token_elem.text
 
 
-def get_inventory_quantity(sku: str, access_token: str) -> int:
-    url = f"https://marketplace.walmartapis.com/v3/inventory?sku={sku}"
+# ---------------- New ATS workflow ----------------
+
+def fetch_all_ats(access_token: str, limit: int = 50) -> pd.DataFrame:
+    """
+    Pull all SKUs and total available-to-sell (ATS) across nodes using /v3/inventories (nextCursor).
+    Returns: DataFrame with ['sku','ats']
+    """
+    base_url = "https://marketplace.walmartapis.com/v3/inventories"
     headers = {
         "Authorization": f"Bearer {access_token}",
         "WM_SEC.ACCESS_TOKEN": access_token,
         "WM_SVC.NAME": "Walmart Marketplace",
         "WM_QOS.CORRELATION_ID": str(uuid.uuid4()),
-        "Accept": "application/json"
+        "Accept": "application/json",
     }
 
-    response = requests.get(url, headers=headers)
-    if response.status_code == 200:
-        return response.json().get("quantity", {}).get("amount")
-    else:
-        print(f"⚠️ Inventory fetch failed for {sku} ({response.status_code})")
-        return None
+    rows = []
+    cursor: Optional[str] = None
+    while True:
+        params = {"limit": limit}
+        if cursor:
+            params["nextCursor"] = cursor
+
+        r = requests.get(base_url, headers=headers, params=params)
+        if r.status_code != 200:
+            raise RuntimeError(f"❌ /inventories {r.status_code}: {r.text}")
 
+        payload = r.json()
+        inventories = (payload.get("elements") or {}).get("inventories", []) or []
+        cursor = (payload.get("meta") or {}).get("nextCursor")
 
-def get_item_ids_and_quantities(access_token: str) -> pd.DataFrame:
+        for inv in inventories:
+            sku = inv.get("sku")
+            nodes = inv.get("nodes") or []
+            ats = sum((n.get("availToSellQty", {}) or {}).get("amount", 0) or 0 for n in nodes)
+            rows.append({"sku": sku, "ats": ats})
+
+        if not cursor:
+            break
+
+    return pd.DataFrame(rows, columns=["sku", "ats"])
+
+
+def _extract_gtin_like(item: dict) -> Optional[str]:
+    """
+    Prefer 'gtin', fallback to 'upc', then scan common identifier shapes.
+    """
+    gtin = item.get("gtin")
+    if gtin:
+        return gtin
+    upc = item.get("upc")
+    if upc:
+        return upc
+
+    candidates = []
+    for key in ("productIdentifiers", "identifiers", "additionalProductAttributes", "attributes"):
+        obj = item.get(key)
+        if isinstance(obj, list):
+            for e in obj:
+                if not isinstance(e, dict):
+                    continue
+                t = (e.get("productIdType") or e.get("type") or "").upper()
+                v = e.get("productId") or e.get("id") or e.get("value")
+                if v and t in {"GTIN", "UPC"}:
+                    candidates.append((t, v))
+        elif isinstance(obj, dict):
+            for t, v in obj.items():
+                if isinstance(v, str) and t.upper() in {"GTIN", "UPC"}:
+                    candidates.append((t.upper(), v))
+
+    for t, v in candidates:
+        if t == "GTIN":
+            return v
+    for t, v in candidates:
+        if t == "UPC":
+            return v
+    return None
+
+
+def _get_total_items(access_token: str) -> int:
+    """
+    Probe /v3/items to read total count from meta (meta.totalCount or totalItems).
+    """
+    url = "https://marketplace.walmartapis.com/v3/items"
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "WM_SEC.ACCESS_TOKEN": access_token,
+        "WM_SVC.NAME": "Walmart Marketplace",
+        "WM_QOS.CORRELATION_ID": str(uuid.uuid4()),
+        "Accept": "application/json",
+        "WM_GLOBAL_VERSION": "3.1",
+        "WM_MARKET": "us",
+    }
+    params = {"limit": 1}
+    r = requests.get(url, headers=headers, params=params)
+    if r.status_code != 200:
+        raise RuntimeError(f"❌ /items (probe) {r.status_code}: {r.text}")
+    data = r.json()
+    meta = data.get("meta") or {}
+    total = meta.get("totalCount")
+    if total is None:
+        total = data.get("totalItems")
+    if total is None:
+        total = len(data.get("ItemResponse", []) or [])
+    return int(total)
+
+
+def fetch_all_items_with_gtin_cursor(
+    access_token: str,
+    limit: Optional[int] = None
+) -> pd.DataFrame:
+    """
+    Pull items via /v3/items with nextCursor.
+    If limit is None, auto-sets to total items reported by meta.
+    Returns: ['sku','gtin','productName']
+    """
     url = "https://marketplace.walmartapis.com/v3/items"
     headers = {
         "Authorization": f"Bearer {access_token}",
         "WM_SEC.ACCESS_TOKEN": access_token,
         "WM_SVC.NAME": "Walmart Marketplace",
         "WM_QOS.CORRELATION_ID": str(uuid.uuid4()),
-        "Accept": "application/json"
+        "Accept": "application/json",
+        "WM_GLOBAL_VERSION": "3.1",
+        "WM_MARKET": "us",
     }
 
-    records = []
-    offset = 0
-    limit = 365
-    total_items = None
-    item_counter = 0
+    if limit is None:
+        try:
+            limit = _get_total_items(access_token)
+        except Exception as e:
+            print(f"⚠️ Could not detect total items automatically: {e}. Falling back to 200.")
+            limit = 200
 
-    while True:
-        full_url = f"{url}?limit={limit}&offset={offset}"
-        response = requests.get(full_url, headers=headers)
-        if response.status_code != 200:
-            print(f"❌ Error {response.status_code}: {response.text}")
-            break
+    base_params = {"limit": limit}
+    recs = []
+    cursor: Optional[str] = None
 
-        data = response.json()
-        if total_items is None:
-            total_items = data.get("totalItems", 0)
-            print(f"📦 Total items to process: {total_items}")
+    while True:
+        q = dict(base_params)
+        if cursor:
+            q["nextCursor"] = cursor
+
+        resp = requests.get(url, headers=headers, params=q)
+        if resp.status_code != 200:
+            # If large limit is rejected, back off and paginate.
+            if q.get("limit", 0) > 500:
+                print(f"ℹ️ Large limit={q['limit']} not accepted. Backing off to 200 with pagination.")
+                base_params["limit"] = 200
+                continue
+            raise RuntimeError(f"❌ /items {resp.status_code}: {resp.text}")
+
+        data = resp.json()
+        items = data.get("ItemResponse", []) or []
+        cursor = (data.get("meta") or {}).get("nextCursor")
+
+        for it in items:
+            recs.append({
+                "sku": it.get("sku"),
+                "gtin": _extract_gtin_like(it),
+                "productName": it.get("productName"),
+            })
 
-        items = data.get("ItemResponse", [])
-        if not items:
+        if not cursor:
             break
 
-        for item in items:
-            item_counter += 1
-            item_name = item.get('productName')
-            sku = item.get("sku")
-            gtin = item.get("gtin")
-            qty = get_inventory_quantity(sku, access_token)
-            time.sleep(0.1)
+    return pd.DataFrame(recs, columns=["sku", "gtin", "productName"])
 
-            print(f"🔢 [{item_counter}/{total_items}] gtin: {gtin} | Product: {item_name[:20]} | Qty: {qty}")
 
-            records.append({
-                'productName': item_name,
-                "gtin": gtin,
-                "quantity": qty
-            })
+def fetch_inventory_with_gtin_cursor(access_token: str) -> pd.DataFrame:
+    """
+    Join ATS (from /inventories) with GTIN/productName (from /items) on SKU.
+    Returns: ['sku','gtin','productName','ats']
+    """
+    ats_df = fetch_all_ats(access_token)
+    items_df = fetch_all_items_with_gtin_cursor(access_token, limit=None)
+    merged = ats_df.merge(items_df, on="sku", how="left")
+    return merged[["sku", "gtin", "productName", "ats"]]
 
-        offset += limit
-        if offset >= total_items:
-            break
-
-    return pd.DataFrame(records)
 
+# ---------------- Google Drive helpers ----------------
 
 def list_sheets_in_folder(folder_id):
     query = (
@@ -151,101 +261,94 @@ def load_sheet_as_dataframe(sheet_id, range_name="Sheet1"):
     values = result.get("values", [])
     if not values:
         return pd.DataFrame()
-
-    # First row = header
-    df = pd.DataFrame(values[1:], columns=values[0])
-    return df
+    return pd.DataFrame(values[1:], columns=values[0])
 
 
-sheet_index = {}
-walmart_data = {"df": None}
-
+# --- reactive state ---
+sheet_index = reactive.Value({})          # Shiny tracks changes
+walmart_df  = reactive.Value(pd.DataFrame())  # <- Store ATS join here
 
 def server(input, output, session):
+    # Populate dropdown once (or whenever your Drive listing changes)
     @reactive.Effect
-    def init_dropdown_from_folder():
-        global sheet_index
+    def _init_dropdown_from_folder():
         try:
             sheets = list_sheets_in_folder(GOOGLE_FOLDER_ID)
-            sheet_index = {s['name']: s['id'] for s in sheets}
-            sheet_names = list(sheet_index.keys())
-            print(f"[DEBUG] Found {len(sheet_names)} sheets: {sheet_names}")
-
-            ui.update_select("sheet_dropdown_check", choices=sheet_names)
-
+            idx = {s['name']: s['id'] for s in sheets}
+            sheet_index.set(idx)
+            ui.update_select("sheet_dropdown_check", choices=list(idx.keys()))
         except Exception as e:
             print(f"[ERROR] Failed to list folder contents: {e}")
 
+    # Button press -> fetch token -> fetch ATS+items -> set reactive value
     @reactive.Effect
     @reactive.event(input.load_walmart_data)
-    def load_walmart_data():
+    def _load_walmart_data():
         try:
             print("[DEBUG] Getting Walmart token...")
             token = get_walmart_token(WALMART_CLIENT_ID, WALMART_CLIENT_SECRET)
-            print("[DEBUG] Fetching Walmart inventory data...")
-            walmart_data["df"] = get_item_ids_and_quantities(token)
-            print(f"[DEBUG] Loaded {len(walmart_data['df'])} Walmart items")
+            print("[DEBUG] Fetching Walmart ATS + GTIN ...")
+            df = fetch_inventory_with_gtin_cursor(token)     # <- your new ATS workflow
+            walmart_df.set(df)                               # <- invalidate dependents
+            print(f"[DEBUG] Loaded {len(df)} Walmart items (ATS)")
         except Exception as e:
             print(f"[ERROR] Failed to load Walmart data: {e}")
-            walmart_data["df"] = pd.DataFrame()  # Set empty DataFrame on error
+            walmart_df.set(pd.DataFrame())
 
     @output
     @render.text
     def walmart_status():
-        if walmart_data["df"] is None:
+        df = walmart_df()  # establish dependency
+        if df.empty:
             return "Click 'Load Walmart Data' to fetch inventory data"
-        elif len(walmart_data["df"]) == 0:
-            return "Error loading Walmart data"
-        else:
-            return f"Walmart data loaded: {len(walmart_data['df'])} items"
+        return f"Walmart data loaded: {len(df)} items (ATS)"
 
     @output
     @render.table
     def results_check():
-        sheet_name = input.sheet_dropdown()
-        
+        # Re-run when either the dropdown changes or the walmart_df changes
+        _ = walmart_df()  # establish dependency on data
+        selected = input.sheet_dropdown_check()
+
+        idx = sheet_index()
+        if not selected or selected not in idx:
+            return pd.DataFrame({"status": ["Select a sheet to compare"]})
+
+        if walmart_df().empty:
+            return pd.DataFrame({"status": ["Click 'Load Walmart Data' first"]})
+
         try:
-            if not sheet_name or sheet_name not in sheet_index:
-                return pd.DataFrame({"status": ["Select a sheet to compare"]})
-            
-            if walmart_data["df"] is None:
-                return pd.DataFrame({"status": ["Loading Walmart data..."]})
-
-            print(f"[DEBUG] Loading sheet: {sheet_name}")
-            sheet_id = sheet_index[sheet_name]
+            sheet_id = idx[selected]
             google_df = load_sheet_as_dataframe(sheet_id)
-            
             if google_df.empty:
                 return pd.DataFrame({"error": ["Google sheet is empty"]})
-            
-            walmart_df = walmart_data["df"]
-            
-            # Merge the data - adjust column names as needed
-            if 'walmart_gtin' in google_df.columns:
+
+            wdf = walmart_df()[["sku", "gtin", "productName", "ats"]]
+
+            if "walmart_gtin" in google_df.columns:
                 merged = google_df.merge(
-                    walmart_df[['gtin', 'productName', 'quantity']],
-                    left_on='walmart_gtin',
-                    right_on='gtin',
-                    how='inner'
+                    wdf[["gtin", "productName", "ats"]],
+                    left_on="walmart_gtin",
+                    right_on="gtin",
+                    how="inner",
                 )
-                
-                # Show discrepancies
-                if 'walmart_quantity' in merged.columns:
-                    discrepancies = merged[
-                        merged['walmart_quantity'].astype(int) !=
-                        merged['quantity'].astype(int)
-                    ][['productName', 'walmart_quantity', 'quantity']]
-                    
+                compare_col = (
+                    "walmart_ats"
+                    if "walmart_ats" in merged.columns
+                    else ("walmart_quantity" if "walmart_quantity" in merged.columns else None)
+                )
+                if compare_col:
+                    lhs = pd.to_numeric(merged[compare_col], errors="coerce").fillna(0).astype(int)
+                    rhs = pd.to_numeric(merged["ats"], errors="coerce").fillna(0).astype(int)
+                    discrepancies = merged.loc[lhs.ne(rhs), ["productName", compare_col, "ats"]]
                     if discrepancies.empty:
-                        return pd.DataFrame({
-                            "status": ["No quantity discrepancies found"]
-                        })
+                        return pd.DataFrame({"status": ["No ATS discrepancies found"]})
                     return discrepancies
                 else:
-                    return merged[['productName', 'gtin', 'quantity']]
+                    return merged[["productName", "gtin", "ats"]]
             else:
-                # If no walmart_gtin column, just show the merged data
-                return walmart_df
-                
+                # No GTIN in sheet: just show ATS snapshot
+                return wdf
+
         except Exception as e:
-            return pd.DataFrame({"error": [f"Error: {str(e)}"]})
+            return pd.DataFrame({"error": [f"Error: {e}"]})