""" HubSpot Tasks → Supabase (incremental since a millisecond cursor) Usage from orchestrator: import hubspot_tasks hubspot_tasks.main(since_ms=) Direct CLI: # epoch ms python hubspot_tasks.py 1754025600000 # ISO-8601 python hubspot_tasks.py 2026-01-01T00:00:00Z # Back-compat date (floors to 00:00Z) python hubspot_tasks.py 2026-01-01 """ import os import re import time import logging import datetime from typing import List, Dict, Optional, Tuple, Union import httpx import hubspot from dotenv import load_dotenv from supabase import create_client from hubspot.crm.objects.tasks import ApiException as TasksApiException from hubspot_utils import ( parse_ts, try_parse_int, deduplicate_by_key ) from supabase_utils import ( insert_into_supabase_table, update_sync_metadata ) # ----------------------------------------------------------------------------- # Constants # ----------------------------------------------------------------------------- TASK_PROPERTIES = [ "hs_created_by", "hs_created_by_user_id", "hs_createdate", "hs_date_entered_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342", "hs_engagement_source", "hs_object_id", "hs_object_source_label", "hs_pipeline", "hs_pipeline_stage", "hs_task_missed_due_date", "hs_task_status", "hs_task_subject", "hs_queue_membership_ids", "hs_lastmodifieddate", ] TASK_ENUM_PROPERTIES = [ "hs_object_source_label", "hs_task_status", "hs_queue_membership_ids", ] # ----------------------------------------------------------------------------- # Logging # ----------------------------------------------------------------------------- logging.basicConfig( filename=f"logs/hubspot_tasks_pipeline_{datetime.datetime.now().strftime('%Y-%m-%d')}.log", filemode="a", level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", ) # ----------------------------------------------------------------------------- # Environment # ----------------------------------------------------------------------------- load_dotenv() HUBSPOT_TOKEN = os.getenv("HUBSPOT_TOKEN") SUPABASE_URL = os.getenv("SUPABASE_URL") SUPABASE_SERVICE_ROLE_KEY = os.getenv("SUPABASE_SERVICE_ROLE_KEY") # Optional bootstrap cursor if orchestrator doesn't provide one BOOTSTRAP_SINCE_MS_ENV = os.getenv("HUBSPOT_TASKS_SINCE_MS") if not HUBSPOT_TOKEN: raise RuntimeError("HUBSPOT_TOKEN is not set") if not SUPABASE_URL or not SUPABASE_SERVICE_ROLE_KEY: raise RuntimeError("Supabase env vars are not set") hubspot_client = hubspot.Client.create(access_token=HUBSPOT_TOKEN) supabase_client = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY) # ----------------------------------------------------------------------------- # Time helpers # ----------------------------------------------------------------------------- def _ensure_utc(dt: datetime.datetime) -> datetime.datetime: if dt.tzinfo is None: dt = dt.replace(tzinfo=datetime.timezone.utc) return dt.astimezone(datetime.timezone.utc) def floor_to_utc_midnight(dt: datetime.datetime) -> datetime.datetime: dt = _ensure_utc(dt) return dt.replace(hour=0, minute=0, second=0, microsecond=0) def _parse_iso_like_to_dt(value: str) -> datetime.datetime: if isinstance(value, str) and value.endswith("Z"): value = value[:-1] + "+00:00" dt = datetime.datetime.fromisoformat(value) return _ensure_utc(dt) def to_epoch_ms(dt_or_str: Union[str, datetime.datetime]) -> int: if isinstance(dt_or_str, str): dt = _parse_iso_like_to_dt(dt_or_str) elif isinstance(dt_or_str, datetime.datetime): dt = _ensure_utc(dt_or_str) else: raise TypeError(f"Unsupported type for to_epoch_ms: {type(dt_or_str)}") return int(dt.timestamp() * 1000) def parse_any_ts_ms(value: Optional[Union[str, int, float]]) -> Optional[int]: """ Accepts ms-epoch / sec-epoch / ISO-8601; returns ms since epoch or None. """ if value is None: return None try: v = int(str(value)) if v < 10_000_000_000_000: # seconds → ms v *= 1000 return v except ValueError: pass try: return to_epoch_ms(str(value)) except Exception: logging.warning("Could not parse timestamp value=%r", value) return None # ----------------------------------------------------------------------------- # Pipeline / Stage and enum label maps # ----------------------------------------------------------------------------- def get_enum_label_maps() -> Dict[str, Dict[str, str]]: """ Retrieves label mappings for all enumeration properties on the task object. For each property in TASK_ENUM_PROPERTIES, calls the HubSpot Properties API to get option value-to-label mappings. Pipeline and stage labels are fetched separately from the Pipelines API and stored under 'hs_pipeline' and 'hs_pipeline_stage'. :return: A dict of {property_name: {value: label}}. Empty dict per property on failure. """ label_maps: Dict[str, Dict[str, str]] = {} for prop_name in TASK_ENUM_PROPERTIES: try: prop_info = hubspot_client.crm.properties.core_api.get_by_name( "tasks", prop_name ) label_maps[prop_name] = { opt.value: opt.label for opt in (prop_info.options or []) } except Exception as e: logging.warning( "Failed to fetch options for property '%s': %s", prop_name, str(e) ) label_maps[prop_name] = {} # Pipeline stages are nested under pipelines, not in property options. # Override hs_pipeline and hs_pipeline_stage with pipeline API data. try: pipelines_api = hubspot_client.crm.pipelines.pipelines_api response = pipelines_api.get_all(object_type="tasks") pipeline_map: Dict[str, str] = {} stage_map: Dict[str, str] = {} for pipeline in response.results: pipeline_map[pipeline.id] = pipeline.label for stage in pipeline.stages: stage_map[stage.id] = stage.label label_maps["hs_pipeline"] = pipeline_map label_maps["hs_pipeline_stage"] = stage_map except Exception as e: logging.warning("Failed to fetch task pipeline/stage mappings: %s", str(e)) return label_maps # ----------------------------------------------------------------------------- # Search IDs (ts > since_ms) with property fallback # ----------------------------------------------------------------------------- def _search_task_ids_from(since_ms: int, prop: str) -> List[str]: """ Search tasks where {prop} > since_ms (epoch-ms). Sort ascending so we can advance the cursor monotonically. """ url = "https://api.hubapi.com/crm/v3/objects/tasks/search" headers = { "Authorization": f"Bearer {HUBSPOT_TOKEN}", "Content-Type": "application/json", "Accept": "application/json", } payload = { "filterGroups": [{ "filters": [ {"propertyName": prop, "operator": "GT", "value": str(since_ms)}, ] }], "limit": 100, "sorts": [{"propertyName": prop, "direction": "ASCENDING"}], } ids: List[str] = [] after: Optional[str] = None with httpx.Client(timeout=30.0) as client: while True: body = dict(payload) if after: body["after"] = after resp = client.post(url, headers=headers, json=body) if resp.status_code >= 400: try: logging.error("Task search error for prop '%s': %s", prop, resp.json()) except Exception: logging.error("Task search error for prop '%s': %s", prop, resp.text) resp.raise_for_status() data = resp.json() ids.extend([obj["id"] for obj in data.get("results", []) or []]) after = (data.get("paging") or {}).get("next", {}).get("after") if not after: break time.sleep(0.1) return ids def search_task_ids_after_ms(since_ms: int) -> Tuple[List[str], str]: """ Try these properties in order; return (ids, prop_used) for the first successful search: 1) hs_lastmodifieddate 2) hs_createdate """ props_to_try = ["hs_lastmodifieddate", "hs_createdate"] last_err = None for prop in props_to_try: try: ids = _search_task_ids_from(since_ms, prop) logging.info("Task search with '%s' returned %d IDs.", prop, len(ids)) return ids, prop except httpx.HTTPStatusError as e: last_err = e continue if last_err: raise last_err return [], "hs_lastmodifieddate" # ----------------------------------------------------------------------------- # Read-by-ID → map rows and track max cursor ts # ----------------------------------------------------------------------------- def read_tasks_by_ids( task_ids: List[str], cursor_prop: str, ) -> Tuple[List[Dict], Optional[int]]: """ Read tasks by ID from HubSpot and map them to Supabase row dicts. :param task_ids: List of HubSpot task object IDs to fetch. :param cursor_prop: The property name used in the search (to track max ts). :return: (all_tasks, max_ts_ms) where max_ts_ms is the highest value of cursor_prop seen across all fetched tasks, or None if unavailable. """ if not task_ids: return [], None all_tasks: List[Dict] = [] label_maps = get_enum_label_maps() max_ts_ms: Optional[int] = None def get_label(prop_name: str, value: Optional[str]) -> str: """Return the human-readable label for an enumeration value, or ''.""" if value is None: return "" return label_maps.get(prop_name, {}).get(value, "") for i, tid in enumerate(task_ids, start=1): try: record = hubspot_client.crm.objects.basic_api.get_by_id( object_type="tasks", object_id=tid, properties=TASK_PROPERTIES, archived=False, ) props = record.properties or {} # Track max timestamp for the cursor property used in the search cursor_val = props.get(cursor_prop) ts_ms = parse_any_ts_ms(cursor_val) if ts_ms is not None and (max_ts_ms is None or ts_ms > max_ts_ms): max_ts_ms = ts_ms # hs_task_missed_due_date comes back as the string "true"/"false" missed_due_raw = props.get("hs_task_missed_due_date") if missed_due_raw is None: missed_due = None else: missed_due = missed_due_raw.lower() == "true" all_tasks.append({ "task_id": try_parse_int(props.get("hs_object_id")), "hubspot_created_by": try_parse_int(props.get("hs_created_by")), "hubspot_created_by_user_id": try_parse_int(props.get("hs_created_by_user_id")), "hubspot_createdate": parse_ts(props.get("hs_createdate")), "date_entered_completed": parse_ts(props.get( "hs_date_entered_61bafb31_e7fa_46ed_aaa9_1322438d6e67_1866552342" )), "engagement_source": props.get("hs_engagement_source"), "record_source": props.get("hs_object_source_label"), "record_source_label": get_label( "hs_object_source_label", props.get("hs_object_source_label") ), "pipeline": props.get("hs_pipeline"), "pipeline_label": get_label("hs_pipeline", props.get("hs_pipeline")), "pipeline_stage": props.get("hs_pipeline_stage"), "pipeline_stage_label": get_label( "hs_pipeline_stage", props.get("hs_pipeline_stage") ), "task_missed_due_date": missed_due, "task_status": props.get("hs_task_status"), "task_status_label": get_label("hs_task_status", props.get("hs_task_status")), "task_subject": props.get("hs_task_subject"), "queue_membership_ids": props.get("hs_queue_membership_ids"), }) if i % 200 == 0: logging.info("Read %d tasks...", i) time.sleep(0.05) except httpx.HTTPStatusError as e: logging.error("HTTP error reading task %s: %s", tid, e) except (TasksApiException, httpx.HTTPError) as e: logging.error("Error reading task %s: %s", tid, e) return all_tasks, max_ts_ms # ----------------------------------------------------------------------------- # Upsert # ----------------------------------------------------------------------------- def upsert_tasks(tasks: List[Dict]) -> None: if tasks: insert_into_supabase_table(supabase_client, "hubspot_tasks", tasks) print(f"Upserted {len(tasks)} tasks.") # ----------------------------------------------------------------------------- # Main (timestamp cursor) # ----------------------------------------------------------------------------- def main(since_ms: Optional[int] = None): """ Orchestrates: 1) Search task IDs with > since_ms (property fallback) 2) Read full tasks (track max timestamp for ) 3) Upsert into Supabase 4) Update sync metadata with last_sync_time """ # Resolve since_ms if since_ms is None and BOOTSTRAP_SINCE_MS_ENV: try: since_ms = int(BOOTSTRAP_SINCE_MS_ENV) except ValueError: raise RuntimeError("HUBSPOT_TASKS_SINCE_MS must be an integer (ms) if set.") if since_ms is None: # Default: today@00:00:00Z for first run today0 = floor_to_utc_midnight(datetime.datetime.now(datetime.timezone.utc)) since_ms = to_epoch_ms(today0) print(f"Searching tasks with timestamp > {since_ms} ...") ids, cursor_prop = search_task_ids_after_ms(since_ms) print(f"Search property: {cursor_prop}. Found {len(ids)} task IDs.") now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat() if not ids: print("No tasks beyond the cursor. Updating sync metadata and exiting.") update_sync_metadata(supabase_client, "hubspot_tasks", now_iso) return print("Reading tasks...") tasks, max_ts_ms = read_tasks_by_ids(ids, cursor_prop) print("Upserting into Supabase...") upsert_tasks(tasks) # Advance cursor to max timestamp we actually ingested for the chosen property new_cursor_ms = max_ts_ms if max_ts_ms is not None else since_ms update_sync_metadata(supabase_client, "hubspot_tasks", now_iso) print(f"Tasks sync complete. Advanced cursor to {new_cursor_ms} using prop '{cursor_prop}'.") # ----------------------------------------------------------------------------- # CLI # ----------------------------------------------------------------------------- def _parse_cli_arg_to_ms(arg: str) -> int: """ Accept: - integer epoch ms - ISO-8601 (Z or offset) - YYYY-MM-DD (floors to 00:00Z) """ # epoch ms or seconds if re.fullmatch(r"\d{10,13}", arg): v = int(arg) if v < 10_000_000_000_000: # seconds -> ms v *= 1000 return v # YYYY-MM-DD if re.fullmatch(r"\d{4}-\d{2}-\d{2}", arg): d = datetime.datetime.strptime(arg, "%Y-%m-%d").replace(tzinfo=datetime.timezone.utc) return to_epoch_ms(floor_to_utc_midnight(d)) # ISO-8601 return to_epoch_ms(arg) if __name__ == "__main__": import sys if len(sys.argv) > 1: try: since = _parse_cli_arg_to_ms(sys.argv[1]) except Exception as e: print( f"Invalid timestamp. Provide epoch ms, ISO-8601, or YYYY-MM-DD. Error: {e}" ) sys.exit(1) main(since_ms=since) else: main()