chore: remove tracked generated files and update gitignore

.gitignore

@@ -66,4 +66,8 @@ Real Estate Intelligence Layer.md
 grant_privileges.sql
 agent_prompt.md
 dynamic_tool_calling_features.txt
-Frontend_API_Updates.md
+Frontend_API_Updates.md
+*.npy
+section_titles.json
+test_rpcs.py
+test_api.py

all_rpcs.md

@@ -0,0 +1,330 @@
+| schema_name | function_name             | identity_args                                                                                           | result_type                                                                                                                                                                                                                                                                                  | security_definer | function_definition                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| ----------- | ------------------------- | ------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| public      | get_dashboard_kpis        |                                                                                                         | json                                                                                                                                                                                                                                                                                         | true             | CREATE OR REPLACE FUNCTION public.get_dashboard_kpis()
+ RETURNS json
+ LANGUAGE sql
+ SECURITY DEFINER
+AS $function$
+  SELECT json_build_object(
+    'pricing', json_build_object(
+      'products_tracked',   (SELECT COUNT(*) FROM products),
+      'price_changes_7d',   (
+        SELECT COUNT(*) FROM price_history
+        WHERE created_at >= NOW() - INTERVAL '7 days'
+        AND price != (
+          SELECT price FROM price_history ph2
+          WHERE ph2.product_id = price_history.product_id
+          AND ph2.created_at < price_history.created_at
+          ORDER BY ph2.created_at DESC LIMIT 1
+        )
+      ),
+      'spikes_7d',          (
+        SELECT COUNT(*) FROM v_price_volatility
+        WHERE ABS(pct_above_trailing_avg) >= 25
+        AND created_at >= NOW() - INTERVAL '7 days'
+      ),
+      'last_scrape_status', (
+        SELECT last_status FROM v_scrape_health
+        WHERE job_type = 'PRICE_MONITOR'
+        ORDER BY last_started_at DESC LIMIT 1
+      ),
+      'tracking_since',     (SELECT MIN(created_at) FROM products)
+    ),
+    'leads', json_build_object(
+      'targets_crawled',      (SELECT COUNT(*) FROM lead_targets),
+      'new_leads_7d',         (SELECT COUNT(*) FROM leads WHERE created_at >= NOW() - INTERVAL '7 days'),
+      'total_leads_all_time', (SELECT COUNT(*) FROM leads),
+      'last_scrape_status',   (
+        SELECT last_status FROM v_scrape_health
+        WHERE job_type = 'LEAD_GEN'
+        ORDER BY last_started_at DESC LIMIT 1
+      )
+    ),
+    'real_estate', json_build_object(
+      'properties_tracked', (SELECT COUNT(DISTINCT property_id) FROM v_rate_volatility),
+      'rate_changes_7d',    (
+        SELECT COUNT(*) FROM rate_history
+        WHERE created_at >= NOW() - INTERVAL '7 days'
+        AND nightly_rate IS NOT NULL
+        AND nightly_rate != (
+          SELECT nightly_rate FROM rate_history rh2
+          WHERE rh2.property_id = rate_history.property_id
+          AND rh2.created_at < rate_history.created_at
+          AND rh2.nightly_rate IS NOT NULL
+          ORDER BY rh2.created_at DESC LIMIT 1
+        )
+      ),
+      'spikes_7d',          (
+        SELECT COUNT(*) FROM v_rate_volatility
+        WHERE ABS(pct_above_trailing_avg) >= 25
+        AND recorded_at >= NOW() - INTERVAL '7 days'
+      ),
+      'tracking_since',     (SELECT MIN(created_at) FROM properties),
+      'last_scrape_status', (
+        SELECT COALESCE(json_object_agg(platform, last_status), '{}'::json)
+        FROM v_scrape_health
+        WHERE job_type = 'REAL_ESTATE_MONITOR'
+      )
+    )
+  );
+$function$
+ |
+| public      | get_distance_km           | property_a_id uuid, property_b_id uuid                                                                  | TABLE(distance_km numeric)                                                                                                                                                                                                                                                                   | true             | CREATE OR REPLACE FUNCTION public.get_distance_km(property_a_id uuid, property_b_id uuid)
+ RETURNS TABLE(distance_km numeric)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+  WITH a AS (
+    SELECT latitude, longitude
+    FROM public.properties
+    WHERE id = property_a_id
+  ),
+  b AS (
+    SELECT latitude, longitude
+    FROM public.properties
+    WHERE id = property_b_id
+  )
+  SELECT
+    (
+      6371 * 2 * ASIN(
+        SQRT(
+          POWER(SIN(RADIANS(b.latitude - a.latitude) / 2), 2) +
+          COS(RADIANS(a.latitude)) * COS(RADIANS(b.latitude)) *
+          POWER(SIN(RADIANS(b.longitude - a.longitude) / 2), 2)
+        )
+      )
+    )::numeric(12,4) AS distance_km
+  FROM a, b
+  WHERE a.latitude IS NOT NULL
+    AND a.longitude IS NOT NULL
+    AND b.latitude IS NOT NULL
+    AND b.longitude IS NOT NULL;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| public      | get_market_averages       | market_param text                                                                                       | TABLE(market character varying, active_properties bigint, avg_nightly_rate numeric, min_nightly_rate numeric, max_nightly_rate numeric)                                                                                                                                                      | true             | CREATE OR REPLACE FUNCTION public.get_market_averages(market_param text DEFAULT NULL::text)
+ RETURNS TABLE(market character varying, active_properties bigint, avg_nightly_rate numeric, min_nightly_rate numeric, max_nightly_rate numeric)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+    SELECT 
+        market,
+        COUNT(DISTINCT property_id) AS active_properties,
+        ROUND(AVG(nightly_rate), 2) AS avg_nightly_rate,
+        MIN(nightly_rate) AS min_nightly_rate,
+        MAX(nightly_rate) AS max_nightly_rate
+    FROM public.v_rate_volatility
+    WHERE (market_param IS NULL OR LOWER(market) = LOWER(market_param))
+      AND is_active = true
+      AND nightly_rate IS NOT NULL
+    GROUP BY market;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| public      | get_properties_by_filter  | p_market text, p_platform text, p_available boolean, p_bedrooms integer                                 | TABLE(property_id uuid, property_name character varying, market character varying, platform character varying, bedrooms integer, avg_rating numeric, review_count integer, nightly_rate numeric, pct_above_trailing_avg numeric, is_available boolean, recorded_at timestamp with time zone) | true             | CREATE OR REPLACE FUNCTION public.get_properties_by_filter(p_market text DEFAULT NULL::text, p_platform text DEFAULT NULL::text, p_available boolean DEFAULT NULL::boolean, p_bedrooms integer DEFAULT NULL::integer)
+ RETURNS TABLE(property_id uuid, property_name character varying, market character varying, platform character varying, bedrooms integer, avg_rating numeric, review_count integer, nightly_rate numeric, pct_above_trailing_avg numeric, is_available boolean, recorded_at timestamp with time zone)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+    SELECT DISTINCT ON (property_id)
+        property_id,
+        property_name,
+        market,
+        platform,
+        bedrooms,
+        avg_rating,
+        review_count,
+        nightly_rate,
+        pct_above_trailing_avg,
+        is_available,
+        recorded_at
+    FROM public.v_rate_volatility
+    WHERE (p_market IS NULL OR LOWER(market) = LOWER(p_market))
+      AND (p_platform IS NULL OR LOWER(platform) = LOWER(p_platform))
+      AND (p_available IS NULL OR is_available = p_available)
+      AND (p_bedrooms IS NULL OR bedrooms = p_bedrooms)
+    ORDER BY property_id, recorded_at DESC;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| public      | get_property_rate_changes | property_search text, days_param integer, compare_window_days integer                                   | TABLE(property_id uuid, property_name character varying, market character varying, platform character varying, stay_date timestamp with time zone, nightly_rate numeric, trailing_avg_rate numeric, pct_above_trailing_avg numeric, prev_nightly_rate numeric, pct_change_vs_prev numeric)   | true             | CREATE OR REPLACE FUNCTION public.get_property_rate_changes(property_search text, days_param integer DEFAULT 14, compare_window_days integer DEFAULT 1)
+ RETURNS TABLE(property_id uuid, property_name character varying, market character varying, platform character varying, stay_date timestamp with time zone, nightly_rate numeric, trailing_avg_rate numeric, pct_above_trailing_avg numeric, prev_nightly_rate numeric, pct_change_vs_prev numeric)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+  WITH base AS (
+    SELECT
+      rh.property_id,
+      p.name AS property_name,
+      p.market,
+      p.platform,
+      rh.stay_date,
+      rh.nightly_rate,
+      -- compute trailing avg as avg of previous 7 days (excluding current day)
+      AVG(rh.nightly_rate) OVER (
+        PARTITION BY rh.property_id
+        ORDER BY rh.stay_date
+        ROWS BETWEEN 7 PRECEDING AND 1 PRECEDING
+      ) AS trailing_avg_rate,
+      LAG(rh.nightly_rate, compare_window_days) OVER (
+        PARTITION BY rh.property_id
+        ORDER BY rh.stay_date
+      ) AS prev_nightly_rate
+    FROM public.v_rate_volatility rh
+    JOIN public.properties p
+      ON p.id = rh.property_id
+    WHERE (rh.property_id::text = property_search OR LOWER(rh.property_name) = LOWER(property_search) OR LOWER(rh.property_name) LIKE '%' || LOWER(property_search) || '%')
+      AND rh.recorded_at >= (NOW() - (days_param || ' days')::interval)
+  )
+  SELECT
+    property_id,
+    property_name,
+    market,
+    platform,
+    stay_date,
+    nightly_rate,
+    trailing_avg_rate,
+    CASE
+      WHEN trailing_avg_rate IS NULL OR trailing_avg_rate = 0 OR nightly_rate IS NULL THEN NULL
+      ELSE ((nightly_rate - trailing_avg_rate) / trailing_avg_rate) * 100
+    END AS pct_above_trailing_avg,
+    prev_nightly_rate,
+    CASE
+      WHEN prev_nightly_rate IS NULL OR prev_nightly_rate = 0 OR nightly_rate IS NULL THEN NULL
+      ELSE ((nightly_rate - prev_nightly_rate) / prev_nightly_rate) * 100
+    END AS pct_change_vs_prev
+  FROM base
+  WHERE stay_date IS NOT NULL
+  ORDER BY stay_date ASC;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| public      | get_property_rate_history | property_search text, days_param integer                                                                | TABLE(property_id uuid, property_name character varying, market character varying, stay_date timestamp with time zone, nightly_rate numeric, trailing_avg_rate numeric, pct_above_trailing_avg numeric, is_available boolean)                                                                | true             | CREATE OR REPLACE FUNCTION public.get_property_rate_history(property_search text, days_param integer DEFAULT 30)
+ RETURNS TABLE(property_id uuid, property_name character varying, market character varying, stay_date timestamp with time zone, nightly_rate numeric, trailing_avg_rate numeric, pct_above_trailing_avg numeric, is_available boolean)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+    SELECT 
+        property_id,
+        property_name,
+        market,
+        stay_date,
+        nightly_rate,
+        trailing_avg_rate,
+        pct_above_trailing_avg,
+        is_available
+    FROM public.v_rate_volatility
+    WHERE (property_id::TEXT = property_search OR LOWER(property_name) LIKE '%' || LOWER(property_search) || '%')
+      AND stay_date >= (NOW() - (days_param || ' days')::INTERVAL)
+    ORDER BY stay_date ASC;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| public      | get_spike_alerts          | threshold_param numeric, days_param integer                                                             | TABLE(property_id uuid, property_name character varying, market character varying, platform character varying, nightly_rate numeric, trailing_avg_rate numeric, pct_above_trailing_avg numeric, recorded_at timestamp with time zone)                                                        | true             | CREATE OR REPLACE FUNCTION public.get_spike_alerts(threshold_param numeric DEFAULT 25.0, days_param integer DEFAULT 7)
+ RETURNS TABLE(property_id uuid, property_name character varying, market character varying, platform character varying, nightly_rate numeric, trailing_avg_rate numeric, pct_above_trailing_avg numeric, recorded_at timestamp with time zone)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+    SELECT 
+        property_id,
+        property_name,
+        market,
+        platform,
+        nightly_rate,
+        trailing_avg_rate,
+        pct_above_trailing_avg,
+        recorded_at
+    FROM public.v_rate_volatility
+    WHERE ABS(pct_above_trailing_avg) >= threshold_param
+      AND recorded_at >= (NOW() - (days_param || ' days')::INTERVAL)
+    ORDER BY recorded_at DESC;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| public      | get_tracked_markets       | p_platform text                                                                                         | TABLE(market character varying, active_properties bigint)                                                                                                                                                                                                                                    | true             | CREATE OR REPLACE FUNCTION public.get_tracked_markets(p_platform text DEFAULT NULL::text)
+ RETURNS TABLE(market character varying, active_properties bigint)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+  SELECT p.market,
+         COUNT(DISTINCT p.id) AS active_properties
+  FROM public.properties p
+  WHERE p.is_active = true
+    AND (p_platform IS NULL OR LOWER(p.platform) = LOWER(p_platform))
+    AND p.market IS NOT NULL
+  GROUP BY p.market
+  ORDER BY active_properties DESC, market;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+| public      | match_re_methodology      | query_embedding vector, match_threshold double precision, match_count integer                           | TABLE(id uuid, section_title character varying, chunk_content text, similarity double precision)                                                                                                                                                                                             | true             | CREATE OR REPLACE FUNCTION public.match_re_methodology(query_embedding vector, match_threshold double precision DEFAULT 0.5, match_count integer DEFAULT 3)
+ RETURNS TABLE(id uuid, section_title character varying, chunk_content text, similarity double precision)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+    SELECT
+        kb.id,
+        kb.section_title,
+        kb.chunk_content,
+        1 - (kb.embedding <=> query_embedding) AS similarity
+    FROM public.re_knowledge_base kb
+    WHERE 1 - (kb.embedding <=> query_embedding) > match_threshold
+    ORDER BY kb.embedding <=> query_embedding
+    LIMIT match_count;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| public      | rls_auto_enable           |                                                                                                         | event_trigger                                                                                                                                                                                                                                                                                | true             | CREATE OR REPLACE FUNCTION public.rls_auto_enable()
+ RETURNS event_trigger
+ LANGUAGE plpgsql
+ SECURITY DEFINER
+ SET search_path TO 'pg_catalog'
+AS $function$
+DECLARE
+  cmd record;
+BEGIN
+  FOR cmd IN
+    SELECT *
+    FROM pg_event_trigger_ddl_commands()
+    WHERE command_tag IN ('CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO')
+      AND object_type IN ('table','partitioned table')
+  LOOP
+     IF cmd.schema_name IS NOT NULL AND cmd.schema_name IN ('public') AND cmd.schema_name NOT IN ('pg_catalog','information_schema') AND cmd.schema_name NOT LIKE 'pg_toast%' AND cmd.schema_name NOT LIKE 'pg_temp%' THEN
+      BEGIN
+        EXECUTE format('alter table if exists %s enable row level security', cmd.object_identity);
+        RAISE LOG 'rls_auto_enable: enabled RLS on %', cmd.object_identity;
+      EXCEPTION
+        WHEN OTHERS THEN
+          RAISE LOG 'rls_auto_enable: failed to enable RLS on %', cmd.object_identity;
+      END;
+     ELSE
+        RAISE LOG 'rls_auto_enable: skip % (either system schema or not in enforced list: %.)', cmd.object_identity, cmd.schema_name;
+     END IF;
+  END LOOP;
+END;
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| public      | search_properties         | p_search text, p_market text, p_platform text, p_bedrooms integer, p_available boolean, p_limit integer | TABLE(property_id uuid, property_name character varying, market character varying, platform character varying, bedrooms integer, avg_rating numeric, review_count integer, nightly_rate numeric, pct_above_trailing_avg numeric, is_available boolean, recorded_at timestamp with time zone) | true             | CREATE OR REPLACE FUNCTION public.search_properties(p_search text DEFAULT NULL::text, p_market text DEFAULT NULL::text, p_platform text DEFAULT NULL::text, p_bedrooms integer DEFAULT NULL::integer, p_available boolean DEFAULT NULL::boolean, p_limit integer DEFAULT 50)
+ RETURNS TABLE(property_id uuid, property_name character varying, market character varying, platform character varying, bedrooms integer, avg_rating numeric, review_count integer, nightly_rate numeric, pct_above_trailing_avg numeric, is_available boolean, recorded_at timestamp with time zone)
+ LANGUAGE sql
+ STABLE SECURITY DEFINER
+AS $function$
+  SELECT DISTINCT ON (rvv.property_id)
+    rvv.property_id,
+    rvv.property_name,
+    rvv.market,
+    rvv.platform,
+    rvv.bedrooms,
+    rvv.avg_rating,
+    rvv.review_count,
+    rvv.nightly_rate,
+    rvv.pct_above_trailing_avg,
+    rvv.is_available,
+    rvv.recorded_at
+  FROM public.v_rate_volatility rvv
+  WHERE
+    (p_market IS NULL OR LOWER(rvv.market) = LOWER(p_market))
+    AND (p_platform IS NULL OR LOWER(rvv.platform) = LOWER(p_platform))
+    AND (p_bedrooms IS NULL OR rvv.bedrooms = p_bedrooms)
+    AND (p_available IS NULL OR rvv.is_available = p_available)
+    AND (
+      p_search IS NULL
+      OR rvv.property_name ILIKE '%' || p_search || '%'
+      OR rvv.property_id::text = p_search
+    )
+  ORDER BY rvv.property_id, rvv.recorded_at DESC
+  LIMIT LEAST(GREATEST(p_limit, 1), 200);
+$function$
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |

kb_embeddings.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d025e65f12c0809aa1266d93e87c9c8da61ef1bad866b25ba3ccc3af825f832e
-size 23168

methodology_kb.json

@@ -0,0 +1,145 @@
+[
+  {
+    "section_title": "Calculation Formulas: 7-Day Trailing Average",
+    "chunk_content": "The '7-day trailing average' acts as the pricing baseline for all volatility tracking. Mathematically, it is calculated by taking the mean of a property's nightly rates over a 7-day lookback window. Crucially, this window strictly excludes the current day to ensure that incomplete intraday scraping does not skew the baseline."
+  },
+  {
+    "section_title": "Threshold Definitions: Volatility and Spike Alerts",
+    "chunk_content": "Pricing anomalies, or 'spikes', are mathematically triggered when a property's current nightly rate deviates significantly from its 7-day trailing average. By default, the system flags a spike alert if the absolute percentage deviation is greater than or equal to 25%."
+  },
+  {
+    "section_title": "System Limits: Search and History Windows",
+
+    "chunk_content": "When interacting with the data, property searches will return a maximum of 200 properties at a time. For historical rate tracking, you can look back up to 90 days of history, and compare current rates to past rates up to 14 days ago."
+  },
+  {
+    "section_title": "Global Filters Behavior",
+    "chunk_content": "The Real Estate Rate Monitor page utilizes several global filters to drill down into the data: \n- **Market**: Filters the properties by geographic region (e.g., 'Miami', 'NYC'). \n- **Platform**: Filters properties by listing platform (e.g., 'Airbnb', 'Vrbo'). \n- **Status**: Allows filtering by 'Currently Tracked' (active properties), 'Untracked/Removed' (inactive properties), or 'All Historical' to include everything. \n- **Bedrooms**: Filters properties to match an exact bedroom count. \n- **Stay Dates**: A date range picker ('Start Date' to 'End Date') that filters properties where the check-in 'Stay Date' falls within the selected window. All filters dynamically recalculate the underlying dataset and update the KPIs, maps, and charts in real-time."
+  },
+  {
+    "section_title": "Dashboard KPIs",
+    "chunk_content": "The top-level Key Performance Indicators (KPIs) update dynamically based on the active global filters: \n- **Properties Tracked**: Displays the count of unique properties that match the current filters. \n- **Rate Changes (7D)**: Represents the total number of nightly rate changes detected across all tracked properties over the last 7 days. \n- **25%+ Spikes (7D)**: Represents the count of 'Pricing Anomalies', defined as instances where a property's current nightly rate deviates by 25% or more (either surging up or dropping down) from its own 7-day trailing average, recorded within the last 7 days."
+  },
+  {
+    "section_title": "Visual Indicators and Tooltips",
+    "chunk_content": "There is use specific visual rules to convey property health and status: \n- **Map Colors**: On the geographic map, an Orange marker indicates an 'Active' and available property with normal pricing. An Amber marker indicates a 'Pricing Anomaly' (a rate deviation >= 25% from its average). A Gray marker indicates the property is 'Unavailable' for the tracked dates. \n- **Stale Badge**: If the latest data point for a property is older than 24 hours, the UI fades the row to 60% opacity, labels it 'Stale', and displays 'YES (STALE)' under availability. \n- **Unavailable Tooltip**: When a property is unavailable, a tooltip clarifies: 'Reflects a 2-night stay starting today rather than the property's full calendar. Other dates may still be bookable.' \n- **Sparklines**: If a property is currently unavailable, the 'Rate' column renders a mini line chart (sparkline) showing the trajectory of its last 5 known prices."
+  },
+  {
+    "section_title": "Chart Logic: Nightly Rate History",
+    "chunk_content": "The 'Nightly Rate History' interactive line chart visualizes the pricing trend for a single selected property over time. It plots two distinct series: \n1. **Solid Line (Primary Color)**: The actual 'Nightly Rate' recorded at specific scrape times. \n2. **Dashed Line (Gray/Muted)**: The '7-day Trailing Avg', serving as a benchmark to easily spot when the current rate is surging or dropping below historical norms."
+  },
+  {
+    "section_title": "Property Rate Snapshot Table",
+    "chunk_content": "The snapshot table lists the latest recorded status for properties matching the filters. Its columns include: \n- **Property**: Name and an external link to the live listing. \n- **Market / Platform**: The geographic market and booking site. \n- **Beds / Rating**: Bedroom count, star rating out of 5.0, and total review count. \n- **Stay Date**: The specific check-in date being tracked. \n- **Rate**: The current nightly rate. If unavailable, it shows a sparkline of the last 5 known prices instead. \n- **vs 7d Avg**: The percentage difference between the current rate and the 7-day average. Positive values are green, negative values are red, and deviations >= 25% turn amber and bold. \n- **Avail.**: Shows 'YES', 'NO', or 'YES (STALE)'. \n- **Last Checked**: Human-readable time since the property was last scraped (e.g., '15m ago' or 'As of Aug 8')."
+  },
+  {
+    "section_title": "Scraping Frequency",
+    "chunk_content": "Listings are monitored exactly four times per day. This frequent checking ensures that we capture any short-term price adjustments and booking status changes throughout the daily cycle."
+  },
+  {
+    "section_title": "Stale Data Status",
+    "chunk_content": "A property is flagged as 'Stale' if our system fails to fetch new pricing information for over 12 hours. This acts as an immediate warning that the data displayed may no longer reflect the live market conditions."
+  },
+  {
+    "section_title": "2-Night Minimum Booking Rule",
+    "chunk_content": "The system defines a property as 'Unavailable' if it does not have a consecutive two-night opening starting from the current date. We enforce a two-night check-in window because most premium short-term rentals require a minimum two-night stay, rendering single-night gaps essentially unbookable for our target audience."
+  },
+  {
+    "section_title": "Lookahead Calendar Limits",
+    "chunk_content": "Our availability checks do not scan the entire upcoming calendar year. Instead, they focus strictly on immediate, short-term availability (the upcoming 2-night window) to gauge current market tightness and sudden surges in demand."
+  },
+  {
+    "section_title": "Primary Tracking Markets",
+    "chunk_content": "We actively track and monitor properties in two primary markets: the Miami area and the New York City/New Jersey Metro region. These locations were selected due to their high volume of short-term rentals and volatile pricing dynamics."
+  },
+  {
+    "section_title": "2026 World Cup Strategy",
+    "chunk_content": "Our strategic focus on the Miami and NYC/NJ Metro markets was directly tied to the 2026 FIFA World Cup Final. By monitoring these specific areas, we captured early rate surges, supply constraints, and pricing anomalies as demand built for the event."
+  },
+  {
+    "section_title": "Active Platform Monitoring",
+    "chunk_content": "Airbnb is our primary platform for active daily monitoring. We continuously fetch live rates and availability from Airbnb because it provides the most reliable and consistent visibility into daily pricing changes for our targeted properties."
+  },
+  {
+    "section_title": "Historical Platform Constraints",
+    "chunk_content": "Properties listed on Vrbo are currently marked as 'Historical' within our system. This is due to platform accessibility configurations that make daily automated fetching difficult. While we no longer fetch fresh daily rates from Vrbo, their past data remains visible to serve as a baseline comparison against current market trends."
+  },
+  {
+    "section_title": "Defining Pricing Anomalies",
+    "chunk_content": "A pricing anomaly occurs when a property's nightly rate suddenly deviates significantly from its recent norms. Our system specifically flags a change as an anomaly if the rate spikes or drops by 25 percent or more compared to its trailing average."
+  },
+  {
+    "section_title": "The 7-Day Trailing Average",
+    "chunk_content": "To determine what a 'normal' price is for a property, we calculate a 7-day trailing average. This means we take the mean average of the nightly rates recorded over the previous six days, creating a rolling benchmark that smooths out daily fluctuations."
+  },
+  {
+    "section_title": "Property Inactivity and Removal",
+    "chunk_content": "If a property is removed from our designated tracking list, the system automatically marks it as inactive. This ensures that our data models and averages only reflect properties that are actively managed and currently relevant to our analysis."
+  },
+  {
+    "section_title": "Handling Unavailable Pricing",
+    "chunk_content": "When a property is booked or otherwise unavailable for the target check-in window, the platform does not report a zero dollar rate. Instead, the rate is treated as a null or unknown value to prevent zero-dollar entries from artificially lowering the market averages."
+  },
+  {
+    "section_title": "Market Average Calculation",
+  
+    "chunk_content": "When calculating the average nightly rate for a specific market, the system only includes properties that are currently active and have a known, non-null nightly rate. This ensures the reported average accurately reflects the actual booking cost."
+  },
+  {
+    "section_title": "Discounted Rate Capture",
+    "chunk_content": "When a platform displays a discounted price (e.g., a crossed-out original price next to a lower promotional price), our system is designed to capture the final discounted amount as the actual nightly rate, ensuring our data reflects what a customer would actually pay."
+  },
+  {
+    "section_title": "Host Identification and Fallbacks",
+
+    "chunk_content": "We extract the host's name or profile identifier for each listing to track multi-property operators. If the host's name is not readily available in the standard data structure, the system uses alternative methods to locate and save the host's profile link for later analysis."
+  },
+  {
+    "section_title": "Property Key Grouping",
+    "chunk_content": "To prevent duplicate counting, our system uses a unique property key to group listings that represent the exact same physical rental unit, even if that unit is listed across multiple different booking platforms."
+  },
+  {
+    "section_title": "Rate Volatility Tracking",
+    "chunk_content": "We maintain a dedicated view of rate volatility that compares the current recorded rate of every property against its 7-day historical benchmark. This allows analysts to instantly sort and identify the most aggressive price changes in a market."
+  },
+  {
+    "section_title": "Consecutive Failure Tolerance",
+    "chunk_content": "If the system repeatedly encounters \"not found\" errors when attempting to view a property, it increments a failure counter. This helps distinguish between a temporary network glitch and a property that has been permanently removed by the host."
+  },
+  {
+    "section_title": "Event-Driven Rate Surges",
+    "chunk_content": "The system expects to see significant, sudden rate surges in localized geographic pockets. By isolating data to specific markets like Miami and NYC, analysts can attribute these surges to major events, such as the World Cup, rather than seasonal trends."
+  },
+  {
+    "section_title": "Missing Data Prevention",
+    "chunk_content": "To maintain data integrity, any automated extraction that fails to find standard pricing formats will escalate to a secondary analysis method. This ensures we maximize our data capture rate even when booking platforms subtly change their page layouts."
+  },
+  {
+    "section_title": "Data Isolation by Platform",
+    "chunk_content": "While we monitor multiple listing sites, data is strictly categorized by platform. This prevents the distinct pricing algorithms, fees, and booking rules of Airbnb from being improperly mixed with historical data from platforms like Vrbo."
+  },
+  {
+    "section_title": "Automated Stale Record Filtering",
+    "chunk_content": "When calculating live market averages or compiling reports on active inventory, the system automatically filters out any property records marked as 'Stale'. This guarantees that decision-makers are only reviewing the most current, actionable intelligence."
+  },
+      {
+        "section_title": "Availability & 2-Night Window Definition",
+        "chunk_content": "A listing marked as 'Unavailable' means no open booking dates were detected within a 2-night check-in window starting from the current date. The system does not look ahead across full calendar months; it tracks consecutive 2-night availability as an immediate signal."
+    },
+    {
+        "section_title": "7-Day Trailing Average Benchmark",
+        "chunk_content": "The 7-day trailing average rate is calculated per property by taking the mean nightly price recorded across the prior 6 daily scrapes. Rate volatility alerts trigger when a property's current nightly rate strays 25% or more above or below this baseline."
+    },
+    {
+        "section_title": "Scrape Cadence & Data Refresh",
+        "chunk_content": "Listings are scraped 4 times daily to capture short-term rate adjustments and booking updates. Status 'Stale' indicates a scraper job failed to return fresh price points within the last 12 hours."
+    },
+    {
+        "section_title": "World Cup 2026 Strategic Focus",
+        "chunk_content": "The Real Estate Rate Monitor specifically tracks short-term rental inventory across NYC/NJ Metro and Miami markets to capture rate surges, supply constraints, and pricing dynamic anomalies leading up to the 2026 World Cup Final."
+    },
+    {
+        "section_title": "Vrbo Historical Tracking Status",
+        "chunk_content": "Vrbo properties are flagged as 'Historical' following platform scraping accessibility adjustments. Historical listings remain visible for baseline comparisons, but fresh daily rates are actively tracked via Airbnb endpoints."
+    }
+]

services/groq_service.py

@@ -1,31 +1,55 @@
 import json
 import re
+import os
+import numpy as np
 from groq import Groq
 from services.tools import REAL_ESTATE_TOOLS
 from services.supabase_service import execute_tool_rpc, search_methodology_rag
+from services.embedding_service import get_embedding_model
 from config import GROQ_API_KEY
 
 session_history = {}
 
 groq_client = Groq(api_key=GROQ_API_KEY)
+embedder = get_embedding_model()
+
+try:
+    if os.path.exists("section_title_embeddings.npy") and os.path.exists("section_titles.json"):
+        section_title_embeddings = np.load("section_title_embeddings.npy")
+        with open("section_titles.json", "r", encoding="utf-8") as f:
+            section_titles = json.load(f)
+        print(f"Loaded {len(section_titles)} local section titles for pre-routing.")
+    else:
+        section_title_embeddings = None
+        section_titles = []
+except Exception as e:
+    print(f"Failed to load local section title embeddings: {e}")
+    section_title_embeddings = None
+    section_titles = []
 
 ROUTER_PROMPT = """You are the classification router for the Joule Dynamics Real Estate Intelligence Layer.
-Analyze the user query and classify it into EXACTLY ONE of six classifications:
+Analyze the user query and classify it into EXACTLY ONE of five classifications:
 
 1. "OUT_OF_SCOPE": Query asks about Leads, Lead-capture, Pricing Monitor data, general web crawling, or cross-system topics outside of the /real-estate page.
 2. "PATH_A": Query asks a live-data question (prices, spikes, availability, market averages, KPIs, specific listing rates).
 3. "PATH_B": Query asks a methodology/system design question (7-day average definition, 2-night check-in window, 4x daily scrape cadence, Vrbo status, World Cup strategy).
-4. "PATH_C": Query asks a general real estate market context question that does not require live data metrics from our system.
-5. "BOTH": Query requires BOTH explaining a methodology concept AND fetching live data metrics.
-6. "GREETING": User is saying hello, thanking the assistant, or making casual conversation without asking a specific question.
+4. "BOTH": Query requires BOTH explaining a methodology concept AND fetching live data metrics.
+5. "GREETING": User is saying hello, thanking the assistant, or making casual conversation without asking a specific question.
 
 Respond ONLY with valid JSON matching this schema:
-{"classification": "OUT_OF_SCOPE" | "PATH_A" | "PATH_B" | "PATH_C" | "BOTH" | "GREETING", "reason": "1-sentence justification"}
+{"classification": "OUT_OF_SCOPE" | "PATH_A" | "PATH_B" | "BOTH" | "GREETING", "reason": "1-sentence justification"}
 """
 
 SYNTHESIS_PROMPT = """You are the B2B Real Estate Intelligence Assistant for Joule Dynamics.
 You provide precise data analysis to real estate investors and property managers reviewing short-term rental market performance.
 
+IMMUTABLE SYSTEM BOUNDARIES & HARD FACTS:
+1. TRACKED MARKETS: You ONLY track two markets: 'NYC/NJ Metro' and 'Miami'. 
+2. TRACKED PLATFORMS: You ONLY track two platforms: 'Airbnb' (Active daily tracking) and 'Vrbo' (Historical data only).
+3. ABSOLUTE FORBIDDEN ENTITIES: You must NEVER list, suggest, or mention any other cities (e.g., Los Angeles, Chicago, Houston, Orlando) or other booking platforms (e.g., Booking.com, Expedia, Tripadvisor). If asked about them, state plainly that they are outside Joule Dynamics' current tracking scope.
+4. ZERO FABRICATION: Every single price, rate change percentage, property count, and availability status MUST come directly from a returned tool output JSON. If a tool returns no data or an error, state: "I don't have that information in the current real estate scope."
+5. NO RAW SQL: Never attempt to write or generate SQL queries. Rely strictly on the registered tool RPCs provided.
+
 OPERATIONAL RULES:
 1. NEVER FABRICATE DATA: Rely strictly on returned tool outputs or retrieved methodology chunks. NEVER write ad-hoc SQL. You must exclusively use the registered tools provided.
 2. ZERO GUESSING: If data or methodology is missing, state plainly: "I don't have that information in the current real estate scope."
@@ -38,11 +62,33 @@ OPERATIONAL RULES:
 """
 
 async def process_chat_message(user_query: str, session_id: str, session_context: dict) -> dict:
+    global section_title_embeddings, section_titles
+
     if session_id not in session_history:
         session_history[session_id] = []
 
-    # STEP 1: Routing Classification (llama-3.1-8b-instant)
-    router_messages = [{"role": "system", "content": ROUTER_PROMPT}]
+    # STEP 1: Pre-Router Local Vector Search
+    pre_check_hint = ""
+    if section_title_embeddings is None and os.path.exists("section_title_embeddings.npy"):
+        try:
+            section_title_embeddings = np.load("section_title_embeddings.npy")
+            with open("section_titles.json", "r", encoding="utf-8") as f:
+                section_titles = json.load(f)
+        except Exception as e:
+            print(f"Lazy load failed: {e}")
+
+    if section_title_embeddings is not None and len(section_titles) > 0:
+        query_emb = embedder.encode([user_query], normalize_embeddings=True)[0]
+        sims = section_title_embeddings @ query_emb
+        top_idx = np.argsort(sims)[::-1][:3]
+        matched_titles = [section_titles[i] for i in top_idx if sims[i] >= 0.45]
+        
+        if matched_titles:
+            pre_check_hint = f"\n\nLocal Methodology Pre-Check: High similarity match with section titles: {matched_titles}. Consider classifying as PATH_B or BOTH."
+
+    router_sys_prompt = ROUTER_PROMPT + pre_check_hint
+    router_messages = [{"role": "system", "content": router_sys_prompt}]
+    
     # To save tokens on routing, only include the last 4 messages of history
     router_messages.extend(session_history[session_id][-4:])
     router_messages.append({"role": "user", "content": user_query})
@@ -63,7 +109,7 @@ async def process_chat_message(user_query: str, session_id: str, session_context
         routing = {}
         
     classification = routing.get("classification", "PATH_A")
-    if classification not in ["OUT_OF_SCOPE", "PATH_A", "PATH_B", "PATH_C", "BOTH", "GREETING"]:
+    if classification not in ["OUT_OF_SCOPE", "PATH_A", "PATH_B", "BOTH", "GREETING"]:
         classification = "PATH_A"
 
     # Guardrail: Immediate short-circuit if Out of Scope
@@ -86,10 +132,6 @@ async def process_chat_message(user_query: str, session_id: str, session_context
     tool_results = []
     rag_chunks = []
 
-    # STEP 2: Execute Vector Search if Path B or Both
-    if classification in ["PATH_B", "BOTH"]:
-        rag_chunks = await search_methodology_rag(user_query)
-
     # STEP 2: Execute Vector Search if Path B or Both
     if classification in ["PATH_B", "BOTH"]:
         rag_chunks = await search_methodology_rag(user_query)
@@ -97,11 +139,6 @@ async def process_chat_message(user_query: str, session_id: str, session_context
     messages = [
         {"role": "system", "content": SYNTHESIS_PROMPT}
     ]
-    if classification == "PATH_C":
-        messages.append({
-            "role": "system",
-            "content": "Note: Because this is a PATH_C query, you must prepend a strict disclaimer stating: 'Note: This is general market context, not live data from the Joule Dynamics tracking system.'"
-        })
 
     messages.extend(session_history[session_id])
 

unified_ingest.py

@@ -1,41 +1,19 @@
 import os
 import uuid
+import json
 import numpy as np
 from services.embedding_service import get_embedding_model
 from kb_docs import KB_DOCS
 from supabase import create_client, Client
 from config import SUPABASE_URL, SUPABASE_KEY
 
-METHODOLOGY_DOCS = [
-    {
-        "section_title": "Availability & 2-Night Window Definition",
-        "chunk_content": "A listing marked as 'Unavailable' means no open booking dates were detected within a 2-night check-in window starting from the current date. The system does not look ahead across full calendar months; it tracks consecutive 2-night availability as an immediate signal."
-    },
-    {
-        "section_title": "7-Day Trailing Average Benchmark",
-        "chunk_content": "The 7-day trailing average rate is calculated per property by taking the mean nightly price recorded across the prior 6 daily scrapes. Rate volatility alerts trigger when a property's current nightly rate strays 25% or more above or below this baseline."
-    },
-    {
-        "section_title": "Scrape Cadence & Data Refresh",
-        "chunk_content": "Listings are scraped 4 times daily to capture short-term rate adjustments and booking updates. Status 'Stale' indicates a scraper job failed to return fresh price points within the last 12 hours."
-    },
-    {
-        "section_title": "World Cup 2026 Strategic Focus",
-        "chunk_content": "The Real Estate Rate Monitor specifically tracks short-term rental inventory across NYC/NJ Metro and Miami markets to capture rate surges, supply constraints, and pricing dynamic anomalies leading up to the 2026 World Cup Final."
-    },
-    {
-        "section_title": "Vrbo Historical Tracking Status",
-        "chunk_content": "Vrbo properties are flagged as 'Historical' following platform scraping accessibility adjustments. Historical listings remain visible for baseline comparisons, but fresh daily rates are actively tracked via Airbnb endpoints."
-    }
-]
-
 def ensure_ingested():
     """
     Idempotent function that seeds both the local numpy embeddings and the remote Supabase database.
     """
     print("Verifying ingestion state...")
     embedder = get_embedding_model()
-    
+
     # 1. Local Numpy KB for Amara (Idempotent)
     if not os.path.exists("kb_embeddings.npy"):
         print("kb_embeddings.npy not found, generating local embeddings...")
@@ -55,19 +33,25 @@ def ensure_ingested():
             kb_embeddings = embedder.encode(texts, normalize_embeddings=True)
             np.save("kb_embeddings.npy", kb_embeddings)
             print(f"Embedded {len(texts)} KB docs and saved to kb_embeddings.npy")
-            
-    # 2. Remote Supabase KB for Real Estate (Idempotent)
+
+    # 2. Read Real Estate Methodology KB
+    methodology_docs = []
+    if os.path.exists("methodology_kb.json"):
+        with open("methodology_kb.json", "r", encoding="utf-8") as f:
+            methodology_docs = json.load(f)
+
+    # 3. Remote Supabase KB for Real Estate (Idempotent)
     supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
-    
-    # Check what already exists in Supabase
     try:
         existing_res = supabase.table("re_knowledge_base").select("section_title").execute()
         existing_titles = {row["section_title"] for row in existing_res.data}
     except Exception as e:
         print(f"Failed to query Supabase: {e}")
         existing_titles = set()
-        
-    for doc in METHODOLOGY_DOCS:
+
+    for doc in methodology_docs:
+        if "section_title" not in doc or "chunk_content" not in doc:
+            continue
         if doc["section_title"] not in existing_titles:
             print(f"Seeding missing chunk to Supabase: {doc['section_title']}")
             vector = embedder.encode(doc["chunk_content"]).tolist()
@@ -75,7 +59,7 @@ def ensure_ingested():
                 "id": str(uuid.uuid4()),
                 "section_title": doc["section_title"],
                 "chunk_content": doc["chunk_content"],
-                "embedding": vector
+                "embedding": vector,
             }
             try:
                 supabase.table("re_knowledge_base").insert(payload).execute()
@@ -83,5 +67,28 @@ def ensure_ingested():
             except Exception as e:
                 print(f"Failed to seed {doc['section_title']}: {e}")
 
+    # 4. Local Pre-Router Embeddings for section_titles (Idempotent)
+    section_titles = [doc["section_title"] for doc in methodology_docs if "section_title" in doc]
+    
+    regenerate_titles = False
+    if not os.path.exists("section_title_embeddings.npy") or not os.path.exists("section_titles.json"):
+        regenerate_titles = True
+    else:
+        try:
+            with open("section_titles.json", "r", encoding="utf-8") as f:
+                saved_titles = json.load(f)
+            if saved_titles != section_titles:
+                regenerate_titles = True
+        except Exception:
+            regenerate_titles = True
+
+    if regenerate_titles and section_titles:
+        print("Generating section_title_embeddings.npy...")
+        title_embeddings = embedder.encode(section_titles, normalize_embeddings=True)
+        np.save("section_title_embeddings.npy", title_embeddings)
+        with open("section_titles.json", "w", encoding="utf-8") as f:
+            json.dump(section_titles, f, indent=2)
+        print(f"Saved {len(section_titles)} section title embeddings for pre-routing.")
+
 if __name__ == "__main__":
     ensure_ingested()