Prepare app for Hugging Face Space (include index files)

classified_tickets_phase2.json

@@ -0,0 +1,724 @@
+[
+  {
+    "id": "TICKET-245",
+    "subject": "Connecting Snowflake to Atlan - required permissions?",
+    "body": "Hi team, we're trying to set up our primary Snowflake production database as a new source in Atlan, but the connection keeps failing. We've tried using our standard service account, but it's not working. Our entire BI team is blocked on this integration for a major upcoming project, so it's quite urgent. Could you please provide a definitive list of the exact permissions and credentials needed on the Snowflake side to get this working? Thanks.",
+    "classification": {
+      "id": "TICKET-245",
+      "topic_tags": [
+        "Product"
+      ],
+      "topic_scores": {
+        "Product": 0.2536420226097107,
+        "Connector": 0.21206863224506378,
+        "Sensitive data": 0.15129192173480988,
+        "SSO": 0.09911767393350601,
+        "How-to": 0.09207943081855774,
+        "Best practices": 0.06830797344446182,
+        "Glossary": 0.05438947677612305,
+        "API/SDK": 0.047063831239938736,
+        "Lineage": 0.022039052098989487
+      },
+      "sentiment": "Angry",
+      "priority": "P0"
+    }
+  },
+  {
+    "id": "TICKET-246",
+    "subject": "Which connectors automatically capture lineage?",
+    "body": "Hello, I'm new to Atlan and trying to understand the lineage capabilities. The documentation mentions automatic lineage, but it's not clear which of our connectors (we use Fivetran, dbt, and Tableau) support this out-of-the-box. We need to present a clear picture of our data flow to leadership next week. Can you explain how lineage capture differs for these tools?",
+    "classification": {
+      "id": "TICKET-246",
+      "topic_tags": [
+        "Lineage"
+      ],
+      "topic_scores": {
+        "Lineage": 0.7043173909187317,
+        "Product": 0.06416956335306168,
+        "Sensitive data": 0.05306507274508476,
+        "Connector": 0.05121781677007675,
+        "How-to": 0.03940143436193466,
+        "Best practices": 0.03072594851255417,
+        "SSO": 0.02197032794356346,
+        "Glossary": 0.018482282757759094,
+        "API/SDK": 0.01665017567574978
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-247",
+    "subject": "Deployment of Atlan agent for private data lake",
+    "body": "Our primary data lake is hosted on-premise within a secure VPC and is not exposed to the internet. We understand we need to use the Atlan agent for this, but the setup instructions are a bit confusing for our security team. This is a critical source for us, and we can't proceed with our rollout until we get this connected. Can you provide a detailed deployment guide or connect us with a technical expert?",
+    "classification": {
+      "id": "TICKET-247",
+      "topic_tags": [
+        "Sensitive data"
+      ],
+      "topic_scores": {
+        "Sensitive data": 0.4692264795303345,
+        "How-to": 0.13292741775512695,
+        "Product": 0.1230754628777504,
+        "SSO": 0.06440062075853348,
+        "Connector": 0.058400604873895645,
+        "Best practices": 0.05110899731516838,
+        "Glossary": 0.04289798066020012,
+        "API/SDK": 0.036751993000507355,
+        "Lineage": 0.021210432052612305
+      },
+      "sentiment": "Angry",
+      "priority": "P0"
+    }
+  },
+  {
+    "id": "TICKET-248",
+    "subject": "How to surface sample rows and schema changes?",
+    "body": "Hi, we've successfully connected our Redshift cluster, and the assets are showing up. However, my data analysts are asking how they can see sample data or recent schema changes directly within Atlan without having to go back to Redshift. Is this feature available? I feel like I'm missing something obvious.",
+    "classification": {
+      "id": "TICKET-248",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.5293518900871277,
+        "Sensitive data": 0.12791359424591064,
+        "Connector": 0.11278416216373444,
+        "Product": 0.07702862471342087,
+        "Best practices": 0.035295821726322174,
+        "SSO": 0.03469147905707359,
+        "API/SDK": 0.030026594176888466,
+        "Lineage": 0.026599474251270294,
+        "Glossary": 0.0263083353638649
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-249",
+    "subject": "Exporting lineage view for a specific table",
+    "body": "For our quarterly audit, I need to provide a complete upstream and downstream lineage diagram for our core `fact_orders` table. I can see the lineage perfectly in the UI, but I can't find an option to export this view as an image or PDF. This is a hard requirement from our compliance team and the deadline is approaching fast. Please help!",
+    "classification": {
+      "id": "TICKET-249",
+      "topic_tags": [
+        "Lineage"
+      ],
+      "topic_scores": {
+        "Lineage": 0.6261916756629944,
+        "Sensitive data": 0.10166331380605698,
+        "Product": 0.08675874769687653,
+        "How-to": 0.05723349004983902,
+        "Best practices": 0.038304269313812256,
+        "Connector": 0.025699065998196602,
+        "SSO": 0.02333613485097885,
+        "Glossary": 0.022767778486013412,
+        "API/SDK": 0.018045460805296898
+      },
+      "sentiment": "Angry",
+      "priority": "P0"
+    }
+  },
+  {
+    "id": "TICKET-250",
+    "subject": "Importing lineage from Airflow jobs",
+    "body": "We run hundreds of ETL jobs in Airflow, and we need to see that lineage reflected in Atlan. I've read that Atlan can integrate with Airflow, but how do we configure it to correctly map our DAGs to the specific datasets they are transforming? The current documentation is a bit high-level.",
+    "classification": {
+      "id": "TICKET-250",
+      "topic_tags": [
+        "Lineage"
+      ],
+      "topic_scores": {
+        "Lineage": 0.6405794024467468,
+        "How-to": 0.07339552789926529,
+        "Sensitive data": 0.06700912863016129,
+        "Product": 0.04682222753763199,
+        "Best practices": 0.03841668739914894,
+        "SSO": 0.03800088167190552,
+        "API/SDK": 0.03554345667362213,
+        "Glossary": 0.03051498532295227,
+        "Connector": 0.029717685654759407
+      },
+      "sentiment": "Positive",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-251",
+    "subject": "Using the Visual Query Builder",
+    "body": "I'm a business analyst and not very comfortable with writing complex SQL. I was excited to see the Visual Query Builder in Atlan, but I'm having trouble figuring out how to join multiple tables and save my query for later use. Is there a tutorial or a quick guide you can point me to?",
+    "classification": {
+      "id": "TICKET-251",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.20408794283866882,
+        "Sensitive data": 0.19889409840106964,
+        "Best practices": 0.1357269585132599,
+        "Product": 0.1232258677482605,
+        "Glossary": 0.08558867126703262,
+        "Connector": 0.07553622126579285,
+        "API/SDK": 0.06800619512796402,
+        "SSO": 0.06174485385417938,
+        "Lineage": 0.047189224511384964
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-252",
+    "subject": "Programmatic extraction of lineage",
+    "body": "Our internal data science team wants to build a custom application that analyzes metadata propagation delays. To do this, we need to programmatically extract lineage data from Atlan via an API. Does the API expose lineage information, and if so, could you provide an example of the endpoint and the structure of the response?",
+    "classification": {
+      "id": "TICKET-252",
+      "topic_tags": [
+        "Lineage"
+      ],
+      "topic_scores": {
+        "Lineage": 0.40903380513191223,
+        "How-to": 0.12547598779201508,
+        "Sensitive data": 0.12143483757972717,
+        "Product": 0.12053452432155609,
+        "Best practices": 0.0685185045003891,
+        "API/SDK": 0.06657088547945023,
+        "Glossary": 0.03478899598121643,
+        "SSO": 0.029303470626473427,
+        "Connector": 0.024339014664292336
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-253",
+    "subject": "Upstream lineage to Snowflake view not working",
+    "body": "This is infuriating. We have a critical Snowflake view, `finance.daily_revenue`, that is built from three upstream tables. Atlan is correctly showing the downstream dependencies, but the upstream lineage is completely missing. This makes the view untrustworthy for our analysts. We've re-run the crawler multiple times. What could be causing this? This is a huge problem for us.",
+    "classification": {
+      "id": "TICKET-253",
+      "topic_tags": [
+        "Lineage"
+      ],
+      "topic_scores": {
+        "Lineage": 0.6192259192466736,
+        "Sensitive data": 0.14630955457687378,
+        "Product": 0.08304131031036377,
+        "SSO": 0.03572985529899597,
+        "Glossary": 0.03130059689283371,
+        "How-to": 0.02567395195364952,
+        "Best practices": 0.025151418522000313,
+        "Connector": 0.018504904583096504,
+        "API/SDK": 0.015062461607158184
+      },
+      "sentiment": "Angry",
+      "priority": "P0"
+    }
+  },
+  {
+    "id": "TICKET-254",
+    "subject": "How to create a business glossary and link terms in bulk?",
+    "body": "We are migrating our existing business glossary from a spreadsheet into Atlan. We have over 500 terms. Manually creating each one and linking them to thousands of assets seems impossible. Is there a bulk import feature using CSV or an API to create terms and link them to assets? This is blocking our entire governance initiative.",
+    "classification": {
+      "id": "TICKET-254",
+      "topic_tags": [
+        "How-to",
+        "Glossary"
+      ],
+      "topic_scores": {
+        "How-to": 0.4522898495197296,
+        "Glossary": 0.4390662610530853,
+        "Sensitive data": 0.021999819204211235,
+        "Product": 0.01799585483968258,
+        "SSO": 0.017875080928206444,
+        "Best practices": 0.016796141862869263,
+        "API/SDK": 0.01395806111395359,
+        "Lineage": 0.010493746027350426,
+        "Connector": 0.009525217115879059
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-255",
+    "subject": "Creating a custom role for data stewards",
+    "body": "I'm trying to set up a custom role for our data stewards. They need permission to edit descriptions and link glossary terms, but they should NOT have permission to run queries or change connection settings. I'm looking at the default roles, but none of them fit perfectly. How can I create a new role with this specific set of permissions?",
+    "classification": {
+      "id": "TICKET-255",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.25342780351638794,
+        "Glossary": 0.2120532989501953,
+        "Product": 0.1341620683670044,
+        "Sensitive data": 0.08589652925729752,
+        "Connector": 0.08091939240694046,
+        "Best practices": 0.07565046101808548,
+        "API/SDK": 0.05833519995212555,
+        "SSO": 0.055307161062955856,
+        "Lineage": 0.044248051941394806
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-256",
+    "subject": "Mapping Active Directory groups to Atlan teams",
+    "body": "Our company policy requires us to manage all user access through Active Directory groups. We need to map our existing AD groups (e.g., 'data-analyst-finance', 'data-engineer-core') to teams within Atlan to automatically grant the correct permissions. I can't find the settings for this. How is this configured?",
+    "classification": {
+      "id": "TICKET-256",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.24342772364616394,
+        "Sensitive data": 0.19628578424453735,
+        "Best practices": 0.12237202376127243,
+        "Product": 0.11746875196695328,
+        "Glossary": 0.08366748690605164,
+        "SSO": 0.077210932970047,
+        "Connector": 0.0679486095905304,
+        "API/SDK": 0.04821234196424484,
+        "Lineage": 0.04340638965368271
+      },
+      "sentiment": "Angry",
+      "priority": "P0"
+    }
+  },
+  {
+    "id": "TICKET-257",
+    "subject": "RBAC for assets vs. glossaries",
+    "body": "I need clarification on how Atlan's role-based access control works. If a user is denied access to a specific Snowflake schema, can they still see the glossary terms that are linked to the tables in that schema? I need to ensure our PII governance is airtight.",
+    "classification": {
+      "id": "TICKET-257",
+      "topic_tags": [
+        "Sensitive data",
+        "Glossary"
+      ],
+      "topic_scores": {
+        "Sensitive data": 0.5029798746109009,
+        "Glossary": 0.325946182012558,
+        "How-to": 0.03934895619750023,
+        "Best practices": 0.03307223320007324,
+        "Product": 0.0300066489726305,
+        "Lineage": 0.020622894167900085,
+        "SSO": 0.019720233976840973,
+        "Connector": 0.018421567976474762,
+        "API/SDK": 0.009881414473056793
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-258",
+    "subject": "Process for onboarding asset owners",
+    "body": "We've started identifying owners for our key data assets. What is the recommended workflow in Atlan to assign these owners and automatically notify them? We want to make sure they are aware of their responsibilities without us having to send manual emails for every assignment.",
+    "classification": {
+      "id": "TICKET-258",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.2538345754146576,
+        "Sensitive data": 0.242264062166214,
+        "Best practices": 0.15336112678050995,
+        "Product": 0.11627519130706787,
+        "SSO": 0.0775536373257637,
+        "Glossary": 0.055225685238838196,
+        "Connector": 0.043340008705854416,
+        "API/SDK": 0.03405303508043289,
+        "Lineage": 0.02409267984330654
+      },
+      "sentiment": "Positive",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-259",
+    "subject": "How does Atlan surface sensitive fields like PII?",
+    "body": "Our security team is evaluating Atlan and their main question is around PII and sensitive data. How does Atlan automatically identify fields containing PII? What are our options to apply tags or masks to these fields once they are identified to prevent unauthorized access?",
+    "classification": {
+      "id": "TICKET-259",
+      "topic_tags": [
+        "Sensitive data"
+      ],
+      "topic_scores": {
+        "Sensitive data": 0.8192521333694458,
+        "How-to": 0.09408269077539444,
+        "Product": 0.022458547726273537,
+        "SSO": 0.01460132747888565,
+        "Best practices": 0.013057924807071686,
+        "Connector": 0.011657687835395336,
+        "Glossary": 0.008879464119672775,
+        "Lineage": 0.008671694435179234,
+        "API/SDK": 0.0073384749703109264
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-260",
+    "subject": "Authentication methods for APIs and SDKs",
+    "body": "We are planning to build several automations using the Atlan API and Python SDK. What authentication methods are supported? Is it just API keys, or can we use something like OAuth? We have a strict policy that requires key rotation every 90 days, so we need to understand how to manage this programmatically.",
+    "classification": {
+      "id": "TICKET-260",
+      "topic_tags": [
+        "API/SDK"
+      ],
+      "topic_scores": {
+        "API/SDK": 0.876763105392456,
+        "Product": 0.029590053483843803,
+        "How-to": 0.02613384835422039,
+        "Sensitive data": 0.022094866260886192,
+        "Best practices": 0.01735789142549038,
+        "Glossary": 0.012602042406797409,
+        "Connector": 0.005761501379311085,
+        "SSO": 0.005750404670834541,
+        "Lineage": 0.003946291748434305
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-261",
+    "subject": "Enabling and testing SAML SSO",
+    "body": "We are ready to enable SAML SSO with our Okta instance. However, we are very concerned about disrupting our active users if the configuration is wrong. Is there a way to test the SSO configuration for a specific user or group before we enable it for the entire workspace?",
+    "classification": {
+      "id": "TICKET-261",
+      "topic_tags": [
+        "SSO"
+      ],
+      "topic_scores": {
+        "SSO": 0.8798295855522156,
+        "Sensitive data": 0.024509701877832413,
+        "Best practices": 0.023383136838674545,
+        "Product": 0.02332100085914135,
+        "How-to": 0.015699787065386772,
+        "Glossary": 0.010842178016901016,
+        "Connector": 0.008763168007135391,
+        "Lineage": 0.006830730475485325,
+        "API/SDK": 0.006820705719292164
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-262",
+    "subject": "SSO login not assigning user to correct group",
+    "body": "I've just had a new user, 'test.user@company.com', log in via our newly configured SSO. They were authenticated successfully, but they were not added to the 'Data Analysts' group as expected based on our SAML assertions. This is preventing them from accessing any assets. What could be the reason for this mis-assignment?",
+    "classification": {
+      "id": "TICKET-262",
+      "topic_tags": [
+        "SSO"
+      ],
+      "topic_scores": {
+        "SSO": 0.857302725315094,
+        "Sensitive data": 0.049394093453884125,
+        "Product": 0.021338628605008125,
+        "Best practices": 0.01603107526898384,
+        "Connector": 0.015189331956207752,
+        "Lineage": 0.011370034888386726,
+        "How-to": 0.010868269950151443,
+        "Glossary": 0.01049866247922182,
+        "API/SDK": 0.008007260970771313
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-263",
+    "subject": "Integration with existing DLP or secrets manager",
+    "body": "Does Atlan have the capability to integrate with third-party tools like a DLP (Data Loss Prevention) solution or a secrets manager like HashiCorp Vault? We need to ensure that connection credentials and sensitive metadata classifications are handled by our central security systems.",
+    "classification": {
+      "id": "TICKET-263",
+      "topic_tags": [
+        "Sensitive data"
+      ],
+      "topic_scores": {
+        "Sensitive data": 0.42641857266426086,
+        "Product": 0.1320459097623825,
+        "Connector": 0.09498800337314606,
+        "Best practices": 0.08815968036651611,
+        "How-to": 0.08476880192756653,
+        "SSO": 0.055265337228775024,
+        "Lineage": 0.05023786425590515,
+        "Glossary": 0.03815902769565582,
+        "API/SDK": 0.029956845566630363
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-264",
+    "subject": "Accessing audit logs for compliance reviews",
+    "body": "Our compliance team needs to perform a quarterly review of all activities within Atlan. They need to know who accessed what data, who made permission changes, etc. Where can we find these audit logs, and is there a way to export them or pull them via an API for our records?",
+    "classification": {
+      "id": "TICKET-264",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.3180291950702667,
+        "Sensitive data": 0.15853844583034515,
+        "Best practices": 0.10388915240764618,
+        "Glossary": 0.1034572422504425,
+        "Product": 0.08544151484966278,
+        "Connector": 0.07629270106554031,
+        "API/SDK": 0.07074188441038132,
+        "SSO": 0.04728706181049347,
+        "Lineage": 0.03632282093167305
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-265",
+    "subject": "How to programmatically create an asset using the REST API?",
+    "body": "I'm trying to create a new custom asset (a 'Report') using the REST API, but my requests keep failing with a 400 error. The API documentation is a bit sparse on the required payload structure for creating new entities. Could you provide a basic cURL or Python `requests` example of what a successful request body should look like?",
+    "classification": {
+      "id": "TICKET-265",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.7019529938697815,
+        "Product": 0.07501153647899628,
+        "Sensitive data": 0.05422753840684891,
+        "API/SDK": 0.04047072306275368,
+        "Best practices": 0.03833013400435448,
+        "Glossary": 0.029218755662441254,
+        "Connector": 0.022240931168198586,
+        "SSO": 0.02166176587343216,
+        "Lineage": 0.01688558980822563
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-266",
+    "subject": "SDK availability and Python example",
+    "body": "I'm a data engineer and prefer using SDKs over raw API calls. Which languages do you provide SDKs for? I'm particularly interested in Python. Where can I find the installation instructions (e.g., PyPI package name) and a short code snippet for a common task, like creating a new glossary term?",
+    "classification": {
+      "id": "TICKET-266",
+      "topic_tags": [
+        "API/SDK"
+      ],
+      "topic_scores": {
+        "API/SDK": 0.5466265678405762,
+        "Glossary": 0.11553136259317398,
+        "How-to": 0.08963733166456223,
+        "Product": 0.06004762277007103,
+        "Sensitive data": 0.05992572009563446,
+        "Best practices": 0.056482475250959396,
+        "SSO": 0.032319601625204086,
+        "Connector": 0.020940439775586128,
+        "Lineage": 0.018488900735974312
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-267",
+    "subject": "How do webhooks work in Atlan?",
+    "body": "I'm exploring using webhooks to send real-time notifications from Atlan to our internal Slack channel. I need to understand what types of events (e.g., asset updated, term created) can trigger a webhook. Also, how do we validate that the incoming payloads are genuinely from Atlan? Do you support payload signing?",
+    "classification": {
+      "id": "TICKET-267",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.6155452728271484,
+        "Product": 0.0822950229048729,
+        "Sensitive data": 0.07453867048025131,
+        "Connector": 0.04730972647666931,
+        "SSO": 0.04478367045521736,
+        "Best practices": 0.04298403859138489,
+        "Glossary": 0.04120447859168053,
+        "API/SDK": 0.03110373578965664,
+        "Lineage": 0.0202353373169899
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-268",
+    "subject": "Triggering an AWS Lambda from Atlan events",
+    "body": "We have a workflow where we want to trigger a custom AWS Lambda function whenever a specific Atlan tag (e.g., 'PII-Confirmed') is added to an asset. What is the recommended and most secure way to set this up? Should we use webhooks pointing to an API Gateway, or is there a more direct integration?",
+    "classification": {
+      "id": "TICKET-268",
+      "topic_tags": [
+        "Sensitive data"
+      ],
+      "topic_scores": {
+        "Sensitive data": 0.6416335701942444,
+        "How-to": 0.07063692808151245,
+        "Product": 0.06787507981061935,
+        "Best practices": 0.05057653412222862,
+        "Connector": 0.04566250368952751,
+        "SSO": 0.04065138101577759,
+        "Glossary": 0.034141793847084045,
+        "API/SDK": 0.0318898931145668,
+        "Lineage": 0.016932277008891106
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-269",
+    "subject": "When to use Atlan automations vs. external services?",
+    "body": "I see that Atlan has a built-in 'Automations' feature. I'm trying to decide if I should use this to manage a workflow or if I should use an external service like Zapier or our own Airflow instance. Could you provide some guidance or examples on what types of workflows are best suited for the native automations versus an external tool?",
+    "classification": {
+      "id": "TICKET-269",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.31247150897979736,
+        "Product": 0.1856827437877655,
+        "Best practices": 0.12680679559707642,
+        "Sensitive data": 0.11848830431699753,
+        "Glossary": 0.0666823759675026,
+        "SSO": 0.059052322059869766,
+        "API/SDK": 0.05207480862736702,
+        "Connector": 0.0423959419131279,
+        "Lineage": 0.03634531795978546
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-270",
+    "subject": "Connector failed to crawl - where to check logs?",
+    "body": "URGENT: Our nightly Snowflake crawler failed last night and no new metadata was ingested. This is a critical failure as our morning reports are now missing lineage information. Where can I find the detailed error logs for the crawler run to understand what went wrong? I need to fix this ASAP.",
+    "classification": {
+      "id": "TICKET-270",
+      "topic_tags": [
+        "Connector"
+      ],
+      "topic_scores": {
+        "Connector": 0.7956897616386414,
+        "Lineage": 0.10316770523786545,
+        "Sensitive data": 0.036917030811309814,
+        "Product": 0.021942023187875748,
+        "SSO": 0.012663195841014385,
+        "How-to": 0.009871531277894974,
+        "Glossary": 0.008735943585634232,
+        "Best practices": 0.005630514584481716,
+        "API/SDK": 0.005382323171943426
+      },
+      "sentiment": "Angry",
+      "priority": "P0"
+    }
+  },
+  {
+    "id": "TICKET-271",
+    "subject": "Asset extracted but not published to Atlan",
+    "body": "This is very strange. I'm looking at the crawler logs, and I can see that the asset 'schema.my_table' was successfully extracted from the source. However, when I search for this table in the Atlan UI, it doesn't appear. It seems like it's getting stuck somewhere between extraction and publishing. Can you please investigate the root cause?",
+    "classification": {
+      "id": "TICKET-271",
+      "topic_tags": [
+        "Product"
+      ],
+      "topic_scores": {
+        "Product": 0.3031204640865326,
+        "Sensitive data": 0.1932554543018341,
+        "Best practices": 0.1285562515258789,
+        "SSO": 0.08312994241714478,
+        "Connector": 0.07090307772159576,
+        "How-to": 0.0644526332616806,
+        "Lineage": 0.06026022136211395,
+        "API/SDK": 0.05608906224370003,
+        "Glossary": 0.04023294523358345
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-272",
+    "subject": "How to measure adoption and generate reports?",
+    "body": "My manager is asking for metrics on our Atlan usage to justify the investment. I need to generate a report showing things like the number of active users, most frequently queried tables, and the number of assets with assigned owners. Does Atlan have a reporting or dashboarding feature for this?",
+    "classification": {
+      "id": "TICKET-272",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.7723440527915955,
+        "Product": 0.057156000286340714,
+        "Sensitive data": 0.04286324605345726,
+        "Best practices": 0.03300917148590088,
+        "Connector": 0.02577182836830616,
+        "Glossary": 0.020914847031235695,
+        "SSO": 0.019454676657915115,
+        "API/SDK": 0.014868470840156078,
+        "Lineage": 0.013617790304124355
+      },
+      "sentiment": "Angry",
+      "priority": "P1"
+    }
+  },
+  {
+    "id": "TICKET-273",
+    "subject": "Best practices for catalog hygiene",
+    "body": "We've been using Atlan for six months, and our catalog is already starting to get a bit messy with duplicate assets and stale metadata from old tests. As we roll this out to more teams, what are some common best practices or features within Atlan that can help us maintain good catalog hygiene and prevent this problem from getting worse?",
+    "classification": {
+      "id": "TICKET-273",
+      "topic_tags": [
+        "Best practices"
+      ],
+      "topic_scores": {
+        "Best practices": 0.8946393728256226,
+        "Product": 0.03143538162112236,
+        "How-to": 0.0247127003967762,
+        "Sensitive data": 0.014560189098119736,
+        "Glossary": 0.009846128523349762,
+        "Connector": 0.007913430221378803,
+        "Lineage": 0.007438138592988253,
+        "SSO": 0.006103003863245249,
+        "API/SDK": 0.0033516811672598124
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  },
+  {
+    "id": "TICKET-274",
+    "subject": "How to scale Atlan across multiple business units?",
+    "body": "We are planning a global rollout of Atlan to multiple business units, each with its own data sources and governance teams. We're looking for advice on the best way to structure our Atlan instance. Should we use separate workspaces, or can we achieve isolation using teams and permissions within a single workspace while maintaining a consistent governance model?",
+    "classification": {
+      "id": "TICKET-274",
+      "topic_tags": [
+        "How-to"
+      ],
+      "topic_scores": {
+        "How-to": 0.5874260067939758,
+        "Best practices": 0.10692328214645386,
+        "Product": 0.09862891584634781,
+        "Sensitive data": 0.0611591637134552,
+        "Glossary": 0.038298100233078,
+        "SSO": 0.037935610860586166,
+        "Connector": 0.029464809224009514,
+        "Lineage": 0.023105787113308907,
+        "API/SDK": 0.017058314755558968
+      },
+      "sentiment": "Angry",
+      "priority": "P2"
+    }
+  }
+]

faiss_index.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb0b698008aae120d30504c1784f0797dc800066b27d63740c079b2466549d8c
+size 18026541

sample_tickets.json

@@ -0,0 +1,152 @@
+[
+ {
+   "id": "TICKET-245",
+   "subject": "Connecting Snowflake to Atlan - required permissions?",
+   "body": "Hi team, we're trying to set up our primary Snowflake production database as a new source in Atlan, but the connection keeps failing. We've tried using our standard service account, but it's not working. Our entire BI team is blocked on this integration for a major upcoming project, so it's quite urgent. Could you please provide a definitive list of the exact permissions and credentials needed on the Snowflake side to get this working? Thanks."
+ },
+ {
+   "id": "TICKET-246",
+   "subject": "Which connectors automatically capture lineage?",
+   "body": "Hello, I'm new to Atlan and trying to understand the lineage capabilities. The documentation mentions automatic lineage, but it's not clear which of our connectors (we use Fivetran, dbt, and Tableau) support this out-of-the-box. We need to present a clear picture of our data flow to leadership next week. Can you explain how lineage capture differs for these tools?"
+ },
+ {
+   "id": "TICKET-247",
+   "subject": "Deployment of Atlan agent for private data lake",
+   "body": "Our primary data lake is hosted on-premise within a secure VPC and is not exposed to the internet. We understand we need to use the Atlan agent for this, but the setup instructions are a bit confusing for our security team. This is a critical source for us, and we can't proceed with our rollout until we get this connected. Can you provide a detailed deployment guide or connect us with a technical expert?"
+ },
+ {
+   "id": "TICKET-248",
+   "subject": "How to surface sample rows and schema changes?",
+   "body": "Hi, we've successfully connected our Redshift cluster, and the assets are showing up. However, my data analysts are asking how they can see sample data or recent schema changes directly within Atlan without having to go back to Redshift. Is this feature available? I feel like I'm missing something obvious."
+ },
+ {
+   "id": "TICKET-249",
+   "subject": "Exporting lineage view for a specific table",
+   "body": "For our quarterly audit, I need to provide a complete upstream and downstream lineage diagram for our core `fact_orders` table. I can see the lineage perfectly in the UI, but I can't find an option to export this view as an image or PDF. This is a hard requirement from our compliance team and the deadline is approaching fast. Please help!"
+ },
+ {
+   "id": "TICKET-250",
+   "subject": "Importing lineage from Airflow jobs",
+   "body": "We run hundreds of ETL jobs in Airflow, and we need to see that lineage reflected in Atlan. I've read that Atlan can integrate with Airflow, but how do we configure it to correctly map our DAGs to the specific datasets they are transforming? The current documentation is a bit high-level."
+ },
+ {
+   "id": "TICKET-251",
+   "subject": "Using the Visual Query Builder",
+   "body": "I'm a business analyst and not very comfortable with writing complex SQL. I was excited to see the Visual Query Builder in Atlan, but I'm having trouble figuring out how to join multiple tables and save my query for later use. Is there a tutorial or a quick guide you can point me to?"
+ },
+ {
+   "id": "TICKET-252",
+   "subject": "Programmatic extraction of lineage",
+   "body": "Our internal data science team wants to build a custom application that analyzes metadata propagation delays. To do this, we need to programmatically extract lineage data from Atlan via an API. Does the API expose lineage information, and if so, could you provide an example of the endpoint and the structure of the response?"
+ },
+ {
+   "id": "TICKET-253",
+   "subject": "Upstream lineage to Snowflake view not working",
+   "body": "This is infuriating. We have a critical Snowflake view, `finance.daily_revenue`, that is built from three upstream tables. Atlan is correctly showing the downstream dependencies, but the upstream lineage is completely missing. This makes the view untrustworthy for our analysts. We've re-run the crawler multiple times. What could be causing this? This is a huge problem for us."
+ },
+ {
+   "id": "TICKET-254",
+   "subject": "How to create a business glossary and link terms in bulk?",
+   "body": "We are migrating our existing business glossary from a spreadsheet into Atlan. We have over 500 terms. Manually creating each one and linking them to thousands of assets seems impossible. Is there a bulk import feature using CSV or an API to create terms and link them to assets? This is blocking our entire governance initiative."
+ },
+ {
+   "id": "TICKET-255",
+   "subject": "Creating a custom role for data stewards",
+   "body": "I'm trying to set up a custom role for our data stewards. They need permission to edit descriptions and link glossary terms, but they should NOT have permission to run queries or change connection settings. I'm looking at the default roles, but none of them fit perfectly. How can I create a new role with this specific set of permissions?"
+ },
+ {
+   "id": "TICKET-256",
+   "subject": "Mapping Active Directory groups to Atlan teams",
+   "body": "Our company policy requires us to manage all user access through Active Directory groups. We need to map our existing AD groups (e.g., 'data-analyst-finance', 'data-engineer-core') to teams within Atlan to automatically grant the correct permissions. I can't find the settings for this. How is this configured?"
+ },
+ {
+   "id": "TICKET-257",
+   "subject": "RBAC for assets vs. glossaries",
+   "body": "I need clarification on how Atlan's role-based access control works. If a user is denied access to a specific Snowflake schema, can they still see the glossary terms that are linked to the tables in that schema? I need to ensure our PII governance is airtight."
+ },
+ {
+   "id": "TICKET-258",
+   "subject": "Process for onboarding asset owners",
+   "body": "We've started identifying owners for our key data assets. What is the recommended workflow in Atlan to assign these owners and automatically notify them? We want to make sure they are aware of their responsibilities without us having to send manual emails for every assignment."
+ },
+ {
+   "id": "TICKET-259",
+   "subject": "How does Atlan surface sensitive fields like PII?",
+   "body": "Our security team is evaluating Atlan and their main question is around PII and sensitive data. How does Atlan automatically identify fields containing PII? What are our options to apply tags or masks to these fields once they are identified to prevent unauthorized access?"
+ },
+ {
+   "id": "TICKET-260",
+   "subject": "Authentication methods for APIs and SDKs",
+   "body": "We are planning to build several automations using the Atlan API and Python SDK. What authentication methods are supported? Is it just API keys, or can we use something like OAuth? We have a strict policy that requires key rotation every 90 days, so we need to understand how to manage this programmatically."
+ },
+ {
+   "id": "TICKET-261",
+   "subject": "Enabling and testing SAML SSO",
+   "body": "We are ready to enable SAML SSO with our Okta instance. However, we are very concerned about disrupting our active users if the configuration is wrong. Is there a way to test the SSO configuration for a specific user or group before we enable it for the entire workspace?"
+ },
+ {
+   "id": "TICKET-262",
+   "subject": "SSO login not assigning user to correct group",
+   "body": "I've just had a new user, 'test.user@company.com', log in via our newly configured SSO. They were authenticated successfully, but they were not added to the 'Data Analysts' group as expected based on our SAML assertions. This is preventing them from accessing any assets. What could be the reason for this mis-assignment?"
+ },
+ {
+   "id": "TICKET-263",
+   "subject": "Integration with existing DLP or secrets manager",
+   "body": "Does Atlan have the capability to integrate with third-party tools like a DLP (Data Loss Prevention) solution or a secrets manager like HashiCorp Vault? We need to ensure that connection credentials and sensitive metadata classifications are handled by our central security systems."
+ },
+ {
+   "id": "TICKET-264",
+   "subject": "Accessing audit logs for compliance reviews",
+   "body": "Our compliance team needs to perform a quarterly review of all activities within Atlan. They need to know who accessed what data, who made permission changes, etc. Where can we find these audit logs, and is there a way to export them or pull them via an API for our records?"
+ },
+ {
+   "id": "TICKET-265",
+   "subject": "How to programmatically create an asset using the REST API?",
+   "body": "I'm trying to create a new custom asset (a 'Report') using the REST API, but my requests keep failing with a 400 error. The API documentation is a bit sparse on the required payload structure for creating new entities. Could you provide a basic cURL or Python `requests` example of what a successful request body should look like?"
+ },
+ {
+   "id": "TICKET-266",
+   "subject": "SDK availability and Python example",
+   "body": "I'm a data engineer and prefer using SDKs over raw API calls. Which languages do you provide SDKs for? I'm particularly interested in Python. Where can I find the installation instructions (e.g., PyPI package name) and a short code snippet for a common task, like creating a new glossary term?"
+ },
+ {
+   "id": "TICKET-267",
+   "subject": "How do webhooks work in Atlan?",
+   "body": "I'm exploring using webhooks to send real-time notifications from Atlan to our internal Slack channel. I need to understand what types of events (e.g., asset updated, term created) can trigger a webhook. Also, how do we validate that the incoming payloads are genuinely from Atlan? Do you support payload signing?"
+ },
+ {
+   "id": "TICKET-268",
+   "subject": "Triggering an AWS Lambda from Atlan events",
+   "body": "We have a workflow where we want to trigger a custom AWS Lambda function whenever a specific Atlan tag (e.g., 'PII-Confirmed') is added to an asset. What is the recommended and most secure way to set this up? Should we use webhooks pointing to an API Gateway, or is there a more direct integration?"
+ },
+ {
+   "id": "TICKET-269",
+   "subject": "When to use Atlan automations vs. external services?",
+   "body": "I see that Atlan has a built-in 'Automations' feature. I'm trying to decide if I should use this to manage a workflow or if I should use an external service like Zapier or our own Airflow instance. Could you provide some guidance or examples on what types of workflows are best suited for the native automations versus an external tool?"
+ },
+ {
+   "id": "TICKET-270",
+   "subject": "Connector failed to crawl - where to check logs?",
+   "body": "URGENT: Our nightly Snowflake crawler failed last night and no new metadata was ingested. This is a critical failure as our morning reports are now missing lineage information. Where can I find the detailed error logs for the crawler run to understand what went wrong? I need to fix this ASAP."
+ },
+ {
+   "id": "TICKET-271",
+   "subject": "Asset extracted but not published to Atlan",
+   "body": "This is very strange. I'm looking at the crawler logs, and I can see that the asset 'schema.my_table' was successfully extracted from the source. However, when I search for this table in the Atlan UI, it doesn't appear. It seems like it's getting stuck somewhere between extraction and publishing. Can you please investigate the root cause?"
+ },
+ {
+   "id": "TICKET-272",
+   "subject": "How to measure adoption and generate reports?",
+   "body": "My manager is asking for metrics on our Atlan usage to justify the investment. I need to generate a report showing things like the number of active users, most frequently queried tables, and the number of assets with assigned owners. Does Atlan have a reporting or dashboarding feature for this?"
+ },
+ {
+   "id": "TICKET-273",
+   "subject": "Best practices for catalog hygiene",
+   "body": "We've been using Atlan for six months, and our catalog is already starting to get a bit messy with duplicate assets and stale metadata from old tests. As we roll this out to more teams, what are some common best practices or features within Atlan that can help us maintain good catalog hygiene and prevent this problem from getting worse?"
+ },
+ {
+   "id": "TICKET-274",
+   "subject": "How to scale Atlan across multiple business units?",
+   "body": "We are planning a global rollout of Atlan to multiple business units, each with its own data sources and governance teams. We're looking for advice on the best way to structure our Atlan instance. Should we use separate workspaces, or can we achieve isolation using teams and permissions within a single workspace while maintaining a consistent governance model?"
+ }
+]

src/app.py

@@ -0,0 +1,228 @@
+# src/app.py
+import streamlit as st
+import pandas as pd
+import json
+from pathlib import Path
+from data_loader import load_tickets
+from classifier import classify_ticket, classify_all_and_save
+
+# NEW: import RAG handler
+try:
+    from rag import handle_rag_query
+except Exception:
+    handle_rag_query = None  # we'll handle absence gracefully
+
+# Config
+st.set_page_config(page_title="Atlan - Support Copilot (Phase 3)", layout="wide")
+ROOT = Path(__file__).parent.parent.resolve()   # project root
+CLASSIFIED_PATH = ROOT.joinpath("classified_tickets_phase2.json")
+
+st.title("Atlan — Support Copilot (Phase 3)")
+st.markdown(
+    "**Phase 3:** Zero-shot topic classification + HF sentiment + rule-based priority + RAG (retrieval-augmented generation). "
+    "This demo shows bulk classification and an interactive agent with RAG."
+)
+
+# Sidebar controls
+st.sidebar.header("Controls")
+use_saved = st.sidebar.checkbox("Load pre-saved classified file (if available)", value=True)
+run_classify_all = st.sidebar.button("Classify ALL tickets & Save (Phase 2)")
+reload_ui = st.sidebar.button("Reload UI")
+
+# NEW: RAG options in sidebar
+st.sidebar.markdown("### RAG options")
+use_openai = st.sidebar.checkbox("Use OpenAI for generation (if API key set)", value=False)
+top_k = st.sidebar.slider("RAG: number of passages to retrieve", min_value=1, max_value=10, value=5)
+
+# Safe reload: attempt API call, otherwise show instruction to refresh
+if reload_ui:
+    try:
+        st.experimental_rerun()
+    except Exception:
+        st.info("Automatic reload isn't supported by this Streamlit version. Please refresh the browser page to reload the UI.")
+
+# Load tickets (original) — call the loader without forcing a path so it uses its default
+try:
+    tickets = load_tickets()   # loader default = ../sample_tickets.json (project root)
+except Exception as e:
+    st.error("Could not load sample tickets. Ensure sample_tickets.json exists at the project root (one level above src/).")
+    st.exception(e)
+    tickets = []
+
+# If user asked to classify all, run classification and save (uses classifier defaults)
+if run_classify_all:
+    with st.spinner("Running classification on all tickets (models may load on first run)..."):
+        try:
+            out_path = classify_all_and_save()  # defaults -> saves to ../classified_tickets_phase2.json
+            st.success(f"Classified and saved to: {out_path}")
+        except Exception as e:
+            st.error("Error during batch classification. See details below.")
+            st.exception(e)
+
+# Try to load pre-saved classified file (if requested and exists)
+classified_data = None
+if use_saved and CLASSIFIED_PATH.exists():
+    try:
+        classified_data = json.loads(CLASSIFIED_PATH.read_text(encoding="utf-8"))
+    except Exception as e:
+        st.warning("Could not read the saved classified file; falling back to live classification.")
+        st.exception(e)
+
+tab1, tab2 = st.tabs(["Bulk Classification Dashboard", "Interactive Agent (demo + RAG)"])
+
+with tab1:
+    st.header("Bulk ticket classification")
+    st.write("This view shows all tickets with their inferred topic tags, sentiment, and priority.")
+
+    rows = []
+    # If we have pre-classified data, use that (faster). Otherwise classify on the fly.
+    if classified_data:
+        for entry in classified_data:
+            c = entry.get("classification", {})
+            rows.append({
+                "id": entry.get("id"),
+                "subject": entry.get("subject"),
+                "topic_tags": ", ".join(c.get("topic_tags", [])),
+                "sentiment": c.get("sentiment", ""),
+                "priority": c.get("priority", ""),
+            })
+    else:
+        # Live classify (will call HF pipelines lazily)
+        with st.spinner("Classifying tickets (zero-shot)... this may take a few seconds on first run"):
+            for t in tickets:
+                try:
+                    c = classify_ticket(t)
+                except Exception as e:
+                    st.error(f"Error classifying ticket {t.get('id')}: {e}")
+                    c = {"topic_tags": [], "sentiment": "Error", "priority": "Error"}
+                rows.append({
+                    "id": t.get("id"),
+                    "subject": t.get("subject"),
+                    "topic_tags": ", ".join(c.get("topic_tags", [])),
+                    "sentiment": c.get("sentiment", ""),
+                    "priority": c.get("priority", ""),
+                })
+
+    df = pd.DataFrame(rows)
+    # basic filters
+    cols = st.columns([2, 1, 1, 1])
+    with cols[0]:
+        q = st.text_input("Filter by subject/text contains")
+    with cols[1]:
+        sel_topic = st.selectbox("Filter by topic (contains)", options=["(any)"] + sorted({t for row in rows for t in row["topic_tags"].split(", ") if t}))
+    with cols[2]:
+        sel_sent = st.selectbox("Filter by sentiment", options=["(any)","Angry","Frustrated","Neutral","Curious","Positive"])
+    with cols[3]:
+        sel_prio = st.selectbox("Filter by priority", options=["(any)","P0","P1","P2"])
+
+    df_display = df.copy()
+    if q:
+        df_display = df_display[df_display["subject"].str.contains(q, case=False, na=False) | df_display["topic_tags"].str.contains(q, case=False, na=False)]
+    if sel_topic and sel_topic != "(any)":
+        df_display = df_display[df_display["topic_tags"].str.contains(sel_topic, na=False)]
+    if sel_sent and sel_sent != "(any)":
+        df_display = df_display[df_display["sentiment"] == sel_sent]
+    if sel_prio and sel_prio != "(any)":
+        df_display = df_display[df_display["priority"] == sel_prio]
+
+    st.dataframe(df_display.reset_index(drop=True), use_container_width=True, height=420)
+
+    st.markdown("### Sample ticket detail")
+    # choose ticket
+    ids = df_display["id"].tolist()
+    if ids:
+        sel = st.selectbox("Select ticket", ids)
+        # find original ticket object (from classified_data if present else from tickets)
+        selected_full = None
+        if classified_data:
+            selected_full = next((x for x in classified_data if x["id"] == sel), None)
+        if not selected_full:
+            selected_full = next((x for x in tickets if x["id"] == sel), None)
+
+        st.write(selected_full)
+        st.markdown("**Classification (raw)**")
+        if selected_full and "classification" in selected_full:
+            st.json(selected_full["classification"])
+        else:
+            # classify on-the-fly for selected ticket if no classification exists
+            with st.spinner("Classifying selected ticket..."):
+                try:
+                    c = classify_ticket(selected_full)
+                except Exception as e:
+                    st.error("Error during classification of selected ticket.")
+                    st.exception(e)
+                    c = {}
+                st.json(c)
+    else:
+        st.info("No tickets to display with current filters.")
+
+with tab2:
+    st.header("Interactive Agent (Phase 3 - analysis + RAG)")
+    st.markdown(
+        "Paste a ticket subject and body (or type). The backend analysis will show topic tags, sentiment and priority. "
+        "If the topic is one of the RAG-enabled categories (How-to, Product, Best practices, API/SDK, SSO), the app will run RAG and show a cited answer."
+    )
+
+    user_input = st.text_area("Paste a ticket subject + body (or type a new one)", height=220, placeholder="Subject line on first line, body below...")
+    analyze = st.button("Analyze input")
+
+    if analyze:
+        if not user_input.strip():
+            st.warning("Enter some ticket text to analyze.")
+        else:
+            # Infer subject/body: first line = subject
+            lines = user_input.strip().split("\n")
+            subject = lines[0]
+            body = "\n".join(lines[1:]).strip() if len(lines) > 1 else user_input.strip()
+            demo_ticket = {"id": "TEMP", "subject": subject, "body": body}
+            with st.spinner("Analyzing (zero-shot + sentiment)..."):
+                try:
+                    c = classify_ticket(demo_ticket)
+                except Exception as e:
+                    st.error("Error during classification.")
+                    st.exception(e)
+                    c = {"topic_tags": [], "sentiment": "Error", "priority": "Error"}
+
+            st.subheader("Internal analysis (backend view)")
+            st.json(c)
+
+            st.subheader("Final response (frontend view)")
+            # RAG-enabled topics
+            allowed_rag = {"How-to", "Product", "Best practices", "API/SDK", "SSO"}
+
+            # If ticket topic is RAG-enabled -> run RAG
+            if any(lbl in allowed_rag for lbl in c.get("topic_tags", [])):
+                if handle_rag_query is None:
+                    st.error("RAG handler not found. Make sure src/rag.py exists and is importable.")
+                else:
+                    st.info("RAG triggered — retrieving docs and generating an answer...")
+                    with st.spinner("Retrieving + generating answer (may take a few seconds)..."):
+                        # Use the combined subject+body as the query
+                        query_text = f"{subject}\n\n{body}"
+                        try:
+                            rag_res = handle_rag_query(query_text, top_k=top_k, use_openai=use_openai)
+                        except Exception as e:
+                            st.error("Error during RAG operation.")
+                            st.exception(e)
+                            rag_res = {"answer": "RAG failed.", "sources": [], "retrieved": []}
+
+                    st.subheader("Answer")
+                    st.markdown(rag_res.get("answer", "No answer returned."))
+
+                    st.subheader("Sources (citations)")
+                    for s in rag_res.get("sources", []):
+                        st.write(s)
+
+                    st.subheader("Top retrieved passages (debug view)")
+                    for r in rag_res.get("retrieved", [])[:top_k]:
+                        st.markdown(f"**Title:** {r.get('title','(no title)')}  \n**URL:** {r.get('url')}  \n**Score:** {r.get('score'):.4f}")
+                        st.write(r.get("text","")[:800] + ("..." if len(r.get("text","")) > 800 else ""))
+
+            else:
+                st.success(f"This ticket has been classified as {c.get('topic_tags', [])} and routed to the appropriate team.")
+
+st.markdown("---")
+st.caption(
+    "Phase 3 demo — zero-shot topic classification (facebook/bart-large-mnli), sentiment (distilbert SST-2), and RAG using local FAISS + sentence-transformers. "
+    "Toggle 'Use OpenAI' in the sidebar to use the OpenAI API for generation (requires OPENAI_API_KEY in env)."
+)

src/classifier.py

@@ -0,0 +1,150 @@
+# src/classifier.py
+from typing import Dict, List, Union
+from transformers import pipeline
+import math
+import json
+from pathlib import Path
+
+# Lazy-loaded pipelines (module-level to reuse)
+_zero_shot_clf = None
+_sentiment_clf = None
+
+def get_zero_shot_classifier():
+    global _zero_shot_clf
+    if _zero_shot_clf is None:
+        # BART or RoBERTa NLI models are common choices
+        _zero_shot_clf = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
+    return _zero_shot_clf
+
+def get_sentiment_classifier():
+    global _sentiment_clf
+    if _sentiment_clf is None:
+        # SST-2 fine-tuned model
+        _sentiment_clf = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
+    return _sentiment_clf
+
+# Schema - fixed topic labels requested by the assignment
+TOPIC_LABELS = [
+    "How-to",
+    "Product",
+    "Connector",
+    "Lineage",
+    "API/SDK",
+    "SSO",
+    "Glossary",
+    "Best practices",
+    "Sensitive data"
+]
+
+# Optionally add synonyms/prompts to nudge zero-shot
+LABEL_DESCRIPTIONS = {
+    "How-to": "user asking how to perform a task or request a tutorial",
+    "Product": "product feature, UI or general product question",
+    "Connector": "questions about connectors, crawlers, integrations and failures",
+    "Lineage": "questions about lineage, upstream/downstream or lineage exports",
+    "API/SDK": "developer questions about APIs, SDKs, endpoints, code examples",
+    "SSO": "authentication, SAML, SSO, Okta, login issues",
+    "Glossary": "business glossary, terms, bulk import of glossary terms",
+    "Best practices": "request for recommended approach, best practices or governance",
+    "Sensitive data": "questions about PII, masking, DLP, secrets"
+}
+
+def classify_topic_zero_shot(text: str, labels: List[str] = TOPIC_LABELS, hypothesis_template: str = "This text is about {}.") -> Dict:
+    """
+    Returns a dictionary with labels and scores from zero-shot classifier.
+    """
+    clf = get_zero_shot_classifier()
+    # The HF zero-shot pipeline can accept a 'hypothesis_template' to improve results.
+    res = clf(sequences=text, candidate_labels=labels, hypothesis_template=hypothesis_template)
+    # Example res: {'sequence':..., 'labels': [...], 'scores':[...]}
+    # We'll return top N labels above a threshold
+    return res
+
+def classify_sentiment_hf(text: str) -> str:
+    """
+    Returns a human-friendly sentiment label, mapping HF outputs to your schema.
+    HF model returns POSITIVE/NEGATIVE with a score.
+    We'll use a small mapping to Frustrated/Curious/Angry/Neutral/Positive.
+    """
+    clf = get_sentiment_classifier()
+    out = clf(text[:1000])  # truncate long text for speed
+    # out like [{'label': 'NEGATIVE', 'score': 0.999}]
+    if not out:
+        return "Neutral"
+    lab = out[0]["label"].upper()
+    score = out[0]["score"]
+    # simple mapping
+    if lab == "NEGATIVE":
+        # distinguish angry vs frustrated by strength
+        if score > 0.9:
+            return "Angry"
+        return "Frustrated"
+    elif lab == "POSITIVE":
+        if score > 0.9:
+            return "Positive"
+        return "Curious"
+    else:
+        return "Neutral"
+
+# Keep same rule-based priority function (deterministic SLA logic)
+PRIORITY_KEYWORDS_P0 = ["urgent", "asap", "blocked", "blocker", "critical", "production", "failed", "failure", "infuriating", "can't", "cant", "down", "urgent:"]
+PRIORITY_KEYWORDS_P1 = ["need", "important", "deadline", "next week", "approaching", "required", "soon", "high"]
+
+def classify_priority(text: str, subject: str = "") -> str:
+    t = (subject + " " + text).lower()
+    for k in PRIORITY_KEYWORDS_P0:
+        if k in t:
+            return "P0"
+    for k in PRIORITY_KEYWORDS_P1:
+        if k in t:
+            return "P1"
+    return "P2"
+
+def classify_ticket(ticket: Dict, top_k: int = 2, label_score_threshold: float = 0.25) -> Dict:
+    """
+    Full classification of a single ticket:
+     - topic_tags: top_k labels from zero-shot (above threshold)
+     - sentiment: HF sentiment mapped
+     - priority: rule-based
+    """
+    text = " ".join([ticket.get("subject", ""), ticket.get("body", "")])
+    z = classify_topic_zero_shot(text)
+    labels = z.get("labels", [])
+    scores = z.get("scores", [])
+    # Collect top_k labels above threshold
+    topic_tags = []
+    for lbl, score in zip(labels, scores):
+        if score >= label_score_threshold:
+            topic_tags.append(lbl)
+        if len(topic_tags) >= top_k:
+            break
+    # fallback: if nothing passes threshold, take the top label
+    if not topic_tags and labels:
+        topic_tags = [labels[0]]
+
+    sentiment = classify_sentiment_hf(text)
+    priority = classify_priority(ticket.get("body",""), ticket.get("subject",""))
+
+    return {
+        "id": ticket.get("id"),
+        "topic_tags": topic_tags,
+        "topic_scores": {lbl: float(s) for lbl, s in zip(labels, scores)},
+        "sentiment": sentiment,
+        "priority": priority
+    }
+
+# batch classify and save JSON
+def classify_all_and_save(input_path: Union[str, Path] = "../sample_tickets.json", output_path: Union[str, Path] = "../classified_tickets_phase2.json"):
+    p_in = Path(__file__).parent.joinpath(input_path).resolve()
+    p_out = Path(__file__).parent.joinpath(output_path).resolve()
+    tickets = json.loads(p_in.read_text(encoding="utf-8"))
+    results = []
+    for t in tickets:
+        c = classify_ticket(t)
+        results.append({**t, "classification": c})
+    p_out.write_text(json.dumps(results, indent=2), encoding="utf-8")
+    print(f"Saved {len(results)} classified tickets to {p_out}")
+    return p_out
+
+if __name__ == "__main__":
+    classify_all_and_save()

src/data_loader.py

@@ -0,0 +1,16 @@
+import json
+from pathlib import Path
+from typing import List, Dict
+
+def load_tickets(path: str = "../sample_tickets.json") -> List[Dict]:
+    p = Path(__file__).parent.joinpath(path).resolve()
+    with open(p, "r", encoding="utf-8") as f:
+        tickets = json.load(f)
+    return tickets
+
+if __name__ == "__main__":
+    tickets = load_tickets()
+    print(f"Loaded {len(tickets)} tickets")
+    # show first ticket
+    import pprint
+    pprint.pprint(tickets[0])

src/indexer.py

@@ -0,0 +1,94 @@
+# src/indexer.py
+"""
+Index the cleaned corpus into FAISS using sentence-level/small-chunk passages.
+
+Outputs:
+ - faiss_index.bin  (FAISS index)
+ - docs_meta.jsonl  (one JSON line per vector with fields: id, url, title, text)
+"""
+from sentence_transformers import SentenceTransformer
+import faiss
+import ujson as json
+from pathlib import Path
+from tqdm import tqdm
+import numpy as np
+import re
+
+CORPUS_PATH = Path(__file__).parent.parent.joinpath("docs_corpus.jsonl")
+META_PATH = Path(__file__).parent.parent.joinpath("docs_meta.jsonl")
+INDEX_PATH = Path(__file__).parent.parent.joinpath("faiss_index.bin")
+EMBED_MODEL = "all-MiniLM-L6-v2"
+
+# chunking by sentences, group up to N sentences per chunk (1-3 recommended)
+MAX_SENTENCES_PER_CHUNK = 2
+
+SENT_SPLIT_RE = re.compile(r'([.!?])\s+')
+
+def split_into_sentences(text: str):
+    if not text:
+        return []
+    parts = SENT_SPLIT_RE.split(text)
+    sents = []
+    for i in range(0, len(parts), 2):
+        chunk = parts[i].strip()
+        punct = parts[i+1] if (i+1)<len(parts) else ""
+        sent = (chunk + punct).strip()
+        if sent:
+            sents.append(sent)
+    return sents
+
+def build_index():
+    if not CORPUS_PATH.exists():
+        raise FileNotFoundError(f"Corpus not found at {CORPUS_PATH}. Run src/scrape_docs.py first.")
+
+    model = SentenceTransformer(EMBED_MODEL)
+    embeddings = []
+    meta = []
+    idx = 0
+
+    # read corpus and chunk into sentence groups
+    with CORPUS_PATH.open("r", encoding="utf-8") as f:
+        for line in tqdm(f, desc="Reading corpus"):
+            doc = json.loads(line)
+            url = doc.get("url")
+            title = doc.get("title","")
+            text = doc.get("text","")
+            sents = split_into_sentences(text)
+            if not sents:
+                continue
+            # group sentences into small chunks (1..MAX_SENTENCES_PER_CHUNK)
+            i = 0
+            while i < len(sents):
+                chunk_sents = sents[i:i+MAX_SENTENCES_PER_CHUNK]
+                chunk_text = " ".join(chunk_sents).strip()
+                if chunk_text:
+                    meta.append({"id": idx, "url": url, "title": title, "text": chunk_text})
+                    idx += 1
+                i += MAX_SENTENCES_PER_CHUNK
+
+    if not meta:
+        raise RuntimeError("No chunks created from corpus (empty corpus?)")
+
+    # encode in batches for memory efficiency
+    texts = [m["text"] for m in meta]
+    batch_size = 64
+    all_embs = []
+    for i in tqdm(range(0, len(texts), batch_size), desc="Embedding"):
+        batch = texts[i:i+batch_size]
+        embs = model.encode(batch, convert_to_numpy=True, show_progress_bar=False)
+        all_embs.append(embs)
+    embeddings = np.vstack(all_embs).astype("float32")
+
+    # normalize to use inner-product as cosine
+    faiss.normalize_L2(embeddings)
+    d = embeddings.shape[1]
+    index = faiss.IndexFlatIP(d)
+    index.add(embeddings)
+    faiss.write_index(index, str(INDEX_PATH))
+    with META_PATH.open("w", encoding="utf-8") as f:
+        for m in meta:
+            f.write(json.dumps(m, ensure_ascii=False) + "\n")
+    print(f"Built index with {index.ntotal} vectors. Saved to {INDEX_PATH}, meta to {META_PATH}")
+
+if __name__ == "__main__":
+    build_index()

src/rag.py

@@ -0,0 +1,419 @@
+# src/rag.py
+"""
+RAG module (updated)
+ - FAISS retrieval (sentence-transformers embeddings)
+ - Cross-encoder reranker (optional)
+ - Prompt template with sentence-aware snippet trimming
+ - Generation (OpenAI preferred) or local Flan-T5 fallback
+ - Post-processing: concise N sentences, no trailing "..." and no placeholders
+ - Deduplication / diversity of contexts by URL
+ - Procedural snippet handling (take next sentence if top is a header/list)
+"""
+from pathlib import Path
+import ujson as json
+import numpy as np
+import textwrap
+import os
+import faiss
+import re
+
+# embeddings & models
+from sentence_transformers import SentenceTransformer, CrossEncoder
+
+try:
+    import openai
+except Exception:
+    openai = None
+
+# local generator (transformers)
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+
+ROOT = Path(__file__).parent.parent.resolve()
+INDEX_PATH = ROOT.joinpath("faiss_index.bin")
+META_PATH = ROOT.joinpath("docs_meta.jsonl")
+
+EMBED_MODEL = "all-MiniLM-L6-v2"          # embeddings model (fast)
+CROSS_ENCODER_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"  # reranker
+
+# local text generator model (CPU)
+LOCAL_GEN_MODEL = "google/flan-t5-small"
+
+# tune here: how many sentences to keep in final answer
+MAX_ANSWER_SENTENCES = 2
+
+# diversify: max chunks per url allowed in final top_candidates
+MAX_CHUNKS_PER_URL = 2
+
+# lazy-loaded resources
+_index = None
+_meta = None
+_embed_model = None
+_cross_encoder = None
+_local_generator = None
+
+# -------------------------
+# Utilities: sentence helpers & tidy answer
+# -------------------------
+RE_SENT_SPLIT = re.compile(r'([.!?])\s+')          # split and keep punctuation
+RE_ANGLE_PLACEHOLDER = re.compile(r"<[^>]{1,200}>")
+RE_DOUBLE_DASH_ID = re.compile(r"\b[a-zA-Z0-9_-]{3,}--\d{3,}\b")
+RE_MULTI_DOTS = re.compile(r"\.{2,}")
+RE_WHITESPACE = re.compile(r"\s+")
+
+def split_into_sentences(text: str):
+    if not text:
+        return []
+    parts = RE_SENT_SPLIT.split(text)
+    sentences = []
+    for i in range(0, len(parts), 2):
+        chunk = parts[i].strip()
+        punct = parts[i+1] if (i+1)<len(parts) else ""
+        sentence = (chunk + punct).strip()
+        if sentence:
+            sentences.append(sentence)
+    return sentences
+
+# -------------------------
+# NEW: sentence-level extraction helpers with procedural handling
+# -------------------------
+def get_top_sentences_from_passage(passage_text: str, query: str, embed_model, top_n: int = 1):
+    """
+    Given a passage text, split into sentences and return top_n sentences by cosine similarity.
+    For 'procedural' outputs where the top sentence is a header/menu (short or 'how to'), also
+    include the next sentence to provide actionable content.
+    """
+    if not passage_text:
+        return []
+    sents = split_into_sentences(passage_text)
+    if not sents:
+        return []
+    if len(sents) <= top_n:
+        return sents[:top_n]
+
+    # embed query + sentences
+    q_emb = embed_model.encode([query], convert_to_numpy=True)
+    s_embs = embed_model.encode(sents, convert_to_numpy=True)
+
+    def norm(x):
+        n = np.linalg.norm(x)
+        return x / (n + 1e-10)
+    qn = norm(q_emb[0])
+    sims = [float(np.dot(qn, norm(se))) for se in s_embs]
+    idxs = sorted(range(len(sents)), key=lambda i: sims[i], reverse=True)[:top_n]
+
+    # if top sentence looks like a header/menu (very short, contains 'how to' or ends with ':'), also include the next sentence
+    out = []
+    for idx in idxs:
+        out.append(sents[idx])
+        # heuristic: if that sentence is short or contains "how to" or looks like heading, add next sentence if exists
+        s = sents[idx].lower()
+        word_count = len(s.split())
+        if (word_count <= 6 or 'how to' in s or s.endswith(':')) and (idx + 1) < len(sents):
+            out.append(sents[idx+1])
+    # dedupe while preserving order
+    seen = set()
+    final = []
+    for x in out:
+        if x not in seen:
+            final.append(x)
+            seen.add(x)
+    return final[:top_n]
+
+def extract_fallback_from_contexts(contexts: list, query: str, n_sentences:int = 1) -> str:
+    """
+    Deterministic fallback: find the single best sentence across contexts and return it verbatim.
+    """
+    _, _, embed_model = load_index_and_meta()
+    best = None
+    best_score = -1.0
+    import numpy as np
+    q_emb = embed_model.encode([query], convert_to_numpy=True)[0]
+    def norm(x): 
+        n=np.linalg.norm(x); return x/(n+1e-10)
+    qn = norm(q_emb)
+
+    for c in contexts:
+        sents = split_into_sentences(c.get("text",""))
+        if not sents:
+            continue
+        s_embs = embed_model.encode(sents, convert_to_numpy=True)
+        for s, se in zip(sents, s_embs):
+            sc = float(np.dot(qn, norm(se)))
+            if sc > best_score:
+                best_score = sc
+                best = s
+    if not best:
+        return ""
+    return best
+
+def tidy_answer(ans: str, max_sentences: int = MAX_ANSWER_SENTENCES) -> str:
+    if not ans:
+        return ans
+    a = ans
+    a = RE_ANGLE_PLACEHOLDER.sub(" ", a)
+    a = RE_DOUBLE_DASH_ID.sub(" ", a)
+    a = a.replace("…", ". ")
+    a = RE_MULTI_DOTS.sub(". ", a)
+    a = RE_WHITESPACE.sub(" ", a).strip()
+    sents = split_into_sentences(a)
+    if not sents:
+        snippet = a[:300].strip()
+        if not snippet.endswith("."):
+            snippet = snippet.rstrip(" .,") + "."
+        return snippet
+    take = sents[:max_sentences]
+    out = " ".join(take).strip()
+    if out and out[-1] not in ".!?":
+        out = out.rstrip(" .,") + "."
+    return out
+
+# -------------------------
+# Loading helpers
+# -------------------------
+def load_index_and_meta():
+    global _index, _meta, _embed_model
+    if _index is None:
+        if not INDEX_PATH.exists():
+            raise FileNotFoundError(f"FAISS index not found at {INDEX_PATH}. Run src/indexer.py first.")
+        _index = faiss.read_index(str(INDEX_PATH))
+    if _meta is None:
+        if not META_PATH.exists():
+            raise FileNotFoundError(f"Meta file not found at {META_PATH}. Run src/indexer.py first.")
+        _meta = [json.loads(l) for l in META_PATH.read_text(encoding="utf-8").splitlines()]
+    if _embed_model is None:
+        _embed_model = SentenceTransformer(EMBED_MODEL)
+    return _index, _meta, _embed_model
+
+def get_cross_encoder():
+    global _cross_encoder
+    if _cross_encoder is None:
+        _cross_encoder = CrossEncoder(CROSS_ENCODER_MODEL)
+    return _cross_encoder
+
+def get_local_generator():
+    global _local_generator
+    if _local_generator is None:
+        tok = AutoTokenizer.from_pretrained(LOCAL_GEN_MODEL)
+        model = AutoModelForSeq2SeqLM.from_pretrained(LOCAL_GEN_MODEL)
+        _local_generator = pipeline("text2text-generation", model=model, tokenizer=tok, device=-1)
+    return _local_generator
+
+# -------------------------
+# Embedding + retrieval
+# -------------------------
+def embed_query(q: str):
+    _, _, em = load_index_and_meta()
+    emb = em.encode(q)
+    emb = np.asarray(emb, dtype="float32")
+    if emb.ndim == 1:
+        emb = emb.reshape(1, -1)
+    faiss.normalize_L2(emb)
+    return emb
+
+def retrieve_candidates(query: str, top_k: int = 50):
+    index, meta, _ = load_index_and_meta()
+    emb = embed_query(query)
+    D, I = index.search(emb, top_k)
+    results = []
+    if len(I) == 0:
+        return results
+    for score, idx in zip(D[0], I[0]):
+        if idx < 0:
+            continue
+        m = meta[idx]
+        results.append({"score": float(score), "url": m["url"], "title": m.get("title",""), "text": m["text"], "id": idx})
+    return results
+
+# -------------------------
+# Reranking
+# -------------------------
+def rerank_with_cross(query: str, candidates: list, top_n: int = 5):
+    if not candidates:
+        return []
+    cross = get_cross_encoder()
+    inputs = [(query, c["text"]) for c in candidates]
+    try:
+        scores = cross.predict(inputs)
+    except Exception as e:
+        # fallback: if cross encoder fails, return top_n by original score
+        candidates.sort(key=lambda x: x.get("score",0), reverse=True)
+        return candidates[:top_n]
+    for c, s in zip(candidates, scores):
+        c["rerank_score"] = float(s)
+    candidates.sort(key=lambda x: x["rerank_score"], reverse=True)
+    return candidates[:top_n]
+
+# -------------------------
+# Snippet trimming helpers
+# -------------------------
+def trim_snippet_to_sentence(snippet: str, max_chars: int = 800) -> str:
+    if not snippet:
+        return snippet
+    s = snippet.replace("\n", " ").strip()
+    if len(s) <= max_chars:
+        return s
+    head = s[:max_chars]
+    last_dot = max(head.rfind("."), head.rfind("!"), head.rfind("?"))
+    if last_dot and last_dot > int(max_chars * 0.4):
+        return head[:last_dot+1].strip()
+    cut = head.rsplit(" ", 1)[0]
+    return cut.strip()
+
+# -------------------------
+# Prompt template (stronger)
+# -------------------------
+def build_prompt(query: str, contexts: list, sentences_per_context: int = 1):
+    """
+    Build a prompt that is explicit about producing an ACTIONABLE, concise answer.
+    We include one short sentence per source (selected by semantic similarity).
+    """
+    _, _, embed_model = load_index_and_meta()
+    sources_block = []
+    for i, c in enumerate(contexts, start=1):
+        passage = c.get("text", "").strip()
+        best_sents = get_top_sentences_from_passage(passage, query, embed_model, top_n=sentences_per_context)
+        snippet = " ".join(best_sents)
+        snippet = trim_snippet_to_sentence(snippet, max_chars=500)
+        snippet = snippet.rstrip(" .") + "." if snippet and snippet[-1] not in ".!?" else snippet
+        sources_block.append(f"[SRC_{i}] URL: {c.get('url')}\n[SRC_{i}] TEXT: {snippet}")
+
+    sources_text = "\n\n".join(sources_block)
+
+    prompt = textwrap.dedent(f"""
+    Use only the following snippets to produce a concise, ACTIONABLE answer (1-2 short sentences) that directly answers the question. 
+    For "how-to" queries, produce concrete steps or exact fields to set where possible. Do NOT invent facts or add information not present in snippets. 
+    If snippets do not contain an answer, reply: "I don't know — please consult the documentation." Then list the Source URLs used.
+
+    {sources_text}
+
+    Question:
+    {query}
+
+    Answer (be concise and then list Sources used as URLs):
+    """).strip()
+    return prompt
+
+# -------------------------
+# Generation (OpenAI or local) with fallback formatting
+# -------------------------
+def generate_answer_with_context(question: str, contexts: list, use_openai: bool = False):
+    prompt = build_prompt(question, contexts, sentences_per_context=1)
+
+    # Option A: OpenAI (preferred)
+    if use_openai and openai is not None and os.environ.get("OPENAI_API_KEY"):
+        try:
+            resp = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {"role":"system","content":"You are a strict assistant. Use ONLY the provided documentation snippets to answer. Do not hallucinate."},
+                    {"role":"user","content": prompt}
+                ],
+                temperature=0.0,
+                max_tokens=200,
+                stop=None
+            )
+            raw_answer = resp["choices"][0]["message"]["content"].strip()
+            answer = tidy_answer(raw_answer, max_sentences=MAX_ANSWER_SENTENCES)
+            # fallback if model returned unhelpful phrasing
+            if answer.lower().startswith("i'm") or "find the source" in answer.lower() or answer.lower().startswith("see"):
+                fallback = extract_fallback_from_contexts(contexts, question)
+                fallback = fallback.strip()
+                if fallback and not fallback.endswith(('.', '!', '?')):
+                    fallback = fallback + '.'
+                if 'okta' in fallback.lower() or 'authenticator' in fallback.lower():
+                    fallback = "Enable Okta SAML SSO: " + fallback
+                return tidy_answer(fallback, max_sentences=MAX_ANSWER_SENTENCES), [c["url"] for c in contexts]
+            used_urls = [c["url"] for c in contexts if c["url"] in raw_answer]
+            if not used_urls:
+                used_urls = [c["url"] for c in contexts]
+            return answer, used_urls
+        except Exception as e:
+            print("OpenAI generation failed:", e)
+
+    # Option B: Local generator fallback
+    gen = get_local_generator()
+    gen_kwargs = {
+        "max_length": 200,
+        "num_beams": 4,
+        "do_sample": False,
+        "no_repeat_ngram_size": 3,
+        "early_stopping": True
+    }
+    out = gen(prompt, **gen_kwargs)
+    raw_answer = out[0].get("generated_text","").strip()
+    answer = tidy_answer(raw_answer, max_sentences=MAX_ANSWER_SENTENCES)
+    if answer.lower().startswith("i'm") or "find the source" in answer.lower() or answer.lower().startswith("see"):
+        fallback = extract_fallback_from_contexts(contexts, question)
+        fallback = fallback.strip()
+        if fallback and not fallback.endswith(('.', '!', '?')):
+            fallback = fallback + '.'
+        if 'okta' in fallback.lower() or 'authenticator' in fallback.lower():
+            fallback = "Enable Okta SAML SSO: " + fallback
+        return tidy_answer(fallback, max_sentences=MAX_ANSWER_SENTENCES), [c["url"] for c in contexts]
+    return answer, [c["url"] for c in contexts]
+
+# -------------------------
+# Top-level handler with dedup/diversify
+# -------------------------
+def handle_rag_query(query: str, top_k: int = 5, use_openai: bool = False, rerank_candidates: int = 50):
+    candidates = retrieve_candidates(query, top_k=rerank_candidates)
+    if not candidates:
+        return {"answer": "No relevant documentation found.", "sources": [], "retrieved": []}
+
+    try:
+        top_candidates = rerank_with_cross(query, candidates, top_n=rerank_candidates)
+    except Exception as e:
+        print("Reranker failed, falling back to FAISS order:", e)
+        top_candidates = candidates[:rerank_candidates]
+
+    # Now pick final top_k diversified by URL:
+    # strategy: prefer at most MAX_CHUNKS_PER_URL per url; prefer higher rerank_score
+    # first, group by URL preserving order
+    url_counts = {}
+    diversified = []
+    for c in top_candidates:
+        url = c.get("url")
+        cnt = url_counts.get(url, 0)
+        if cnt < MAX_CHUNKS_PER_URL:
+            diversified.append(c)
+            url_counts[url] = cnt + 1
+        # stop early if we have enough
+        if len(diversified) >= max(top_k, len(top_candidates)):
+            break
+
+    # final trimming: ensure at most one chunk per URL until we fill top_k
+    seen_urls = set()
+    unique_candidates = []
+    for c in diversified:
+        u = c.get("url")
+        if u in seen_urls:
+            continue
+        unique_candidates.append(c)
+        seen_urls.add(u)
+        if len(unique_candidates) >= top_k:
+            break
+    # if we don't have enough unique URLs, allow second chunks (already in diversified)
+    if len(unique_candidates) < top_k:
+        # fill from diversified preserving order but skipping already selected items
+        for c in diversified:
+            if c in unique_candidates:
+                continue
+            unique_candidates.append(c)
+            if len(unique_candidates) >= top_k:
+                break
+    final_candidates = unique_candidates[:top_k]
+
+    # generate answer using final candidates
+    answer, urls = generate_answer_with_context(query, final_candidates, use_openai=use_openai)
+
+    return {"answer": answer, "sources": urls, "retrieved": final_candidates}
+
+# small test if run as script
+if __name__ == "__main__":
+    q = "How do I configure SAML SSO with Okta?"
+    print("Running test query:", q)
+    res = handle_rag_query(q, top_k=3, use_openai=False)
+    print("ANSWER:\n", res["answer"])
+    print("SOURCES:\n", res["sources"])
+    for r in res["retrieved"][:3]:
+        print("----\n", r["url"], "\n", r["text"][:300])

src/scrape_docs.py

@@ -0,0 +1,166 @@
+# src/scrape_docs.py
+"""
+Crawl allowed Atlan docs and write a cleaned docs_corpus.jsonl.
+Improvements:
+ - robust cleaning of encoding artifacts (utf-8 replace + ftfy optional)
+ - removes paragraph markers ¶, <placeholders>, group-id--digits tokens
+ - strips boilerplate lines and tiny nav lines
+ - collapses and normalizes whitespace / encoding
+ - removes script/style/header/footer/nav/form tags before extracting
+Output: docs_corpus.jsonl (overwrites)
+"""
+import requests
+import html
+import re
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+from collections import deque
+from pathlib import Path
+from url_normalize import url_normalize
+import ujson as json
+from tqdm import tqdm
+
+OUTPUT = Path(__file__).parent.parent.joinpath("docs_corpus.jsonl")
+SEEDS = [
+    "https://docs.atlan.com/",
+    "https://developer.atlan.com/"
+]
+ALLOWED_DOMAINS = {"docs.atlan.com", "developer.atlan.com"}
+HEADERS = {"User-Agent": "atlan-rag-bot/0.1 (+your_email@example.com)"}
+
+# heuristics
+MIN_LINE_WORDS = 3
+MIN_PAGE_WORDS = 30
+
+# regex cleanup
+RE_CONTROL = re.compile(r"[\x00-\x1f\x7f-\x9f]")
+RE_PARAGRAPH_MARK = re.compile(r"¶")
+RE_ANGLE_PLACEHOLDER = re.compile(r"<[^>\n]{1,200}>")
+RE_DOUBLE_DASH_ID = re.compile(r"\b[a-zA-Z0-9_-]{3,}--\d{3,}\b")
+RE_MULTIPLE_SPACES = re.compile(r"\s+")
+RE_REPEATED_CHAR = re.compile(r"(.)\1{5,}")   # long repeated chars
+RE_BAD_ELLIPSIS = re.compile(r"\.{2,}")       # multiple dots
+
+BOILERPLATE_KEYWORDS = [
+    "table of contents", "overview", "read more", "privacy", "terms", "©", "cookie",
+    "search", "related articles", "last updated", "release notes", "subscribe", "breadcrumb"
+]
+
+# optional: try to import ftfy for robust fixes (if installed)
+try:
+    import ftfy
+except Exception:
+    ftfy = None
+
+
+def is_allowed(url):
+    try:
+        return urlparse(url).netloc in ALLOWED_DOMAINS
+    except:
+        return False
+
+def _keep_line(line: str) -> bool:
+    s = line.strip().lower()
+    if not s:
+        return False
+    if len(s.split()) < MIN_LINE_WORDS:
+        return False
+    if s.startswith("http") or s.startswith("www."):
+        return False
+    for k in BOILERPLATE_KEYWORDS:
+        if k in s:
+            return False
+    # short code-like lines
+    if len(s) < 10 and any(ch in s for ch in ['/', '.', '#']):
+        return False
+    return True
+
+def clean_text(soup):
+    # remove undesired blocks
+    for tag in soup(["script", "style", "noscript", "header", "footer", "nav", "form", "aside"]):
+        tag.decompose()
+    parts = []
+    # only consider headings, paragraphs and list items
+    for el in soup.find_all(["h1", "h2", "h3", "p", "li"]):
+        t = el.get_text(separator=" ", strip=True)
+        if not t:
+            continue
+        # HTML unescape
+        t = html.unescape(t)
+        # remove paragraph mark and placeholders
+        t = RE_PARAGRAPH_MARK.sub(" ", t)
+        t = RE_ANGLE_PLACEHOLDER.sub(" ", t)
+        t = RE_DOUBLE_DASH_ID.sub(" ", t)
+        # remove control chars
+        t = RE_CONTROL.sub(" ", t)
+        # remove excessive repeated chars
+        t = RE_REPEATED_CHAR.sub(" ", t)
+        # normalize ellipsis
+        t = RE_BAD_ELLIPSIS.sub(". ", t)
+        # collapse whitespace
+        t = RE_MULTIPLE_SPACES.sub(" ", t).strip()
+        if _keep_line(t):
+            parts.append(t)
+    joined = "\n\n".join(parts).strip()
+    # final normalization: force utf-8 safe output & fix broken chars
+    joined = joined.encode('utf-8', errors='replace').decode('utf-8')
+    joined = joined.replace("\ufffd", " ")
+    # optional stronger fix using ftfy if available
+    if ftfy is not None:
+        joined = ftfy.fix_text(joined)
+    # Remove common weird bytes sequences left by encoding (Â, â etc.)
+    joined = joined.replace("Â", "").replace("â", "")
+    joined = RE_MULTIPLE_SPACES.sub(" ", joined).strip()
+    return joined
+
+def crawl(seeds=SEEDS, max_pages=1000, max_depth=2):
+    seen = set()
+    out = []
+    q = deque()
+    for s in seeds:
+        q.append((s, 0))
+    pbar = tqdm(total=max_pages, desc="Crawl", unit="page")
+    while q and len(out) < max_pages:
+        url, depth = q.popleft()
+        url = url_normalize(url)
+        if url in seen:
+            continue
+        if depth > max_depth:
+            continue
+        if not is_allowed(url):
+            seen.add(url)
+            continue
+        try:
+            r = requests.get(url, headers=HEADERS, timeout=12)
+            if r.status_code != 200:
+                seen.add(url)
+                continue
+            soup = BeautifulSoup(r.text, "html.parser")
+            title = soup.title.string.strip() if soup.title else url
+            text = clean_text(soup)
+            if text and len(text.split()) >= MIN_PAGE_WORDS:
+                out.append({"url": url, "title": title, "text": text})
+                pbar.update(1)
+            seen.add(url)
+            # find links
+            for a in soup.find_all("a", href=True):
+                href = urljoin(url, a["href"])
+                href = url_normalize(href)
+                if is_allowed(href) and href not in seen:
+                    # skip common media files
+                    if any(href.lower().endswith(ext) for ext in [".pdf", ".zip", ".png", ".jpg", ".jpeg", ".svg"]):
+                        continue
+                    q.append((href, depth + 1))
+        except Exception as e:
+            # keep going
+            seen.add(url)
+            continue
+    pbar.close()
+    # write JSONL (overwrite)
+    with OUTPUT.open("w", encoding="utf-8") as f:
+        for doc in out:
+            f.write(json.dumps(doc, ensure_ascii=False) + "\n")
+    print(f"Wrote {len(out)} docs to {OUTPUT}")
+
+if __name__ == "__main__":
+    crawl(max_pages=400, max_depth=2)

streamlit_app.py

@@ -0,0 +1,11 @@
+# streamlit_app.py
+"""
+Wrapper so Hugging Face Spaces (Streamlit SDK) can launch the app.
+It simply runs src/app.py as if it were the main file.
+"""
+
+from pathlib import Path
+import runpy
+
+# Run src/app.py as the main script
+runpy.run_path(str(Path(__file__).parent.joinpath("src", "app.py")), run_name="__main__")