Updating app with latest changes

backend/api/main.py

@@ -113,16 +113,26 @@ async def search(
     query: str = Form(...),
     top_k: int = Form(5),
     add_to_dataset: bool = Form(False),
-    protection: bool = Form(False),
+    pii_detection: bool = Form(False),
     hallucination_detection: bool = Form(False),
     induce_hallucination: bool = Form(False),
     project_name: str = Form(...),
     logstream_name: str = Form(...),
     dataset_name: str = Form(...),
 ) -> JSONResponse:
+    logger.info("=" * 80)
+    logger.info("SEARCH REQUEST RECEIVED")
+    logger.info(f"Query: {query}")
+    logger.info(f"Top K: {top_k}")
+    logger.info(f"Add to Dataset: {add_to_dataset}")
+    logger.info(f"PII Detection: {pii_detection}")
+    logger.info(f"Hallucination Detection: {hallucination_detection}")
+    logger.info(f"Induce Hallucination: {induce_hallucination}")
+    logger.info("=" * 80)
+    
     response, redacted_response, original_response, context_adherence_score, pii_flag = rag_app.run(
         query,
-        protect_enabled=protection,
+        pii_detection=pii_detection,
         top_k=top_k,
         hallucination_detection=hallucination_detection,
         induce_hallucination=induce_hallucination,

backend/api/templates/index.html

@@ -56,7 +56,7 @@
                     type="number"
                     id="top_k"
                     name="top_k"
-                    value="5"
+                    value="10"
                     class="mt-1 p-2 w-full border rounded"
                 />
             </label>
@@ -72,11 +72,11 @@
             <label class="flex items-center space-x-2">
                 <input
                     type="checkbox"
-                    id="protection"
-                    name="protection"
+                    id="pii_detection"
+                    name="pii_detection"
                     class="form-checkbox text-green-600 focus:ring-green-500"
                 />
-                <span>Enable Galileo Protection</span>
+                <span>Enable PII Detection</span>
             </label>
             <label class="flex items-center space-x-2">
                 <input
@@ -152,8 +152,8 @@
             if (urlParams.has('project_name')) {
                 $('#project_name').val(urlParams.get('project_name'));
             }
-            if (urlParams.has('protection')) {
-                $('#protection').prop('checked', urlParams.get('protection') === 'true');
+            if (urlParams.has('pii_detection')) {
+                $('#pii_detection').prop('checked', urlParams.get('pii_detection') === 'true');
             }
             if (urlParams.has('hallucination_detection')) {
                 $('#hallucination_detection').prop('checked', urlParams.get('hallucination_detection') === 'true');
@@ -168,7 +168,7 @@
                 const query = $('#query').val();
                 const top_k = $('#top_k').val();
                 const add_to_dataset = $('#add_to_dataset').is(':checked');
-                const protection = $('#protection').is(':checked');
+                const pii_detection = $('#pii_detection').is(':checked');
                 const hallucination_detection = $('#hallucination_detection').is(':checked');
                 const induce_hallucination = $('#induce_hallucination').is(':checked');
                 const project_name = $('#project_name').val();
@@ -188,7 +188,7 @@
                         query: query,
                         top_k: top_k,
                         add_to_dataset: add_to_dataset,
-                        protection: protection,
+                        pii_detection: pii_detection,
                         hallucination_detection: hallucination_detection,
                         induce_hallucination: induce_hallucination,
                         project_name: project_name,
@@ -209,12 +209,12 @@
                         if (piiDetected) {
                             // Build specific PII warning message
                             const detectedTypes = [];
-                            if (piiFlag.phone_number) detectedTypes.push('phone number');
+                            if (piiFlag.phone_number) detectedTypes.push('<span style="color:yellow; font-weight: bold">phone number</span>');
                             if (piiFlag.email) detectedTypes.push('<span style="color:yellow; font-weight: bold">email address</span>');
                             if (piiFlag.name) detectedTypes.push('<span style="color:yellow; font-weight: bold">personal name</span>');
                             if (piiFlag.company) detectedTypes.push('<span style="color:yellow; font-weight: bold">company name</span>');
                             
-                            let piiMessage = 'Sensitive personally identifiable information detected! The following types of PII were found: ';
+                            let piiMessage = 'Sensitive personally identifiable information detected! The following types of PII were found: <br>';
                             if (detectedTypes.length === 1) {
                                 piiMessage += detectedTypes[0];
                             } else if (detectedTypes.length === 2) {
@@ -262,7 +262,6 @@
                                                 <style>
                                                     pii {
                                                         font-weight: bold;
-                                                        text-decoration: line-through;
                                                         background-color: yellow;
                                                     }
                                                 </style>
@@ -323,7 +322,7 @@
                                                     <input
                                                         type="number"
                                                         id="retry_top_k"
-                                                        value="5"
+                                                        value="10"
                                                         min="1"
                                                         max="100"
                                                         class="ml-2 p-1 w-16 border border-blue-300 rounded text-sm"
@@ -402,7 +401,7 @@
                 const dataset_name = $('#dataset_name').val();
                 const logstream_name = $('#logstream_name').val();
                 const add_to_dataset = $('#add_to_dataset').is(':checked');
-                const protection = $('#protection').is(':checked');
+                const pii_detection = $('#pii_detection').is(':checked');
                 const hallucination_detection = $('#hallucination_detection').is(':checked');
                 const induce_hallucination = $('#induce_hallucination').is(':checked');
                 // Create URL parameters to reload with form pre-filled
@@ -413,7 +412,7 @@
                 if (project_name) params.set('project_name', project_name);
                 if (logstream_name) params.set('logstream_name', logstream_name);
                 if (add_to_dataset) params.set('add_to_dataset', 'true');
-                if (protection) params.set('protection', 'true');
+                if (pii_detection) params.set('pii_detection', 'true');
                 if (hallucination_detection) params.set('hallucination_detection', 'true');
                 if (induce_hallucination) params.set('induce_hallucination', 'true');
                 // Reload the page with parameters

backend/classes/galileo_platform.py

@@ -1,16 +1,11 @@
+import json
 from pydantic import BaseModel
 from dotenv import load_dotenv
-from typing import Any
 
-from galileo import GalileoLogger, GalileoScorers, StageType
+from galileo import GalileoLogger
 from galileo.log_streams import create_log_stream, get_log_stream
 from galileo.projects import create_project, get_project
-from galileo.protect import invoke_protect
-from galileo.stages import create_protect_stage, get_protect_stage
-from galileo_core.schemas.protect.action import OverrideAction
-from galileo_core.schemas.protect.payload import Payload
-from galileo_core.schemas.protect.rule import Rule, RuleOperator
-from galileo_core.schemas.protect.ruleset import Ruleset
+import galileo_protect as gp
 
 load_dotenv()
 
@@ -26,7 +21,7 @@ class GalileoPlatform:
         self.config = config
         self.protect_stage_id = self.get_protect_stage_id()
 
-    def get_logger(self, project_name: str, logstream_name: str) -> GalileoLogger | None:
+    def get_logger(self, project_name: str, logstream_name: str):
         """Get or create a Galileo Logger."""
         if not get_project(name=project_name):
             _ = create_project(name=project_name)
@@ -36,76 +31,161 @@ class GalileoPlatform:
             _ = create_log_stream(name=logstream_name, project_name=project_name)
             print(f"Logstream {logstream_name} created in project {project_name}")
 
+        return GalileoLogger(
+            project=project_name,
+            log_stream=logstream_name,
+        )
+
+
+    def get_protect_stage_id(self) -> str:
+        """Get or create a Galileo Protect stage."""
+        try:
+            protect_project = gp.get_project(
+                project_name=self.config.protect_project_name
+            )
+            print(f"Protect project {self.config.protect_project_name} found")
+        except Exception as _:
+            protect_project = gp.create_project(name=self.config.protect_project_name)
+            print(f"Protect project {self.config.protect_project_name} created")
+        protect_project_id = protect_project.id
+
         try:
-            return GalileoLogger(
-                project=project_name,
-                log_stream=logstream_name,
+            protect_stage = gp.get_stage(
+                project_id=protect_project_id, stage_name=self.config.protect_stage_name
             )
-        except Exception as e:
-            print(f"Failed to create logger: {e}")
+            print(f"Protect stage {self.config.protect_stage_name} found")
+        except Exception as _:
+            protect_stage = gp.create_stage(
+                project_id=protect_project_id,
+                name=self.config.protect_stage_name,
+            )
+
+        return protect_stage.id
+
+    def run_protect_v1(
+        self,
+        input: str,
+        output: str,
+        hallucination_detection: bool = False,
+        pii_detection: bool = False,
+        logger: GalileoLogger | None = None,
+    ) -> dict | None:
+        """Run Galileo Protect on input and output."""
+
+        # print(f"Running Galileo Protect with PII detection: {pii_detection} and Hallucination detection: {hallucination_detection}")
+
+        rulesets = []
+
+        if not (hallucination_detection or pii_detection):
             return None
 
-    def get_protect_stage_id(self) -> str | None:
-        """Get or create a Galileo Protect stage."""
-        if not get_project(name=self.config.protect_project_name):
-            _ = create_project(name=self.config.protect_project_name)
-            print(f"Project {self.config.protect_project_name} created")
+        if hallucination_detection:
+            rulesets.append(gp.Ruleset(
+                rules=[
+                    gp.Rule(
+                        metric=gp.RuleMetrics.context_adherence_luna,
+                        operator=gp.RuleOperator.lte,
+                        target_value=0.8,
+                    ),
+                ],
+                action=gp.OverrideAction(
+                    choices=["I cannot provide a reliable answer to this question based on the available information! Please try again."]
+                ),
+            ))
+
+        if pii_detection:
+            rulesets.extend([
+                gp.Ruleset(
+                    rules=[
+                        gp.Rule(
+                            metric=gp.RuleMetrics.pii,
+                            operator=gp.RuleOperator.any,
+                            target_value=["email", "phone_number", "name"],
+                        ),
+                    ],
+                    action=gp.OverrideAction(
+                        choices=["Sorry, the output contains PII."]
+                    ),
+                ),  
+                gp.Ruleset(
+                    rules=[
+                        gp.Rule(
+                            metric="fairfield_company_pii_scorer_0",
+                            operator=gp.RuleOperator.gte,
+                            target_value=0.1,
+                        )
+                    ],
+                    action=gp.OverrideAction(
+                        choices=["Sorry, the output contains PII."]
+                    ),
+                )
+            ])
 
-        protect_stage = get_protect_stage(
-            project_name=self.config.protect_project_name,
-            stage_name=self.config.protect_stage_name,
+        response = gp.invoke(
+            payload=gp.Payload(input=input, output=output),
+            prioritized_rulesets=rulesets,
+            stage_id=self.protect_stage_id,
         )
-        if not protect_stage:
-            _ = create_protect_stage(
-                project_name=self.config.protect_project_name,
-                name=self.config.protect_stage_name,
-                stage_type=StageType.local
-            )
-            print(f"Protect stage {self.config.protect_stage_name} created in project {self.config.protect_project_name}")
-            protect_stage = get_protect_stage(
-                project_name=self.config.protect_project_name,
-                stage_name=self.config.protect_stage_name,
+
+        if logger:
+            logger.add_protect_span(
+                payload=gp.Payload(input=input, output=output),
+                response=response,
             )
-        return protect_stage.id
 
-    def run_protect(self, input: str, output: str, logger: GalileoLogger | None = None) -> dict[Any, Any]:
+        return dict(response)
+
+    def run_protect_v2(
+        self,
+        input: str,
+        output: str,
+        pii_detection: bool = False,
+        hallucination_detection: bool = False,
+        logger: GalileoLogger | None= None,
+    ) -> dict:
         """Run Galileo Protect on input and output."""
-        response = invoke_protect(
-            payload=Payload(input=input, output=output),
+
+        if not (pii_detection or hallucination_detection):
+            return None
+
+        # print(f"Running Galileo Protect v2 with PII detection: {pii_detection} and Hallucination detection: {hallucination_detection}")
+
+        response = gp.invoke(
+            payload=gp.Payload(input=input, output=output),
             prioritized_rulesets=[
-                Ruleset(
+                gp.Ruleset(
                     rules=[
-                        Rule(
-                            metric=GalileoScorers.context_adherence_luna,
-                            operator=RuleOperator.lte,
-                            target_value=0.01,
+                        gp.Rule(
+                            metric=gp.RuleMetrics.context_adherence_luna,
+                            operator=gp.RuleOperator.lte,
+                            target_value=0.8,
                         ),
                     ],
-                    action=OverrideAction(
-                        choices=["Sorry, the input is hallucinatory."]
+                    action=gp.OverrideAction(
+                        choices=["I cannot provide a reliable answer to this question based on the available information! Please try again."]
                     ),
                 ),
-                Ruleset(
+                gp.Ruleset(
                     rules=[
-                        Rule(
-                            metric=GalileoScorers.output_pii,
-                            operator=RuleOperator.any,
+                        gp.Rule(
+                            metric=gp.RuleMetrics.pii,
+                            operator=gp.RuleOperator.any,
                             target_value=["email", "phone_number", "name"],
                         )
                     ],
-                    action=OverrideAction(
+                    action=gp.OverrideAction(
                         choices=["Sorry, the output contains PII."]
                     ),
                 ),
-                Ruleset(
+                gp.Ruleset(
                     rules=[
-                        Rule(
-                            metric="deutsche_bank_company_pii_scorer_0",
-                            operator=RuleOperator.gte,
+                        gp.Rule(
+                            metric="fairfield_company_pii_scorer_0",
+                            operator=gp.RuleOperator.gte,
                             target_value=0.1,
                         )
                     ],
-                    action=OverrideAction(
+                    action=gp.OverrideAction(
                         choices=["Sorry, the output contains PII."]
                     ),
                 )
@@ -113,10 +193,30 @@ class GalileoPlatform:
             stage_id=self.protect_stage_id,
         )
 
+        response_dict = response.model_dump()
+        filtered_ruleset_results = []
+
+        metrics_to_exclude = set()
+        if not hallucination_detection:
+            metrics_to_exclude.add('context_adherence_luna')
+        if not pii_detection:
+            metrics_to_exclude.add('pii')
+            metrics_to_exclude.add('fairfield_company_pii_scorer_0')
+        
+        for metric in metrics_to_exclude:
+            response_dict["metric_results"].pop(metric, None)
+        
+        for ruleset in response_dict["ruleset_results"]:
+            metric = ruleset["rules"][0]["metric"]
+            if metric not in metrics_to_exclude:
+                filtered_ruleset_results.append(ruleset)
+
+        response_dict["ruleset_results"] = filtered_ruleset_results
+
         if logger:
             logger.add_protect_span(
-                payload=Payload(input=input, output=output),
-                response=response,
+                payload=gp.Payload(input=input, output=output),
+                response=gp.Response.model_validate(response_dict),
             )
 
-        return dict(response)
+        return response_dict

backend/classes/rag_application.py

@@ -1,3 +1,4 @@
+import re
 import time
 from typing import List, Union
 
@@ -72,7 +73,7 @@ class RAGApplication:
         self,
         query: str,
         prompt_template: str = None,
-        protect_enabled: bool = False,
+        pii_detection: bool = False,
         top_k: int = 5,
         hallucination_detection: bool = False,
         induce_hallucination: bool = False,
@@ -129,10 +130,12 @@ class RAGApplication:
 
             formatted_prompt = f"{prompt}\n\nQUESTION: {query}\n\nCONTEXT: {context}"
 
-            result = self.config.generative_model.generate_response(
+            llm_result = self.config.generative_model.generate_response(
                 formatted_prompt
             )
 
+            result = llm_result
+
             if induce_hallucination:
                 original_result = result
                 hallucinatory_prompt = self.config.hallucinatory_prompt_template.format(question=query, context=context, original_response=result)
@@ -172,23 +175,37 @@ class RAGApplication:
 
             start_time = time.time()
             
-            protect_response = self.config.galileo_platform.run_protect(
-                context, result, galileo_logger
+            # protect_response_v1 = self.config.galileo_platform.run_protect_v1(
+            #     input=context,
+            #     output=result,
+            #     hallucination_detection=hallucination_detection,
+            #     pii_detection=pii_detection,
+            #     logger=galileo_logger,
+            # )
+            # print("Protect response v1:", protect_response_v1)
+
+            protect_response = self.config.galileo_platform.run_protect_v2(
+                input=context,
+                output=result,
+                pii_detection=pii_detection,
+                hallucination_detection=hallucination_detection,
+                logger=galileo_logger,
             )
+            # print("Protect response v2:", protect_response)
 
-            if protect_enabled and protect_response["text"] != result:
+            if pii_detection and protect_response and protect_response["text"] != result:
                 pii_flag["phone_number"] = "phone_number" in protect_response["metric_results"]["pii"]["value"]
                 pii_flag["email"] = "email" in protect_response["metric_results"]["pii"]["value"]
                 pii_flag["name"] = "name" in protect_response["metric_results"]["pii"]["value"]
-                pii_flag["company"] = protect_response["metric_results"]["deutsche_bank_company_pii_0"]["value"]>0.1
+                pii_flag["company"] = protect_response["metric_results"]["fairfield_company_pii_scorer_0"]["value"]>0.1
                 redacted_result = self.get_redacted_result(result, pii_flag)
-                # redacted_result = re.sub(r'<pii>(.*?)</pii>', r'<pii>REDACTED</pii>', redacted_result)
                 result = redacted_result.replace("<pii>", "<tag>").replace("</pii>", "</tag>")
+                redacted_result = re.sub(r'<pii>(.*?)</pii>', r'<pii>REDACTED</pii>', redacted_result)
 
             if hallucination_detection:
                 context_adherence_score = protect_response["metric_results"]["context_adherence_luna"]["value"]
 
-            galileo_logger.conclude(output=result)
+            galileo_logger.conclude(output=llm_result)
             galileo_logger.flush()
 
             return result, redacted_result, original_result, context_adherence_score, pii_flag

data/processed/vector_db/rfp_data_milvus.db

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20c91eb1ccbb74b7fa6c38ea48dae8e440731f088d790742d753ffaee5346cb9
-size 249856
+oid sha256:6d48c75024c05b50782d45b796c2f18cdaace4d1bf007c7dbcface1e1d23d907
+size 192512