Add large model file to Git LFS

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+rai_privacy_package/privacy/rai_privacy/privacy/privacy/util/face_detect/face_detector/res10_300x300_ssd_iter_140000.caffemodel filter=lfs diff=lfs merge=lfs -text
+rai_privacy_package/privacy/rai_privacy/privacy/privacy/util/face_detect/doc/10.jpg filter=lfs diff=lfs merge=lfs -text
+rai_privacy_package/privacy/rai_privacy/privacy/privacy/util/face_detect/doc/5.jpg filter=lfs diff=lfs merge=lfs -text
+rai_privacy_package/privacy/rai_privacy/privacy/privacy/util/face_detect/doc/8.jpg filter=lfs diff=lfs merge=lfs -text

LICENSE.md

@@ -0,0 +1,8 @@
+
+MIT license https://opensource.org/licenses/MIT Copyright 2024-2025 Infosys Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

api-spec/rai-privacy.yaml

@@ -0,0 +1,389 @@
+openapi: 3.0.2
+info:
+  title: Infosys Responsible AI - responsible-ai-privacy - OpenAPI 3.0
+  description: API specs for Infosys Responsible AI Privacy pillar in OpenAPI 3.0 format
+  termsOfService: https://www.infosys.com
+  contact:
+    email: aina@infosys.com
+  license:
+    name: Infosys
+    url: https://www.infosys.com
+  version: v$version
+security:
+  - oauth_auth:
+    - write:users
+    - read:users
+paths:
+  /api/v1/privacy/pii/analyze:
+    post:
+      tags:
+        - PII Privacy
+      summary: Analyze
+      operationId: analyze_api_v1_privacy_pii_analyze_post
+      security:
+        - oauth_auth:
+          - write:users
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/PIIAnalyzeRequest'
+        required: true
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PIIAnalyzeResponse'
+        '401':
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '403':
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '422':
+          description: Validation Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+  /api/v1/privacy/pii/anonymize:
+    post:
+      tags:
+        - PII Privacy
+      summary: Anonymize
+      operationId: anonymize_api_v1_privacy_pii_anonymize_post
+      security:
+        - oauth_auth:
+          - write:users
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/PIIAnonymizeRequest'
+        required: true
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PIIAnonymizeResponse'
+        '401':
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '403':
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '422':
+          description: Validation Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+  /api/v1/privacy/pii/image/analyze:
+    post:
+      tags:
+        - PII Privacy
+      summary: Image Analyze
+      operationId: image_analyze_api_v1_privacy_pii_image_analyze_post
+      security:
+        - oauth_auth:
+          - write:users
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/Body_image_analyze_api_v1_privacy_pii_image_analyze_post'
+        required: true
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PIIImageAnalyzeResponse'
+        '401':
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '403':
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '422':
+          description: Validation Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+  /api/v1/privacy/pii/image/anonymize:
+    post:
+      tags:
+        - PII Privacy
+      summary: Image Anonymize
+      operationId: image_anonymize_api_v1_privacy_pii_image_anonymize_post
+      security:
+        - oauth_auth:
+          - write:users
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/Body_image_anonymize_api_v1_privacy_pii_image_anonymize_post'
+        required: true
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '403':
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '422':
+          description: Validation Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+  /api/v1/privacy/pii/image/verify:
+    post:
+      tags:
+        - PII Privacy
+      summary: Image Verify
+      operationId: image_verify_api_v1_privacy_pii_image_verify_post
+      security:
+        - oauth_auth:
+          - write:users
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/Body_image_verify_api_v1_privacy_pii_image_verify_post'
+        required: true
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: Unauthorized
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '403':
+          description: Forbidden
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '422':
+          description: Validation Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+components:
+  schemas:
+    Body_image_analyze_api_v1_privacy_pii_image_analyze_post:
+      title: Body_image_analyze_api_v1_privacy_pii_image_analyze_post
+      required:
+        - payload
+      type: object
+      properties:
+        payload:
+          title: Payload
+          type: string
+          format: binary
+    Body_image_anonymize_api_v1_privacy_pii_image_anonymize_post:
+      title: Body_image_anonymize_api_v1_privacy_pii_image_anonymize_post
+      required:
+        - payload
+      type: object
+      properties:
+        payload:
+          title: Payload
+          type: string
+          format: binary
+    Body_image_verify_api_v1_privacy_pii_image_verify_post:
+      title: Body_image_verify_api_v1_privacy_pii_image_verify_post
+      required:
+        - payload
+      type: object
+      properties:
+        payload:
+          title: Payload
+          type: string
+          format: binary
+    HTTPValidationError:
+      title: HTTPValidationError
+      type: object
+      properties:
+        detail:
+          title: Detail
+          type: array
+          items:
+            $ref: '#/components/schemas/ValidationError'
+    PIIAnalyzeRequest:
+      title: PIIAnalyzeRequest
+      required:
+        - inputText
+      type: object
+      properties:
+        inputText:
+          title: Inputtext
+          type: string
+          example: John Smith's SSN is 012884567
+    PIIAnalyzeResponse:
+      title: PIIAnalyzeResponse
+      required:
+        - PIIEntities
+      type: object
+      properties:
+        PIIEntities:
+          title: Piientities
+          type: array
+          items:
+            $ref: '#/components/schemas/PIIEntity'
+    PIIAnonymizeRequest:
+      title: PIIAnonymizeRequest
+      required:
+        - inputText
+      type: object
+      properties:
+        inputText:
+          title: Inputtext
+          type: string
+          example: John Smith's SSN is 012884567
+        piiEntitiesToBeRedacted:
+          title: Piientitiestoberedacted
+          type: array
+          items: {}
+          example:
+            - US_SSN
+        redactionType:
+          title: Redactiontype
+          type: string
+          example: replace
+    PIIAnonymizeResponse:
+      title: PIIAnonymizeResponse
+      required:
+        - anonymizedText
+      type: object
+      properties:
+        anonymizedText:
+          title: Anonymizedtext
+          type: string
+          example: John Smith's SSN is <US_SSN>
+    PIIEntity:
+      title: PIIEntity
+      required:
+        - type
+        - beginOffset
+        - endOffset
+        - confidenceScore
+      type: object
+      properties:
+        type:
+          title: Type
+          type: string
+          example: US_SSN
+        beginOffset:
+          title: Beginoffset
+          type: integer
+          example: 19
+        endOffset:
+          title: Endoffset
+          type: integer
+          example: 28
+        confidenceScore:
+          title: ConfidenceScore
+          type: number
+          example: 0.25
+    PIIImageAnalyzeResponse:
+      title: PIIImageAnalyzeResponse
+      required:
+        - PIIEntities
+      type: object
+      properties:
+        PIIEntities:
+          title: Piientities
+          type: array
+          items:
+            $ref: '#/components/schemas/PIIImageEntity'
+    PIIImageEntity:
+      title: PIIImageEntity
+      required:
+        - type
+      type: object
+      properties:
+        type:
+          title: Type
+          type: string
+          example: US_SSN
+    ValidationError:
+      title: ValidationError
+      required:
+        - loc
+        - msg
+        - type
+      type: object
+      properties:
+        loc:
+          title: Location
+          type: array
+          items:
+            anyOf:
+              - type: string
+              - type: integer
+        msg:
+          title: Message
+          type: string
+        type:
+          title: Error Type
+          type: string
+  securitySchemes:
+    oauth_auth:
+      type: oauth2
+      flows:
+        authorizationCode:
+          authorizationUrl: https://example.com/oauth/authorize
+          tokenUrl: https://example.com/oauth/token
+          scopes:
+            write:users: modify user profile
+            
+tags:
+  - name: PII Privacy
+    description: Operations required for a PII entity (e.g. IN_ADHAAR, IN_PAN, US_SSN etc)
+    externalDocs:
+      description: Find out more
+      url: https://www.infosys.com

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/build_config.yaml

@@ -0,0 +1,14 @@
+-
+    name: presidio_analyzer
+    version: 4.1.0
+    build: 0.0.1
+    author: Amit Hegde
+    author_email: amitumamaheshwar.h@infosys.com
+    description: Infosys Intelligent Assistant
+    long_description: Infosys Intelligent Assistant
+    classifiers: ["Programming Language :: Python :: 3",
+                  "License :: OSI Approved :: MIT License",
+                  "Operating System :: OS Independent",]
+    package_dir: {"": "presidio_analyzer"}
+    packages: presidio_analyzer
+    python_requires: ['>=3.6']

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/create_wheel_file.py

@@ -0,0 +1,44 @@
+__copyright__ = """ 2020 - 2021 Infosys Limited, Bangalore, India. All Rights Reserved.  
+Version: 2.5.0.0 
+Except for any free or open source software components embedded in this Infosys proprietary software program (“Program”), this Program is protected by copyright laws, international treaties and other pending or existing intellectual property rights in India, the United States and other countries.
+Except as expressly permitted, any unauthorized reproduction, storage, transmission in any form or by any means (including without limitation electronic, mechanical, printing, photocopying, recording or otherwise), or any distribution of this Program, or any portion of it, may result in severe civil and criminal penalties, and will be prosecuted to the maximum extent possible under the law.
+"""
+import yaml
+import subprocess
+import os
+with open(r'.\build_config.yaml') as build_file:
+    build_config_list = yaml.safe_load(build_file)
+
+
+for build_config in build_config_list:
+
+    try:
+        print(build_config)
+
+        if os.path.exists(f"./{build_config['packages']}"):
+
+            setup_str = f"import setuptools\r" \
+                        f"setuptools.setup(\r \
+                name='{build_config['name']}',\r \
+                version='{build_config['version']}',\r \
+                author='{build_config['author']}',\r \
+                author_email='{build_config['author_email']}',\r \
+                description='{build_config['description']}',\r \
+                long_description='{build_config['long_description']}',\r \
+                classifiers={build_config['classifiers']},\r \
+                package_dir={build_config['package_dir']},\r \
+                packages=setuptools.find_packages(where='{build_config['packages']}'),\r \
+                python_requires='{build_config['python_requires'][0]}',\r \
+                )"
+
+            with open('setup.py','w') as file:
+                file.write(setup_str)
+
+            subprocess.run(["python", "-m","build"])
+            wheel_file = f"{build_config['name']}-{build_config['version']}_build_{build_config['build']}-py3-none-any.whl"
+            print(f"wheel_file: {wheel_file}")
+            subprocess.run(["python", "-m", "pyc_wheel", f"dist\{wheel_file}"])
+        else:
+            print(f"Path does not exist ./{build_config['packages']}")
+    except Exception as e:
+        print("Exception occurred")

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/dist/presidio_analyzer-4.0.5.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:effdee5c88badc2a4605dcabc7fe1ff43df586df0a7c2be3f4dbc4d440c7e4d6
+size 44375

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/dist/presidio_analyzer-4.0.6.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c49ca4ee3acda590bb69b68697e02cbfc81b89bd8dcfcaf9ff90b07fec062515
+size 44656

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/dist/presidio_analyzer-4.1.0.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:595ba3a58a473cc94a2a5c421eea075c5db52cb0181335a92f3a222f5cc76736
+size 44675

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: presidio_analyzer
+Version: 4.1.0
+Summary: Infosys Intelligent Assistant
+Author: Amit Hegde
+Author-email: amitumamaheshwar.h@infosys.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+
+Infosys Intelligent Assistant

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer.egg-info/SOURCES.txt

@@ -0,0 +1,64 @@
+setup.py
+presidio_analyzer/presidio_analyzer/__init__.py
+presidio_analyzer/presidio_analyzer/analysis_explanation.py
+presidio_analyzer/presidio_analyzer/analyzer_engine.py
+presidio_analyzer/presidio_analyzer/analyzer_request.py
+presidio_analyzer/presidio_analyzer/app_tracer.py
+presidio_analyzer/presidio_analyzer/batch_analyzer_engine.py
+presidio_analyzer/presidio_analyzer/dict_analyzer_result.py
+presidio_analyzer/presidio_analyzer/entity_recognizer.py
+presidio_analyzer/presidio_analyzer/local_recognizer.py
+presidio_analyzer/presidio_analyzer/pattern.py
+presidio_analyzer/presidio_analyzer/pattern_recognizer.py
+presidio_analyzer/presidio_analyzer/recognizer_result.py
+presidio_analyzer/presidio_analyzer/remote_recognizer.py
+presidio_analyzer/presidio_analyzer.egg-info/PKG-INFO
+presidio_analyzer/presidio_analyzer.egg-info/SOURCES.txt
+presidio_analyzer/presidio_analyzer.egg-info/dependency_links.txt
+presidio_analyzer/presidio_analyzer.egg-info/top_level.txt
+presidio_analyzer/presidio_analyzer/context_aware_enhancers/__init__.py
+presidio_analyzer/presidio_analyzer/context_aware_enhancers/context_aware_enhancer.py
+presidio_analyzer/presidio_analyzer/context_aware_enhancers/lemma_context_aware_enhancer.py
+presidio_analyzer/presidio_analyzer/nlp_engine/__init__.py
+presidio_analyzer/presidio_analyzer/nlp_engine/client_nlp_engine.py
+presidio_analyzer/presidio_analyzer/nlp_engine/nlp_artifacts.py
+presidio_analyzer/presidio_analyzer/nlp_engine/nlp_engine.py
+presidio_analyzer/presidio_analyzer/nlp_engine/nlp_engine_provider.py
+presidio_analyzer/presidio_analyzer/nlp_engine/spacy_nlp_engine.py
+presidio_analyzer/presidio_analyzer/nlp_engine/stanza_nlp_engine.py
+presidio_analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/Aadhaar_Number.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/PAN_Number.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/__init__.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/au_abn_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/au_acn_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/au_medicare_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/au_tfn_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/credit_card_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/crypto_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/data_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/date_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/email_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/es_nif_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/iban_patterns.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/iban_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/ip_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_driver_license_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_fiscal_code_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_identity_card_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_passport_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/it_vat_code.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/medical_license_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/phone_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/sg_fin_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/spacy_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/stanza_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/transformers_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/uk_nhs_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/url_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/us_driver_license_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/us_itin_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/us_passport_recognizer.py
+presidio_analyzer/presidio_analyzer/predefined_recognizers/us_ssn_recognizer.py
+presidio_analyzer/presidio_analyzer/recognizer_registry/__init__.py
+presidio_analyzer/presidio_analyzer/recognizer_registry/recognizer_registry.py

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer.egg-info/top_level.txt

@@ -0,0 +1 @@
+presidio_analyzer

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/__init__.py

@@ -0,0 +1,52 @@
+"""Presidio analyzer package."""
+
+import logging
+
+from presidio_analyzer.pattern import Pattern
+from presidio_analyzer.analysis_explanation import AnalysisExplanation
+from presidio_analyzer.recognizer_result import RecognizerResult
+from presidio_analyzer.dict_analyzer_result import DictAnalyzerResult
+from presidio_analyzer.entity_recognizer import EntityRecognizer
+from presidio_analyzer.local_recognizer import LocalRecognizer
+from presidio_analyzer.pattern_recognizer import PatternRecognizer
+from presidio_analyzer.remote_recognizer import RemoteRecognizer
+from presidio_analyzer.recognizer_registry import RecognizerRegistry
+from presidio_analyzer.analyzer_engine import AnalyzerEngine
+from presidio_analyzer.batch_analyzer_engine import BatchAnalyzerEngine
+from presidio_analyzer.analyzer_request import AnalyzerRequest
+from presidio_analyzer.context_aware_enhancers import ContextAwareEnhancer
+from presidio_analyzer.context_aware_enhancers import LemmaContextAwareEnhancer
+
+
+# Define default loggers behavior
+
+# 1. presidio_analyzer logger
+
+logging.getLogger("presidio_analyzer").addHandler(logging.NullHandler())
+
+# 2. decision_process logger.
+# Setting the decision process trace here as we would want it
+# to be activated using a parameter to AnalyzeEngine and not by default.
+
+decision_process_logger = logging.getLogger("decision_process")
+ch = logging.StreamHandler()
+formatter = logging.Formatter("[%(asctime)s][%(name)s][%(levelname)s]%(message)s")
+ch.setFormatter(formatter)
+decision_process_logger.addHandler(ch)
+decision_process_logger.setLevel("INFO")
+__all__ = [
+    "Pattern",
+    "AnalysisExplanation",
+    "RecognizerResult",
+    "DictAnalyzerResult",
+    "EntityRecognizer",
+    "LocalRecognizer",
+    "PatternRecognizer",
+    "RemoteRecognizer",
+    "RecognizerRegistry",
+    "AnalyzerEngine",
+    "AnalyzerRequest",
+    "ContextAwareEnhancer",
+    "LemmaContextAwareEnhancer",
+    "BatchAnalyzerEngine",
+]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/analysis_explanation.py

@@ -0,0 +1,64 @@
+from typing import Dict
+
+
+class AnalysisExplanation:
+    """
+    Hold tracing information to explain why PII entities were identified as such.
+
+    :param recognizer: name of recognizer that made the decision
+    :param original_score: recognizer's confidence in result
+    :param pattern_name: name of pattern
+            (if decision was made by a PatternRecognizer)
+    :param pattern: regex pattern that was applied (if PatternRecognizer)
+    :param validation_result: result of a validation (e.g. checksum)
+    :param textual_explanation: Free text for describing
+            a decision of a logic or model
+    """
+
+    def __init__(
+        self,
+        recognizer: str,
+        original_score: float,
+        pattern_name: str = None,
+        pattern: str = None,
+        validation_result: float = None,
+        textual_explanation: str = None,
+    ):
+
+        self.recognizer = recognizer
+        self.pattern_name = pattern_name
+        self.pattern = pattern
+        self.original_score = original_score
+        self.score = original_score
+        self.textual_explanation = textual_explanation
+        self.score_context_improvement = 0
+        self.supportive_context_word = ""
+        self.validation_result = validation_result
+
+    def __repr__(self):
+        """Create string representation of the object."""
+        return str(self.__dict__)
+
+    def set_improved_score(self, score: float) -> None:
+        """Update the score and calculate the difference from the original score."""
+        self.score = score
+        self.score_context_improvement = self.score - self.original_score
+
+    def set_supportive_context_word(self, word: str) -> None:
+        """Set the context word which helped increase the score."""
+        self.supportive_context_word = word
+
+    def append_textual_explanation_line(self, text: str) -> None:
+        """Append a new line to textual_explanation field."""
+        if self.textual_explanation is None:
+            self.textual_explanation = text
+        else:
+            self.textual_explanation = "{}\n{}".format(self.textual_explanation, text)
+
+    def to_dict(self) -> Dict:
+        """
+        Serialize self to dictionary.
+
+        :return: a dictionary
+        """
+        return self.__dict__

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/analyzer_engine.py

@@ -0,0 +1,372 @@
+import json
+import logging
+from typing import List, Optional
+
+from presidio_analyzer import (
+    RecognizerRegistry,
+    RecognizerResult,
+    EntityRecognizer,
+)
+from presidio_analyzer.app_tracer import AppTracer
+from presidio_analyzer.context_aware_enhancers import (
+    ContextAwareEnhancer,
+    LemmaContextAwareEnhancer,
+)
+from presidio_analyzer.nlp_engine import NlpEngine, NlpEngineProvider, NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class AnalyzerEngine:
+    """
+    Entry point for Presidio Analyzer.
+
+    Orchestrating the detection of PII entities and all related logic.
+
+    :param registry: instance of type RecognizerRegistry
+    :param nlp_engine: instance of type NlpEngine
+    (for example SpacyNlpEngine)
+    :param app_tracer: instance of type AppTracer, used to trace the logic
+    used during each request for interpretability reasons.
+    :param log_decision_process: bool,
+    defines whether the decision process within the analyzer should be logged or not.
+    :param default_score_threshold: Minimum confidence value
+    for detected entities to be returned
+    :param supported_languages: List of possible languages this engine could be run on.
+    Used for loading the right NLP models and recognizers for these languages.
+    :param context_aware_enhancer: instance of type ContextAwareEnhancer for enhancing
+    confidence score based on context words, (LemmaContextAwareEnhancer will be created
+    by default if None passed)
+    """
+
+    def __init__(
+        self,
+        registry: RecognizerRegistry = None,
+        nlp_engine: NlpEngine = None,
+        app_tracer: AppTracer = None,
+        log_decision_process: bool = False,
+        default_score_threshold: float = 0,
+        supported_languages: List[str] = None,
+        context_aware_enhancer: Optional[ContextAwareEnhancer] = None,
+    ):
+        if not supported_languages:
+            supported_languages = ["en"]
+
+        if not nlp_engine:
+            logger.info("nlp_engine not provided, creating default.")
+            provider = NlpEngineProvider()
+            nlp_engine = provider.create_engine()
+
+        if not registry:
+            logger.info("registry not provided, creating default.")
+            registry = RecognizerRegistry()
+        if not app_tracer:
+            app_tracer = AppTracer()
+        self.app_tracer = app_tracer
+
+        self.supported_languages = supported_languages
+
+        self.nlp_engine = nlp_engine
+        self.registry = registry
+
+        # load all recognizers
+        if not registry.recognizers:
+            registry.load_predefined_recognizers(
+                nlp_engine=self.nlp_engine, languages=self.supported_languages
+            )
+
+        self.log_decision_process = log_decision_process
+        self.default_score_threshold = default_score_threshold
+
+        if not context_aware_enhancer:
+            logger.debug(
+                "context aware enhancer not provided, creating default"
+                + " lemma based enhancer."
+            )
+            context_aware_enhancer = LemmaContextAwareEnhancer()
+
+        self.context_aware_enhancer = context_aware_enhancer
+
+    def get_recognizers(self, language: Optional[str] = None) -> List[EntityRecognizer]:
+        """
+        Return a list of PII recognizers currently loaded.
+
+        :param language: Return the recognizers supporting a given language.
+        :return: List of [Recognizer] as a RecognizersAllResponse
+        """
+        if not language:
+            languages = self.supported_languages
+        else:
+            languages = [language]
+
+        recognizers = []
+        for language in languages:
+            logger.info(f"Fetching all recognizers for language {language}")
+            recognizers.extend(
+                self.registry.get_recognizers(language=language, all_fields=True)
+            )
+
+        return list(set(recognizers))
+
+    def get_supported_entities(self, language: Optional[str] = None) -> List[str]:
+        """
+        Return a list of the entities that can be detected.
+
+        :param language: Return only entities supported in a specific language.
+        :return: List of entity names
+        """
+        recognizers = self.get_recognizers(language=language)
+        supported_entities = []
+        for recognizer in recognizers:
+            supported_entities.extend(recognizer.get_supported_entities())
+
+        return list(set(supported_entities))
+
+    def analyze(
+        self,
+        text: str,
+        language: str,
+        entities: Optional[List[str]] = None,
+        correlation_id: Optional[str] = None,
+        score_threshold: Optional[float] = None,
+        return_decision_process: Optional[bool] = False,
+        ad_hoc_recognizers: Optional[List[EntityRecognizer]] = None,
+        context: Optional[List[str]] = None,
+        allow_list: Optional[List[str]] = None,
+        nlp_artifacts: Optional[NlpArtifacts] = None,
+    ) -> List[RecognizerResult]:
+        """
+        Find PII entities in text using different PII recognizers for a given language.
+
+        :param text: the text to analyze
+        :param language: the language of the text
+        :param entities: List of PII entities that should be looked for in the text.
+        If entities=None then all entities are looked for.
+        :param correlation_id: cross call ID for this request
+        :param score_threshold: A minimum value for which
+        to return an identified entity
+        :param return_decision_process: Whether the analysis decision process steps
+        returned in the response.
+        :param ad_hoc_recognizers: List of recognizers which will be used only
+        for this specific request.
+        :param context: List of context words to enhance confidence score if matched
+        with the recognized entity's recognizer context
+        :param allow_list: List of words that the user defines as being allowed to keep
+        in the text
+        :param nlp_artifacts: precomputed NlpArtifacts
+        :return: an array of the found entities in the text
+
+        :example:
+
+        >>> from presidio_analyzer import AnalyzerEngine
+
+        >>> # Set up the engine, loads the NLP module (spaCy model by default)
+        >>> # and other PII recognizers
+        >>> analyzer = AnalyzerEngine()
+
+        >>> # Call analyzer to get results
+        >>> results = analyzer.analyze(text='My phone number is 212-555-5555', entities=['PHONE_NUMBER'], language='en') # noqa D501
+        >>> print(results)
+        [type: PHONE_NUMBER, start: 19, end: 31, score: 0.85]
+        """
+        all_fields = not entities
+
+        recognizers = self.registry.get_recognizers(
+            language=language,
+            entities=entities,
+            all_fields=all_fields,
+            ad_hoc_recognizers=ad_hoc_recognizers,
+        )
+
+        if all_fields:
+            # Since all_fields=True, list all entities by iterating
+            # over all recognizers
+            entities = self.get_supported_entities(language=language)
+
+        # run the nlp pipeline over the given text, store the results in
+        # a NlpArtifacts instance
+        if not nlp_artifacts:
+            nlp_artifacts = self.nlp_engine.process_text(text, language)
+
+        if self.log_decision_process:
+            self.app_tracer.trace(
+                correlation_id, "nlp artifacts:" + nlp_artifacts.to_json()
+            )
+
+        results = []
+        for recognizer in recognizers:
+            # Lazy loading of the relevant recognizers
+            if not recognizer.is_loaded:
+                recognizer.load()
+                recognizer.is_loaded = True
+
+            # analyze using the current recognizer and append the results
+            current_results = recognizer.analyze(
+                text=text, entities=entities, nlp_artifacts=nlp_artifacts
+            )
+
+            if current_results:
+                # add recognizer name to recognition metadata inside results
+                # if not exists
+                self.__add_recognizer_id_if_not_exists(current_results, recognizer)
+                results.extend(current_results)
+
+
+        results = self._enhance_using_context(
+            text, results, nlp_artifacts, recognizers, context
+        )
+
+        if self.log_decision_process:
+            self.app_tracer.trace(
+                correlation_id,
+                json.dumps([str(result.to_dict()) for result in results]),
+            )
+
+        # Remove duplicates or low score results
+        results = EntityRecognizer.remove_duplicates(results)
+        results = self.__remove_low_scores(results, score_threshold)
+
+        if allow_list:
+            results = self._remove_allow_list(results, allow_list, text)
+
+        if not return_decision_process:
+            results = self.__remove_decision_process(results)
+
+        return results
+
+    def _enhance_using_context(
+        self,
+        text: str,
+        raw_results: List[RecognizerResult],
+        nlp_artifacts: NlpArtifacts,
+        recognizers: List[EntityRecognizer],
+        context: Optional[List[str]] = None,
+    ) -> List[RecognizerResult]:
+        """
+        Enhance confidence score using context words.
+
+        :param text: The actual text that was analyzed
+        :param raw_results: Recognizer results which didn't take
+                            context into consideration
+        :param nlp_artifacts: The nlp artifacts contains elements
+                              such as lemmatized tokens for better
+                              accuracy of the context enhancement process
+        :param recognizers: the list of recognizers
+        :param context: list of context words
+        """
+        results = []
+
+        for recognizer in recognizers:
+            recognizer_results = [
+                r
+                for r in raw_results
+                if r.recognition_metadata[RecognizerResult.RECOGNIZER_IDENTIFIER_KEY]
+                == recognizer.id
+            ]
+            other_recognizer_results = [
+                r
+                for r in raw_results
+                if r.recognition_metadata[RecognizerResult.RECOGNIZER_IDENTIFIER_KEY]
+                != recognizer.id
+            ]
+
+            # enhance score using context in recognizer level if implemented
+            recognizer_results = recognizer.enhance_using_context(
+                text=text,
+                # each recognizer will get access to all recognizer results
+                # to allow related entities contex enhancement
+                raw_recognizer_results=recognizer_results,
+                other_raw_recognizer_results=other_recognizer_results,
+                nlp_artifacts=nlp_artifacts,
+                context=context,
+            )
+
+            results.extend(recognizer_results)
+
+        # Update results in case surrounding words or external context are relevant to
+        # the context words.
+        results = self.context_aware_enhancer.enhance_using_context(
+            text=text,
+            raw_results=results,
+            nlp_artifacts=nlp_artifacts,
+            recognizers=recognizers,
+            context=context,
+        )
+
+        return results
+
+    def __remove_low_scores(
+        self, results: List[RecognizerResult], score_threshold: float = None
+    ) -> List[RecognizerResult]:
+        """
+        Remove results for which the confidence is lower than the threshold.
+
+        :param results: List of RecognizerResult
+        :param score_threshold: float value for minimum possible confidence
+        :return: List[RecognizerResult]
+        """
+        if score_threshold is None:
+            score_threshold = self.default_score_threshold
+
+        new_results = [result for result in results if result.score >= score_threshold]
+        return new_results
+
+    @staticmethod
+    def _remove_allow_list(
+        results: List[RecognizerResult], allow_list: List[str], text: str
+    ) -> List[RecognizerResult]:
+        """
+        Remove results which are part of the allow list.
+
+        :param results: List of RecognizerResult
+        :param allow_list: list of allowed terms
+        :param text: the text to analyze
+        :return: List[RecognizerResult]
+        """
+        new_results = []
+        for result in results:
+            word = text[result.start : result.end]
+            # if the word is not specified to be allowed, keep in the PII entities
+            if word not in allow_list:
+                new_results.append(result)
+
+        return new_results
+
+    @staticmethod
+    def __add_recognizer_id_if_not_exists(
+        results: List[RecognizerResult], recognizer: EntityRecognizer
+    ):
+        """Ensure recognition metadata with recognizer id existence.
+
+        Ensure recognizer result list contains recognizer id inside recognition
+        metadata dictionary, and if not create it. recognizer_id is needed
+        for context aware enhancement.
+
+        :param results: List of RecognizerResult
+        :param recognizer: Entity recognizer
+        """
+        for result in results:
+            if not result.recognition_metadata:
+                result.recognition_metadata = dict()
+            if (
+                RecognizerResult.RECOGNIZER_IDENTIFIER_KEY
+                not in result.recognition_metadata
+            ):
+                result.recognition_metadata[
+                    RecognizerResult.RECOGNIZER_IDENTIFIER_KEY
+                ] = recognizer.id
+            if RecognizerResult.RECOGNIZER_NAME_KEY not in result.recognition_metadata:
+                result.recognition_metadata[
+                    RecognizerResult.RECOGNIZER_NAME_KEY
+                ] = recognizer.name
+
+    @staticmethod
+    def __remove_decision_process(
+        results: List[RecognizerResult],
+    ) -> List[RecognizerResult]:
+        """Remove decision process / analysis explanation from response."""
+
+        for result in results:
+            result.analysis_explanation = None
+
+        return results

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/analyzer_request.py

@@ -0,0 +1,36 @@
+from typing import Dict
+
+from presidio_analyzer import PatternRecognizer
+
+
+class AnalyzerRequest:
+    """
+    Analyzer request data.
+
+    :param req_data: A request dictionary with the following fields:
+        text: the text to analyze
+        language: the language of the text
+        entities: List of PII entities that should be looked for in the text.
+        If entities=None then all entities are looked for.
+        correlation_id: cross call ID for this request
+        score_threshold: A minimum value for which to return an identified entity
+        log_decision_process: Should the decision points within the analysis
+        be logged
+        return_decision_process: Should the decision points within the analysis
+        returned as part of the response
+    """
+
+    def __init__(self, req_data: Dict):
+        self.text = req_data.get("text")
+        self.language = req_data.get("language")
+        self.entities = req_data.get("entities")
+        self.correlation_id = req_data.get("correlation_id")
+        self.score_threshold = req_data.get("score_threshold")
+        self.return_decision_process = req_data.get("return_decision_process")
+        ad_hoc_recognizers = req_data.get("ad_hoc_recognizers")
+        self.ad_hoc_recognizers = []
+        if ad_hoc_recognizers:
+            self.ad_hoc_recognizers = [
+                PatternRecognizer.from_dict(rec) for rec in ad_hoc_recognizers
+            ]
+        self.context = req_data.get("context")

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/app_tracer.py

@@ -0,0 +1,27 @@
+import logging
+
+
+class AppTracer:
+    """
+    Allow logging/tracing the system's decisions.
+
+    Relevant in cases where we want to know which modules were used for detection,
+    which logic was utilized, what results were given and potentially why.
+    This can be useful for analyzing the detection accuracy of the system.
+    :param enabled: Whether tracing should be activated.
+    """
+
+    def __init__(self, enabled: bool = True):
+        self.logger = logging.getLogger("decision_process")
+        self.enabled = enabled
+
+    def trace(self, request_id: str, trace_data: str) -> None:
+        """
+        Write a value associated with a decision for a specific request into the trace.
+
+        Tracing for further inspection if needed.
+        :param request_id: A unique ID, to correlate across calls.
+        :param trace_data: A string to write to the log.
+        """
+        if self.enabled:
+            self.logger.info("[%s][%s]", request_id, trace_data)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/batch_analyzer_engine.py

@@ -0,0 +1,145 @@
+import logging
+from typing import List, Iterable, Dict, Union, Any, Optional, Iterator, Tuple
+
+from presidio_analyzer import DictAnalyzerResult, RecognizerResult, AnalyzerEngine
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class BatchAnalyzerEngine:
+    """
+    Batch analysis of documents (tables, lists, dicts).
+
+    Wrapper class to run Presidio Analyzer Engine on multiple values,
+    either lists/iterators of strings, or dictionaries.
+
+    :param: analyzer_engine: AnalyzerEngine instance to use
+    for handling the values in those collections.
+    """
+
+    def __init__(self, analyzer_engine: Optional[AnalyzerEngine] = None):
+
+        self.analyzer_engine = analyzer_engine
+        if not analyzer_engine:
+            self.analyzer_engine = AnalyzerEngine()
+
+    def analyze_iterator(
+        self,
+        texts: Iterable[Union[str, bool, float, int]],
+        language: str,
+        **kwargs,
+    ) -> List[List[RecognizerResult]]:
+        """
+        Analyze an iterable of strings.
+
+        :param texts: An list containing strings to be analyzed.
+        :param language: Input language
+        :param kwargs: Additional parameters for the `AnalyzerEngine.analyze` method.
+        """
+
+        # validate types
+        texts = self._validate_types(texts)
+
+        # Process the texts as batch for improved performance
+        nlp_artifacts_batch: Iterator[
+            Tuple[str, NlpArtifacts]
+        ] = self.analyzer_engine.nlp_engine.process_batch(
+            texts=texts, language=language
+        )
+
+        list_results = []
+        for text, nlp_artifacts in nlp_artifacts_batch:
+            results = self.analyzer_engine.analyze(
+                text=str(text), nlp_artifacts=nlp_artifacts, language=language, **kwargs
+            )
+
+            list_results.append(results)
+
+        return list_results
+
+    def analyze_dict(
+        self,
+        input_dict: Dict[str, Union[Any, Iterable[Any]]],
+        language: str,
+        keys_to_skip: Optional[List[str]] = None,
+        **kwargs,
+    ) -> Iterator[DictAnalyzerResult]:
+        """
+        Analyze a dictionary of keys (strings) and values/iterable of values.
+
+        Non-string values are returned as is.
+
+        :param input_dict: The input dictionary for analysis
+        :param language: Input language
+        :param keys_to_skip: Keys to ignore during analysis
+        :param kwargs: Additional keyword arguments
+        for the `AnalyzerEngine.analyze` method.
+        Use this to pass arguments to the analyze method,
+        such as `ad_hoc_recognizers`, `context`, `return_decision_process`.
+        See `AnalyzerEngine.analyze` for the full list.
+        """
+
+        context = []
+        if "context" in kwargs:
+            context = kwargs["context"]
+            del kwargs["context"]
+
+        if not keys_to_skip:
+            keys_to_skip = []
+
+        for key, value in input_dict.items():
+            if not value or key in keys_to_skip:
+                yield DictAnalyzerResult(key=key, value=value, recognizer_results=[])
+                continue  # skip this key as requested
+
+            # Add the key as an additional context
+            specific_context = context[:]
+            specific_context.append(key)
+
+            if type(value) in (str, int, bool, float):
+                results: List[RecognizerResult] = self.analyzer_engine.analyze(
+                    text=str(value), language=language, context=[key], **kwargs
+                )
+            elif isinstance(value, dict):
+                new_keys_to_skip = self._get_nested_keys_to_skip(key, keys_to_skip)
+                results = self.analyze_dict(
+                    input_dict=value,
+                    language=language,
+                    context=specific_context,
+                    keys_to_skip=new_keys_to_skip,
+                    **kwargs,
+                )
+            elif isinstance(value, Iterable):
+                # Recursively iterate nested dicts
+
+                results: List[List[RecognizerResult]] = self.analyze_iterator(
+                    texts=value,
+                    language=language,
+                    context=specific_context,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(f"type {type(value)} is unsupported.")
+
+            yield DictAnalyzerResult(key=key, value=value, recognizer_results=results)
+
+    @staticmethod
+    def _validate_types(value_iterator: Iterable[Any]) -> Iterator[Any]:
+        for val in value_iterator:
+            if val and not type(val) in (int, float, bool, str):
+                err_msg = (
+                    "Analyzer.analyze_iterator only works "
+                    "on primitive types (int, float, bool, str). "
+                    "Lists of objects are not yet supported."
+                )
+                logger.error(err_msg)
+                raise ValueError(err_msg)
+            yield val
+
+    @staticmethod
+    def _get_nested_keys_to_skip(key, keys_to_skip):
+        new_keys_to_skip = [
+            k.replace(f"{key}.", "") for k in keys_to_skip if k.startswith(key)
+        ]
+        return new_keys_to_skip

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/context_aware_enhancers/__init__.py

@@ -0,0 +1,5 @@
+"""Recognizer registry init."""
+from .context_aware_enhancer import ContextAwareEnhancer
+from .lemma_context_aware_enhancer import LemmaContextAwareEnhancer
+
+__all__ = ["ContextAwareEnhancer", "LemmaContextAwareEnhancer"]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/context_aware_enhancers/context_aware_enhancer.py

@@ -0,0 +1,68 @@
+import logging
+from abc import abstractmethod
+from typing import List, Optional
+
+from presidio_analyzer import RecognizerResult
+from presidio_analyzer import EntityRecognizer
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class ContextAwareEnhancer:
+    """
+    A class representing an abstract context aware enhancer.
+
+    Context words might enhance confidence score of a recognized entity,
+    ContextAwareEnhancer is an abstract class to be inherited by a context aware
+    enhancer logic.
+
+    :param context_similarity_factor: How much to enhance confidence of match entity
+    :param min_score_with_context_similarity: Minimum confidence score
+    :param context_prefix_count: how many words before the entity to match context
+    :param context_suffix_count: how many words after the entity to match context
+    """
+
+    MIN_SCORE = 0
+    MAX_SCORE = 1.0
+
+    def __init__(
+        self,
+        context_similarity_factor: float,
+        min_score_with_context_similarity: float,
+        context_prefix_count: int,
+        context_suffix_count: int,
+    ):
+
+        self.context_similarity_factor = context_similarity_factor
+        self.min_score_with_context_similarity = min_score_with_context_similarity
+        self.context_prefix_count = context_prefix_count
+        self.context_suffix_count = context_suffix_count
+
+    @abstractmethod
+    def enhance_using_context(
+        self,
+        text: str,
+        raw_results: List[RecognizerResult],
+        nlp_artifacts: NlpArtifacts,
+        recognizers: List[EntityRecognizer],
+        context: Optional[List[str]] = None,
+    ) -> List[RecognizerResult]:
+        """
+        Update results in case surrounding words are relevant to the context words.
+
+        Using the surrounding words of the actual word matches, look
+        for specific strings that if found contribute to the score
+        of the result, improving the confidence that the match is
+        indeed of that PII entity type
+
+        :param text: The actual text that was analyzed
+        :param raw_results: Recognizer results which didn't take
+                            context into consideration
+        :param nlp_artifacts: The nlp artifacts contains elements
+                              such as lemmatized tokens for better
+                              accuracy of the context enhancement process
+        :param recognizers: the list of recognizers
+        :param context: list of context words
+        """
+        return raw_results

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/context_aware_enhancers/lemma_context_aware_enhancer.py

@@ -0,0 +1,334 @@
+import copy
+import logging
+from typing import List, Optional
+
+from presidio_analyzer import RecognizerResult
+from presidio_analyzer import EntityRecognizer
+from presidio_analyzer.nlp_engine import NlpArtifacts
+from presidio_analyzer.context_aware_enhancers import ContextAwareEnhancer
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class LemmaContextAwareEnhancer(ContextAwareEnhancer):
+    """
+    A class representing a lemma based context aware enhancer logic.
+
+    Context words might enhance confidence score of a recognized entity,
+    LemmaContextAwareEnhancer is an implementation of Lemma based context aware logic,
+    it compares spacy lemmas of each word in context of the matched entity to given
+    context and the recognizer context words,
+    if matched it enhance the recognized entity confidence score by a given factor.
+
+    :param context_similarity_factor: How much to enhance confidence of match entity
+    :param min_score_with_context_similarity: Minimum confidence score
+    :param context_prefix_count: how many words before the entity to match context
+    :param context_suffix_count: how many words after the entity to match context
+    """
+
+    def __init__(
+        self,
+        context_similarity_factor: float = 0.35,
+        min_score_with_context_similarity: float = 0.4,
+        context_prefix_count: int = 5,
+        context_suffix_count: int = 0,
+    ):
+        super().__init__(
+            context_similarity_factor=context_similarity_factor,
+            min_score_with_context_similarity=min_score_with_context_similarity,
+            context_prefix_count=context_prefix_count,
+            context_suffix_count=context_suffix_count,
+        )
+
+    def enhance_using_context(
+        self,
+        text: str,
+        raw_results: List[RecognizerResult],
+        nlp_artifacts: NlpArtifacts,
+        recognizers: List[EntityRecognizer],
+        context: Optional[List[str]] = None,
+    ) -> List[RecognizerResult]:
+        """
+        Update results in case the lemmas of surrounding words or input context
+        words are identical to the context words.
+
+        Using the surrounding words of the actual word matches, look
+        for specific strings that if found contribute to the score
+        of the result, improving the confidence that the match is
+        indeed of that PII entity type
+
+        :param text: The actual text that was analyzed
+        :param raw_results: Recognizer results which didn't take
+                            context into consideration
+        :param nlp_artifacts: The nlp artifacts contains elements
+                              such as lemmatized tokens for better
+                              accuracy of the context enhancement process
+        :param recognizers: the list of recognizers
+        :param context: list of context words
+        """  # noqa D205 D400
+
+        # create a deep copy of the results object, so we can manipulate it
+        results = copy.deepcopy(raw_results)
+
+        # create recognizer context dictionary
+        recognizers_dict = {recognizer.id: recognizer for recognizer in recognizers}
+
+        # Create empty list in None or lowercase all context words in the list
+        if not context:
+            context = []
+        else:
+            context = [word.lower() for word in context]
+
+        # Sanity
+        if nlp_artifacts is None:
+            logger.warning("NLP artifacts were not provided")
+            return results
+
+        for result in results:
+            recognizer = None
+            # get recognizer matching the result, if found.
+            if (
+                result.recognition_metadata
+                and RecognizerResult.RECOGNIZER_IDENTIFIER_KEY
+                in result.recognition_metadata.keys()
+            ):
+                recognizer = recognizers_dict.get(
+                    result.recognition_metadata[
+                        RecognizerResult.RECOGNIZER_IDENTIFIER_KEY
+                    ]
+                )
+
+            if not recognizer:
+                logger.debug(
+                    "Recognizer name not found as part of the "
+                    "recognition_metadata dict in the RecognizerResult. "
+                )
+                continue
+
+            # skip recognizer result if the recognizer doesn't support
+            # context enhancement
+            if not recognizer.context:
+                logger.debug(
+                    "recognizer '%s' does not support context enhancement",
+                    recognizer.name,
+                )
+                continue
+
+            # skip context enhancement if already boosted by recognizer level
+            if result.recognition_metadata.get(
+                RecognizerResult.IS_SCORE_ENHANCED_BY_CONTEXT_KEY
+            ):
+                logger.debug("result score already boosted, skipping")
+                continue
+
+            # extract lemmatized context from the surrounding of the match
+            word = text[result.start : result.end]
+
+            surrounding_words = self._extract_surrounding_words(
+                nlp_artifacts=nlp_artifacts, word=word, start=result.start
+            )
+
+            # combine other sources of context with surrounding words
+            surrounding_words.extend(context)
+
+            supportive_context_word = self._find_supportive_word_in_context(
+                surrounding_words, recognizer.context
+            )
+            if supportive_context_word != "":
+                result.score += self.context_similarity_factor
+                result.score = max(result.score, self.min_score_with_context_similarity)
+                result.score = min(result.score, ContextAwareEnhancer.MAX_SCORE)
+
+                # Update the explainability object with context information
+                # helped to improve the score
+                result.analysis_explanation.set_supportive_context_word(
+                    supportive_context_word
+                )
+                result.analysis_explanation.set_improved_score(result.score)
+        return results
+
+    @staticmethod
+    def _find_supportive_word_in_context(
+        context_list: List[str], recognizer_context_list: List[str]
+    ) -> str:
+        """
+        Find words in the text which are relevant for context evaluation.
+
+        A word is considered a supportive context word if there's exact match
+        between a keyword in context_text and any keyword in context_list.
+
+        :param context_list words before and after the matched entity within
+               a specified window size
+        :param recognizer_context_list a list of words considered as
+                context keywords manually specified by the recognizer's author
+        """
+        word = ""
+        # If the context list is empty, no need to continue
+        if context_list is None or recognizer_context_list is None:
+            return word
+
+        for predefined_context_word in recognizer_context_list:
+            # result == true only if any of the predefined context words
+            # is found exactly or as a substring in any of the collected
+            # context words
+            result = next(
+                (
+                    True
+                    for keyword in context_list
+                    if predefined_context_word in keyword
+                ),
+                False,
+            )
+            if result:
+                logger.debug("Found context keyword '%s'", predefined_context_word)
+                word = predefined_context_word
+                break
+
+        return word
+
+    def _extract_surrounding_words(
+        self, nlp_artifacts: NlpArtifacts, word: str, start: int
+    ) -> List[str]:
+        """Extract words surrounding another given word.
+
+        The text from which the context is extracted is given in the nlp
+        doc.
+
+        :param nlp_artifacts: An abstraction layer which holds different
+                              items which are the result of a NLP pipeline
+                              execution on a given text
+        :param word: The word to look for context around
+        :param start: The start index of the word in the original text
+        """
+        if not nlp_artifacts.tokens:
+            logger.info("Skipping context extraction due to lack of NLP artifacts")
+            # if there are no nlp artifacts, this is ok, we can
+            # extract context and we return a valid, yet empty
+            # context
+            return [""]
+
+        # Get the already prepared words in the given text, in their
+        # LEMMATIZED version
+        lemmatized_keywords = nlp_artifacts.keywords
+
+        # since the list of tokens is not necessarily aligned
+        # with the actual index of the match, we look for the
+        # token index which corresponds to the match
+        token_index = self._find_index_of_match_token(
+            word, start, nlp_artifacts.tokens, nlp_artifacts.tokens_indices
+        )
+
+        # index i belongs to the PII entity, take the preceding n words
+        # and the successing m words into a context list
+
+        backward_context = self._add_n_words_backward(
+            token_index,
+            self.context_prefix_count,
+            nlp_artifacts.lemmas,
+            lemmatized_keywords,
+        )
+        forward_context = self._add_n_words_forward(
+            token_index,
+            self.context_suffix_count,
+            nlp_artifacts.lemmas,
+            lemmatized_keywords,
+        )
+
+        context_list = []
+        context_list.extend(backward_context)
+        context_list.extend(forward_context)
+        context_list = list(set(context_list))
+        logger.debug("Context list is: %s", " ".join(context_list))
+        return context_list
+
+    @staticmethod
+    def _find_index_of_match_token(
+        word: str, start: int, tokens, tokens_indices: List[int]  # noqa ANN001
+    ) -> int:
+        found = False
+        # we use the known start index of the original word to find the actual
+        # token at that index, we are not checking for equivilance since the
+        # token might be just a substring of that word (e.g. for phone number
+        # 555-124564 the first token might be just '555' or for a match like '
+        # rocket' the actual token will just be 'rocket' hence the misalignment
+        # of indices)
+        # Note: we are iterating over the original tokens (not the lemmatized)
+        i = -1
+        for i, token in enumerate(tokens, 0):
+            # Either we found a token with the exact location, or
+            # we take a token which its characters indices covers
+            # the index we are looking for.
+            if (tokens_indices[i] == start) or (start < tokens_indices[i] + len(token)):
+                # found the interesting token, the one that around it
+                # we take n words, we save the matching lemma
+                found = True
+                break
+
+        if not found:
+            raise ValueError(
+                "Did not find word '" + word + "' "
+                "in the list of tokens although it "
+                "is expected to be found"
+            )
+        return i
+
+    @staticmethod
+    def _add_n_words(
+        index: int,
+        n_words: int,
+        lemmas: List[str],
+        lemmatized_filtered_keywords: List[str],
+        is_backward: bool,
+    ) -> List[str]:
+        """
+        Prepare a string of context words.
+
+        Return a list of words which surrounds a lemma at a given index.
+        The words will be collected only if exist in the filtered array
+
+        :param index: index of the lemma that its surrounding words we want
+        :param n_words: number of words to take
+        :param lemmas: array of lemmas
+        :param lemmatized_filtered_keywords: the array of filtered
+               lemmas from the original sentence,
+        :param is_backward: if true take the preceeding words, if false,
+                            take the successing words
+        """
+        i = index
+        context_words = []
+        # The entity itself is no interest to us...however we want to
+        # consider it anyway for cases were it is attached with no spaces
+        # to an interesting context word, so we allow it and add 1 to
+        # the number of collected words
+
+        # collect at most n words (in lower case)
+        remaining = n_words + 1
+        while 0 <= i < len(lemmas) and remaining > 0:
+            lower_lemma = lemmas[i].lower()
+            if lower_lemma in lemmatized_filtered_keywords:
+                context_words.append(lower_lemma)
+                remaining -= 1
+            i = i - 1 if is_backward else i + 1
+        return context_words
+
+    def _add_n_words_forward(
+        self,
+        index: int,
+        n_words: int,
+        lemmas: List[str],
+        lemmatized_filtered_keywords: List[str],
+    ) -> List[str]:
+        return self._add_n_words(
+            index, n_words, lemmas, lemmatized_filtered_keywords, False
+        )
+
+    def _add_n_words_backward(
+        self,
+        index: int,
+        n_words: int,
+        lemmas: List[str],
+        lemmatized_filtered_keywords: List[str],
+    ) -> List[str]:
+        return self._add_n_words(
+            index, n_words, lemmas, lemmatized_filtered_keywords, True
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/dict_analyzer_result.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+from typing import List, Union, Iterator
+
+from presidio_analyzer import RecognizerResult
+
+
+@dataclass
+class DictAnalyzerResult:
+    """
+    Data class for holding the output of the Presidio Analyzer on dictionaries.
+
+    :param key: key in dictionary
+    :param value: value to run analysis on (either string or list of strings)
+    :param recognizer_results: Analyzer output for one value.
+    Could be either:
+     - A list of recognizer results if the input is one string
+     - A list of lists of recognizer results, if the input is a list of strings.
+     - An iterator of a DictAnalyzerResult, if the input is a dictionary.
+     In this case the recognizer_results would be the iterator
+     of the DictAnalyzerResults next level in the dictionary.
+    """
+
+    key: str
+    value: Union[str, List[str], dict]
+    recognizer_results: Union[
+        List[RecognizerResult],
+        List[List[RecognizerResult]],
+        Iterator["DictAnalyzerResult"],
+    ]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/entity_recognizer.py

@@ -0,0 +1,199 @@
+import logging
+from abc import abstractmethod
+from typing import List, Dict, Optional
+
+from presidio_analyzer import RecognizerResult
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class EntityRecognizer:
+    """
+    A class representing an abstract PII entity recognizer.
+
+    EntityRecognizer is an abstract class to be inherited by
+    Recognizers which hold the logic for recognizing specific PII entities.
+
+    EntityRecognizer exposes a method called enhance_using_context which
+    can be overridden in case a custom context aware enhancement is needed
+    in derived class of a recognizer.
+
+    :param supported_entities: the entities supported by this recognizer
+    (for example, phone number, address, etc.)
+    :param supported_language: the language supported by this recognizer.
+    The supported langauge code is iso6391Name
+    :param name: the name of this recognizer (optional)
+    :param version: the recognizer current version
+    :param context: a list of words which can help boost confidence score
+    when they appear in context of the matched entity
+    """
+
+    MIN_SCORE = 0
+    MAX_SCORE = 1.0
+
+    def __init__(
+        self,
+        supported_entities: List[str],
+        name: str = None,
+        supported_language: str = "en",
+        version: str = "0.0.1",
+        context: Optional[List[str]] = None,
+    ):
+
+        self.supported_entities = supported_entities
+
+        if name is None:
+            self.name = self.__class__.__name__  # assign class name as name
+        else:
+            self.name = name
+
+        self._id = f"{self.name}_{id(self)}"
+
+        self.supported_language = supported_language
+        self.version = version
+        self.is_loaded = False
+        self.context = context if context else []
+
+        self.load()
+        logger.info("Loaded recognizer: %s", self.name)
+        self.is_loaded = True
+
+    @property
+    def id(self):
+        """Return a unique identifier of this recognizer."""
+
+        return self._id
+
+    @abstractmethod
+    def load(self) -> None:
+        """
+        Initialize the recognizer assets if needed.
+
+        (e.g. machine learning models)
+        """
+
+    @abstractmethod
+    def analyze(
+        self, text: str, entities: List[str], nlp_artifacts: NlpArtifacts
+    ) -> List[RecognizerResult]:
+        """
+        Analyze text to identify entities.
+
+        :param text: The text to be analyzed
+        :param entities: The list of entities this recognizer is able to detect
+        :param nlp_artifacts: A group of attributes which are the result of
+        an NLP process over the input text.
+        :return: List of results detected by this recognizer.
+        """
+        return None
+
+    def enhance_using_context(
+        self,
+        text: str,
+        raw_recognizer_results: List[RecognizerResult],
+        other_raw_recognizer_results: List[RecognizerResult],
+        nlp_artifacts: NlpArtifacts,
+        context: Optional[List[str]] = None,
+    ) -> List[RecognizerResult]:
+        """Enhance confidence score using context of the entity.
+
+        Override this method in derived class in case a custom logic
+        is needed, otherwise return value will be equal to
+        raw_results.
+
+        in case a result score is boosted, derived class need to update
+        result.recognition_metadata[RecognizerResult.IS_SCORE_ENHANCED_BY_CONTEXT_KEY]
+
+        :param text: The actual text that was analyzed
+        :param raw_recognizer_results: This recognizer's results, to be updated
+        based on recognizer specific context.
+        :param other_raw_recognizer_results: Other recognizer results matched in
+        the given text to allow related entity context enhancement
+        :param nlp_artifacts: The nlp artifacts contains elements
+                              such as lemmatized tokens for better
+                              accuracy of the context enhancement process
+        :param context: list of context words
+        """
+        return raw_recognizer_results
+
+    def get_supported_entities(self) -> List[str]:
+        """
+        Return the list of entities this recognizer can identify.
+
+        :return: A list of the supported entities by this recognizer
+        """
+        return self.supported_entities
+
+    def get_supported_language(self) -> str:
+        """
+        Return the language this recognizer can support.
+
+        :return: A list of the supported language by this recognizer
+        """
+        return self.supported_language
+
+    def get_version(self) -> str:
+        """
+        Return the version of this recognizer.
+
+        :return: The current version of this recognizer
+        """
+        return self.version
+
+    def to_dict(self) -> Dict:
+        """
+        Serialize self to dictionary.
+
+        :return: a dictionary
+        """
+        return_dict = {
+            "supported_entities": self.supported_entities,
+            "supported_language": self.supported_language,
+            "name": self.name,
+            "version": self.version,
+        }
+        return return_dict
+
+    @classmethod
+    def from_dict(cls, entity_recognizer_dict: Dict) -> "EntityRecognizer":
+        """
+        Create EntityRecognizer from a dict input.
+
+        :param entity_recognizer_dict: Dict containing keys and values for instantiation
+        """
+        return cls(**entity_recognizer_dict)
+
+    @staticmethod
+    def remove_duplicates(results: List[RecognizerResult]) -> List[RecognizerResult]:
+        """
+        Remove duplicate results.
+
+        Remove duplicates in case the two results
+        have identical start and ends and types.
+        :param results: List[RecognizerResult]
+        :return: List[RecognizerResult]
+        """
+        results = list(set(results))
+        results = sorted(results, key=lambda x: (-x.score, x.start, -(x.end - x.start)))
+        filtered_results = []
+
+        for result in results:
+            if result.score == 0:
+                continue
+
+            to_keep = result not in filtered_results  # equals based comparison
+            if to_keep:
+                for filtered in filtered_results:
+                    # If result is contained in one of the other results
+                    if (
+                        result.contained_in(filtered)
+                        and result.entity_type == filtered.entity_type
+                    ):
+                        to_keep = False
+                        break
+
+            if to_keep:
+                filtered_results.append(result)
+
+        return filtered_results

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/local_recognizer.py

@@ -0,0 +1,7 @@
+from abc import ABC
+
+from presidio_analyzer import EntityRecognizer
+
+
+class LocalRecognizer(ABC, EntityRecognizer):
+    """PII entity recognizer which runs on the same process as the AnalyzerEngine."""

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/__init__.py

@@ -0,0 +1,19 @@
+"""NLP engine package. Performs text pre-processing."""
+
+from .nlp_artifacts import NlpArtifacts
+from .nlp_engine import NlpEngine
+from .spacy_nlp_engine import SpacyNlpEngine
+from .client_nlp_engine import ClientNlpEngine
+from .stanza_nlp_engine import StanzaNlpEngine
+from .transformers_nlp_engine import TransformersNlpEngine
+from .nlp_engine_provider import NlpEngineProvider
+
+__all__ = [
+    "NlpArtifacts",
+    "NlpEngine",
+    "SpacyNlpEngine",
+    "ClientNlpEngine",
+    "StanzaNlpEngine",
+    "NlpEngineProvider",
+    "TransformersNlpEngine",
+]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/client_nlp_engine.py

@@ -0,0 +1,108 @@
+import logging
+# import logging
+
+try:
+    import client
+    import spacy_client
+except ImportError:
+    client = None
+
+from typing import Optional, Dict, Iterator, Tuple, Union, List
+
+import spacy
+from spacy.language import Language
+from spacy.tokens import Doc
+
+from presidio_analyzer.nlp_engine import NlpArtifacts, NlpEngine
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class ClientNlpEngine(NlpEngine):
+    """
+    SpacyNlpEngine is an abstraction layer over the nlp module.
+
+    It provides processing functionality as well as other queries
+    on tokens.
+    The SpacyNlpEngine uses SpaCy as its NLP module
+    """
+
+    
+    engine_name="spacy"
+    
+    is_available = bool(spacy)
+   
+
+    def __init__(self, models: Optional[Dict[str, str]] = None):
+        """
+        Initialize a wrapper on spaCy functionality.
+
+        :param models: Dictionary with the name of the spaCy model per language.
+        For example: models = {"en": "en_core_web_lg"}
+        """
+        if not models:
+            models = {"en": "en_core_web_lg"}
+        logger.debug(f"Loading SpaCy models: {models.values()}")
+
+        self.nlp = {
+            lang_code: spacy.load(model_name, disable=["parser"])
+            for lang_code, model_name in models.items()
+        }
+
+        
+
+    def process_text(self, text: str, language: str) -> NlpArtifacts:
+        """Execute the SpaCy NLP pipeline on the given text and language."""
+
+        doc = self.nlp[language](text)
+        return self._doc_to_nlp_artifact(doc, language)
+
+    def process_batch(
+        self,
+        texts: Union[List[str], List[Tuple[str, object]]],
+        language: str,
+        as_tuples: bool = False,
+    ) -> Iterator[Optional[NlpArtifacts]]:
+        """Execute the NLP pipeline on a batch of texts using spacy pipe."""
+        texts = (str(text) for text in texts)
+        docs = self.nlp[language].pipe(texts, as_tuples=as_tuples)
+        for doc in docs:
+            yield doc.text, self._doc_to_nlp_artifact(doc, language)
+
+    def is_stopword(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a stop word.
+
+        (within the given language)
+        """
+        return self.nlp[language].vocab[word].is_stop
+
+    def is_punct(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a punctuation word.
+
+        (within the given language).
+        """
+        return self.nlp[language].vocab[word].is_punct
+
+    def get_nlp(self, language: str) -> Language:
+        """
+        Return the language model loaded for a language.
+
+        :param language: Name of language
+        :return: Language model from spaCy
+        """
+        return self.nlp[language]
+
+    def _doc_to_nlp_artifact(self, doc: Doc, language: str) -> NlpArtifacts:
+        lemmas = [token.lemma_ for token in doc]
+        tokens_indices = [token.idx for token in doc]
+        entities = doc.ents
+        return NlpArtifacts(
+            entities=entities,
+            tokens=doc,
+            tokens_indices=tokens_indices,
+            lemmas=lemmas,
+            nlp_engine=self,
+            language=language,
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/nlp_artifacts.py

@@ -0,0 +1,74 @@
+import json
+from typing import List
+
+from spacy.tokens import Doc, Span
+
+
+class NlpArtifacts:
+    """
+    NlpArtifacts is an abstraction layer over the results of an NLP pipeline.
+
+    processing over a given text, it holds attributes such as entities,
+    tokens and lemmas which can be used by any recognizer
+    """
+
+    def __init__(
+        self,
+        entities: List[Span],
+        tokens: Doc,
+        tokens_indices: List[int],
+        lemmas: List[str],
+        nlp_engine,  # noqa ANN001
+        language: str,
+    ):
+        self.entities = entities
+        self.tokens = tokens
+        self.lemmas = lemmas
+        self.tokens_indices = tokens_indices
+        self.keywords = self.set_keywords(nlp_engine, lemmas, language)
+        self.nlp_engine = nlp_engine
+
+    @staticmethod
+    def set_keywords(
+        nlp_engine, lemmas: List[str], language: str  # noqa ANN001
+    ) -> List[str]:
+        """
+        Return keywords fpr text.
+
+        Extracts lemmas with certain conditions as keywords.
+        """
+        if not nlp_engine:
+            return []
+        keywords = [
+            k.lower()
+            for k in lemmas
+            if not nlp_engine.is_stopword(k, language)
+            and not nlp_engine.is_punct(k, language)
+            and k != "-PRON-"
+            and k != "be"
+        ]
+
+        # best effort, try even further to break tokens into sub tokens,
+        # this can result in reducing false negatives
+        keywords = [i.split(":") for i in keywords]
+
+        # splitting the list can, if happened, will result in list of lists,
+        # we flatten the list
+        keywords = [item for sublist in keywords for item in sublist]
+        return keywords
+
+    def to_json(self) -> str:
+        """Convert nlp artifacts to json."""
+
+        return_dict = self.__dict__.copy()
+
+        # Ignore NLP engine as it's not serializable currently
+        del return_dict["nlp_engine"]
+
+        # Converting spaCy tokens and spans to string as they are not serializable
+        if "tokens" in return_dict:
+            return_dict["tokens"] = [token.text for token in self.tokens]
+        if "entities" in return_dict:
+            return_dict["entities"] = [entity.text for entity in self.entities]
+
+        return json.dumps(return_dict)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/nlp_engine.py

@@ -0,0 +1,42 @@
+from abc import ABC, abstractmethod
+from typing import Iterable, Iterator, Tuple
+
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+
+class NlpEngine(ABC):
+    """
+    NlpEngine is an abstraction layer over the nlp module.
+
+    It provides NLP preprocessing functionality as well as other queries
+    on tokens.
+    """
+
+    @abstractmethod
+    def process_text(self, text: str, language: str) -> NlpArtifacts:
+        """Execute the NLP pipeline on the given text and language."""
+
+    @abstractmethod
+    def process_batch(
+        self, texts: Iterable[str], language: str, **kwargs
+    ) -> Iterator[Tuple[str, NlpArtifacts]]:
+        """Execute the NLP pipeline on a batch of texts.
+
+        Returns a tuple of (text, NlpArtifacts)
+        """
+
+    @abstractmethod
+    def is_stopword(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a stop word.
+
+        (within the given language)
+        """
+
+    @abstractmethod
+    def is_punct(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a punctuation word.
+
+        (within the given language)
+        """

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/nlp_engine_provider.py

@@ -0,0 +1,128 @@
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Union, Tuple
+
+import yaml
+
+from presidio_analyzer.nlp_engine import (
+    StanzaNlpEngine,
+    SpacyNlpEngine,
+    NlpEngine,
+    ClientNlpEngine,
+    TransformersNlpEngine,
+)
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class NlpEngineProvider:
+    """Create different NLP engines from configuration.
+
+    :param nlp_engines: List of available NLP engines.
+    Default: (SpacyNlpEngine, StanzaNlpEngine)
+    :param nlp_configuration: Dict containing nlp configuration
+    :example: configuration:
+            {
+                "nlp_engine_name": "spacy",
+                "models": [{"lang_code": "en",
+                            "model_name": "en_core_web_lg"
+                          }]
+            }
+    Nlp engine names available by default: spacy, stanza.
+    :param conf_file: Path to yaml file containing nlp engine configuration.
+    """
+
+    def __init__(
+        self,
+        nlp_engines: Optional[Tuple] = None,
+        conf_file: Optional[Union[Path, str]] = None,
+        nlp_configuration: Optional[Dict] = None,
+    ):
+        
+        if not nlp_engines:
+            nlp_engines = (SpacyNlpEngine, StanzaNlpEngine, TransformersNlpEngine,ClientNlpEngine)
+        
+        
+
+        self.nlp_engines = {
+            engine.engine_name: engine for engine in nlp_engines if engine.is_available
+        }
+        
+        logger.debug(
+            f"Loaded these available nlp engines: {list(self.nlp_engines.keys())}"
+        )
+
+        if conf_file and nlp_configuration:
+            raise ValueError(
+                "Either conf_file or nlp_configuration should be provided, not both."
+            )
+
+        if nlp_configuration:
+            self.nlp_configuration = nlp_configuration
+
+        if conf_file:
+            self.nlp_configuration = self._read_nlp_conf(conf_file)
+
+        if not conf_file and not nlp_configuration:
+            conf_file = self._get_full_conf_path()
+            logger.debug(f"Reading default conf file from {conf_file}")
+            self.nlp_configuration = self._read_nlp_conf(conf_file)
+
+    def create_engine(self) -> NlpEngine:
+        """Create an NLP engine instance."""
+        if (
+            not self.nlp_configuration
+            or not self.nlp_configuration.get("models")
+            or not self.nlp_configuration.get("nlp_engine_name")
+        ):
+            raise ValueError(
+                "Illegal nlp configuration. "
+                "Configuration should include nlp_engine_name and models "
+                "(list of model_name for each lang_code)."
+            )
+        nlp_engine_name = self.nlp_configuration["nlp_engine_name"]
+        if nlp_engine_name not in self.nlp_engines:
+            raise ValueError(
+                f"NLP engine '{nlp_engine_name}' is not available. "
+                "Make sure you have all required packages installed"
+            )
+        try:
+            nlp_engine_class = self.nlp_engines[nlp_engine_name]
+            nlp_engine_opts = {
+                m["lang_code"]: m["model_name"]
+                for m in self.nlp_configuration["models"]
+            }
+            engine = nlp_engine_class(nlp_engine_opts)
+            logger.info(
+                f"Created NLP engine: {engine.engine_name}. "
+                f"Loaded models: {list(engine.nlp.keys())}"
+            )
+            return engine
+        except KeyError:
+            raise ValueError("Wrong NLP engine configuration")
+
+    @staticmethod
+    def _read_nlp_conf(conf_file: Union[Path, str]) -> dict:
+        """Read the nlp configuration from a provided yaml file."""
+
+        if not Path(conf_file).exists():
+            nlp_configuration = {
+                "nlp_engine_name": "spacy",
+                "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}],
+            }
+            logger.warning(
+                f"configuration file {conf_file} not found.  "
+                f"Using default config: {nlp_configuration}."
+            )
+
+        else:
+            nlp_configuration = yaml.safe_load(open(conf_file))
+
+        return nlp_configuration
+
+    @staticmethod
+    def _get_full_conf_path(
+        default_conf_file: Union[Path, str] = "default.yaml"
+    ) -> Path:
+        """Return a Path to the default conf file."""
+        return Path(Path(__file__).parent.parent.parent, "conf", default_conf_file)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/spacy_nlp_engine.py

@@ -0,0 +1,96 @@
+import logging
+from typing import Optional, Dict, Iterator, Tuple, Union, List
+
+import spacy
+from spacy.language import Language
+from spacy.tokens import Doc
+
+from presidio_analyzer.nlp_engine import NlpArtifacts, NlpEngine
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class SpacyNlpEngine(NlpEngine):
+    """
+    SpacyNlpEngine is an abstraction layer over the nlp module.
+
+    It provides processing functionality as well as other queries
+    on tokens.
+    The SpacyNlpEngine uses SpaCy as its NLP module
+    """
+
+    engine_name = "spacy"
+    is_available = bool(spacy)
+   
+
+    def __init__(self, models: Optional[Dict[str, str]] = None):
+        """
+        Initialize a wrapper on spaCy functionality.
+
+        :param models: Dictionary with the name of the spaCy model per language.
+        For example: models = {"en": "en_core_web_lg"}
+        """
+        if not models:
+            models = {"en": "en_core_web_lg"}
+        logger.debug(f"Loading SpaCy models: {models.values()}")
+
+        self.nlp = {
+            lang_code: spacy.load(model_name, disable=["parser"])
+            for lang_code, model_name in models.items()
+        }
+
+    def process_text(self, text: str, language: str) -> NlpArtifacts:
+        """Execute the SpaCy NLP pipeline on the given text and language."""
+
+        doc = self.nlp[language](text)
+        return self._doc_to_nlp_artifact(doc, language)
+
+    def process_batch(
+        self,
+        texts: Union[List[str], List[Tuple[str, object]]],
+        language: str,
+        as_tuples: bool = False,
+    ) -> Iterator[Optional[NlpArtifacts]]:
+        """Execute the NLP pipeline on a batch of texts using spacy pipe."""
+        texts = (str(text) for text in texts)
+        docs = self.nlp[language].pipe(texts, as_tuples=as_tuples)
+        for doc in docs:
+            yield doc.text, self._doc_to_nlp_artifact(doc, language)
+
+    def is_stopword(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a stop word.
+
+        (within the given language)
+        """
+        return self.nlp[language].vocab[word].is_stop
+
+    def is_punct(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a punctuation word.
+
+        (within the given language).
+        """
+        return self.nlp[language].vocab[word].is_punct
+
+    def get_nlp(self, language: str) -> Language:
+        """
+        Return the language model loaded for a language.
+
+        :param language: Name of language
+        :return: Language model from spaCy
+        """
+        return self.nlp[language]
+
+    def _doc_to_nlp_artifact(self, doc: Doc, language: str) -> NlpArtifacts:
+        lemmas = [token.lemma_ for token in doc]
+        tokens_indices = [token.idx for token in doc]
+        entities = doc.ents
+        return NlpArtifacts(
+            entities=entities,
+            tokens=doc,
+            tokens_indices=tokens_indices,
+            lemmas=lemmas,
+            nlp_engine=self,
+            language=language,
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/stanza_nlp_engine.py

@@ -0,0 +1,39 @@
+import logging
+
+try:
+    import stanza
+    import spacy_stanza
+except ImportError:
+    stanza = None
+
+from presidio_analyzer.nlp_engine import SpacyNlpEngine
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class StanzaNlpEngine(SpacyNlpEngine):
+    """
+    StanzaNlpEngine is an abstraction layer over the nlp module.
+
+    It provides processing functionality as well as other queries
+    on tokens.
+    The StanzaNlpEngine uses spacy-stanza and stanza as its NLP module
+
+    :param models: Dictionary with the name of the stanza model per language.
+    For example: models = {"en": "en"}
+    """
+
+    engine_name = "stanza"
+    is_available = bool(stanza)
+    def __init__(self, models=None):  # noqa ANN201
+        if not models:
+            models = {"en": "en"}
+        logger.debug(f"Loading Stanza models: {models.values()}")
+
+        self.nlp = {
+            lang_code: spacy_stanza.load_pipeline(
+                model_name,
+                processors="tokenize,pos,lemma,ner",
+            )
+            for lang_code, model_name in models.items()
+        }

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py

@@ -0,0 +1,155 @@
+import logging
+from typing import Optional, Dict
+
+import spacy
+from spacy.language import Language
+from spacy.tokens import Doc, Span
+
+from presidio_analyzer.nlp_engine import SpacyNlpEngine
+
+
+try:
+    import torch
+    import transformers
+    from transformers import (
+        AutoTokenizer,
+        AutoModelForTokenClassification,
+        pipeline,
+    )
+except ImportError:
+    torch = None
+    transformers = None
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+@Language.factory(
+    "transformers",
+    default_config={"pretrained_model_name_or_path": "dslim/bert-base-NER"},
+)
+def create_transformer_component(nlp, name, pretrained_model_name_or_path: str):
+    """Spacy Language factory for creating custom component."""
+    return TransformersComponent(
+        pretrained_model_name_or_path=pretrained_model_name_or_path
+    )
+
+
+class TransformersComponent:
+    """
+    Custom component to use in spacy pipeline.
+
+    Using HaggingFace transformers pretrained models for entity recognition.
+    :param pretrained_model_name_or_path: HaggingFace pretrained_model_name_or_path
+    """
+
+    def __init__(self, pretrained_model_name_or_path: str) -> None:
+        Span.set_extension("confidence_score", default=1.0, force=True)
+        tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path)
+        model = AutoModelForTokenClassification.from_pretrained(
+            pretrained_model_name_or_path
+        )
+        self.nlp = pipeline(
+            "ner", model=model, tokenizer=tokenizer, aggregation_strategy="simple"
+        )
+
+    def __call__(self, doc: Doc) -> Doc:
+        """Write transformers results to doc entities."""
+
+        res = self.nlp(doc.text)
+        ents = []
+        for d in res:
+            span = doc.char_span(
+                d["start"], d["end"], label=d["entity_group"], alignment_mode="expand"
+            )
+            if span is not None:
+                span._.confidence_score = d["score"]
+                ents.append(span)
+            else:
+                logger.warning(
+                    f"Transformers model returned {d} but no valid span was found."
+                )
+        doc.ents = ents
+        return doc
+
+
+class TransformersNlpEngine(SpacyNlpEngine):
+    """
+
+    SpacyTransformersNlpEngine is a transformers based NlpEngine.
+
+    It comprises a spacy pipeline used for tokenization,
+    lemmatization, pos, and a transformers component for NER.
+
+    Both the underlying spacy pipeline and the transformers engine could be
+    configured by the user.
+
+    :param models: a dictionary containing the model names per language.
+    :example:
+    {
+        "en": {
+            "spacy": "en_core_web_sm",
+            "transformers": "dslim/bert-base-NER"
+        }
+    }
+
+    Note that since the spaCy model is not used for NER,
+    we recommend using a simple model, such as en_core_web_sm for English.
+    For potential Transformers models, see a list of models here:
+    https://huggingface.co/models?pipeline_tag=token-classification
+    It is further recommended to fine-tune these models
+    to the specific scenario in hand.
+    """
+
+    engine_name = "transformers"
+    is_available = bool(spacy) and bool(transformers)
+   
+
+    def __init__(self, models: Optional[Dict[str, Dict[str, str]]] = None):
+        # default models if not specified
+        if not models:
+            models = {
+                "en": {"spacy": "en_core_web_sm", "transformers": "dslim/bert-base-NER"}
+            }
+        # validate models type
+        elif type(models) is not dict:
+            logger.error(f"''models' argument must be dict, not {type(models)}")
+            raise KeyError(f"Expected 'models' argument to be dict, not {type(models)}")
+        # validate models[model_lang] type is dict for all model_lang
+        elif any(
+            [type(model_dict) is not dict for model_lang, model_dict in models.items()]
+        ):
+            # elif type(models["model_name"]) is not dict:
+            logger.error(
+                "'models.model_name' argument must be dict,"
+                f"not {type(models['model_name'])}"
+            )
+            raise KeyError(
+                "Expected 'models.model_name' argument to be dict,"
+                f"not {type(models['model_name'])}"
+            )
+        # chack that model_name dict includes the keys: "spacy" and "transformers"
+        elif any(
+            [
+                any([key not in model_dict for key in ("spacy", "transformers")])
+                for model_lang, model_dict in models.items()
+            ]
+        ):
+            logger.error(
+                "'models.model_name' must contains 'spacy' and 'transformers' keys"
+            )
+            raise KeyError(
+                "Expected keys ('spacy' and 'transformers') was not found in "
+                "models.model_name dict"
+            )
+
+        logger.debug(f"Loading SpaCy and transformers models: {models.values()}")
+
+        self.nlp = {}
+        for lang_code, model_name in models.items():
+            nlp = spacy.load(model_name["spacy"], disable=["parser", "ner"])
+            nlp.add_pipe(
+                "transformers",
+                config={"pretrained_model_name_or_path": model_name["transformers"]},
+                last=True,
+            )
+            self.nlp[lang_code] = nlp

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/pattern.py

@@ -0,0 +1,45 @@
+import json
+from typing import Dict
+
+
+class Pattern:
+    """
+    A class that represents a regex pattern.
+
+    :param name: the name of the pattern
+    :param regex: the regex pattern to detect
+    :param score: the pattern's strength (values varies 0-1)
+    """
+
+    def __init__(self, name: str, regex: str, score: float):
+
+        self.name = name
+        self.regex = regex
+        self.score = score
+
+    def to_dict(self) -> Dict:
+        """
+        Turn this instance into a dictionary.
+
+        :return: a dictionary
+        """
+        return_dict = {"name": self.name, "score": self.score, "regex": self.regex}
+        return return_dict
+
+    @classmethod
+    def from_dict(cls, pattern_dict: Dict) -> "Pattern":
+        """
+        Load an instance from a dictionary.
+
+        :param pattern_dict: a dictionary holding the pattern's parameters
+        :return: a Pattern instance
+        """
+        return cls(**pattern_dict)
+
+    def __repr__(self):
+        """Return string representation of instance."""
+        return json.dumps(self.to_dict())
+
+    def __str__(self):
+        """Return string representation of instance."""
+        return json.dumps(self.to_dict())

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/pattern_recognizer.py

@@ -0,0 +1,253 @@
+import datetime
+import logging
+from typing import List, Optional, Dict
+
+import regex as re
+
+from presidio_analyzer import (
+    LocalRecognizer,
+    Pattern,
+    RecognizerResult,
+    EntityRecognizer,
+    AnalysisExplanation,
+)
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+logger = logging.getLogger("presidio_analyzer")
+
+
+class PatternRecognizer(LocalRecognizer):
+    """
+    PII entity recognizer using regular expressions or deny-lists.
+
+    :param patterns: A list of patterns to detect
+    :param deny_list: A list of words to detect,
+    in case our recognizer uses a predefined list of words (deny list)
+    :param context: list of context words
+    :param deny_list_score: confidence score for a term
+    identified using a deny-list
+    """
+
+    def __init__(
+        self,
+        supported_entity: str,
+        name: str = None,
+        supported_language: str = "en",
+        patterns: List[Pattern] = None,
+        deny_list: List[str] = None,
+        context: List[str] = None,
+        deny_list_score: float = 1.0,
+        version: str = "0.0.1",
+    ):
+
+        if not supported_entity:
+            raise ValueError("Pattern recognizer should be initialized with entity")
+
+        if not patterns and not deny_list:
+            raise ValueError(
+                "Pattern recognizer should be initialized with patterns"
+                " or with deny list"
+            )
+
+        super().__init__(
+            supported_entities=[supported_entity],
+            supported_language=supported_language,
+            name=name,
+            version=version,
+        )
+        if patterns is None:
+            self.patterns = []
+        else:
+            self.patterns = patterns
+        self.context = context
+        self.deny_list_score = deny_list_score
+
+        if deny_list:
+            deny_list_pattern = self._deny_list_to_regex(deny_list)
+            self.patterns.append(deny_list_pattern)
+            self.deny_list = deny_list
+        else:
+            self.deny_list = []
+
+    def load(self):  # noqa D102
+        pass
+
+    def analyze(
+        self,
+        text: str,
+        entities: List[str],
+        nlp_artifacts: NlpArtifacts = None,
+        regex_flags: int = None,
+    ) -> List[RecognizerResult]:
+        """
+        Analyzes text to detect PII using regular expressions or deny-lists.
+
+        :param text: Text to be analyzed
+        :param entities: Entities this recognizer can detect
+        :param nlp_artifacts: Output values from the NLP engine
+        :param regex_flags:
+        :return:
+        """
+        results = []
+
+        if self.patterns:
+            pattern_result = self.__analyze_patterns(text, regex_flags)
+            results.extend(pattern_result)
+
+        return results
+
+    def _deny_list_to_regex(self, deny_list: List[str]) -> Pattern:
+        """
+        Convert a list of words to a matching regex.
+
+        To be analyzed by the analyze method as any other regex patterns.
+
+        :param deny_list: the list of words to detect
+        :return:the regex of the words for detection
+        """
+
+        # Escape deny list elements as preparation for regex
+        escaped_deny_list = [re.escape(element) for element in deny_list]
+        regex = r"(?:^|(?<=\W))(" + "|".join(escaped_deny_list) + r")(?:(?=\W)|$)"
+        return Pattern(name="deny_list", regex=regex, score=self.deny_list_score)
+
+    def validate_result(self, pattern_text: str) -> Optional[bool]:
+        """
+        Validate the pattern logic e.g., by running checksum on a detected pattern.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool indicating whether the validation was successful.
+        """
+        return None
+
+    def invalidate_result(self, pattern_text: str) -> Optional[bool]:
+        """
+        Logic to check for result invalidation by running pruning logic.
+
+        For example, each SSN number group should not consist of all the same digits.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool indicating whether the result is invalidated
+        """
+        return None
+
+    @staticmethod
+    def build_regex_explanation(
+        recognizer_name: str,
+        pattern_name: str,
+        pattern: str,
+        original_score: float,
+        validation_result: bool,
+    ) -> AnalysisExplanation:
+        """
+        Construct an explanation for why this entity was detected.
+
+        :param recognizer_name: Name of recognizer detecting the entity
+        :param pattern_name: Regex pattern name which detected the entity
+        :param pattern: Regex pattern logic
+        :param original_score: Score given by the recognizer
+        :param validation_result: Whether validation was used and its result
+        :return: Analysis explanation
+        """
+        explanation = AnalysisExplanation(
+            recognizer=recognizer_name,
+            original_score=original_score,
+            pattern_name=pattern_name,
+            pattern=pattern,
+            validation_result=validation_result,
+        )
+        return explanation
+
+    def __analyze_patterns(
+        self, text: str, flags: int = None
+    ) -> List[RecognizerResult]:
+        """
+        Evaluate all patterns in the provided text.
+
+        Including words in the provided deny-list
+
+        :param text: text to analyze
+        :param flags: regex flags
+        :return: A list of RecognizerResult
+        """
+        flags = flags if flags else re.DOTALL | re.MULTILINE
+        results = []
+        for pattern in self.patterns:
+            match_start_time = datetime.datetime.now()
+            matches = re.finditer(pattern.regex, text, flags=flags)
+            match_time = datetime.datetime.now() - match_start_time
+            logger.debug(
+                "--- match_time[%s]: %s.%s seconds",
+                pattern.name,
+                match_time.seconds,
+                match_time.microseconds,
+            )
+
+            for match in matches:
+                start, end = match.span()
+                current_match = text[start:end]
+
+                # Skip empty results
+                if current_match == "":
+                    continue
+
+                score = pattern.score
+
+                validation_result = self.validate_result(current_match)
+                description = self.build_regex_explanation(
+                    self.name, pattern.name, pattern.regex, score, validation_result
+                )
+                pattern_result = RecognizerResult(
+                    entity_type=self.supported_entities[0],
+                    start=start,
+                    end=end,
+                    score=score,
+                    analysis_explanation=description,
+                    recognition_metadata={
+                        RecognizerResult.RECOGNIZER_NAME_KEY: self.name,
+                        RecognizerResult.RECOGNIZER_IDENTIFIER_KEY: self.id,
+                    },
+                )
+
+                if validation_result is not None:
+                    if validation_result:
+                        pattern_result.score = EntityRecognizer.MAX_SCORE
+                    else:
+                        pattern_result.score = EntityRecognizer.MIN_SCORE
+
+                invalidation_result = self.invalidate_result(current_match)
+                if invalidation_result is not None and invalidation_result:
+                    pattern_result.score = EntityRecognizer.MIN_SCORE
+
+                if pattern_result.score > EntityRecognizer.MIN_SCORE:
+                    results.append(pattern_result)
+
+                # Update analysis explanation score following validation or invalidation
+                description.score = pattern_result.score
+
+        results = EntityRecognizer.remove_duplicates(results)
+        return results
+
+    def to_dict(self) -> Dict:
+        """Serialize instance into a dictionary."""
+        return_dict = super().to_dict()
+
+        return_dict["patterns"] = [pat.to_dict() for pat in self.patterns]
+        return_dict["deny_list"] = self.deny_list
+        return_dict["context"] = self.context
+        return_dict["supported_entity"] = return_dict["supported_entities"][0]
+        del return_dict["supported_entities"]
+
+        return return_dict
+
+    @classmethod
+    def from_dict(cls, entity_recognizer_dict: Dict) -> "PatternRecognizer":
+        """Create instance from a serialized dict."""
+        patterns = entity_recognizer_dict.get("patterns")
+        if patterns:
+            patterns_list = [Pattern.from_dict(pat) for pat in patterns]
+            entity_recognizer_dict["patterns"] = patterns_list
+
+        return cls(**entity_recognizer_dict)

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/Aadhaar_Number.py

@@ -0,0 +1,46 @@
+from typing import Optional, List
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class Aadhaar_Number(PatternRecognizer):
+    """
+    Recognizes US bank number using regex.
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    """
+
+    PATTERNS = [
+        Pattern(name="aadhaar_number_pattern", regex="[2-9]{1}[0-9]{3}\s{1}[0-9]{4}\s{1}[0-9]{4}", score=0.5),
+    ]
+
+    CONTEXT = [
+        "bank"
+        # Task #603: Support keyphrases: change to "checking account"
+        # as part of keyphrase change
+        "check",
+        "account",
+        "account#",
+        "acct",
+        "save",
+        "debit",
+    ]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "AADHAR_NUMBER",
+    ):
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/PAN_Number.py

@@ -0,0 +1,46 @@
+from typing import Optional, List
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class PAN_Number(PatternRecognizer):
+    """
+    Recognizes US bank number using regex.
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    """
+
+    PATTERNS = [
+        Pattern(name="pan_number_pattern", regex="[A-Z]{5}[0-9]{4}[A-Z]{1}", score=0.5),
+    ]
+
+    CONTEXT = [
+        "bank"
+        # Task #603: Support keyphrases: change to "checking account"
+        # as part of keyphrase change
+        "check",
+        "account",
+        "account#",
+        "acct",
+        "save",
+        "debit",
+    ]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "PAN_Number",
+    ):
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/__init__.py

@@ -0,0 +1,77 @@
+"""Predefined recognizers package. Holds all the default recognizers."""
+
+from presidio_analyzer.predefined_recognizers.transformers_recognizer import (
+    TransformersRecognizer,
+)
+from .PAN_Number import PAN_Number
+from .credit_card_recognizer import CreditCardRecognizer
+from .crypto_recognizer import CryptoRecognizer
+from .date_recognizer import DateRecognizer
+from .email_recognizer import EmailRecognizer
+from .iban_recognizer import IbanRecognizer
+from .ip_recognizer import IpRecognizer
+from .medical_license_recognizer import MedicalLicenseRecognizer
+from .phone_recognizer import PhoneRecognizer
+from .sg_fin_recognizer import SgFinRecognizer
+from .spacy_recognizer import SpacyRecognizer
+from .stanza_recognizer import StanzaRecognizer
+from .uk_nhs_recognizer import NhsRecognizer
+from .url_recognizer import UrlRecognizer
+from .Aadhaar_Number import Aadhaar_Number
+from .data_recognizer import ClientListRecognizer
+from .us_driver_license_recognizer import UsLicenseRecognizer
+from .us_itin_recognizer import UsItinRecognizer
+from .us_passport_recognizer import UsPassportRecognizer
+from .us_ssn_recognizer import UsSsnRecognizer
+from .es_nif_recognizer import EsNifRecognizer
+from .au_abn_recognizer import AuAbnRecognizer
+from .au_acn_recognizer import AuAcnRecognizer
+from .au_tfn_recognizer import AuTfnRecognizer
+from .au_medicare_recognizer import AuMedicareRecognizer
+from .it_driver_license_recognizer import ItDriverLicenseRecognizer
+from .it_fiscal_code_recognizer import ItFiscalCodeRecognizer
+from .it_vat_code import ItVatCodeRecognizer
+from .it_identity_card_recognizer import ItIdentityCardRecognizer
+from .it_passport_recognizer import ItPassportRecognizer
+
+NLP_RECOGNIZERS = {
+    "spacy": SpacyRecognizer,
+    "stanza": StanzaRecognizer,
+    "transformers": TransformersRecognizer,
+    "client":ClientListRecognizer
+}
+
+__all__ = [
+    "AbaRoutingRecognizer",
+    "CreditCardRecognizer",
+    "CryptoRecognizer",
+    "DateRecognizer",
+    "EmailRecognizer",
+    "IbanRecognizer",
+    "IpRecognizer",
+    "NhsRecognizer",
+    "MedicalLicenseRecognizer",
+    "PhoneRecognizer",
+    "SgFinRecognizer",
+    "UrlRecognizer",
+    "UsBankRecognizer",
+    "UsItinRecognizer",
+    "UsLicenseRecognizer",
+    "UsPassportRecognizer",
+    "UsSsnRecognizer",
+    "EsNifRecognizer",
+    "SpacyRecognizer",
+    "ClientListRecognizer",
+    "StanzaRecognizer",
+    "NLP_RECOGNIZERS",
+    "AuAbnRecognizer",
+    "AuAcnRecognizer",
+    "AuTfnRecognizer",
+    "AuMedicareRecognizer",
+    "TransformersRecognizer",
+    "ItDriverLicenseRecognizer",
+    "ItFiscalCodeRecognizer",
+    "ItVatCodeRecognizer",
+    "ItIdentityCardRecognizer",
+    "ItPassportRecognizer",
+]

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/au_abn_recognizer.py

@@ -0,0 +1,93 @@
+from typing import Optional, List, Tuple
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class AuAbnRecognizer(PatternRecognizer):
+    """
+    Recognizes Australian Business Number ("ABN").
+
+    The Australian Business Number (ABN) is a unique 11
+    digit identifier issued to all entities registered in
+    the Australian Business Register (ABR).
+    The 11 digit ABN is structured as a 9 digit identifier
+    with two leading check digits.
+    The leading check digits are derived using a modulus 89 calculation.
+    This recognizer identifies ABN using regex, context words and checksum.
+    Reference: https://abr.business.gov.au/Help/AbnFormat
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    :param replacement_pairs: List of tuples with potential replacement values
+    for different strings to be used during pattern matching.
+    This can allow a greater variety in input, for example by removing dashes or spaces.
+    """
+
+    PATTERNS = [
+        Pattern(
+            "ABN (Medium)",
+            r"\b\d{2}\s\d{3}\s\d{3}\s\d{3}\b",
+            0.1,
+        ),
+        Pattern(
+            "ABN (Low)",
+            r"\b\d{11}\b",
+            0.01,
+        ),
+    ]
+
+    CONTEXT = [
+        "australian business number",
+        "abn",
+    ]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "AU_ABN",
+        replacement_pairs: Optional[List[Tuple[str, str]]] = None,
+    ):
+        self.replacement_pairs = (
+            replacement_pairs if replacement_pairs else [("-", ""), (" ", "")]
+        )
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str) -> bool:
+        """
+        Validate the pattern logic e.g., by running checksum on a detected pattern.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool indicating whether the validation was successful.
+        """
+        # Pre-processing before validation checks
+        text = self.__sanitize_value(pattern_text, self.replacement_pairs)
+        abn_list = [int(digit) for digit in text if not digit.isspace()]
+
+        # Set weights based on digit position
+        weight = [10, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19]
+
+        # Perform checksums
+        abn_list[0] = 9 if abn_list[0] == 0 else abn_list[0] - 1
+        sum_product = 0
+        for i in range(11):
+            sum_product += abn_list[i] * weight[i]
+        remainder = sum_product % 89
+        return remainder == 0
+
+    @staticmethod
+    def __sanitize_value(text: str, replacement_pairs: List[Tuple[str, str]]) -> str:
+        for search_string, replacement_string in replacement_pairs:
+            text = text.replace(search_string, replacement_string)
+        return text

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/au_acn_recognizer.py

@@ -0,0 +1,90 @@
+from typing import Optional, List, Tuple
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class AuAcnRecognizer(PatternRecognizer):
+    """
+    Recognizes Australian Company Number ("ACN").
+
+    The Australian Company Number (ACN) is a nine digit number
+    with the last digit being a check digit calculated using a
+    modified modulus 10 calculation.
+    This recognizer identifies ACN using regex, context words, and checksum.
+    Reference: https://asic.gov.au/
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    :param replacement_pairs: List of tuples with potential replacement values
+    for different strings to be used during pattern matching.
+    This can allow a greater variety in input, for example by removing dashes or spaces.
+    """
+
+    PATTERNS = [
+        Pattern(
+            "ACN (Medium)",
+            r"\b\d{3}\s\d{3}\s\d{3}\b",
+            0.1,
+        ),
+        Pattern(
+            "ACN (Low)",
+            r"\b\d{9}\b",
+            0.01,
+        ),
+    ]
+
+    CONTEXT = [
+        "australian company number",
+        "acn",
+    ]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "AU_ACN",
+        replacement_pairs: Optional[List[Tuple[str, str]]] = None,
+    ):
+        self.replacement_pairs = (
+            replacement_pairs if replacement_pairs else [("-", ""), (" ", "")]
+        )
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str) -> bool:
+        """
+        Validate the pattern logic e.g., by running checksum on a detected pattern.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool indicating whether the validation was successful.
+        """
+        # Pre-processing before validation checks
+        text = self.__sanitize_value(pattern_text, self.replacement_pairs)
+        acn_list = [int(digit) for digit in text if not digit.isspace()]
+
+        # Set weights based on digit position
+        weight = [8, 7, 6, 5, 4, 3, 2, 1]
+
+        # Perform checksums
+        sum_product = 0
+        for i in range(8):
+            sum_product += acn_list[i] * weight[i]
+        remainder = sum_product % 10
+        complement = 10 - remainder
+        return complement == acn_list[-1]
+
+    @staticmethod
+    def __sanitize_value(text: str, replacement_pairs: List[Tuple[str, str]]) -> str:
+        for search_string, replacement_string in replacement_pairs:
+            text = text.replace(search_string, replacement_string)
+        return text

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/au_medicare_recognizer.py

@@ -0,0 +1,89 @@
+from typing import Optional, List, Tuple
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class AuMedicareRecognizer(PatternRecognizer):
+    """
+    Recognizes Australian Medicare number using regex, context words, and checksum.
+
+    Medicare number is a unique identifier issued by Australian Government
+    that enables the cardholder to receive a rebates of medical expenses
+    under Australia's Medicare system.
+    It uses a modulus 10 checksum scheme to validate the number.
+    Reference: https://en.wikipedia.org/wiki/Medicare_card_(Australia)
+
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    :param replacement_pairs: List of tuples with potential replacement values
+    for different strings to be used during pattern matching.
+    This can allow a greater variety in input, for example by removing dashes or spaces.
+    """
+
+    PATTERNS = [
+        Pattern(
+            "Australian Medicare Number (Medium)",
+            r"\b[2-6]\d{3}\s\d{5}\s\d\b",
+            0.1,
+        ),
+        Pattern(
+            "Australian Medicare Number (Low)",
+            r"\b[2-6]\d{9}\b",
+            0.01,
+        ),
+    ]
+
+    CONTEXT = [
+        "medicare",
+    ]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "AU_MEDICARE",
+        replacement_pairs: Optional[List[Tuple[str, str]]] = None,
+    ):
+        self.replacement_pairs = (
+            replacement_pairs if replacement_pairs else [("-", ""), (" ", "")]
+        )
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str) -> bool:
+        """
+        Validate the pattern logic e.g., by running checksum on a detected pattern.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool indicating whether the validation was successful.
+        """
+        # Pre-processing before validation checks
+        text = self.__sanitize_value(pattern_text, self.replacement_pairs)
+        medicare_list = [int(digit) for digit in text if not digit.isspace()]
+
+        # Set weights based on digit position
+        weight = [1, 3, 7, 9, 1, 3, 7, 9]
+
+        # Perform checksums
+        sum_product = 0
+        for i in range(8):
+            sum_product += medicare_list[i] * weight[i]
+        remainder = sum_product % 10
+        return remainder == medicare_list[8]
+
+    @staticmethod
+    def __sanitize_value(text: str, replacement_pairs: List[Tuple[str, str]]) -> str:
+        for search_string, replacement_string in replacement_pairs:
+            text = text.replace(search_string, replacement_string)
+        return text

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/au_tfn_recognizer.py

@@ -0,0 +1,95 @@
+from typing import Optional, List, Tuple
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class AuTfnRecognizer(PatternRecognizer):
+    """
+    Recognizes Australian Tax File Numbers ("TFN").
+
+    The tax file number (TFN) is a unique identifier
+    issued by the Australian Taxation Office
+    to each taxpaying entity — an individual, company,
+    superannuation fund, partnership, or trust.
+    The TFN consists of a nine digit number, usually
+    presented in the format NNN NNN NNN.
+    TFN includes a check digit for detecting erroneous
+    number based on simple modulo 11.
+    This recognizer uses regex, context words,
+    and checksum to identify TFN.
+    Reference: https://www.ato.gov.au/individuals/tax-file-number/
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    :param replacement_pairs: List of tuples with potential replacement values
+    for different strings to be used during pattern matching.
+    This can allow a greater variety in input, for example by removing dashes or spaces.
+    """
+
+    PATTERNS = [
+        Pattern(
+            "TFN (Medium)",
+            r"\b\d{3}\s\d{3}\s\d{3}\b",
+            0.1,
+        ),
+        Pattern(
+            "TFN (Low)",
+            r"\b\d{9}\b",
+            0.01,
+        ),
+    ]
+
+    CONTEXT = [
+        "tax file number",
+        "tfn",
+    ]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "AU_TFN",
+        replacement_pairs: Optional[List[Tuple[str, str]]] = None,
+    ):
+        self.replacement_pairs = (
+            replacement_pairs if replacement_pairs else [("-", ""), (" ", "")]
+        )
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str) -> bool:
+        """
+        Validate the pattern logic e.g., by running checksum on a detected pattern.
+
+        :param pattern_text: the text to validated.
+        Only the part in text that was detected by the regex engine
+        :return: A bool indicating whether the validation was successful.
+        """
+        # Pre-processing before validation checks
+        text = self.__sanitize_value(pattern_text, self.replacement_pairs)
+        tfn_list = [int(digit) for digit in text if not digit.isspace()]
+
+        # Set weights based on digit position
+        weight = [1, 4, 3, 7, 5, 8, 6, 9, 10]
+
+        # Perform checksums
+        sum_product = 0
+        for i in range(9):
+            sum_product += tfn_list[i] * weight[i]
+        remainder = sum_product % 11
+        return remainder == 0
+
+    @staticmethod
+    def __sanitize_value(text: str, replacement_pairs: List[Tuple[str, str]]) -> str:
+        for search_string, replacement_string in replacement_pairs:
+            text = text.replace(search_string, replacement_string)
+        return text

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/credit_card_recognizer.py

@@ -0,0 +1,85 @@
+from typing import List, Tuple, Optional
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class CreditCardRecognizer(PatternRecognizer):
+    """
+    Recognize common credit card numbers using regex + checksum.
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    :param replacement_pairs: List of tuples with potential replacement values
+    for different strings to be used during pattern matching.
+    This can allow a greater variety in input, for example by removing dashes or spaces.
+    """
+
+    PATTERNS = [
+        Pattern(
+            "All Credit Cards (weak)",
+            r"\b((4\d{3})|(5[0-5]\d{2})|(6\d{3})|(1\d{3})|(3\d{3}))[- ]?(\d{3,4})[- ]?(\d{3,4})[- ]?(\d{3,5})\b",  # noqa: E501
+            0.3,
+        ),
+    ]
+
+    CONTEXT = [
+        "credit",
+        "card",
+        "visa",
+        "mastercard",
+        "cc ",
+        "amex",
+        "discover",
+        "jcb",
+        "diners",
+        "maestro",
+        "instapayment",
+    ]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "CREDIT_CARD",
+        replacement_pairs: Optional[List[Tuple[str, str]]] = None,
+    ):
+
+        self.replacement_pairs = (
+            replacement_pairs if replacement_pairs else [("-", ""), (" ", "")]
+        )
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str) -> bool:  # noqa D102
+        sanitized_value = self.__sanitize_value(pattern_text, self.replacement_pairs)
+        checksum = self.__luhn_checksum(sanitized_value)
+
+        return checksum
+
+    @staticmethod
+    def __luhn_checksum(sanitized_value: str) -> bool:
+        def digits_of(n: str) -> List[int]:
+            return [int(dig) for dig in str(n)]
+
+        digits = digits_of(sanitized_value)
+        odd_digits = digits[-1::-2]
+        even_digits = digits[-2::-2]
+        checksum = sum(odd_digits)
+        for d in even_digits:
+            checksum += sum(digits_of(str(d * 2)))
+        return checksum % 10 == 0
+
+    @staticmethod
+    def __sanitize_value(text: str, replacement_pairs: List[Tuple[str, str]]) -> str:
+        for search_string, replacement_string in replacement_pairs:
+            text = text.replace(search_string, replacement_string)
+        return text

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/crypto_recognizer.py

@@ -0,0 +1,54 @@
+from hashlib import sha256
+from typing import List, Optional
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+# Copied from:
+# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
+
+
+class CryptoRecognizer(PatternRecognizer):
+    """Recognize common crypto account numbers using regex + checksum.
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    """
+
+    PATTERNS = [
+        Pattern("Crypto (Medium)", r"\b[13][a-km-zA-HJ-NP-Z1-9]{26,33}\b", 0.5),
+    ]
+
+    CONTEXT = ["wallet", "btc", "bitcoin", "crypto"]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "CRYPTO",
+    ):
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str) -> bool:  # noqa D102
+        try:
+            bcbytes = self.__decode_base58(pattern_text, 25)
+            return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
+        except ValueError:
+            return False
+
+    @staticmethod
+    def __decode_base58(bc: str, length: int) -> bytes:
+        digits58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+        n = 0
+        for char in bc:
+            n = n * 58 + digits58.index(char)
+        return n.to_bytes(length, "big")

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/data_recognizer.py

@@ -0,0 +1,184 @@
+import logging
+from typing import Optional, List, Tuple, Set
+import spacy
+from spacy.matcher import PhraseMatcher
+from presidio_analyzer.predefined_recognizers.spacy_recognizer import SpacyRecognizer
+# from presidio_analyzer.predefined_recognizers import SpacyRecognizer
+from presidio_analyzer import RecognizerResult
+import copy
+
+
+
+
+from presidio_analyzer import (
+    RecognizerResult,
+    LocalRecognizer,
+    AnalysisExplanation,
+)
+
+logger = logging.getLogger("presidio_analyzer")
+# terms = ["1&1 Telecommunication SE","1010 data services LLC","AMA",
+#          "A O Smith Corporations","ABBMST","Addidas India","CITI","Cisco Systems","ERICSSON","Gati Ltd","IBM",
+#          "Infosys Ltd","Intel Corporation","Johnson","JTC Corporation","NSC Global","SUZUKI MOTOR CORPORATION",
+#          "Synopsys Ltd","TIBCOO", "T-Mobile UK","Toyota Systems Corporation","TSB Bank","UBS Bank"
+#         ,"United Health Corporation","Vodafone quickcom","Voltas","VOLVO CARS","WIPRO LIMITED",
+#          "Walmart", "CVS Health", "Walgreens Boots Alliance"]
+terms=[]
+class DataList:
+    # def __init__(self,val) -> None:
+        # self.Entiity=val
+    entity=[]
+    def setData(values):
+        terms.extend(values)
+        # print(terms)
+    def resetData():
+        terms.clear()
+    # def setEntity(val):
+    #     DataList.Entity=val
+    #     ClientListRecognizer(supported_entities=val)
+    # def getE():
+        # return self.Entiity
+        
+
+nlp = spacy.load("en_core_web_lg")
+        
+
+
+
+
+class ClientListRecognizer(SpacyRecognizer):     
+    """
+    Recognize PII entities using a spaCy NLP model.
+
+    Since the spaCy pipeline is ran by the AnalyzerEngine,
+    this recognizer only extracts the entities from the NlpArtifacts
+    and replaces their types to align with Presidio's.
+
+    :param supported_language: Language this recognizer supports
+    :param supported_entities: The entities this recognizer can detect
+    :param ner_strength: Default confidence for NER prediction
+    :param check_label_groups: Tuple containing Presidio entity names
+    and spaCy entity names, for verifying that the right entity
+    is translated into a Presidio entity.
+    """
+
+    ENTITIES = DataList.entity
+
+    DEFAULT_EXPLANATION = "Identified as {} by Spacy's Named Entity Recognition"
+
+    CHECK_LABEL_GROUPS = [
+        # ({"LOCATION"}, {"GPE", "LOC"}),
+        # ({"PERSON", "PER"}, {"PERSON", "PER"}),
+        # ({"DATE_TIME"}, {"DATE", "TIME"}),
+        # ({"NRP"}, {"NORP"}),
+        # ({"ORGANIZATION"}, {"ORG"}),
+        # ()
+    ]
+        
+    
+
+    
+
+    def __init__(
+        self,
+        supported_language: str = "en",
+        supported_entities: Optional[List[str]] = None,
+        ner_strength: float = 0.85,
+        check_label_groups: Optional[Tuple[Set, Set]] = None,
+        context: Optional[List[str]] = None,
+    ):
+        self.ner_strength = ner_strength
+        self.check_label_groups = (
+            check_label_groups if check_label_groups else self.CHECK_LABEL_GROUPS
+        )
+        supported_entities = supported_entities if supported_entities else self.ENTITIES
+        # print("=========",supported_entities)
+        super().__init__(
+            supported_entities=supported_entities,
+            supported_language=supported_language,
+            context=context,
+        )
+
+    def load(self) -> None:  # noqa D102
+        # no need to load anything as the analyze method already receives
+        # preprocessed nlp artifacts
+        pass
+    
+    
+    def build_spacy_explanation(
+        self, original_score: float, explanation: str
+    ) -> AnalysisExplanation:
+        """
+        Create explanation for why this result was detected.
+
+        :param original_score: Score given by this recognizer
+        :param explanation: Explanation string
+        :return:
+        """
+        explanation = AnalysisExplanation(
+            recognizer=self.__class__.__name__,
+            original_score=original_score,
+            textual_explanation=explanation,
+        )
+        return explanation
+    
+    def analyze(self, text, entities, nlp_artifacts=None):  # noqa D102
+        
+        # print("=========",self.supported_entities)
+       
+    #     matcher = PhraseMatcher(nlp.vocab)
+       
+    # # Only run nlp.make_doc to speed things up
+    #     patterns = [nlp.make_doc(text) for text in terms]
+    
+    #     matcher.add("TerminologyList", patterns)
+        # result = []
+        
+        matcher = PhraseMatcher(nlp.vocab)
+        
+        # Only run nlp.make_doc to speed things up
+        patterns = [nlp.make_doc(text) for text in terms]
+        
+        matcher.add("TerminologyList", patterns)
+        
+        results = []
+        # result =[]
+       
+        doc = nlp(text)
+        doc1 = str(doc)
+        
+        matches = matcher(doc)
+        for match_id, start, end in matches:
+            span = doc[start:end]
+            
+            if doc1.find(str(span)):
+                doc1=doc1.replace(str(span.text),"<COMPANY_NAME>")
+            etype=copy.deepcopy(DataList.entity)    
+            spacy_result = RecognizerResult(
+                    
+                    entity_type=etype[0],
+                    start=span.start_char,
+                    end=span.end_char,
+                    score=self.ner_strength,
+                    # analysis_explanation=explanation,
+                    recognition_metadata={
+                        RecognizerResult.RECOGNIZER_NAME_KEY: self.name,
+                        RecognizerResult.RECOGNIZER_IDENTIFIER_KEY: self.id,
+                    },
+                )
+            
+
+            results.append(spacy_result)
+
+       
+      
+
+        return results
+
+    @staticmethod
+    def __check_label(
+        entity: str, label: str, check_label_groups: Tuple[Set, Set]
+    ) -> bool:
+        return any(
+            [entity in egrp and label in lgrp for egrp, lgrp in check_label_groups]
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/date_recognizer.py

@@ -0,0 +1,127 @@
+from typing import Optional, List
+
+from presidio_analyzer import Pattern, PatternRecognizer, RecognizerResult
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+import regex as re
+
+
+class DateRecognizer(PatternRecognizer):
+    """
+    Recognize date using regex.
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    """
+
+    PATTERNS = [
+        Pattern(
+            "mm/dd/yyyy or mm/dd/yy",
+            r"\b(([1-9]|0[1-9]|1[0-2])/([1-9]|0[1-9]|[1-2][0-9]|3[0-1])/(\d{4}|\d{2}))\b",  # noqa: E501
+            0.6,
+        ),
+        Pattern(
+            "dd/mm/yyyy or dd/mm/yy",
+            r"\b(([1-9]|0[1-9]|[1-2][0-9]|3[0-1])/([1-9]|0[1-9]|1[0-2])/(\d{4}|\d{2}))\b",  # noqa: E501
+            0.6,
+        ),
+        Pattern(
+            "yyyy/mm/dd",
+            r"\b(\d{4}/([1-9]|0[1-9]|1[0-2])/([1-9]|0[1-9]|[1-2][0-9]|3[0-1]))\b",
+            0.6,
+        ),
+        Pattern(
+            "mm-dd-yyyy",
+            r"\b(([1-9]|0[1-9]|1[0-2])-([1-9]|0[1-9]|[1-2][0-9]|3[0-1])-\d{4})\b",
+            0.6,
+        ),
+        Pattern(
+            "dd-mm-yyyy",
+            r"\b(([1-9]|0[1-9]|[1-2][0-9]|3[0-1])-([1-9]|0[1-9]|1[0-2])-\d{4})\b",
+            0.6,
+        ),
+        Pattern(
+            "yyyy-mm-dd",
+            r"\b(\d{4}-([1-9]|0[1-9]|1[0-2])-([1-9]|0[1-9]|[1-2][0-9]|3[0-1]))\b",
+            0.6,
+        ),
+        Pattern(
+            "dd.mm.yyyy or dd.mm.yy",
+            r"\b(([1-9]|0[1-9]|[1-2][0-9]|3[0-1])\.([1-9]|0[1-9]|1[0-2])\.(\d{4}|\d{2}))\b",  # noqa: E501
+            0.6,
+        ),
+        Pattern(
+            "dd-MMM-yyyy or dd-MMM-yy",
+            r"\b(([1-9]|0[1-9]|[1-2][0-9]|3[0-1])-(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)-(\d{4}|\d{2}))\b",  # noqa: E501
+            0.6,
+        ),
+        Pattern(
+            "MMM-yyyy or MMM-yy",
+            r"\b((JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)-(\d{4}|\d{2}))\b",  # noqa: E501
+            0.6,
+        ),
+        Pattern(
+            "dd-MMM or dd-MMM",
+            r"\b(([1-9]|0[1-9]|[1-2][0-9]|3[0-1])-(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC))\b",  # noqa: E501
+            0.6,
+        ),
+        Pattern(
+            "mm/yyyy or m/yyyy",
+            r"\b(([1-9]|0[1-9]|1[0-2])/\d{4})\b",
+            0.2,
+        ),
+        Pattern(
+            "mm/yy or m/yy",
+            r"\b(([1-9]|0[1-9]|1[0-2])/\d{2})\b",
+            0.1,
+        ),
+    ]
+
+    CONTEXT = ["date", "birthday"]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "DATE_TIME",
+    ):
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def analyze(
+        self,
+        text: str,
+        entities: List[str],
+        nlp_artifacts: NlpArtifacts = None,
+        regex_flags: int = None,
+    ) -> List[RecognizerResult]:
+        """
+        Analyzes text to detect PII using regular expressions or deny-lists.
+
+        :param text: Text to be analyzed
+        :param entities: Entities this recognizer can detect
+        :param nlp_artifacts: Output values from the NLP engine
+        :param regex_flags:
+        :return:
+        """
+        regex_flags = (
+            regex_flags | re.IGNORECASE
+            if regex_flags
+            else re.DOTALL | re.MULTILINE | re.IGNORECASE
+        )  # noqa: E501
+
+        return super().analyze(
+            text=text,
+            entities=entities,
+            nlp_artifacts=nlp_artifacts,
+            regex_flags=regex_flags,
+        )

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/email_recognizer.py

@@ -0,0 +1,46 @@
+from typing import List, Optional
+
+import tldextract
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class EmailRecognizer(PatternRecognizer):
+    """
+    Recognize email addresses using regex.
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    """
+
+    PATTERNS = [
+        Pattern(
+            "Email (Medium)",
+            r"\b((([!#$%&'*+\-/=?^_`{|}~\w])|([!#$%&'*+\-/=?^_`{|}~\w][!#$%&'*+\-/=?^_`{|}~\.\w]{0,}[!#$%&'*+\-/=?^_`{|}~\w]))[@]\w+([-.]\w+)*\.\w+([-.]\w+)*)\b",  # noqa: E501
+            0.5,
+        ),
+    ]
+
+    CONTEXT = ["email"]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "en",
+        supported_entity: str = "EMAIL_ADDRESS",
+    ):
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str):  # noqa D102
+        result = tldextract.extract(pattern_text)
+        return result.fqdn != ""

presidio_analyzer/presidio_analyzer/Infosys_presidio_analyzer/presidio_analyzer/presidio_analyzer/predefined_recognizers/es_nif_recognizer.py

@@ -0,0 +1,58 @@
+from typing import List, Tuple, Optional
+
+from presidio_analyzer import Pattern, PatternRecognizer
+
+
+class EsNifRecognizer(PatternRecognizer):
+    """
+    Recognize NIF number using regex and checksum.
+
+    :param patterns: List of patterns to be used by this recognizer
+    :param context: List of context words to increase confidence in detection
+    :param supported_language: Language this recognizer supports
+    :param supported_entity: The entity this recognizer can detect
+    :param replacement_pairs: List of tuples with potential replacement values
+    for different strings to be used during pattern matching.
+    This can allow a greater variety in input, for example by removing dashes or spaces.
+    """
+
+    PATTERNS = [
+        Pattern(
+            "NIF",
+            r"\b[0-9]?[0-9]{7}[-]?[A-Z]\b",
+            0.5,
+        ),
+    ]
+
+    CONTEXT = ["documento nacional de identidad", "DNI", "NIF", "identificación"]
+
+    def __init__(
+        self,
+        patterns: Optional[List[Pattern]] = None,
+        context: Optional[List[str]] = None,
+        supported_language: str = "es",
+        supported_entity: str = "ES_NIF",
+        replacement_pairs: Optional[List[Tuple[str, str]]] = None,
+    ):
+        self.replacement_pairs = (
+            replacement_pairs if replacement_pairs else [("-", ""), (" ", "")]
+        )
+        patterns = patterns if patterns else self.PATTERNS
+        context = context if context else self.CONTEXT
+        super().__init__(
+            supported_entity=supported_entity,
+            patterns=patterns,
+            context=context,
+            supported_language=supported_language,
+        )
+
+    def validate_result(self, pattern_text: str) -> bool:  # noqa D102
+        pattern_text = EsNifRecognizer.__sanitize_value(pattern_text)
+        letter = pattern_text[-1]
+        number = int("".join(filter(str.isdigit, pattern_text)))
+        letters = "TRWAGMYFPDXBNJZSQVHLCKE"
+        return letter == letters[number % 23]
+
+    @staticmethod
+    def __sanitize_value(text: str) -> str:
+        return text.replace("-", "").replace(" ", "")