Upload 7 files

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.11.3
+
+WORKDIR /app
+
+COPY requirements.txt /app/
+
+RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
+
+COPY . /app/
+
+CMD ["Gradio app.py" ]

README.md

@@ -1,13 +1,12 @@
 ---
-title: Quality Check App
-emoji: 👁
-colorFrom: yellow
-colorTo: gray
+title: Intern Qa Check
+emoji: 🏢
+colorFrom: pink
+colorTo: red
 sdk: gradio
-sdk_version: 4.1.2
+sdk_version: 3.28.1
 app_file: app.py
 pinned: false
-license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,164 @@
+# standard libraries
+import os
+
+# third party libraries
+import gradio as gr
+import validators
+from validators import ValidationFailure
+import gspread
+from google.oauth2.service_account import Credentials 
+from pydrive.auth import GoogleAuth
+from pydrive.drive import GoogleDrive
+
+import etl
+from generic import matched_super_generic
+
+cred_filename = 'heliumhealth-a05d595e5991.json'
+# cred_file_path = os.path.join('credentials', cred_filename)
+
+# google auth scopes
+scopes = ['https://www.googleapis.com/auth/spreadsheets',
+          'https://www.googleapis.com/auth/drive']
+
+# create credentials 
+credentials = Credentials.from_service_account_file(
+    cred_filename, scopes=scopes )
+
+# authorize google spreadsheet
+gc = gspread.authorize(credentials)
+
+gauth = GoogleAuth()
+drive = GoogleDrive(gauth)
+
+
+def prescription_quality_check(gs, sheet_type):
+    """
+    - Loads, transforms prescription data 
+    - Performs RX Norm check for prescription data 
+
+    Args:
+        gs (Object): authenticated google sheet instance
+        sheet_type (str): sheet type always: prescription
+    """
+    # load data from sheet
+    pres_data, super_data = etl.load_data(gs, sheet_type)
+
+    # process data for use
+    pres_data, super_data = etl.preprocess_data(pres_data, super_data, sheet_type)
+
+    # run the matching algorithm
+    pres_data = pres_data.apply(matched_super_generic, 
+                                axis=1, 
+                                args=[
+                                    super_data, 
+                                    'RX Norm [Super Generic]',
+                                    'SUPER_GENERIC', 
+                                    'match'])
+
+    # post process data
+    pres_data = pres_data[['Unstructured Name', 'RX Norm [Super Generic]', 'match', 'Number']]
+    
+    # write output to sheet 
+    etl.output_data(gs, pres_data, sheet_type)
+
+
+def diagnosis_quality_check(gs, sheet_type):
+    """
+    - Loads, transforms prescription data 
+    - Performs RX Norm check for prescription data 
+
+    Args:
+        gs (Object): google-sheet instance 
+        sheet_type (str): sheet type always diagnosis
+    """
+    # load data from sheet
+    diag_data, super_data = etl.load_data(gs, sheet_type)
+    # process data for use
+    diag_data, super_data = etl.preprocess_data(diag_data, super_data, sheet_type)
+    
+    
+    # run the matching algorithm
+    diag_data = diag_data.apply(matched_super_generic, 
+                                axis=1, 
+                                args=[
+                                    super_data, 
+                                    'ICD10 Diagnosis',
+                                    'Name', 
+                                    'match'])
+
+    # post processing 
+    diag_data = diag_data[['Unstructured Name', 'ICD10 Diagnosis', 'match', 'Number']]
+
+    # write output to sheet 
+    etl.output_data(gs, diag_data, sheet_type)
+
+def quality_check_main(sheet_type, sheet_url):
+    """
+   main check function
+
+    Args:
+        sheet_type (str): sheet type either: prescription or diagnosis 
+        sheet_url (url): url to data sheet
+
+    Raises:
+        gr.exceptions.Error: Value error for invalid urls 
+
+    Returns:
+        message (str): success or failure messages.
+    """
+    result = validators.url(sheet_url)
+
+    # open the google sheet for reading 
+    gs = gc.open_by_url(sheet_url)
+    
+    # Check if the input is valid url
+    # ToDO: Error message displayed is not explanatory - Fix it 
+    if isinstance(result, ValidationFailure):
+        raise gr.exceptions.Error(message="Please enter a valid URL")
+    
+    if sheet_type == 'diagnosis':
+        diagnosis_quality_check(gs, sheet_type)
+    elif sheet_type == 'prescription':
+        prescription_quality_check(gs, sheet_type)
+
+    output_text = f"{sheet_type} Q/A successfully. Please check the result sheet for output"
+    return output_text
+
+
+
+with gr.Blocks() as demo:
+  gr.Markdown(
+      """
+      # Quality Assurance App
+
+      ## Instructions
+
+      * Add IAM User to sheet you want to test
+      * Gsheet tabs required for diagnosis:  
+        * Diagnosis, ICD10 Diagnosis, resultd
+      * Gsheet tabs required for prescription:  
+        * Prescriptions, Super_generic_Prescriptions, result
+      * Data headers required for diagnosis
+        * Unstructured Name, ICD10 Diagnosis, Number
+      * Data headers required for prescription
+        * Unstructured Name, RX Norm [Super Generic], Number
+      """
+  )
+  # inputs
+  sheet_type = gr.Dropdown(['prescription', 'diagnosis'], label="QA Type")
+  url = gr.Textbox(label="URL", placeholder="Enter sheet url ...")
+  # outputs
+  output = gr.Textbox(label="Output Box")
+  
+  run_btn = gr.Button("Run")
+  run_btn.click(
+      fn=quality_check_main,
+      inputs=[
+          sheet_type,
+          url
+          ],
+      outputs=output, 
+      api_name="quality_check"
+  )
+
+demo.launch()

etl.py

@@ -0,0 +1,105 @@
+import gradio as gr
+import pandas as pd
+from gspread_dataframe import set_with_dataframe 
+
+def load_data(gs, d_type):
+    """_summary_
+
+    Args:
+        gs (_type_): _description_
+        d_type (_type_): _description_
+
+    Raises:
+        gr.exception.Error: _description_
+
+    Returns:
+        _type_: _description_
+    """
+    in_sheets_dict = {
+        "diagnosis": ["Diagnosis", "ICD10 Diagnosis"],
+        "prescription": ["Prescriptions", "Super_generic_Prescriptions"]
+    }
+
+    # open data and super sheet
+    try:
+        data_sheet = gs.worksheet(in_sheets_dict[d_type][0])
+        super_sheet = gs.worksheet(in_sheets_dict[d_type][1])
+    except:
+        raise gr.exception.Error(message='Permission denied. Please add IAM user to the sheet and try again')
+    
+    # read data from sheets
+    data = pd.DataFrame(data_sheet.get_all_records())
+    super_data = pd.DataFrame(super_sheet.get_all_records())
+    
+    return data, super_data
+
+def preprocess_data(data, super_data, d_type):
+    """
+    _summary_
+
+    Args:
+        data (_type_): _description_
+        super_data (_type_): _description_
+        d_type (_type_): _description_
+
+    Raises:
+        gr.exception.Error: _description_
+
+    Returns:
+        _type_: _description_
+    """
+    cols_dict = {
+        "diagnosis": ['Unstructured Name', 'ICD10 Diagnosis', 'Number'],
+        "prescription": ['Unstructured Name', 'RX Norm [Super Generic]', 'Number']
+    }
+
+    super_dict = {
+        "diagnosis": "Name", 
+        "prescription": "SUPER_GENERIC"
+    }
+    try:
+        # remove white spaces from columns
+        data.columns = data.columns.str.strip()
+        super_data.columns = super_data.columns.str.strip()
+        # select relevant columns
+        data = data[cols_dict[d_type]]
+
+        # remove all white spaces from column data
+        super_data[super_dict[d_type]] = super_data[super_dict[d_type]].str.strip()
+        data[cols_dict[d_type][1]] = data[cols_dict[d_type][1]].str.strip()
+    except:
+        raise gr.exception.Error(message='Unmatched named columns')
+    
+    return data, super_data
+
+def output_data(gs, data, dtype):
+    """_summary_
+
+    Args:
+        gs (_type_): _description_
+        data (_type_): _description_
+        dtype (_type_): _description_
+
+    Raises:
+        gr.exception.Error: _description_
+    """
+    out_sheet_dict = {
+        "diagnosis": "resultd", 
+        "prescription": "result"
+    }
+
+    # write output to sheet 
+    try:
+        out_worksheet = gs.worksheet(out_sheet_dict[dtype])
+        # clear the worksheet 
+        out_worksheet.clear()
+    except:
+        raise gr.exception.Error("Result sheet does not exist")
+
+    # write dataframe to work sheet
+    set_with_dataframe(
+        worksheet=out_worksheet, 
+        dataframe=data, 
+        include_index=False,
+        include_column_header=True, 
+        resize=True)

generic.py

@@ -0,0 +1,22 @@
+def matched_super_generic(row, 
+                          generic_data, 
+                          colname, 
+                          super_colname, 
+                          match_col):
+    """_summary_
+
+    Args:
+        row (_type_): _description_
+        generic_data (_type_): _description_
+        colname (_type_): _description_
+        super_colname (_type_): _description_
+        match_col (_type_): _description_
+
+    Returns:
+        _type_: _description_
+    """
+    diag = row[colname]
+    match_ = generic_data[generic_data[super_colname] == diag]
+    if not match_.empty:
+        row[match_col] = match_[super_colname].iloc[0]
+    return row

heliumhealth-a05d595e5991.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "heliumhealth",
+  "private_key_id": "a05d595e59911b601cf718b32e967a0f4e1f463d",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEugIBADANBgkqhkiG9w0BAQEFAASCBKQwggSgAgEAAoIBAQCzJJ2dCd32LwAc\njoTiCB2shYwGRFvApmF3/RUkT9BZDg5S+imfuXSsHBgueKIEAW7E0UBGVbioKQC0\nEzKh/+aLVdZEYdqyNdrx4g5HmuDn4sQyd/zP0J3cUa/LlF/cND6HsExJcdkjL9rV\nVZEQZhpVrgB9J6Ui5dZsrN9RWHguP8awI6JFqsfF2aKWJ4Z+Gl9YlxMhCqcwBalF\nf4MUOwnXSK1t0QMXdEAg9zA+wwySYjCz/nVOIrEBB1lHp1mEiwpv2KtuX3f2GGRt\nSGwOThDWJPn6EuLcX2mlxEs7ylEs1hYcKfROPzBMSRX5g/m5lKkRuZLsBpqKZj5G\nrB62FasXAgMBAAECggEADIMutRTG2Zaf0d7MfkMl23J6fMeShwq3/RsCuoYOneHC\n813AwYTBFvJhaAGKA3CcLYgWDrOXEbvHbkx2+WS31BybOiyBi3gCgmmRKMQzGE84\n9/N0h4lhNU3JUxVaerV1dqHEOk8G5RPHG6Z92uDezwQ3yYYW3TCjgPS0JmjhAA3b\nAnAgATuiY4ivscR28xSTDsexQhDnVGdM0ms4hrAUV0nM+sGLbXil6hTBoRi/iEmq\nGESeFsDOcdJKhIAvs94FwhHGLljCniP+2/jN6Wg8+wTFwUpaH0NpV6tfHYvGe8Bp\natfVZwCqwP5+J3dXHuaNUwe/cUK6HSVcGARXp5vubQKBgQD7xXVg2WoMONpGHWnD\nxk3T085JiC+MgBK+y5MzNLTtVxCNzsv3I70lu8ilv7ytfy35bnQ2dx8Ych1Jjc7z\n5kQs9rRxtY5iZvyTRaBB4Zfyr2uGxrR/lMJizsH+M0my48zPiHWLdqJSTupBmufr\n2kFeRIbphEl6u5fir9s5c/+EGwKBgQC2JuCQXrYHryn2vs6qf+Zf/RgzK5+3irHy\nEeYbedEpe6dCy5dWjlh7sXR23938XTXtnwmKWMuovdnLybBXvuWacOsqz5wYuXAn\nK6V8oBPpV9Aj+nVSft26E8nMzNX1lXbvxBgq1GvOUk7Mbi9p4nm6gjKNKYo1/U9P\nH+S9VHQMtQKBgAxqkH9WOYSgySLsEGs3PF9V8rZtoOqs5j/Cil/cGZAa2xYjPKvT\nd2CFAkAqVIO54eqLJ/AHr+Dkv80A0VP15ybQg9WXvo3bxlj89gpJtdSiEgtzgTNJ\ngsycpbSDkv7ffRo/AI0ALMEiYysZGJbpDJA5kO2zOGx1E+h6A7WxoshBAn8UolLO\nB+yW8kDOss62gcaXGRSkt0xgflWqFlz6v9Hx4RARgP6jz3w2huOqk7GR5P027c0m\n3ugzNU52x2Iyjm10EVaSgvIr2tXZmhglBf07cbciXDzuG3ECozs49/tE1qmif5Q9\nRdLwjGJgxhqY5A7mEdmoJAyEES3qyLIgwHBlAoGAS0sA8A5Ay+BbaAGufqSaBnxt\nQgMGNYUgyeG6edksXMRPMqTWbQ5xdjEmv38nQ8AT9RVMZ9TfZqCts1fCD+FgTcAU\niAasrJMvd4mCBhWZWPwGJN+yaeZHvdbrXsSEo3kkyiwjFip03CCJtDJO3MwyowOb\nIxC34Nn4A6XUGjaF7Io=\n-----END PRIVATE KEY-----\n",
+  "client_email": "test-gs@heliumhealth.iam.gserviceaccount.com",
+  "client_id": "102772078802762265627",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-gs%40heliumhealth.iam.gserviceaccount.com"
+}

requirements.txt

@@ -0,0 +1,87 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+async-timeout==4.0.2
+attrs==23.1.0
+beautifulsoup4==4.12.2
+cachetools==5.3.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+decorator==5.1.1
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.12.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+google==3.0.0
+google-api-core==2.11.0
+google-api-python-client==2.86.0
+google-auth==2.17.3
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==1.0.0
+googleapis-common-protos==1.59.0
+gradio==3.28.1
+gradio_client==0.1.4
+gspread==5.8.0
+gspread-dataframe==3.3.0
+h11==0.14.0
+httpcore==0.17.0
+httplib2==0.22.0
+httpx==0.24.0
+huggingface-hub==0.14.1
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.2
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.3
+oauth2client==4.1.3
+oauthlib==3.2.2
+orjson==3.8.11
+packaging==23.1
+pandas==2.0.1
+Pillow==9.5.0
+protobuf==4.22.3
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pydantic==1.10.7
+PyDrive==1.3.1
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0
+requests==2.29.0
+requests-oauthlib==1.3.1
+rsa==4.9
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.4.1
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+uritemplate==4.1.1
+urllib3==1.26.15
+uvicorn==0.22.0
+validators==0.20.0
+websockets==11.0.2
+yarl==1.9.2