Hesam Fallahian commited on
Commit
87a356d
·
1 Parent(s): f2973b9

initial commit

Browse files
Files changed (4) hide show
  1. .gitignore +130 -0
  2. articles_db.csv +60 -0
  3. main.py +222 -0
  4. requirements.txt +20 -0
.gitignore ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ pip-wheel-metadata/
24
+ share/python-wheels/
25
+ *.egg-info/
26
+ .installed.cfg
27
+ *.egg
28
+ MANIFEST
29
+
30
+ # PyInstaller
31
+ # Usually these files are written by a python script from a template
32
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
33
+ *.manifest
34
+ *.spec
35
+
36
+ # Installer logs
37
+ pip-log.txt
38
+ pip-delete-this-directory.txt
39
+
40
+ # Unit test / coverage reports
41
+ htmlcov/
42
+ .tox/
43
+ .nox/
44
+ .coverage
45
+ .coverage.*
46
+ .cache
47
+ nosetests.xml
48
+ coverage.xml
49
+ *.cover
50
+ *.py,cover
51
+ .hypothesis/
52
+ .pytest_cache/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ target/
76
+
77
+ # Jupyter Notebook
78
+ .ipynb_checkpoints
79
+
80
+ # IPython
81
+ profile_default/
82
+ ipython_config.py
83
+
84
+ # pyenv
85
+ .python-version
86
+
87
+ # pipenv
88
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
90
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
91
+ # install all needed dependencies.
92
+ #Pipfile.lock
93
+
94
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95
+ __pypackages__/
96
+
97
+ # Celery stuff
98
+ celerybeat-schedule
99
+ celerybeat.pid
100
+
101
+ # SageMath parsed files
102
+ *.sage.py
103
+
104
+ # Environments
105
+ .env
106
+ .venv
107
+ env/
108
+ venv/
109
+ ENV/
110
+ env.bak/
111
+ venv.bak/
112
+
113
+ # Spyder project settings
114
+ .spyderproject
115
+ .spyproject
116
+
117
+ # Rope project settings
118
+ .ropeproject
119
+
120
+ # mkdocs documentation
121
+ /site
122
+
123
+ # mypy
124
+ .mypy_cache/
125
+ .dmypy.json
126
+ dmypy.json
127
+
128
+ # Pyre type checker
129
+ .pyre/
130
+ .idea
articles_db.csv ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ id,Title,Link,Context,Category,SourceType,FileName,Description
2
+ 1,2020 FORTIFIED Standard,https://fortifiedhome.org/wp-content/uploads/2020-FORTIFIED-Home-Standard.pdf,Re-Roofing and Roof,Construction Standards,file,2020_fortified_standard.pdf,Short description …
3
+ 2,Sealed Roof Deck Supplemental Deck Attachment,https://fortifiedhome.org/wp-content/uploads/2015-04_technical-bulletin_sealed-roof-deck-supplemental-deck-attachment.pdf,Sealed Roof Deck,Technical Bulletin,file,sealed_roof_deck_supplemental_deck_attachment.pdf,Short description …
4
+ 3,Roof Flashing,https://fortifiedhome.org/wp-content/uploads/2017-01_technical-bulletin_flashing.pdf,Roof Flashing,Technical Bulletin,file,roof_flashing.pdf,Short description …
5
+ 4,Metal Panel Roof Covering Guidance,https://fortifiedhome.org/wp-content/uploads/2019-01_technical-bulletin_metal-roof-panel-selection_revised.pdf,Metal Panel and Roof,Technical Bulletin,file,metal_panel_roof_covering_guidance.pdf,Short description …
6
+ 5,Design Pressure Guidance for Roof Coverings,https://fortifiedhome.org/wp-content/uploads/2020-01_technical-bulletin_design-pressure-guidance-for-roof-coverings_revised.pdf,Roof and Sealed Roof Deck,Technical Bulletin,file,design_pressure_guidance_for_roof_coverings.pdf,Short description …
7
+ 6,Garage Door Requirements,https://fortifiedhome.org/wp-content/uploads/2021-01_technical-bulletin_garage-door-requirements.pdf,General,Technical Bulletin,file,garage_door_requirements.pdf,Short description …
8
+ 7,PA - Vycor Product Advisory,https://fortifiedhome.org/wp-content/uploads/Vycor_Product_Advisory.pdf,General,Technical Bulletin,file,pa_vycor_product_advisory.pdf,Short description …
9
+ 8,Corrosion Resistant Fasteners,https://fortifiedhome.org/wp-content/uploads/TB_FH_2021-02.pdf,"New Roof, Re-Roofing, Retrofit, and Roof",Technical Bulletin,file,corrosion_resistant_fasteners.pdf,Short description …
10
+ 9,Sealed Roof Deck for Wood Shake and Shingle Roof Systems,https://fortifiedhome.org/wp-content/uploads/TB-FH-2021-03_SRD_options_for_wood_shake.pdf,"Re-Roofing, Retrofit, and Roof",Technical Bulletin,file,sealed_roof_deck_for_wood_shake_and_shingle_roof_systems.pdf,Short description …
11
+ 10,IBHS Guidance Choosing the Right Tape,https://fortifiedhome.org/wp-content/uploads/Choosing-the-Right-Tape_FINAL.pdf,Roof,Technical Bulletin,file,ibhs_guidance_choosing_the_right_tape.pdf,Short description …
12
+ 11,Roof Sheathing Nail Pattern Documentation Requirements,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022_01-Roof-Sheathing-Attachment-Documentation.pdf,Roof,Technical Bulletin,file,roof_sheathing_nail_pattern_documentation_requirements.pdf,Short description …
13
+ 12,FORTIFIED Roof Identification,https://fortifiedhome.org/wp-content/uploads/FORTIFIED-ROOF-IDENTIFICATIONR10.pdf,Roof,Technical Bulletin,file,fortified_roof_identification.pdf,Short description …
14
+ 13,The FORTIFIED Definition of Roof,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022-02-Definition-of-Roof.pdf,Roof,Technical Bulletin,file,the_fortified_definition_of_roof.pdf,Short description …
15
+ 14,FORTIFIED Home Requirements for Elevated Roof-Mounted Decks,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022-03-Elevated-Rooftop-Decks.pdf,Roof and Sealed Roof Deck,Technical Bulletin,file,fortified_home_requirements_for_elevated_roof_mounted_decks.pdf,Short description …
16
+ 15,Product Substitution Due to Supply Chain Issues and Product Availability,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022_04-Supply-Chain-Issues.pdf,Roof,Technical Bulletin,file,product_substitution_due_to_supply_chain_issues_and_product_availability.pdf,Short description …
17
+ 16,Requirements for Re-Roofing Over Existing Self-Adhered Membranes,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022-05-Reroofing-Over-Self-Adhered-Membranes.pdf,General,Technical Bulletin,file,requirements_for_re_roofing_over_existing_self_adhered_membranes.pdf,Short description …
18
+ 17,Foundation Requirements for FORTIFIED Home Eligibility,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022-06-Foundation-Requirements-FORTIFIED-Home.pdf,Roof,Technical Bulletin,file,foundation_requirements_for_fortified_home_eligibility.pdf,Short description …
19
+ 18,Removal of Egress Requirement for Entry Doors,https://fortifiedhome.org/wp-content/uploads/TB-Update-Removal-of-Egress-Requirement.pdf,Roof,Technical Bulletin,file,removal_of_egress_requirement_for_entry_doors.pdf,Short description …
20
+ 19,Product Advisory FH Eligibility of Existing Fiberglass Resin Deck Coatings as Roof Covers,https://fortifiedhome.org/wp-content/uploads/PA_FH_2023-01-Fiberglass-resin-deck-coatings-as-roof-covers.pdf,General,Technical Bulletin,file,product_advisory_fh_eligibility_of_existing_fiberglass_resin_deck_coatings_as_roof_covers.pdf,Short description …
21
+ 20,Fastener Type Requirements for Asphalt Shingles and Drip Edge,https://fortifiedhome.org/wp-content/uploads/TB-Update-Fastener-Requirements-for-Asphalt-Shingles-and-Drip-Edge.pdf,Roof,Technical Bulletin,file,fastener_type_requirements_for_asphalt_shingles_and_drip_edge.pdf,Short description …
22
+ 21,Eligibility and Compliance of Accessory Roof Structures,https://fortifiedhome.org/wp-content/uploads/TB-2023-03_Eligibility-and-Compliance-Accessory-Roof-Structures.pdf,General,Technical Bulletin,file,eligibility_and_compliance_of_accessory_roof_structures.pdf,Short description …
23
+ 22,Use of Factory Seconds or Unlabeled Material in FORTIFIED,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-04-Factory-Seconds-or-Unlabeled-Materials.pdf,General,Technical Bulletin,file,use_of_factory_seconds_or_unlabeled_material_in_fortified.pdf,Short description …
24
+ 23,FORTIFIED Requirements for Cement and Clay Hip and Ridge Tile Installed Over Asphalt Shingle Roof Cover,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-05-Ridge-or-Hip-Tiles-with-Asphalt-Shingles.pdf,General,Technical Bulletin,file,fortified_requirements_for_cement_and_clay_hip_and_ridge_tile_installed_over_asphalt_shingle_roof_cover.pdf,Short description …
25
+ 24,2-ply Synthetic Underlayment is an Approved Sealed Roof Deck Option on Asphalt Shingle and Metal Roofs,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-06-2-Ply-Synthetic-SRD-Method.pdf,General,Technical Bulletin,file,2_ply_synthetic_underlayment_is_an_approved_sealed_roof_deck_option_on_asphalt_shingle_and_metal_roofs.pdf,Short description …
26
+ 25,FORTIFIED Home Requirements for Homes with Excessive Gaps Between Wood Decking Boards,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023_07-Gapped-Decking.pdf,General,Technical Bulletin,file,fortified_home_requirements_for_homes_with_excessive_gaps_between_wood_decking_boards.pdf,Short description …
27
+ 26,Eligibility Requirements for Homes Constructed to the HUD Code,https://fortifiedhome.org/wp-content/uploads/TB_2023_08-Eligibility-of-HUD-Homes.pdf,General,Technical Bulletin,file,eligibility_requirements_for_homes_constructed_to_the_hud_code.pdf,Short description …
28
+ 27,FORTIFIED Guidance on Leaf Guards and Gutters,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-09-Leaf-Guards-and-Gutters.pdf,General,Technical Bulletin,file,fortified_guidance_on_leaf_guards_and_gutters.pdf,Short description …
29
+ 28,Redesignation Policy Update,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-10-Redesignation-Policy-Update.pdf,General,Technical Bulletin,file,redesignation_policy_update.pdf,Short description …
30
+ 29,Companion Details for Roll Widths Greater than 36 Inches,https://fortifiedhome.org/wp-content/uploads/Fastening-Synthetic-Underlayment.pdf,General,Technical Bulletin,file,companion_details_for_roll_widths_greater_than_36_inches.pdf,Short description …
31
+ 30,Underlayment Fastening for Taped Seams and Underlayment Sealed Roof Deck Installations Using Rolls Wider Than 36 Inches,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-11-Underlayment-Fastening.pdf,General,Technical Bulletin,file,underlayment_fastening_for_taped_seams_and_underlayment_sealed_roof_deck_installations_using_rolls_wider_than_36_inches.pdf,Short description …
32
+ 31, FORTIFIED Home Requirements for Elevated Roof-Mounted HVAC Units,https://fortifiedhome.org/wp-content/uploads/TB_FH_2024-01-Elevated-Roof-Mounted-HVAC-Units.pdf,Sealed Roof Deck,Technical Bulletin,file,fortified_home_requirements_for_elevated_roof_mounted_hvac_units.pdf,Short description …
33
+ 32,FORTIFIED Guidance ‚Äì Rain Diverters,https://fortifiedhome.org/wp-content/uploads/TB_FH_2024-02-Rain-Diverters.pdf,Sealed Roof Deck,Technical Bulletin,file,fortified_guidance_rain_diverters.pdf,Short description …
34
+ 33,Dog Doors in FORTIFIED Homes,https://fortifiedhome.org/wp-content/uploads/FH-2024-03-Dog-Doors-in-FORTIFIED-Homes.pdf,General,Technical Bulletin,file,dog_doors_in_fortified_homes.pdf,Short description …
35
+ 34,Modular Home Starter Kit ‚Äì Full Set,https://fortifiedhome.org/wp-content/uploads/Modular-Home-Toolkit-2024.pdf,General,Modular Home,file,modular_home_starter_kit_full_set.pdf,Short description …
36
+ 35,Post Storm Audit Processes,https://fortifiedhome.org/wp-content/uploads/Post-Storm-Audit-Processes.pdf,list_only,Technical Bulletin,file,post_storm_audit_processes.pdf,Short description …
37
+ 36,FORTIFIED Roofing Contractor Handbook,https://fortifiedhome.org/wp-content/uploads/FORTIFIED_Roof_Contractor_Handbook.pdf,General,Hankbook,file,fortified_roofing_contractor_handbook.pdf,Short description …
38
+ 37,FORTIFIED Evaluator Handbook,https://fortifiedhome.org/wp-content/uploads/FORTIFIED_Home_Evaluator_Handbook.pdf,General,Hankbook,file,fortified_evaluator_handbook.pdf,Short description …
39
+ 38,FORTIFIED Professional Handbook,https://fortifiedhome.org/wp-content/uploads/FORTIFIED-Home-Professional-Handbook.pdf,General,Hankbook,file,fortified_professional_handbook.pdf,Short description …
40
+ 39,Certified FORTIFIED Roofing Contractor Agreement,https://fortifiedhome.org/wp-content/uploads/FORTIFIED-Roofing-Contractor-Agreement_5-25-22.pdf,General,Agreement,file,certified_fortified_roofing_contractor_agreement.pdf,Short description …
41
+ 40,Certified FORTIFIED Evaluator Agreement,https://fortifiedhome.org/wp-content/uploads/FORTIFIED_Home_Evaluator-Agreement.pdf,General,Agreement,file,certified_fortified_evaluator_agreement.pdf,Short description …
42
+ 41,FORTIFIED Professional Agreement,https://fortifiedhome.org/wp-content/uploads/FORTIFIED-Professional-Agreement.pdf,General,Agreement,file,fortified_professional_agreement.pdf,Short description …
43
+ 42,What to Do After a Hurricane,https://fortifiedhome.org/article/what-to-do-after-a-hurricane/,General,Article,web,,Short description …
44
+ 43,Extend the Life of Your Roof,https://fortifiedhome.org/article/extend-the-life-of-your-roof/,General,Article,web,,Short description …
45
+ 44,How to Select a Roofing Contractor,https://fortifiedhome.org/article/how-to-select-a-roofing-contractor/,General,Article,web,,Short description …
46
+ 45,The Next Line of Defense,https://fortifiedhome.org/article/the-next-line-of-defense/,General,Article,web,,Short description …
47
+ 46,9 Questions You Should Absolutely Ask your Roofer,https://fortifiedhome.org/article/9-questions-you-should-absolutely-ask-your-roofer/,General,Article,web,,Short description …
48
+ 47,Strengthen Your Home Against Hurricanes and Severe Weather,https://fortifiedhome.org/article/strengthen-your-home-against-hurricanes-and-severe-weather/,General,Article,web,,Short description …
49
+ 48,8 Ways to Know If You Need a New Roof,https://fortifiedhome.org/article/8-ways-to-know-if-you-need-a-new-roof/,General,Article,web,,Short description …
50
+ 49,Defend Against Frozen Pipes,https://fortifiedhome.org/article/preventing-frozen-pipes/,General,Article,web,,Short description …
51
+ 50,Hail Protection That Works,https://fortifiedhome.org/article/hail-protection-that-works/,General,Article,web,,Short description …
52
+ 51,Shut the Door,https://fortifiedhome.org/article/shut-the-door/,General,Article,web,,Short description …
53
+ 52,Don t Jeopardize Your Home s Resilience,https://fortifiedhome.org/article/dont-jeopardize-your-homes-resilience/,General,Article,web,,Short description …
54
+ 53, Resilience gains ground as storms increase in frequency and intensity,https://fortifiedhome.org/article/resilience-gains-ground-as-storms-increase-in-frequency-and-intensity/,General,Article,web,,Short description …
55
+ 54,Reinforce Your Property Against Hurricanes,https://fortifiedhome.org/article/reinforce-your-property-against-hurricanes/,General,Article,web,,Short description …
56
+ 55,Can You Weather the Cost of The Next Storm?,https://fortifiedhome.org/article/can-you-weather-the-cost-of-the-next-storm/,General,Article,web,,Short description …
57
+ 56,Is Your Building Code Leaving You Vulnerable to Severe Weather?,https://fortifiedhome.org/article/is-your-building-code-leaving-you-vulnerable-to-severe-weather/,General,Article,web,,Short description …
58
+ 57,Renewing Your Designation,https://fortifiedhome.org/renew-your-designation/,General,Article,web,,Short description …
59
+ 58,How to Become a FORTIFIED Certified Service Provider,https://fortifiedhome.org/how-to-become-fortified-certified/,General,Article,web,,Short description …
60
+ 59,Frequently Asked Questions,https://fortifiedhome.org/frequently-asked-questions/,General,FAQ,web,,Short description …
main.py ADDED
@@ -0,0 +1,222 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio
2
+ from langchain_openai import OpenAIEmbeddings
3
+ from langchain_openai import ChatOpenAI
4
+ from langchain_chroma import Chroma
5
+ import chromadb
6
+ from chromadb.config import Settings
7
+ from langchain_core.prompts import ChatPromptTemplate
8
+ from typing import List
9
+
10
+ from langchain_core.documents import Document
11
+ from langchain_core.runnables import RunnablePassthrough
12
+ from langchain_core.output_parsers import XMLOutputParser
13
+ import gradio as gr
14
+ import pandas as pd
15
+ import logging
16
+ from langchain_core.exceptions import OutputParserException
17
+ from langchain_core.prompts import MessagesPlaceholder
18
+ import os
19
+ import uuid
20
+ import uuid
21
+ import hashlib
22
+
23
+ from langchain_community.chat_message_histories import ChatMessageHistory
24
+ from langchain_core.chat_history import BaseChatMessageHistory
25
+ from langchain_core.messages import (
26
+ AIMessage,
27
+ HumanMessage,
28
+ SystemMessage,
29
+ ToolMessage,
30
+ trim_messages,
31
+ )
32
+
33
+
34
+ # Constants
35
+ PERSIST_DIRECTORY = "chroma_store"
36
+ K_VALUE = 5
37
+ store = {}
38
+
39
+ xml_system = """You are a helpful AI assistant for the FORTIFIED program, a voluntary initiative for construction
40
+ and re-roofing designed to strengthen homes and commercial buildings against severe weather threats, including
41
+ high winds, hail, hurricanes, and tornadoes. Your users may include homeowners, insurance agents, realtors,
42
+ design professionals and construction professionals. When given a user question, you should consult the provided
43
+ technical and general documents containing FORTIFIED standards, instructions, and program information to deliver
44
+ accurate and relevant responses. If the information necessary to answer the inquiry is not available within these
45
+ documents, simply respond: ‘There is no information relevant to your inquiry in our current resources.
46
+ Please contact FORTIFIED customer support for further assistance.’
47
+ Remember, you must return both an answer and citations. A citation consists of a VERBATIM quote that
48
+ justifies the answer and the ID and also Source Name of the quote article. Return a citation for every quote across all articles
49
+ that justify the answer. Use the following format for your final output:
50
+ <cited_answer>
51
+ <answer></answer>
52
+ <citations>
53
+ <citation><source_id></source_id><source></source><quote></quote></citation>
54
+ <citation><source_id></source_id><source></source><quote></quote></citation>
55
+ ...
56
+ </citations>
57
+ </cited_answer>
58
+ Here are the articles:{context}"""
59
+
60
+ xml_prompt = ChatPromptTemplate.from_messages([("system", xml_system), ("human", "{input}")])
61
+
62
+ llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
63
+
64
+ contextualize_q_system_prompt = (
65
+ "Given a chat history and the latest user question "
66
+ "which might reference context in the chat history, "
67
+ "formulate a standalone question which can be understood "
68
+ "without the chat history. Do NOT answer the question, "
69
+ "just reformulate it if needed and otherwise return it as is."
70
+ )
71
+
72
+ contextualize_q_prompt = ChatPromptTemplate.from_messages(
73
+ [
74
+ ("system", contextualize_q_system_prompt),
75
+ MessagesPlaceholder("chat_history"),
76
+ ("human", "{input}"),
77
+ ]
78
+ )
79
+
80
+ runnable = contextualize_q_prompt | llm
81
+
82
+ def format_docs_xml(docs: List[Document]) -> str:
83
+ formatted_docs = [
84
+ f"<source id=\"{i}\">\n<source>{doc.metadata['source']}</source>\n<article_snippet>{doc.page_content}</article_snippet>\n</source>"
85
+ for i, doc in enumerate(docs)
86
+ ]
87
+ return f"\n\n<sources>\n{chr(10).join(formatted_docs)}\n</sources>"
88
+
89
+
90
+
91
+ rag_chain_from_docs = (
92
+ RunnablePassthrough.assign(context=(lambda x: format_docs_xml(x["context"])))
93
+ | xml_prompt
94
+ | llm
95
+ | XMLOutputParser()
96
+ )
97
+ settings = Settings(persist_directory=PERSIST_DIRECTORY)
98
+ vectordb = Chroma(embedding_function=OpenAIEmbeddings(), persist_directory=PERSIST_DIRECTORY)
99
+ retriever = vectordb.as_retriever(search_type="mmr", search_kwargs={"k": K_VALUE})
100
+ retrieve_docs = (lambda x: x["input"]) | retriever
101
+ chain = RunnablePassthrough.assign(context=retrieve_docs).assign(
102
+ answer=rag_chain_from_docs
103
+ )
104
+
105
+ def get_article_info(df, file_name):
106
+ if ".pdf" in file_name:
107
+ title = df[df["FileName"] == file_name]["Title"].iloc[0]
108
+ link = df[df["FileName"] == file_name]["Link"].iloc[0]
109
+ else:
110
+ title = df[df["Link"] == file_name]["Title"].iloc[0]
111
+ link = file_name
112
+ return title, link
113
+
114
+
115
+ df = pd.read_csv("articles_db.csv")
116
+
117
+
118
+ def vectordb_search(query):
119
+ titles, links = [], []
120
+ question_search = retriever.invoke(query)
121
+ for item in question_search:
122
+ edited_item = item.metadata["source"].replace("Articles/", "")
123
+ title, link = get_article_info(df, edited_item)
124
+ if title not in titles:
125
+ titles.append(title)
126
+ if link not in links:
127
+ links.append(link)
128
+ return "\n".join([f"- [{title}]({link})" for title, link in zip(titles, links)])
129
+
130
+ def get_session_history(session_id: str) -> BaseChatMessageHistory:
131
+ if session_id not in store:
132
+ store[session_id] = ChatMessageHistory()
133
+ return store[session_id]
134
+
135
+
136
+ def generate_unique_string():
137
+ mac = uuid.getnode()
138
+ mac_str = f'{mac:012x}'
139
+ try:
140
+ with open('/proc/cpuinfo') as f:
141
+ cpu_info = f.read()
142
+ except FileNotFoundError:
143
+ cpu_info = str(uuid.getnode())
144
+ unique_string = hashlib.sha256((mac_str + cpu_info).encode()).hexdigest()
145
+ return unique_string
146
+
147
+ def llm_response(query):
148
+ titles, links, res_titles, res_links = [], [], [], []
149
+ unique_id = generate_unique_string()
150
+ config = {"configurable": {"thread_id": unique_id }}
151
+ try:
152
+ filtered_history = trim_messages(get_session_history(unique_id).messages, strategy="last", token_counter=len, max_tokens=5,
153
+ start_on="human", end_on=("human", "tool"), include_system=True,)
154
+ modified_query = runnable.invoke({"input": query, "chat_history": filtered_history}).content
155
+ result = chain.invoke({"input": modified_query}, config=config)
156
+ answer = result['answer']['cited_answer'][0]["answer"]
157
+ history = get_session_history(unique_id)
158
+ history.add_user_message(modified_query)
159
+ history.add_ai_message(answer)
160
+ citations = result['answer']['cited_answer'][1]['citations']
161
+ for citation in citations:
162
+ edited_item = citation['citation'][1]["source"].replace("Articles/", "")
163
+ title, link = get_article_info(df, edited_item)
164
+ if title not in titles:
165
+ titles.append(title)
166
+ if link not in links:
167
+ links.append(link)
168
+ question_search = retriever.invoke(query)
169
+ for res_item in question_search:
170
+ edited_item = res_item.metadata["source"].replace("Articles/", "")
171
+ res_title, res_link = get_article_info(df, edited_item)
172
+ if res_title not in res_titles:
173
+ res_titles.append(res_title)
174
+ if res_link not in res_links:
175
+ res_links.append(res_link)
176
+
177
+ # Build the answer with superscript citations
178
+ answer_with_citations = f"{answer}"
179
+ for i, (title, link) in enumerate(zip(titles, links), start=1):
180
+ answer_with_citations += f" <sup>[[{i}]({link})]</sup> " # Append superscript citation numbers to the answer text
181
+
182
+ # Build the references section with clickable links
183
+ citations_section = "\n\nCitations:\n" + "\n".join(
184
+ [f"[{i}]: [{title}]({link})" for i, (title, link) in enumerate(zip(titles, links), start=1)]
185
+ )
186
+ # Combine answer and citations for final markdown output
187
+ markdown_list = f"{answer_with_citations}{citations_section}"
188
+ markdown_list += f"\n\n\nHere is a list of resources that can provide more information about your inquiry:\n"
189
+ markdown_list += "\n".join(
190
+ [f"- [{res_title}]({res_link})" for res_title, res_link in zip(res_titles, res_links)])
191
+ except OutputParserException:
192
+ markdown_list = "There is no information relevant to your inquiry in my current resources. Please contact [FORTIFIED customer support](https://fortifiedhome.org/contact/) for further assistance."
193
+ return markdown_list
194
+
195
+
196
+
197
+ with gr.Blocks() as fortified:
198
+ gr.Markdown("## FORTIFIED AI Assistant!")
199
+ gr.Markdown("### I'll try to answer any questions related to FORTIFIED program. Tell me what's on your mind?")
200
+ with gr.Row():
201
+ with gr.Column():
202
+ chatbot = gr.Chatbot(type="messages", height=400)
203
+ msg = gr.Textbox(label="Hit the Enter to send your question", placeholder="What's on your mind?", show_copy_button=True)
204
+ send = gr.Button("Send")
205
+
206
+ def user(user_message, history: list):
207
+ return "", history + [{"role": "user", "content": user_message}]
208
+
209
+ def bot(history: list):
210
+ bot_message = llm_response(history[-1]['content'])
211
+ history.append({"role": "assistant", "content": ""})
212
+ for character in bot_message:
213
+ history[-1]['content'] += character
214
+ yield history
215
+
216
+
217
+ msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(bot, chatbot, chatbot)
218
+ send.click(user, [msg, chatbot], [msg, chatbot], queue=False).then(bot, chatbot, chatbot)
219
+
220
+ if __name__ == "__main__":
221
+ fortified.launch(share=True)
222
+ gradio.deploy()
requirements.txt ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pandas
2
+ langchain
3
+ langchain-community
4
+ python-dotenv
5
+ langchain_ollama
6
+ langchain-openai
7
+ tiktoken
8
+ langgraph
9
+ beautifulsoup4
10
+ tqdm
11
+ ipywidgets
12
+ widgetsnbextension
13
+ pandas-profiling
14
+ pypdf
15
+ gradio
16
+ gradio_log
17
+ langchain_chroma
18
+ defusedxml
19
+ fonttools
20
+ lark