Spaces:
Running
Running
Hesam Fallahian commited on
Commit ·
87a356d
1
Parent(s): f2973b9
initial commit
Browse files- .gitignore +130 -0
- articles_db.csv +60 -0
- main.py +222 -0
- requirements.txt +20 -0
.gitignore
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C extensions
|
| 7 |
+
*.so
|
| 8 |
+
|
| 9 |
+
# Distribution / packaging
|
| 10 |
+
.Python
|
| 11 |
+
build/
|
| 12 |
+
develop-eggs/
|
| 13 |
+
dist/
|
| 14 |
+
downloads/
|
| 15 |
+
eggs/
|
| 16 |
+
.eggs/
|
| 17 |
+
lib/
|
| 18 |
+
lib64/
|
| 19 |
+
parts/
|
| 20 |
+
sdist/
|
| 21 |
+
var/
|
| 22 |
+
wheels/
|
| 23 |
+
pip-wheel-metadata/
|
| 24 |
+
share/python-wheels/
|
| 25 |
+
*.egg-info/
|
| 26 |
+
.installed.cfg
|
| 27 |
+
*.egg
|
| 28 |
+
MANIFEST
|
| 29 |
+
|
| 30 |
+
# PyInstaller
|
| 31 |
+
# Usually these files are written by a python script from a template
|
| 32 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 33 |
+
*.manifest
|
| 34 |
+
*.spec
|
| 35 |
+
|
| 36 |
+
# Installer logs
|
| 37 |
+
pip-log.txt
|
| 38 |
+
pip-delete-this-directory.txt
|
| 39 |
+
|
| 40 |
+
# Unit test / coverage reports
|
| 41 |
+
htmlcov/
|
| 42 |
+
.tox/
|
| 43 |
+
.nox/
|
| 44 |
+
.coverage
|
| 45 |
+
.coverage.*
|
| 46 |
+
.cache
|
| 47 |
+
nosetests.xml
|
| 48 |
+
coverage.xml
|
| 49 |
+
*.cover
|
| 50 |
+
*.py,cover
|
| 51 |
+
.hypothesis/
|
| 52 |
+
.pytest_cache/
|
| 53 |
+
|
| 54 |
+
# Translations
|
| 55 |
+
*.mo
|
| 56 |
+
*.pot
|
| 57 |
+
|
| 58 |
+
# Django stuff:
|
| 59 |
+
*.log
|
| 60 |
+
local_settings.py
|
| 61 |
+
db.sqlite3
|
| 62 |
+
db.sqlite3-journal
|
| 63 |
+
|
| 64 |
+
# Flask stuff:
|
| 65 |
+
instance/
|
| 66 |
+
.webassets-cache
|
| 67 |
+
|
| 68 |
+
# Scrapy stuff:
|
| 69 |
+
.scrapy
|
| 70 |
+
|
| 71 |
+
# Sphinx documentation
|
| 72 |
+
docs/_build/
|
| 73 |
+
|
| 74 |
+
# PyBuilder
|
| 75 |
+
target/
|
| 76 |
+
|
| 77 |
+
# Jupyter Notebook
|
| 78 |
+
.ipynb_checkpoints
|
| 79 |
+
|
| 80 |
+
# IPython
|
| 81 |
+
profile_default/
|
| 82 |
+
ipython_config.py
|
| 83 |
+
|
| 84 |
+
# pyenv
|
| 85 |
+
.python-version
|
| 86 |
+
|
| 87 |
+
# pipenv
|
| 88 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
| 89 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
| 90 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
| 91 |
+
# install all needed dependencies.
|
| 92 |
+
#Pipfile.lock
|
| 93 |
+
|
| 94 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
| 95 |
+
__pypackages__/
|
| 96 |
+
|
| 97 |
+
# Celery stuff
|
| 98 |
+
celerybeat-schedule
|
| 99 |
+
celerybeat.pid
|
| 100 |
+
|
| 101 |
+
# SageMath parsed files
|
| 102 |
+
*.sage.py
|
| 103 |
+
|
| 104 |
+
# Environments
|
| 105 |
+
.env
|
| 106 |
+
.venv
|
| 107 |
+
env/
|
| 108 |
+
venv/
|
| 109 |
+
ENV/
|
| 110 |
+
env.bak/
|
| 111 |
+
venv.bak/
|
| 112 |
+
|
| 113 |
+
# Spyder project settings
|
| 114 |
+
.spyderproject
|
| 115 |
+
.spyproject
|
| 116 |
+
|
| 117 |
+
# Rope project settings
|
| 118 |
+
.ropeproject
|
| 119 |
+
|
| 120 |
+
# mkdocs documentation
|
| 121 |
+
/site
|
| 122 |
+
|
| 123 |
+
# mypy
|
| 124 |
+
.mypy_cache/
|
| 125 |
+
.dmypy.json
|
| 126 |
+
dmypy.json
|
| 127 |
+
|
| 128 |
+
# Pyre type checker
|
| 129 |
+
.pyre/
|
| 130 |
+
.idea
|
articles_db.csv
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
id,Title,Link,Context,Category,SourceType,FileName,Description
|
| 2 |
+
1,2020 FORTIFIED Standard,https://fortifiedhome.org/wp-content/uploads/2020-FORTIFIED-Home-Standard.pdf,Re-Roofing and Roof,Construction Standards,file,2020_fortified_standard.pdf,Short description …
|
| 3 |
+
2,Sealed Roof Deck Supplemental Deck Attachment,https://fortifiedhome.org/wp-content/uploads/2015-04_technical-bulletin_sealed-roof-deck-supplemental-deck-attachment.pdf,Sealed Roof Deck,Technical Bulletin,file,sealed_roof_deck_supplemental_deck_attachment.pdf,Short description …
|
| 4 |
+
3,Roof Flashing,https://fortifiedhome.org/wp-content/uploads/2017-01_technical-bulletin_flashing.pdf,Roof Flashing,Technical Bulletin,file,roof_flashing.pdf,Short description …
|
| 5 |
+
4,Metal Panel Roof Covering Guidance,https://fortifiedhome.org/wp-content/uploads/2019-01_technical-bulletin_metal-roof-panel-selection_revised.pdf,Metal Panel and Roof,Technical Bulletin,file,metal_panel_roof_covering_guidance.pdf,Short description …
|
| 6 |
+
5,Design Pressure Guidance for Roof Coverings,https://fortifiedhome.org/wp-content/uploads/2020-01_technical-bulletin_design-pressure-guidance-for-roof-coverings_revised.pdf,Roof and Sealed Roof Deck,Technical Bulletin,file,design_pressure_guidance_for_roof_coverings.pdf,Short description …
|
| 7 |
+
6,Garage Door Requirements,https://fortifiedhome.org/wp-content/uploads/2021-01_technical-bulletin_garage-door-requirements.pdf,General,Technical Bulletin,file,garage_door_requirements.pdf,Short description …
|
| 8 |
+
7,PA - Vycor Product Advisory,https://fortifiedhome.org/wp-content/uploads/Vycor_Product_Advisory.pdf,General,Technical Bulletin,file,pa_vycor_product_advisory.pdf,Short description …
|
| 9 |
+
8,Corrosion Resistant Fasteners,https://fortifiedhome.org/wp-content/uploads/TB_FH_2021-02.pdf,"New Roof, Re-Roofing, Retrofit, and Roof",Technical Bulletin,file,corrosion_resistant_fasteners.pdf,Short description …
|
| 10 |
+
9,Sealed Roof Deck for Wood Shake and Shingle Roof Systems,https://fortifiedhome.org/wp-content/uploads/TB-FH-2021-03_SRD_options_for_wood_shake.pdf,"Re-Roofing, Retrofit, and Roof",Technical Bulletin,file,sealed_roof_deck_for_wood_shake_and_shingle_roof_systems.pdf,Short description …
|
| 11 |
+
10,IBHS Guidance Choosing the Right Tape,https://fortifiedhome.org/wp-content/uploads/Choosing-the-Right-Tape_FINAL.pdf,Roof,Technical Bulletin,file,ibhs_guidance_choosing_the_right_tape.pdf,Short description …
|
| 12 |
+
11,Roof Sheathing Nail Pattern Documentation Requirements,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022_01-Roof-Sheathing-Attachment-Documentation.pdf,Roof,Technical Bulletin,file,roof_sheathing_nail_pattern_documentation_requirements.pdf,Short description …
|
| 13 |
+
12,FORTIFIED Roof Identification,https://fortifiedhome.org/wp-content/uploads/FORTIFIED-ROOF-IDENTIFICATIONR10.pdf,Roof,Technical Bulletin,file,fortified_roof_identification.pdf,Short description …
|
| 14 |
+
13,The FORTIFIED Definition of Roof,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022-02-Definition-of-Roof.pdf,Roof,Technical Bulletin,file,the_fortified_definition_of_roof.pdf,Short description …
|
| 15 |
+
14,FORTIFIED Home Requirements for Elevated Roof-Mounted Decks,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022-03-Elevated-Rooftop-Decks.pdf,Roof and Sealed Roof Deck,Technical Bulletin,file,fortified_home_requirements_for_elevated_roof_mounted_decks.pdf,Short description …
|
| 16 |
+
15,Product Substitution Due to Supply Chain Issues and Product Availability,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022_04-Supply-Chain-Issues.pdf,Roof,Technical Bulletin,file,product_substitution_due_to_supply_chain_issues_and_product_availability.pdf,Short description …
|
| 17 |
+
16,Requirements for Re-Roofing Over Existing Self-Adhered Membranes,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022-05-Reroofing-Over-Self-Adhered-Membranes.pdf,General,Technical Bulletin,file,requirements_for_re_roofing_over_existing_self_adhered_membranes.pdf,Short description …
|
| 18 |
+
17,Foundation Requirements for FORTIFIED Home Eligibility,https://fortifiedhome.org/wp-content/uploads/TB_FH_2022-06-Foundation-Requirements-FORTIFIED-Home.pdf,Roof,Technical Bulletin,file,foundation_requirements_for_fortified_home_eligibility.pdf,Short description …
|
| 19 |
+
18,Removal of Egress Requirement for Entry Doors,https://fortifiedhome.org/wp-content/uploads/TB-Update-Removal-of-Egress-Requirement.pdf,Roof,Technical Bulletin,file,removal_of_egress_requirement_for_entry_doors.pdf,Short description …
|
| 20 |
+
19,Product Advisory FH Eligibility of Existing Fiberglass Resin Deck Coatings as Roof Covers,https://fortifiedhome.org/wp-content/uploads/PA_FH_2023-01-Fiberglass-resin-deck-coatings-as-roof-covers.pdf,General,Technical Bulletin,file,product_advisory_fh_eligibility_of_existing_fiberglass_resin_deck_coatings_as_roof_covers.pdf,Short description …
|
| 21 |
+
20,Fastener Type Requirements for Asphalt Shingles and Drip Edge,https://fortifiedhome.org/wp-content/uploads/TB-Update-Fastener-Requirements-for-Asphalt-Shingles-and-Drip-Edge.pdf,Roof,Technical Bulletin,file,fastener_type_requirements_for_asphalt_shingles_and_drip_edge.pdf,Short description …
|
| 22 |
+
21,Eligibility and Compliance of Accessory Roof Structures,https://fortifiedhome.org/wp-content/uploads/TB-2023-03_Eligibility-and-Compliance-Accessory-Roof-Structures.pdf,General,Technical Bulletin,file,eligibility_and_compliance_of_accessory_roof_structures.pdf,Short description …
|
| 23 |
+
22,Use of Factory Seconds or Unlabeled Material in FORTIFIED,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-04-Factory-Seconds-or-Unlabeled-Materials.pdf,General,Technical Bulletin,file,use_of_factory_seconds_or_unlabeled_material_in_fortified.pdf,Short description …
|
| 24 |
+
23,FORTIFIED Requirements for Cement and Clay Hip and Ridge Tile Installed Over Asphalt Shingle Roof Cover,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-05-Ridge-or-Hip-Tiles-with-Asphalt-Shingles.pdf,General,Technical Bulletin,file,fortified_requirements_for_cement_and_clay_hip_and_ridge_tile_installed_over_asphalt_shingle_roof_cover.pdf,Short description …
|
| 25 |
+
24,2-ply Synthetic Underlayment is an Approved Sealed Roof Deck Option on Asphalt Shingle and Metal Roofs,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-06-2-Ply-Synthetic-SRD-Method.pdf,General,Technical Bulletin,file,2_ply_synthetic_underlayment_is_an_approved_sealed_roof_deck_option_on_asphalt_shingle_and_metal_roofs.pdf,Short description …
|
| 26 |
+
25,FORTIFIED Home Requirements for Homes with Excessive Gaps Between Wood Decking Boards,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023_07-Gapped-Decking.pdf,General,Technical Bulletin,file,fortified_home_requirements_for_homes_with_excessive_gaps_between_wood_decking_boards.pdf,Short description …
|
| 27 |
+
26,Eligibility Requirements for Homes Constructed to the HUD Code,https://fortifiedhome.org/wp-content/uploads/TB_2023_08-Eligibility-of-HUD-Homes.pdf,General,Technical Bulletin,file,eligibility_requirements_for_homes_constructed_to_the_hud_code.pdf,Short description …
|
| 28 |
+
27,FORTIFIED Guidance on Leaf Guards and Gutters,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-09-Leaf-Guards-and-Gutters.pdf,General,Technical Bulletin,file,fortified_guidance_on_leaf_guards_and_gutters.pdf,Short description …
|
| 29 |
+
28,Redesignation Policy Update,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-10-Redesignation-Policy-Update.pdf,General,Technical Bulletin,file,redesignation_policy_update.pdf,Short description …
|
| 30 |
+
29,Companion Details for Roll Widths Greater than 36 Inches,https://fortifiedhome.org/wp-content/uploads/Fastening-Synthetic-Underlayment.pdf,General,Technical Bulletin,file,companion_details_for_roll_widths_greater_than_36_inches.pdf,Short description …
|
| 31 |
+
30,Underlayment Fastening for Taped Seams and Underlayment Sealed Roof Deck Installations Using Rolls Wider Than 36 Inches,https://fortifiedhome.org/wp-content/uploads/TB_FH_2023-11-Underlayment-Fastening.pdf,General,Technical Bulletin,file,underlayment_fastening_for_taped_seams_and_underlayment_sealed_roof_deck_installations_using_rolls_wider_than_36_inches.pdf,Short description …
|
| 32 |
+
31, FORTIFIED Home Requirements for Elevated Roof-Mounted HVAC Units,https://fortifiedhome.org/wp-content/uploads/TB_FH_2024-01-Elevated-Roof-Mounted-HVAC-Units.pdf,Sealed Roof Deck,Technical Bulletin,file,fortified_home_requirements_for_elevated_roof_mounted_hvac_units.pdf,Short description …
|
| 33 |
+
32,FORTIFIED Guidance ‚Äì Rain Diverters,https://fortifiedhome.org/wp-content/uploads/TB_FH_2024-02-Rain-Diverters.pdf,Sealed Roof Deck,Technical Bulletin,file,fortified_guidance_rain_diverters.pdf,Short description …
|
| 34 |
+
33,Dog Doors in FORTIFIED Homes,https://fortifiedhome.org/wp-content/uploads/FH-2024-03-Dog-Doors-in-FORTIFIED-Homes.pdf,General,Technical Bulletin,file,dog_doors_in_fortified_homes.pdf,Short description …
|
| 35 |
+
34,Modular Home Starter Kit ‚Äì Full Set,https://fortifiedhome.org/wp-content/uploads/Modular-Home-Toolkit-2024.pdf,General,Modular Home,file,modular_home_starter_kit_full_set.pdf,Short description …
|
| 36 |
+
35,Post Storm Audit Processes,https://fortifiedhome.org/wp-content/uploads/Post-Storm-Audit-Processes.pdf,list_only,Technical Bulletin,file,post_storm_audit_processes.pdf,Short description …
|
| 37 |
+
36,FORTIFIED Roofing Contractor Handbook,https://fortifiedhome.org/wp-content/uploads/FORTIFIED_Roof_Contractor_Handbook.pdf,General,Hankbook,file,fortified_roofing_contractor_handbook.pdf,Short description …
|
| 38 |
+
37,FORTIFIED Evaluator Handbook,https://fortifiedhome.org/wp-content/uploads/FORTIFIED_Home_Evaluator_Handbook.pdf,General,Hankbook,file,fortified_evaluator_handbook.pdf,Short description …
|
| 39 |
+
38,FORTIFIED Professional Handbook,https://fortifiedhome.org/wp-content/uploads/FORTIFIED-Home-Professional-Handbook.pdf,General,Hankbook,file,fortified_professional_handbook.pdf,Short description …
|
| 40 |
+
39,Certified FORTIFIED Roofing Contractor Agreement,https://fortifiedhome.org/wp-content/uploads/FORTIFIED-Roofing-Contractor-Agreement_5-25-22.pdf,General,Agreement,file,certified_fortified_roofing_contractor_agreement.pdf,Short description …
|
| 41 |
+
40,Certified FORTIFIED Evaluator Agreement,https://fortifiedhome.org/wp-content/uploads/FORTIFIED_Home_Evaluator-Agreement.pdf,General,Agreement,file,certified_fortified_evaluator_agreement.pdf,Short description …
|
| 42 |
+
41,FORTIFIED Professional Agreement,https://fortifiedhome.org/wp-content/uploads/FORTIFIED-Professional-Agreement.pdf,General,Agreement,file,fortified_professional_agreement.pdf,Short description …
|
| 43 |
+
42,What to Do After a Hurricane,https://fortifiedhome.org/article/what-to-do-after-a-hurricane/,General,Article,web,,Short description …
|
| 44 |
+
43,Extend the Life of Your Roof,https://fortifiedhome.org/article/extend-the-life-of-your-roof/,General,Article,web,,Short description …
|
| 45 |
+
44,How to Select a Roofing Contractor,https://fortifiedhome.org/article/how-to-select-a-roofing-contractor/,General,Article,web,,Short description …
|
| 46 |
+
45,The Next Line of Defense,https://fortifiedhome.org/article/the-next-line-of-defense/,General,Article,web,,Short description …
|
| 47 |
+
46,9 Questions You Should Absolutely Ask your Roofer,https://fortifiedhome.org/article/9-questions-you-should-absolutely-ask-your-roofer/,General,Article,web,,Short description …
|
| 48 |
+
47,Strengthen Your Home Against Hurricanes and Severe Weather,https://fortifiedhome.org/article/strengthen-your-home-against-hurricanes-and-severe-weather/,General,Article,web,,Short description …
|
| 49 |
+
48,8 Ways to Know If You Need a New Roof,https://fortifiedhome.org/article/8-ways-to-know-if-you-need-a-new-roof/,General,Article,web,,Short description …
|
| 50 |
+
49,Defend Against Frozen Pipes,https://fortifiedhome.org/article/preventing-frozen-pipes/,General,Article,web,,Short description …
|
| 51 |
+
50,Hail Protection That Works,https://fortifiedhome.org/article/hail-protection-that-works/,General,Article,web,,Short description …
|
| 52 |
+
51,Shut the Door,https://fortifiedhome.org/article/shut-the-door/,General,Article,web,,Short description …
|
| 53 |
+
52,Don t Jeopardize Your Home s Resilience,https://fortifiedhome.org/article/dont-jeopardize-your-homes-resilience/,General,Article,web,,Short description …
|
| 54 |
+
53, Resilience gains ground as storms increase in frequency and intensity,https://fortifiedhome.org/article/resilience-gains-ground-as-storms-increase-in-frequency-and-intensity/,General,Article,web,,Short description …
|
| 55 |
+
54,Reinforce Your Property Against Hurricanes,https://fortifiedhome.org/article/reinforce-your-property-against-hurricanes/,General,Article,web,,Short description …
|
| 56 |
+
55,Can You Weather the Cost of The Next Storm?,https://fortifiedhome.org/article/can-you-weather-the-cost-of-the-next-storm/,General,Article,web,,Short description …
|
| 57 |
+
56,Is Your Building Code Leaving You Vulnerable to Severe Weather?,https://fortifiedhome.org/article/is-your-building-code-leaving-you-vulnerable-to-severe-weather/,General,Article,web,,Short description …
|
| 58 |
+
57,Renewing Your Designation,https://fortifiedhome.org/renew-your-designation/,General,Article,web,,Short description …
|
| 59 |
+
58,How to Become a FORTIFIED Certified Service Provider,https://fortifiedhome.org/how-to-become-fortified-certified/,General,Article,web,,Short description …
|
| 60 |
+
59,Frequently Asked Questions,https://fortifiedhome.org/frequently-asked-questions/,General,FAQ,web,,Short description …
|
main.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio
|
| 2 |
+
from langchain_openai import OpenAIEmbeddings
|
| 3 |
+
from langchain_openai import ChatOpenAI
|
| 4 |
+
from langchain_chroma import Chroma
|
| 5 |
+
import chromadb
|
| 6 |
+
from chromadb.config import Settings
|
| 7 |
+
from langchain_core.prompts import ChatPromptTemplate
|
| 8 |
+
from typing import List
|
| 9 |
+
|
| 10 |
+
from langchain_core.documents import Document
|
| 11 |
+
from langchain_core.runnables import RunnablePassthrough
|
| 12 |
+
from langchain_core.output_parsers import XMLOutputParser
|
| 13 |
+
import gradio as gr
|
| 14 |
+
import pandas as pd
|
| 15 |
+
import logging
|
| 16 |
+
from langchain_core.exceptions import OutputParserException
|
| 17 |
+
from langchain_core.prompts import MessagesPlaceholder
|
| 18 |
+
import os
|
| 19 |
+
import uuid
|
| 20 |
+
import uuid
|
| 21 |
+
import hashlib
|
| 22 |
+
|
| 23 |
+
from langchain_community.chat_message_histories import ChatMessageHistory
|
| 24 |
+
from langchain_core.chat_history import BaseChatMessageHistory
|
| 25 |
+
from langchain_core.messages import (
|
| 26 |
+
AIMessage,
|
| 27 |
+
HumanMessage,
|
| 28 |
+
SystemMessage,
|
| 29 |
+
ToolMessage,
|
| 30 |
+
trim_messages,
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# Constants
|
| 35 |
+
PERSIST_DIRECTORY = "chroma_store"
|
| 36 |
+
K_VALUE = 5
|
| 37 |
+
store = {}
|
| 38 |
+
|
| 39 |
+
xml_system = """You are a helpful AI assistant for the FORTIFIED program, a voluntary initiative for construction
|
| 40 |
+
and re-roofing designed to strengthen homes and commercial buildings against severe weather threats, including
|
| 41 |
+
high winds, hail, hurricanes, and tornadoes. Your users may include homeowners, insurance agents, realtors,
|
| 42 |
+
design professionals and construction professionals. When given a user question, you should consult the provided
|
| 43 |
+
technical and general documents containing FORTIFIED standards, instructions, and program information to deliver
|
| 44 |
+
accurate and relevant responses. If the information necessary to answer the inquiry is not available within these
|
| 45 |
+
documents, simply respond: ‘There is no information relevant to your inquiry in our current resources.
|
| 46 |
+
Please contact FORTIFIED customer support for further assistance.’
|
| 47 |
+
Remember, you must return both an answer and citations. A citation consists of a VERBATIM quote that
|
| 48 |
+
justifies the answer and the ID and also Source Name of the quote article. Return a citation for every quote across all articles
|
| 49 |
+
that justify the answer. Use the following format for your final output:
|
| 50 |
+
<cited_answer>
|
| 51 |
+
<answer></answer>
|
| 52 |
+
<citations>
|
| 53 |
+
<citation><source_id></source_id><source></source><quote></quote></citation>
|
| 54 |
+
<citation><source_id></source_id><source></source><quote></quote></citation>
|
| 55 |
+
...
|
| 56 |
+
</citations>
|
| 57 |
+
</cited_answer>
|
| 58 |
+
Here are the articles:{context}"""
|
| 59 |
+
|
| 60 |
+
xml_prompt = ChatPromptTemplate.from_messages([("system", xml_system), ("human", "{input}")])
|
| 61 |
+
|
| 62 |
+
llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
|
| 63 |
+
|
| 64 |
+
contextualize_q_system_prompt = (
|
| 65 |
+
"Given a chat history and the latest user question "
|
| 66 |
+
"which might reference context in the chat history, "
|
| 67 |
+
"formulate a standalone question which can be understood "
|
| 68 |
+
"without the chat history. Do NOT answer the question, "
|
| 69 |
+
"just reformulate it if needed and otherwise return it as is."
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
contextualize_q_prompt = ChatPromptTemplate.from_messages(
|
| 73 |
+
[
|
| 74 |
+
("system", contextualize_q_system_prompt),
|
| 75 |
+
MessagesPlaceholder("chat_history"),
|
| 76 |
+
("human", "{input}"),
|
| 77 |
+
]
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
runnable = contextualize_q_prompt | llm
|
| 81 |
+
|
| 82 |
+
def format_docs_xml(docs: List[Document]) -> str:
|
| 83 |
+
formatted_docs = [
|
| 84 |
+
f"<source id=\"{i}\">\n<source>{doc.metadata['source']}</source>\n<article_snippet>{doc.page_content}</article_snippet>\n</source>"
|
| 85 |
+
for i, doc in enumerate(docs)
|
| 86 |
+
]
|
| 87 |
+
return f"\n\n<sources>\n{chr(10).join(formatted_docs)}\n</sources>"
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
rag_chain_from_docs = (
|
| 92 |
+
RunnablePassthrough.assign(context=(lambda x: format_docs_xml(x["context"])))
|
| 93 |
+
| xml_prompt
|
| 94 |
+
| llm
|
| 95 |
+
| XMLOutputParser()
|
| 96 |
+
)
|
| 97 |
+
settings = Settings(persist_directory=PERSIST_DIRECTORY)
|
| 98 |
+
vectordb = Chroma(embedding_function=OpenAIEmbeddings(), persist_directory=PERSIST_DIRECTORY)
|
| 99 |
+
retriever = vectordb.as_retriever(search_type="mmr", search_kwargs={"k": K_VALUE})
|
| 100 |
+
retrieve_docs = (lambda x: x["input"]) | retriever
|
| 101 |
+
chain = RunnablePassthrough.assign(context=retrieve_docs).assign(
|
| 102 |
+
answer=rag_chain_from_docs
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
def get_article_info(df, file_name):
|
| 106 |
+
if ".pdf" in file_name:
|
| 107 |
+
title = df[df["FileName"] == file_name]["Title"].iloc[0]
|
| 108 |
+
link = df[df["FileName"] == file_name]["Link"].iloc[0]
|
| 109 |
+
else:
|
| 110 |
+
title = df[df["Link"] == file_name]["Title"].iloc[0]
|
| 111 |
+
link = file_name
|
| 112 |
+
return title, link
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
df = pd.read_csv("articles_db.csv")
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def vectordb_search(query):
|
| 119 |
+
titles, links = [], []
|
| 120 |
+
question_search = retriever.invoke(query)
|
| 121 |
+
for item in question_search:
|
| 122 |
+
edited_item = item.metadata["source"].replace("Articles/", "")
|
| 123 |
+
title, link = get_article_info(df, edited_item)
|
| 124 |
+
if title not in titles:
|
| 125 |
+
titles.append(title)
|
| 126 |
+
if link not in links:
|
| 127 |
+
links.append(link)
|
| 128 |
+
return "\n".join([f"- [{title}]({link})" for title, link in zip(titles, links)])
|
| 129 |
+
|
| 130 |
+
def get_session_history(session_id: str) -> BaseChatMessageHistory:
|
| 131 |
+
if session_id not in store:
|
| 132 |
+
store[session_id] = ChatMessageHistory()
|
| 133 |
+
return store[session_id]
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def generate_unique_string():
|
| 137 |
+
mac = uuid.getnode()
|
| 138 |
+
mac_str = f'{mac:012x}'
|
| 139 |
+
try:
|
| 140 |
+
with open('/proc/cpuinfo') as f:
|
| 141 |
+
cpu_info = f.read()
|
| 142 |
+
except FileNotFoundError:
|
| 143 |
+
cpu_info = str(uuid.getnode())
|
| 144 |
+
unique_string = hashlib.sha256((mac_str + cpu_info).encode()).hexdigest()
|
| 145 |
+
return unique_string
|
| 146 |
+
|
| 147 |
+
def llm_response(query):
|
| 148 |
+
titles, links, res_titles, res_links = [], [], [], []
|
| 149 |
+
unique_id = generate_unique_string()
|
| 150 |
+
config = {"configurable": {"thread_id": unique_id }}
|
| 151 |
+
try:
|
| 152 |
+
filtered_history = trim_messages(get_session_history(unique_id).messages, strategy="last", token_counter=len, max_tokens=5,
|
| 153 |
+
start_on="human", end_on=("human", "tool"), include_system=True,)
|
| 154 |
+
modified_query = runnable.invoke({"input": query, "chat_history": filtered_history}).content
|
| 155 |
+
result = chain.invoke({"input": modified_query}, config=config)
|
| 156 |
+
answer = result['answer']['cited_answer'][0]["answer"]
|
| 157 |
+
history = get_session_history(unique_id)
|
| 158 |
+
history.add_user_message(modified_query)
|
| 159 |
+
history.add_ai_message(answer)
|
| 160 |
+
citations = result['answer']['cited_answer'][1]['citations']
|
| 161 |
+
for citation in citations:
|
| 162 |
+
edited_item = citation['citation'][1]["source"].replace("Articles/", "")
|
| 163 |
+
title, link = get_article_info(df, edited_item)
|
| 164 |
+
if title not in titles:
|
| 165 |
+
titles.append(title)
|
| 166 |
+
if link not in links:
|
| 167 |
+
links.append(link)
|
| 168 |
+
question_search = retriever.invoke(query)
|
| 169 |
+
for res_item in question_search:
|
| 170 |
+
edited_item = res_item.metadata["source"].replace("Articles/", "")
|
| 171 |
+
res_title, res_link = get_article_info(df, edited_item)
|
| 172 |
+
if res_title not in res_titles:
|
| 173 |
+
res_titles.append(res_title)
|
| 174 |
+
if res_link not in res_links:
|
| 175 |
+
res_links.append(res_link)
|
| 176 |
+
|
| 177 |
+
# Build the answer with superscript citations
|
| 178 |
+
answer_with_citations = f"{answer}"
|
| 179 |
+
for i, (title, link) in enumerate(zip(titles, links), start=1):
|
| 180 |
+
answer_with_citations += f" <sup>[[{i}]({link})]</sup> " # Append superscript citation numbers to the answer text
|
| 181 |
+
|
| 182 |
+
# Build the references section with clickable links
|
| 183 |
+
citations_section = "\n\nCitations:\n" + "\n".join(
|
| 184 |
+
[f"[{i}]: [{title}]({link})" for i, (title, link) in enumerate(zip(titles, links), start=1)]
|
| 185 |
+
)
|
| 186 |
+
# Combine answer and citations for final markdown output
|
| 187 |
+
markdown_list = f"{answer_with_citations}{citations_section}"
|
| 188 |
+
markdown_list += f"\n\n\nHere is a list of resources that can provide more information about your inquiry:\n"
|
| 189 |
+
markdown_list += "\n".join(
|
| 190 |
+
[f"- [{res_title}]({res_link})" for res_title, res_link in zip(res_titles, res_links)])
|
| 191 |
+
except OutputParserException:
|
| 192 |
+
markdown_list = "There is no information relevant to your inquiry in my current resources. Please contact [FORTIFIED customer support](https://fortifiedhome.org/contact/) for further assistance."
|
| 193 |
+
return markdown_list
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
with gr.Blocks() as fortified:
|
| 198 |
+
gr.Markdown("## FORTIFIED AI Assistant!")
|
| 199 |
+
gr.Markdown("### I'll try to answer any questions related to FORTIFIED program. Tell me what's on your mind?")
|
| 200 |
+
with gr.Row():
|
| 201 |
+
with gr.Column():
|
| 202 |
+
chatbot = gr.Chatbot(type="messages", height=400)
|
| 203 |
+
msg = gr.Textbox(label="Hit the Enter to send your question", placeholder="What's on your mind?", show_copy_button=True)
|
| 204 |
+
send = gr.Button("Send")
|
| 205 |
+
|
| 206 |
+
def user(user_message, history: list):
|
| 207 |
+
return "", history + [{"role": "user", "content": user_message}]
|
| 208 |
+
|
| 209 |
+
def bot(history: list):
|
| 210 |
+
bot_message = llm_response(history[-1]['content'])
|
| 211 |
+
history.append({"role": "assistant", "content": ""})
|
| 212 |
+
for character in bot_message:
|
| 213 |
+
history[-1]['content'] += character
|
| 214 |
+
yield history
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(bot, chatbot, chatbot)
|
| 218 |
+
send.click(user, [msg, chatbot], [msg, chatbot], queue=False).then(bot, chatbot, chatbot)
|
| 219 |
+
|
| 220 |
+
if __name__ == "__main__":
|
| 221 |
+
fortified.launch(share=True)
|
| 222 |
+
gradio.deploy()
|
requirements.txt
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
pandas
|
| 2 |
+
langchain
|
| 3 |
+
langchain-community
|
| 4 |
+
python-dotenv
|
| 5 |
+
langchain_ollama
|
| 6 |
+
langchain-openai
|
| 7 |
+
tiktoken
|
| 8 |
+
langgraph
|
| 9 |
+
beautifulsoup4
|
| 10 |
+
tqdm
|
| 11 |
+
ipywidgets
|
| 12 |
+
widgetsnbextension
|
| 13 |
+
pandas-profiling
|
| 14 |
+
pypdf
|
| 15 |
+
gradio
|
| 16 |
+
gradio_log
|
| 17 |
+
langchain_chroma
|
| 18 |
+
defusedxml
|
| 19 |
+
fonttools
|
| 20 |
+
lark
|