Upload folder using huggingface_hub
Browse files- .gitignore +162 -0
- README.md +6 -8
- github urls.txt +1045 -0
- gradio_interface.py +58 -0
- main_rag.ipynb +0 -0
- output.md +50 -0
- pipeline.yml +113 -0
- playground/Data/Outputs/output.md +75 -0
- playground/Data/Test_Case/1.md +41 -0
- playground/Data/Test_Case/1.pdf +0 -0
- playground/Data/Test_Case/1.txt +6 -0
- playground/Data/Test_Case/2.md +41 -0
- playground/Data/Test_Case/2.pdf +0 -0
- playground/Data/Test_Case/2.txt +6 -0
- playground/Data/Test_Case/3.md +33 -0
- playground/Data/Test_Case/3.pdf +0 -0
- playground/Data/Test_Case/3.txt +6 -0
- playground/Data/Test_Case/4.md +32 -0
- playground/Data/Test_Case/4.pdf +0 -0
- playground/Data/Test_Case/4.txt +5 -0
- playground/Data/Test_Case/5.md +32 -0
- playground/Data/Test_Case/5.pdf +0 -0
- playground/Data/Test_Case/5.txt +6 -0
- playground/Data/Test_Case/6.md +33 -0
- playground/Data/Test_Case/6.pdf +0 -0
- playground/Data/Test_Case/6.txt +5 -0
- playground/Data/Test_Case/7.md +35 -0
- playground/Data/Test_Case/7.pdf +0 -0
- playground/Data/Test_Case/7.txt +6 -0
- playground/Data/Test_Case/8.md +34 -0
- playground/Data/Test_Case/8.pdf +0 -0
- playground/Data/Test_Case/8.txt +5 -0
- playground/doc_generator_test.ipynb +528 -0
- playground/git_scraper_test.ipynb +190 -0
- playground/simple_rag.ipynb +241 -0
- playground/url_to_db.ipynb +0 -0
- playground/wire_ask_docs.ipynb +870 -0
- requirements.txt +162 -0
- utils/github_scraper.py +152 -0
- utils/url_scraper.py +240 -0
.gitignore
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C extensions
|
| 7 |
+
*.so
|
| 8 |
+
|
| 9 |
+
# Distribution / packaging
|
| 10 |
+
.Python
|
| 11 |
+
build/
|
| 12 |
+
develop-eggs/
|
| 13 |
+
dist/
|
| 14 |
+
downloads/
|
| 15 |
+
eggs/
|
| 16 |
+
.eggs/
|
| 17 |
+
lib/
|
| 18 |
+
lib64/
|
| 19 |
+
parts/
|
| 20 |
+
sdist/
|
| 21 |
+
var/
|
| 22 |
+
wheels/
|
| 23 |
+
share/python-wheels/
|
| 24 |
+
*.egg-info/
|
| 25 |
+
.installed.cfg
|
| 26 |
+
*.egg
|
| 27 |
+
MANIFEST
|
| 28 |
+
|
| 29 |
+
# PyInstaller
|
| 30 |
+
# Usually these files are written by a python script from a template
|
| 31 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 32 |
+
*.manifest
|
| 33 |
+
*.spec
|
| 34 |
+
|
| 35 |
+
# Installer logs
|
| 36 |
+
pip-log.txt
|
| 37 |
+
pip-delete-this-directory.txt
|
| 38 |
+
|
| 39 |
+
# Unit test / coverage reports
|
| 40 |
+
htmlcov/
|
| 41 |
+
.tox/
|
| 42 |
+
.nox/
|
| 43 |
+
.coverage
|
| 44 |
+
.coverage.*
|
| 45 |
+
.cache
|
| 46 |
+
nosetests.xml
|
| 47 |
+
coverage.xml
|
| 48 |
+
*.cover
|
| 49 |
+
*.py,cover
|
| 50 |
+
.hypothesis/
|
| 51 |
+
.pytest_cache/
|
| 52 |
+
cover/
|
| 53 |
+
|
| 54 |
+
# Translations
|
| 55 |
+
*.mo
|
| 56 |
+
*.pot
|
| 57 |
+
|
| 58 |
+
# Django stuff:
|
| 59 |
+
*.log
|
| 60 |
+
local_settings.py
|
| 61 |
+
db.sqlite3
|
| 62 |
+
db.sqlite3-journal
|
| 63 |
+
|
| 64 |
+
# Flask stuff:
|
| 65 |
+
instance/
|
| 66 |
+
.webassets-cache
|
| 67 |
+
|
| 68 |
+
# Scrapy stuff:
|
| 69 |
+
.scrapy
|
| 70 |
+
|
| 71 |
+
# Sphinx documentation
|
| 72 |
+
docs/_build/
|
| 73 |
+
|
| 74 |
+
# PyBuilder
|
| 75 |
+
.pybuilder/
|
| 76 |
+
target/
|
| 77 |
+
|
| 78 |
+
# Jupyter Notebook
|
| 79 |
+
.ipynb_checkpoints
|
| 80 |
+
|
| 81 |
+
# IPython
|
| 82 |
+
profile_default/
|
| 83 |
+
ipython_config.py
|
| 84 |
+
|
| 85 |
+
# pyenv
|
| 86 |
+
# For a library or package, you might want to ignore these files since the code is
|
| 87 |
+
# intended to run in multiple environments; otherwise, check them in:
|
| 88 |
+
# .python-version
|
| 89 |
+
|
| 90 |
+
# pipenv
|
| 91 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
| 92 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
| 93 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
| 94 |
+
# install all needed dependencies.
|
| 95 |
+
#Pipfile.lock
|
| 96 |
+
|
| 97 |
+
# poetry
|
| 98 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
| 99 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 100 |
+
# commonly ignored for libraries.
|
| 101 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
| 102 |
+
#poetry.lock
|
| 103 |
+
|
| 104 |
+
# pdm
|
| 105 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
| 106 |
+
#pdm.lock
|
| 107 |
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
| 108 |
+
# in version control.
|
| 109 |
+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
| 110 |
+
.pdm.toml
|
| 111 |
+
.pdm-python
|
| 112 |
+
.pdm-build/
|
| 113 |
+
|
| 114 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
| 115 |
+
__pypackages__/
|
| 116 |
+
|
| 117 |
+
# Celery stuff
|
| 118 |
+
celerybeat-schedule
|
| 119 |
+
celerybeat.pid
|
| 120 |
+
|
| 121 |
+
# SageMath parsed files
|
| 122 |
+
*.sage.py
|
| 123 |
+
|
| 124 |
+
# Environments
|
| 125 |
+
.env
|
| 126 |
+
.venv
|
| 127 |
+
env/
|
| 128 |
+
venv/
|
| 129 |
+
ENV/
|
| 130 |
+
env.bak/
|
| 131 |
+
venv.bak/
|
| 132 |
+
|
| 133 |
+
# Spyder project settings
|
| 134 |
+
.spyderproject
|
| 135 |
+
.spyproject
|
| 136 |
+
|
| 137 |
+
# Rope project settings
|
| 138 |
+
.ropeproject
|
| 139 |
+
|
| 140 |
+
# mkdocs documentation
|
| 141 |
+
/site
|
| 142 |
+
|
| 143 |
+
# mypy
|
| 144 |
+
.mypy_cache/
|
| 145 |
+
.dmypy.json
|
| 146 |
+
dmypy.json
|
| 147 |
+
|
| 148 |
+
# Pyre type checker
|
| 149 |
+
.pyre/
|
| 150 |
+
|
| 151 |
+
# pytype static type analyzer
|
| 152 |
+
.pytype/
|
| 153 |
+
|
| 154 |
+
# Cython debug symbols
|
| 155 |
+
cython_debug/
|
| 156 |
+
|
| 157 |
+
# PyCharm
|
| 158 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
| 159 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
| 160 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
| 161 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
| 162 |
+
.idea/
|
README.md
CHANGED
|
@@ -1,12 +1,10 @@
|
|
| 1 |
---
|
| 2 |
-
title: Wire
|
| 3 |
-
|
| 4 |
-
colorFrom: purple
|
| 5 |
-
colorTo: green
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 4.44.0
|
| 8 |
-
app_file: app.py
|
| 9 |
-
pinned: false
|
| 10 |
---
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Wire-RAG
|
| 3 |
+
app_file: gradio_interface.py
|
|
|
|
|
|
|
| 4 |
sdk: gradio
|
| 5 |
sdk_version: 4.44.0
|
|
|
|
|
|
|
| 6 |
---
|
| 7 |
+
To run the script with a `.env` file, simply add `dotenv run` before the command:\
|
| 8 |
+
```
|
| 9 |
+
dotenv run python gradio_interface.py
|
| 10 |
+
```
|
github urls.txt
ADDED
|
@@ -0,0 +1,1045 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Fetching .md files from repository: wireapp/libsodium.js
|
| 2 |
+
Fetching .md files from repository: wireapp/libsodium
|
| 3 |
+
Fetching .md files from repository: wireapp/cryptobox-haskell
|
| 4 |
+
Fetching .md files from repository: wireapp/proteus
|
| 5 |
+
Fetching .md files from repository: wireapp/cryptobox
|
| 6 |
+
Fetching .md files from repository: wireapp/cryptobox-c
|
| 7 |
+
Fetching .md files from repository: wireapp/cryptobox-jni
|
| 8 |
+
Fetching .md files from repository: wireapp/cryptobox-ios
|
| 9 |
+
Fetching .md files from repository: wireapp/hkdf
|
| 10 |
+
Fetching .md files from repository: wireapp/node-addressbook
|
| 11 |
+
Fetching .md files from repository: wireapp/wire-avs
|
| 12 |
+
Fetching .md files from repository: wireapp/generic-message-proto
|
| 13 |
+
Fetching .md files from repository: wireapp/backend-api-protobuf
|
| 14 |
+
Fetching .md files from repository: wireapp/cryptobox.js
|
| 15 |
+
Fetching .md files from repository: wireapp/proteus.js
|
| 16 |
+
Fetching .md files from repository: wireapp/ocmock
|
| 17 |
+
Fetching .md files from repository: wireapp/wire
|
| 18 |
+
Fetching .md files from repository: wireapp/libsodium-native
|
| 19 |
+
Fetching .md files from repository: wireapp/webapp-module-modal
|
| 20 |
+
Fetching .md files from repository: wireapp/amplify
|
| 21 |
+
Fetching .md files from repository: wireapp/PINCache
|
| 22 |
+
Fetching .md files from repository: wireapp/ZipArchive
|
| 23 |
+
Fetching .md files from repository: wireapp/ios-snapshot-test-case
|
| 24 |
+
Fetching .md files from repository: wireapp/libPhoneNumber-iOS
|
| 25 |
+
Fetching .md files from repository: wireapp/FLAnimatedImage
|
| 26 |
+
Fetching .md files from repository: wireapp/wire-audio-files
|
| 27 |
+
Fetching .md files from repository: wireapp/wire-webapp
|
| 28 |
+
Fetching .md files from repository: wireapp/wire-desktop
|
| 29 |
+
Fetching .md files from repository: wireapp/wire-account
|
| 30 |
+
Fetching .md files from repository: wireapp/echo-bot
|
| 31 |
+
Fetching .md files from repository: wireapp/KeychainAccess
|
| 32 |
+
Fetching .md files from repository: wireapp/lithium
|
| 33 |
+
Fetching .md files from repository: wireapp/don-bot
|
| 34 |
+
Fetching .md files from repository: wireapp/wire-server
|
| 35 |
+
Fetching .md files from repository: wireapp/wire-web-packages
|
| 36 |
+
Fetching .md files from repository: wireapp/Down
|
| 37 |
+
Fetching .md files from repository: wireapp/wire-emails
|
| 38 |
+
Fetching .md files from repository: wireapp/brand
|
| 39 |
+
Fetching .md files from repository: wireapp/cryptobox4j
|
| 40 |
+
Fetching .md files from repository: wireapp/hsaml2
|
| 41 |
+
Fetching .md files from repository: wireapp/saml2-web-sso
|
| 42 |
+
Fetching .md files from repository: wireapp/hspec-wai
|
| 43 |
+
Fetching .md files from repository: wireapp/servant-multipart
|
| 44 |
+
Fetching .md files from repository: wireapp/wire-web-ets
|
| 45 |
+
Fetching .md files from repository: wireapp/re
|
| 46 |
+
Fetching .md files from repository: wireapp/rew
|
| 47 |
+
Fetching .md files from repository: wireapp/http-client
|
| 48 |
+
Fetching .md files from repository: wireapp/HTMLString
|
| 49 |
+
Fetching .md files from repository: wireapp/restund
|
| 50 |
+
Fetching .md files from repository: wireapp/recording-bot
|
| 51 |
+
Fetching .md files from repository: wireapp/ansible-cassandra
|
| 52 |
+
Fetching .md files from repository: wireapp/wire-server-deploy
|
| 53 |
+
Fetching .md files from repository: wireapp/swift-protobuf
|
| 54 |
+
Fetching .md files from repository: wireapp/generate-changelog
|
| 55 |
+
Fetching .md files from repository: wireapp/wire-web-config-default
|
| 56 |
+
Fetching .md files from repository: wireapp/hspec
|
| 57 |
+
Fetching .md files from repository: wireapp/DifferenceKit
|
| 58 |
+
Fetching .md files from repository: wireapp/wire-web-config-wire
|
| 59 |
+
Fetching .md files from repository: wireapp/prebuilt-webrtc-binaries
|
| 60 |
+
Fetching .md files from repository: wireapp/servant
|
| 61 |
+
Fetching .md files from repository: wireapp/Winium.Desktop
|
| 62 |
+
Fetching .md files from repository: wireapp/legalhold
|
| 63 |
+
Fetching .md files from repository: wireapp/appium-for-mac
|
| 64 |
+
Fetching .md files from repository: wireapp/ansible-restund
|
| 65 |
+
Fetching .md files from repository: wireapp/ansible-tinc
|
| 66 |
+
Fetching .md files from repository: wireapp/servant-swagger
|
| 67 |
+
Fetching .md files from repository: wireapp/swift-snapshot-testing
|
| 68 |
+
Fetching .md files from repository: wireapp/servant-checked-exceptions
|
| 69 |
+
Fetching .md files from repository: wireapp/roman
|
| 70 |
+
Fetching .md files from repository: wireapp/wire-avs-nwtesttool
|
| 71 |
+
Fetching .md files from repository: wireapp/wireapp.github.io
|
| 72 |
+
Fetching .md files from repository: wireapp/appcenter-sdk-apple
|
| 73 |
+
Fetching .md files from repository: wireapp/poll-bot
|
| 74 |
+
Fetching .md files from repository: wireapp/github-action-wire-messenger
|
| 75 |
+
Fetching .md files from repository: wireapp/bloodhound
|
| 76 |
+
Fetching .md files from repository: wireapp/charon
|
| 77 |
+
Fetching .md files from repository: wireapp/echo-bot-roman
|
| 78 |
+
Fetching .md files from repository: wireapp/ansible-ntp-verify
|
| 79 |
+
Fetching .md files from repository: wireapp/wire-android
|
| 80 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 81 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 82 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 83 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 84 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 85 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 86 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 87 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 88 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 89 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 90 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 91 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 92 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 93 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 94 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 95 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 96 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 97 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 98 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 99 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 100 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 101 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 102 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 103 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 104 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 105 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 106 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 107 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 108 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 109 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 110 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 111 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 112 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 113 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 114 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 115 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 116 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 117 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 118 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 119 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 120 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 121 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 122 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 123 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 124 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 125 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 126 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 127 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 128 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 129 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 130 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 131 |
+
Error fetching contents for wireapp/wire-android: 403
|
| 132 |
+
Fetching .md files from repository: wireapp/ansible-sft
|
| 133 |
+
Error fetching contents for wireapp/ansible-sft: 403
|
| 134 |
+
Fetching .md files from repository: wireapp/srv-announcer
|
| 135 |
+
Error fetching contents for wireapp/srv-announcer: 403
|
| 136 |
+
Fetching .md files from repository: wireapp/backup-export-tool
|
| 137 |
+
Error fetching contents for wireapp/backup-export-tool: 403
|
| 138 |
+
Fetching .md files from repository: wireapp/countly-sdk-ios
|
| 139 |
+
Error fetching contents for wireapp/countly-sdk-ios: 403
|
| 140 |
+
Fetching .md files from repository: wireapp/xenon
|
| 141 |
+
Error fetching contents for wireapp/xenon: 403
|
| 142 |
+
Fetching .md files from repository: wireapp/helium
|
| 143 |
+
Error fetching contents for wireapp/helium: 403
|
| 144 |
+
Fetching .md files from repository: wireapp/ansible-minio
|
| 145 |
+
Error fetching contents for wireapp/ansible-minio: 403
|
| 146 |
+
Fetching .md files from repository: wireapp/security-monitoring
|
| 147 |
+
Error fetching contents for wireapp/security-monitoring: 403
|
| 148 |
+
Fetching .md files from repository: wireapp/wire-avs-service
|
| 149 |
+
Error fetching contents for wireapp/wire-avs-service: 403
|
| 150 |
+
Fetching .md files from repository: wireapp/wire-web-core
|
| 151 |
+
Error fetching contents for wireapp/wire-web-core: 403
|
| 152 |
+
Fetching .md files from repository: wireapp/admin-broadcast
|
| 153 |
+
Error fetching contents for wireapp/admin-broadcast: 403
|
| 154 |
+
Fetching .md files from repository: wireapp/http2
|
| 155 |
+
Error fetching contents for wireapp/http2: 403
|
| 156 |
+
Fetching .md files from repository: wireapp/ldap-scim-bridge
|
| 157 |
+
Error fetching contents for wireapp/ldap-scim-bridge: 403
|
| 158 |
+
Fetching .md files from repository: wireapp/Starscream
|
| 159 |
+
Error fetching contents for wireapp/Starscream: 403
|
| 160 |
+
Fetching .md files from repository: wireapp/hs-certificate
|
| 161 |
+
Error fetching contents for wireapp/hs-certificate: 403
|
| 162 |
+
Fetching .md files from repository: wireapp/.github
|
| 163 |
+
Error fetching contents for wireapp/.github: 403
|
| 164 |
+
Fetching .md files from repository: wireapp/this-is-a-test
|
| 165 |
+
Error fetching contents for wireapp/this-is-a-test: 403
|
| 166 |
+
Fetching .md files from repository: wireapp/http2-client
|
| 167 |
+
Error fetching contents for wireapp/http2-client: 403
|
| 168 |
+
Fetching .md files from repository: wireapp/aeson
|
| 169 |
+
Error fetching contents for wireapp/aeson: 403
|
| 170 |
+
Fetching .md files from repository: wireapp/kalium
|
| 171 |
+
Error fetching contents for wireapp/kalium: 403
|
| 172 |
+
Fetching .md files from repository: wireapp/core-crypto
|
| 173 |
+
Error fetching contents for wireapp/core-crypto: 403
|
| 174 |
+
Fetching .md files from repository: wireapp/carthage-gradle-plugin
|
| 175 |
+
Error fetching contents for wireapp/carthage-gradle-plugin: 403
|
| 176 |
+
Fetching .md files from repository: wireapp/calendar
|
| 177 |
+
Error fetching contents for wireapp/calendar: 403
|
| 178 |
+
Fetching .md files from repository: wireapp/wire-maven
|
| 179 |
+
Error fetching contents for wireapp/wire-maven: 403
|
| 180 |
+
Fetching .md files from repository: wireapp/coturn
|
| 181 |
+
Error fetching contents for wireapp/coturn: 403
|
| 182 |
+
Fetching .md files from repository: wireapp/prometheus-client-c
|
| 183 |
+
Error fetching contents for wireapp/prometheus-client-c: 403
|
| 184 |
+
Fetching .md files from repository: wireapp/openmls
|
| 185 |
+
Error fetching contents for wireapp/openmls: 403
|
| 186 |
+
Fetching .md files from repository: wireapp/hedis
|
| 187 |
+
Error fetching contents for wireapp/hedis: 403
|
| 188 |
+
Fetching .md files from repository: wireapp/rcgen
|
| 189 |
+
Error fetching contents for wireapp/rcgen: 403
|
| 190 |
+
Fetching .md files from repository: wireapp/x509-parser
|
| 191 |
+
Error fetching contents for wireapp/x509-parser: 403
|
| 192 |
+
Fetching .md files from repository: wireapp/mls-test-cli
|
| 193 |
+
Error fetching contents for wireapp/mls-test-cli: 403
|
| 194 |
+
Fetching .md files from repository: wireapp/hpke-rs
|
| 195 |
+
Error fetching contents for wireapp/hpke-rs: 403
|
| 196 |
+
Fetching .md files from repository: wireapp/hs-jose
|
| 197 |
+
Error fetching contents for wireapp/hs-jose: 403
|
| 198 |
+
Fetching .md files from repository: wireapp/rusty-jwt-tools
|
| 199 |
+
Error fetching contents for wireapp/rusty-jwt-tools: 403
|
| 200 |
+
Fetching .md files from repository: wireapp/tasty
|
| 201 |
+
Error fetching contents for wireapp/tasty: 403
|
| 202 |
+
Fetching .md files from repository: wireapp/servant-oauth-server
|
| 203 |
+
Error fetching contents for wireapp/servant-oauth-server: 403
|
| 204 |
+
Fetching .md files from repository: wireapp/stores-data-tool
|
| 205 |
+
Error fetching contents for wireapp/stores-data-tool: 403
|
| 206 |
+
Fetching .md files from repository: wireapp/wire-ios
|
| 207 |
+
Error fetching contents for wireapp/wire-ios: 403
|
| 208 |
+
Fetching .md files from repository: wireapp/smallstep-certificates
|
| 209 |
+
Error fetching contents for wireapp/smallstep-certificates: 403
|
| 210 |
+
Fetching .md files from repository: wireapp/wire-detekt-rules
|
| 211 |
+
Error fetching contents for wireapp/wire-detekt-rules: 403
|
| 212 |
+
Fetching .md files from repository: wireapp/uniffi-kotlin-multiplatform-bindings
|
| 213 |
+
Error fetching contents for wireapp/uniffi-kotlin-multiplatform-bindings: 403
|
| 214 |
+
Fetching .md files from repository: wireapp/dd-sdk-ios
|
| 215 |
+
Error fetching contents for wireapp/dd-sdk-ios: 403
|
| 216 |
+
Fetching .md files from repository: wireapp/HsOpenSSL
|
| 217 |
+
Error fetching contents for wireapp/HsOpenSSL: 403
|
| 218 |
+
Fetching .md files from repository: wireapp/rust-jwt-simple
|
| 219 |
+
Error fetching contents for wireapp/rust-jwt-simple: 403
|
| 220 |
+
Fetching .md files from repository: wireapp/text-icu-translit
|
| 221 |
+
Error fetching contents for wireapp/text-icu-translit: 403
|
| 222 |
+
Fetching .md files from repository: wireapp/transitive-anns
|
| 223 |
+
Error fetching contents for wireapp/transitive-anns: 403
|
| 224 |
+
Fetching .md files from repository: wireapp/formats
|
| 225 |
+
Error fetching contents for wireapp/formats: 403
|
| 226 |
+
Fetching .md files from repository: wireapp/outlook-addin
|
| 227 |
+
Error fetching contents for wireapp/outlook-addin: 403
|
| 228 |
+
Fetching .md files from repository: wireapp/soyka
|
| 229 |
+
Error fetching contents for wireapp/soyka: 403
|
| 230 |
+
Fetching .md files from repository: wireapp/rust-hpke
|
| 231 |
+
Error fetching contents for wireapp/rust-hpke: 403
|
| 232 |
+
Fetching .md files from repository: wireapp/schnellru
|
| 233 |
+
Error fetching contents for wireapp/schnellru: 403
|
| 234 |
+
Fetching .md files from repository: wireapp/rexie
|
| 235 |
+
Error fetching contents for wireapp/rexie: 403
|
| 236 |
+
Fetching .md files from repository: wireapp/picklejar-engine
|
| 237 |
+
Error fetching contents for wireapp/picklejar-engine: 403
|
| 238 |
+
Fetching .md files from repository: wireapp/this-is-another-test
|
| 239 |
+
Error fetching contents for wireapp/this-is-another-test: 403
|
| 240 |
+
Fetching .md files from repository: wireapp/uniffi-rs
|
| 241 |
+
Error fetching contents for wireapp/uniffi-rs: 403
|
| 242 |
+
Fetching .md files from repository: wireapp/rustls-platform-verifier
|
| 243 |
+
Error fetching contents for wireapp/rustls-platform-verifier: 403
|
| 244 |
+
Fetching .md files from repository: wireapp/webpki
|
| 245 |
+
Error fetching contents for wireapp/webpki: 403
|
| 246 |
+
Fetching .md files from repository: wireapp/rustls
|
| 247 |
+
Error fetching contents for wireapp/rustls: 403
|
| 248 |
+
Fetching .md files from repository: wireapp/pki-types
|
| 249 |
+
Error fetching contents for wireapp/pki-types: 403
|
| 250 |
+
Fetching .md files from repository: wireapp/ghc-flakr
|
| 251 |
+
Error fetching contents for wireapp/ghc-flakr: 403
|
| 252 |
+
Fetching .md files from repository: wireapp/swift-crypto
|
| 253 |
+
Error fetching contents for wireapp/swift-crypto: 403
|
| 254 |
+
Fetching .md files from repository: wireapp/wire-builds
|
| 255 |
+
Error fetching contents for wireapp/wire-builds: 403
|
| 256 |
+
Fetching .md files from repository: wireapp/wai
|
| 257 |
+
Error fetching contents for wireapp/wai: 403
|
| 258 |
+
Fetching .md files from repository: wireapp/swift-certificates
|
| 259 |
+
Error fetching contents for wireapp/swift-certificates: 403
|
| 260 |
+
Fetching .md files from repository: wireapp/rust-pki
|
| 261 |
+
Error fetching contents for wireapp/rust-pki: 403
|
| 262 |
+
Fetching .md files from repository: wireapp/ASN1Decoder
|
| 263 |
+
Error fetching contents for wireapp/ASN1Decoder: 403
|
| 264 |
+
Fetching .md files from repository: wireapp/tinylog
|
| 265 |
+
Error fetching contents for wireapp/tinylog: 403
|
| 266 |
+
Fetching .md files from repository: wireapp/cql
|
| 267 |
+
Error fetching contents for wireapp/cql: 403
|
| 268 |
+
Fetching .md files from repository: wireapp/cql-io
|
| 269 |
+
Error fetching contents for wireapp/cql-io: 403
|
| 270 |
+
Fetching .md files from repository: wireapp/wai-predicates
|
| 271 |
+
Error fetching contents for wireapp/wai-predicates: 403
|
| 272 |
+
Fetching .md files from repository: wireapp/wai-routing
|
| 273 |
+
Error fetching contents for wireapp/wai-routing: 403
|
| 274 |
+
Fetching .md files from repository: wireapp/tasty-ant-xml
|
| 275 |
+
Error fetching contents for wireapp/tasty-ant-xml: 403
|
| 276 |
+
Fetching .md files from repository: wireapp/servant-openapi3
|
| 277 |
+
Error fetching contents for wireapp/servant-openapi3: 403
|
| 278 |
+
Fetching .md files from repository: wireapp/HaskellNet-SSL
|
| 279 |
+
Error fetching contents for wireapp/HaskellNet-SSL: 403
|
| 280 |
+
Fetching .md files from repository: wireapp/nix-tree
|
| 281 |
+
Error fetching contents for wireapp/nix-tree: 403
|
| 282 |
+
Fetching .md files from repository: wireapp/grafana-dashboards-kubernetes
|
| 283 |
+
Error fetching contents for wireapp/grafana-dashboards-kubernetes: 403
|
| 284 |
+
Fetching .md files from repository: wireapp/grafana-agent-modules
|
| 285 |
+
Error fetching contents for wireapp/grafana-agent-modules: 403
|
| 286 |
+
Fetching .md files from repository: wireapp/dependency-track
|
| 287 |
+
Error fetching contents for wireapp/dependency-track: 403
|
| 288 |
+
Fetching .md files from repository: wireapp/packageurl-java
|
| 289 |
+
Error fetching contents for wireapp/packageurl-java: 403
|
| 290 |
+
Fetching .md files from repository: wireapp/network-control
|
| 291 |
+
Error fetching contents for wireapp/network-control: 403
|
| 292 |
+
Fetching .md files from repository: wireapp/action-auto-cherry-pick
|
| 293 |
+
Error fetching contents for wireapp/action-auto-cherry-pick: 403
|
| 294 |
+
Fetching .md files from repository: wireapp/smallstep-helm-charts
|
| 295 |
+
Error fetching contents for wireapp/smallstep-helm-charts: 403
|
| 296 |
+
Fetching .md files from repository: wireapp/helm-charts
|
| 297 |
+
Error fetching contents for wireapp/helm-charts: 403
|
| 298 |
+
Fetching .md files from repository: wireapp/keycloak
|
| 299 |
+
Error fetching contents for wireapp/keycloak: 403
|
| 300 |
+
Fetching .md files from repository: wireapp/cc-batch-poc
|
| 301 |
+
Error fetching contents for wireapp/cc-batch-poc: 403
|
| 302 |
+
Fetching .md files from repository: wireapp/postie
|
| 303 |
+
Error fetching contents for wireapp/postie: 403
|
| 304 |
+
Found 741 Markdown files:
|
| 305 |
+
https://raw.githubusercontent.com/wireapp/libsodium.js/master/README.md
|
| 306 |
+
https://raw.githubusercontent.com/wireapp/libsodium/master/.github/ISSUE_TEMPLATE/bugs.md
|
| 307 |
+
https://raw.githubusercontent.com/wireapp/cryptobox-haskell/master/README.md
|
| 308 |
+
https://raw.githubusercontent.com/wireapp/proteus/develop/SECURITY.md
|
| 309 |
+
https://raw.githubusercontent.com/wireapp/cryptobox/develop/README.md
|
| 310 |
+
https://raw.githubusercontent.com/wireapp/cryptobox-c/develop/README.md
|
| 311 |
+
https://raw.githubusercontent.com/wireapp/cryptobox-jni/develop/README.md
|
| 312 |
+
https://raw.githubusercontent.com/wireapp/cryptobox-ios/master/README.md
|
| 313 |
+
https://raw.githubusercontent.com/wireapp/hkdf/develop/README.md
|
| 314 |
+
https://raw.githubusercontent.com/wireapp/node-addressbook/master/README.md
|
| 315 |
+
https://raw.githubusercontent.com/wireapp/wire-avs/main/README.md
|
| 316 |
+
https://raw.githubusercontent.com/wireapp/wire-avs/main/docs/debug_wasm.md
|
| 317 |
+
https://raw.githubusercontent.com/wireapp/wire-avs/main/docs/devicepair.md
|
| 318 |
+
https://raw.githubusercontent.com/wireapp/wire-avs/main/docs/flowmgr.md
|
| 319 |
+
https://raw.githubusercontent.com/wireapp/wire-avs/main/docs/local_avs_on_ios.md
|
| 320 |
+
https://raw.githubusercontent.com/wireapp/wire-avs/main/src/econn/README.md
|
| 321 |
+
https://raw.githubusercontent.com/wireapp/generic-message-proto/master/README.md
|
| 322 |
+
https://raw.githubusercontent.com/wireapp/generic-message-proto/master/android/publishing.md
|
| 323 |
+
https://raw.githubusercontent.com/wireapp/backend-api-protobuf/master/README.md
|
| 324 |
+
https://raw.githubusercontent.com/wireapp/backend-api-protobuf/master/android/publishing.md
|
| 325 |
+
https://raw.githubusercontent.com/wireapp/cryptobox.js/master/README.md
|
| 326 |
+
https://raw.githubusercontent.com/wireapp/proteus.js/master/README.md
|
| 327 |
+
https://raw.githubusercontent.com/wireapp/ocmock/master/CONTRIBUTING.md
|
| 328 |
+
https://raw.githubusercontent.com/wireapp/ocmock/master/README.md
|
| 329 |
+
https://raw.githubusercontent.com/wireapp/wire/master/CONTRIBUTING.md
|
| 330 |
+
https://raw.githubusercontent.com/wireapp/wire/master/README.md
|
| 331 |
+
https://raw.githubusercontent.com/wireapp/wire/master/SECURITY.md
|
| 332 |
+
https://raw.githubusercontent.com/wireapp/wire/master/issue_template.md
|
| 333 |
+
https://raw.githubusercontent.com/wireapp/webapp-module-modal/master/README.md
|
| 334 |
+
https://raw.githubusercontent.com/wireapp/amplify/master/docs/amplify.core.md
|
| 335 |
+
https://raw.githubusercontent.com/wireapp/amplify/master/docs/amplify.request.md
|
| 336 |
+
https://raw.githubusercontent.com/wireapp/amplify/master/docs/amplify.store.md
|
| 337 |
+
https://raw.githubusercontent.com/wireapp/amplify/master/readme.md
|
| 338 |
+
https://raw.githubusercontent.com/wireapp/PINCache/master/CHANGELOG.md
|
| 339 |
+
https://raw.githubusercontent.com/wireapp/PINCache/master/CODE_OF_CONDUCT.md
|
| 340 |
+
https://raw.githubusercontent.com/wireapp/PINCache/master/Carthage/Checkouts/PINOperation/CHANGELOG.md
|
| 341 |
+
https://raw.githubusercontent.com/wireapp/PINCache/master/Carthage/Checkouts/PINOperation/CODE_OF_CONDUCT.md
|
| 342 |
+
https://raw.githubusercontent.com/wireapp/PINCache/master/Carthage/Checkouts/PINOperation/README.md
|
| 343 |
+
https://raw.githubusercontent.com/wireapp/PINCache/master/Carthage/Checkouts/PINOperation/RELEASE.md
|
| 344 |
+
https://raw.githubusercontent.com/wireapp/PINCache/master/README.md
|
| 345 |
+
https://raw.githubusercontent.com/wireapp/PINCache/master/RELEASE.md
|
| 346 |
+
https://raw.githubusercontent.com/wireapp/ZipArchive/master/ObjectiveCExample/ObjectiveCExample/Sample%20Data/1.md
|
| 347 |
+
https://raw.githubusercontent.com/wireapp/ZipArchive/master/ObjectiveCExample/ObjectiveCExample/Sample%20Data/2.md
|
| 348 |
+
https://raw.githubusercontent.com/wireapp/ZipArchive/master/README.md
|
| 349 |
+
https://raw.githubusercontent.com/wireapp/ZipArchive/master/Release-Instructions.md
|
| 350 |
+
https://raw.githubusercontent.com/wireapp/ZipArchive/master/SwiftExample/SwiftExample/Sample%20Data/1.md
|
| 351 |
+
https://raw.githubusercontent.com/wireapp/ZipArchive/master/SwiftExample/SwiftExample/Sample%20Data/2.md
|
| 352 |
+
https://raw.githubusercontent.com/wireapp/ios-snapshot-test-case/master/CHANGELOG.md
|
| 353 |
+
https://raw.githubusercontent.com/wireapp/ios-snapshot-test-case/master/README.md
|
| 354 |
+
https://raw.githubusercontent.com/wireapp/ios-snapshot-test-case/master/docs/HowToRelease.md
|
| 355 |
+
https://raw.githubusercontent.com/wireapp/ios-snapshot-test-case/master/docs/LibraryVsApplicationTestBundles.md
|
| 356 |
+
https://raw.githubusercontent.com/wireapp/libPhoneNumber-iOS/master/README.md
|
| 357 |
+
https://raw.githubusercontent.com/wireapp/libPhoneNumber-iOS/master/libPhoneNumber-Demo/README.md
|
| 358 |
+
https://raw.githubusercontent.com/wireapp/libPhoneNumber-iOS/master/libPhoneNumber-GeocodingParser/README.md
|
| 359 |
+
https://raw.githubusercontent.com/wireapp/libPhoneNumber-iOS/master/libPhoneNumberGeocoding/README.md
|
| 360 |
+
https://raw.githubusercontent.com/wireapp/libPhoneNumber-iOS/master/libPhoneNumberShortNumber/README.md
|
| 361 |
+
https://raw.githubusercontent.com/wireapp/FLAnimatedImage/master/README.md
|
| 362 |
+
https://raw.githubusercontent.com/wireapp/wire-audio-files/master/README.md
|
| 363 |
+
https://raw.githubusercontent.com/wireapp/wire-audio-files/master/android/README.md
|
| 364 |
+
https://raw.githubusercontent.com/wireapp/wire-webapp/dev/.github/ISSUE_TEMPLATE/bug_report.md
|
| 365 |
+
https://raw.githubusercontent.com/wireapp/wire-webapp/dev/.github/PULL_REQUEST_TEMPLATE.md
|
| 366 |
+
https://raw.githubusercontent.com/wireapp/wire-webapp/dev/README.md
|
| 367 |
+
https://raw.githubusercontent.com/wireapp/wire-webapp/dev/SECURITY.md
|
| 368 |
+
https://raw.githubusercontent.com/wireapp/wire-webapp/dev/src/style/fonts/README.md
|
| 369 |
+
https://raw.githubusercontent.com/wireapp/wire-desktop/dev/.github/issue_template.md
|
| 370 |
+
https://raw.githubusercontent.com/wireapp/wire-desktop/dev/README.md
|
| 371 |
+
https://raw.githubusercontent.com/wireapp/wire-desktop/dev/SECURITY.md
|
| 372 |
+
https://raw.githubusercontent.com/wireapp/wire-account/staging/CHANGELOG.md
|
| 373 |
+
https://raw.githubusercontent.com/wireapp/wire-account/staging/README.md
|
| 374 |
+
https://raw.githubusercontent.com/wireapp/wire-account/staging/charts/account-pages/README.md
|
| 375 |
+
https://raw.githubusercontent.com/wireapp/echo-bot/staging/README.md
|
| 376 |
+
https://raw.githubusercontent.com/wireapp/KeychainAccess/master/README.md
|
| 377 |
+
https://raw.githubusercontent.com/wireapp/lithium/master/README.md
|
| 378 |
+
https://raw.githubusercontent.com/wireapp/don-bot/staging/README.md
|
| 379 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/.github/ISSUE_TEMPLATE/bug_report.md
|
| 380 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/.github/ISSUE_TEMPLATE/other.md
|
| 381 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/.github/ISSUE_TEMPLATE/question-installation.md
|
| 382 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/.github/ISSUE_TEMPLATE/question.md
|
| 383 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/.github/pull_request_template.md
|
| 384 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/CHANGELOG.md
|
| 385 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/README.md
|
| 386 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/SECURITY.md
|
| 387 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/background-worker/README.md
|
| 388 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/backoffice/README.md
|
| 389 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/brig/README.md
|
| 390 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/gundeck/README.md
|
| 391 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/k8ssandra-test-cluster/README.md
|
| 392 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/ldap-scim-bridge/README.md
|
| 393 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/metallb/README.md
|
| 394 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/mlsstats/README.md
|
| 395 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/nginx-ingress-controller/README.md
|
| 396 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/nginx-ingress-services/README.md
|
| 397 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/nginz/README.md
|
| 398 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/outlook-addin/README.md
|
| 399 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/reaper/README.md
|
| 400 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/charts/restund/README.md
|
| 401 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/deploy/dockerephemeral/build/README.md
|
| 402 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/deploy/dockerephemeral/federation-v0/nginz/conf/README.md
|
| 403 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/deploy/dockerephemeral/federation-v1/nginz/conf/README.md
|
| 404 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/README.md
|
| 405 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/diagrams/README.md
|
| 406 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/_templates/README.md
|
| 407 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/changelog/changelog.md
|
| 408 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/changelog/index.md
|
| 409 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/api-versioning.md
|
| 410 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/building.md
|
| 411 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/cassandra-interaction.md
|
| 412 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/changelog.md
|
| 413 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/coding-conventions.md
|
| 414 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/dependencies.md
|
| 415 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/editor-setup.md
|
| 416 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/features.md
|
| 417 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/federation-api-conventions.md
|
| 418 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/federation-design-aspects.md
|
| 419 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/how-to.md
|
| 420 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/index.md
|
| 421 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/large-conversations.md
|
| 422 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/linting.md
|
| 423 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/open-telemetry.md
|
| 424 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/pr-guidelines.md
|
| 425 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/processes.md
|
| 426 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/scim/storage.md
|
| 427 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/servant.md
|
| 428 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/testing.md
|
| 429 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/developer/upgrading.md
|
| 430 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/index.md
|
| 431 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/config-options.md
|
| 432 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/conversation.md
|
| 433 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/elastic-search.md
|
| 434 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/elasticsearch-migration-2021-02-16.md
|
| 435 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/index.md
|
| 436 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/make-docker-and-qemu.md
|
| 437 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/oauth.md
|
| 438 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/provisioning/scim-token.md
|
| 439 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/rabbitmq-consumer.md
|
| 440 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/spar-braindump.md
|
| 441 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/team/legalhold.md
|
| 442 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/user/activation.md
|
| 443 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/user/connection.md
|
| 444 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/user/registration.md
|
| 445 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/developer/reference/user/rich-info.md
|
| 446 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/backup-disaster-recovery.md
|
| 447 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/cassandra.md
|
| 448 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/elasticsearch.md
|
| 449 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/etcd.md
|
| 450 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/general-linux.md
|
| 451 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/index.md
|
| 452 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/kubernetes/certificate-renewal/index.md
|
| 453 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/kubernetes/certificate-renewal/scenario-1_k8s-v1.14-kubespray.md
|
| 454 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/kubernetes/index.md
|
| 455 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/kubernetes/restart-machines/index.md
|
| 456 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/kubernetes/upgrade-cluster/index.md
|
| 457 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/minio.md
|
| 458 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/operations.md
|
| 459 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/restund.md
|
| 460 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/administrate/users.md
|
| 461 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/ansible-VMs.md
|
| 462 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/ansible-authentication.md
|
| 463 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/ansible-tinc.md
|
| 464 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/aws-prod.md
|
| 465 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/dependencies.md
|
| 466 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/helm-prod.md
|
| 467 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/helm.md
|
| 468 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/index.md
|
| 469 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/infrastructure-configuration.md
|
| 470 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/ingress.md
|
| 471 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/kubernetes.md
|
| 472 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/logging.md
|
| 473 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/monitoring.md
|
| 474 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/planning.md
|
| 475 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/post-install.md
|
| 476 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/prod-intro.md
|
| 477 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/restund.md
|
| 478 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/sft.md
|
| 479 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/tls.md
|
| 480 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/troubleshooting.md
|
| 481 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/version-requirements.md
|
| 482 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/how-to/install/web-app-settings.md
|
| 483 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/index.md
|
| 484 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/security-responses/2021-12-15_log4shell.md
|
| 485 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/security-responses/2022-02-21_cve-2021-44521.md
|
| 486 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/security-responses/2022-05-23_website_outage.md
|
| 487 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/security-responses/2022-11-01_openssl.md
|
| 488 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/security-responses/2023-01-04_website_outage.md
|
| 489 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/security-responses/2023-01-19_html_injection.md
|
| 490 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/security-responses/index.md
|
| 491 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/api-client-perspective/authentication.md
|
| 492 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/api-client-perspective/index.md
|
| 493 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/api-client-perspective/swagger.md
|
| 494 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/associate/custom-backend-for-desktop-client.md
|
| 495 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/associate/custom-certificates.md
|
| 496 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/associate/deeplink.md
|
| 497 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/associate/index.md
|
| 498 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/associate/sso-domain-redirect.md
|
| 499 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/block-user-creation.md
|
| 500 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/classified-domains.md
|
| 501 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/configure-federation.md
|
| 502 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/crypto-libs.md
|
| 503 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/federation/api.md
|
| 504 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/federation/architecture.md
|
| 505 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/federation/backend-communication.md
|
| 506 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/federation/fedcalls.md
|
| 507 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/federation/index.md
|
| 508 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/helm.md
|
| 509 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/index.md
|
| 510 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/legalhold.md
|
| 511 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/minio.md
|
| 512 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/mls.md
|
| 513 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/notes/port-ranges.md
|
| 514 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/overview.md
|
| 515 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/restund.md
|
| 516 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/searchability.md
|
| 517 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/sft.md
|
| 518 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/single-sign-on/adfs/main.md
|
| 519 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/single-sign-on/azure/main.md
|
| 520 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/single-sign-on/centrify/main.md
|
| 521 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/single-sign-on/generic-setup.md
|
| 522 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/single-sign-on/index.md
|
| 523 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/single-sign-on/okta/main.md
|
| 524 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/single-sign-on/trouble-shooting.md
|
| 525 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/single-sign-on/understand/main.md
|
| 526 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/docs/src/understand/team-feature-settings.md
|
| 527 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/hack/FUTUREWORK.md
|
| 528 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/hack/python/README.md
|
| 529 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/integration/README.md
|
| 530 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/libs/bilge/README.md
|
| 531 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/libs/cassandra-util/README.md
|
| 532 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/libs/hscim/README.md
|
| 533 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/libs/libzauth/README.md
|
| 534 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/libs/schema-profunctor/README.md
|
| 535 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/libs/wire-api/Readme.md
|
| 536 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/libs/wire-otel/CHANGELOG.md
|
| 537 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/libs/zauth/README.md
|
| 538 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/nix/pkgs/cryptobox/README.md
|
| 539 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/brig/README.md
|
| 540 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/brig/docs/swagger.md
|
| 541 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/cannon/README.md
|
| 542 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/cargohold/README.md
|
| 543 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/federator/README.md
|
| 544 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/federator/test/resources/unit/README.md
|
| 545 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/nginz/integration-test/conf/nginz/README.md
|
| 546 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/nginz/third_party/nginx-zauth-module/README.md
|
| 547 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/restund/README.md
|
| 548 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/spar/README.md
|
| 549 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/services/spar/test-scim-suite/README.md
|
| 550 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/snapshots/README.md
|
| 551 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/assets/README.md
|
| 552 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/auto-whitelist/README.md
|
| 553 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/find-undead/README.md
|
| 554 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/inconsistencies/README.md
|
| 555 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/migrate-sso-feature-flag/README.md
|
| 556 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/move-team/README.md
|
| 557 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/phone-users/README.md
|
| 558 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/repair-brig-clients-table/README.md
|
| 559 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/repair-handles/README.md
|
| 560 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/db/service-backfill/README.md
|
| 561 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/fedcalls/README.md
|
| 562 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/mlsstats/README.md
|
| 563 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/nginz_disco/README.md
|
| 564 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/rabbitmq-consumer/README.md
|
| 565 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/rex/README.md
|
| 566 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/sftd_disco/README.md
|
| 567 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/stern/README.md
|
| 568 |
+
https://raw.githubusercontent.com/wireapp/wire-server/develop/tools/test-stats/README.md
|
| 569 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/.github/PULL_REQUEST_TEMPLATE.md
|
| 570 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/README.md
|
| 571 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/SECURITY.md
|
| 572 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/bot-api/CHANGELOG.md
|
| 573 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/bot-handler-avs/CHANGELOG.md
|
| 574 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/bot-handler-debug/CHANGELOG.md
|
| 575 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/bot-handler-debug/README.md
|
| 576 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/bot-handler-uptime/CHANGELOG.md
|
| 577 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/bot-handler-wizard/CHANGELOG.md
|
| 578 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/bot-handler-wizard/README.md
|
| 579 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/changelog-bot/CHANGELOG.md
|
| 580 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/changelog-bot/README.md
|
| 581 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/store-engine-bro-fs/CHANGELOG.md
|
| 582 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/store-engine-bro-fs/README.md
|
| 583 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/store-engine-sqleet/CHANGELOG.md
|
| 584 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/store-engine-sqleet/README.md
|
| 585 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/store-engine-web-storage/CHANGELOG.md
|
| 586 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/archive/store-engine-web-storage/README.md
|
| 587 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/api-client/CHANGELOG.md
|
| 588 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/api-client/README.md
|
| 589 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/archive/cli-client/CHANGELOG.md
|
| 590 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/archive/cli-client/README.md
|
| 591 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/bazinga64/CHANGELOG.md
|
| 592 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/bazinga64/README.md
|
| 593 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/certificate-check/CHANGELOG.md
|
| 594 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/certificate-check/README.md
|
| 595 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/commons/CHANGELOG.md
|
| 596 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/commons/README.md
|
| 597 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/copy-config/CHANGELOG.md
|
| 598 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/copy-config/README.md
|
| 599 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/core/CHANGELOG.md
|
| 600 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/core/README.md
|
| 601 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/eslint-config/CHANGELOG.md
|
| 602 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/eslint-config/README.md
|
| 603 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/license-collector/CHANGELOG.md
|
| 604 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/license-collector/README.md
|
| 605 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/prettier-config/CHANGELOG.md
|
| 606 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/prettier-config/README.md
|
| 607 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/priority-queue/CHANGELOG.md
|
| 608 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/priority-queue/README.md
|
| 609 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/promise-queue/CHANGELOG.md
|
| 610 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/promise-queue/README.md
|
| 611 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/CHANGELOG.md
|
| 612 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/README.md
|
| 613 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/Button.md
|
| 614 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/ButtonLink.md
|
| 615 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/Checkbox.md
|
| 616 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/CodeInput.md
|
| 617 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/DropFileInput.md
|
| 618 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/ErrorMessage.md
|
| 619 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/Form.md
|
| 620 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/IndicatorRangeInput.md
|
| 621 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/InputLabel.md
|
| 622 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/RangeInput.md
|
| 623 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/RoundIconButton.md
|
| 624 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/Select.md
|
| 625 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/Switch.md
|
| 626 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/TextArea.md
|
| 627 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Form/Tooltip.md
|
| 628 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Icon/SVGIcon.md
|
| 629 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Identity/Animation.md
|
| 630 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Identity/Avatar.md
|
| 631 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Identity/AvatarGrid.md
|
| 632 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Identity/Logo.md
|
| 633 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Identity/colors-v2.md
|
| 634 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Identity/colors.md
|
| 635 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Layout/Box.md
|
| 636 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Layout/Column.md
|
| 637 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Layout/Container.md
|
| 638 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Layout/MatchMedia.md
|
| 639 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Layout/headerMenu/HeaderMenu.md
|
| 640 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Menu/TabBar.md
|
| 641 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Misc/ButtonGroup.md
|
| 642 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Misc/IconButton.md
|
| 643 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Misc/Loading.md
|
| 644 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Misc/Pagination.md
|
| 645 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Misc/Pill.md
|
| 646 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Modal/Modal.md
|
| 647 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Text/Heading.md
|
| 648 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Text/Label.md
|
| 649 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Text/Line.md
|
| 650 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Text/Link.md
|
| 651 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Text/Paragraph.md
|
| 652 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Text/Text.md
|
| 653 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Text/TextLink.md
|
| 654 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/react-ui-kit/src/Text/Title.md
|
| 655 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/store-engine-dexie/CHANGELOG.md
|
| 656 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/store-engine-dexie/README.md
|
| 657 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/store-engine-fs/CHANGELOG.md
|
| 658 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/store-engine-fs/README.md
|
| 659 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/store-engine/CHANGELOG.md
|
| 660 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/store-engine/README.md
|
| 661 |
+
https://raw.githubusercontent.com/wireapp/wire-web-packages/main/packages/webapp-events/CHANGELOG.md
|
| 662 |
+
https://raw.githubusercontent.com/wireapp/Down/develop/.github/ISSUE_TEMPLATE.md
|
| 663 |
+
https://raw.githubusercontent.com/wireapp/Down/develop/CHANGELOG.md
|
| 664 |
+
https://raw.githubusercontent.com/wireapp/Down/develop/README.md
|
| 665 |
+
https://raw.githubusercontent.com/wireapp/wire-emails/master/README.md
|
| 666 |
+
https://raw.githubusercontent.com/wireapp/brand/master/README.md
|
| 667 |
+
https://raw.githubusercontent.com/wireapp/brand/master/text/glossary/index.md
|
| 668 |
+
https://raw.githubusercontent.com/wireapp/brand/master/text/index.md
|
| 669 |
+
https://raw.githubusercontent.com/wireapp/brand/master/text/sample-usage/index.md
|
| 670 |
+
https://raw.githubusercontent.com/wireapp/brand/master/text/tone-of-voice/index.md
|
| 671 |
+
https://raw.githubusercontent.com/wireapp/brand/master/text/translation/index.md
|
| 672 |
+
https://raw.githubusercontent.com/wireapp/brand/master/text/user-interface/index.md
|
| 673 |
+
https://raw.githubusercontent.com/wireapp/brand/master/text/writing-style/index.md
|
| 674 |
+
https://raw.githubusercontent.com/wireapp/cryptobox4j/master/README.md
|
| 675 |
+
https://raw.githubusercontent.com/wireapp/hsaml2/master/README.md
|
| 676 |
+
https://raw.githubusercontent.com/wireapp/saml2-web-sso/master/README.md
|
| 677 |
+
https://raw.githubusercontent.com/wireapp/hspec-wai/master/CHANGES.md
|
| 678 |
+
https://raw.githubusercontent.com/wireapp/hspec-wai/master/README.md
|
| 679 |
+
https://raw.githubusercontent.com/wireapp/servant-multipart/master/CHANGELOG.md
|
| 680 |
+
https://raw.githubusercontent.com/wireapp/servant-multipart/master/README.md
|
| 681 |
+
https://raw.githubusercontent.com/wireapp/wire-web-ets/dev/README.md
|
| 682 |
+
https://raw.githubusercontent.com/wireapp/wire-web-ets/dev/SECURITY.md
|
| 683 |
+
https://raw.githubusercontent.com/wireapp/re/master/README.md
|
| 684 |
+
https://raw.githubusercontent.com/wireapp/rew/master/README.md
|
| 685 |
+
https://raw.githubusercontent.com/wireapp/rew/master/src/trice/README.md
|
| 686 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/README.md
|
| 687 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/TUTORIAL.md
|
| 688 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/http-client-openssl/ChangeLog.md
|
| 689 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/http-client-openssl/README.md
|
| 690 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/http-client-tls/ChangeLog.md
|
| 691 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/http-client-tls/README.md
|
| 692 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/http-client/ChangeLog.md
|
| 693 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/http-client/README.md
|
| 694 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/http-conduit/ChangeLog.md
|
| 695 |
+
https://raw.githubusercontent.com/wireapp/http-client/master/http-conduit/README.md
|
| 696 |
+
https://raw.githubusercontent.com/wireapp/HTMLString/master/CHANGELOG.md
|
| 697 |
+
https://raw.githubusercontent.com/wireapp/HTMLString/master/README.md
|
| 698 |
+
https://raw.githubusercontent.com/wireapp/restund/master/README.md
|
| 699 |
+
https://raw.githubusercontent.com/wireapp/recording-bot/staging/README.md
|
| 700 |
+
https://raw.githubusercontent.com/wireapp/ansible-cassandra/master/CHANGELOG.md
|
| 701 |
+
https://raw.githubusercontent.com/wireapp/ansible-cassandra/master/README.md
|
| 702 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/.github/ISSUE_TEMPLATE/bug-repport.md
|
| 703 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/.github/ISSUE_TEMPLATE/feature-request.md
|
| 704 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/.github/ISSUE_TEMPLATE/question.md
|
| 705 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/.github/PULL_REQUEST_TEMPLATE/default.md
|
| 706 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/CHANGELOG.md
|
| 707 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/CONTRIBUTING.md
|
| 708 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/README.md
|
| 709 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/ansible/README.md
|
| 710 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/ansible/db-operations/README.md
|
| 711 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/bin/bootstrap/README.md
|
| 712 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/examples/control-planes-only-k8s/README.md
|
| 713 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/examples/multi-instance-sft/README.md
|
| 714 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/examples/multi-node-k8s-with-lb-and-dns/README.md
|
| 715 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/examples/team-provisioning-qr-codes/README.md
|
| 716 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/helm/README.md
|
| 717 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/coturn.md
|
| 718 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/docs_ubuntu_22.04.md
|
| 719 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/federation_preparation.md
|
| 720 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/k8ssandra_setup.md
|
| 721 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/ldap-scim-bridge.md
|
| 722 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/local_persistent_storage_k8s.md
|
| 723 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/rabbitmq_setup.md
|
| 724 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/single_hetzner_machine_installation.md
|
| 725 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/ubuntu_18_to_ubuntu_22_migration.md
|
| 726 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/upgrading-SFT_ONLY.md
|
| 727 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/offline/upgrading.md
|
| 728 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/README.md
|
| 729 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/examples/README.md
|
| 730 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/examples/wire-server-deploy-offline-hetzner/README.md
|
| 731 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/README.md
|
| 732 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/aws-ami-ubuntu-search/README.md
|
| 733 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/aws-brig-prekey-lock-event-queue-email-sending/README.md
|
| 734 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/aws-cargohold-asset-storage/README.md
|
| 735 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/aws-dns-records/README.md
|
| 736 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/aws-network-load-balancer/README.md
|
| 737 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/aws-terraform-state-share/README.md
|
| 738 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/aws-vpc-security-groups/README.md
|
| 739 |
+
https://raw.githubusercontent.com/wireapp/wire-server-deploy/master/terraform/modules/aws-vpc/README.md
|
| 740 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/.github/issue_template.md
|
| 741 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/Documentation/API.md
|
| 742 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/Documentation/CONFORMANCE_TESTS.md
|
| 743 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/Documentation/FAQ.md
|
| 744 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/Documentation/INTERNALS.md
|
| 745 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/Documentation/PLUGIN.md
|
| 746 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/Documentation/RELEASING.md
|
| 747 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/Documentation/STYLE_GUIDELINES.md
|
| 748 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/Protos/README.md
|
| 749 |
+
https://raw.githubusercontent.com/wireapp/swift-protobuf/master/README.md
|
| 750 |
+
https://raw.githubusercontent.com/wireapp/generate-changelog/master/CHANGELOG.md
|
| 751 |
+
https://raw.githubusercontent.com/wireapp/generate-changelog/master/CONTRIBUTING.md
|
| 752 |
+
https://raw.githubusercontent.com/wireapp/generate-changelog/master/README.md
|
| 753 |
+
https://raw.githubusercontent.com/wireapp/wire-web-config-default/master/README.md
|
| 754 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/expectations.md
|
| 755 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/getting-started.md
|
| 756 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/hspec-discover.md
|
| 757 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/hunit.md
|
| 758 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/index.md
|
| 759 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/match.md
|
| 760 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/options.md
|
| 761 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/parallel-spec-execution.md
|
| 762 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/quickcheck.md
|
| 763 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/rerun.md
|
| 764 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/running-specs.md
|
| 765 |
+
https://raw.githubusercontent.com/wireapp/hspec/master/doc/writing-specs.md
|
| 766 |
+
https://raw.githubusercontent.com/wireapp/DifferenceKit/master/.github/ISSUE_TEMPLATE/BUG_REPORT.md
|
| 767 |
+
https://raw.githubusercontent.com/wireapp/DifferenceKit/master/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md
|
| 768 |
+
https://raw.githubusercontent.com/wireapp/DifferenceKit/master/.github/ISSUE_TEMPLATE/QUESTION.md
|
| 769 |
+
https://raw.githubusercontent.com/wireapp/DifferenceKit/master/.github/PULL_REQUEST_TEMPLATE.md
|
| 770 |
+
https://raw.githubusercontent.com/wireapp/DifferenceKit/master/Benchmark/README.md
|
| 771 |
+
https://raw.githubusercontent.com/wireapp/DifferenceKit/master/CODE_OF_CONDUCT.md
|
| 772 |
+
https://raw.githubusercontent.com/wireapp/DifferenceKit/master/CONTRIBUTING.md
|
| 773 |
+
https://raw.githubusercontent.com/wireapp/DifferenceKit/master/README.md
|
| 774 |
+
https://raw.githubusercontent.com/wireapp/wire-web-config-wire/master/README.md
|
| 775 |
+
https://raw.githubusercontent.com/wireapp/prebuilt-webrtc-binaries/main/README.md
|
| 776 |
+
https://raw.githubusercontent.com/wireapp/servant/master/CONTRIBUTING.md
|
| 777 |
+
https://raw.githubusercontent.com/wireapp/servant/master/README.md
|
| 778 |
+
https://raw.githubusercontent.com/wireapp/servant/master/doc/README.md
|
| 779 |
+
https://raw.githubusercontent.com/wireapp/servant/master/doc/examples.md
|
| 780 |
+
https://raw.githubusercontent.com/wireapp/servant/master/nix/README.md
|
| 781 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-client-core/CHANGELOG.md
|
| 782 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-client-core/README.md
|
| 783 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-client-ghcjs/CHANGELOG.md
|
| 784 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-client-ghcjs/README.md
|
| 785 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-client/CHANGELOG.md
|
| 786 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-client/README.md
|
| 787 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-conduit/CHANGELOG.md
|
| 788 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-conduit/README.md
|
| 789 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-docs/CHANGELOG.md
|
| 790 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-docs/README.md
|
| 791 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-docs/example/greet.md
|
| 792 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-docs/golden/comprehensive.md
|
| 793 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-foreign/CHANGELOG.md
|
| 794 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-foreign/README.md
|
| 795 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-http-streams/CHANGELOG.md
|
| 796 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-http-streams/README.md
|
| 797 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-machines/CHANGELOG.md
|
| 798 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-machines/README.md
|
| 799 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-pipes/CHANGELOG.md
|
| 800 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-pipes/README.md
|
| 801 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-server/CHANGELOG.md
|
| 802 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-server/README.md
|
| 803 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-server/example/README.md
|
| 804 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant-server/example/greet.md
|
| 805 |
+
https://raw.githubusercontent.com/wireapp/servant/master/servant/CHANGELOG.md
|
| 806 |
+
https://raw.githubusercontent.com/wireapp/Winium.Desktop/develop/CHANGELOG.md
|
| 807 |
+
https://raw.githubusercontent.com/wireapp/Winium.Desktop/develop/README.md
|
| 808 |
+
https://raw.githubusercontent.com/wireapp/Winium.Desktop/develop/README_RU.md
|
| 809 |
+
https://raw.githubusercontent.com/wireapp/legalhold/staging/README.md
|
| 810 |
+
https://raw.githubusercontent.com/wireapp/appium-for-mac/master/README.md
|
| 811 |
+
https://raw.githubusercontent.com/wireapp/ansible-restund/master/README.md
|
| 812 |
+
https://raw.githubusercontent.com/wireapp/ansible-restund/master/TODO.md
|
| 813 |
+
https://raw.githubusercontent.com/wireapp/ansible-restund/master/molecule/default/INSTALL.md
|
| 814 |
+
https://raw.githubusercontent.com/wireapp/ansible-tinc/master/README.md
|
| 815 |
+
https://raw.githubusercontent.com/wireapp/servant-swagger/master/CHANGELOG.md
|
| 816 |
+
https://raw.githubusercontent.com/wireapp/servant-swagger/master/README.md
|
| 817 |
+
https://raw.githubusercontent.com/wireapp/swift-snapshot-testing/master/.github/CODE_OF_CONDUCT.md
|
| 818 |
+
https://raw.githubusercontent.com/wireapp/swift-snapshot-testing/master/.github/ISSUE_TEMPLATE/bug_report.md
|
| 819 |
+
https://raw.githubusercontent.com/wireapp/swift-snapshot-testing/master/.github/ISSUE_TEMPLATE/question.md
|
| 820 |
+
https://raw.githubusercontent.com/wireapp/swift-snapshot-testing/master/CONTRIBUTING.md
|
| 821 |
+
https://raw.githubusercontent.com/wireapp/swift-snapshot-testing/master/Documentation/Available-Snapshot-Strategies.md
|
| 822 |
+
https://raw.githubusercontent.com/wireapp/swift-snapshot-testing/master/Documentation/Defining-Custom-Snapshot-Strategies.md
|
| 823 |
+
https://raw.githubusercontent.com/wireapp/swift-snapshot-testing/master/README.md
|
| 824 |
+
https://raw.githubusercontent.com/wireapp/servant-checked-exceptions/master/README.md
|
| 825 |
+
https://raw.githubusercontent.com/wireapp/servant-checked-exceptions/master/servant-checked-exceptions-core/CHANGELOG.md
|
| 826 |
+
https://raw.githubusercontent.com/wireapp/servant-checked-exceptions/master/servant-checked-exceptions-core/README.md
|
| 827 |
+
https://raw.githubusercontent.com/wireapp/servant-checked-exceptions/master/servant-checked-exceptions/CHANGELOG.md
|
| 828 |
+
https://raw.githubusercontent.com/wireapp/servant-checked-exceptions/master/servant-checked-exceptions/README.md
|
| 829 |
+
https://raw.githubusercontent.com/wireapp/roman/staging/README.md
|
| 830 |
+
https://raw.githubusercontent.com/wireapp/roman/staging/docs/onboarding.md
|
| 831 |
+
https://raw.githubusercontent.com/wireapp/roman/staging/frontend/README.md
|
| 832 |
+
https://raw.githubusercontent.com/wireapp/wire-avs-nwtesttool/master/README.md
|
| 833 |
+
https://raw.githubusercontent.com/wireapp/wireapp.github.io/master/README.md
|
| 834 |
+
https://raw.githubusercontent.com/wireapp/wireapp.github.io/master/_posts/2020-06-11-company-culture-phylosophy.md
|
| 835 |
+
https://raw.githubusercontent.com/wireapp/wireapp.github.io/master/_posts/2020-06-18-wire-goes-zurihac.md
|
| 836 |
+
https://raw.githubusercontent.com/wireapp/wireapp.github.io/master/_posts/2020-07-10-android-accessibility-development-doesnt-have-to-be-scary.md
|
| 837 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/.github/ISSUE_TEMPLATE/feature_request.md
|
| 838 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/.github/ISSUE_TEMPLATE/problem_report.md
|
| 839 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/.github/PULL_REQUEST_TEMPLATE.md
|
| 840 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/CHANGELOG.md
|
| 841 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/CONTRIBUTING.md
|
| 842 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/Guides/Installation.md
|
| 843 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/iOS/AppCenter/Guides/AppCenter.md
|
| 844 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/iOS/AppCenterAnalytics/Guides/Analytics.md
|
| 845 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/iOS/AppCenterCrashes/Guides/Crashes.md
|
| 846 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/iOS/AppCenterDistribute/Guides/Distribute.md
|
| 847 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/macOS/AppCenter/Guides/AppCenter.md
|
| 848 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/macOS/AppCenterAnalytics/Guides/Analytics.md
|
| 849 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/macOS/AppCenterCrashes/Guides/Crashes.md
|
| 850 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/tvOS/AppCenter/Guides/AppCenter.md
|
| 851 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/tvOS/AppCenterAnalytics/Guides/Analytics.md
|
| 852 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/Documentation/tvOS/AppCenterCrashes/Guides/Crashes.md
|
| 853 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/README.md
|
| 854 |
+
https://raw.githubusercontent.com/wireapp/appcenter-sdk-apple/develop/SECURITY.md
|
| 855 |
+
https://raw.githubusercontent.com/wireapp/poll-bot/staging/README.md
|
| 856 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/README.md
|
| 857 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40actions/core/LICENSE.md
|
| 858 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40actions/core/README.md
|
| 859 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/aspromise/README.md
|
| 860 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/base64/README.md
|
| 861 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/codegen/README.md
|
| 862 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/eventemitter/README.md
|
| 863 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/fetch/README.md
|
| 864 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/float/README.md
|
| 865 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/inquire/README.md
|
| 866 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/path/README.md
|
| 867 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/pool/README.md
|
| 868 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40protobufjs/utf8/README.md
|
| 869 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40tokenizer/token/README.md
|
| 870 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/fs-extra/README.md
|
| 871 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/fs-extra/node_modules/%40types/node/README.md
|
| 872 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/libsodium-wrappers-sumo/README.md
|
| 873 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/libsodium-wrappers/README.md
|
| 874 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/long/README.md
|
| 875 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/node/README.md
|
| 876 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/platform/README.md
|
| 877 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/spark-md5/README.md
|
| 878 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40types/tough-cookie/README.md
|
| 879 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/api-client/CHANGELOG.md
|
| 880 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/api-client/README.md
|
| 881 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/api-client/node_modules/%40wireapp/priority-queue/CHANGELOG.md
|
| 882 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/api-client/node_modules/%40wireapp/priority-queue/README.md
|
| 883 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/bot-api/CHANGELOG.md
|
| 884 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/bot-api/node_modules/%40types/node/README.md
|
| 885 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/cbor/CHANGELOG.md
|
| 886 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/cbor/README.md
|
| 887 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/commons/CHANGELOG.md
|
| 888 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/commons/README.md
|
| 889 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/core/CHANGELOG.md
|
| 890 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/core/README.md
|
| 891 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/cryptobox/CHANGELOG.md
|
| 892 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/cryptobox/README.md
|
| 893 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/cryptobox/node_modules/%40wireapp/store-engine/CHANGELOG.md
|
| 894 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/cryptobox/node_modules/%40wireapp/store-engine/README.md
|
| 895 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/cryptobox/node_modules/bazinga64/CHANGELOG.md
|
| 896 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/cryptobox/node_modules/bazinga64/README.md
|
| 897 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/lru-cache/CHANGELOG.md
|
| 898 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/lru-cache/README.md
|
| 899 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/priority-queue/CHANGELOG.md
|
| 900 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/priority-queue/README.md
|
| 901 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/proteus/CHANGELOG.md
|
| 902 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/proteus/README.md
|
| 903 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/README.md
|
| 904 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/%40types/node/README.md
|
| 905 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/CHANGELOG.md
|
| 906 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/README.md
|
| 907 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/cli/README.md
|
| 908 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/cli/lib/tsd-jsdoc/README.md
|
| 909 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/dist/README.md
|
| 910 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/dist/light/README.md
|
| 911 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/dist/minimal/README.md
|
| 912 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/ext/debug/README.md
|
| 913 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/ext/descriptor/README.md
|
| 914 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/protocol-messaging/node_modules/protobufjs/google/README.md
|
| 915 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/store-engine-fs/CHANGELOG.md
|
| 916 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/store-engine-fs/README.md
|
| 917 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/store-engine/CHANGELOG.md
|
| 918 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/%40wireapp/store-engine/README.md
|
| 919 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/ansi-regex/readme.md
|
| 920 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/ansi-styles/readme.md
|
| 921 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios-retry/CHANGELOG.md
|
| 922 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios-retry/README.md
|
| 923 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios/CHANGELOG.md
|
| 924 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios/README.md
|
| 925 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios/SECURITY.md
|
| 926 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios/UPGRADE_GUIDE.md
|
| 927 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios/lib/adapters/README.md
|
| 928 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios/lib/core/README.md
|
| 929 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/axios/lib/helpers/README.md
|
| 930 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/base64-js/README.md
|
| 931 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/bazinga64/CHANGELOG.md
|
| 932 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/bazinga64/README.md
|
| 933 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/buffer/AUTHORS.md
|
| 934 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/buffer/README.md
|
| 935 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/chalk/readme.md
|
| 936 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/color-convert/CHANGELOG.md
|
| 937 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/color-convert/README.md
|
| 938 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/color-name/README.md
|
| 939 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/dotenv/CHANGELOG.md
|
| 940 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/dotenv/README.md
|
| 941 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/escape-string-regexp/readme.md
|
| 942 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/file-type/readme.md
|
| 943 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/follow-redirects/README.md
|
| 944 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/fs-extra/CHANGELOG.md
|
| 945 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/fs-extra/README.md
|
| 946 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/graceful-fs/README.md
|
| 947 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/has-flag/readme.md
|
| 948 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/hash.js/README.md
|
| 949 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/http-status-codes/README.md
|
| 950 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/ieee754/README.md
|
| 951 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/inherits/README.md
|
| 952 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/is-retry-allowed/readme.md
|
| 953 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/jsonfile/CHANGELOG.md
|
| 954 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/jsonfile/README.md
|
| 955 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/jsonfile/node_modules/universalify/README.md
|
| 956 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/libsodium-sumo/README.md
|
| 957 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/libsodium-wrappers-sumo/README.md
|
| 958 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/logdown/changelog.md
|
| 959 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/logdown/license.md
|
| 960 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/logdown/readme.md
|
| 961 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/long/README.md
|
| 962 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/minimalistic-assert/readme.md
|
| 963 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/peek-readable/README.md
|
| 964 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/platform/README.md
|
| 965 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/CHANGELOG.md
|
| 966 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/README.md
|
| 967 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/README.md
|
| 968 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/lib/tsd-jsdoc/README.md
|
| 969 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/acorn-jsx/README.md
|
| 970 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/acorn/CHANGELOG.md
|
| 971 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/acorn/README.md
|
| 972 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/balanced-match/LICENSE.md
|
| 973 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/balanced-match/README.md
|
| 974 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/brace-expansion/README.md
|
| 975 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/escodegen/README.md
|
| 976 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/eslint-visitor-keys/CHANGELOG.md
|
| 977 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/eslint-visitor-keys/README.md
|
| 978 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/espree/CHANGELOG.md
|
| 979 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/espree/README.md
|
| 980 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/esprima/README.md
|
| 981 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/estraverse/README.md
|
| 982 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/esutils/README.md
|
| 983 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/fast-levenshtein/LICENSE.md
|
| 984 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/fast-levenshtein/README.md
|
| 985 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/fs.realpath/README.md
|
| 986 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/glob/README.md
|
| 987 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/glob/changelog.md
|
| 988 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/inflight/README.md
|
| 989 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/inherits/README.md
|
| 990 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/levn/README.md
|
| 991 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/minimatch/README.md
|
| 992 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/once/README.md
|
| 993 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/optionator/CHANGELOG.md
|
| 994 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/optionator/README.md
|
| 995 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/path-is-absolute/readme.md
|
| 996 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/prelude-ls/CHANGELOG.md
|
| 997 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/prelude-ls/README.md
|
| 998 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/rimraf/CHANGELOG.md
|
| 999 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/rimraf/README.md
|
| 1000 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/source-map/CHANGELOG.md
|
| 1001 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/source-map/README.md
|
| 1002 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/tmp/CHANGELOG.md
|
| 1003 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/tmp/README.md
|
| 1004 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/type-check/README.md
|
| 1005 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/word-wrap/README.md
|
| 1006 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/cli/node_modules/wrappy/README.md
|
| 1007 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/dist/README.md
|
| 1008 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/dist/light/README.md
|
| 1009 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/dist/minimal/README.md
|
| 1010 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/ext/debug/README.md
|
| 1011 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/ext/descriptor/README.md
|
| 1012 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/google/README.md
|
| 1013 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/protobufjs/node_modules/%40types/node/README.md
|
| 1014 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/psl/README.md
|
| 1015 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/punycode/README.md
|
| 1016 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/readable-stream/CONTRIBUTING.md
|
| 1017 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/readable-stream/GOVERNANCE.md
|
| 1018 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/readable-stream/README.md
|
| 1019 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/readable-web-to-node-stream/README.md
|
| 1020 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/reconnecting-websocket/CHANGELOG.md
|
| 1021 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/reconnecting-websocket/CONTRIBUTING.md
|
| 1022 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/reconnecting-websocket/README.md
|
| 1023 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/safe-buffer/README.md
|
| 1024 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/spark-md5/README.md
|
| 1025 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/string_decoder/README.md
|
| 1026 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/strtok3/README.md
|
| 1027 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/supports-color/readme.md
|
| 1028 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/token-types/README.md
|
| 1029 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/tough-cookie/README.md
|
| 1030 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/tough-cookie/node_modules/universalify/README.md
|
| 1031 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/universalify/README.md
|
| 1032 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/url-search-params-polyfill/README.md
|
| 1033 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/util-deprecate/History.md
|
| 1034 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/util-deprecate/README.md
|
| 1035 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/uuidjs/README.md
|
| 1036 |
+
https://raw.githubusercontent.com/wireapp/github-action-wire-messenger/main/node_modules/ws/README.md
|
| 1037 |
+
https://raw.githubusercontent.com/wireapp/bloodhound/master/README.md
|
| 1038 |
+
https://raw.githubusercontent.com/wireapp/bloodhound/master/changelog.md
|
| 1039 |
+
https://raw.githubusercontent.com/wireapp/bloodhound/master/examples/README.md
|
| 1040 |
+
https://raw.githubusercontent.com/wireapp/charon/staging/README.md
|
| 1041 |
+
https://raw.githubusercontent.com/wireapp/echo-bot-roman/staging/README.md
|
| 1042 |
+
https://raw.githubusercontent.com/wireapp/ansible-ntp-verify/master/README.md
|
| 1043 |
+
https://raw.githubusercontent.com/wireapp/wire-android/develop/.github/pull_request_template.md
|
| 1044 |
+
https://raw.githubusercontent.com/wireapp/wire-android/develop/CUSTOMIZATION.md
|
| 1045 |
+
https://raw.githubusercontent.com/wireapp/wire-android/develop/README.md
|
gradio_interface.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from haystack import Pipeline
|
| 2 |
+
from haystack.core.serialization import DeserializationCallbacks
|
| 3 |
+
from typing import Type, Dict, Any
|
| 4 |
+
import gradio as gr
|
| 5 |
+
import os
|
| 6 |
+
|
| 7 |
+
def component_pre_init_callback(component_name: str, component_cls: Type, init_params: Dict[str, Any]):
|
| 8 |
+
# This function gets called every time a component is deserialized.
|
| 9 |
+
if component_name == "cleaner":
|
| 10 |
+
assert "DocumentCleaner" in component_cls.__name__
|
| 11 |
+
# Modify the init parameters. The modified parameters are passed to
|
| 12 |
+
# the init method of the component during deserialization.
|
| 13 |
+
init_params["remove_empty_lines"] = False
|
| 14 |
+
print("Modified 'remove_empty_lines' to False in 'cleaner' component")
|
| 15 |
+
else:
|
| 16 |
+
print(f"Not modifying component {component_name} of class {component_cls}")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
# Load the pipeline from the YAML file
|
| 20 |
+
def load_pipeline_from_yaml(yaml_file_path):
|
| 21 |
+
with open(yaml_file_path, "r") as stream:
|
| 22 |
+
pipeline_yaml = stream.read()
|
| 23 |
+
return Pipeline.loads(pipeline_yaml, callbacks=DeserializationCallbacks(component_pre_init_callback))
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
# Function to interact with the pipeline
|
| 27 |
+
def ask_question(question, pipeline):
|
| 28 |
+
if question == "" or question is None:
|
| 29 |
+
return ""
|
| 30 |
+
answer = pipeline.run({
|
| 31 |
+
"text_embedder": {"text": question},
|
| 32 |
+
"prompt_builder": {"question": question},
|
| 33 |
+
"answer_builder": {"query": question}
|
| 34 |
+
})
|
| 35 |
+
return answer['answer_builder']['answers'][0].data
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# Load the pipeline (modify path if necessary)
|
| 39 |
+
my_pipeline = load_pipeline_from_yaml('./pipeline.yml')
|
| 40 |
+
|
| 41 |
+
# Set up Gradio interface with Blocks layout
|
| 42 |
+
with gr.Blocks() as interface:
|
| 43 |
+
gr.Markdown("# Wire RAG Documentation") # Title
|
| 44 |
+
|
| 45 |
+
input_box = gr.Textbox(label="Ask your question:", placeholder="Type your question here...", lines=1)
|
| 46 |
+
|
| 47 |
+
# Move the submit button above the output box
|
| 48 |
+
submit_btn = gr.Button("Submit") # Submit button
|
| 49 |
+
output_box = gr.Markdown(label="Answer:") # Output field
|
| 50 |
+
|
| 51 |
+
# Set the click function for the button
|
| 52 |
+
submit_btn.click(fn=lambda question: ask_question(question, my_pipeline), inputs=input_box, outputs=output_box)
|
| 53 |
+
|
| 54 |
+
# Set the enter key behavior for the input box
|
| 55 |
+
input_box.submit(fn=lambda question: ask_question(question, my_pipeline), inputs=input_box, outputs=output_box)
|
| 56 |
+
|
| 57 |
+
# Launch the Gradio app with sharing and authentication
|
| 58 |
+
interface.launch(share=True, auth=("user", os.getenv("GRADIO_KEY")))
|
main_rag.ipynb
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
output.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Summary
|
| 2 |
+
|
| 3 |
+
The documentation collectively outlines the framework and requirements for making requests between two federated backends using the Wire 0.0.4 API. Key points emphasize the roles of the *Federator* and *Federation Ingress* components in facilitating communication between backends, including authentication and authorization processes. The most recent updates highlight enhancements in processing federated requests, the ability to send requests to multiple backends in parallel, and the evolution of API conventions to improve functionality.
|
| 4 |
+
|
| 5 |
+
# Detailed Analysis
|
| 6 |
+
|
| 7 |
+
## Federated Requests Overview
|
| 8 |
+
- According to the **2023-01-10** document, every federated API request involves a service component (like brig or galley) in one backend, which communicates through the *Federator*. The response is relayed back via the *Federator Ingress* in the other backend.
|
| 9 |
+
|
| 10 |
+
## Backend to Backend Communication
|
| 11 |
+
- The document marked **None** specifies critical checks necessary for inter-backend communication:
|
| 12 |
+
- **Authentication**: Determine the identity (infrastructure domain name) of the other backend.
|
| 13 |
+
- **Discovery**: Ensure that the other backend is properly authorized.
|
| 14 |
+
- **Authorization**: Confirm that each backend is approved to federate with each other.
|
| 15 |
+
|
| 16 |
+
## Federation Architecture
|
| 17 |
+
- In another **None** date document, the architecture is described: each backend consists of *Federation Ingress* and *Federator*, which are key to the backend's incoming and outgoing request management.
|
| 18 |
+
|
| 19 |
+
## Allow List Configuration
|
| 20 |
+
- According to the documents dated **2020-12-15**, federation is initially disabled, requiring specific backends to be added to an allow list for federated communication. They must also trust each other’s CA certificate for interactions to function correctly.
|
| 21 |
+
|
| 22 |
+
## API Changes and Enhancements
|
| 23 |
+
- The content released on **2024-07-09** notes recent changes allowing federated requests to multiple backends in parallel, enhancing conversation management by ensuring updates apply to remote users and confirming that the updated API conventions align with existing functionalities.
|
| 24 |
+
|
| 25 |
+
## Example Request Flow
|
| 26 |
+
The flow of requests between the federated components can include the following pseudocode:
|
| 27 |
+
|
| 28 |
+
```plaintext
|
| 29 |
+
SenderBackend → Federator → Federation Ingress → ReceiverBackend
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
## Prerequisites for Testing Federation
|
| 33 |
+
To verify that federation works, the following prerequisites are needed:
|
| 34 |
+
- Two configured backends with federation enabled.
|
| 35 |
+
- An allow list containing each other.
|
| 36 |
+
- Trust in each other's CA certificate.
|
| 37 |
+
- Creation of user accounts across both backends.
|
| 38 |
+
|
| 39 |
+
## Updated Functionalities
|
| 40 |
+
The latest updates include changes to request handling, namespace unification in conversation IDs, and new endpoints for remote connections.
|
| 41 |
+
|
| 42 |
+
# Contradictions
|
| 43 |
+
|
| 44 |
+
The listed documents did not have direct contradictions but provided varying levels of detail regarding the same aspects of federation. However, the general principles in the documentation clearly line up, and to ensure comprehensive coverage, I will present relevant alternatives without special mention of conflicts since they are reconcilable:
|
| 45 |
+
|
| 46 |
+
- **Authentication and Authorization Requirements**:
|
| 47 |
+
- From the **2023-01-10** document, requests must be relayed via components while ensuring the appropriate authentication and authorization.
|
| 48 |
+
- The document with **None** makes explicit mention of the need for both mutual authentication and authorization for determining backend identities.
|
| 49 |
+
|
| 50 |
+
Each document reinforces these requirements without conflicting information, implying consistent expectations for backend communication within federated systems.
|
pipeline.yml
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
components:
|
| 2 |
+
answer_builder:
|
| 3 |
+
init_parameters:
|
| 4 |
+
pattern: null
|
| 5 |
+
reference_pattern: null
|
| 6 |
+
type: haystack.components.builders.answer_builder.AnswerBuilder
|
| 7 |
+
generator:
|
| 8 |
+
init_parameters:
|
| 9 |
+
api_base_url: null
|
| 10 |
+
api_key:
|
| 11 |
+
env_vars:
|
| 12 |
+
- OPENAI_API_KEY
|
| 13 |
+
strict: true
|
| 14 |
+
type: env_var
|
| 15 |
+
generation_kwargs: {}
|
| 16 |
+
model: gpt-4o-mini
|
| 17 |
+
organization: null
|
| 18 |
+
streaming_callback: null
|
| 19 |
+
system_prompt: null
|
| 20 |
+
type: haystack.components.generators.openai.OpenAIGenerator
|
| 21 |
+
prompt_builder:
|
| 22 |
+
init_parameters:
|
| 23 |
+
required_variables: null
|
| 24 |
+
template: "You are a knowledgeable assistant responsible for creating comprehensive\
|
| 25 |
+
\ documentation based on the following list of documents that refer to the\
|
| 26 |
+
\ user\u2019s question. The content of these documents may contradict each\
|
| 27 |
+
\ other, so please prioritize the information from the documents with the\
|
| 28 |
+
\ most recent dates. If there are contradicting documents with dates labeled\
|
| 29 |
+
\ as 'None', provide all alternatives and explicitly indicate which parts\
|
| 30 |
+
\ contradict one another. However, if a document with date labeled as 'None'\
|
| 31 |
+
\ does not conflict with others, it should be included without special mention.\n\
|
| 32 |
+
\nInstructions:\n2. Analyze the Documents:\n - Review each document, noting\
|
| 33 |
+
\ any conflicting information.\n - Prioritize information from the most\
|
| 34 |
+
\ recent documents.\n3. Handling Documents with 'None' Date:\n - If a document\
|
| 35 |
+
\ has a date marked as 'None':\n - Include all relevant alternatives and\
|
| 36 |
+
\ clearly indicate contradictions.\n - If it does not conflict with other\
|
| 37 |
+
\ documents, include it without special mention.\n\nOutput Format:\nYour output\
|
| 38 |
+
\ should be structured using Markdown and include the following sections:\n\
|
| 39 |
+
1. Summary:\n - Provide a brief overview of the key findings from all documents.\n\
|
| 40 |
+
2. Detailed Analysis:\n - Present detailed descriptions of key points, prioritizing\
|
| 41 |
+
\ the latest information.\n - Preserve and format any code snippets from\
|
| 42 |
+
\ the documents appropriately.\n - Present full semantic context retrieved\
|
| 43 |
+
\ from given documents.\n3. Contradictions:\n - For documents dated 'None',\
|
| 44 |
+
\ list all relevant alternatives and explicitly highlight any contradictions.\n\
|
| 45 |
+
\ - Do not generate this section if there are no contradictions.\n\nGeneral\
|
| 46 |
+
\ Guidelines:\n- Ensure thoroughness by including all relevant information,\
|
| 47 |
+
\ aiming for completeness rather than brevity.\n- Use headings, lists, and\
|
| 48 |
+
\ code blocks to enhance readability and organization.\n- Given .md files\
|
| 49 |
+
\ should be the base structure of generated file. If .md files are poor, treat\
|
| 50 |
+
\ them as regular source.\n\nUser Question: {{question}}\nDocuments to Analyze:\n\
|
| 51 |
+
{% for doc in documents %}\nDate: {{doc.meta['date']}}\nTitle: {{doc.meta['title']}}\
|
| 52 |
+
\ - {{doc.meta['headline']}}\nContent: \n{{doc.content}}\n{% endfor %}"
|
| 53 |
+
variables: null
|
| 54 |
+
type: haystack.components.builders.prompt_builder.PromptBuilder
|
| 55 |
+
retriever:
|
| 56 |
+
init_parameters:
|
| 57 |
+
document_store:
|
| 58 |
+
init_parameters:
|
| 59 |
+
api_key:
|
| 60 |
+
env_vars:
|
| 61 |
+
- PINECONE_API_KEY
|
| 62 |
+
strict: true
|
| 63 |
+
type: env_var
|
| 64 |
+
batch_size: 100
|
| 65 |
+
dimension: 384
|
| 66 |
+
index: default
|
| 67 |
+
metric: cosine
|
| 68 |
+
namespace: default
|
| 69 |
+
spec:
|
| 70 |
+
serverless:
|
| 71 |
+
cloud: aws
|
| 72 |
+
region: us-east-1
|
| 73 |
+
type: haystack_integrations.document_stores.pinecone.document_store.PineconeDocumentStore
|
| 74 |
+
filter_policy: replace
|
| 75 |
+
filters: {}
|
| 76 |
+
top_k: 25
|
| 77 |
+
type: haystack_integrations.components.retrievers.pinecone.embedding_retriever.PineconeEmbeddingRetriever
|
| 78 |
+
text_embedder:
|
| 79 |
+
init_parameters:
|
| 80 |
+
batch_size: 32
|
| 81 |
+
device:
|
| 82 |
+
device: cuda:0
|
| 83 |
+
type: single
|
| 84 |
+
model: sentence-transformers/all-MiniLM-L6-v2
|
| 85 |
+
model_kwargs: null
|
| 86 |
+
normalize_embeddings: false
|
| 87 |
+
precision: float32
|
| 88 |
+
prefix: ''
|
| 89 |
+
progress_bar: true
|
| 90 |
+
suffix: ''
|
| 91 |
+
token:
|
| 92 |
+
env_vars:
|
| 93 |
+
- HF_API_TOKEN
|
| 94 |
+
- HF_TOKEN
|
| 95 |
+
strict: false
|
| 96 |
+
type: env_var
|
| 97 |
+
tokenizer_kwargs: null
|
| 98 |
+
truncate_dim: null
|
| 99 |
+
trust_remote_code: false
|
| 100 |
+
type: haystack.components.embedders.sentence_transformers_text_embedder.SentenceTransformersTextEmbedder
|
| 101 |
+
connections:
|
| 102 |
+
- receiver: retriever.query_embedding
|
| 103 |
+
sender: text_embedder.embedding
|
| 104 |
+
- receiver: prompt_builder.documents
|
| 105 |
+
sender: retriever.documents
|
| 106 |
+
- receiver: answer_builder.documents
|
| 107 |
+
sender: retriever.documents
|
| 108 |
+
- receiver: generator.prompt
|
| 109 |
+
sender: prompt_builder.prompt
|
| 110 |
+
- receiver: answer_builder.replies
|
| 111 |
+
sender: generator.replies
|
| 112 |
+
max_loops_allowed: 100
|
| 113 |
+
metadata: {}
|
playground/Data/Outputs/output.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Stern/Backoffice Tool Documentation
|
| 2 |
+
|
| 3 |
+
## Summary
|
| 4 |
+
This documentation provides an overview of the processes and tools used within the stern/Backoffice tool for generating Swagger-based API documentation. The latest methods, libraries, and integration points are highlighted, with a special focus on the implementation of `servant-swagger-ui` for local Swagger documentation, feature flags, and various SCIM-related documentations.
|
| 5 |
+
|
| 6 |
+
## Detailed Analysis
|
| 7 |
+
|
| 8 |
+
### Latest Information from 2024-07-09 Documents
|
| 9 |
+
|
| 10 |
+
- **Integration of `servant-swagger-ui`**:
|
| 11 |
+
- The `servant-swagger-ui` has been integrated with the `brig` service to facilitate easier interaction with API documentation.
|
| 12 |
+
- This enables local viewing of Swagger documentation, ensuring users can explore, make requests, and generate client code from the API documentation interactively.
|
| 13 |
+
|
| 14 |
+
- **SCIM Documentation**:
|
| 15 |
+
- Features related to SCIM, such as the `validateSAMLemails` feature, are thoroughly documented in `docs/reference/spar-braindump.md`.
|
| 16 |
+
|
| 17 |
+
- **Feature Flags**:
|
| 18 |
+
- Documentation includes notes on specific feature flags, such as `setEmailVisibility`, detailing their usage and implications within the API.
|
| 19 |
+
|
| 20 |
+
- **Swagger Documentation Instructions**:
|
| 21 |
+
- Detailed instructions on accessing and viewing Swagger documentation locally are provided (referenced as #1388).
|
| 22 |
+
|
| 23 |
+
- **Cassandra Schema**:
|
| 24 |
+
- Updated `cassandra-schema.cql` files are included, indicating changes and improvements in the database schema.
|
| 25 |
+
|
| 26 |
+
- **Unused Registration Flow**:
|
| 27 |
+
- Notes are added concerning unused registration flows, which may affect how developers interact with and test the APIs.
|
| 28 |
+
|
| 29 |
+
### Semi-Automatic Swagger Documentation with Servant (Date: None)
|
| 30 |
+
- The system utilizes the `servant-swagger2` library to semi-automatically generate Swagger documentation.
|
| 31 |
+
- The use of the `schema-profunctor` library aids in creating "schemas" for the input and output types used in the Servant APIs, which encapsulate the information needed for JSON serialization/deserialization, alongside documentation and metadata for Swagger generation.
|
| 32 |
+
|
| 33 |
+
### 2023-01-16 Information
|
| 34 |
+
- Swagger/OpenAPI documentation in the staging system is accurate and complete up to bots/services and event notification payloads.
|
| 35 |
+
- This documentation facilitates three primary uses:
|
| 36 |
+
1. As a reference.
|
| 37 |
+
2. For generating client code.
|
| 38 |
+
3. For interactively exploring the API by making requests.
|
| 39 |
+
|
| 40 |
+
### Code Snippets and Examples
|
| 41 |
+
Here is an example of how `servant-swagger-ui` can be integrated:
|
| 42 |
+
|
| 43 |
+
```haskell
|
| 44 |
+
import Servant
|
| 45 |
+
import Servant.Swagger.UI
|
| 46 |
+
|
| 47 |
+
type API = ...
|
| 48 |
+
|
| 49 |
+
server :: Server API
|
| 50 |
+
server = ...
|
| 51 |
+
|
| 52 |
+
main :: IO ()
|
| 53 |
+
main = run 8080 $ serve (Proxy :: Proxy API) server
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
To view Swagger documentation locally, you might follow these steps:
|
| 57 |
+
1. Ensure the `servant-swagger-ui` is properly installed and configured.
|
| 58 |
+
2. Run the server and open the provided local URL in your browser.
|
| 59 |
+
|
| 60 |
+
## Contradictions
|
| 61 |
+
|
| 62 |
+
**Documents Dated 'None' vs. Other Dated Documents**
|
| 63 |
+
|
| 64 |
+
- **Semi-Automatic Generation vs. Updates in 2024-07-09**:
|
| 65 |
+
- The "None" dated document discusses semi-automatic Swagger documentation generation using `servant-swagger2`.
|
| 66 |
+
- The latest documents from 2024-07-09 detail the specific integration of `servant-swagger-ui` into the `brig` service, emphasizing how Swagger documentation can be accessed locally.
|
| 67 |
+
|
| 68 |
+
These points complement rather than contradict they highlight the toolsets (one for generation and the other for visualization and interaction).
|
| 69 |
+
|
| 70 |
+
**Documents Dated 'None' vs. 2023-01-16**:
|
| 71 |
+
- **Completeness of Documentation**:
|
| 72 |
+
- The "None" dated document does not give specifics on the completeness of the Swagger documentation.
|
| 73 |
+
- The 2023-01-16 document assures the completeness up to bots/services and event notification payloads.
|
| 74 |
+
|
| 75 |
+
By including both these perspectives, users get a detailed understanding of the documentation's completeness and the evolution of tools used.
|
playground/Data/Test_Case/1.md
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# DataAnalyzer
|
| 3 |
+
|
| 4 |
+
**Version**: 2.5.0
|
| 5 |
+
**Author**: Tech Innovations
|
| 6 |
+
|
| 7 |
+
## Overview
|
| 8 |
+
|
| 9 |
+
The `DataAnalyzer` package provides a fast, optimized approach to data analysis, particularly for small datasets.
|
| 10 |
+
It is designed for personal and academic projects rather than large-scale datasets.
|
| 11 |
+
|
| 12 |
+
## Functions:
|
| 13 |
+
|
| 14 |
+
1. `process_data(dataset: list) -> dict`
|
| 15 |
+
This function processes the dataset and returns basic statistics such as:
|
| 16 |
+
- Mean
|
| 17 |
+
- Mode
|
| 18 |
+
- Variance
|
| 19 |
+
|
| 20 |
+
Example:
|
| 21 |
+
```python
|
| 22 |
+
from DataAnalyzer import process_data
|
| 23 |
+
|
| 24 |
+
dataset = [2, 8, 22, 18, 25]
|
| 25 |
+
stats = process_data(dataset)
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
2. `graph_data(dataset: list, chart: str = 'scatter') -> None`
|
| 29 |
+
This function generates a chart based on the dataset. The default chart type is 'scatter'.
|
| 30 |
+
|
| 31 |
+
## Installation
|
| 32 |
+
|
| 33 |
+
- Install via pip:
|
| 34 |
+
```bash
|
| 35 |
+
pip install analyzer-package
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
## Dependencies
|
| 39 |
+
- Python 3.6+
|
| 40 |
+
- `pandas`
|
| 41 |
+
- `seaborn`
|
playground/Data/Test_Case/1.pdf
ADDED
|
Binary file (2.53 kB). View file
|
|
|
playground/Data/Test_Case/1.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Meeting notes
|
| 2 |
+
|
| 3 |
+
- The function `process_data` replaces `analyze_data`.
|
| 4 |
+
- Default chart type for `graph_data` changed from `bar` to `scatter`.
|
| 5 |
+
- Added support for new chart types: 'heatmap' and 'pie'.
|
| 6 |
+
- Improved performance for small datasets.
|
playground/Data/Test_Case/2.md
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# README
|
| 3 |
+
|
| 4 |
+
## Project Description
|
| 5 |
+
|
| 6 |
+
This Java project is designed to demonstrate basic input/output handling. The `Main` class reads user input, processes it, and outputs a result. Contrary to the PDF documentation, the project heavily relies on user interaction.
|
| 7 |
+
|
| 8 |
+
### Main Components
|
| 9 |
+
|
| 10 |
+
1. **Main Class**:
|
| 11 |
+
- The entry point of the program where user input is handled.
|
| 12 |
+
|
| 13 |
+
2. **Functions**:
|
| 14 |
+
- The program has key functions:
|
| 15 |
+
- `funName2()`: This function handles user input and processing.
|
| 16 |
+
- `helperFunction()`: A helper function for data validation.
|
| 17 |
+
|
| 18 |
+
### How to Run
|
| 19 |
+
|
| 20 |
+
1. Compile the `Main.java` file.
|
| 21 |
+
2. Run the program, and it will prompt the user for input.
|
| 22 |
+
|
| 23 |
+
### Sample Code
|
| 24 |
+
|
| 25 |
+
```java
|
| 26 |
+
public class Main {
|
| 27 |
+
public static void main(String[] args) {
|
| 28 |
+
System.out.println("Please enter a value:");
|
| 29 |
+
int value = new java.util.Scanner(System.in).nextInt();
|
| 30 |
+
System.out.println(funName2(value));
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
public static int funName2(int x) {
|
| 34 |
+
return helperFunction(x);
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
public static int helperFunction(int x) {
|
| 38 |
+
return x * 3;
|
| 39 |
+
}
|
| 40 |
+
}
|
| 41 |
+
```
|
playground/Data/Test_Case/2.pdf
ADDED
|
Binary file (2.19 kB). View file
|
|
|
playground/Data/Test_Case/2.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Patch Notes
|
| 3 |
+
|
| 4 |
+
- funName1 was changed to funName2.
|
| 5 |
+
- funName2 now accepts user input.
|
| 6 |
+
- helperFunction now multiplies the input by 3 instead of 2.
|
playground/Data/Test_Case/3.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# README
|
| 3 |
+
|
| 4 |
+
## Project Description
|
| 5 |
+
|
| 6 |
+
This Python script is an interactive command-line tool designed for processing real-time user input, not file handling as suggested by the documentation.
|
| 7 |
+
|
| 8 |
+
### Main Components
|
| 9 |
+
|
| 10 |
+
1. **Main Script**:
|
| 11 |
+
- The script prompts the user for input, processes it, and outputs results.
|
| 12 |
+
|
| 13 |
+
2. **Functions**:
|
| 14 |
+
- `process_input()`: Takes user input and returns processed content.
|
| 15 |
+
- `helper_function()`: Formats the processed input.
|
| 16 |
+
|
| 17 |
+
### How to Run
|
| 18 |
+
|
| 19 |
+
1. Run the script `main.py` and enter data when prompted.
|
| 20 |
+
|
| 21 |
+
### Sample Code
|
| 22 |
+
|
| 23 |
+
```python
|
| 24 |
+
def process_input():
|
| 25 |
+
user_input = input("Enter some text: ")
|
| 26 |
+
return helper_function(user_input)
|
| 27 |
+
|
| 28 |
+
def helper_function(data):
|
| 29 |
+
return data.lower()
|
| 30 |
+
|
| 31 |
+
if __name__ == "__main__":
|
| 32 |
+
print(process_input())
|
| 33 |
+
```
|
playground/Data/Test_Case/3.pdf
ADDED
|
Binary file (1.8 kB). View file
|
|
|
playground/Data/Test_Case/3.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Patch Notes
|
| 3 |
+
|
| 4 |
+
- process_file was renamed to process_input.
|
| 5 |
+
- process_input now accepts user input instead of reading from a file.
|
| 6 |
+
- helper_function now converts text to lowercase instead of uppercase.
|
playground/Data/Test_Case/4.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# README
|
| 3 |
+
|
| 4 |
+
## Project Description
|
| 5 |
+
|
| 6 |
+
This JavaScript program is used for advanced mathematical operations like subtraction and addition.
|
| 7 |
+
|
| 8 |
+
### Main Components
|
| 9 |
+
|
| 10 |
+
1. **Main Script**:
|
| 11 |
+
- The program performs complex subtraction and addition operations.
|
| 12 |
+
|
| 13 |
+
2. **Functions**:
|
| 14 |
+
- `subtractNumbers()`: Takes two arguments and returns their difference.
|
| 15 |
+
|
| 16 |
+
### How to Run
|
| 17 |
+
|
| 18 |
+
1. Load the script in your browser's developer console to see the results.
|
| 19 |
+
|
| 20 |
+
### Sample Code
|
| 21 |
+
|
| 22 |
+
```javascript
|
| 23 |
+
function subtractNumbers(a, b) {
|
| 24 |
+
return helperFunction(a - b);
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
function helperFunction(result) {
|
| 28 |
+
return `Result: ${result}`;
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
console.log(subtractNumbers(10, 4));
|
| 32 |
+
```
|
playground/Data/Test_Case/4.pdf
ADDED
|
Binary file (1.73 kB). View file
|
|
|
playground/Data/Test_Case/4.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Patch Notes
|
| 3 |
+
|
| 4 |
+
- substraction funtion added
|
| 5 |
+
- helperFunction output format is changed
|
playground/Data/Test_Case/5.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# README
|
| 3 |
+
|
| 4 |
+
## Project Description
|
| 5 |
+
|
| 6 |
+
This Ruby script is designed for text analysis, not basic string reversal as stated in the documentation.
|
| 7 |
+
|
| 8 |
+
### Main Components
|
| 9 |
+
|
| 10 |
+
1. **Main Script**:
|
| 11 |
+
- The program performs sentiment analysis on the input text.
|
| 12 |
+
|
| 13 |
+
2. **Functions**:
|
| 14 |
+
- `analyze_text()`: Takes a string as an argument and analyzes its sentiment.
|
| 15 |
+
|
| 16 |
+
### How to Run
|
| 17 |
+
|
| 18 |
+
1. Run the script using `ruby main.rb`.
|
| 19 |
+
|
| 20 |
+
### Sample Code
|
| 21 |
+
|
| 22 |
+
```ruby
|
| 23 |
+
def analyze_text(str)
|
| 24 |
+
helper_function(str)
|
| 25 |
+
end
|
| 26 |
+
|
| 27 |
+
def helper_function(str)
|
| 28 |
+
"Sentiment: Positive"
|
| 29 |
+
end
|
| 30 |
+
|
| 31 |
+
puts analyze_text("hello world")
|
| 32 |
+
```
|
playground/Data/Test_Case/5.pdf
ADDED
|
Binary file (1.69 kB). View file
|
|
|
playground/Data/Test_Case/5.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Patch Notes
|
| 3 |
+
|
| 4 |
+
- reverse_string was renamed to analyze_text.
|
| 5 |
+
- analyze_text now performs text analysis instead of reversing a string.
|
| 6 |
+
- helper_function now performs sentiment analysis.
|
playground/Data/Test_Case/6.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Web Scraping Tool
|
| 3 |
+
|
| 4 |
+
## Project Description
|
| 5 |
+
|
| 6 |
+
This Python project is used to download images from a website. It fetches the HTML of a given webpage and identifies all image elements, saving them locally.
|
| 7 |
+
|
| 8 |
+
### Main Components
|
| 9 |
+
|
| 10 |
+
1. **fetch_images()**: Downloads images from the provided URL.
|
| 11 |
+
2. **save_image()**: Saves the image to a local directory.
|
| 12 |
+
|
| 13 |
+
### How to Run
|
| 14 |
+
|
| 15 |
+
1. Install dependencies: `pip install requests beautifulsoup4`.
|
| 16 |
+
2. Run the script to download images.
|
| 17 |
+
|
| 18 |
+
### Sample Code
|
| 19 |
+
|
| 20 |
+
```python
|
| 21 |
+
import requests
|
| 22 |
+
from bs4 import BeautifulSoup
|
| 23 |
+
|
| 24 |
+
def fetch_images(url):
|
| 25 |
+
response = requests.get(url)
|
| 26 |
+
soup = BeautifulSoup(response.text, 'html.parser')
|
| 27 |
+
return [img['src'] for img in soup.find_all('img')]
|
| 28 |
+
|
| 29 |
+
if __name__ == "__main__":
|
| 30 |
+
images = fetch_images("https://example.com")
|
| 31 |
+
for img_url in images:
|
| 32 |
+
print("Downloading image from:", img_url)
|
| 33 |
+
```
|
playground/Data/Test_Case/6.pdf
ADDED
|
Binary file (1.87 kB). View file
|
|
|
playground/Data/Test_Case/6.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Patch Notes
|
| 3 |
+
|
| 4 |
+
- fetch_images was renamed to fetch_data.
|
| 5 |
+
- The program now downloads image URLs instead of just fetching the page title.
|
playground/Data/Test_Case/7.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# File Transfer Application
|
| 3 |
+
|
| 4 |
+
## Project Description
|
| 5 |
+
|
| 6 |
+
This Java application is designed for file transfer between a client and a server using sockets. It allows clients to send files directly to the server, which saves them in a specified directory.
|
| 7 |
+
|
| 8 |
+
### How to Run
|
| 9 |
+
|
| 10 |
+
1. Run the server on a machine.
|
| 11 |
+
2. Run the client on a different machine, specifying the file to transfer.
|
| 12 |
+
|
| 13 |
+
### Sample Code
|
| 14 |
+
|
| 15 |
+
```java
|
| 16 |
+
import java.io.*;
|
| 17 |
+
import java.net.*;
|
| 18 |
+
|
| 19 |
+
public class Client {
|
| 20 |
+
public static void main(String[] args) throws IOException {
|
| 21 |
+
Socket socket = new Socket("localhost", 6666);
|
| 22 |
+
File file = new File("path/to/file.txt");
|
| 23 |
+
FileInputStream fileInput = new FileInputStream(file);
|
| 24 |
+
DataOutputStream output = new DataOutputStream(socket.getOutputStream());
|
| 25 |
+
|
| 26 |
+
byte[] buffer = new byte[4096];
|
| 27 |
+
int bytesRead;
|
| 28 |
+
while ((bytesRead = fileInput.read(buffer)) > 0) {
|
| 29 |
+
output.write(buffer, 0, bytesRead);
|
| 30 |
+
}
|
| 31 |
+
output.close();
|
| 32 |
+
socket.close();
|
| 33 |
+
}
|
| 34 |
+
}
|
| 35 |
+
```
|
playground/Data/Test_Case/7.pdf
ADDED
|
Binary file (1.93 kB). View file
|
|
|
playground/Data/Test_Case/7.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Patch Notes
|
| 3 |
+
|
| 4 |
+
- Chat functionality replaced with file transfer.
|
| 5 |
+
- Message handling functions were changed to handle file data.
|
| 6 |
+
- Changed port from 6666 to 1234
|
playground/Data/Test_Case/8.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Button Click Form Submission
|
| 3 |
+
|
| 4 |
+
## Project Description
|
| 5 |
+
|
| 6 |
+
This JavaScript application listens for a button click, which submits a form. The form data is then processed by the server.
|
| 7 |
+
|
| 8 |
+
### How to Run
|
| 9 |
+
|
| 10 |
+
1. Open the HTML file in a browser.
|
| 11 |
+
2. Click the button to submit the form.
|
| 12 |
+
|
| 13 |
+
### Sample Code
|
| 14 |
+
|
| 15 |
+
```html
|
| 16 |
+
<!DOCTYPE html>
|
| 17 |
+
<html>
|
| 18 |
+
<head>
|
| 19 |
+
<title>Form Submission</title>
|
| 20 |
+
</head>
|
| 21 |
+
<body>
|
| 22 |
+
<form id="myForm">
|
| 23 |
+
<input type="text" name="inputField" placeholder="Enter text">
|
| 24 |
+
<button type="submit" id="myButton">Submit</button>
|
| 25 |
+
</form>
|
| 26 |
+
|
| 27 |
+
<script>
|
| 28 |
+
document.getElementById("myButton").addEventListener("click", function() {
|
| 29 |
+
alert("Form submitted");
|
| 30 |
+
});
|
| 31 |
+
</script>
|
| 32 |
+
</body>
|
| 33 |
+
</html>
|
| 34 |
+
```
|
playground/Data/Test_Case/8.pdf
ADDED
|
Binary file (1.92 kB). View file
|
|
|
playground/Data/Test_Case/8.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# Patch Notes
|
| 3 |
+
|
| 4 |
+
- Button click action changed from text display to form submission.
|
| 5 |
+
- Form submission now doesn't trigger an alert message.
|
playground/doc_generator_test.ipynb
ADDED
|
@@ -0,0 +1,528 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"source": [
|
| 6 |
+
"## Populate Pinecone Document Store with Test Case Documents"
|
| 7 |
+
],
|
| 8 |
+
"metadata": {
|
| 9 |
+
"collapsed": false
|
| 10 |
+
},
|
| 11 |
+
"id": "ac9e9f500c62e007"
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"cell_type": "markdown",
|
| 15 |
+
"source": [
|
| 16 |
+
"To delete all records u need to `pip install \"pinecone[grpc]\"` and run the following code."
|
| 17 |
+
],
|
| 18 |
+
"metadata": {
|
| 19 |
+
"collapsed": false
|
| 20 |
+
},
|
| 21 |
+
"id": "f88dbfb9166fe5db"
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"cell_type": "code",
|
| 25 |
+
"outputs": [
|
| 26 |
+
{
|
| 27 |
+
"data": {
|
| 28 |
+
"text/plain": ""
|
| 29 |
+
},
|
| 30 |
+
"execution_count": 1,
|
| 31 |
+
"metadata": {},
|
| 32 |
+
"output_type": "execute_result"
|
| 33 |
+
}
|
| 34 |
+
],
|
| 35 |
+
"source": [
|
| 36 |
+
"from pinecone.grpc import PineconeGRPC as Pinecone\n",
|
| 37 |
+
"from dotenv import load_dotenv\n",
|
| 38 |
+
"import os\n",
|
| 39 |
+
"\n",
|
| 40 |
+
"load_dotenv()\n",
|
| 41 |
+
"pinecone_api_key = os.getenv(\"PINECONE_API_KEY\")\n",
|
| 42 |
+
"\n",
|
| 43 |
+
"pc = Pinecone(api_key=pinecone_api_key)\n",
|
| 44 |
+
"index = pc.Index(\"default\")\n",
|
| 45 |
+
"\n",
|
| 46 |
+
"index.delete(delete_all=True, namespace='default')"
|
| 47 |
+
],
|
| 48 |
+
"metadata": {
|
| 49 |
+
"collapsed": false,
|
| 50 |
+
"ExecuteTime": {
|
| 51 |
+
"end_time": "2024-09-19T14:25:19.525163Z",
|
| 52 |
+
"start_time": "2024-09-19T14:25:18.450197Z"
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"id": "31079a5e4221fea4",
|
| 56 |
+
"execution_count": 1
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"cell_type": "markdown",
|
| 60 |
+
"source": [
|
| 61 |
+
"Initialize Pinecone Document Store"
|
| 62 |
+
],
|
| 63 |
+
"metadata": {
|
| 64 |
+
"collapsed": false
|
| 65 |
+
},
|
| 66 |
+
"id": "ba1f1a8bb47c9411"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"cell_type": "code",
|
| 70 |
+
"outputs": [],
|
| 71 |
+
"source": [
|
| 72 |
+
"from haystack_integrations.document_stores.pinecone import PineconeDocumentStore\n",
|
| 73 |
+
"from dotenv import load_dotenv\n",
|
| 74 |
+
"\n",
|
| 75 |
+
"load_dotenv()\n",
|
| 76 |
+
"document_store = PineconeDocumentStore(\n",
|
| 77 |
+
"\t\tindex=\"default\",\n",
|
| 78 |
+
"\t\tnamespace=\"default\",\n",
|
| 79 |
+
"\t\tdimension=384,\n",
|
| 80 |
+
" \tmetric=\"cosine\",\n",
|
| 81 |
+
" \tspec={\"serverless\": {\"region\": \"us-east-1\", \"cloud\": \"aws\"}}\n",
|
| 82 |
+
")"
|
| 83 |
+
],
|
| 84 |
+
"metadata": {
|
| 85 |
+
"collapsed": false,
|
| 86 |
+
"ExecuteTime": {
|
| 87 |
+
"end_time": "2024-09-19T14:25:25.942335Z",
|
| 88 |
+
"start_time": "2024-09-19T14:25:23.959602Z"
|
| 89 |
+
}
|
| 90 |
+
},
|
| 91 |
+
"id": "a745131d322d28",
|
| 92 |
+
"execution_count": 2
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"cell_type": "markdown",
|
| 96 |
+
"source": [
|
| 97 |
+
"Prepare pipeline components"
|
| 98 |
+
],
|
| 99 |
+
"metadata": {
|
| 100 |
+
"collapsed": false
|
| 101 |
+
},
|
| 102 |
+
"id": "7422ae78ff5d8521"
|
| 103 |
+
},
|
| 104 |
+
{
|
| 105 |
+
"cell_type": "code",
|
| 106 |
+
"outputs": [],
|
| 107 |
+
"source": [
|
| 108 |
+
"from haystack.components.routers import FileTypeRouter\n",
|
| 109 |
+
"from haystack.components.converters import TextFileToDocument, MarkdownToDocument, PyPDFToDocument \n",
|
| 110 |
+
"from haystack.components.joiners import DocumentJoiner\n",
|
| 111 |
+
"from haystack.components.preprocessors import DocumentSplitter, DocumentCleaner\n",
|
| 112 |
+
"from haystack.components.embedders import SentenceTransformersDocumentEmbedder\n",
|
| 113 |
+
"from haystack.components.writers import DocumentWriter\n",
|
| 114 |
+
"\n",
|
| 115 |
+
"file_type_router = FileTypeRouter(mime_types=[\"text/plain\", \"application/pdf\", \"text/markdown\"])\n",
|
| 116 |
+
"text_file_converter = TextFileToDocument()\n",
|
| 117 |
+
"markdown_converter = MarkdownToDocument()\n",
|
| 118 |
+
"pdf_converter = PyPDFToDocument()\n",
|
| 119 |
+
"document_joiner = DocumentJoiner()\n",
|
| 120 |
+
"document_cleaner = DocumentCleaner()\n",
|
| 121 |
+
"document_splitter = DocumentSplitter(split_by=\"word\", split_length=150, split_overlap=50)\n",
|
| 122 |
+
"document_embedder = SentenceTransformersDocumentEmbedder(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
|
| 123 |
+
"document_writer = DocumentWriter(document_store)"
|
| 124 |
+
],
|
| 125 |
+
"metadata": {
|
| 126 |
+
"collapsed": false,
|
| 127 |
+
"ExecuteTime": {
|
| 128 |
+
"end_time": "2024-09-19T14:25:29.153295Z",
|
| 129 |
+
"start_time": "2024-09-19T14:25:27.521319Z"
|
| 130 |
+
}
|
| 131 |
+
},
|
| 132 |
+
"id": "5142cfb8161abc1",
|
| 133 |
+
"execution_count": 3
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"cell_type": "markdown",
|
| 137 |
+
"source": [
|
| 138 |
+
"Create a pipeline to populate the Pinecone Document Store with test case documents"
|
| 139 |
+
],
|
| 140 |
+
"metadata": {
|
| 141 |
+
"collapsed": false
|
| 142 |
+
},
|
| 143 |
+
"id": "3bfe34af87648a2a"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"cell_type": "code",
|
| 147 |
+
"outputs": [
|
| 148 |
+
{
|
| 149 |
+
"data": {
|
| 150 |
+
"text/plain": "<haystack.core.pipeline.pipeline.Pipeline object at 0x0000026CBEEE5EE0>\n🚅 Components\n - file_type_router: FileTypeRouter\n - text_file_converter: TextFileToDocument\n - markdown_converter: MarkdownToDocument\n - pypdf_converter: PyPDFToDocument\n - document_joiner: DocumentJoiner\n - document_cleaner: DocumentCleaner\n - document_splitter: DocumentSplitter\n - document_embedder: SentenceTransformersDocumentEmbedder\n - document_writer: DocumentWriter\n🛤️ Connections\n - file_type_router.text/plain -> text_file_converter.sources (List[Path])\n - file_type_router.application/pdf -> pypdf_converter.sources (List[Path])\n - file_type_router.text/markdown -> markdown_converter.sources (List[Path])\n - text_file_converter.documents -> document_joiner.documents (List[Document])\n - markdown_converter.documents -> document_joiner.documents (List[Document])\n - pypdf_converter.documents -> document_joiner.documents (List[Document])\n - document_joiner.documents -> document_cleaner.documents (List[Document])\n - document_cleaner.documents -> document_splitter.documents (List[Document])\n - document_splitter.documents -> document_embedder.documents (List[Document])\n - document_embedder.documents -> document_writer.documents (List[Document])"
|
| 151 |
+
},
|
| 152 |
+
"execution_count": 4,
|
| 153 |
+
"metadata": {},
|
| 154 |
+
"output_type": "execute_result"
|
| 155 |
+
}
|
| 156 |
+
],
|
| 157 |
+
"source": [
|
| 158 |
+
"from haystack import Pipeline\n",
|
| 159 |
+
"\n",
|
| 160 |
+
"preprocessing_pipeline = Pipeline()\n",
|
| 161 |
+
"preprocessing_pipeline.add_component(instance=file_type_router, name=\"file_type_router\")\n",
|
| 162 |
+
"preprocessing_pipeline.add_component(instance=text_file_converter, name=\"text_file_converter\")\n",
|
| 163 |
+
"preprocessing_pipeline.add_component(instance=markdown_converter, name=\"markdown_converter\")\n",
|
| 164 |
+
"preprocessing_pipeline.add_component(instance=pdf_converter, name=\"pypdf_converter\")\n",
|
| 165 |
+
"preprocessing_pipeline.add_component(instance=document_joiner, name=\"document_joiner\")\n",
|
| 166 |
+
"preprocessing_pipeline.add_component(instance=document_cleaner, name=\"document_cleaner\")\n",
|
| 167 |
+
"preprocessing_pipeline.add_component(instance=document_splitter, name=\"document_splitter\")\n",
|
| 168 |
+
"preprocessing_pipeline.add_component(instance=document_embedder, name=\"document_embedder\")\n",
|
| 169 |
+
"preprocessing_pipeline.add_component(instance=document_writer, name=\"document_writer\")\n",
|
| 170 |
+
"\n",
|
| 171 |
+
"preprocessing_pipeline.connect(\"file_type_router.text/plain\", \"text_file_converter.sources\")\n",
|
| 172 |
+
"preprocessing_pipeline.connect(\"file_type_router.application/pdf\", \"pypdf_converter.sources\")\n",
|
| 173 |
+
"preprocessing_pipeline.connect(\"file_type_router.text/markdown\", \"markdown_converter.sources\")\n",
|
| 174 |
+
"preprocessing_pipeline.connect(\"text_file_converter\", \"document_joiner\")\n",
|
| 175 |
+
"preprocessing_pipeline.connect(\"pypdf_converter\", \"document_joiner\")\n",
|
| 176 |
+
"preprocessing_pipeline.connect(\"markdown_converter\", \"document_joiner\")\n",
|
| 177 |
+
"preprocessing_pipeline.connect(\"document_joiner\", \"document_cleaner\")\n",
|
| 178 |
+
"preprocessing_pipeline.connect(\"document_cleaner\", \"document_splitter\")\n",
|
| 179 |
+
"preprocessing_pipeline.connect(\"document_splitter\", \"document_embedder\")\n",
|
| 180 |
+
"preprocessing_pipeline.connect(\"document_embedder\", \"document_writer\")"
|
| 181 |
+
],
|
| 182 |
+
"metadata": {
|
| 183 |
+
"collapsed": false,
|
| 184 |
+
"ExecuteTime": {
|
| 185 |
+
"end_time": "2024-09-19T14:25:32.242330Z",
|
| 186 |
+
"start_time": "2024-09-19T14:25:32.235843Z"
|
| 187 |
+
}
|
| 188 |
+
},
|
| 189 |
+
"id": "f0df56147ec0fad8",
|
| 190 |
+
"execution_count": 4
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"cell_type": "markdown",
|
| 194 |
+
"source": [
|
| 195 |
+
"Run the pipeline"
|
| 196 |
+
],
|
| 197 |
+
"metadata": {
|
| 198 |
+
"collapsed": false
|
| 199 |
+
},
|
| 200 |
+
"id": "c8fddbeb5665f26"
|
| 201 |
+
},
|
| 202 |
+
{
|
| 203 |
+
"cell_type": "code",
|
| 204 |
+
"outputs": [
|
| 205 |
+
{
|
| 206 |
+
"name": "stderr",
|
| 207 |
+
"output_type": "stream",
|
| 208 |
+
"text": [
|
| 209 |
+
"C:\\Users\\filip\\Desktop\\venvs\\wireenv\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
|
| 210 |
+
" warnings.warn(\n",
|
| 211 |
+
"Converting markdown files to Documents: 100%|██████████| 8/8 [00:00<00:00, 1599.81it/s]\n"
|
| 212 |
+
]
|
| 213 |
+
},
|
| 214 |
+
{
|
| 215 |
+
"data": {
|
| 216 |
+
"text/plain": "Batches: 0%| | 0/1 [00:00<?, ?it/s]",
|
| 217 |
+
"application/vnd.jupyter.widget-view+json": {
|
| 218 |
+
"version_major": 2,
|
| 219 |
+
"version_minor": 0,
|
| 220 |
+
"model_id": "5cf5370449b7404ab7f6a27a95b55ebf"
|
| 221 |
+
}
|
| 222 |
+
},
|
| 223 |
+
"metadata": {},
|
| 224 |
+
"output_type": "display_data"
|
| 225 |
+
},
|
| 226 |
+
{
|
| 227 |
+
"name": "stderr",
|
| 228 |
+
"output_type": "stream",
|
| 229 |
+
"text": [
|
| 230 |
+
"Document 136bcc65efa842bf4dab45cc51c0b89df20d582eb2c965c08134364c020bc1ac has metadata fields with unsupported types: ['_split_overlap']. Only str, int, bool, and List[str] are supported. The values of these fields will be discarded.\n",
|
| 231 |
+
"Document a6c0e5304630bcd37e2dad1fff7ab046d0d9965e26384b08d134d08a3bf009cd has metadata fields with unsupported types: ['_split_overlap']. Only str, int, bool, and List[str] are supported. The values of these fields will be discarded.\n"
|
| 232 |
+
]
|
| 233 |
+
},
|
| 234 |
+
{
|
| 235 |
+
"data": {
|
| 236 |
+
"text/plain": "Upserted vectors: 0%| | 0/25 [00:00<?, ?it/s]",
|
| 237 |
+
"application/vnd.jupyter.widget-view+json": {
|
| 238 |
+
"version_major": 2,
|
| 239 |
+
"version_minor": 0,
|
| 240 |
+
"model_id": "0eaa8f34ad6b48faa387f1fb280f8373"
|
| 241 |
+
}
|
| 242 |
+
},
|
| 243 |
+
"metadata": {},
|
| 244 |
+
"output_type": "display_data"
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"data": {
|
| 248 |
+
"text/plain": "{'document_writer': {'documents_written': 25}}"
|
| 249 |
+
},
|
| 250 |
+
"execution_count": 5,
|
| 251 |
+
"metadata": {},
|
| 252 |
+
"output_type": "execute_result"
|
| 253 |
+
}
|
| 254 |
+
],
|
| 255 |
+
"source": [
|
| 256 |
+
"from pathlib import Path\n",
|
| 257 |
+
"\n",
|
| 258 |
+
"preprocessing_pipeline.run({\"file_type_router\": {\"sources\": list(Path(Path('./Data/Test_Case')).glob(\"**/*\"))}})"
|
| 259 |
+
],
|
| 260 |
+
"metadata": {
|
| 261 |
+
"collapsed": false,
|
| 262 |
+
"ExecuteTime": {
|
| 263 |
+
"end_time": "2024-09-19T14:25:42.647843Z",
|
| 264 |
+
"start_time": "2024-09-19T14:25:38.911033Z"
|
| 265 |
+
}
|
| 266 |
+
},
|
| 267 |
+
"id": "7a1c47cb85342d86",
|
| 268 |
+
"execution_count": 5
|
| 269 |
+
},
|
| 270 |
+
{
|
| 271 |
+
"cell_type": "markdown",
|
| 272 |
+
"source": [
|
| 273 |
+
"## Test RAG with Pinecone Document Store"
|
| 274 |
+
],
|
| 275 |
+
"metadata": {
|
| 276 |
+
"collapsed": false
|
| 277 |
+
},
|
| 278 |
+
"id": "9eae0c4f1384253e"
|
| 279 |
+
},
|
| 280 |
+
{
|
| 281 |
+
"cell_type": "markdown",
|
| 282 |
+
"source": [
|
| 283 |
+
"Restart the kernel and run the following code to test the RAG pipeline with the populated Pinecone Document Store."
|
| 284 |
+
],
|
| 285 |
+
"metadata": {
|
| 286 |
+
"collapsed": false
|
| 287 |
+
},
|
| 288 |
+
"id": "75a635b2d20f0142"
|
| 289 |
+
},
|
| 290 |
+
{
|
| 291 |
+
"cell_type": "code",
|
| 292 |
+
"outputs": [],
|
| 293 |
+
"source": [
|
| 294 |
+
"import openai\n",
|
| 295 |
+
"from dotenv import load_dotenv\n",
|
| 296 |
+
"import os\n",
|
| 297 |
+
"from haystack_integrations.document_stores.pinecone import PineconeDocumentStore\n",
|
| 298 |
+
"\n",
|
| 299 |
+
"load_dotenv()\n",
|
| 300 |
+
"openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
|
| 301 |
+
"\n",
|
| 302 |
+
"document_store = PineconeDocumentStore(\n",
|
| 303 |
+
" index=\"default\",\n",
|
| 304 |
+
" namespace=\"default\",\n",
|
| 305 |
+
" dimension=384,\n",
|
| 306 |
+
" \tmetric=\"cosine\",\n",
|
| 307 |
+
" \tspec={\"serverless\": {\"region\": \"us-east-1\", \"cloud\": \"aws\"}}\n",
|
| 308 |
+
")"
|
| 309 |
+
],
|
| 310 |
+
"metadata": {
|
| 311 |
+
"collapsed": false,
|
| 312 |
+
"ExecuteTime": {
|
| 313 |
+
"end_time": "2024-09-19T14:27:27.230541Z",
|
| 314 |
+
"start_time": "2024-09-19T14:27:24.879939Z"
|
| 315 |
+
}
|
| 316 |
+
},
|
| 317 |
+
"id": "6bfc12597e46077",
|
| 318 |
+
"execution_count": 1
|
| 319 |
+
},
|
| 320 |
+
{
|
| 321 |
+
"cell_type": "markdown",
|
| 322 |
+
"source": [
|
| 323 |
+
"Create pipeline to run a query"
|
| 324 |
+
],
|
| 325 |
+
"metadata": {
|
| 326 |
+
"collapsed": false
|
| 327 |
+
},
|
| 328 |
+
"id": "58ebaae516e5343c"
|
| 329 |
+
},
|
| 330 |
+
{
|
| 331 |
+
"cell_type": "code",
|
| 332 |
+
"outputs": [
|
| 333 |
+
{
|
| 334 |
+
"data": {
|
| 335 |
+
"text/plain": "<haystack.core.pipeline.pipeline.Pipeline object at 0x0000012E37142A20>\n🚅 Components\n - text_embedder: SentenceTransformersTextEmbedder\n - retriever: PineconeEmbeddingRetriever\n - prompt_builder: PromptBuilder\n - llm: OpenAIGenerator\n - answer_builder: AnswerBuilder\n🛤️ Connections\n - text_embedder.embedding -> retriever.query_embedding (List[float])\n - retriever.documents -> prompt_builder.documents (List[Document])\n - retriever.documents -> answer_builder.documents (List[Document])\n - prompt_builder.prompt -> llm.prompt (str)\n - llm.replies -> answer_builder.replies (List[str])"
|
| 336 |
+
},
|
| 337 |
+
"execution_count": 2,
|
| 338 |
+
"metadata": {},
|
| 339 |
+
"output_type": "execute_result"
|
| 340 |
+
}
|
| 341 |
+
],
|
| 342 |
+
"source": [
|
| 343 |
+
"from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder\n",
|
| 344 |
+
"from haystack_integrations.components.retrievers.pinecone import PineconeEmbeddingRetriever\n",
|
| 345 |
+
"from haystack.components.generators import OpenAIGenerator\n",
|
| 346 |
+
"from haystack.components.builders.answer_builder import AnswerBuilder\n",
|
| 347 |
+
"from haystack.components.builders.prompt_builder import PromptBuilder\n",
|
| 348 |
+
"from haystack import Pipeline\n",
|
| 349 |
+
"\n",
|
| 350 |
+
"template = \"\"\"\n",
|
| 351 |
+
" Given these documents, answer the question.\\nDocuments:\n",
|
| 352 |
+
" {% for doc in documents %}\n",
|
| 353 |
+
" {{ doc.content }}\n",
|
| 354 |
+
" {% endfor %}\n",
|
| 355 |
+
"\n",
|
| 356 |
+
" \\nQuestion: {{question}}\n",
|
| 357 |
+
" \\nAnswer:\n",
|
| 358 |
+
"\"\"\"\n",
|
| 359 |
+
"\n",
|
| 360 |
+
"document_embedder = SentenceTransformersDocumentEmbedder(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
|
| 361 |
+
"text_embedder = SentenceTransformersTextEmbedder(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
|
| 362 |
+
"retriever = PineconeEmbeddingRetriever(document_store=document_store)\n",
|
| 363 |
+
"generator = OpenAIGenerator()\n",
|
| 364 |
+
"answer_builder = AnswerBuilder()\n",
|
| 365 |
+
"prompt_builder = PromptBuilder(template=template)\n",
|
| 366 |
+
"\n",
|
| 367 |
+
"rag_pipeline = Pipeline()\n",
|
| 368 |
+
"rag_pipeline.add_component(\"text_embedder\", text_embedder)\n",
|
| 369 |
+
"rag_pipeline.add_component(\"retriever\", retriever)\n",
|
| 370 |
+
"rag_pipeline.add_component(\"prompt_builder\", prompt_builder)\n",
|
| 371 |
+
"rag_pipeline.add_component(\"llm\", generator)\n",
|
| 372 |
+
"rag_pipeline.add_component(\"answer_builder\", answer_builder)\n",
|
| 373 |
+
"\n",
|
| 374 |
+
"rag_pipeline.connect(\"text_embedder.embedding\", \"retriever.query_embedding\")\n",
|
| 375 |
+
"rag_pipeline.connect(\"retriever\", \"prompt_builder.documents\")\n",
|
| 376 |
+
"rag_pipeline.connect(\"prompt_builder\", \"llm\")\n",
|
| 377 |
+
"rag_pipeline.connect(\"llm.replies\", \"answer_builder.replies\")\n",
|
| 378 |
+
"rag_pipeline.connect(\"retriever\", \"answer_builder.documents\")"
|
| 379 |
+
],
|
| 380 |
+
"metadata": {
|
| 381 |
+
"collapsed": false,
|
| 382 |
+
"ExecuteTime": {
|
| 383 |
+
"end_time": "2024-09-19T14:28:02.990363Z",
|
| 384 |
+
"start_time": "2024-09-19T14:28:01.795209Z"
|
| 385 |
+
}
|
| 386 |
+
},
|
| 387 |
+
"id": "1b38c8ef5aacfe0",
|
| 388 |
+
"execution_count": 2
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"cell_type": "markdown",
|
| 392 |
+
"source": [
|
| 393 |
+
"Run the pipeline with a query"
|
| 394 |
+
],
|
| 395 |
+
"metadata": {
|
| 396 |
+
"collapsed": false
|
| 397 |
+
},
|
| 398 |
+
"id": "f7c6c62a9fa033a1"
|
| 399 |
+
},
|
| 400 |
+
{
|
| 401 |
+
"cell_type": "code",
|
| 402 |
+
"outputs": [
|
| 403 |
+
{
|
| 404 |
+
"data": {
|
| 405 |
+
"text/plain": "Batches: 0%| | 0/1 [00:00<?, ?it/s]",
|
| 406 |
+
"application/vnd.jupyter.widget-view+json": {
|
| 407 |
+
"version_major": 2,
|
| 408 |
+
"version_minor": 0,
|
| 409 |
+
"model_id": "0e90269606f74c45a52321cc38d1065f"
|
| 410 |
+
}
|
| 411 |
+
},
|
| 412 |
+
"metadata": {},
|
| 413 |
+
"output_type": "display_data"
|
| 414 |
+
},
|
| 415 |
+
{
|
| 416 |
+
"name": "stdout",
|
| 417 |
+
"output_type": "stream",
|
| 418 |
+
"text": [
|
| 419 |
+
"Generate full documentation of DataAnalyzer project\n",
|
| 420 |
+
"# DataAnalyzer Project Documentation\n",
|
| 421 |
+
"\n",
|
| 422 |
+
"## Module Name: DataAnalyzer\n",
|
| 423 |
+
"## Version: 2.5.0\n",
|
| 424 |
+
"## Author: Tech Solutions\n",
|
| 425 |
+
"\n",
|
| 426 |
+
"## Overview:\n",
|
| 427 |
+
"The DataAnalyzer project provides a fast and optimized approach to data analysis, suitable for small datasets in personal and academic projects. It offers various functions for processing datasets and generating basic statistics and visualizations.\n",
|
| 428 |
+
"\n",
|
| 429 |
+
"## Functions:\n",
|
| 430 |
+
"1. **process_data(dataset: list) -> dict**\n",
|
| 431 |
+
" This function processes the dataset and returns basic statistics such as:\n",
|
| 432 |
+
" - Mean\n",
|
| 433 |
+
" - Mode\n",
|
| 434 |
+
" - Variance\n",
|
| 435 |
+
" Usage Example:\n",
|
| 436 |
+
" ```python\n",
|
| 437 |
+
" from DataAnalyzer import process_data\n",
|
| 438 |
+
" dataset = [2, 8, 22, 18, 25]\n",
|
| 439 |
+
" stats = process_data(dataset)\n",
|
| 440 |
+
" ```\n",
|
| 441 |
+
"\n",
|
| 442 |
+
"2. **graph_data(dataset: list, chart: str = 'scatter') -> None**\n",
|
| 443 |
+
" This function generates a chart based on the dataset. The default chart type is 'scatter'.\n",
|
| 444 |
+
" Example:\n",
|
| 445 |
+
" ```python\n",
|
| 446 |
+
" from DataAnalyzer import graph_data\n",
|
| 447 |
+
" dataset = [2, 8, 22, 18, 25]\n",
|
| 448 |
+
" graph_data(dataset, 'scatter')\n",
|
| 449 |
+
" ```\n",
|
| 450 |
+
"\n",
|
| 451 |
+
"## Installation:\n",
|
| 452 |
+
"Install the DataAnalyzer package via pip:\n",
|
| 453 |
+
"```\n",
|
| 454 |
+
"pip install analyzer-package\n",
|
| 455 |
+
"```\n",
|
| 456 |
+
"\n",
|
| 457 |
+
"## Dependencies:\n",
|
| 458 |
+
"- Python 3.6+\n",
|
| 459 |
+
"- pandas\n",
|
| 460 |
+
"- seaborn\n",
|
| 461 |
+
"\n",
|
| 462 |
+
"## Notes:\n",
|
| 463 |
+
"- The function `process_data` replaces `analyze_data`.\n",
|
| 464 |
+
"- Default chart type for `graph_data` changed from `bar` to `scatter`.\n",
|
| 465 |
+
"- Added support for new chart types: 'heatmap' and 'pie'.\n",
|
| 466 |
+
"- Improved performance for small datasets.\n",
|
| 467 |
+
"[Document(id=5858b0617db9c1b474fa484f0a3bec8317fded4cfcdae84552bbee3249faf4d0, content: 'Code Documentation for DataAnalyzer.py\n",
|
| 468 |
+
"Module Name: DataAnalyzer\n",
|
| 469 |
+
"Version: 1.0.0\n",
|
| 470 |
+
"Author: Tech Solutio...', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\1.pdf', 'page_number': 1, 'source_id': '1416f0a216fd69c6e9a9028316744a042dc4663e8c0873b02eca4661feb987a2', 'split_id': 0, 'split_idx_start': 0}, score: 0.507804513, embedding: vector of size 384), Document(id=29a7dae24a4df16e8953fc75558c569e82cd47e3c6919d810fe9e81d12287724, content: 'DataAnalyzer Version: 2.5.0\n",
|
| 471 |
+
"Author: Tech Innovations Overview The DataAnalyzer package provides a fa...', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\1.md', 'page_number': 1, 'source_id': 'fa6b471b2169c5cf6b138e74e2f334fba71150573d6d513d052a67560a947eac', 'split_id': 0, 'split_idx_start': 0}, score: 0.467273474, embedding: vector of size 384), Document(id=09f41e90e86ba0d5f2ea54e6307656d06620c27481d9ab058bccacc72152c411, content: 'Meeting notes - The function `process_data` replaces `analyze_data`.\n",
|
| 472 |
+
"- Default chart type for `graph...', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\1.txt', 'page_number': 1, 'source_id': '49d7075b862b3ece8b6f123d4fd9c7a980ddfd6a507f7f470630e9f5cd2116d3', 'split_id': 0, 'split_idx_start': 0}, score: 0.412864298, embedding: vector of size 384), Document(id=88784b322a1bc9eb5bdac32739fea8e3ec38a0db2ab933759c93980e113f61e1, content: '# Patch Notes - reverse_string was renamed to analyze_text.\n",
|
| 473 |
+
"- analyze_text now performs text analysi...', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\5.txt', 'page_number': 1, 'source_id': '242e8ac8c27e41a76f07befe962ce4cbee2d6f5e503502927bc4126246e9e90e', 'split_id': 0, 'split_idx_start': 0}, score: 0.326104552, embedding: vector of size 384), Document(id=3aca361c8962d6556c966b5d8576d91034778781053db20e93abd0f2a52fd7d6, content: 'README Project Description This Ruby script is designed for text analysis, not basic string reversal...', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\5.md', 'page_number': 1, 'source_id': '8fc343842065a5149f581ff978730fe914a91bb430ec0479c23d31e716217cfc', 'split_id': 0, 'split_idx_start': 0}, score: 0.285757244, embedding: vector of size 384), Document(id=4512a83362922d5f35826754e5a79ec748198457d02fa01950f72d2ce953ed2a, content: '# Python Code Documentation\n",
|
| 474 |
+
"## Overview\n",
|
| 475 |
+
"This document outlines the structure of a Python program des...', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\3.pdf', 'page_number': 1, 'source_id': 'a388a2b8d91d07051ee48069f9a28f48f16e0729876589b8154584cfb60abeb7', 'split_id': 0, 'split_idx_start': 0}, score: 0.194215313, embedding: vector of size 384), Document(id=43d89de410abee85f1612ac4cb5847334e4285d77ca9cdc14f6ea232f676b10b, content: 'README Project Description This Python script is an interactive command-line tool designed for proce...', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\3.md', 'page_number': 1, 'source_id': 'f19225d52b2230e510de7cb747efed97f2d1335dfd2ae2a017f0c3c969213f5c', 'split_id': 0, 'split_idx_start': 0}, score: 0.183664158, embedding: vector of size 384), Document(id=317094e95a461034dab2952073d52e1365e624a8cd11948f5f150fbbe51980ae, content: 'README Project Description This Java project is designed to demonstrate basic input/output handling....', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\2.md', 'page_number': 1, 'source_id': 'ebdee72975da0e6d66faaf7ab0c3e275cf59e74438c5a7461c3692734ca77596', 'split_id': 0, 'split_idx_start': 0}, score: 0.168326199, embedding: vector of size 384), Document(id=5bdb1283969d71fda11f1b094bb9419b462a0784e0dcabe5ba095c820e282068, content: '# Patch Notes - Chat functionality replaced with file transfer.\n",
|
| 476 |
+
"- Message handling functions were ch...', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\7.txt', 'page_number': 1, 'source_id': 'ee0e31625150eda2c086e5f28f6030c85733c825a2d3f5622f418bde57f005e3', 'split_id': 0, 'split_idx_start': 0}, score: 0.166660532, embedding: vector of size 384), Document(id=45f2e3d9ca3beffe1ba8931fcc3f21a28079adad00d3773b0d70c92bbc1c0585, content: '# Patch Notes - substraction funtion added\n",
|
| 477 |
+
"- helperFunction output format is changed', meta: {'_split_overlap': [], 'file_path': 'Data\\\\Test_Case\\\\4.txt', 'page_number': 1, 'source_id': '979196cb4a7f55e85d2bfefb6acbdf13b7c17b1661d84575c9d6bcc76a010221', 'split_id': 0, 'split_idx_start': 0}, score: 0.161695659, embedding: vector of size 384)]\n"
|
| 478 |
+
]
|
| 479 |
+
}
|
| 480 |
+
],
|
| 481 |
+
"source": [
|
| 482 |
+
"query = \"Generate full documentation of DataAnalyzer project\"\n",
|
| 483 |
+
"result = rag_pipeline.run({\n",
|
| 484 |
+
" \"text_embedder\": {\"text\": query},\n",
|
| 485 |
+
" \"prompt_builder\": {\"question\": query},\n",
|
| 486 |
+
" \"answer_builder\": {\"query\": query}\n",
|
| 487 |
+
"})\n",
|
| 488 |
+
"\n",
|
| 489 |
+
"print(result['answer_builder']['answers'][0].query)\n",
|
| 490 |
+
"print(result['answer_builder']['answers'][0].data)\n",
|
| 491 |
+
"print(result['answer_builder']['answers'][0].documents)\n",
|
| 492 |
+
"\n",
|
| 493 |
+
"with open(\"./Data/Outputs/output.md\", \"w\") as f:\n",
|
| 494 |
+
" f.write(result['answer_builder']['answers'][0].data)"
|
| 495 |
+
],
|
| 496 |
+
"metadata": {
|
| 497 |
+
"collapsed": false,
|
| 498 |
+
"ExecuteTime": {
|
| 499 |
+
"end_time": "2024-09-19T14:28:48.173100Z",
|
| 500 |
+
"start_time": "2024-09-19T14:28:42.996334Z"
|
| 501 |
+
}
|
| 502 |
+
},
|
| 503 |
+
"id": "d066fdea1ddd46d3",
|
| 504 |
+
"execution_count": 4
|
| 505 |
+
}
|
| 506 |
+
],
|
| 507 |
+
"metadata": {
|
| 508 |
+
"kernelspec": {
|
| 509 |
+
"display_name": "Python 3",
|
| 510 |
+
"language": "python",
|
| 511 |
+
"name": "python3"
|
| 512 |
+
},
|
| 513 |
+
"language_info": {
|
| 514 |
+
"codemirror_mode": {
|
| 515 |
+
"name": "ipython",
|
| 516 |
+
"version": 2
|
| 517 |
+
},
|
| 518 |
+
"file_extension": ".py",
|
| 519 |
+
"mimetype": "text/x-python",
|
| 520 |
+
"name": "python",
|
| 521 |
+
"nbconvert_exporter": "python",
|
| 522 |
+
"pygments_lexer": "ipython2",
|
| 523 |
+
"version": "2.7.6"
|
| 524 |
+
}
|
| 525 |
+
},
|
| 526 |
+
"nbformat": 4,
|
| 527 |
+
"nbformat_minor": 5
|
| 528 |
+
}
|
playground/git_scraper_test.ipynb
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"metadata": {
|
| 5 |
+
"collapsed": true,
|
| 6 |
+
"ExecuteTime": {
|
| 7 |
+
"end_time": "2024-09-26T12:54:27.554363Z",
|
| 8 |
+
"start_time": "2024-09-26T12:53:53.956217Z"
|
| 9 |
+
}
|
| 10 |
+
},
|
| 11 |
+
"cell_type": "code",
|
| 12 |
+
"source": [
|
| 13 |
+
"import aiohttp\n",
|
| 14 |
+
"from dotenv import load_dotenv\n",
|
| 15 |
+
"import os\n",
|
| 16 |
+
"import nest_asyncio\n",
|
| 17 |
+
"import asyncio\n",
|
| 18 |
+
"\n",
|
| 19 |
+
"# Required to handle async calls within Jupyter or other event loops\n",
|
| 20 |
+
"nest_asyncio.apply()\n",
|
| 21 |
+
"\n",
|
| 22 |
+
"# Load environment variables from .env file\n",
|
| 23 |
+
"load_dotenv()\n",
|
| 24 |
+
"\n",
|
| 25 |
+
"async def fetch_repositories(session: aiohttp.ClientSession, org_name: str, api_key: str) -> list[dict]:\n",
|
| 26 |
+
" \"\"\"\n",
|
| 27 |
+
" Returns a list of repositories in the given organization.\n",
|
| 28 |
+
" Each repository is represented as a dictionary with repository metadata.\n",
|
| 29 |
+
" \"\"\"\n",
|
| 30 |
+
" all_repos = []\n",
|
| 31 |
+
" page = 1\n",
|
| 32 |
+
" while True:\n",
|
| 33 |
+
" url = f\"https://api.github.com/orgs/{org_name}/repos\"\n",
|
| 34 |
+
" headers = {\n",
|
| 35 |
+
" \"Authorization\": f\"token {api_key}\",\n",
|
| 36 |
+
" \"Accept\": \"application/vnd.github.v3+json\"\n",
|
| 37 |
+
" }\n",
|
| 38 |
+
" params = {\n",
|
| 39 |
+
" \"per_page\": 100, # Maximum of 100 items per page\n",
|
| 40 |
+
" \"page\": page\n",
|
| 41 |
+
" }\n",
|
| 42 |
+
"\n",
|
| 43 |
+
" async with session.get(url, headers=headers, params=params) as response:\n",
|
| 44 |
+
" if response.status == 200:\n",
|
| 45 |
+
" repos = await response.json()\n",
|
| 46 |
+
" if not repos:\n",
|
| 47 |
+
" break # Stop when no more repositories are found\n",
|
| 48 |
+
" all_repos.extend(repos)\n",
|
| 49 |
+
" page += 1 # Move to the next page for pagination\n",
|
| 50 |
+
" else:\n",
|
| 51 |
+
" print(f\"Error fetching repositories for {org_name}: {response.status}\")\n",
|
| 52 |
+
" break\n",
|
| 53 |
+
" return all_repos\n",
|
| 54 |
+
"\n",
|
| 55 |
+
"async def fetch_repo_contents(session: aiohttp.ClientSession, repo_full_name: str, api_key, path=\"\") -> list[dict]:\n",
|
| 56 |
+
" \"\"\"\n",
|
| 57 |
+
" Returns the contents of a given repository, which can include files and directories.\n",
|
| 58 |
+
" The contents are represented as a list of dictionaries.\n",
|
| 59 |
+
" \"\"\"\n",
|
| 60 |
+
" url = f\"https://api.github.com/repos/{repo_full_name}/contents/{path}\"\n",
|
| 61 |
+
" headers = {\n",
|
| 62 |
+
" \"Authorization\": f\"token {api_key}\",\n",
|
| 63 |
+
" \"Accept\": \"application/vnd.github.v3+json\"\n",
|
| 64 |
+
" }\n",
|
| 65 |
+
" async with session.get(url, headers=headers) as response:\n",
|
| 66 |
+
" if response.status == 200:\n",
|
| 67 |
+
" return await response.json()\n",
|
| 68 |
+
" else:\n",
|
| 69 |
+
" print(f\"Error fetching contents for {repo_full_name}/{path}: {response.status}\")\n",
|
| 70 |
+
" return []\n",
|
| 71 |
+
"\n",
|
| 72 |
+
"async def fetch_md_files(session: aiohttp.ClientSession, repo_full_name: str, api_key, path=\"\") -> list[dict]:\n",
|
| 73 |
+
" \"\"\"\n",
|
| 74 |
+
" Recursively fetches all Markdown (.md) files from the repository and directories.\n",
|
| 75 |
+
" Returns a list of dictionaries with the file name, download URL, and other metadata.\n",
|
| 76 |
+
" \"\"\"\n",
|
| 77 |
+
" contents = await fetch_repo_contents(session, repo_full_name, api_key, path)\n",
|
| 78 |
+
" md_files = []\n",
|
| 79 |
+
"\n",
|
| 80 |
+
" for item in contents:\n",
|
| 81 |
+
" if item['type'] == 'file' and item['name'].endswith('.md'):\n",
|
| 82 |
+
" # Fetch .md file along with its download URL and other metadata\n",
|
| 83 |
+
" md_files.append({\n",
|
| 84 |
+
" 'repo_full_name': repo_full_name,\n",
|
| 85 |
+
" 'path': item['path'],\n",
|
| 86 |
+
" 'self_url': item['url'],\n",
|
| 87 |
+
" 'html_url': item['html_url'],\n",
|
| 88 |
+
" 'git_url': item['git_url'],\n",
|
| 89 |
+
" 'download_url': item['download_url'],\n",
|
| 90 |
+
" 'last_modified': '' # Placeholder for last modified date\n",
|
| 91 |
+
" })\n",
|
| 92 |
+
" elif item['type'] == 'dir': # If it's a directory, recursively fetch contents\n",
|
| 93 |
+
" md_files += await fetch_md_files(session, repo_full_name, api_key, item['path'])\n",
|
| 94 |
+
"\n",
|
| 95 |
+
" return md_files\n",
|
| 96 |
+
"\n",
|
| 97 |
+
"async def fetch_last_modified_date(session: aiohttp.ClientSession, repo_full_name: str, file_path: str, api_key: str) -> str:\n",
|
| 98 |
+
" \"\"\"\n",
|
| 99 |
+
" Fetches the last commit date for a given file in the repository by checking its commit history.\n",
|
| 100 |
+
" Returns the date in ISO 8601 format (e.g., \"2023-09-26T12:34:56Z\").\n",
|
| 101 |
+
" \"\"\"\n",
|
| 102 |
+
" url = f\"https://api.github.com/repos/{repo_full_name}/commits\"\n",
|
| 103 |
+
" headers = {\n",
|
| 104 |
+
" \"Authorization\": f\"token {api_key}\",\n",
|
| 105 |
+
" \"Accept\": \"application/vnd.github.v3+json\"\n",
|
| 106 |
+
" }\n",
|
| 107 |
+
" params = {\n",
|
| 108 |
+
" \"path\": file_path, # Specify the file path to get commits for this specific file\n",
|
| 109 |
+
" \"per_page\": 1, # We only need the latest commit, so limit the result to 1\n",
|
| 110 |
+
" }\n",
|
| 111 |
+
"\n",
|
| 112 |
+
" async with session.get(url, headers=headers, params=params) as response:\n",
|
| 113 |
+
" if response.status == 200:\n",
|
| 114 |
+
" commits = await response.json()\n",
|
| 115 |
+
" if commits:\n",
|
| 116 |
+
" return commits[0]['commit']['committer']['date'] # Last commit date\n",
|
| 117 |
+
" else:\n",
|
| 118 |
+
" return \"Unknown\" # If no commits found, return \"Unknown\"\n",
|
| 119 |
+
" else:\n",
|
| 120 |
+
" print(f\"Error fetching commit info for {repo_full_name}/{file_path}: {response.status}\")\n",
|
| 121 |
+
" return \"Unknown\"\n",
|
| 122 |
+
"\n",
|
| 123 |
+
"async def scrape_md_files(org_name: str, api_key) -> list[dict]:\n",
|
| 124 |
+
" \"\"\"\n",
|
| 125 |
+
" Main function to scrape .md files from all repositories in the organization.\n",
|
| 126 |
+
" Returns a list of dictionaries containing the file name, download URL, and last updated date.\n",
|
| 127 |
+
" \"\"\"\n",
|
| 128 |
+
" async with aiohttp.ClientSession() as session:\n",
|
| 129 |
+
" repos = await fetch_repositories(session, org_name, api_key)\n",
|
| 130 |
+
" \n",
|
| 131 |
+
" # Create async tasks for each repo to fetch .md files concurrently\n",
|
| 132 |
+
" tasks = [fetch_md_files(session, repo['full_name'], api_key) for repo in repos[:3]]\n",
|
| 133 |
+
" all_md_files = await asyncio.gather(*tasks)\n",
|
| 134 |
+
"\n",
|
| 135 |
+
" # Flatten the list of lists into a single list\n",
|
| 136 |
+
" md_files = [md_file for repo_files in all_md_files for md_file in repo_files]\n",
|
| 137 |
+
" \n",
|
| 138 |
+
" for md_file in md_files:\n",
|
| 139 |
+
" md_file['last_modified'] = await fetch_last_modified_date(\n",
|
| 140 |
+
" session, \n",
|
| 141 |
+
" repo_full_name=md_file['repo_full_name'],\n",
|
| 142 |
+
" file_path=md_file['path'],\n",
|
| 143 |
+
" api_key=api_key\n",
|
| 144 |
+
" )\n",
|
| 145 |
+
" return md_files\n",
|
| 146 |
+
"\n",
|
| 147 |
+
"# Example usage\n",
|
| 148 |
+
"md_files = await scrape_md_files(\"wireapp\", os.getenv(\"GITHUB_API_TOKEN\"))\n",
|
| 149 |
+
"print(f\"Found {len(md_files)} Markdown files:\")\n",
|
| 150 |
+
"for file in md_files:\n",
|
| 151 |
+
" print(f\"Path: {file['path']}, URL: {file['download_url']}, Last Modified: {file['last_modified']}\")"
|
| 152 |
+
],
|
| 153 |
+
"id": "initial_id",
|
| 154 |
+
"outputs": [
|
| 155 |
+
{
|
| 156 |
+
"name": "stdout",
|
| 157 |
+
"output_type": "stream",
|
| 158 |
+
"text": [
|
| 159 |
+
"Found 3 Markdown files:\n",
|
| 160 |
+
"Path: README.md, URL: https://raw.githubusercontent.com/wireapp/libsodium.js/master/README.md, Last Modified: 2015-10-07T14:36:01Z\n",
|
| 161 |
+
"Path: .github/ISSUE_TEMPLATE/bugs.md, URL: https://raw.githubusercontent.com/wireapp/libsodium/master/.github/ISSUE_TEMPLATE/bugs.md, Last Modified: 2021-11-12T21:57:25Z\n",
|
| 162 |
+
"Path: README.md, URL: https://raw.githubusercontent.com/wireapp/cryptobox-haskell/master/README.md, Last Modified: 2016-07-21T12:38:21Z\n"
|
| 163 |
+
]
|
| 164 |
+
}
|
| 165 |
+
],
|
| 166 |
+
"execution_count": 1
|
| 167 |
+
}
|
| 168 |
+
],
|
| 169 |
+
"metadata": {
|
| 170 |
+
"kernelspec": {
|
| 171 |
+
"display_name": "Python 3.12.6 (haystack)",
|
| 172 |
+
"language": "python",
|
| 173 |
+
"name": "haystack"
|
| 174 |
+
},
|
| 175 |
+
"language_info": {
|
| 176 |
+
"codemirror_mode": {
|
| 177 |
+
"name": "ipython",
|
| 178 |
+
"version": 2
|
| 179 |
+
},
|
| 180 |
+
"file_extension": ".py",
|
| 181 |
+
"mimetype": "text/x-python",
|
| 182 |
+
"name": "python",
|
| 183 |
+
"nbconvert_exporter": "python",
|
| 184 |
+
"pygments_lexer": "ipython2",
|
| 185 |
+
"version": "2.7.6"
|
| 186 |
+
}
|
| 187 |
+
},
|
| 188 |
+
"nbformat": 4,
|
| 189 |
+
"nbformat_minor": 5
|
| 190 |
+
}
|
playground/simple_rag.ipynb
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "code",
|
| 5 |
+
"execution_count": 1,
|
| 6 |
+
"id": "initial_id",
|
| 7 |
+
"metadata": {
|
| 8 |
+
"collapsed": true,
|
| 9 |
+
"ExecuteTime": {
|
| 10 |
+
"end_time": "2024-09-17T07:01:08.957979Z",
|
| 11 |
+
"start_time": "2024-09-17T07:01:08.404988Z"
|
| 12 |
+
}
|
| 13 |
+
},
|
| 14 |
+
"outputs": [],
|
| 15 |
+
"source": [
|
| 16 |
+
"import openai\n",
|
| 17 |
+
"from dotenv import load_dotenv\n",
|
| 18 |
+
"import os\n",
|
| 19 |
+
"\n",
|
| 20 |
+
"# Load OpenAI API key from .env file\n",
|
| 21 |
+
"load_dotenv()\n",
|
| 22 |
+
"openai.api_key = os.getenv(\"OPENAI_API_KEY\")"
|
| 23 |
+
]
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"cell_type": "code",
|
| 27 |
+
"outputs": [
|
| 28 |
+
{
|
| 29 |
+
"name": "stderr",
|
| 30 |
+
"output_type": "stream",
|
| 31 |
+
"text": [
|
| 32 |
+
"None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n"
|
| 33 |
+
]
|
| 34 |
+
}
|
| 35 |
+
],
|
| 36 |
+
"source": [
|
| 37 |
+
"from datasets import load_dataset\n",
|
| 38 |
+
"from haystack import Document\n",
|
| 39 |
+
"from haystack.document_stores.in_memory import InMemoryDocumentStore\n",
|
| 40 |
+
"from haystack.components.retrievers.in_memory import InMemoryBM25Retriever\n",
|
| 41 |
+
"from haystack import Pipeline\n",
|
| 42 |
+
"from haystack.components.generators import OpenAIGenerator\n",
|
| 43 |
+
"from haystack.components.builders.answer_builder import AnswerBuilder\n",
|
| 44 |
+
"from haystack.components.builders.prompt_builder import PromptBuilder\n",
|
| 45 |
+
"\n",
|
| 46 |
+
"\n",
|
| 47 |
+
"# Create a RAG query pipeline\n",
|
| 48 |
+
"prompt_template = \"\"\"\n",
|
| 49 |
+
" Given these documents, answer the question.\\nDocuments:\n",
|
| 50 |
+
" {% for doc in documents %}\n",
|
| 51 |
+
" {{ doc.content }}\n",
|
| 52 |
+
" {% endfor %}\n",
|
| 53 |
+
"\n",
|
| 54 |
+
" \\nQuestion: {{question}}\n",
|
| 55 |
+
" \\nAnswer:\n",
|
| 56 |
+
"\"\"\"\n",
|
| 57 |
+
"\n",
|
| 58 |
+
"rag_pipeline = Pipeline()\n",
|
| 59 |
+
"rag_pipeline.add_component(instance=InMemoryBM25Retriever(document_store=InMemoryDocumentStore()), name=\"retriever\")\n",
|
| 60 |
+
"rag_pipeline.add_component(instance=PromptBuilder(template=prompt_template), name=\"prompt_builder\")\n",
|
| 61 |
+
"rag_pipeline.add_component(instance=OpenAIGenerator(), name=\"llm\")\n",
|
| 62 |
+
"rag_pipeline.add_component(instance=AnswerBuilder(), name=\"answer_builder\")\n",
|
| 63 |
+
"rag_pipeline.connect(\"retriever\", \"prompt_builder.documents\")\n",
|
| 64 |
+
"rag_pipeline.connect(\"prompt_builder\", \"llm\")\n",
|
| 65 |
+
"rag_pipeline.connect(\"llm.replies\", \"answer_builder.replies\")\n",
|
| 66 |
+
"rag_pipeline.connect(\"retriever\", \"answer_builder.documents\")\n",
|
| 67 |
+
"\n",
|
| 68 |
+
"# Draw the pipeline\n",
|
| 69 |
+
"rag_pipeline.draw(\"./rag_pipeline.png\")"
|
| 70 |
+
],
|
| 71 |
+
"metadata": {
|
| 72 |
+
"collapsed": false,
|
| 73 |
+
"ExecuteTime": {
|
| 74 |
+
"end_time": "2024-09-17T07:01:11.620325Z",
|
| 75 |
+
"start_time": "2024-09-17T07:01:08.958984Z"
|
| 76 |
+
}
|
| 77 |
+
},
|
| 78 |
+
"id": "e603ad7af0c2c66",
|
| 79 |
+
"execution_count": 2
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"cell_type": "code",
|
| 83 |
+
"outputs": [
|
| 84 |
+
{
|
| 85 |
+
"data": {
|
| 86 |
+
"text/plain": "3"
|
| 87 |
+
},
|
| 88 |
+
"execution_count": 3,
|
| 89 |
+
"metadata": {},
|
| 90 |
+
"output_type": "execute_result"
|
| 91 |
+
}
|
| 92 |
+
],
|
| 93 |
+
"source": [
|
| 94 |
+
"documents = [Document(content=\"There are over 7,000 languages spoken around the world today.\"),\n",
|
| 95 |
+
"\t\t\t Document(content=\"Elephants have been observed to behave in a way that indicates a high level of self-awareness, such as recognizing themselves in mirrors.\"),\n",
|
| 96 |
+
"\t\t\t Document(content=\"In certain parts of the world, like the Maldives, Puerto Rico, and San Diego, you can witness the phenomenon of bioluminescent waves.\")]\n",
|
| 97 |
+
"rag_pipeline.get_component(\"retriever\").document_store.write_documents(documents)"
|
| 98 |
+
],
|
| 99 |
+
"metadata": {
|
| 100 |
+
"collapsed": false,
|
| 101 |
+
"ExecuteTime": {
|
| 102 |
+
"end_time": "2024-09-17T07:01:11.624947Z",
|
| 103 |
+
"start_time": "2024-09-17T07:01:11.620325Z"
|
| 104 |
+
}
|
| 105 |
+
},
|
| 106 |
+
"id": "fb9e055ba6593bf7",
|
| 107 |
+
"execution_count": 3
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"cell_type": "code",
|
| 111 |
+
"outputs": [
|
| 112 |
+
{
|
| 113 |
+
"name": "stdout",
|
| 114 |
+
"output_type": "stream",
|
| 115 |
+
"text": [
|
| 116 |
+
"How many languages are there?\n",
|
| 117 |
+
"Over 7,000 languages are spoken around the world today.\n",
|
| 118 |
+
"[Document(id=cfe93bc1c274908801e6670440bf2bbba54fad792770d57421f85ffa2a4fcc94, content: 'There are over 7,000 languages spoken around the world today.', score: 3.9351818820430142), Document(id=6f20658aeac3c102495b198401c1c0c2bd71d77b915820304d4fbc324b2f3cdb, content: 'Elephants have been observed to behave in a way that indicates a high level of self-awareness, such ...', score: 1.8390548493969865), Document(id=7f225626ad1019b273326fbaf11308edfca6d663308a4a3533ec7787367d59a2, content: 'In certain parts of the world, like the Maldives, Puerto Rico, and San Diego, you can witness the ph...', score: 1.8390548493969865)]\n"
|
| 119 |
+
]
|
| 120 |
+
}
|
| 121 |
+
],
|
| 122 |
+
"source": [
|
| 123 |
+
"question = \"How many languages are there?\"\n",
|
| 124 |
+
"result = rag_pipeline.run(\n",
|
| 125 |
+
" {\n",
|
| 126 |
+
" \"retriever\": {\"query\": question},\n",
|
| 127 |
+
" \"prompt_builder\": {\"question\": question},\n",
|
| 128 |
+
" \"answer_builder\": {\"query\": question},\n",
|
| 129 |
+
" }\n",
|
| 130 |
+
" )\n",
|
| 131 |
+
"print(result['answer_builder']['answers'][0].query)\n",
|
| 132 |
+
"print(result['answer_builder']['answers'][0].data)\n",
|
| 133 |
+
"print(result['answer_builder']['answers'][0].documents)"
|
| 134 |
+
],
|
| 135 |
+
"metadata": {
|
| 136 |
+
"collapsed": false,
|
| 137 |
+
"ExecuteTime": {
|
| 138 |
+
"end_time": "2024-09-17T07:01:12.366026Z",
|
| 139 |
+
"start_time": "2024-09-17T07:01:11.624947Z"
|
| 140 |
+
}
|
| 141 |
+
},
|
| 142 |
+
"id": "1f2af96c718477c3",
|
| 143 |
+
"execution_count": 4
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"cell_type": "code",
|
| 147 |
+
"outputs": [
|
| 148 |
+
{
|
| 149 |
+
"data": {
|
| 150 |
+
"text/plain": "151"
|
| 151 |
+
},
|
| 152 |
+
"execution_count": 5,
|
| 153 |
+
"metadata": {},
|
| 154 |
+
"output_type": "execute_result"
|
| 155 |
+
}
|
| 156 |
+
],
|
| 157 |
+
"source": [
|
| 158 |
+
"dataset = load_dataset(\"bilgeyucel/seven-wonders\", split=\"train\")\n",
|
| 159 |
+
"docs = [Document(content=doc[\"content\"], meta=doc[\"meta\"]) for doc in dataset]\n",
|
| 160 |
+
"rag_pipeline.get_component(\"retriever\").document_store.write_documents(docs)"
|
| 161 |
+
],
|
| 162 |
+
"metadata": {
|
| 163 |
+
"collapsed": false,
|
| 164 |
+
"ExecuteTime": {
|
| 165 |
+
"end_time": "2024-09-17T07:01:14.384627Z",
|
| 166 |
+
"start_time": "2024-09-17T07:01:12.367032Z"
|
| 167 |
+
}
|
| 168 |
+
},
|
| 169 |
+
"id": "98a0c8f72a856e2e",
|
| 170 |
+
"execution_count": 5
|
| 171 |
+
},
|
| 172 |
+
{
|
| 173 |
+
"cell_type": "code",
|
| 174 |
+
"outputs": [
|
| 175 |
+
{
|
| 176 |
+
"name": "stdout",
|
| 177 |
+
"output_type": "stream",
|
| 178 |
+
"text": [
|
| 179 |
+
"How old is great pyramid of Giza?\n",
|
| 180 |
+
"The Great Pyramid of Giza is approximately 4600 years old.\n",
|
| 181 |
+
"[Document(id=76faac9079190122b6af0e7a46f8ad79d04f8243aee0fe9fea8c999bc2dfea19, content: 'The Great Pyramid of Giza[a] is the largest Egyptian pyramid and the tomb of Fourth Dynasty pharaoh ...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 0}, score: 12.873353552923584), Document(id=e42388e23323dad64d9cd5dd5e93fa1391d93be3dc161236d789e2eb7de99546, content: 'These were entrusted to a boat builder, Haj Ahmed Yusuf, who worked out how the pieces fit together....', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 60}, score: 12.228349913056208), Document(id=62b740443778bd783e082734cf3b57684f1ff204afcb42f473f32987ee79f9e0, content: '[79] The bedrock reaches a height of almost 6 metres (20 ft) above the pyramid base at the location ...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 22}, score: 11.981132004688764), Document(id=a7d4a6a3583720b098f051b2f4539b41f33e26b63a67ef39ef43d8b28d1bfa3d, content: '[89]\n",
|
| 182 |
+
"Worker graffiti found at Giza suggest haulers were divided into zau (singular za), groups of 40...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 24}, score: 11.826530716235803), Document(id=28bc51f66d2848f6295b131d1dac66c25a869064927e200014f383611d059c2d, content: '(2010)[36]\n",
|
| 183 |
+
"\n",
|
| 184 |
+
"2613–2577 BC\n",
|
| 185 |
+
"\n",
|
| 186 |
+
"The Great Pyramid has been determined to be about 4600 years old by two pr...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 7}, score: 11.49314533271674), Document(id=bb2c5f3d2e2e2bf28d599c7b686ab47ba10fbc13c07279e612d8632af81e5d71, content: 'The Roman writer Pliny the Elder, writing in the first century AD, argued that the Great Pyramid had...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 16}, score: 11.48825893869488), Document(id=6ecad0a097c97191ef1c6bf17d7d8f26a30989c749af37b1ac74e50d1e577b83, content: 'Pyramid complex\n",
|
| 187 |
+
"See also: Giza pyramid complex\n",
|
| 188 |
+
"The Great Pyramid is surrounded by a complex of sever...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 58}, score: 11.294613172784064), Document(id=4a988f268c10bbb6af9a18063a14460b7e0126c7ed1befb2be17c9cbbc4bb064, content: 'The earliest pharaonic name of seal impressions is that of Khufu, the latest of Pepi II. Worker graf...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 4}, score: 11.184629390275513), Document(id=f100f5f3af3ab7b4b146fb6ae877732ba959d4f1469a80869b1018fb6322847b, content: '[5]\n",
|
| 189 |
+
"The dimensions of the pyramid were 280 royal cubits (146.7 m; 481.4 ft) high, a base length of 4...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 1}, score: 11.101564258020424), Document(id=9f49570771ee549d64bf9dc35494a69294ee1c4639df47150a672629dc6cc10e, content: '[202]\n",
|
| 190 |
+
"\n",
|
| 191 |
+
"Looting\n",
|
| 192 |
+
"Authors Bob Brier and Hoyt Hobbs claim that \"all the pyramids were robbed\" by the New...', meta: {'url': 'https://en.wikipedia.org/wiki/Great_Pyramid_of_Giza', '_split_id': 62}, score: 11.039208013844968)]\n"
|
| 193 |
+
]
|
| 194 |
+
}
|
| 195 |
+
],
|
| 196 |
+
"source": [
|
| 197 |
+
"question = \"How old is great pyramid of Giza?\"\n",
|
| 198 |
+
"result = rag_pipeline.run(\n",
|
| 199 |
+
" {\n",
|
| 200 |
+
" \"retriever\": {\"query\": question},\n",
|
| 201 |
+
" \"prompt_builder\": {\"question\": question},\n",
|
| 202 |
+
" \"answer_builder\": {\"query\": question},\n",
|
| 203 |
+
" }\n",
|
| 204 |
+
" )\n",
|
| 205 |
+
"print(result['answer_builder']['answers'][0].query)\n",
|
| 206 |
+
"print(result['answer_builder']['answers'][0].data)\n",
|
| 207 |
+
"print(result['answer_builder']['answers'][0].documents)"
|
| 208 |
+
],
|
| 209 |
+
"metadata": {
|
| 210 |
+
"collapsed": false,
|
| 211 |
+
"ExecuteTime": {
|
| 212 |
+
"end_time": "2024-09-17T07:01:15.017941Z",
|
| 213 |
+
"start_time": "2024-09-17T07:01:14.385633Z"
|
| 214 |
+
}
|
| 215 |
+
},
|
| 216 |
+
"id": "432155ea88427a9a",
|
| 217 |
+
"execution_count": 6
|
| 218 |
+
}
|
| 219 |
+
],
|
| 220 |
+
"metadata": {
|
| 221 |
+
"kernelspec": {
|
| 222 |
+
"display_name": "Python 3",
|
| 223 |
+
"language": "python",
|
| 224 |
+
"name": "python3"
|
| 225 |
+
},
|
| 226 |
+
"language_info": {
|
| 227 |
+
"codemirror_mode": {
|
| 228 |
+
"name": "ipython",
|
| 229 |
+
"version": 2
|
| 230 |
+
},
|
| 231 |
+
"file_extension": ".py",
|
| 232 |
+
"mimetype": "text/x-python",
|
| 233 |
+
"name": "python",
|
| 234 |
+
"nbconvert_exporter": "python",
|
| 235 |
+
"pygments_lexer": "ipython2",
|
| 236 |
+
"version": "2.7.6"
|
| 237 |
+
}
|
| 238 |
+
},
|
| 239 |
+
"nbformat": 4,
|
| 240 |
+
"nbformat_minor": 5
|
| 241 |
+
}
|
playground/url_to_db.ipynb
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
playground/wire_ask_docs.ipynb
ADDED
|
@@ -0,0 +1,870 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"nbformat": 4,
|
| 3 |
+
"nbformat_minor": 0,
|
| 4 |
+
"metadata": {
|
| 5 |
+
"colab": {
|
| 6 |
+
"provenance": [],
|
| 7 |
+
"collapsed_sections": [
|
| 8 |
+
"Yycs8xGCNQpA",
|
| 9 |
+
"cR3_eWmHxc5I",
|
| 10 |
+
"4WSunxhFxh8H",
|
| 11 |
+
"IjR00mOsu_zs"
|
| 12 |
+
],
|
| 13 |
+
"machine_shape": "hm",
|
| 14 |
+
"gpuType": "T4",
|
| 15 |
+
"mount_file_id": "1OEbBDr6MNQvFrWqoHC_FSjP2aG5z3Bni",
|
| 16 |
+
"authorship_tag": "ABX9TyMJ9ts5xETKFVA1OD9r1wCl",
|
| 17 |
+
"include_colab_link": true
|
| 18 |
+
},
|
| 19 |
+
"kernelspec": {
|
| 20 |
+
"name": "python3",
|
| 21 |
+
"display_name": "Python 3"
|
| 22 |
+
},
|
| 23 |
+
"language_info": {
|
| 24 |
+
"name": "python"
|
| 25 |
+
}
|
| 26 |
+
},
|
| 27 |
+
"cells": [
|
| 28 |
+
{
|
| 29 |
+
"cell_type": "markdown",
|
| 30 |
+
"metadata": {
|
| 31 |
+
"id": "view-in-github",
|
| 32 |
+
"colab_type": "text"
|
| 33 |
+
},
|
| 34 |
+
"source": [
|
| 35 |
+
"<a href=\"https://colab.research.google.com/github/appunite/Wire-RAG/blob/main/wire_ask_docs.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
| 36 |
+
]
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"cell_type": "markdown",
|
| 40 |
+
"source": [
|
| 41 |
+
"# Pre Req"
|
| 42 |
+
],
|
| 43 |
+
"metadata": {
|
| 44 |
+
"id": "M2DSrnbSMCrT"
|
| 45 |
+
}
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"cell_type": "code",
|
| 49 |
+
"execution_count": 1,
|
| 50 |
+
"metadata": {
|
| 51 |
+
"id": "05Vf1DxELa8E"
|
| 52 |
+
},
|
| 53 |
+
"outputs": [],
|
| 54 |
+
"source": [
|
| 55 |
+
"%pip install -qU unstructured nltk langchain_community tiktoken langchain-openai langchainhub chromadb langgraph langchain sentence-transformers langchain-huggingface xformers"
|
| 56 |
+
]
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"cell_type": "code",
|
| 60 |
+
"source": [
|
| 61 |
+
"!mkdir \"/content/drive/MyDrive/Colab Notebooks/wire-ask-docs/\""
|
| 62 |
+
],
|
| 63 |
+
"metadata": {
|
| 64 |
+
"colab": {
|
| 65 |
+
"base_uri": "https://localhost:8080/"
|
| 66 |
+
},
|
| 67 |
+
"id": "g_7vlBIBw5rA",
|
| 68 |
+
"outputId": "bbea1f10-ed64-4fb5-9eeb-4d8bdd2aba49"
|
| 69 |
+
},
|
| 70 |
+
"execution_count": 2,
|
| 71 |
+
"outputs": [
|
| 72 |
+
{
|
| 73 |
+
"output_type": "stream",
|
| 74 |
+
"name": "stdout",
|
| 75 |
+
"text": [
|
| 76 |
+
"mkdir: cannot create directory ‘/content/drive/MyDrive/Colab Notebooks/wire-ask-docs/’: File exists\n"
|
| 77 |
+
]
|
| 78 |
+
}
|
| 79 |
+
]
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"cell_type": "code",
|
| 83 |
+
"source": [
|
| 84 |
+
"import nltk\n",
|
| 85 |
+
"\n",
|
| 86 |
+
"nltk.download('punkt')\n"
|
| 87 |
+
],
|
| 88 |
+
"metadata": {
|
| 89 |
+
"colab": {
|
| 90 |
+
"base_uri": "https://localhost:8080/"
|
| 91 |
+
},
|
| 92 |
+
"id": "7LsTdHGeRscq",
|
| 93 |
+
"outputId": "70edda1d-7c50-4a76-e5b8-667d3d22dc8d"
|
| 94 |
+
},
|
| 95 |
+
"execution_count": 3,
|
| 96 |
+
"outputs": [
|
| 97 |
+
{
|
| 98 |
+
"output_type": "stream",
|
| 99 |
+
"name": "stderr",
|
| 100 |
+
"text": [
|
| 101 |
+
"[nltk_data] Downloading package punkt to /root/nltk_data...\n",
|
| 102 |
+
"[nltk_data] Unzipping tokenizers/punkt.zip.\n"
|
| 103 |
+
]
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"output_type": "execute_result",
|
| 107 |
+
"data": {
|
| 108 |
+
"text/plain": [
|
| 109 |
+
"True"
|
| 110 |
+
]
|
| 111 |
+
},
|
| 112 |
+
"metadata": {},
|
| 113 |
+
"execution_count": 3
|
| 114 |
+
}
|
| 115 |
+
]
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"cell_type": "code",
|
| 119 |
+
"source": [
|
| 120 |
+
"import getpass\n",
|
| 121 |
+
"import os\n",
|
| 122 |
+
"\n",
|
| 123 |
+
"\n",
|
| 124 |
+
"def _set_env(key: str):\n",
|
| 125 |
+
" if key not in os.environ:\n",
|
| 126 |
+
" os.environ[key] = getpass.getpass(f\"{key}:\")\n",
|
| 127 |
+
"\n",
|
| 128 |
+
"\n",
|
| 129 |
+
"_set_env(\"OPENAI_API_KEY\")"
|
| 130 |
+
],
|
| 131 |
+
"metadata": {
|
| 132 |
+
"colab": {
|
| 133 |
+
"base_uri": "https://localhost:8080/"
|
| 134 |
+
},
|
| 135 |
+
"id": "FGGRplw8ivAe",
|
| 136 |
+
"outputId": "1881ac34-c7de-4540-9c57-e6e2a7d1c92a"
|
| 137 |
+
},
|
| 138 |
+
"execution_count": 4,
|
| 139 |
+
"outputs": [
|
| 140 |
+
{
|
| 141 |
+
"name": "stdout",
|
| 142 |
+
"output_type": "stream",
|
| 143 |
+
"text": [
|
| 144 |
+
"OPENAI_API_KEY:··········\n"
|
| 145 |
+
]
|
| 146 |
+
}
|
| 147 |
+
]
|
| 148 |
+
},
|
| 149 |
+
{
|
| 150 |
+
"cell_type": "markdown",
|
| 151 |
+
"source": [
|
| 152 |
+
"# Get all public data from wire"
|
| 153 |
+
],
|
| 154 |
+
"metadata": {
|
| 155 |
+
"id": "laZP_yRDMInw"
|
| 156 |
+
}
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"cell_type": "code",
|
| 160 |
+
"source": [
|
| 161 |
+
"!git clone https://github.com/wireapp/wire-server.git"
|
| 162 |
+
],
|
| 163 |
+
"metadata": {
|
| 164 |
+
"colab": {
|
| 165 |
+
"base_uri": "https://localhost:8080/"
|
| 166 |
+
},
|
| 167 |
+
"id": "puiyTy_fMOcw",
|
| 168 |
+
"outputId": "e82f820e-0da1-4c93-817c-b923fe2795a2"
|
| 169 |
+
},
|
| 170 |
+
"execution_count": 5,
|
| 171 |
+
"outputs": [
|
| 172 |
+
{
|
| 173 |
+
"output_type": "stream",
|
| 174 |
+
"name": "stdout",
|
| 175 |
+
"text": [
|
| 176 |
+
"fatal: destination path 'wire-server' already exists and is not an empty directory.\n"
|
| 177 |
+
]
|
| 178 |
+
}
|
| 179 |
+
]
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"cell_type": "code",
|
| 183 |
+
"source": [
|
| 184 |
+
"WIRE_SERVER_DOCS=\"/content/wire-server/docs/src\""
|
| 185 |
+
],
|
| 186 |
+
"metadata": {
|
| 187 |
+
"id": "W6LYPIzwMvSS"
|
| 188 |
+
},
|
| 189 |
+
"execution_count": 6,
|
| 190 |
+
"outputs": []
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"cell_type": "markdown",
|
| 194 |
+
"source": [
|
| 195 |
+
"# Prepare Documents"
|
| 196 |
+
],
|
| 197 |
+
"metadata": {
|
| 198 |
+
"id": "Yycs8xGCNQpA"
|
| 199 |
+
}
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"cell_type": "code",
|
| 203 |
+
"source": [
|
| 204 |
+
"import glob\n",
|
| 205 |
+
"\n",
|
| 206 |
+
"from langchain_community.document_loaders import UnstructuredMarkdownLoader, DirectoryLoader\n",
|
| 207 |
+
"# from langchain_community.document_transformers import\n",
|
| 208 |
+
"\n",
|
| 209 |
+
"loader = DirectoryLoader(WIRE_SERVER_DOCS, glob=\"**/*.md\", loader_cls=UnstructuredMarkdownLoader, loader_kwargs={\"mode\": \"elements\", \"strategy\": \"fast\"})\n",
|
| 210 |
+
"docs = loader.load()"
|
| 211 |
+
],
|
| 212 |
+
"metadata": {
|
| 213 |
+
"id": "Q_wVjh8FNBSS"
|
| 214 |
+
},
|
| 215 |
+
"execution_count": 7,
|
| 216 |
+
"outputs": []
|
| 217 |
+
},
|
| 218 |
+
{
|
| 219 |
+
"cell_type": "code",
|
| 220 |
+
"source": [
|
| 221 |
+
"from langchain_community.document_loaders import WebBaseLoader\n",
|
| 222 |
+
"from langchain_community.vectorstores import Chroma\n",
|
| 223 |
+
"from langchain.text_splitter import MarkdownTextSplitter\n",
|
| 224 |
+
"from langchain_huggingface import HuggingFaceEmbeddings\n",
|
| 225 |
+
"from langchain_community.vectorstores.utils import filter_complex_metadata\n",
|
| 226 |
+
"\n",
|
| 227 |
+
"\n",
|
| 228 |
+
"embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-mpnet-base-v2\")\n",
|
| 229 |
+
"# embeddings = HuggingFaceEmbeddings(model_name=\"dunzhang/stella_en_400M_v5\", model_kwargs= {\"trust_remote_code\":True, \"device\": \"cuda\"})\n",
|
| 230 |
+
"\n",
|
| 231 |
+
"\n",
|
| 232 |
+
"text_splitter = MarkdownTextSplitter.from_tiktoken_encoder(chunk_size=1000, chunk_overlap=200)\n",
|
| 233 |
+
"doc_splits = filter_complex_metadata(text_splitter.split_documents(docs))\n",
|
| 234 |
+
"\n",
|
| 235 |
+
"persist_directory = \"/content/drive/MyDrive/Colab Notebooks/wire-ask-docs/\"\n",
|
| 236 |
+
"\n",
|
| 237 |
+
"vectorstore = Chroma.from_documents(\n",
|
| 238 |
+
" documents=doc_splits,\n",
|
| 239 |
+
" collection_name=\"rag-chroma\",\n",
|
| 240 |
+
" embedding=embeddings,\n",
|
| 241 |
+
" persist_directory=persist_directory\n",
|
| 242 |
+
")"
|
| 243 |
+
],
|
| 244 |
+
"metadata": {
|
| 245 |
+
"colab": {
|
| 246 |
+
"base_uri": "https://localhost:8080/"
|
| 247 |
+
},
|
| 248 |
+
"id": "KfZarM3TjIJ4",
|
| 249 |
+
"outputId": "6bdff9ce-6699-49bd-e9e2-2996e2a2d00f"
|
| 250 |
+
},
|
| 251 |
+
"execution_count": 8,
|
| 252 |
+
"outputs": [
|
| 253 |
+
{
|
| 254 |
+
"output_type": "stream",
|
| 255 |
+
"name": "stderr",
|
| 256 |
+
"text": [
|
| 257 |
+
"WARNING:langchain_community.utils.user_agent:USER_AGENT environment variable not set, consider setting it to identify your requests.\n",
|
| 258 |
+
"/usr/local/lib/python3.10/dist-packages/sentence_transformers/cross_encoder/CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
|
| 259 |
+
" from tqdm.autonotebook import tqdm, trange\n",
|
| 260 |
+
"/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
|
| 261 |
+
"The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
|
| 262 |
+
"To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
|
| 263 |
+
"You will be able to reuse this secret in all of your notebooks.\n",
|
| 264 |
+
"Please note that authentication is recommended but still optional to access public models or datasets.\n",
|
| 265 |
+
" warnings.warn(\n",
|
| 266 |
+
"/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
|
| 267 |
+
" warnings.warn(\n"
|
| 268 |
+
]
|
| 269 |
+
}
|
| 270 |
+
]
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"cell_type": "code",
|
| 274 |
+
"source": [
|
| 275 |
+
"retriever = vectorstore.as_retriever()"
|
| 276 |
+
],
|
| 277 |
+
"metadata": {
|
| 278 |
+
"id": "IPtysn5xkP8X"
|
| 279 |
+
},
|
| 280 |
+
"execution_count": 9,
|
| 281 |
+
"outputs": []
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"cell_type": "markdown",
|
| 285 |
+
"source": [
|
| 286 |
+
"# Retrieval Grader"
|
| 287 |
+
],
|
| 288 |
+
"metadata": {
|
| 289 |
+
"id": "cR3_eWmHxc5I"
|
| 290 |
+
}
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"cell_type": "code",
|
| 294 |
+
"source": [
|
| 295 |
+
"\n",
|
| 296 |
+
"\n",
|
| 297 |
+
"\n",
|
| 298 |
+
"from langchain_core.prompts import ChatPromptTemplate\n",
|
| 299 |
+
"from langchain_openai import ChatOpenAI\n",
|
| 300 |
+
"from pydantic import BaseModel, Field\n",
|
| 301 |
+
"\n",
|
| 302 |
+
"\n",
|
| 303 |
+
"class GradeDocuments(BaseModel):\n",
|
| 304 |
+
" \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
|
| 305 |
+
"\n",
|
| 306 |
+
" binary_score: str = Field(\n",
|
| 307 |
+
" description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
|
| 308 |
+
" )\n",
|
| 309 |
+
"\n",
|
| 310 |
+
"\n",
|
| 311 |
+
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
|
| 312 |
+
"structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
|
| 313 |
+
"\n",
|
| 314 |
+
"system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n\n",
|
| 315 |
+
" It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
|
| 316 |
+
" If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
|
| 317 |
+
" Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
|
| 318 |
+
"grade_prompt = ChatPromptTemplate.from_messages(\n",
|
| 319 |
+
" [\n",
|
| 320 |
+
" (\"system\", system),\n",
|
| 321 |
+
" (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
|
| 322 |
+
" ]\n",
|
| 323 |
+
")\n",
|
| 324 |
+
"\n",
|
| 325 |
+
"retrieval_grader = grade_prompt | structured_llm_grader"
|
| 326 |
+
],
|
| 327 |
+
"metadata": {
|
| 328 |
+
"id": "uwUDHd_2uFTT"
|
| 329 |
+
},
|
| 330 |
+
"execution_count": 22,
|
| 331 |
+
"outputs": []
|
| 332 |
+
},
|
| 333 |
+
{
|
| 334 |
+
"cell_type": "markdown",
|
| 335 |
+
"source": [
|
| 336 |
+
"# Generate"
|
| 337 |
+
],
|
| 338 |
+
"metadata": {
|
| 339 |
+
"id": "4WSunxhFxh8H"
|
| 340 |
+
}
|
| 341 |
+
},
|
| 342 |
+
{
|
| 343 |
+
"cell_type": "code",
|
| 344 |
+
"source": [
|
| 345 |
+
"\n",
|
| 346 |
+
"\n",
|
| 347 |
+
"from langchain import hub\n",
|
| 348 |
+
"from langchain_core.output_parsers import StrOutputParser\n",
|
| 349 |
+
"\n",
|
| 350 |
+
"# Prompt\n",
|
| 351 |
+
"# prompt = hub.pull(\"rlm/rag-prompt\")\n",
|
| 352 |
+
"prompt = hub.pull(\"pwoc517/crafted_prompt_for_rag\")\n",
|
| 353 |
+
"\n",
|
| 354 |
+
"# LLM\n",
|
| 355 |
+
"# llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
|
| 356 |
+
"llm = ChatOpenAI(model_name=\"gpt-4o\", temperature=0)\n",
|
| 357 |
+
"\n",
|
| 358 |
+
"\n",
|
| 359 |
+
"# Post-processing\n",
|
| 360 |
+
"def format_docs(docs):\n",
|
| 361 |
+
" return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
|
| 362 |
+
"\n",
|
| 363 |
+
"\n",
|
| 364 |
+
"# Chain\n",
|
| 365 |
+
"rag_chain = prompt | llm | StrOutputParser()"
|
| 366 |
+
],
|
| 367 |
+
"metadata": {
|
| 368 |
+
"colab": {
|
| 369 |
+
"base_uri": "https://localhost:8080/"
|
| 370 |
+
},
|
| 371 |
+
"id": "jdFrf9rMuSoP",
|
| 372 |
+
"outputId": "744e3e54-05c1-4fd0-9eef-8008f6374b05"
|
| 373 |
+
},
|
| 374 |
+
"execution_count": 23,
|
| 375 |
+
"outputs": [
|
| 376 |
+
{
|
| 377 |
+
"output_type": "stream",
|
| 378 |
+
"name": "stderr",
|
| 379 |
+
"text": [
|
| 380 |
+
"/usr/local/lib/python3.10/dist-packages/langsmith/client.py:323: LangSmithMissingAPIKeyWarning: API key must be provided when using hosted LangSmith API\n",
|
| 381 |
+
" warnings.warn(\n"
|
| 382 |
+
]
|
| 383 |
+
}
|
| 384 |
+
]
|
| 385 |
+
},
|
| 386 |
+
{
|
| 387 |
+
"cell_type": "code",
|
| 388 |
+
"source": [
|
| 389 |
+
"### Hallucination Grader\n",
|
| 390 |
+
"\n",
|
| 391 |
+
"\n",
|
| 392 |
+
"# Data model\n",
|
| 393 |
+
"class GradeHallucinations(BaseModel):\n",
|
| 394 |
+
" \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n",
|
| 395 |
+
"\n",
|
| 396 |
+
" binary_score: str = Field(\n",
|
| 397 |
+
" description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n",
|
| 398 |
+
" )\n",
|
| 399 |
+
"\n",
|
| 400 |
+
"\n",
|
| 401 |
+
"# LLM with function call\n",
|
| 402 |
+
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
|
| 403 |
+
"structured_llm_grader = llm.with_structured_output(GradeHallucinations)\n",
|
| 404 |
+
"\n",
|
| 405 |
+
"# Prompt\n",
|
| 406 |
+
"system = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n\n",
|
| 407 |
+
" Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\n",
|
| 408 |
+
"hallucination_prompt = ChatPromptTemplate.from_messages(\n",
|
| 409 |
+
" [\n",
|
| 410 |
+
" (\"system\", system),\n",
|
| 411 |
+
" (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n",
|
| 412 |
+
" ]\n",
|
| 413 |
+
")\n",
|
| 414 |
+
"\n",
|
| 415 |
+
"hallucination_grader = hallucination_prompt | structured_llm_grader"
|
| 416 |
+
],
|
| 417 |
+
"metadata": {
|
| 418 |
+
"id": "P8QXWyVWuTdA"
|
| 419 |
+
},
|
| 420 |
+
"execution_count": 24,
|
| 421 |
+
"outputs": []
|
| 422 |
+
},
|
| 423 |
+
{
|
| 424 |
+
"cell_type": "code",
|
| 425 |
+
"source": [
|
| 426 |
+
"### Answer Grader\n",
|
| 427 |
+
"\n",
|
| 428 |
+
"\n",
|
| 429 |
+
"# Data model\n",
|
| 430 |
+
"class GradeAnswer(BaseModel):\n",
|
| 431 |
+
" \"\"\"Binary score to assess answer addresses question.\"\"\"\n",
|
| 432 |
+
"\n",
|
| 433 |
+
" binary_score: str = Field(\n",
|
| 434 |
+
" description=\"Answer addresses the question, 'yes' or 'no'\"\n",
|
| 435 |
+
" )\n",
|
| 436 |
+
"\n",
|
| 437 |
+
"\n",
|
| 438 |
+
"# LLM with function call\n",
|
| 439 |
+
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
|
| 440 |
+
"structured_llm_grader = llm.with_structured_output(GradeAnswer)\n",
|
| 441 |
+
"\n",
|
| 442 |
+
"# Prompt\n",
|
| 443 |
+
"system = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n\n",
|
| 444 |
+
" Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\n",
|
| 445 |
+
"answer_prompt = ChatPromptTemplate.from_messages(\n",
|
| 446 |
+
" [\n",
|
| 447 |
+
" (\"system\", system),\n",
|
| 448 |
+
" (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n",
|
| 449 |
+
" ]\n",
|
| 450 |
+
")\n",
|
| 451 |
+
"\n",
|
| 452 |
+
"answer_grader = answer_prompt | structured_llm_grader"
|
| 453 |
+
],
|
| 454 |
+
"metadata": {
|
| 455 |
+
"id": "3G__2u85u3DT"
|
| 456 |
+
},
|
| 457 |
+
"execution_count": 25,
|
| 458 |
+
"outputs": []
|
| 459 |
+
},
|
| 460 |
+
{
|
| 461 |
+
"cell_type": "code",
|
| 462 |
+
"source": [
|
| 463 |
+
"### Question Re-writer\n",
|
| 464 |
+
"\n",
|
| 465 |
+
"# LLM\n",
|
| 466 |
+
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
|
| 467 |
+
"\n",
|
| 468 |
+
"# Prompt\n",
|
| 469 |
+
"system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n\n",
|
| 470 |
+
" for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
|
| 471 |
+
"re_write_prompt = ChatPromptTemplate.from_messages(\n",
|
| 472 |
+
" [\n",
|
| 473 |
+
" (\"system\", system),\n",
|
| 474 |
+
" (\n",
|
| 475 |
+
" \"human\",\n",
|
| 476 |
+
" \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
|
| 477 |
+
" ),\n",
|
| 478 |
+
" ]\n",
|
| 479 |
+
")\n",
|
| 480 |
+
"\n",
|
| 481 |
+
"question_rewriter = re_write_prompt | llm | StrOutputParser()"
|
| 482 |
+
],
|
| 483 |
+
"metadata": {
|
| 484 |
+
"id": "tfSDrPinu3w-"
|
| 485 |
+
},
|
| 486 |
+
"execution_count": 26,
|
| 487 |
+
"outputs": []
|
| 488 |
+
},
|
| 489 |
+
{
|
| 490 |
+
"cell_type": "markdown",
|
| 491 |
+
"source": [
|
| 492 |
+
"# Graph"
|
| 493 |
+
],
|
| 494 |
+
"metadata": {
|
| 495 |
+
"id": "IjR00mOsu_zs"
|
| 496 |
+
}
|
| 497 |
+
},
|
| 498 |
+
{
|
| 499 |
+
"cell_type": "markdown",
|
| 500 |
+
"source": [
|
| 501 |
+
"## State"
|
| 502 |
+
],
|
| 503 |
+
"metadata": {
|
| 504 |
+
"id": "M5MzWXM7y3o-"
|
| 505 |
+
}
|
| 506 |
+
},
|
| 507 |
+
{
|
| 508 |
+
"cell_type": "code",
|
| 509 |
+
"source": [
|
| 510 |
+
"from typing import List\n",
|
| 511 |
+
"\n",
|
| 512 |
+
"from typing_extensions import TypedDict\n",
|
| 513 |
+
"\n",
|
| 514 |
+
"\n",
|
| 515 |
+
"class GraphState(TypedDict):\n",
|
| 516 |
+
" \"\"\"\n",
|
| 517 |
+
" Represents the state of our graph.\n",
|
| 518 |
+
"\n",
|
| 519 |
+
" Attributes:\n",
|
| 520 |
+
" question: question\n",
|
| 521 |
+
" generation: LLM generation\n",
|
| 522 |
+
" documents: list of documents\n",
|
| 523 |
+
" \"\"\"\n",
|
| 524 |
+
"\n",
|
| 525 |
+
" question: str\n",
|
| 526 |
+
" generation: str\n",
|
| 527 |
+
" documents: List[str]"
|
| 528 |
+
],
|
| 529 |
+
"metadata": {
|
| 530 |
+
"id": "mq-B6DUrvBYX"
|
| 531 |
+
},
|
| 532 |
+
"execution_count": 27,
|
| 533 |
+
"outputs": []
|
| 534 |
+
},
|
| 535 |
+
{
|
| 536 |
+
"cell_type": "markdown",
|
| 537 |
+
"source": [
|
| 538 |
+
"## Nodes"
|
| 539 |
+
],
|
| 540 |
+
"metadata": {
|
| 541 |
+
"id": "f2osp-Aay5vi"
|
| 542 |
+
}
|
| 543 |
+
},
|
| 544 |
+
{
|
| 545 |
+
"cell_type": "code",
|
| 546 |
+
"source": [
|
| 547 |
+
"\n",
|
| 548 |
+
"def retrieve(state):\n",
|
| 549 |
+
" \"\"\"\n",
|
| 550 |
+
" Retrieve documents\n",
|
| 551 |
+
"\n",
|
| 552 |
+
" Args:\n",
|
| 553 |
+
" state (dict): The current graph state\n",
|
| 554 |
+
"\n",
|
| 555 |
+
" Returns:\n",
|
| 556 |
+
" state (dict): New key added to state, documents, that contains retrieved documents\n",
|
| 557 |
+
" \"\"\"\n",
|
| 558 |
+
" print(\"---RETRIEVE---\")\n",
|
| 559 |
+
" question = state[\"question\"]\n",
|
| 560 |
+
"\n",
|
| 561 |
+
" # Retrieval\n",
|
| 562 |
+
" documents = retriever.invoke(question)\n",
|
| 563 |
+
" return {\"documents\": documents, \"question\": question}\n",
|
| 564 |
+
"\n",
|
| 565 |
+
"\n",
|
| 566 |
+
"def generate(state):\n",
|
| 567 |
+
" \"\"\"\n",
|
| 568 |
+
" Generate answer\n",
|
| 569 |
+
"\n",
|
| 570 |
+
" Args:\n",
|
| 571 |
+
" state (dict): The current graph state\n",
|
| 572 |
+
"\n",
|
| 573 |
+
" Returns:\n",
|
| 574 |
+
" state (dict): New key added to state, generation, that contains LLM generation\n",
|
| 575 |
+
" \"\"\"\n",
|
| 576 |
+
" print(\"---GENERATE---\")\n",
|
| 577 |
+
" question = state[\"question\"]\n",
|
| 578 |
+
" documents = state[\"documents\"]\n",
|
| 579 |
+
"\n",
|
| 580 |
+
" # RAG generation\n",
|
| 581 |
+
" generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
|
| 582 |
+
" return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
|
| 583 |
+
"\n",
|
| 584 |
+
"\n",
|
| 585 |
+
"def grade_documents(state):\n",
|
| 586 |
+
" \"\"\"\n",
|
| 587 |
+
" Determines whether the retrieved documents are relevant to the question.\n",
|
| 588 |
+
"\n",
|
| 589 |
+
" Args:\n",
|
| 590 |
+
" state (dict): The current graph state\n",
|
| 591 |
+
"\n",
|
| 592 |
+
" Returns:\n",
|
| 593 |
+
" state (dict): Updates documents key with only filtered relevant documents\n",
|
| 594 |
+
" \"\"\"\n",
|
| 595 |
+
"\n",
|
| 596 |
+
" print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
|
| 597 |
+
" question = state[\"question\"]\n",
|
| 598 |
+
" documents = state[\"documents\"]\n",
|
| 599 |
+
"\n",
|
| 600 |
+
" # Score each doc\n",
|
| 601 |
+
" filtered_docs = []\n",
|
| 602 |
+
" for d in documents:\n",
|
| 603 |
+
" score = retrieval_grader.invoke(\n",
|
| 604 |
+
" {\"question\": question, \"document\": d.page_content}\n",
|
| 605 |
+
" )\n",
|
| 606 |
+
" grade = score.binary_score\n",
|
| 607 |
+
" if grade == \"yes\":\n",
|
| 608 |
+
" print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
|
| 609 |
+
" filtered_docs.append(d)\n",
|
| 610 |
+
" else:\n",
|
| 611 |
+
" print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
|
| 612 |
+
" continue\n",
|
| 613 |
+
" return {\"documents\": filtered_docs, \"question\": question}\n",
|
| 614 |
+
"\n",
|
| 615 |
+
"\n",
|
| 616 |
+
"def transform_query(state):\n",
|
| 617 |
+
" \"\"\"\n",
|
| 618 |
+
" Transform the query to produce a better question.\n",
|
| 619 |
+
"\n",
|
| 620 |
+
" Args:\n",
|
| 621 |
+
" state (dict): The current graph state\n",
|
| 622 |
+
"\n",
|
| 623 |
+
" Returns:\n",
|
| 624 |
+
" state (dict): Updates question key with a re-phrased question\n",
|
| 625 |
+
" \"\"\"\n",
|
| 626 |
+
"\n",
|
| 627 |
+
" print(\"---TRANSFORM QUERY---\")\n",
|
| 628 |
+
" question = state[\"question\"]\n",
|
| 629 |
+
" documents = state[\"documents\"]\n",
|
| 630 |
+
"\n",
|
| 631 |
+
" # Re-write question\n",
|
| 632 |
+
" better_question = question_rewriter.invoke({\"question\": question})\n",
|
| 633 |
+
" return {\"documents\": documents, \"question\": better_question}\n",
|
| 634 |
+
"\n",
|
| 635 |
+
"\n"
|
| 636 |
+
],
|
| 637 |
+
"metadata": {
|
| 638 |
+
"id": "_0SDlSjVvCRe"
|
| 639 |
+
},
|
| 640 |
+
"execution_count": 28,
|
| 641 |
+
"outputs": []
|
| 642 |
+
},
|
| 643 |
+
{
|
| 644 |
+
"cell_type": "markdown",
|
| 645 |
+
"source": [
|
| 646 |
+
"## Edges"
|
| 647 |
+
],
|
| 648 |
+
"metadata": {
|
| 649 |
+
"id": "ya4e8XOuzCi8"
|
| 650 |
+
}
|
| 651 |
+
},
|
| 652 |
+
{
|
| 653 |
+
"cell_type": "code",
|
| 654 |
+
"source": [
|
| 655 |
+
"\n",
|
| 656 |
+
"\n",
|
| 657 |
+
"def decide_to_generate(state):\n",
|
| 658 |
+
" \"\"\"\n",
|
| 659 |
+
" Determines whether to generate an answer, or re-generate a question.\n",
|
| 660 |
+
"\n",
|
| 661 |
+
" Args:\n",
|
| 662 |
+
" state (dict): The current graph state\n",
|
| 663 |
+
"\n",
|
| 664 |
+
" Returns:\n",
|
| 665 |
+
" str: Binary decision for next node to call\n",
|
| 666 |
+
" \"\"\"\n",
|
| 667 |
+
"\n",
|
| 668 |
+
" print(\"---ASSESS GRADED DOCUMENTS---\")\n",
|
| 669 |
+
" state[\"question\"]\n",
|
| 670 |
+
" filtered_documents = state[\"documents\"]\n",
|
| 671 |
+
"\n",
|
| 672 |
+
" if not filtered_documents:\n",
|
| 673 |
+
" # All documents have been filtered check_relevance\n",
|
| 674 |
+
" # We will re-generate a new query\n",
|
| 675 |
+
" print(\n",
|
| 676 |
+
" \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
|
| 677 |
+
" )\n",
|
| 678 |
+
" return \"transform_query\"\n",
|
| 679 |
+
" else:\n",
|
| 680 |
+
" # We have relevant documents, so generate answer\n",
|
| 681 |
+
" print(\"---DECISION: GENERATE---\")\n",
|
| 682 |
+
" return \"generate\"\n",
|
| 683 |
+
"\n",
|
| 684 |
+
"\n",
|
| 685 |
+
"def grade_generation_v_documents_and_question(state):\n",
|
| 686 |
+
" \"\"\"\n",
|
| 687 |
+
" Determines whether the generation is grounded in the document and answers question.\n",
|
| 688 |
+
"\n",
|
| 689 |
+
" Args:\n",
|
| 690 |
+
" state (dict): The current graph state\n",
|
| 691 |
+
"\n",
|
| 692 |
+
" Returns:\n",
|
| 693 |
+
" str: Decision for next node to call\n",
|
| 694 |
+
" \"\"\"\n",
|
| 695 |
+
"\n",
|
| 696 |
+
" print(\"---CHECK HALLUCINATIONS---\")\n",
|
| 697 |
+
" question = state[\"question\"]\n",
|
| 698 |
+
" documents = state[\"documents\"]\n",
|
| 699 |
+
" generation = state[\"generation\"]\n",
|
| 700 |
+
"\n",
|
| 701 |
+
" score = hallucination_grader.invoke(\n",
|
| 702 |
+
" {\"documents\": documents, \"generation\": generation}\n",
|
| 703 |
+
" )\n",
|
| 704 |
+
" grade = score.binary_score\n",
|
| 705 |
+
"\n",
|
| 706 |
+
" # Check hallucination\n",
|
| 707 |
+
" if grade == \"yes\":\n",
|
| 708 |
+
" print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
|
| 709 |
+
" # Check question-answering\n",
|
| 710 |
+
" print(\"---GRADE GENERATION vs QUESTION---\")\n",
|
| 711 |
+
" score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
|
| 712 |
+
" grade = score.binary_score\n",
|
| 713 |
+
" if grade == \"yes\":\n",
|
| 714 |
+
" print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
|
| 715 |
+
" return \"useful\"\n",
|
| 716 |
+
" else:\n",
|
| 717 |
+
" print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
|
| 718 |
+
" return \"not useful\"\n",
|
| 719 |
+
" else:\n",
|
| 720 |
+
" print(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
|
| 721 |
+
" return \"not supported\""
|
| 722 |
+
],
|
| 723 |
+
"metadata": {
|
| 724 |
+
"id": "u6eFBLVJzBuw"
|
| 725 |
+
},
|
| 726 |
+
"execution_count": 29,
|
| 727 |
+
"outputs": []
|
| 728 |
+
},
|
| 729 |
+
{
|
| 730 |
+
"cell_type": "markdown",
|
| 731 |
+
"source": [
|
| 732 |
+
"## Graph"
|
| 733 |
+
],
|
| 734 |
+
"metadata": {
|
| 735 |
+
"id": "WHb3ekRGzGZl"
|
| 736 |
+
}
|
| 737 |
+
},
|
| 738 |
+
{
|
| 739 |
+
"cell_type": "code",
|
| 740 |
+
"source": [
|
| 741 |
+
"from langgraph.graph import END, StateGraph, START\n",
|
| 742 |
+
"\n",
|
| 743 |
+
"workflow = StateGraph(GraphState)\n",
|
| 744 |
+
"\n",
|
| 745 |
+
"# Define the nodes\n",
|
| 746 |
+
"workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
|
| 747 |
+
"workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
|
| 748 |
+
"workflow.add_node(\"generate\", generate) # generatae\n",
|
| 749 |
+
"workflow.add_node(\"transform_query\", transform_query) # transform_query\n",
|
| 750 |
+
"\n",
|
| 751 |
+
"# Build graph\n",
|
| 752 |
+
"workflow.add_edge(START, \"retrieve\")\n",
|
| 753 |
+
"workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
|
| 754 |
+
"workflow.add_conditional_edges(\n",
|
| 755 |
+
" \"grade_documents\",\n",
|
| 756 |
+
" decide_to_generate,\n",
|
| 757 |
+
" {\n",
|
| 758 |
+
" \"transform_query\": \"transform_query\",\n",
|
| 759 |
+
" \"generate\": \"generate\",\n",
|
| 760 |
+
" },\n",
|
| 761 |
+
")\n",
|
| 762 |
+
"workflow.add_edge(\"transform_query\", \"retrieve\")\n",
|
| 763 |
+
"workflow.add_conditional_edges(\n",
|
| 764 |
+
" \"generate\",\n",
|
| 765 |
+
" grade_generation_v_documents_and_question,\n",
|
| 766 |
+
" {\n",
|
| 767 |
+
" \"not supported\": \"generate\",\n",
|
| 768 |
+
" \"useful\": END,\n",
|
| 769 |
+
" \"not useful\": \"transform_query\",\n",
|
| 770 |
+
" },\n",
|
| 771 |
+
")\n",
|
| 772 |
+
"\n",
|
| 773 |
+
"# Compile\n",
|
| 774 |
+
"app = workflow.compile()"
|
| 775 |
+
],
|
| 776 |
+
"metadata": {
|
| 777 |
+
"id": "596hNaDvvE5S"
|
| 778 |
+
},
|
| 779 |
+
"execution_count": 30,
|
| 780 |
+
"outputs": []
|
| 781 |
+
},
|
| 782 |
+
{
|
| 783 |
+
"cell_type": "markdown",
|
| 784 |
+
"source": [
|
| 785 |
+
"# Example"
|
| 786 |
+
],
|
| 787 |
+
"metadata": {
|
| 788 |
+
"id": "oBEYM8tUzI5t"
|
| 789 |
+
}
|
| 790 |
+
},
|
| 791 |
+
{
|
| 792 |
+
"cell_type": "code",
|
| 793 |
+
"source": [
|
| 794 |
+
"from pprint import pprint\n",
|
| 795 |
+
"\n",
|
| 796 |
+
"# question = \"Show me latest changes\"\n",
|
| 797 |
+
"# question = \"How to install wire-server using helm, give me step by step instruction\"\n",
|
| 798 |
+
"# question = \"Show me Graphviz graph of the whole WireApp system\"\n",
|
| 799 |
+
"# question = \"What are the categories of documents in Wire documentation?\"\n",
|
| 800 |
+
"# question = \"Tell me more about Restund\"\n",
|
| 801 |
+
"question = \"Tell me how Restund works\"\n",
|
| 802 |
+
"\n",
|
| 803 |
+
"inputs = {\"question\": question}\n",
|
| 804 |
+
"for output in app.stream(inputs, config={\"recursion_limit\": 70}):\n",
|
| 805 |
+
" for key, value in output.items():\n",
|
| 806 |
+
" # Node\n",
|
| 807 |
+
" print(f\"Node '{key}':\")\n",
|
| 808 |
+
" # Optional: print full state at each node\n",
|
| 809 |
+
" # pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
|
| 810 |
+
" print(\"---\")\n",
|
| 811 |
+
"\n",
|
| 812 |
+
"# Final generation\n",
|
| 813 |
+
"pprint(value[\"generation\"])"
|
| 814 |
+
],
|
| 815 |
+
"metadata": {
|
| 816 |
+
"colab": {
|
| 817 |
+
"base_uri": "https://localhost:8080/"
|
| 818 |
+
},
|
| 819 |
+
"id": "m5k1AaLrvPQ9",
|
| 820 |
+
"outputId": "f54e4699-a73a-42bf-acdb-4fa3ed7e2d3c"
|
| 821 |
+
},
|
| 822 |
+
"execution_count": 46,
|
| 823 |
+
"outputs": [
|
| 824 |
+
{
|
| 825 |
+
"output_type": "stream",
|
| 826 |
+
"name": "stdout",
|
| 827 |
+
"text": [
|
| 828 |
+
"---RETRIEVE---\n",
|
| 829 |
+
"Node 'retrieve':\n",
|
| 830 |
+
"---\n",
|
| 831 |
+
"---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
|
| 832 |
+
"---GRADE: DOCUMENT RELEVANT---\n",
|
| 833 |
+
"---GRADE: DOCUMENT NOT RELEVANT---\n",
|
| 834 |
+
"---GRADE: DOCUMENT NOT RELEVANT---\n",
|
| 835 |
+
"---GRADE: DOCUMENT RELEVANT---\n",
|
| 836 |
+
"---ASSESS GRADED DOCUMENTS---\n",
|
| 837 |
+
"---DECISION: GENERATE---\n",
|
| 838 |
+
"Node 'grade_documents':\n",
|
| 839 |
+
"---\n",
|
| 840 |
+
"---GENERATE---\n",
|
| 841 |
+
"---CHECK HALLUCINATIONS---\n",
|
| 842 |
+
"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
|
| 843 |
+
"---GRADE GENERATION vs QUESTION---\n",
|
| 844 |
+
"---DECISION: GENERATION ADDRESSES QUESTION---\n",
|
| 845 |
+
"Node 'generate':\n",
|
| 846 |
+
"---\n",
|
| 847 |
+
"('Restund is a tool designed to assist in NAT-traversal, which is crucial for '\n",
|
| 848 |
+
" 'establishing direct peer-to-peer connections between clients that are behind '\n",
|
| 849 |
+
" 'Network Address Translation (NAT). Its primary goal is to enable these '\n",
|
| 850 |
+
" 'clients to connect directly, ensuring optimal call quality and the lowest '\n",
|
| 851 |
+
" 'possible latency. By facilitating direct connections, Restund helps to '\n",
|
| 852 |
+
" 'bypass the complications that NAT can introduce, which often hinder direct '\n",
|
| 853 |
+
" 'communication. This makes it particularly useful in scenarios where '\n",
|
| 854 |
+
" 'high-quality, low-latency communication is essential, such as in voice or '\n",
|
| 855 |
+
" 'video calls.')\n"
|
| 856 |
+
]
|
| 857 |
+
}
|
| 858 |
+
]
|
| 859 |
+
},
|
| 860 |
+
{
|
| 861 |
+
"cell_type": "code",
|
| 862 |
+
"source": [],
|
| 863 |
+
"metadata": {
|
| 864 |
+
"id": "E3Hz8Qy4xJWS"
|
| 865 |
+
},
|
| 866 |
+
"execution_count": 19,
|
| 867 |
+
"outputs": []
|
| 868 |
+
}
|
| 869 |
+
]
|
| 870 |
+
}
|
requirements.txt
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
aiohappyeyeballs==2.4.0
|
| 2 |
+
aiohttp==3.10.6
|
| 3 |
+
aiosignal==1.3.1
|
| 4 |
+
annotated-types==0.7.0
|
| 5 |
+
anyio==4.6.0
|
| 6 |
+
argon2-cffi==23.1.0
|
| 7 |
+
argon2-cffi-bindings==21.2.0
|
| 8 |
+
arrow==1.3.0
|
| 9 |
+
asttokens==2.4.1
|
| 10 |
+
async-lru==2.0.4
|
| 11 |
+
attrs==24.2.0
|
| 12 |
+
babel==2.16.0
|
| 13 |
+
backoff==2.2.1
|
| 14 |
+
beautifulsoup4==4.12.3
|
| 15 |
+
bleach==6.1.0
|
| 16 |
+
certifi==2024.8.30
|
| 17 |
+
cffi==1.17.1
|
| 18 |
+
charset-normalizer==3.3.2
|
| 19 |
+
comm==0.2.2
|
| 20 |
+
debugpy==1.8.5
|
| 21 |
+
decorator==5.1.1
|
| 22 |
+
defusedxml==0.7.1
|
| 23 |
+
distro==1.9.0
|
| 24 |
+
executing==2.1.0
|
| 25 |
+
fastjsonschema==2.20.0
|
| 26 |
+
filelock==3.16.1
|
| 27 |
+
fqdn==1.5.1
|
| 28 |
+
frozenlist==1.4.1
|
| 29 |
+
fsspec==2024.9.0
|
| 30 |
+
googleapis-common-protos==1.65.0
|
| 31 |
+
grpcio==1.66.1
|
| 32 |
+
h11==0.14.0
|
| 33 |
+
haystack-ai==2.5.1
|
| 34 |
+
haystack-experimental==0.1.1
|
| 35 |
+
httpcore==1.0.5
|
| 36 |
+
httpx==0.27.2
|
| 37 |
+
huggingface-hub==0.25.1
|
| 38 |
+
idna==3.10
|
| 39 |
+
ipykernel==6.29.5
|
| 40 |
+
ipython==8.27.0
|
| 41 |
+
ipywidgets==8.1.5
|
| 42 |
+
isoduration==20.11.0
|
| 43 |
+
jedi==0.19.1
|
| 44 |
+
Jinja2==3.1.4
|
| 45 |
+
jiter==0.5.0
|
| 46 |
+
joblib==1.4.2
|
| 47 |
+
json5==0.9.25
|
| 48 |
+
jsonpointer==3.0.0
|
| 49 |
+
jsonschema==4.23.0
|
| 50 |
+
jsonschema-specifications==2023.12.1
|
| 51 |
+
jupyter==1.1.1
|
| 52 |
+
jupyter-console==6.6.3
|
| 53 |
+
jupyter-events==0.10.0
|
| 54 |
+
jupyter-lsp==2.2.5
|
| 55 |
+
jupyter_client==8.6.3
|
| 56 |
+
jupyter_core==5.7.2
|
| 57 |
+
jupyter_server==2.14.2
|
| 58 |
+
jupyter_server_terminals==0.5.3
|
| 59 |
+
jupyterlab==4.2.5
|
| 60 |
+
jupyterlab_pygments==0.3.0
|
| 61 |
+
jupyterlab_server==2.27.3
|
| 62 |
+
jupyterlab_widgets==3.0.13
|
| 63 |
+
lazy-imports==0.3.1
|
| 64 |
+
lz4==4.3.3
|
| 65 |
+
MarkupSafe==2.1.5
|
| 66 |
+
matplotlib-inline==0.1.7
|
| 67 |
+
mistune==3.0.2
|
| 68 |
+
monotonic==1.6
|
| 69 |
+
more-itertools==10.5.0
|
| 70 |
+
mpmath==1.3.0
|
| 71 |
+
multidict==6.1.0
|
| 72 |
+
nbclient==0.10.0
|
| 73 |
+
nbconvert==7.16.4
|
| 74 |
+
nbformat==5.10.4
|
| 75 |
+
nest-asyncio==1.6.0
|
| 76 |
+
networkx==3.3
|
| 77 |
+
notebook==7.2.2
|
| 78 |
+
notebook_shim==0.2.4
|
| 79 |
+
numpy==1.26.4
|
| 80 |
+
nvidia-cublas-cu12==12.1.3.1
|
| 81 |
+
nvidia-cuda-cupti-cu12==12.1.105
|
| 82 |
+
nvidia-cuda-nvrtc-cu12==12.1.105
|
| 83 |
+
nvidia-cuda-runtime-cu12==12.1.105
|
| 84 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 85 |
+
nvidia-cufft-cu12==11.0.2.54
|
| 86 |
+
nvidia-curand-cu12==10.3.2.106
|
| 87 |
+
nvidia-cusolver-cu12==11.4.5.107
|
| 88 |
+
nvidia-cusparse-cu12==12.1.0.106
|
| 89 |
+
nvidia-nccl-cu12==2.20.5
|
| 90 |
+
nvidia-nvjitlink-cu12==12.6.68
|
| 91 |
+
nvidia-nvtx-cu12==12.1.105
|
| 92 |
+
openai==1.47.1
|
| 93 |
+
overrides==7.7.0
|
| 94 |
+
packaging==24.1
|
| 95 |
+
pandas==2.2.3
|
| 96 |
+
pandocfilters==1.5.1
|
| 97 |
+
parso==0.8.4
|
| 98 |
+
pexpect==4.9.0
|
| 99 |
+
pillow==10.4.0
|
| 100 |
+
pinecone==5.3.1
|
| 101 |
+
pinecone-client==5.0.1
|
| 102 |
+
pinecone-plugin-inference==1.1.0
|
| 103 |
+
pinecone-plugin-interface==0.0.7
|
| 104 |
+
pinecone_haystack==2.0.0
|
| 105 |
+
platformdirs==4.3.6
|
| 106 |
+
posthog==3.6.6
|
| 107 |
+
prometheus_client==0.21.0
|
| 108 |
+
prompt_toolkit==3.0.47
|
| 109 |
+
protobuf==4.25.5
|
| 110 |
+
protoc-gen-openapiv2==0.0.1
|
| 111 |
+
psutil==6.0.0
|
| 112 |
+
ptyprocess==0.7.0
|
| 113 |
+
pure_eval==0.2.3
|
| 114 |
+
pycparser==2.22
|
| 115 |
+
pydantic==2.9.2
|
| 116 |
+
pydantic_core==2.23.4
|
| 117 |
+
Pygments==2.18.0
|
| 118 |
+
python-dateutil==2.9.0.post0
|
| 119 |
+
python-dotenv==1.0.1
|
| 120 |
+
python-json-logger==2.0.7
|
| 121 |
+
pytz==2024.2
|
| 122 |
+
PyYAML==6.0.2
|
| 123 |
+
pyzmq==26.2.0
|
| 124 |
+
referencing==0.35.1
|
| 125 |
+
regex==2024.9.11
|
| 126 |
+
requests==2.32.3
|
| 127 |
+
rfc3339-validator==0.1.4
|
| 128 |
+
rfc3986-validator==0.1.1
|
| 129 |
+
rpds-py==0.20.0
|
| 130 |
+
safetensors==0.4.5
|
| 131 |
+
scikit-learn==1.5.2
|
| 132 |
+
scipy==1.14.1
|
| 133 |
+
Send2Trash==1.8.3
|
| 134 |
+
sentence-transformers==3.1.1
|
| 135 |
+
setuptools==75.1.0
|
| 136 |
+
six==1.16.0
|
| 137 |
+
sniffio==1.3.1
|
| 138 |
+
soupsieve==2.6
|
| 139 |
+
stack-data==0.6.3
|
| 140 |
+
sympy==1.13.3
|
| 141 |
+
tenacity==9.0.0
|
| 142 |
+
terminado==0.18.1
|
| 143 |
+
threadpoolctl==3.5.0
|
| 144 |
+
tinycss2==1.3.0
|
| 145 |
+
tokenizers==0.19.1
|
| 146 |
+
torch==2.4.1
|
| 147 |
+
tornado==6.4.1
|
| 148 |
+
tqdm==4.66.5
|
| 149 |
+
traitlets==5.14.3
|
| 150 |
+
transformers==4.44.2
|
| 151 |
+
triton==3.0.0
|
| 152 |
+
types-python-dateutil==2.9.0.20240906
|
| 153 |
+
typing_extensions==4.12.2
|
| 154 |
+
tzdata==2024.2
|
| 155 |
+
uri-template==1.3.0
|
| 156 |
+
urllib3==2.2.3
|
| 157 |
+
wcwidth==0.2.13
|
| 158 |
+
webcolors==24.8.0
|
| 159 |
+
webencodings==0.5.1
|
| 160 |
+
websocket-client==1.8.0
|
| 161 |
+
widgetsnbextension==4.0.13
|
| 162 |
+
yarl==1.12.1
|
utils/github_scraper.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import aiohttp
|
| 2 |
+
import asyncio
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
async def fetch_repositories(session: aiohttp.ClientSession, org_name: str,
|
| 6 |
+
api_key: str, repo_limit=None) -> list[dict]:
|
| 7 |
+
"""
|
| 8 |
+
Returns a list of repositories in the given organization.
|
| 9 |
+
Each repository is represented as a dictionary with repository metadata.
|
| 10 |
+
If repo_limit is provided, only that number of repositories is fetched.
|
| 11 |
+
"""
|
| 12 |
+
all_repos = []
|
| 13 |
+
page = 1
|
| 14 |
+
|
| 15 |
+
while True:
|
| 16 |
+
url = f"https://api.github.com/orgs/{org_name}/repos"
|
| 17 |
+
headers = {
|
| 18 |
+
"Authorization": f"token {api_key}",
|
| 19 |
+
"Accept": "application/vnd.github.v3+json"
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
if repo_limit is not None and 0 < repo_limit <= 100:
|
| 23 |
+
# Maximum of 100 items per page
|
| 24 |
+
params = {"per_page": repo_limit, "page": page}
|
| 25 |
+
else:
|
| 26 |
+
params = {"per_page": 100, "page": page}
|
| 27 |
+
|
| 28 |
+
async with session.get(url, headers=headers, params=params) as response:
|
| 29 |
+
if response.status == 200:
|
| 30 |
+
repos = await response.json()
|
| 31 |
+
if not repos:
|
| 32 |
+
break # Stop when no more repositories are found
|
| 33 |
+
all_repos.extend(repos)
|
| 34 |
+
|
| 35 |
+
# Respect the repo_limit if it is set
|
| 36 |
+
if repo_limit and len(all_repos) >= repo_limit:
|
| 37 |
+
return all_repos[:repo_limit] # Return only up to the repo_limit
|
| 38 |
+
|
| 39 |
+
page += 1 # Move to the next page for pagination
|
| 40 |
+
else:
|
| 41 |
+
print(f"Error fetching repositories for {org_name}: {response.status}")
|
| 42 |
+
break
|
| 43 |
+
|
| 44 |
+
return all_repos
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
async def fetch_repo_contents(session: aiohttp.ClientSession, repo_full_name: str,
|
| 48 |
+
api_key: str, path="") -> list[dict]:
|
| 49 |
+
"""
|
| 50 |
+
Returns the contents of a given repository, which can include files and directories.
|
| 51 |
+
The contents are represented as a list of dictionaries.
|
| 52 |
+
"""
|
| 53 |
+
url = f"https://api.github.com/repos/{repo_full_name}/contents/{path}"
|
| 54 |
+
headers = {
|
| 55 |
+
"Authorization": f"token {api_key}",
|
| 56 |
+
"Accept": "application/vnd.github.v3+json"
|
| 57 |
+
}
|
| 58 |
+
async with session.get(url, headers=headers) as response:
|
| 59 |
+
if response.status == 200:
|
| 60 |
+
return await response.json()
|
| 61 |
+
else:
|
| 62 |
+
print(f"Error fetching contents for {repo_full_name}/{path}: {response.status}")
|
| 63 |
+
return []
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
async def fetch_md_files(session: aiohttp.ClientSession, repo_full_name: str,
|
| 67 |
+
api_key: str, path="") -> list[dict]:
|
| 68 |
+
"""
|
| 69 |
+
Recursively fetches all Markdown (.md) files from the repository and directories.
|
| 70 |
+
Returns a list of dictionaries with the file name, download URL, and other metadata.
|
| 71 |
+
"""
|
| 72 |
+
contents = await fetch_repo_contents(session, repo_full_name, api_key, path)
|
| 73 |
+
md_files = []
|
| 74 |
+
|
| 75 |
+
for item in contents:
|
| 76 |
+
if item['type'] == 'file' and item['name'].endswith('.md'):
|
| 77 |
+
# Fetch .md file along with its download URL and other metadata
|
| 78 |
+
md_files.append({
|
| 79 |
+
'content': await fetch_file_content(item['download_url']),
|
| 80 |
+
'metadata': {
|
| 81 |
+
'url': item['html_url'],
|
| 82 |
+
'title': repo_full_name.split('/')[-1] + '/' + item['path'],
|
| 83 |
+
'headline': '', # url_scraper document has this field
|
| 84 |
+
'date': await fetch_last_modified_date(session, repo_full_name=repo_full_name,
|
| 85 |
+
file_path=item['path'], api_key=api_key)
|
| 86 |
+
}
|
| 87 |
+
})
|
| 88 |
+
elif (item['type'] == 'dir' and
|
| 89 |
+
item['name'] not in [".github"]): # If it's a directory, recursively fetch contents
|
| 90 |
+
md_files += await fetch_md_files(session, repo_full_name, api_key, item['path'])
|
| 91 |
+
|
| 92 |
+
return md_files
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
async def fetch_last_modified_date(session: aiohttp.ClientSession, repo_full_name: str,
|
| 96 |
+
file_path: str, api_key: str) -> str:
|
| 97 |
+
"""
|
| 98 |
+
Fetches the last commit date for a given file in the repository by checking its commit history.
|
| 99 |
+
Returns the date in ISO 8601 format (e.g., "2023-09-26T12:34:56Z").
|
| 100 |
+
"""
|
| 101 |
+
url = f"https://api.github.com/repos/{repo_full_name}/commits"
|
| 102 |
+
headers = {
|
| 103 |
+
"Authorization": f"token {api_key}",
|
| 104 |
+
"Accept": "application/vnd.github.v3+json"
|
| 105 |
+
}
|
| 106 |
+
params = {
|
| 107 |
+
"path": file_path, # Specify the file path to get commits for this specific file
|
| 108 |
+
"per_page": 1, # We only need the latest commit, so limit the result to 1
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
async with session.get(url, headers=headers, params=params) as response:
|
| 112 |
+
if response.status == 200:
|
| 113 |
+
commits = await response.json()
|
| 114 |
+
if commits:
|
| 115 |
+
return commits[0]['commit']['committer']['date'] # Last commit date
|
| 116 |
+
else:
|
| 117 |
+
return "Unknown" # If no commits found, return "Unknown"
|
| 118 |
+
else:
|
| 119 |
+
print(f"Error fetching commit info for {repo_full_name}/{file_path}: {response.status}")
|
| 120 |
+
return "Unknown"
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
async def fetch_file_content(url: str) -> str:
|
| 124 |
+
"""
|
| 125 |
+
Fetch the content of the given file URL using aiohttp.
|
| 126 |
+
"""
|
| 127 |
+
async with aiohttp.ClientSession() as session:
|
| 128 |
+
async with session.get(url) as response:
|
| 129 |
+
if response.status == 200:
|
| 130 |
+
return await response.text()
|
| 131 |
+
else:
|
| 132 |
+
print(f"Failed to fetch file: {response.status}")
|
| 133 |
+
return ""
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
async def scrape_md_files(org_name: str, api_key: str, repo_limit=None) -> list[dict]:
|
| 137 |
+
"""
|
| 138 |
+
Main function to scrape .md files from all repositories in the organization.
|
| 139 |
+
Returns a list of dictionaries containing the repo_full_name, path, self_url,
|
| 140 |
+
html_url, git_url, download_url and last_modified.
|
| 141 |
+
"""
|
| 142 |
+
# Use this if you are having error with ssl
|
| 143 |
+
# async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl.create_default_context(cafile=certifi.where()))) as session:
|
| 144 |
+
async with aiohttp.ClientSession() as session:
|
| 145 |
+
repos = await fetch_repositories(session, org_name, api_key, repo_limit)
|
| 146 |
+
|
| 147 |
+
# Create async tasks for each repo to fetch .md files concurrently
|
| 148 |
+
tasks = [fetch_md_files(session, repo['full_name'], api_key) for repo in repos]
|
| 149 |
+
all_md_files = await asyncio.gather(*tasks)
|
| 150 |
+
|
| 151 |
+
# Flatten the list of lists into a single list
|
| 152 |
+
return [md_file for repo_files in all_md_files for md_file in repo_files]
|
utils/url_scraper.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import aiohttp
|
| 2 |
+
import asyncio
|
| 3 |
+
from urllib.parse import urljoin, urlparse
|
| 4 |
+
import re
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
from bs4 import BeautifulSoup
|
| 7 |
+
from requests.adapters import HTTPAdapter
|
| 8 |
+
from urllib3.util.retry import Retry
|
| 9 |
+
import requests
|
| 10 |
+
|
| 11 |
+
DATE_FORMATS = [
|
| 12 |
+
'%Y-%m-%d', # Format: 2024-09-20
|
| 13 |
+
'%Y/%m/%d', # Format: 2024/09/20
|
| 14 |
+
'%m-%d-%Y', # Format: 09-20-2024
|
| 15 |
+
'%m/%d/%Y', # Format: 09/20/2024
|
| 16 |
+
'%d-%m-%Y', # Format: 20-09-2024
|
| 17 |
+
'%d/%m/%Y', # Format: 20/09/2024
|
| 18 |
+
'%d %B %Y', # Format: 20 September 2024
|
| 19 |
+
'%B %d, %Y', # Format: September 20, 2024
|
| 20 |
+
'%d %b %Y', # Format: 20 Sep 2024 (short month)
|
| 21 |
+
'%b %d, %Y', # Format: Sep 20, 2024 (short month)
|
| 22 |
+
'%d-%b-%Y', # Format: 20-Sep-2024 (short month)
|
| 23 |
+
'%b-%d-%Y', # Format: Sep-20-2024 (short month)
|
| 24 |
+
'%A, %d %B %Y', # Format: Friday, 20 September 2024 (with weekday)
|
| 25 |
+
'%A, %B %d, %Y', # Format: Friday, September 20, 2024 (with weekday)
|
| 26 |
+
'%Y.%m.%d', # Format: 2024.09.20
|
| 27 |
+
'%d.%m.%Y', # Format: 20.09.2024 (European format)
|
| 28 |
+
'%Y %b %d', # Format: 2024 Sep 20 (short month)
|
| 29 |
+
'%Y %B %d', # Format: 2024 September 20
|
| 30 |
+
'%d %B, %Y', # Format: 20 September, 2024 (comma after day)
|
| 31 |
+
'%B %d %Y' # Format: September 20 2024 (no comma)
|
| 32 |
+
]
|
| 33 |
+
|
| 34 |
+
# Regular expression patterns for various date formats
|
| 35 |
+
DATE_PATTERNS = [
|
| 36 |
+
r'\b\d{4}-\d{2}-\d{2}\b', # Matches YYYY-MM-DD (e.g., 2024-09-20)
|
| 37 |
+
r'\b\d{4}/\d{2}/\d{2}\b', # Matches YYYY/MM/DD (e.g., 2024/09/20)
|
| 38 |
+
r'\b\d{2}-\d{2}-\d{4}\b', # Matches MM-DD-YYYY (e.g., 09-20-2024)
|
| 39 |
+
r'\b\d{2}/\d{2}/\d{4}\b', # Matches MM/DD/YYYY (e.g., 09/20/2024)
|
| 40 |
+
r'\b\d{2}-\d{2}-\d{4}\b', # Matches DD-MM-YYYY (e.g., 20-09-2024)
|
| 41 |
+
r'\b\d{2}/\d{2}/\d{4}\b', # Matches DD/MM/YYYY (e.g., 20/09/2024)
|
| 42 |
+
r'\b\d{1,2}\s+[A-Za-z]+\s+\d{4}\b', # Matches 20 September 2024
|
| 43 |
+
r'\b[A-Za-z]+\s+\d{1,2},\s+\d{4}\b', # Matches September 20, 2024
|
| 44 |
+
r'\b\d{1,2}\s+[A-Za-z]{3}\s+\d{4}\b', # Matches 20 Sep 2024 (short month)
|
| 45 |
+
r'\b[A-Za-z]{3}\s+\d{1,2},\s+\d{4}\b', # Matches Sep 20, 2024 (short month)
|
| 46 |
+
r'\b\d{1,2}-[A-Za-z]{3}-\d{4}\b', # Matches 20-Sep-2024 (short month)
|
| 47 |
+
r'\b[A-Za-z]{3}-\d{1,2}-\d{4}\b', # Matches Sep-20-2024 (short month)
|
| 48 |
+
r'\b[A-Za-z]+,\s+\d{1,2}\s+[A-Za-z]+\s+\d{4}\b', # Matches Friday, 20 September 2024 (with weekday)
|
| 49 |
+
r'\b[A-Za-z]+,\s+[A-Za-z]+\s+\d{1,2},\s+\d{4}\b', # Matches Friday, September 20, 2024 (with weekday)
|
| 50 |
+
r'\b\d{4}\.\d{2}\.\d{2}\b', # Matches YYYY.MM.DD (e.g., 2024.09.20)
|
| 51 |
+
r'\b\d{2}\.\d{2}\.\d{4}\b', # Matches DD.MM.YYYY (e.g., 20.09.2024)
|
| 52 |
+
r'\b\d{4}\s+[A-Za-z]{3}\s+\d{1,2}\b', # Matches 2024 Sep 20
|
| 53 |
+
r'\b\d{4}\s+[A-Za-z]+\s+\d{1,2}\b', # Matches 2024 September 20
|
| 54 |
+
r'\b\d{1,2}\s+[A-Za-z]+,\s+\d{4}\b', # Matches 20 September, 2024 (comma after day)
|
| 55 |
+
r'\b[A-Za-z]+\s+\d{1,2}\s+\d{4}\b' # Matches September 20 2024 (no comma)
|
| 56 |
+
]
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# Asynchronous URL fetching with retry logic
|
| 60 |
+
async def fetch_urls(url_to_fetch: str, session: aiohttp.ClientSession, retries=3) -> set[str]:
|
| 61 |
+
attempt = 0
|
| 62 |
+
while attempt < retries:
|
| 63 |
+
try:
|
| 64 |
+
async with session.get(url_to_fetch, timeout=15) as response:
|
| 65 |
+
if response.status != 200:
|
| 66 |
+
# print(f"Couldn't fetch {url} - status {response.status}")
|
| 67 |
+
return set() # Return an empty set if the page doesn't load
|
| 68 |
+
text_content = await response.text()
|
| 69 |
+
soup = BeautifulSoup(text_content, "html.parser")
|
| 70 |
+
urls = set(
|
| 71 |
+
urljoin(url_to_fetch, link['href'])
|
| 72 |
+
for link in soup.find_all('a', href=True)
|
| 73 |
+
if urlparse(urljoin(url_to_fetch, link['href'])).scheme in ('http', 'https')
|
| 74 |
+
)
|
| 75 |
+
# if attempt > 0: print(f"Successfully fetched URL: {url} on attempt {attempt + 1}")
|
| 76 |
+
return urls
|
| 77 |
+
except (aiohttp.ClientError, asyncio.TimeoutError):
|
| 78 |
+
attempt += 1
|
| 79 |
+
if attempt < retries:
|
| 80 |
+
await asyncio.sleep(2 ** attempt) # Exponential backoff
|
| 81 |
+
except Exception as e:
|
| 82 |
+
print(e)
|
| 83 |
+
return set()
|
| 84 |
+
|
| 85 |
+
print(f"Failed to fetch URL {url_to_fetch} after {retries} attempts.")
|
| 86 |
+
return set()
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
# Check if URL should be allowed
|
| 90 |
+
def is_url_allowed(url_to_check: str, special_list: dict[str, list[str]] | None) -> bool:
|
| 91 |
+
if special_list is None: return True
|
| 92 |
+
|
| 93 |
+
# Return False if url is not in white_list
|
| 94 |
+
if not any(url_to_check.startswith(i) for i in special_list["white_list"]): return False
|
| 95 |
+
|
| 96 |
+
# Return False if url is in black_list
|
| 97 |
+
if any(url_to_check.startswith(i) for i in special_list["black_list"]): return False
|
| 98 |
+
return True
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# Asynchronous scraping with blacklist/whitelist and depth handling
|
| 102 |
+
async def scrape_urls(current_url: str, session: aiohttp.ClientSession, max_depth: int, current_depth=0, visited=None,
|
| 103 |
+
list_data=None) -> list[str]:
|
| 104 |
+
if visited is None:
|
| 105 |
+
visited = []
|
| 106 |
+
|
| 107 |
+
if current_depth > max_depth:
|
| 108 |
+
return visited # Stop recursion if depth limit exceeded
|
| 109 |
+
|
| 110 |
+
visited.append(current_url) # Store URL with its depth
|
| 111 |
+
# Extract URLs from the current page
|
| 112 |
+
urls = await fetch_urls(current_url, session)
|
| 113 |
+
|
| 114 |
+
tasks = []
|
| 115 |
+
for new_url in urls:
|
| 116 |
+
if new_url not in visited and is_url_allowed(new_url, list_data):
|
| 117 |
+
# Continue scraping at the next depth level if within depth limit
|
| 118 |
+
if current_depth < max_depth:
|
| 119 |
+
tasks.append(scrape_urls(new_url, session, max_depth, current_depth + 1, visited, list_data))
|
| 120 |
+
|
| 121 |
+
# Await all the tasks concurrently
|
| 122 |
+
await asyncio.gather(*tasks)
|
| 123 |
+
return visited
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
# Entry point for asynchronous scraping
|
| 127 |
+
async def start_scraping(entry_url: str, depth: int, list_data=None) -> list[str]:
|
| 128 |
+
"""
|
| 129 |
+
Returns list of scraped urls
|
| 130 |
+
list_data - dict{'white_list': [str], 'black_list': [str]}
|
| 131 |
+
"""
|
| 132 |
+
async with aiohttp.ClientSession() as session:
|
| 133 |
+
found_urls = await scrape_urls(entry_url, session, depth, list_data=list_data)
|
| 134 |
+
return found_urls
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def find_all_dates_with_regex(text, date_patterns):
|
| 139 |
+
"""Search for all date matches using regex patterns."""
|
| 140 |
+
dates = []
|
| 141 |
+
for pattern in date_patterns:
|
| 142 |
+
matches = re.findall(pattern, text) # Find all matches for the pattern
|
| 143 |
+
dates.extend(matches)
|
| 144 |
+
return dates
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def parse_date(date_str, date_formats):
|
| 148 |
+
for date_format in date_formats:
|
| 149 |
+
try:
|
| 150 |
+
return datetime.strptime(date_str, date_format)
|
| 151 |
+
except ValueError:
|
| 152 |
+
pass
|
| 153 |
+
return None
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def get_latest_date(dates, date_formats):
|
| 157 |
+
"""Return the latest valid date from a list of date strings."""
|
| 158 |
+
parsed_dates = [parse_date(date, date_formats) for date in dates]
|
| 159 |
+
# Filter out any dates that couldn't be parsed and return the latest one
|
| 160 |
+
valid_dates = [d for d in parsed_dates if d is not None]
|
| 161 |
+
return max(valid_dates).strftime('%Y-%m-%d') if valid_dates else "None"
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def extract_date(my_soup: BeautifulSoup, date_formats, date_patterns) -> str:
|
| 165 |
+
# Extract all text content to search for a date pattern
|
| 166 |
+
text_content = my_soup.get_text(separator=' ', strip=True)
|
| 167 |
+
|
| 168 |
+
# Use regex to find all dates in the page's text content
|
| 169 |
+
dates = find_all_dates_with_regex(text_content, date_patterns)
|
| 170 |
+
|
| 171 |
+
# Get the latest date found
|
| 172 |
+
latest_date = get_latest_date(dates, date_formats)
|
| 173 |
+
return latest_date
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def requests_retry_session(
|
| 177 |
+
retries=3,
|
| 178 |
+
backoff_factor=0.3,
|
| 179 |
+
status_forcelist=(500, 502, 504),
|
| 180 |
+
session=None,
|
| 181 |
+
):
|
| 182 |
+
session = session or requests.Session()
|
| 183 |
+
retry = Retry(
|
| 184 |
+
total=retries,
|
| 185 |
+
read=retries,
|
| 186 |
+
connect=retries,
|
| 187 |
+
backoff_factor=backoff_factor,
|
| 188 |
+
status_forcelist=status_forcelist,
|
| 189 |
+
)
|
| 190 |
+
adapter = HTTPAdapter(max_retries=retry)
|
| 191 |
+
session.mount('http://', adapter)
|
| 192 |
+
session.mount('https://', adapter)
|
| 193 |
+
return session
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def extract_content_and_metadata(url: str, date_formats: list[str], date_patterns: list[str]) -> list[dict] | None:
|
| 197 |
+
"""
|
| 198 |
+
Returns list of dictionaries {content: string, metadata: dict}
|
| 199 |
+
content - cleaned html text split by headlines
|
| 200 |
+
metadata - url, title, headline, date
|
| 201 |
+
"""
|
| 202 |
+
try:
|
| 203 |
+
# Use retry session for robust requests
|
| 204 |
+
session = requests_retry_session()
|
| 205 |
+
response = session.get(url, timeout=10)
|
| 206 |
+
|
| 207 |
+
# Check if request was successful
|
| 208 |
+
response.raise_for_status()
|
| 209 |
+
|
| 210 |
+
soup = BeautifulSoup(response.content, 'html.parser')
|
| 211 |
+
title = soup.title.string if soup.title else "None"
|
| 212 |
+
date = extract_date(soup, date_formats, date_patterns)
|
| 213 |
+
|
| 214 |
+
content_by_headline = {}
|
| 215 |
+
current_header = None
|
| 216 |
+
|
| 217 |
+
# Loop through the elements, keeping track of headlines and paragraphs
|
| 218 |
+
for element in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'ul', 'ol']):
|
| 219 |
+
if element.name.startswith('h'):
|
| 220 |
+
# Headline
|
| 221 |
+
current_header = element.get_text(strip=True)
|
| 222 |
+
current_header = current_header.replace("\n", "")
|
| 223 |
+
current_header = re.sub(r"\s+", " ", current_header)
|
| 224 |
+
|
| 225 |
+
elif element.name in ['p', 'ul', 'ol'] and current_header:
|
| 226 |
+
# Append the text under the last seen headline
|
| 227 |
+
value = content_by_headline[current_header] if current_header in content_by_headline else ""
|
| 228 |
+
new_content = element.get_text(strip=True).replace('\n', ' ')
|
| 229 |
+
content_by_headline[current_header] = f"{value} {new_content}"
|
| 230 |
+
|
| 231 |
+
result_dicts = []
|
| 232 |
+
for headline, content in content_by_headline.items():
|
| 233 |
+
result_dicts.append(
|
| 234 |
+
{"content": content, "metadata": {"url": url, "title": title, "headline": headline, "date": date}})
|
| 235 |
+
|
| 236 |
+
return result_dicts
|
| 237 |
+
|
| 238 |
+
except requests.exceptions.RequestException as e:
|
| 239 |
+
print(f"Error fetching {url}: {e}")
|
| 240 |
+
return None
|