Revert "Rework repositories download to avoid using subprocess"

This reverts commit 8d47ff87a17c03ab89c37d106a2e1fc86ca063b0.

main.py

@@ -1,8 +1,7 @@
 from typing import List, Tuple
 from pathlib import Path
+import subprocess
 import os
-import zipfile
-import io
 
 from dotenv import load_dotenv
 from haystack.preview import Pipeline
@@ -19,75 +18,45 @@ from haystack.preview.components.writers import DocumentWriter
 from haystack.preview.components.file_converters import TextFileToDocument
 from haystack.preview.document_stores.memory import MemoryDocumentStore
 import streamlit as st
-import requests
 
 # Load the environment variables, we're going to need it for OpenAI
 load_dotenv()
 
 # This is the list of documentation that we're going to fetch
 DOCUMENTATIONS = [
-    (
-        "DocArray",
-        "https://github.com/docarray/docarray",
-        "/archive/refs/heads/main.zip",
-        "./docs/**/*.md",
-    ),
-    (
-        "Streamlit",
-        "https://github.com/streamlit/docs",
-        "/archive/refs/heads/main.zip",
-        "./content/**/*.md",
-    ),
-    (
-        "Jinja",
-        "https://github.com/pallets/jinja",
-        "/archive/refs/heads/main.zip",
-        "./docs/**/*.rst",
-    ),
-    (
-        "Pandas",
-        "https://github.com/pandas-dev/pandas",
-        "/archive/refs/heads/main.zip",
-        "./docs/source/**/*.rst",
-    ),
+    ("DocArray", "https://github.com/docarray/docarray", "./docs/**/*.md"),
+    ("Streamlit", "https://github.com/streamlit/docs", "./content/**/*.md"),
+    ("Jinja", "https://github.com/pallets/jinja", "./docs/**/*.rst"),
+    ("Pandas", "https://github.com/pandas-dev/pandas", "./docs/source/**/*.rst"),
     (
         "Elasticsearch",
         "https://github.com/elastic/elasticsearch",
-        "/archive/refs/heads/main.zip",
         "./docs/**/*.asciidoc",
     ),
-    (
-        "NumPy",
-        "https://github.com/numpy/numpy",
-        "/archive/refs/heads/main.zip",
-        "./doc/**/*.rst",
-    ),
+    ("NumPy", "https://github.com/numpy/numpy", "./doc/**/*.rst"),
 ]
 
 
 @st.cache_data(show_spinner=False)
 def fetch(documentations: List[Tuple[str, str, str]]):
     files = []
-    docs_path = Path(__file__).parent / "downloaded_docs"
-    for name, url, zip_path, pattern in documentations:
+    for name, url, pattern in documentations:
         st.write(f"Fetching {name} repository")
-        # All projects use `main` as the default branch
-        branch = "main"
-        # The name of the folder depends on the name of the repository
-        # on GitHub plus the branch zip we're downloading
-        repo_folder = docs_path / (url.split("/")[-1] + f"-{branch}")
-        if not repo_folder.exists():
-            res = requests.get(f"{url}{zip_path}", stream=True)
-            zip = zipfile.ZipFile(io.BytesIO(res.content))
-            # The zip file contains a folder with the name of the repository
-            # so we extract directly into the docs folder
-            zip.extractall(docs_path)
-
-        for p in repo_folder.glob(pattern):
+        repo = Path(__file__).parent / "downloaded_docs" / name
+        if not repo.exists():
+            subprocess.run(["git", "clone", "--depth", "1", url, str(repo)], check=True)
+        res = subprocess.run(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+            check=True,
+            capture_output=True,
+            encoding="utf-8",
+        )
+        branch = res.stdout.strip()
+        for p in repo.glob(pattern):
             data = {
                 "path": p,
                 "metadata": {
-                    "url_source": f"{url}/tree/{branch}/{p.relative_to(repo_folder)}",
+                    "url_source": f"{url}/tree/{branch}/{p.relative_to(repo)}",
                     "suffix": p.suffix,
                 },
             }

requirements.txt

@@ -4,4 +4,3 @@ langdetect
 streamlit==1.27.2
 python-dotenv
 watchdog
-requests