Update data handling and API request parameters for aides

.gitignore

@@ -1,4 +1,6 @@
 */__pycache__/*
 __pycache__
 
-.env
+.env
+
+/data/*.json

data/aides_territoires.py

@@ -6,6 +6,9 @@ headersConnexion = {
     'X-AUTH-TOKEN': 'eeb481e42950f1dbfc46dc348e6e32a0c631cc5b94dd7ab874a30c027f9de87c',
 }
 
+perimeter_id = '71054-yvelines'
+organization_type_slugs = 'private-sector'
+
 
 def connexion():
     conn.request("POST", "/api/connexion/", '', headersConnexion)
@@ -43,7 +46,7 @@ def request():
     page = 1
 
     while True:
-        conn.request("GET", f"/api/aids/?page={page}&organization_type_slugs=farmer&perimeter_id=70971-nouvelle-aquitaine", '', headersRequest)
+        conn.request("GET", f"/api/aids/?page={page}&organization_type_slugs={organization_type_slugs}&perimeter_id={perimeter_id}", '', headersRequest)
         res = conn.getresponse().read()
         resData = res.decode("utf-8")
         data = json.loads(resData)
@@ -76,5 +79,5 @@ for aide in aides:
 
 
 # print(subventions)
-with open('aides_territoires.json', 'w', encoding='utf-8') as f:
+with open('data/aides_territoires.json', 'w', encoding='utf-8') as f:
     json.dump(aides, f, ensure_ascii=False, indent=4)

data/get_aides_entreprises.py

@@ -8,10 +8,13 @@ headers = {
     'X-Aidesentreprises-Key': 'waMF2TjO',
 }
 
+profils = "10,14"       # 10: ETI et grande entreprise, 14: Industrie
+territoire = "103"
+
 
 def request(limit=20, offset=0):
     print(f"Requesting {limit} subventions from offset {offset}")
-    conn.request("GET", f"/v1.1/aides?profils=20&territoire=50002&limit={limit}&offset={offset}", payload, headers)
+    conn.request("GET", f"/v1.1/aides?profils={profils}&territoire={territoire}&limit={limit}&offset={offset}", payload, headers)
     res = conn.getresponse()
     data = res.read()
     return json.loads(data.decode("utf-8"))['data']

data/get_les_aides.py

@@ -7,12 +7,15 @@ headers = {
     'X-IDC': 'bcfac1828e5ef1b7cab084379a5f2a871e82ee7c',
 }
 
-requestID = 23994079
-filieres = [ 289, 290 ]
+sirets = [ 88889319500014, 89765311900015, 53476514400025, 89125008600016, 98379850500019 ]
 domaines = [893,883,877,790,793,798,802,805,862,807,810,813,816,820,818]
 
-def request(filiere, domaines):
-    conn.request("GET", f"/aides?ape=A&region=75&domaine={domaines}&filiere={filiere}", payload, headers)
+# region = 11
+# filieres = [ 341 ] # 341: Automobile
+
+def request(domaines, siret = None):
+    # ape=A&region=75
+    conn.request("GET", f"/aides?siret={siret}&domaine={domaines}", payload, headers)
     res = conn.getresponse()
     data = res.read()
     data = json.loads(data.decode("utf-8"))
@@ -25,11 +28,6 @@ def request(filiere, domaines):
     return data['dispositifs']
 
 
-def get_final_type(types):
-
-
-    return None  # Return None if no matching type is found
-
 
 def getAide(aide):
 
@@ -38,16 +36,10 @@ def getAide(aide):
         res = conn.getresponse()
         aide = json.loads(res.read().decode("utf-8"))
 
-        # if 'cci' in aide:
-        #     del aide['cci']
-        # if 'url' in aide:
-        #     del aide['url']
-
-
         aide["metadata"] = {
-            # "type_aide": get_final_type(aide["prets"]),
             "lien": aide['uri'],
-            "Source": f"https://les-aides.fr/"
+            "Source": f"https://les-aides.fr/",
+            "sirets": subventions_sirest[aide['numero']]
         }
 
         return aide
@@ -58,6 +50,7 @@ def getAide(aide):
 
 
 subventions = []
+subventions_sirest = {}
 
 
 # Split domaines into two sections
@@ -66,21 +59,39 @@ domaines_section_1 = "[805,862,807,810,813,816,820,818]"
 
 
 # Function to add dispositifs to subventions list without duplication
-def add_dispositifs(filiere, domaines_section):
-    print(f"Requesting filiere {filiere} and domaines {domaines_section}")
+def is_dispositif_exists(numero):
+    """Check if a dispositif with given numero exists and return it if found."""
+    for subvention in subventions:
+        if isinstance(subvention, dict) and subvention.get("numero") == numero:
+            return subvention
+    return None
+
+def add_dispositifs(domaines_section, siret=None):
+    print(f"Requesting domaines {domaines_section}")
     
-    dispositifs = request(filiere, domaines_section)
+    dispositifs = request(domaines_section, siret)
     for dispositif in dispositifs:
-        if not any(subvention.get("numero") == dispositif["numero"] for subvention in subventions if isinstance(subvention, dict)):
-            subventions.append(dispositif)
+        numero = dispositif.get("numero")
+        existing = is_dispositif_exists(numero)
+        if not existing:
+            subventions.append(dispositif) # Add dispositif to subventions list
+
+        if numero in subventions_sirest:
+            # print(f"Dispositif {numero} already exists for siret {subventions_sirest[numero]}")
+            subventions_sirest[numero].append(siret)
+        else:
+            subventions_sirest[numero] = [siret]
 
 
-# Call request function with different parameters
-for filiere in filieres:
-    add_dispositifs(filiere, domaines_section_1)
-    add_dispositifs(filiere, domaines_section_2)
+for siret in sirets:
+    print(f"Requesting siret {siret}")
+    add_dispositifs(domaines_section_1, siret)
+    add_dispositifs(domaines_section_2, siret)
 
+    # Call request function with different parameters
+    # for filiere in filieres:
 
+print(f"Nb aides : {subventions_sirest}")
 print(f"Nb aides : {len(subventions)}")
 
 for i in range(len(subventions)):
@@ -88,6 +99,5 @@ for i in range(len(subventions)):
     subventions[i] = getAide(subventions[i])
 
 
-# print(subventions)
 with open('data/les_aides.json', 'w', encoding='utf-8') as f:
     json.dump(subventions, f, ensure_ascii=False, indent=4)

data/importToDb.py

@@ -87,8 +87,12 @@ def importAideEntreprise(subvention, source):
             if 'url' in subvention:
                 del subvention['url']
 
+            metadata = subvention.get("metadata", {})
+            if "sirets" in metadata:
+                del metadata["sirets"]
+
             metadata = {
-                **subvention.get("metadata", {}),
+                **metadata,
                 "id_subvention": subvention['numero'],
                 "deadline_date": -1,
                 "id_document": f"aides_{subvention['numero']}"
@@ -130,10 +134,12 @@ def go():
         if file_name.endswith(".json"):
             print(file_name)
             file_path = os.path.join('data', file_name)
-            with open(file_path, "r", encoding="utf-8") as f:
-                data = json.load(f)
+            
+            if file_name == "les_aides.json":
+                with open(file_path, "r", encoding="utf-8") as f:
+                    data = json.load(f)
 
-                loopSubventions(data,source=file_name)
+                    loopSubventions(data,source=file_name)
 
 if __name__ == "__main__":
     go()