GodsDevProject commited on
Commit
7b41160
·
verified ·
1 Parent(s): e857796

Create ingest/generic_public_foia.py

Browse files
Files changed (1) hide show
  1. ingest/generic_public_foia.py +7 -37
ingest/generic_public_foia.py CHANGED
@@ -1,44 +1,14 @@
1
  import abc
2
- import asyncio
3
- import time
4
  from typing import List, Dict
5
- from ingest.robots import allowed
6
- from ingest.health import HealthStatus
7
-
8
 
9
  class GenericFOIAAdapter(abc.ABC):
10
- source_name: str = "UNKNOWN"
11
- base_url: str = ""
12
- source_type: str = "stub" # stub | live
13
-
14
- def __init__(self, rate_limit_seconds: float = 1.0):
15
- self.rate_limit_seconds = rate_limit_seconds
16
- self._last_call = 0.0
17
- self.last_health: HealthStatus | None = None
18
-
19
- async def _rate_limit(self):
20
- now = asyncio.get_event_loop().time()
21
- delta = now - self._last_call
22
- if delta < self.rate_limit_seconds:
23
- await asyncio.sleep(self.rate_limit_seconds - delta)
24
- self._last_call = asyncio.get_event_loop().time()
25
-
26
- async def _guard(self, url: str) -> bool:
27
- return await allowed(url)
28
-
29
- async def _stub_result(self, query: str) -> List[Dict]:
30
- await self._rate_limit()
31
- self.last_health = HealthStatus(ok=True, latency_ms=0)
32
- return [{
33
- "source": self.source_name,
34
- "title": f"{self.source_name} FOIA result for '{query}'",
35
- "url": self.base_url,
36
- "snippet": "Public FOIA reading room placeholder result (stub).",
37
- "live": False,
38
- "extended": False,
39
- "health": self.last_health.__dict__
40
- }]
41
 
42
  @abc.abstractmethod
43
  async def search(self, query: str) -> List[Dict]:
44
- raise NotImplementedError
 
 
 
 
 
1
  import abc
 
 
2
  from typing import List, Dict
 
 
 
3
 
4
  class GenericFOIAAdapter(abc.ABC):
5
+ source_name = "UNKNOWN"
6
+ base_url = ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
 
8
  @abc.abstractmethod
9
  async def search(self, query: str) -> List[Dict]:
10
+ pass
11
+
12
+ def sync_search(self, query: str) -> List[Dict]:
13
+ import asyncio
14
+ return asyncio.run(self.search(query))