Spaces:
Sleeping
Sleeping
Create ingest/generic_public_foia.py
Browse files
ingest/generic_public_foia.py
CHANGED
|
@@ -1,44 +1,14 @@
|
|
| 1 |
import abc
|
| 2 |
-
import asyncio
|
| 3 |
-
import time
|
| 4 |
from typing import List, Dict
|
| 5 |
-
from ingest.robots import allowed
|
| 6 |
-
from ingest.health import HealthStatus
|
| 7 |
-
|
| 8 |
|
| 9 |
class GenericFOIAAdapter(abc.ABC):
|
| 10 |
-
source_name
|
| 11 |
-
base_url
|
| 12 |
-
source_type: str = "stub" # stub | live
|
| 13 |
-
|
| 14 |
-
def __init__(self, rate_limit_seconds: float = 1.0):
|
| 15 |
-
self.rate_limit_seconds = rate_limit_seconds
|
| 16 |
-
self._last_call = 0.0
|
| 17 |
-
self.last_health: HealthStatus | None = None
|
| 18 |
-
|
| 19 |
-
async def _rate_limit(self):
|
| 20 |
-
now = asyncio.get_event_loop().time()
|
| 21 |
-
delta = now - self._last_call
|
| 22 |
-
if delta < self.rate_limit_seconds:
|
| 23 |
-
await asyncio.sleep(self.rate_limit_seconds - delta)
|
| 24 |
-
self._last_call = asyncio.get_event_loop().time()
|
| 25 |
-
|
| 26 |
-
async def _guard(self, url: str) -> bool:
|
| 27 |
-
return await allowed(url)
|
| 28 |
-
|
| 29 |
-
async def _stub_result(self, query: str) -> List[Dict]:
|
| 30 |
-
await self._rate_limit()
|
| 31 |
-
self.last_health = HealthStatus(ok=True, latency_ms=0)
|
| 32 |
-
return [{
|
| 33 |
-
"source": self.source_name,
|
| 34 |
-
"title": f"{self.source_name} FOIA result for '{query}'",
|
| 35 |
-
"url": self.base_url,
|
| 36 |
-
"snippet": "Public FOIA reading room placeholder result (stub).",
|
| 37 |
-
"live": False,
|
| 38 |
-
"extended": False,
|
| 39 |
-
"health": self.last_health.__dict__
|
| 40 |
-
}]
|
| 41 |
|
| 42 |
@abc.abstractmethod
|
| 43 |
async def search(self, query: str) -> List[Dict]:
|
| 44 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import abc
|
|
|
|
|
|
|
| 2 |
from typing import List, Dict
|
|
|
|
|
|
|
|
|
|
| 3 |
|
| 4 |
class GenericFOIAAdapter(abc.ABC):
|
| 5 |
+
source_name = "UNKNOWN"
|
| 6 |
+
base_url = ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
@abc.abstractmethod
|
| 9 |
async def search(self, query: str) -> List[Dict]:
|
| 10 |
+
pass
|
| 11 |
+
|
| 12 |
+
def sync_search(self, query: str) -> List[Dict]:
|
| 13 |
+
import asyncio
|
| 14 |
+
return asyncio.run(self.search(query))
|