Create ingest/generic_public_foia.py

ingest/generic_public_foia.py

@@ -1,7 +1,7 @@
 import abc
 import asyncio
 from typing import List, Dict
-from ingest.utils import can_fetch
+from ingest.utils import robots_allowed
 
 class GenericFOIAAdapter(abc.ABC):
     source_name: str = "UNKNOWN"
@@ -20,7 +20,9 @@ class GenericFOIAAdapter(abc.ABC):
         self._last_call = asyncio.get_event_loop().time()
 
     def robots_allowed(self) -> bool:
-        return can_fetch(self.base_url)
+        if not self.base_url:
+            return False
+        return robots_allowed(self.base_url)
 
     @abc.abstractmethod
     async def search(self, query: str) -> List[Dict]: