feat: implement core configuration management, Celery Beat scheduling, and Redis-based PDF stream processing workers.

app/config.py

@@ -86,6 +86,16 @@ class Settings(BaseSettings):
                 return [origin.strip() for origin in v.split(",")]
         return v
 
+    @field_validator("redis_url", mode="after")
+    @classmethod
+    def validate_redis_url(cls, v: str) -> str:
+        if v.startswith("rediss://") and "ssl_cert_reqs" not in v:
+            separator = "&" if "?" in v else "?"
+            v = f"{v}{separator}ssl_cert_reqs=CERT_NONE"
+            import os
+            os.environ["REDIS_URL"] = v
+        return v
+
     @property
     def is_production(self) -> bool:
         return self.app_env == "production"

app/main.py

@@ -69,6 +69,11 @@ async def lifespan(app: FastAPI):
     init_retrieval_orchestrator()
     logger.info("All services initialized. Ukweli is ready to serve.")
 
+    # Trigger ingestion to start right away
+    from app.scraper.workers.pdf_processor import run_scraper
+    logger.info("Triggering initial crawler run on startup...")
+    run_scraper.delay()
+
     yield
 
     # --- Shutdown ---

app/scraper/workers/pdf_processor.py

@@ -32,6 +32,13 @@ app.config_from_object({
     'task_track_started': True,
     'task_time_limit': 600,
     'worker_prefetch_multiplier': 1,
+    'beat_schedule': {
+        'run-scraper-daily': {
+            'task': 'app.scraper.workers.pdf_processor.run_scraper',
+            'schedule': 86400.0,  # Run daily (every 24 hours)
+            'args': (None,),
+        },
+    },
 })
 
 logger = logging.getLogger(__name__)

entrypoint.sh

@@ -9,6 +9,14 @@ alembic upgrade head || echo "Database migrations failed, proceeding..."
 echo "Starting Celery worker..."
 celery -A app.scraper.workers.pdf_processor worker --loglevel=info --pool=solo &
 
+# Start Celery Beat scheduler in the background
+echo "Starting Celery Beat..."
+celery -A app.scraper.workers.pdf_processor beat --loglevel=info &
+
+# Start Redis stream consumer in the background (prevents blocking solo worker thread)
+echo "Starting Redis stream consumer..."
+PYTHONPATH=. python -c "from app.scraper.workers.pdf_processor import consume_pdf_stream; consume_pdf_stream()" &
+
 # Start FastAPI application
 echo "Starting FastAPI backend..."
 exec uvicorn app.main:app --host 0.0.0.0 --port 7860

tests/test_oag_pipeline.py

@@ -184,5 +184,24 @@ class TestOAGPipeline(unittest.TestCase):
         self.assertIn("oag_kenya", cmd)
         self.assertIn("max_pages=5", cmd)
 
+    def test_redis_url_validation(self):
+        from app.config import Settings
+        
+        # Test default/standard redis URL is untouched
+        settings_redis = Settings(redis_url="redis://localhost:6379")
+        self.assertEqual(settings_redis.redis_url, "redis://localhost:6379")
+        
+        # Test rediss URL without ssl_cert_reqs is updated
+        settings_rediss = Settings(redis_url="rediss://localhost:6379/0")
+        self.assertEqual(settings_rediss.redis_url, "rediss://localhost:6379/0?ssl_cert_reqs=CERT_NONE")
+        
+        # Test rediss URL with existing query param is updated properly
+        settings_rediss_param = Settings(redis_url="rediss://localhost:6379/0?foo=bar")
+        self.assertEqual(settings_rediss_param.redis_url, "rediss://localhost:6379/0?foo=bar&ssl_cert_reqs=CERT_NONE")
+        
+        # Test rediss URL that already has ssl_cert_reqs is untouched
+        settings_rediss_explicit = Settings(redis_url="rediss://localhost:6379/0?ssl_cert_reqs=CERT_REQUIRED")
+        self.assertEqual(settings_rediss_explicit.redis_url, "rediss://localhost:6379/0?ssl_cert_reqs=CERT_REQUIRED")
+
 if __name__ == '__main__':
     unittest.main()

web/src/components.d.ts

@@ -11,16 +11,32 @@ declare module 'vue' {
     AuthModal: typeof import('./components/AuthModal.vue')['default']
     ChatMessage: typeof import('./components/ChatMessage.vue')['default']
     NAlert: typeof import('naive-ui')['NAlert']
+    NAvatar: typeof import('naive-ui')['NAvatar']
+    NBadge: typeof import('naive-ui')['NBadge']
+    NBlockquote: typeof import('naive-ui')['NBlockquote']
     NButton: typeof import('naive-ui')['NButton']
+    NButtonGroup: typeof import('naive-ui')['NButtonGroup']
+    NCard: typeof import('naive-ui')['NCard']
     NConfigProvider: typeof import('naive-ui')['NConfigProvider']
+    NDescriptions: typeof import('naive-ui')['NDescriptions']
+    NDescriptionsItem: typeof import('naive-ui')['NDescriptionsItem']
     NDialogProvider: typeof import('naive-ui')['NDialogProvider']
+    NDivider: typeof import('naive-ui')['NDivider']
+    NEmpty: typeof import('naive-ui')['NEmpty']
     NIcon: typeof import('naive-ui')['NIcon']
     NInput: typeof import('naive-ui')['NInput']
+    NList: typeof import('naive-ui')['NList']
+    NListItem: typeof import('naive-ui')['NListItem']
     NLoadingBarProvider: typeof import('naive-ui')['NLoadingBarProvider']
     NMessageProvider: typeof import('naive-ui')['NMessageProvider']
     NModal: typeof import('naive-ui')['NModal']
     NNotificationProvider: typeof import('naive-ui')['NNotificationProvider']
     NResult: typeof import('naive-ui')['NResult']
+    NSkeleton: typeof import('naive-ui')['NSkeleton']
+    NStep: typeof import('naive-ui')['NStep']
+    NSteps: typeof import('naive-ui')['NSteps']
+    NTag: typeof import('naive-ui')['NTag']
+    NThing: typeof import('naive-ui')['NThing']
     ReportCard: typeof import('./components/ReportCard.vue')['default']
     ReportListItem: typeof import('./components/ReportListItem.vue')['default']
     RouterLink: typeof import('vue-router')['RouterLink']