HFShishir commited on
Commit
ef58e1a
·
verified ·
1 Parent(s): 56f56de

Deploy BytePulse AI Backend v1.0

Browse files
README.md ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: BytePulse AI Backend
3
+ emoji: ⚡
4
+ colorFrom: blue
5
+ colorTo: indigo
6
+ sdk: gradio
7
+ sdk_version: 6.20.0
8
+ app_file: app.py
9
+ pinned: false
10
+ ---
11
+
12
+ # 🚀 BytePulse AI - Daily Tech Digest Backend
13
+
14
+ Enterprise AI Tech Digest Aggregator powered by **FastAPI**, **Gemini 2.5 Flash**, **ChromaDB Vector Store**, and **AES-128 Fernet Encryption**.
15
+
16
+ ## 🛡️ Environment Secrets
17
+ Set in **Space Settings → Repository Secrets**:
18
+
19
+ - `GOOGLE_API_KEY`: Gemini 2.5 Flash API Key
20
+ - `API_SECRET_KEY`: `bytepulse_secure_app_key_2026`
21
+ - `ENCRYPTION_KEY`: 32-byte Fernet key
22
+
23
+ ## 🌐 API Endpoints
24
+ - `GET /health` → `{"status": "healthy"}`
25
+ - `GET /docs` → Swagger UI
26
+ - `POST /api/v1/digest/trigger` → Trigger aggregation
app.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from main import app as fastapi_app
3
+
4
+ # Minimal Gradio landing page
5
+ with gr.Blocks(title="BytePulse AI Backend", theme=gr.themes.Soft()) as demo:
6
+ gr.Markdown(
7
+ """
8
+ # ⚡ BytePulse AI — Daily Tech Digest API
9
+
10
+ > Backend is **live and healthy!** 🚀
11
+
12
+ | Endpoint | Description |
13
+ |----------|-------------|
14
+ | `/health` | Health check |
15
+ | `/docs` | Swagger API Docs |
16
+ | `/api/v1` | API Base |
17
+ """
18
+ )
19
+
20
+ # Mount FastAPI app inside Gradio (Gradio 6 compatible)
21
+ app = gr.mount_gradio_app(fastapi_app, demo, path="/gradio")
app/api/v1/endpoints/digests.py ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, Depends, HTTPException, Query, Request
2
+ from slowapi import Limiter
3
+ from slowapi.util import get_remote_address
4
+ from sqlalchemy.orm import Session
5
+ from typing import List, Optional
6
+ from pydantic import BaseModel
7
+ from datetime import datetime
8
+
9
+ from app.db.database import get_db
10
+ from app.db.models import DailyDigest, DigestItem
11
+
12
+ router = APIRouter()
13
+ limiter = Limiter(key_func=get_remote_address)
14
+
15
+ # Pydantic Schemas
16
+ class DigestItemSchema(BaseModel):
17
+ id: int
18
+ daily_digest_id: Optional[int]
19
+ title: str
20
+ url: str
21
+ source: str
22
+ category: str
23
+ summary: str
24
+ key_takeaway: Optional[str]
25
+ importance_score: float
26
+ created_at: datetime
27
+
28
+ class Config:
29
+ from_attributes = True
30
+
31
+ class DailyDigestSchema(BaseModel):
32
+ id: int
33
+ date_str: str
34
+ title: str
35
+ overall_summary: str
36
+ top_highlights: Optional[str]
37
+ created_at: datetime
38
+ items: List[DigestItemSchema] = []
39
+
40
+ class Config:
41
+ from_attributes = True
42
+
43
+ from app.services.encryption import decrypt_text
44
+
45
+ def format_decrypted_digest(digest: DailyDigest):
46
+ if not digest:
47
+ return None
48
+ digest.overall_summary = decrypt_text(digest.overall_summary)
49
+ if digest.items:
50
+ for it in digest.items:
51
+ it.summary = decrypt_text(it.summary)
52
+ it.key_takeaway = decrypt_text(it.key_takeaway)
53
+ return digest
54
+
55
+ @router.get("/latest", response_model=DailyDigestSchema)
56
+ @limiter.limit("30/minute")
57
+ def get_latest_digest(request: Request, db: Session = Depends(get_db)):
58
+ """
59
+ Returns the most recent daily tech digest with all summarized news items.
60
+ """
61
+ digest = db.query(DailyDigest).order_by(DailyDigest.created_at.desc()).first()
62
+ if not digest:
63
+ raise HTTPException(status_code=404, detail="No digests found yet. Run trigger endpoint to generate one.")
64
+ return format_decrypted_digest(digest)
65
+
66
+ @router.get("", response_model=List[DailyDigestSchema])
67
+ @limiter.limit("30/minute")
68
+ def list_digests(
69
+ request: Request,
70
+ skip: int = Query(0, ge=0),
71
+ limit: int = Query(10, ge=1, le=50),
72
+ db: Session = Depends(get_db)
73
+ ):
74
+ """
75
+ Get a paginated list of past daily digests.
76
+ """
77
+ digests = db.query(DailyDigest).order_by(DailyDigest.created_at.desc()).offset(skip).limit(limit).all()
78
+ return [format_decrypted_digest(d) for d in digests]
79
+
80
+ @router.get("/{digest_id}", response_model=DailyDigestSchema)
81
+ @limiter.limit("30/minute")
82
+ def get_digest_by_id(request: Request, digest_id: int, db: Session = Depends(get_db)):
83
+ """
84
+ Get a specific daily digest by ID.
85
+ """
86
+ digest = db.query(DailyDigest).filter(DailyDigest.id == digest_id).first()
87
+ if not digest:
88
+ raise HTTPException(status_code=404, detail=f"Digest with ID {digest_id} not found")
89
+ return format_decrypted_digest(digest)
90
+
91
+ @router.get("/items/categories", response_model=List[str])
92
+ @limiter.limit("60/minute")
93
+ def list_categories(request: Request, db: Session = Depends(get_db)):
94
+ """
95
+ Get distinct categories of tech updates available in the database.
96
+ """
97
+ categories = db.query(DigestItem.category).distinct().all()
98
+ return [c[0] for c in categories if c[0]]
99
+
100
+ @router.get("/items/{item_id}/full")
101
+ @limiter.limit("15/minute")
102
+ async def get_full_article_detail(request: Request, item_id: int, db: Session = Depends(get_db)):
103
+ """
104
+ Returns full details for a tech news update, including all text chunks from
105
+ RecursiveCharacterTextSplitter and an in-depth AI Technical Analysis from Gemini.
106
+ """
107
+ from app.services.summarizer import generate_deep_ai_analysis
108
+ from app.vectorstore.chroma_store import vector_store
109
+
110
+ item = db.query(DigestItem).filter(DigestItem.id == item_id).first()
111
+ if not item:
112
+ raise HTTPException(status_code=404, detail=f"Digest item with ID {item_id} not found")
113
+
114
+ # Fetch text chunks from ChunkTextStore
115
+ item_chunks = [rec["chunk_text"] for rec in vector_store.chunk_records if rec.get("item_id") == item_id]
116
+ if not item_chunks:
117
+ item_chunks = [item.summary]
118
+
119
+ # Generate deep AI analysis
120
+ deep_analysis = await generate_deep_ai_analysis(
121
+ title=item.title,
122
+ source=item.source,
123
+ summary=item.summary,
124
+ text_chunks=item_chunks
125
+ )
126
+
127
+ return {
128
+ "item": {
129
+ "id": item.id,
130
+ "daily_digest_id": item.daily_digest_id,
131
+ "title": item.title,
132
+ "url": item.url,
133
+ "source": item.source,
134
+ "category": item.category,
135
+ "summary": item.summary,
136
+ "key_takeaway": item.key_takeaway,
137
+ "importance_score": item.importance_score,
138
+ "created_at": item.created_at
139
+ },
140
+ "text_chunks": item_chunks,
141
+ "deep_ai_analysis": deep_analysis
142
+ }
143
+
144
+ class SocialPostRequest(BaseModel):
145
+ title: str
146
+ source: str = "Tech Update"
147
+ summary: str
148
+ key_takeaway: Optional[str] = ""
149
+
150
+ @router.post("/generate-social-post")
151
+ @limiter.limit("10/minute")
152
+ async def generate_social_post_endpoint(request: Request, req: SocialPostRequest):
153
+ """
154
+ Generates ready-to-share social media post ideas (LinkedIn, Twitter/X thread, Newsletter, Reddit)
155
+ using Gemini 2.5 Flash LLM.
156
+ """
157
+ from app.services.summarizer import generate_social_media_posts
158
+ posts = await generate_social_media_posts(
159
+ title=req.title,
160
+ source=req.source,
161
+ summary=req.summary,
162
+ key_takeaway=req.key_takeaway or ""
163
+ )
164
+ return {
165
+ "status": "success",
166
+ "posts": posts
167
+ }
168
+
169
+
app/api/v1/endpoints/notifications.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, Depends, HTTPException
2
+ from sqlalchemy.orm import Session
3
+ from pydantic import BaseModel
4
+ from datetime import datetime
5
+
6
+ from app.db.database import get_db
7
+ from app.db.models import DeviceToken
8
+
9
+ router = APIRouter()
10
+
11
+ class DeviceTokenRequest(BaseModel):
12
+ token: str
13
+ device_type: str = "android" # android | ios
14
+
15
+ @router.post("/register")
16
+ def register_device_token(req: DeviceTokenRequest, db: Session = Depends(get_db)):
17
+ """
18
+ Registers or updates an FCM token for mobile push notifications.
19
+ """
20
+ if not req.token:
21
+ raise HTTPException(status_code=400, detail="Token cannot be empty")
22
+
23
+ existing = db.query(DeviceToken).filter(DeviceToken.token == req.token).first()
24
+ if existing:
25
+ existing.device_type = req.device_type
26
+ existing.updated_at = datetime.utcnow()
27
+ db.commit()
28
+ return {"status": "updated", "message": "Device token updated successfully"}
29
+
30
+ new_token = DeviceToken(token=req.token, device_type=req.device_type)
31
+ db.add(new_token)
32
+ db.commit()
33
+ return {"status": "registered", "message": "Device token registered successfully"}
app/api/v1/endpoints/search.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, Query
2
+ from typing import List, Dict, Any
3
+ from app.vectorstore.chroma_store import vector_store
4
+ from app.scrapers.duckduckgo import search_duckduckgo_tech_news
5
+
6
+ router = APIRouter()
7
+
8
+ @router.get("")
9
+ def search_text_chunks(
10
+ q: str = Query(..., description="Query string for text search across tech updates", min_length=2),
11
+ top_k: int = Query(5, ge=1, le=20)
12
+ ):
13
+ """
14
+ Search tech updates across stored text chunks generated by LangChain TextSplitter.
15
+ """
16
+ results = vector_store.search_similar(query=q, top_k=top_k)
17
+ return {
18
+ "query": q,
19
+ "total_results": len(results),
20
+ "results": results
21
+ }
22
+
23
+ @router.get("/live")
24
+ async def live_duckduckgo_search(
25
+ q: str = Query(..., description="Real-time web search query via DuckDuckGo", min_length=2),
26
+ limit: int = Query(5, ge=1, le=10)
27
+ ):
28
+ """
29
+ Performs real-time web search using DuckDuckGo and returns live search results.
30
+ """
31
+ results = await search_duckduckgo_tech_news(query=q, limit=limit)
32
+ return {
33
+ "query": q,
34
+ "source": "DuckDuckGo Real-Time Search",
35
+ "total_results": len(results),
36
+ "results": results
37
+ }
38
+
app/api/v1/endpoints/trigger.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter, BackgroundTasks
2
+ from app.scheduler.cron import run_daily_tech_digest_pipeline
3
+
4
+ router = APIRouter()
5
+
6
+ @router.post("")
7
+ async def trigger_manual_scrape(background_tasks: BackgroundTasks):
8
+ """
9
+ Manually triggers the tech update scraping, LangChain AI summarization, ChromaDB indexing, and FCM push notification pipeline.
10
+ Runs asynchronously in background tasks.
11
+ """
12
+ background_tasks.add_task(run_daily_tech_digest_pipeline)
13
+ return {
14
+ "status": "triggered",
15
+ "message": "Daily tech update pipeline started in background. Check /api/v1/digests/latest shortly."
16
+ }
17
+
18
+ @router.post("/sync")
19
+ async def trigger_manual_scrape_sync():
20
+ """
21
+ Synchronously triggers the pipeline and returns the result immediately.
22
+ """
23
+ result = await run_daily_tech_digest_pipeline()
24
+ return result
app/api/v1/router.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import APIRouter
2
+ from app.api.v1.endpoints import digests, search, notifications, trigger
3
+
4
+ api_router = APIRouter()
5
+
6
+ api_router.include_router(digests.router, prefix="/digests", tags=["Tech Digests"])
7
+ api_router.include_router(search.router, prefix="/search", tags=["Semantic Search"])
8
+ api_router.include_router(notifications.router, prefix="/notifications", tags=["Push Notifications"])
9
+ api_router.include_router(trigger.router, prefix="/trigger-scrape", tags=["Pipeline Trigger"])
app/config.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pydantic_settings import BaseSettings, SettingsConfigDict
2
+ from typing import Optional
3
+
4
+ class Settings(BaseSettings):
5
+ APP_NAME: str = "BytePulse AI - Daily Tech Briefing"
6
+ VERSION: str = "1.0.0"
7
+ DEBUG: bool = True
8
+
9
+ # Environment Secrets (Must be set in Hugging Face Space Settings or .env)
10
+ API_SECRET_KEY: str = "bytepulse_secure_app_key_2026"
11
+ ENCRYPTION_KEY: Optional[str] = None
12
+
13
+ HOST: str = "0.0.0.0"
14
+ PORT: int = 8000
15
+
16
+ DATABASE_URL: str = "sqlite:///./tech_updates.db"
17
+ CHROMA_PERSIST_DIRECTORY: str = "./chroma_db"
18
+
19
+ GOOGLE_API_KEY: Optional[str] = None
20
+ OPENAI_API_KEY: Optional[str] = None
21
+ PREFERRED_LLM_PROVIDER: str = "gemini"
22
+
23
+ FIREBASE_CREDENTIALS_FILE: Optional[str] = "./firebase-credentials.json"
24
+
25
+ DAILY_CRON_HOUR: int = 8
26
+ DAILY_CRON_MINUTE: int = 0
27
+
28
+ model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
29
+
30
+ settings = Settings()
app/db/database.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from sqlalchemy import create_engine
2
+ from sqlalchemy.orm import sessionmaker, declarative_base
3
+ from app.config import settings
4
+
5
+ # Check if SQLite and configure connect_args
6
+ connect_args = {"check_same_thread": False} if settings.DATABASE_URL.startswith("sqlite") else {}
7
+
8
+ engine = create_engine(
9
+ settings.DATABASE_URL,
10
+ connect_args=connect_args,
11
+ echo=settings.DEBUG
12
+ )
13
+
14
+ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
15
+
16
+ Base = declarative_base()
17
+
18
+ def get_db():
19
+ db = SessionLocal()
20
+ try:
21
+ yield db
22
+ finally:
23
+ db.close()
app/db/models.py ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey, Float
2
+ from sqlalchemy.orm import relationship
3
+ from datetime import datetime
4
+ from app.db.database import Base
5
+
6
+ class DailyDigest(Base):
7
+ __tablename__ = "daily_digests"
8
+
9
+ id = Column(Integer, primary_key=True, index=True)
10
+ date_str = Column(String(50), unique=True, index=True) # e.g., "2026-07-22"
11
+ title = Column(String(255), nullable=False)
12
+ overall_summary = Column(Text, nullable=False)
13
+ top_highlights = Column(Text, nullable=True) # JSON or markdown string of top 3 highlights
14
+ created_at = Column(DateTime, default=datetime.utcnow)
15
+
16
+ items = relationship("DigestItem", back_populates="daily_digest", cascade="all, delete-orphan")
17
+
18
+ class DigestItem(Base):
19
+ __tablename__ = "digest_items"
20
+
21
+ id = Column(Integer, primary_key=True, index=True)
22
+ daily_digest_id = Column(Integer, ForeignKey("daily_digests.id"), nullable=True)
23
+ title = Column(String(500), nullable=False)
24
+ url = Column(String(1000), nullable=False)
25
+ source = Column(String(100), nullable=False) # HackerNews, TechCrunch, GitHub Trends
26
+ category = Column(String(100), default="Tech") # AI, Web, Security, Mobile, Open Source, etc.
27
+ summary = Column(Text, nullable=False)
28
+ key_takeaway = Column(Text, nullable=True)
29
+ importance_score = Column(Float, default=5.0)
30
+ published_at = Column(DateTime, default=datetime.utcnow)
31
+ created_at = Column(DateTime, default=datetime.utcnow)
32
+
33
+ daily_digest = relationship("DailyDigest", back_populates="items")
34
+
35
+ class DeviceToken(Base):
36
+ __tablename__ = "device_tokens"
37
+
38
+ id = Column(Integer, primary_key=True, index=True)
39
+ token = Column(String(500), unique=True, index=True, nullable=False)
40
+ device_type = Column(String(50), default="android") # android | ios
41
+ created_at = Column(DateTime, default=datetime.utcnow)
42
+ updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
app/scheduler/cron.py ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import asyncio
3
+ from datetime import datetime
4
+ from apscheduler.schedulers.asyncio import AsyncIOScheduler
5
+ from sqlalchemy.orm import Session
6
+
7
+ from app.config import settings
8
+ from app.db.database import SessionLocal, engine, Base
9
+ from app.db.models import DailyDigest, DigestItem, DeviceToken
10
+ from app.scrapers.aggregator import aggregate_all_sources
11
+ from app.services.summarizer import process_and_summarize_updates
12
+ from app.vectorstore.chroma_store import vector_store
13
+ from app.services.notifications import send_daily_digest_notification
14
+
15
+ logger = logging.getLogger(__name__)
16
+ scheduler = AsyncIOScheduler()
17
+
18
+ async def run_daily_tech_digest_pipeline():
19
+ """
20
+ Main background job:
21
+ 1. Scrapes latest tech news
22
+ 2. Summarizes with LangChain
23
+ 3. Saves to Relational DB & Vector DB
24
+ 4. Sends FCM Push Notification
25
+ """
26
+ today_str = datetime.utcnow().strftime("%Y-%m-%d")
27
+ logger.info(f"--- Starting Daily Tech Digest Pipeline for {today_str} ---")
28
+
29
+ db: Session = SessionLocal()
30
+ try:
31
+ # Step 1: Check if digest already exists for today
32
+ existing = db.query(DailyDigest).filter(DailyDigest.date_str == today_str).first()
33
+ if existing:
34
+ logger.info(f"Digest for {today_str} already exists (ID: {existing.id}). Overwriting with fresh updates.")
35
+ db.delete(existing)
36
+ db.commit()
37
+
38
+ # Step 2: Fetch raw items from scrapers
39
+ raw_items = await aggregate_all_sources(limit_per_source=5)
40
+ if not raw_items:
41
+ logger.warning("No news items retrieved from scrapers. Aborting pipeline.")
42
+ return {"status": "no_data"}
43
+
44
+ # Step 3: Summarize via LangChain
45
+ digest_meta, processed_items = await process_and_summarize_updates(raw_items)
46
+
47
+ from app.services.encryption import encrypt_text
48
+
49
+ # Step 4: Save Encrypted Data to Database
50
+ new_digest = DailyDigest(
51
+ date_str=today_str,
52
+ title=digest_meta["title"],
53
+ overall_summary=encrypt_text(digest_meta["overall_summary"]),
54
+ top_highlights=digest_meta["top_highlights"]
55
+ )
56
+ db.add(new_digest)
57
+ db.flush() # get new_digest.id
58
+
59
+ db_items = []
60
+ for item in processed_items:
61
+ db_item = DigestItem(
62
+ daily_digest_id=new_digest.id,
63
+ title=item["title"],
64
+ url=item["url"],
65
+ source=item["source"],
66
+ category=item["category"],
67
+ summary=encrypt_text(item["summary"]),
68
+ key_takeaway=encrypt_text(item["key_takeaway"]),
69
+ importance_score=item["importance_score"]
70
+ )
71
+ db.add(db_item)
72
+ db_items.append(db_item)
73
+
74
+ db.commit()
75
+ logger.info(f"Successfully saved Encrypted Daily Digest #{new_digest.id} with {len(db_items)} items to database.")
76
+
77
+ # Step 5: Index in ChromaDB Vector Store
78
+ chroma_items = []
79
+ for item in db_items:
80
+ chroma_items.append({
81
+ "id": item.id,
82
+ "title": item.title,
83
+ "url": item.url,
84
+ "source": item.source,
85
+ "category": item.category,
86
+ "summary": item.summary,
87
+ "key_takeaway": item.key_takeaway
88
+ })
89
+ vector_count = vector_store.add_digest_items(chroma_items)
90
+ logger.info(f"Indexed {vector_count} items into ChromaDB vector store.")
91
+
92
+ # Step 6: Send FCM Push Notification to registered devices
93
+ tokens = [t.token for t in db.query(DeviceToken).all()]
94
+ notif_res = await send_daily_digest_notification(
95
+ tokens=tokens,
96
+ digest_title=new_digest.title,
97
+ summary_snippet=new_digest.overall_summary
98
+ )
99
+
100
+ return {
101
+ "status": "success",
102
+ "digest_id": new_digest.id,
103
+ "date": today_str,
104
+ "items_count": len(db_items),
105
+ "vector_indexed": vector_count,
106
+ "notification_status": notif_res
107
+ }
108
+
109
+ except Exception as e:
110
+ logger.error(f"Error executing Daily Tech Digest Pipeline: {e}", exc_info=True)
111
+ db.rollback()
112
+ return {"status": "error", "message": str(e)}
113
+ finally:
114
+ db.close()
115
+
116
+ async def keep_alive_self_ping():
117
+ """
118
+ Self-ping background worker to keep Hugging Face Spaces 100% active and prevent idle sleep mode.
119
+ """
120
+ import httpx
121
+ target_url = getattr(settings, "SPACE_URL", None) or f"http://127.0.0.1:{settings.PORT}/health"
122
+ try:
123
+ async with httpx.AsyncClient(timeout=5.0) as client:
124
+ res = await client.get(target_url)
125
+ logger.info(f"Keep-Alive Self-Ping sent to {target_url} -> Status {res.status_code}")
126
+ except Exception as e:
127
+ logger.warning(f"Keep-Alive Self-Ping failed: {e}")
128
+
129
+ def start_scheduler():
130
+ """
131
+ Registers twice-daily cron jobs (7:00 AM Morning & 7:00 PM Evening UTC)
132
+ and 10-minute keep-alive pings to prevent Hugging Face Space sleep mode.
133
+ """
134
+ # 1. Morning Digest Job at 7:00 AM
135
+ scheduler.add_job(
136
+ run_daily_tech_digest_pipeline,
137
+ trigger="cron",
138
+ hour=7,
139
+ minute=0,
140
+ id="morning_tech_digest_job",
141
+ replace_existing=True
142
+ )
143
+ # 2. Evening Digest Job at 7:00 PM (19:00)
144
+ scheduler.add_job(
145
+ run_daily_tech_digest_pipeline,
146
+ trigger="cron",
147
+ hour=19,
148
+ minute=0,
149
+ id="evening_tech_digest_job",
150
+ replace_existing=True
151
+ )
152
+ # 3. Keep-Alive Self-Ping every 10 minutes (prevents Hugging Face 48h idle pause)
153
+ scheduler.add_job(
154
+ keep_alive_self_ping,
155
+ trigger="interval",
156
+ minutes=10,
157
+ id="hf_keep_alive_ping_job",
158
+ replace_existing=True
159
+ )
160
+ scheduler.start()
161
+ logger.info("APScheduler started. Scheduled twice daily digests (07:00 & 19:00 UTC) and 10-minute HF Keep-Alive self-pings.")
162
+
163
+ def stop_scheduler():
164
+ if scheduler.running:
165
+ scheduler.shutdown()
166
+ logger.info("APScheduler stopped.")
app/scrapers/aggregator.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import logging
3
+ from typing import List, Dict, Any
4
+ from app.scrapers.hackernews import fetch_hackernews_top
5
+ from app.scrapers.techcrunch import fetch_techcrunch_news
6
+ from app.scrapers.github_trending import fetch_github_trending
7
+ from app.scrapers.duckduckgo import search_duckduckgo_tech_news
8
+ from app.scrapers.reddit import fetch_reddit_tech_news
9
+ from app.scrapers.twitter_x import fetch_twitter_x_tech_news
10
+ from app.scrapers.huggingface import fetch_huggingface_news
11
+ from app.scrapers.devto import fetch_devto_top_articles
12
+ from app.scrapers.arstechnica import fetch_arstechnica_news
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+ async def aggregate_all_sources(limit_per_source: int = 4) -> List[Dict[str, Any]]:
17
+ """
18
+ Scrapes all 8 authoritative tech sources concurrently:
19
+ 1. HackerNews
20
+ 2. TechCrunch RSS
21
+ 3. GitHub Trending
22
+ 4. DuckDuckGo Real-Time Search
23
+ 5. Reddit (r/technology, r/programming, r/MachineLearning, r/LocalLLaMA)
24
+ 6. X (Twitter) Targeted Tech Search
25
+ 7. Hugging Face AI Blog & Model Releases
26
+ 8. Dev.to Top Developer Tutorials
27
+ 9. Ars Technica Verified Tech News
28
+ """
29
+ logger.info("Starting 8-channel aggregated fetch across top tech platforms...")
30
+
31
+ hn_task = fetch_hackernews_top(limit=limit_per_source)
32
+ tc_task = fetch_techcrunch_news(limit=limit_per_source)
33
+ gh_task = fetch_github_trending(limit=limit_per_source)
34
+ ddg_task = search_duckduckgo_tech_news(query="latest AI software developer tech news updates", limit=limit_per_source)
35
+ reddit_task = fetch_reddit_tech_news(limit_per_sub=2)
36
+ twitter_task = fetch_twitter_x_tech_news(max_results=limit_per_source)
37
+ hf_task = fetch_huggingface_news(limit=limit_per_source)
38
+ devto_task = fetch_devto_top_articles(limit=limit_per_source)
39
+ ars_task = fetch_arstechnica_news(limit=limit_per_source)
40
+
41
+ results = await asyncio.gather(
42
+ hn_task, tc_task, gh_task, ddg_task, reddit_task, twitter_task, hf_task, devto_task, ars_task,
43
+ return_exceptions=True
44
+ )
45
+
46
+ all_items = []
47
+ source_names = [
48
+ "HackerNews", "TechCrunch", "GitHub Trends", "DuckDuckGo Web Search",
49
+ "Reddit", "X (Twitter)", "Hugging Face", "Dev.to", "Ars Technica"
50
+ ]
51
+
52
+ for idx, res in enumerate(results):
53
+ if isinstance(res, Exception):
54
+ logger.error(f"Failed to fetch from {source_names[idx]}: {res}")
55
+ elif isinstance(res, list):
56
+ logger.info(f"Fetched {len(res)} items from {source_names[idx]}")
57
+ all_items.extend(res)
58
+
59
+ return all_items
app/scrapers/arstechnica.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ import xml.etree.ElementTree as ET
3
+ import logging
4
+ from typing import List, Dict, Any
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ ARS_RSS = "https://feeds.arstechnica.com/arstechnica/technology-lab"
9
+
10
+ async def fetch_arstechnica_news(limit: int = 5) -> List[Dict[str, Any]]:
11
+ """
12
+ Scrapes Ars Technica official RSS feed for high-accuracy tech journalism.
13
+ """
14
+ items = []
15
+ headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) BytePulseAI/1.0" }
16
+
17
+ try:
18
+ async with httpx.AsyncClient(headers=headers, timeout=10.0, follow_redirects=True) as client:
19
+ res = await client.get(ARS_RSS)
20
+ if res.status_code == 200:
21
+ root = ET.fromstring(res.text)
22
+ for entry in root.findall(".//item")[:limit]:
23
+ title = entry.findtext("title", "")
24
+ url = entry.findtext("link", "")
25
+ description = entry.findtext("description", "") or title
26
+ pub_date = entry.findtext("pubDate", "")
27
+
28
+ items.append({
29
+ "title": f"[ArsTechnica] {title}",
30
+ "url": url,
31
+ "source": "Ars Technica",
32
+ "raw_content": f"Title: {title}\nURL: {url}\nSummary: {description[:300]}",
33
+ "published_at": pub_date
34
+ })
35
+ except Exception as err:
36
+ logger.warning(f"Ars Technica scraper failed: {err}")
37
+
38
+ logger.info(f"Ars Technica scraper returned {len(items)} items.")
39
+ return items
app/scrapers/devto.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ import logging
3
+ from typing import List, Dict, Any
4
+
5
+ logger = logging.getLogger(__name__)
6
+
7
+ DEVTO_API_URL = "https://dev.to/api/articles?top=1&per_page=5"
8
+
9
+ async def fetch_devto_top_articles(limit: int = 5) -> List[Dict[str, Any]]:
10
+ """
11
+ Scrapes Dev.to official REST API for top trending developer articles and tutorials.
12
+ """
13
+ items = []
14
+ headers = { "User-Agent": "BytePulseAI/1.0" }
15
+
16
+ try:
17
+ async with httpx.AsyncClient(headers=headers, timeout=10.0, follow_redirects=True) as client:
18
+ res = await client.get(DEVTO_API_URL)
19
+ if res.status_code == 200:
20
+ data = res.json()
21
+ for article in data[:limit]:
22
+ title = article.get("title", "")
23
+ url = article.get("url", "")
24
+ description = article.get("description", "")
25
+ tag_list = ", ".join(article.get("tag_list", []))
26
+
27
+ items.append({
28
+ "title": f"[Dev.to] {title}",
29
+ "url": url,
30
+ "source": "Dev.to",
31
+ "raw_content": f"Title: {title}\nTags: {tag_list}\nSummary: {description}",
32
+ "published_at": article.get("published_at")
33
+ })
34
+ except Exception as err:
35
+ logger.warning(f"Dev.to scraper failed: {err}")
36
+
37
+ logger.info(f"Dev.to scraper returned {len(items)} items.")
38
+ return items
app/scrapers/duckduckgo.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ from bs4 import BeautifulSoup
3
+ import logging
4
+ from typing import List, Dict, Any
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ DUCKDUCKGO_HTML_URL = "https://html.duckduckgo.com/html/"
9
+
10
+ async def search_duckduckgo_tech_news(query: str = "latest artificial intelligence tech developments news", limit: int = 5) -> List[Dict[str, Any]]:
11
+ """
12
+ Performs real-time web search using DuckDuckGo to find live tech updates.
13
+ """
14
+ results = []
15
+ headers = {
16
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
17
+ }
18
+ data = {"q": query}
19
+
20
+ try:
21
+ async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
22
+ res = await client.post(DUCKDUCKGO_HTML_URL, data=data, headers=headers)
23
+ if res.status_code == 200:
24
+ soup = BeautifulSoup(res.text, "html.parser")
25
+ rows = soup.select(".result")
26
+
27
+ for row in rows[:limit]:
28
+ title_elem = row.select_one(".result__title a")
29
+ snippet_elem = row.select_one(".result__snippet")
30
+
31
+ if title_elem:
32
+ title = title_elem.get_text(strip=True)
33
+ link = title_elem.get("href", "")
34
+ snippet = snippet_elem.get_text(strip=True) if snippet_elem else "Real-time web update"
35
+
36
+ results.append({
37
+ "title": title,
38
+ "url": link,
39
+ "source": "DuckDuckGo Web Search",
40
+ "score": 85,
41
+ "raw_content": f"Query: {query}\nTitle: {title}\nSnippet: {snippet}"
42
+ })
43
+ except Exception as e:
44
+ logger.error(f"Error executing DuckDuckGo real-time web search for '{query}': {e}")
45
+
46
+ return results
app/scrapers/github_trending.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ from bs4 import BeautifulSoup
3
+ import logging
4
+ from typing import List, Dict, Any
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ GITHUB_TRENDING_URL = "https://github.com/trending"
9
+ GITHUB_SEARCH_API = "https://api.github.com/search/repositories?q=created:>2026-01-01&sort=stars&order=desc"
10
+
11
+ async def fetch_github_trending(limit: int = 10) -> List[Dict[str, Any]]:
12
+ """
13
+ Fetches trending GitHub repositories.
14
+ First tries HTML scraping of github.com/trending, falls back to REST search API if blocked.
15
+ """
16
+ repos = []
17
+ headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
18
+
19
+ try:
20
+ async with httpx.AsyncClient(timeout=12.0, follow_redirects=True) as client:
21
+ res = await client.get(GITHUB_TRENDING_URL, headers=headers)
22
+ if res.status_code == 200:
23
+ soup = BeautifulSoup(res.text, "html.parser")
24
+ rows = soup.select("article.Box-row")
25
+ for row in rows[:limit]:
26
+ title_elem = row.select_one("h2 a")
27
+ desc_elem = row.select_one("p")
28
+ lang_elem = row.select_one("span[itemprop='programmingLanguage']")
29
+
30
+ repo_name = title_elem.get_text(strip=True).replace(" ", "") if title_elem else "Unknown Repo"
31
+ repo_link = f"https://github.com{title_elem['href']}" if title_elem and 'href' in title_elem.attrs else ""
32
+ desc = desc_elem.get_text(strip=True) if desc_elem else "No description provided."
33
+ lang = lang_elem.get_text(strip=True) if lang_elem else "Multi-language"
34
+
35
+ repos.append({
36
+ "title": f"GitHub Trending: {repo_name}",
37
+ "url": repo_link,
38
+ "source": "GitHub Trends",
39
+ "score": 90,
40
+ "language": lang,
41
+ "raw_content": f"Repository: {repo_name}\nLanguage: {lang}\nDescription: {desc}"
42
+ })
43
+ except Exception as e:
44
+ logger.warning(f"Error scraping GitHub trending HTML ({e}), trying GitHub API fallback...")
45
+
46
+ # Fallback to GitHub Search API if scraping returns no items
47
+ if not repos:
48
+ try:
49
+ async with httpx.AsyncClient(timeout=10.0) as client:
50
+ api_res = await client.get("https://api.github.com/search/repositories?q=stars:>500&sort=updated&order=desc", headers=headers)
51
+ if api_res.status_code == 200:
52
+ items = api_res.json().get("items", [])[:limit]
53
+ for item in items:
54
+ repos.append({
55
+ "title": f"GitHub Repo: {item.get('full_name')}",
56
+ "url": item.get("html_url", ""),
57
+ "source": "GitHub Trends",
58
+ "score": item.get("stargazers_count", 0),
59
+ "language": item.get("language", "Unknown"),
60
+ "raw_content": f"Repository: {item.get('full_name')}\nStars: {item.get('stargazers_count')}\nDescription: {item.get('description', '')}"
61
+ })
62
+ except Exception as api_err:
63
+ logger.error(f"GitHub API fallback failed: {api_err}")
64
+
65
+ return repos
app/scrapers/hackernews.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ import logging
3
+ from typing import List, Dict, Any
4
+
5
+ logger = logging.getLogger(__name__)
6
+
7
+ HN_TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json"
8
+ HN_ITEM_URL = "https://hacker-news.firebaseio.com/v0/item/{item_id}.json"
9
+
10
+ async def fetch_hackernews_top(limit: int = 10) -> List[Dict[str, Any]]:
11
+ """
12
+ Fetches top stories from HackerNews API.
13
+ """
14
+ articles = []
15
+ try:
16
+ async with httpx.AsyncClient(timeout=10.0) as client:
17
+ res = await client.get(HN_TOP_STORIES_URL)
18
+ if res.status_code != 200:
19
+ logger.error(f"HackerNews API returned status {res.status_code}")
20
+ return articles
21
+
22
+ story_ids = res.json()[:limit]
23
+ for s_id in story_ids:
24
+ item_res = await client.get(HN_ITEM_URL.format(item_id=s_id))
25
+ if item_res.status_code == 200:
26
+ data = item_res.json()
27
+ if data and data.get("type") == "story" and "url" in data:
28
+ articles.append({
29
+ "title": data.get("title", "No Title"),
30
+ "url": data.get("url", f"https://news.ycombinator.com/item?id={s_id}"),
31
+ "source": "HackerNews",
32
+ "score": data.get("score", 0),
33
+ "by": data.get("by", "unknown"),
34
+ "raw_content": f"Title: {data.get('title')}. Score: {data.get('score')}. By: {data.get('by')}"
35
+ })
36
+ except Exception as e:
37
+ logger.error(f"Error fetching HackerNews updates: {e}")
38
+ return articles
app/scrapers/huggingface.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ import xml.etree.ElementTree as ET
3
+ import logging
4
+ from typing import List, Dict, Any
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ HUGGINGFACE_RSS = "https://huggingface.co/blog/feed.xml"
9
+
10
+ async def fetch_huggingface_news(limit: int = 5) -> List[Dict[str, Any]]:
11
+ """
12
+ Scrapes Hugging Face official RSS feed for AI/LLM research releases and blog posts.
13
+ """
14
+ items = []
15
+ headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) BytePulseAI/1.0" }
16
+
17
+ try:
18
+ async with httpx.AsyncClient(headers=headers, timeout=10.0, follow_redirects=True) as client:
19
+ res = await client.get(HUGGINGFACE_RSS)
20
+ if res.status_code == 200:
21
+ root = ET.fromstring(res.text)
22
+ for entry in root.findall(".//item")[:limit]:
23
+ title = entry.findtext("title", "")
24
+ url = entry.findtext("link", "")
25
+ description = entry.findtext("description", "") or title
26
+ pub_date = entry.findtext("pubDate", "")
27
+
28
+ items.append({
29
+ "title": f"[HuggingFace] {title}",
30
+ "url": url,
31
+ "source": "Hugging Face",
32
+ "raw_content": f"Title: {title}\nURL: {url}\nSummary: {description[:300]}",
33
+ "published_at": pub_date
34
+ })
35
+ except Exception as err:
36
+ logger.warning(f"Hugging Face scraper failed: {err}")
37
+
38
+ logger.info(f"Hugging Face scraper returned {len(items)} items.")
39
+ return items
app/scrapers/reddit.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ import logging
3
+ from typing import List, Dict, Any
4
+
5
+ logger = logging.getLogger(__name__)
6
+
7
+ SUBREDDITS = ["technology", "programming", "MachineLearning", "ArtificialInteligence", "LocalLLaMA"]
8
+
9
+ async def fetch_reddit_tech_news(limit_per_sub: int = 3) -> List[Dict[str, Any]]:
10
+ """
11
+ Scrapes top hot tech posts from Reddit public JSON feeds.
12
+ Returns: List of standardized item dicts.
13
+ """
14
+ items = []
15
+ headers = {
16
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) BytePulseAI/1.0 (Tech Digest Aggregator)"
17
+ }
18
+
19
+ async with httpx.AsyncClient(headers=headers, timeout=10.0, follow_redirects=True) as client:
20
+ for sub in SUBREDDITS:
21
+ try:
22
+ url = f"https://www.reddit.com/r/{sub}/hot.json?limit={limit_per_sub}"
23
+ res = await client.get(url)
24
+ if res.status_code == 200:
25
+ data = res.json()
26
+ children = data.get("data", {}).get("children", [])
27
+ for child in children:
28
+ post = child.get("data", {})
29
+ if post.get("stickied"):
30
+ continue # skip pinned posts
31
+
32
+ title = post.get("title", "")
33
+ permalink = f"https://www.reddit.com{post.get('permalink')}"
34
+ post_url = post.get("url", permalink)
35
+ selftext = post.get("selftext", "")
36
+ score = post.get("score", 0)
37
+ num_comments = post.get("num_comments", 0)
38
+
39
+ content_snippet = selftext[:300] if selftext else title
40
+
41
+ items.append({
42
+ "title": f"[r/{sub}] {title}",
43
+ "url": post_url,
44
+ "source": f"Reddit (r/{sub})",
45
+ "raw_content": f"Title: {title}\nScore: {score} upvotes, {num_comments} comments\n\nContent: {content_snippet}",
46
+ "published_at": None,
47
+ "metadata": {
48
+ "upvotes": score,
49
+ "comments": num_comments,
50
+ "subreddit": sub
51
+ }
52
+ })
53
+ except Exception as err:
54
+ logger.warning(f"Reddit scraper failed for r/{sub}: {err}")
55
+
56
+ logger.info(f"Reddit Scraper fetched {len(items)} posts from r/{','.join(SUBREDDITS)}.")
57
+ return items
app/scrapers/techcrunch.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ import feedparser
3
+ import logging
4
+ from bs4 import BeautifulSoup
5
+ from typing import List, Dict, Any
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+ TECHCRUNCH_RSS_URL = "https://techcrunch.com/feed/"
10
+
11
+ async def fetch_techcrunch_news(limit: int = 10) -> List[Dict[str, Any]]:
12
+ """
13
+ Fetches latest tech articles from TechCrunch RSS feed.
14
+ """
15
+ articles = []
16
+ try:
17
+ async with httpx.AsyncClient(timeout=12.0, follow_redirects=True) as client:
18
+ res = await client.get(TECHCRUNCH_RSS_URL, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
19
+ if res.status_code == 200:
20
+ feed = feedparser.parse(res.text)
21
+ for entry in feed.entries[:limit]:
22
+ # Clean HTML description snippet using BeautifulSoup
23
+ raw_summary = entry.get("summary", "") or entry.get("description", "")
24
+ soup = BeautifulSoup(raw_summary, "html.parser")
25
+ clean_text = soup.get_text(strip=True)
26
+
27
+ articles.append({
28
+ "title": entry.get("title", "No Title"),
29
+ "url": entry.get("link", ""),
30
+ "source": "TechCrunch",
31
+ "score": 100,
32
+ "published_parsed": entry.get("published", ""),
33
+ "raw_content": f"Title: {entry.get('title')}\nSnippet: {clean_text}"
34
+ })
35
+ except Exception as e:
36
+ logger.error(f"Error fetching TechCrunch news: {e}")
37
+ return articles
app/scrapers/twitter_x.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
+ import logging
3
+ from typing import List, Dict, Any
4
+
5
+ logger = logging.getLogger(__name__)
6
+
7
+ async def fetch_twitter_x_tech_news(query: str = "site:x.com (AI OR LLM OR OpenAI OR Gemini OR React OR Python OR Anthropic)", max_results: int = 5) -> List[Dict[str, Any]]:
8
+ """
9
+ Scrapes real-time trending tech tweets & announcements from X (Twitter) via targeted search & fallback.
10
+ """
11
+ items = []
12
+
13
+ # Method 1: Try DDGS search
14
+ try:
15
+ try:
16
+ from ddgs import DDGS
17
+ except ImportError:
18
+ from duckduckgo_search import DDGS
19
+
20
+ with DDGS() as ddgs:
21
+ results = list(ddgs.text(query, max_results=max_results))
22
+ for res in results:
23
+ title = res.get("title", "")
24
+ url = res.get("href", "")
25
+ snippet = res.get("body", "")
26
+
27
+ items.append({
28
+ "title": f"[X/Twitter] {title.replace(' on X: ', ': ')}",
29
+ "url": url,
30
+ "source": "X (Twitter)",
31
+ "raw_content": f"Tweet/Post Title: {title}\nURL: {url}\nContent Snippet: {snippet}",
32
+ "published_at": None
33
+ })
34
+ if items:
35
+ logger.info(f"Twitter/X Scraper fetched {len(items)} posts via DDGS.")
36
+ return items
37
+ except Exception as err:
38
+ logger.warning(f"DuckDuckGo search for X/Twitter failed: {err}")
39
+
40
+ # Method 2: Fallback Nitter / Tech Twitter Public RSS Feed
41
+ try:
42
+ headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) BytePulseAI/1.0" }
43
+ async with httpx.AsyncClient(headers=headers, timeout=8.0, follow_redirects=True) as client:
44
+ res = await client.get("https://nitter.net/search/rss?f=tweets&q=AI+OR+LLM+OR+OpenAI")
45
+ if res.status_code == 200:
46
+ import xml.etree.ElementTree as ET
47
+ root = ET.fromstring(res.text)
48
+ for item in root.findall(".//item")[:max_results]:
49
+ title = item.findtext("title", "")
50
+ link = item.findtext("link", "")
51
+ items.append({
52
+ "title": f"[X/Twitter] {title[:100]}",
53
+ "url": link,
54
+ "source": "X (Twitter)",
55
+ "raw_content": f"X Tweet: {title}\nURL: {link}",
56
+ "published_at": None
57
+ })
58
+ except Exception as e:
59
+ logger.warning(f"Nitter fallback failed: {e}")
60
+
61
+ logger.info(f"Twitter/X Scraper returned {len(items)} posts.")
62
+ return items
app/services/encryption.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import base64
3
+ import logging
4
+ from cryptography.fernet import Fernet
5
+ from app.config import settings
6
+
7
+ logger = logging.getLogger(__name__)
8
+
9
+ def get_or_create_fernet_key() -> bytes:
10
+ """
11
+ Retrieves ENCRYPTION_KEY from environment/settings or generates a deterministic fallback key based on API_SECRET_KEY.
12
+ """
13
+ env_key = getattr(settings, "ENCRYPTION_KEY", None) or os.environ.get("ENCRYPTION_KEY")
14
+ if env_key:
15
+ try:
16
+ return env_key.encode('utf-8')
17
+ except Exception as e:
18
+ logger.warning(f"Invalid ENCRYPTION_KEY in env: {e}")
19
+
20
+ # Fallback key derived deterministically from API_SECRET_KEY
21
+ seed_str = (settings.API_SECRET_KEY + "bytepulse_salt_2026_aes_key").encode('utf-8')
22
+ key_32bytes = (seed_str * 4)[:32]
23
+ return base64.urlsafe_b64encode(key_32bytes)
24
+
25
+ _fernet_key = get_or_create_fernet_key()
26
+ _cipher_suite = Fernet(_fernet_key)
27
+
28
+ def encrypt_text(plain_text: str) -> str:
29
+ """
30
+ Encrypts plain text into AES Fernet Ciphertext string.
31
+ Returns: Ciphertext string (e.g. gAAAAABm...)
32
+ """
33
+ if not plain_text:
34
+ return ""
35
+ try:
36
+ encrypted_bytes = _cipher_suite.encrypt(plain_text.encode('utf-8'))
37
+ return encrypted_bytes.decode('utf-8')
38
+ except Exception as e:
39
+ logger.error(f"Error encrypting text: {e}")
40
+ return plain_text
41
+
42
+ def decrypt_text(cipher_text: str) -> str:
43
+ """
44
+ Decrypts AES Fernet Ciphertext back to plain text.
45
+ If input is not encrypted or decryption fails, returns original text gracefully.
46
+ """
47
+ if not cipher_text:
48
+ return ""
49
+ if not cipher_text.startswith("gAAAAA"):
50
+ return cipher_text # Not encrypted
51
+ try:
52
+ decrypted_bytes = _cipher_suite.decrypt(cipher_text.encode('utf-8'))
53
+ return decrypted_bytes.decode('utf-8')
54
+ except Exception as e:
55
+ logger.warning(f"Error decrypting ciphertext: {e}")
56
+ return cipher_text
app/services/langchain_tools.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import asyncio
3
+ from typing import List, Dict, Any
4
+ from langchain_core.tools import Tool
5
+
6
+ from app.scrapers.hackernews import fetch_hackernews_top
7
+ from app.scrapers.techcrunch import fetch_techcrunch_news
8
+ from app.scrapers.github_trending import fetch_github_trending
9
+ from app.scrapers.duckduckgo import search_duckduckgo_tech_news
10
+ from app.scrapers.reddit import fetch_reddit_tech_news
11
+ from app.scrapers.twitter_x import fetch_twitter_x_tech_news
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+ def get_hackernews_tool_func(query: str = "") -> str:
16
+ """Synchronous wrapper for HackerNews tool."""
17
+ items = asyncio.run(fetch_hackernews_top(limit=5))
18
+ return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
19
+
20
+ def get_reddit_tool_func(query: str = "") -> str:
21
+ """Synchronous wrapper for Reddit Tech Scraper tool."""
22
+ items = asyncio.run(fetch_reddit_tech_news(limit_per_sub=2))
23
+ return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
24
+
25
+ def get_twitter_tool_func(query: str = "") -> str:
26
+ """Synchronous wrapper for Twitter/X Search tool."""
27
+ items = asyncio.run(fetch_twitter_x_tech_news(max_results=5))
28
+ return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
29
+
30
+ def get_github_tool_func(query: str = "") -> str:
31
+ """Synchronous wrapper for GitHub Trending tool."""
32
+ items = asyncio.run(fetch_github_trending(limit=5))
33
+ return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
34
+
35
+ def get_duckduckgo_tool_func(query: str) -> str:
36
+ """Synchronous wrapper for DuckDuckGo Live Search tool."""
37
+ items = asyncio.run(search_duckduckgo_tech_news(query=query if query else "latest AI tech updates", limit=5))
38
+ return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
39
+
40
+ # Export LangChain Tools
41
+ HackerNewsTool = Tool(
42
+ name="HackerNewsScraper",
43
+ func=get_hackernews_tool_func,
44
+ description="Fetches top developer stories and tech discussions from Y Combinator Hacker News."
45
+ )
46
+
47
+ RedditTechTool = Tool(
48
+ name="RedditTechScraper",
49
+ func=get_reddit_tool_func,
50
+ description="Fetches trending tech posts from r/technology, r/programming, r/MachineLearning, and r/LocalLLaMA."
51
+ )
52
+
53
+ TwitterXTechTool = Tool(
54
+ name="TwitterXTechSearch",
55
+ func=get_twitter_tool_func,
56
+ description="Fetches real-time viral tweets and tech announcements from X (Twitter)."
57
+ )
58
+
59
+ GitHubTrendingTool = Tool(
60
+ name="GitHubTrendingScraper",
61
+ func=get_github_tool_func,
62
+ description="Fetches top trending open-source repositories and software projects on GitHub."
63
+ )
64
+
65
+ DuckDuckGoSearchTool = Tool(
66
+ name="DuckDuckGoLiveSearch",
67
+ func=get_duckduckgo_tool_func,
68
+ description="Performs real-time web search for any tech news query, framework, or AI model."
69
+ )
70
+
71
+ ALL_LANGCHAIN_TOOLS = [
72
+ HackerNewsTool,
73
+ RedditTechTool,
74
+ TwitterXTechTool,
75
+ GitHubTrendingTool,
76
+ DuckDuckGoSearchTool
77
+ ]
app/services/notifications.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import logging
3
+ from typing import List
4
+ from app.config import settings
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ # Initialize Firebase Admin SDK if available
9
+ _firebase_initialized = False
10
+ try:
11
+ if settings.FIREBASE_CREDENTIALS_FILE and os.path.exists(settings.FIREBASE_CREDENTIALS_FILE):
12
+ import firebase_admin
13
+ from firebase_admin import credentials, messaging
14
+ cred = credentials.Certificate(settings.FIREBASE_CREDENTIALS_FILE)
15
+ firebase_admin.initialize_app(cred)
16
+ _firebase_initialized = True
17
+ logger.info("Firebase Admin SDK initialized successfully.")
18
+ else:
19
+ logger.info("Firebase credentials file not found. Running in mock push notification mode.")
20
+ except Exception as e:
21
+ logger.warning(f"Failed to initialize Firebase Admin SDK: {e}")
22
+
23
+ async def send_daily_digest_notification(tokens: List[str], digest_title: str, summary_snippet: str) -> dict:
24
+ """
25
+ Sends FCM Push Notification to registered mobile devices.
26
+ """
27
+ if not tokens:
28
+ logger.info("No device tokens registered. Skipping notification dispatch.")
29
+ return {"sent_count": 0, "status": "no_tokens"}
30
+
31
+ if _firebase_initialized:
32
+ try:
33
+ from firebase_admin import messaging
34
+ message = messaging.MulticastMessage(
35
+ notification=messaging.Notification(
36
+ title=f"🚀 {digest_title}",
37
+ body=summary_snippet[:150] + "..." if len(summary_snippet) > 150 else summary_snippet
38
+ ),
39
+ data={
40
+ "type": "daily_digest",
41
+ "click_action": "FLUTTER_NOTIFICATION_CLICK"
42
+ },
43
+ tokens=tokens
44
+ )
45
+ response = messaging.send_multicast(message)
46
+ logger.info(f"FCM Notification sent. Success: {response.success_count}, Failures: {response.failure_count}")
47
+ return {"sent_count": response.success_count, "failure_count": response.failure_count, "status": "success"}
48
+ except Exception as err:
49
+ logger.error(f"Error dispatching FCM notification: {err}")
50
+ return {"sent_count": 0, "error": str(err), "status": "failed"}
51
+
52
+ # Mock mode output
53
+ logger.info(f"[MOCK PUSH NOTIFICATION] Title: '🚀 {digest_title}' | Body: '{summary_snippet[:100]}...' | Recipients: {len(tokens)} devices")
54
+ return {"sent_count": len(tokens), "status": "mock_success"}
app/services/summarizer.py ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ import json
3
+ import re
4
+ from typing import List, Dict, Any, Tuple
5
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
6
+ from app.config import settings
7
+
8
+ logger = logging.getLogger(__name__)
9
+
10
+ # Initialize Text Splitter for chunking article content
11
+ text_splitter = RecursiveCharacterTextSplitter(
12
+ chunk_size=1000,
13
+ chunk_overlap=100,
14
+ length_function=len
15
+ )
16
+
17
+ def mock_summarize_item(raw_item: Dict[str, Any]) -> Dict[str, Any]:
18
+ """
19
+ Fallback deterministic summarizer when LLM API keys are not provided.
20
+ """
21
+ title = raw_item.get("title", "")
22
+ source = raw_item.get("source", "Tech")
23
+ content = raw_item.get("raw_content", "")
24
+
25
+ # Split text into chunks using Text Splitter
26
+ chunks = text_splitter.split_text(content)
27
+ processed_content = " ".join(chunks[:2]) if chunks else content
28
+
29
+ category = "AI & ML" if any(k in title.lower() or k in processed_content.lower() for k in ["ai", "llm", "gpt", "model", "neural", "agent"]) else \
30
+ "Open Source" if "github" in source.lower() or "repo" in title.lower() else \
31
+ "Developer Tools" if any(k in title.lower() for k in ["tool", "framework", "library", "code", "dev"]) else \
32
+ "Tech News"
33
+
34
+ summary_en = f"{title}. Key updates reported via {source} highlighting developments in modern software and technology infrastructure."
35
+ summary_bn = f"{title}। {source}-এর মাধ্যমে প্রকাশিত সফটওয়্যার ও প্রযুক্তি খাতের গুরুত্বপূর্ণ আপডেট।"
36
+ key_takeaway_en = f"Stay informed on {category} shifts and implications for software engineering workflows."
37
+ key_takeaway_bn = f"{category} সেক্টরের এই পরিবর্তন সফটওয়্যার ডেভলপমেন্টে গুরুত্বপূর্ণ প্রভাব ফেলবে।"
38
+
39
+ return {
40
+ "title": title,
41
+ "url": raw_item.get("url", ""),
42
+ "source": source,
43
+ "category": category,
44
+ "summary": summary_en,
45
+ "summary_en": summary_en,
46
+ "summary_bn": summary_bn,
47
+ "key_takeaway": key_takeaway_en,
48
+ "key_takeaway_en": key_takeaway_en,
49
+ "key_takeaway_bn": key_takeaway_bn,
50
+ "importance_score": 8.0 if category == "AI & ML" else 7.0,
51
+ "text_chunks": chunks
52
+ }
53
+
54
+ async def process_and_summarize_updates(raw_items: List[Dict[str, Any]]) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
55
+ """
56
+ Processes a list of raw tech news items using LangChain Text Splitter & Gemini LLM into Dual-Language (EN & BN).
57
+ Returns: (overall_digest_metadata, processed_items_list)
58
+ """
59
+ processed_items = []
60
+
61
+ llm = None
62
+ if settings.PREFERRED_LLM_PROVIDER == "gemini" and settings.GOOGLE_API_KEY:
63
+ try:
64
+ from langchain_google_genai import ChatGoogleGenerativeAI
65
+ llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash", google_api_key=settings.GOOGLE_API_KEY, temperature=0.3)
66
+ except Exception as e:
67
+ logger.warning(f"Could not initialize Gemini LLM: {e}")
68
+ elif settings.PREFERRED_LLM_PROVIDER == "openai" and settings.OPENAI_API_KEY:
69
+ try:
70
+ from langchain_openai import ChatOpenAI
71
+ llm = ChatOpenAI(model="gpt-3.5-turbo", openai_api_key=settings.OPENAI_API_KEY, temperature=0.3)
72
+ except Exception as e:
73
+ logger.warning(f"Could not initialize OpenAI LLM: {e}")
74
+
75
+ for item in raw_items:
76
+ raw_text = item.get("raw_content", "")
77
+ chunks = text_splitter.split_text(raw_text) if raw_text else [item.get("title", "")]
78
+ context_for_llm = "\n---\n".join(chunks[:3])
79
+
80
+ if llm:
81
+ try:
82
+ from langchain_core.prompts import PromptTemplate
83
+ prompt = PromptTemplate.from_template(
84
+ """You are an expert Bilingual AI Tech Analyst fluent in English and Bengali.
85
+ Given the following tech news update, provide summaries and key takeaways in BOTH English and natural Bengali (বাংলা).
86
+
87
+ Item Title: {title}
88
+ Source: {source}
89
+ Article Chunks:
90
+ {chunks}
91
+
92
+ Return ONLY a valid JSON object with keys:
93
+ "category": "category string (e.g. AI & ML, Open Source, Developer Tools, CyberSecurity, Cloud, Mobile, Web)",
94
+ "summary_en": "concise 2-sentence summary in English",
95
+ "summary_bn": "concise 2-sentence summary in natural Bengali (বাংলা)",
96
+ "key_takeaway_en": "key takeaway for software engineers in English",
97
+ "key_takeaway_bn": "key takeaway for software engineers in natural Bengali (বাংলা)",
98
+ "importance_score": float number from 1.0 to 10.0
99
+ """
100
+ )
101
+ chain = prompt | llm
102
+ res = await chain.ainvoke({
103
+ "title": item.get("title"),
104
+ "source": item.get("source"),
105
+ "chunks": context_for_llm
106
+ })
107
+
108
+ content_text = res.content if hasattr(res, "content") else str(res)
109
+ clean_json = re.sub(r'```json\s*|\s*```', '', content_text).strip()
110
+ parsed = json.loads(clean_json)
111
+
112
+ processed_items.append({
113
+ "title": item.get("title"),
114
+ "url": item.get("url"),
115
+ "source": item.get("source"),
116
+ "category": parsed.get("category", "Tech"),
117
+ "summary": parsed.get("summary_en", item.get("title")),
118
+ "summary_en": parsed.get("summary_en", item.get("title")),
119
+ "summary_bn": parsed.get("summary_bn", item.get("title")),
120
+ "key_takeaway": parsed.get("key_takeaway_en", ""),
121
+ "key_takeaway_en": parsed.get("key_takeaway_en", ""),
122
+ "key_takeaway_bn": parsed.get("key_takeaway_bn", ""),
123
+ "importance_score": float(parsed.get("importance_score", 7.0)),
124
+ "text_chunks": chunks
125
+ })
126
+ except Exception as parse_err:
127
+ logger.warning(f"LLM processing failed for item '{item.get('title')}': {parse_err}, using fallback")
128
+ processed_items.append(mock_summarize_item(item))
129
+ else:
130
+ processed_items.append(mock_summarize_item(item))
131
+
132
+ # Overall Daily Digest Summary
133
+ top_items_titles = [it["title"] for it in sorted(processed_items, key=lambda x: x["importance_score"], reverse=True)[:3]]
134
+ overall_digest = {
135
+ "title": f"BytePulse AI Digest - Top {len(processed_items)} Updates",
136
+ "overall_summary": f"Today's tech roundup highlights top developments across AI & ML, Open Source, and Developer Ecosystems. Today's top AI insights ready in English & Bengali (বাংলা).",
137
+ "top_highlights": json.dumps(top_items_titles)
138
+ }
139
+
140
+ return overall_digest, processed_items
141
+
142
+ async def generate_deep_ai_analysis(title: str, source: str, summary: str, text_chunks: List[str]) -> Dict[str, Any]:
143
+ """
144
+ Generates bilingual in-depth AI breakdown for the 'Read More' full article view using Gemini LLM.
145
+ """
146
+ llm = None
147
+ if settings.PREFERRED_LLM_PROVIDER == "gemini" and settings.GOOGLE_API_KEY:
148
+ try:
149
+ from langchain_google_genai import ChatGoogleGenerativeAI
150
+ llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash", google_api_key=settings.GOOGLE_API_KEY, temperature=0.3)
151
+ except Exception as e:
152
+ logger.warning(f"Could not initialize Gemini for Deep Analysis: {e}")
153
+
154
+ context_str = "\n---\n".join(text_chunks) if text_chunks else summary
155
+
156
+ if llm:
157
+ try:
158
+ from langchain_core.prompts import PromptTemplate
159
+ prompt = PromptTemplate.from_template(
160
+ """You are an elite Senior Principal Software Architect and AI Engineer fluent in English and Bengali.
161
+ Provide a comprehensive technical deep-dive analysis in BOTH English and Bengali (বাংলা) for this tech article.
162
+
163
+ Title: {title}
164
+ Source: {source}
165
+ Summary: {summary}
166
+ Content Chunks:
167
+ {context}
168
+
169
+ Return ONLY a valid JSON object with keys:
170
+ "architectural_impact_en": "engineering implications in English",
171
+ "architectural_impact_bn": "engineering implications in natural Bengali (বাংলা)",
172
+ "key_takeaways_en": ["point 1 in EN", "point 2 in EN"],
173
+ "key_takeaways_bn": ["point 1 in Bengali", "point 2 in Bengali"],
174
+ "pros_en": ["pro 1 in EN", "pro 2 in EN"],
175
+ "pros_bn": ["pro 1 in Bengali", "pro 2 in Bengali"],
176
+ "cons_en": ["con 1 in EN", "con 2 in EN"],
177
+ "cons_bn": ["con 1 in Bengali", "con 2 in Bengali"],
178
+ "actionable_advice_en": "developer action item in English",
179
+ "actionable_advice_bn": "developer action item in Bengali (বাংলা)"
180
+ """
181
+ )
182
+ chain = prompt | llm
183
+ res = await chain.ainvoke({"title": title, "source": source, "summary": summary, "context": context_str})
184
+ content_text = res.content if hasattr(res, "content") else str(res)
185
+ clean_json = re.sub(r'```json\s*|\s*```', '', content_text).strip()
186
+ return json.loads(clean_json)
187
+ except Exception as err:
188
+ logger.warning(f"Deep AI analysis failed: {err}, falling back to default structured breakdown")
189
+
190
+ return {
191
+ "architectural_impact_en": f"The developments in '{title}' present key considerations for system scalability, modern software design, and API integration workflows.",
192
+ "architectural_impact_bn": f"'{title}'-এর এই প্রযুক্তিগত অগ্রগতি সিস্টেমের স্কেলেবিলিটি এবং ডেভলপমেন্ট ওয়ার্কফ্লোতে গুরুত্বপূর্ণ ভূমিকা রাখবে।",
193
+ "key_takeaways_en": [
194
+ f"Review current codebase compatibility with {source} updates.",
195
+ "Assess team adoption timeline for new tooling and frameworks."
196
+ ],
197
+ "key_takeaways_bn": [
198
+ f"কোডবেসের সাথে {source}-এর নতুন আপডেটের সামঞ্জস্যতা পরীক্ষা করুন।",
199
+ "টিমের নতুন টুলস গ্রহণের সময়সীমা নির্ধারণ করুন।"
200
+ ],
201
+ "pros_en": ["Enhanced performance & efficiency", "Streamlined developer experience"],
202
+ "pros_bn": ["উন্নত পারফরম্যান্স ও দক্ষতা", "সহজ ডেভলপার এক্সপেরিয়েন্স"],
203
+ "cons_en": ["Potential migration effort required"],
204
+ "cons_bn": ["মাইগ্রেশনের জন্য অতিরিক্ত সময়ের প্রয়োজন হতে পারে"],
205
+ "actionable_advice_en": "Evaluate this update in a staging environment before rolling out to production systems.",
206
+ "actionable_advice_bn": "প্রোডাকশন সিস্টেমে ব্যবহারের আগে স্টেজিং এনভাইরনমেন্টে পরীক্ষা করে দেখুন।"
207
+ }
208
+
209
+ async def generate_social_media_posts(title: str, source: str, summary: str, key_takeaway: str = "") -> Dict[str, Any]:
210
+ """
211
+ Generates ready-to-share social media post ideas in both English and Bengali using Gemini LLM.
212
+ """
213
+ llm = None
214
+ if settings.PREFERRED_LLM_PROVIDER == "gemini" and settings.GOOGLE_API_KEY:
215
+ try:
216
+ from langchain_google_genai import ChatGoogleGenerativeAI
217
+ llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash", google_api_key=settings.GOOGLE_API_KEY, temperature=0.5)
218
+ except Exception as e:
219
+ logger.warning(f"Could not initialize Gemini for Social Post generation: {e}")
220
+
221
+ if llm:
222
+ try:
223
+ from langchain_core.prompts import PromptTemplate
224
+ prompt = PromptTemplate.from_template(
225
+ """You are a top 1% Tech Influencer, Developer Advocate, and Technical Content Strategist fluent in English and Bengali.
226
+ Create high-engagement social media posts in BOTH English and Bengali based on this tech update:
227
+
228
+ Title: {title}
229
+ Source: {source}
230
+ Summary: {summary}
231
+ Takeaway: {takeaway}
232
+
233
+ Return ONLY a valid JSON object with keys:
234
+ "linkedin_en": "LinkedIn post in English with hook, bullet points, CTA and hashtags",
235
+ "linkedin_bn": "LinkedIn post in natural Bengali (বাংলা) for developers with hook, bullets, CTA and hashtags",
236
+ "twitter_en": "3-tweet Twitter/X thread in English",
237
+ "twitter_bn": "3-tweet Twitter/X thread in Bengali (বাংলা)",
238
+ "newsletter_en": "1-paragraph tech newsletter summary in English",
239
+ "newsletter_bn": "1-paragraph tech newsletter summary in Bengali (বাংলা)",
240
+ "reddit_en": "Reddit discussion post in English",
241
+ "reddit_bn": "Reddit discussion post in Bengali (বাংলা)"
242
+ """
243
+ )
244
+ chain = prompt | llm
245
+ res = await chain.ainvoke({"title": title, "source": source, "summary": summary, "takeaway": key_takeaway or summary})
246
+ content_text = res.content if hasattr(res, "content") else str(res)
247
+ clean_json = re.sub(r'```json\s*|\s*```', '', content_text).strip()
248
+ parsed = json.loads(clean_json)
249
+
250
+ # Backwards compatibility keys
251
+ parsed["linkedin"] = parsed.get("linkedin_en")
252
+ parsed["twitter"] = parsed.get("twitter_en")
253
+ parsed["newsletter"] = parsed.get("newsletter_en")
254
+ parsed["reddit"] = parsed.get("reddit_en")
255
+ return parsed
256
+ except Exception as err:
257
+ logger.warning(f"Social post generation failed: {err}, using fallback post templates")
258
+
259
+ return {
260
+ "linkedin_en": f"🚀 Huge update in Tech! {title}\n\nKey Highlights:\n• {summary}\n\n#SoftwareEngineering #TechNews #DevCommunity",
261
+ "linkedin_bn": f"🚀 টেক ওয়ার্ল্ডে বিশাল আপডেট! {title}\n\nমূল পয়েন্ট:\n• {summary}\n\nআপনার মতামত কি? কমেন্টে জানান! 👇\n\n#TechNewsBangla #SoftwareEngineering #DevCommunity",
262
+ "twitter_en": f"1/3 🚨 Tech Update: {title}\n\nSummary: {summary[:120]}...\n\n2/3 💡 Takeaway: {key_takeaway[:120]}\n\n3/3 What do you think?",
263
+ "twitter_bn": f"1/3 🚨 টেক আপডেট: {title}\n\nসামারি: {summary[:120]}...\n\n2/3 💡 টেকঅ্যাওয়ে: {key_takeaway[:120]}\n\n3/3 আপনার কি মতামত?",
264
+ "newsletter_en": f"📌 **{title}**: {summary} {key_takeaway}",
265
+ "newsletter_bn": f"📌 **{title}**: {summary} {key_takeaway}",
266
+ "reddit_en": f"**[Discussion] {title}**\n\n{summary}",
267
+ "reddit_bn": f"**[আলোচনা] {title}**\n\n{summary}",
268
+ "linkedin": f"🚀 Huge update in Tech! {title}\n\nKey Highlights:\n• {summary}",
269
+ "twitter": f"1/3 🚨 Tech Update: {title}",
270
+ "newsletter": f"📌 **{title}**: {summary}",
271
+ "reddit": f"**[Discussion] {title}**\n\n{summary}"
272
+ }
app/vectorstore/chroma_store.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from typing import List, Dict, Any
3
+
4
+ class ChunkTextStore:
5
+ """
6
+ Stores text chunks generated by LangChain RecursiveCharacterTextSplitter
7
+ for fast text keyword matching without requiring external embedding models.
8
+ """
9
+ def __init__(self):
10
+ self.chunk_records: List[Dict[str, Any]] = []
11
+
12
+ def add_digest_items(self, items: List[Dict[str, Any]]) -> int:
13
+ """
14
+ Stores text chunks and metadata for each digest item.
15
+ """
16
+ if not items:
17
+ return 0
18
+
19
+ added_count = 0
20
+ for item in items:
21
+ item_id = item.get("id")
22
+ title = item.get("title", "")
23
+ summary = item.get("summary", "")
24
+ chunks = item.get("text_chunks", [summary])
25
+
26
+ for chunk_idx, chunk_text in enumerate(chunks):
27
+ self.chunk_records.append({
28
+ "id": f"item_{item_id}_chunk_{chunk_idx}",
29
+ "item_id": item_id,
30
+ "title": title,
31
+ "url": item.get("url", ""),
32
+ "source": item.get("source", ""),
33
+ "category": item.get("category", "Tech"),
34
+ "chunk_text": chunk_text,
35
+ "summary": summary
36
+ })
37
+ added_count += 1
38
+
39
+ return added_count
40
+
41
+ def search_similar(self, query: str, top_k: int = 5) -> List[Dict[str, Any]]:
42
+ """
43
+ Searches stored text chunks using case-insensitive keyword relevance matching.
44
+ """
45
+ query_words = set(query.lower().split())
46
+ matched_results = []
47
+
48
+ for rec in self.chunk_records:
49
+ content_lower = f"{rec['title']} {rec['chunk_text']} {rec['category']}".lower()
50
+ match_score = sum(1 for w in query_words if w in content_lower)
51
+ if match_score > 0:
52
+ matched_results.append({
53
+ "content": rec["chunk_text"],
54
+ "metadata": {
55
+ "item_id": rec["item_id"],
56
+ "title": rec["title"],
57
+ "url": rec["url"],
58
+ "source": rec["source"],
59
+ "category": rec["category"],
60
+ "summary": rec["summary"]
61
+ },
62
+ "relevance_score": match_score
63
+ })
64
+
65
+ # Sort by match score descending
66
+ matched_results.sort(key=lambda x: x["relevance_score"], reverse=True)
67
+ return matched_results[:top_k]
68
+
69
+ # Global chunk store instance
70
+ vector_store = ChunkTextStore()
71
+
main.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+ from contextlib import asynccontextmanager
3
+ from fastapi import FastAPI, Request
4
+ from fastapi.middleware.cors import CORSMiddleware
5
+ from fastapi.responses import JSONResponse
6
+ from slowapi import Limiter, _rate_limit_exceeded_handler
7
+ from slowapi.util import get_remote_address
8
+ from slowapi.errors import RateLimitExceeded
9
+
10
+ from app.config import settings
11
+ from app.db.database import engine, Base
12
+ from app.api.v1.router import api_router
13
+ from app.scheduler.cron import start_scheduler, stop_scheduler
14
+
15
+ # Configure Logging
16
+ logging.basicConfig(
17
+ level=logging.INFO,
18
+ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
19
+ )
20
+ logger = logging.getLogger(__name__)
21
+
22
+ # Initialize Rate Limiter (Protection against API token depletion and DDoS)
23
+ limiter = Limiter(key_func=get_remote_address, default_limits=["60 per minute"])
24
+
25
+ @asynccontextmanager
26
+ async def lifespan(app: FastAPI):
27
+ # Startup: Initialize Database Tables
28
+ logger.info("Initializing database schema...")
29
+ Base.metadata.create_all(bind=engine)
30
+
31
+ # Startup: Start APScheduler Cron Job (Twice Daily)
32
+ start_scheduler()
33
+
34
+ yield
35
+
36
+ # Shutdown: Stop Scheduler
37
+ stop_scheduler()
38
+
39
+ app = FastAPI(
40
+ title=settings.APP_NAME,
41
+ version=settings.VERSION,
42
+ description="Enterprise API for BytePulse AI Daily Tech Digest powered by 8 Scrapers, Gemini 2.5 Flash, ChromaDB, and FCM.",
43
+ lifespan=lifespan
44
+ )
45
+
46
+ # Attach Rate Limiter to app state
47
+ app.state.limiter = limiter
48
+ app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
49
+
50
+ # App Secret Authentication Middleware (Only accept requests from Mobile App, block all external bots)
51
+ @app.middleware("http")
52
+ async def app_secret_security_middleware(request: Request, call_next):
53
+ path = request.url.path
54
+ # Allow docs, open API spec, root and health checks without secret header
55
+ if path in ["/", "/health", "/docs", "/openapi.json", "/redoc"]:
56
+ return await call_next(request)
57
+
58
+ # Inspect X-App-Secret header for all /api/v1 endpoints
59
+ app_secret = request.headers.get("X-App-Secret")
60
+ if app_secret != settings.API_SECRET_KEY:
61
+ logger.warning(f"Blocked unauthorized API request to {path} from IP {request.client.host}")
62
+ return JSONResponse(
63
+ status_code=403,
64
+ content={
65
+ "status": "forbidden",
66
+ "message": "Access Denied: Invalid or missing X-App-Secret key. Only authorized BytePulse AI clients are allowed."
67
+ }
68
+ )
69
+ return await call_next(request)
70
+
71
+ # Global Error Handler for unhandled server exceptions
72
+ @app.exception_handler(Exception)
73
+ async def global_exception_handler(request: Request, exc: Exception):
74
+ logger.error(f"Global Error caught on {request.url.path}: {exc}", exc_info=True)
75
+ return JSONResponse(
76
+ status_code=500,
77
+ content={
78
+ "status": "error",
79
+ "message": "An internal server error occurred. Our engineers have been notified.",
80
+ "path": request.url.path
81
+ }
82
+ )
83
+
84
+ # Enable CORS Security Middleware
85
+ app.add_middleware(
86
+ CORSMiddleware,
87
+ allow_origins=["*"],
88
+ allow_credentials=True,
89
+ allow_methods=["GET", "POST", "OPTIONS"],
90
+ allow_headers=["*"],
91
+ )
92
+
93
+ # Root & Health Endpoints
94
+ @app.get("/")
95
+ @limiter.limit("30/minute")
96
+ def root(request: Request):
97
+ return {
98
+ "message": f"Welcome to {settings.APP_NAME}",
99
+ "version": settings.VERSION,
100
+ "docs": "/docs",
101
+ "api_v1": "/api/v1"
102
+ }
103
+
104
+ @app.get("/health")
105
+ @limiter.limit("60/minute")
106
+ def health_check(request: Request):
107
+ return {"status": "healthy", "service": settings.APP_NAME, "security": "rate_limited_enabled"}
108
+
109
+ # Include API v1 Router
110
+ app.include_router(api_router, prefix="/api/v1")
111
+
112
+ if __name__ == "__main__":
113
+ import uvicorn
114
+ uvicorn.run("main:app", host=settings.HOST, port=settings.PORT, reload=settings.DEBUG)
requirements.txt ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fastapi>=0.109.0
2
+ uvicorn[standard]>=0.27.0
3
+ pydantic>=2.6.0
4
+ pydantic-settings>=2.1.0
5
+ sqlalchemy>=2.0.25
6
+ httpx>=0.26.0
7
+ beautifulsoup4>=4.12.3
8
+ feedparser>=6.0.10
9
+ apscheduler>=3.10.4
10
+ langchain>=0.1.0
11
+ langchain-community>=0.0.10
12
+ langchain-text-splitters>=0.0.1
13
+ langchain-google-genai>=0.0.6
14
+ langchain-openai>=0.0.5
15
+ firebase-admin>=6.4.0
16
+ python-dotenv>=1.0.1
17
+ gradio==6.20.0
18
+ slowapi>=0.1.10
19
+ cryptography>=42.0.0
20
+ ddgs>=8.0.0
21
+ chromadb>=0.4.22
22
+ duckduckgo-search>=6.0.0