track join-step blocker

app/bot/delivery.py

@@ -9,7 +9,7 @@ from app.bot.keyboards import library as lib_kb
 from app.bot.keyboards.common import share_kb
 from app.core import season
 from app.core.i18n import t
-from app.db.models import Material, User
+from app.db.models import ActivityEvent, Material, User
 from app.db.repos import library as repo
 from app.services import growth
 from app.services import library as lib
@@ -41,6 +41,7 @@ async def deliver_material(bot: Bot, session: AsyncSession, user: User, material
     except TelegramBadRequest:
         return False
     material.downloads = (material.downloads or 0) + 1
+    session.add(ActivityEvent(user_id=user.id, kind="download", ref=str(material.id), n=1))
     res = await xp_service.award(session, user, "download", 1, ref=f"{material.id}:{season.day_key()}")
     if res.leveled_up:
         await xp_service.announce_level(bot, user, res.new_level)

app/bot/handlers/__init__.py

@@ -1,6 +1,6 @@
 from aiogram import Router
 
-from app.bot.handlers import admin, admin_queue, contribute, gate, groups, inline, library, menu, search, start
+from app.bot.handlers import admin, admin_queue, analytics, channel_posts, contribute, gate, groups, inline, library, menu, search, start
 
 
 def build_router() -> Router:
@@ -9,6 +9,8 @@ def build_router() -> Router:
     router.include_router(gate.router)
     router.include_router(admin.router)
     router.include_router(admin_queue.router)
+    router.include_router(analytics.router)
+    router.include_router(channel_posts.router)
     router.include_router(groups.router)
     router.include_router(start.router)
     router.include_router(menu.router)

app/bot/handlers/analytics.py

@@ -0,0 +1,79 @@
+"""/analyze: an admin-only analytics screen with section buttons. Each section is computed on its own,
+so one failing query never hides the others."""
+import logging
+
+from aiogram import F, Router
+from aiogram.filters import Command
+from aiogram.types import CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, Message
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.bot.handlers.admin import IsAdmin
+from app.bot.handlers.channel_posts import is_main_channel
+from app.db.repos import analytics as analytics_repo
+from app.services import analytics as A
+
+log = logging.getLogger(__name__)
+router = Router()
+router.message.filter(IsAdmin())
+router.callback_query.filter(IsAdmin())
+
+SECTIONS = {
+    "growth": ("👥 Growth", A.section_growth),
+    "retention": ("🔁 Retention", A.section_retention),
+    "library": ("📚 Library", A.section_library),
+    "contrib": ("📤 Student files", A.section_contrib),
+    "channel": ("📢 Channel posts", A.section_channel),
+    "insights": ("💡 Insights", A.section_insights),
+}
+
+
+def keyboard(current: str) -> InlineKeyboardMarkup:
+    buttons = [
+        InlineKeyboardButton(text=("• " if key == current else "") + label, callback_data=f"az:{key}")
+        for key, (label, _) in SECTIONS.items()
+    ]
+    rows = [buttons[i : i + 2] for i in range(0, len(buttons), 2)]
+    rows.append([InlineKeyboardButton(text=("• " if current == "home" else "") + "🏠 Overview", callback_data="az:home")])
+    return InlineKeyboardMarkup(inline_keyboard=rows)
+
+
+def render(fn, snap: A.Snapshot) -> str:
+    try:
+        text = fn(snap)
+    except Exception as exc:  # one broken section must not hide the rest
+        log.exception("Analytics section failed")
+        text = f"⚠️ This section failed: {type(exc).__name__}: {exc}"
+    if len(text) > 4000:
+        text = text[: text.rfind("\n", 0, 3900)] + "\n…(cut: too long for one message)"
+    return text
+
+
+@router.message(Command("analyze"))
+async def analyze(message: Message, session: AsyncSession) -> None:
+    snap = await analytics_repo.load(session)
+    await message.answer(render(A.overview, snap), reply_markup=keyboard("home"))
+
+
+@router.callback_query(F.data.startswith("az:"))
+async def section(call: CallbackQuery, session: AsyncSession) -> None:
+    key = call.data.split(":")[1]
+    fn = A.overview if key == "home" else SECTIONS.get(key, (None, A.overview))[1]
+    snap = await analytics_repo.load(session)
+    await call.answer()
+    try:
+        await call.message.edit_text(render(fn, snap), reply_markup=keyboard(key))
+    except Exception:
+        pass  # "message is not modified": the same numbers
+
+
+@router.message(F.chat.type == "private", F.text, F.forward_origin.type == "channel")
+async def add_forwarded_post(message: Message, session: AsyncSession) -> None:
+    """Forward an older channel post (text) here to add it to the channel analysis."""
+    origin = message.forward_origin
+    if not is_main_channel(origin.chat):
+        await message.answer("That post is from another channel; only your main channel is analysed.")
+        return
+    await analytics_repo.save_post(
+        session, chat_id=origin.chat.id, message_id=origin.message_id, posted_at=origin.date, text=message.text
+    )
+    await message.answer("✅ Post added. See it in /analyze → Channel posts (reactions of old posts are unknown).")

app/bot/handlers/channel_posts.py

@@ -0,0 +1,44 @@
+"""Records the main channel's posts and their reactions, so /analyze can link posts to new students.
+Telegram's Bot API does NOT give bots a post's views or shares (only the Telegram app shows those);
+reactions are delivered as counts when the bot is an admin of the channel."""
+import logging
+
+from aiogram import Router
+from aiogram.types import Chat, Message, MessageReactionCountUpdated
+
+from app.config import get_settings
+from app.db.repos import analytics as analytics_repo
+from app.db.session import SessionFactory
+
+log = logging.getLogger(__name__)
+router = Router()
+
+
+def is_main_channel(chat: Chat) -> bool:
+    return (chat.username or "").lower() == get_settings().gate_list[0].lstrip("@").lower()
+
+
+@router.channel_post()
+async def on_channel_post(post: Message) -> None:
+    if not is_main_channel(post.chat):
+        return
+    try:
+        async with SessionFactory() as session:
+            await analytics_repo.save_post(
+                session, chat_id=post.chat.id, message_id=post.message_id, posted_at=post.date, text=post.text or post.caption or ""
+            )
+            await session.commit()
+    except Exception:
+        log.exception("Could not record a channel post")
+
+
+@router.message_reaction_count()
+async def on_reactions(event: MessageReactionCountUpdated) -> None:
+    if not is_main_channel(event.chat):
+        return
+    try:
+        async with SessionFactory() as session:
+            await analytics_repo.set_reactions(session, event.chat.id, event.message_id, sum(r.total_count for r in event.reactions))
+            await session.commit()
+    except Exception:
+        log.exception("Could not record reactions")

app/bot/handlers/groups.py

@@ -12,6 +12,7 @@ from aiogram.types import ChatMemberUpdated, InlineKeyboardButton, InlineKeyboar
 from app.bot.keyboards.common import open_bot_kb
 from app.bot.keyboards.library import clip, course_icon, course_label
 from app.core.i18n import t
+from app.db.models import ActivityEvent
 from app.db.repos import groups as group_repo
 from app.db.session import SessionFactory
 from app.services import growth
@@ -60,6 +61,8 @@ async def find_in_group(message: Message, command: CommandObject) -> None:
         return
     async with SessionFactory() as session:
         courses, materials = await lib.search(session, query, course_limit=4, material_limit=4)
+        session.add(ActivityEvent(user_id=message.from_user.id, kind="find", ref=query[:100].lower(), n=len(courses) + len(materials)))
+        await session.commit()
     if not courses and not materials:
         await message.reply(t("en", "group_find_none"), reply_markup=open_bot_kb(open_link))
         return

app/bot/handlers/inline.py

@@ -18,7 +18,7 @@ from aiogram.types import (
 
 from app.config import get_settings
 from app.core.i18n import t
-from app.db.models import Course, Material
+from app.db.models import ActivityEvent, Course, Material
 from app.db.repos import library as repo
 from app.db.session import SessionFactory
 from app.services import library as lib
@@ -80,3 +80,10 @@ async def inline_search(query: InlineQuery) -> None:
         is_personal=False,
         button=InlineQueryResultsButton(text=t("en", "inline_open_bot"), start_parameter="inline"),
     )
+    if len(text) >= 2:  # remember what was searched (after answering, so the student never waits for it)
+        try:
+            async with SessionFactory() as session:
+                session.add(ActivityEvent(user_id=query.from_user.id, kind="inline", ref=text[:100].lower(), n=len(results)))
+                await session.commit()
+        except Exception:
+            pass

app/bot/handlers/library.py

@@ -40,15 +40,16 @@ async def _show(target: Union[Message, CallbackQuery], text: str, markup) -> Non
         await target.answer(text, reply_markup=markup)
 
 
-async def _course_payload(user: User, session: AsyncSession, page: int):
-    items, total = await lib.courses_page(session, user, page)
+async def _course_payload(user: User, session: AsyncSession, page: int, scope: str = "c"):
+    items, total, more_elsewhere = await lib.courses_page(session, user, page, scope)
+    markup = kb.courses(user.lang, items, page, total, lib.PAGE_SIZE, scope, more_elsewhere)
     if not items:
-        return t(user.lang, "lib_no_courses"), None
-    return t(user.lang, "lib_courses"), kb.courses(user.lang, items, page, total, lib.PAGE_SIZE)
+        return t(user.lang, "lib_no_courses"), (markup if markup.inline_keyboard else None)
+    return t(user.lang, "lib_courses_all" if scope == "a" else "lib_courses"), markup
 
 
-async def _courses(target, user: User, session: AsyncSession, page: int) -> None:
-    text, markup = await _course_payload(user, session, page)
+async def _courses(target, user: User, session: AsyncSession, page: int, scope: str = "c") -> None:
+    text, markup = await _course_payload(user, session, page, scope)
     await _show(target, text, markup)
 
 
@@ -62,6 +63,11 @@ async def on_courses(call: CallbackQuery, user: User, session: AsyncSession) ->
     await _courses(call, user, session, int(call.data.split(":")[2]))
 
 
+@router.callback_query(F.data.startswith("lib:a:"))
+async def on_all_courses(call: CallbackQuery, user: User, session: AsyncSession) -> None:
+    await _courses(call, user, session, int(call.data.split(":")[2]), scope="a")
+
+
 @router.callback_query(F.data.startswith("lib:k:"))
 async def on_kinds(call: CallbackQuery, user: User, session: AsyncSession) -> None:
     course_id = int(call.data.split(":")[2])

app/bot/handlers/search.py

@@ -4,7 +4,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.bot.keyboards import library as kb
 from app.core.i18n import t
-from app.db.models import User
+from app.db.models import ActivityEvent, User
 from app.services import library as lib
 
 router = Router()
@@ -19,6 +19,7 @@ async def search_text(message: Message, user: User, session: AsyncSession) -> No
         await message.answer(t(user.lang, "search_none"))
         return
     courses, materials = await lib.search(session, query)
+    session.add(ActivityEvent(user_id=user.id, kind="search", ref=query[:100].lower(), n=len(courses) + len(materials)))
     if not courses and not materials:
         await message.answer(t(user.lang, "search_none"))
         return

app/bot/keyboards/library.py

@@ -30,11 +30,17 @@ def _nav(lang: str, prefix: str, page: int, total: int, size: int) -> list[Inlin
     return row
 
 
-def courses(lang: str, items: Iterable[Course], page: int, total: int, size: int) -> InlineKeyboardMarkup:
+def courses(
+    lang: str, items: Iterable[Course], page: int, total: int, size: int, scope: str = "c", more_elsewhere: bool = False
+) -> InlineKeyboardMarkup:
     rows = [[_btn(f"{course_icon(c)} {course_label(c)}", f"lib:k:{c.id}")] for c in items]
-    nav = _nav(lang, "lib:c", page, total, size)
+    nav = _nav(lang, f"lib:{scope}", page, total, size)
     if nav:
         rows.append(nav)
+    if scope == "a":
+        rows.append([_btn(t(lang, "btn_my_courses"), "lib:c:0")])
+    elif more_elsewhere:
+        rows.append([_btn(t(lang, "btn_more_courses"), "lib:a:0")])  # courses of the other stream
     return InlineKeyboardMarkup(inline_keyboard=rows)
 
 

app/bot/manual.py

@@ -169,6 +169,15 @@ def sections(f: dict) -> list[tuple[str, str, list[Cmd]]]:
                 "Offline marketing you can measure: each place has its own tag in /growth. The PNGs let you use "
                 "your own design (for example in Canva).",
                 "Test-scan the QR with a phone before printing and keep it at least 3 cm wide.", "Poster + QR for printing"),
+            Cmd("analyze", "/analyze",
+                "Your analytics screen with buttons: Overview, Growth (per day, funnel, sources, who is stuck), "
+                "Retention (daily actives, who comes back), Library (files, downloads, empty courses and the searches "
+                "that found NOTHING), Student files, Channel posts and Insights (what to do next, with the numbers).",
+                "Turns the bot's own data into decisions: what material to add, where students are lost, which posts "
+                "bring students. Every insight shows the numbers behind it.",
+                "Channel views and shares are only visible in the Telegram app (the Bot API does not give them to bots). "
+                "The bot records new channel posts and their reactions by itself; FORWARD an older text post to the bot to "
+                "add it. Small samples (under ~30 students) are only hints.", "Analytics: growth, retention, library, insights"),
             Cmd("announce", "/announce",
                 "Writes the channel announcement for the XP, levels and weekly prizes from your current settings, "
                 "shows a preview and offers a button to post it to the channel.",

app/db/models/__init__.py

@@ -1,3 +1,4 @@
+from app.db.models.analytics import ActivityEvent, ChannelPost
 from app.db.models.course import Course
 from app.db.models.group import BotGroup
 from app.db.models.material import Material
@@ -7,4 +8,4 @@ from app.db.models.submission import Submission
 from app.db.models.user import User
 from app.db.models.xp import SeasonResult, XpEvent
 
-__all__ = ["User", "Course", "Material", "Referral", "PendingMaterial", "XpEvent", "SeasonResult", "BotGroup", "Submission"]
+__all__ = ["User", "Course", "Material", "Referral", "PendingMaterial", "XpEvent", "SeasonResult", "BotGroup", "Submission", "ActivityEvent", "ChannelPost"]

app/db/models/analytics.py

@@ -0,0 +1,33 @@
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import BigInteger, DateTime, Integer, String, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.db.base import Base
+
+
+class ActivityEvent(Base):
+    """Small usage log that feeds /analyze: downloads and searches (so we learn what students look for)."""
+
+    __tablename__ = "activity_events"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    user_id: Mapped[int] = mapped_column(BigInteger, index=True)
+    kind: Mapped[str] = mapped_column(String(12), index=True)  # download | search | inline | find
+    ref: Mapped[str] = mapped_column(String(120))  # material id, or the search words
+    n: Mapped[int] = mapped_column(Integer, default=0)  # for searches: how many results were found
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), index=True)
+
+
+class ChannelPost(Base):
+    """A post in the main channel (seen live, or added by forwarding it to the bot)."""
+
+    __tablename__ = "channel_posts"
+
+    chat_id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=False)
+    message_id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=False)
+    posted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
+    preview: Mapped[Optional[str]] = mapped_column(String(300))
+    tags: Mapped[Optional[str]] = mapped_column(String(120))  # ?start= tags found in the post, comma separated
+    reactions: Mapped[Optional[int]] = mapped_column(Integer)  # None = unknown

app/db/repos/analytics.py

@@ -0,0 +1,97 @@
+"""Loads the plain data the analyzer needs (simple selects; all the maths is in services/analytics.py)."""
+from datetime import date, datetime, timedelta, timezone
+from typing import Optional
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ActivityEvent, ChannelPost, Course, Material, Submission, User, XpEvent
+from app.services import analytics as A
+
+
+def _utc(dt: Optional[datetime]) -> Optional[datetime]:
+    if dt is None:
+        return None
+    return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
+
+
+def _day(dt: datetime) -> date:
+    return _utc(dt).astimezone(A.ET).date()
+
+
+async def load(session: AsyncSession, now: Optional[datetime] = None) -> A.Snapshot:
+    now = now or datetime.now(timezone.utc)
+
+    users = [
+        A.UserRow(
+            id=uid, created=_utc(created), joined=_day(created), source=source or "before tracking",
+            verified=bool(passed), onboarded=year is not None, active_days=active or 0, stream=stream, missing=missing,
+        )
+        for uid, created, source, passed, year, active, stream, missing in (
+            await session.execute(
+                select(User.id, User.created_at, User.source, User.gate_passed, User.year, User.active_days, User.stream, User.gate_missing)
+            )
+        ).all()
+    ]
+
+    checkins: dict[int, set[date]] = {}
+    for uid, day in (await session.execute(select(XpEvent.user_id, XpEvent.day).where(XpEvent.kind == "checkin"))).all():
+        checkins.setdefault(uid, set()).add(date.fromisoformat(day))
+
+    since = now - timedelta(days=30)
+    events = [
+        A.EventRow(kind=kind, ref=ref or "", n=n or 0, day=_day(created), user_id=uid)
+        for kind, ref, n, created, uid in (
+            await session.execute(
+                select(ActivityEvent.kind, ActivityEvent.ref, ActivityEvent.n, ActivityEvent.created_at, ActivityEvent.user_id)
+                .where(ActivityEvent.created_at >= since)
+            )
+        ).all()
+    ]
+
+    materials = [
+        A.MaterialRow(id=i, course_id=cid, title=title or "", downloads=dl or 0, status=status, kind=kind)
+        for i, cid, title, dl, status, kind in (
+            await session.execute(select(Material.id, Material.course_id, Material.title, Material.downloads, Material.status, Material.kind))
+        ).all()
+    ]
+    courses = {cid: name for cid, name in (await session.execute(select(Course.id, Course.name))).all()}
+    subs = [
+        A.SubRow(status=status, user_id=uid, created=_utc(created), reviewed=_utc(reviewed))
+        for status, uid, created, reviewed in (
+            await session.execute(select(Submission.status, Submission.user_id, Submission.created_at, Submission.reviewed_at))
+        ).all()
+    ]
+    posts = [
+        A.PostRow(
+            message_id=mid, posted=_utc(posted), preview=preview or "",
+            tags=[t for t in (tags or "").split(",") if t], reactions=reactions,
+        )
+        for mid, posted, preview, tags, reactions in (
+            await session.execute(
+                select(ChannelPost.message_id, ChannelPost.posted_at, ChannelPost.preview, ChannelPost.tags, ChannelPost.reactions)
+                .order_by(ChannelPost.posted_at.desc()).limit(30)
+            )
+        ).all()
+    ]
+    return A.Snapshot(now=now, users=users, checkins=checkins, events=events, materials=materials, courses=courses, subs=subs, posts=posts)
+
+
+async def save_post(
+    session: AsyncSession, *, chat_id: int, message_id: int, posted_at: datetime, text: str, reactions: Optional[int] = None
+) -> None:
+    post = await session.get(ChannelPost, (chat_id, message_id))
+    preview = " ".join((text or "").split())[:300] or "(media post)"
+    tags = ",".join(A.extract_tags(text or ""))[:120]
+    if post:
+        post.preview, post.tags = preview, tags
+    else:
+        session.add(ChannelPost(chat_id=chat_id, message_id=message_id, posted_at=posted_at, preview=preview, tags=tags, reactions=reactions))
+    await session.flush()
+
+
+async def set_reactions(session: AsyncSession, chat_id: int, message_id: int, total: int) -> None:
+    post = await session.get(ChannelPost, (chat_id, message_id))
+    if post:
+        post.reactions = total
+        await session.flush()

app/db/repos/library.py

@@ -46,6 +46,20 @@ async def list_courses(
     return list(res.scalars())
 
 
+async def count_all_courses(session: AsyncSession, only_with_files: bool = False) -> int:
+    stmt = select(func.count()).select_from(Course)
+    if only_with_files:
+        stmt = stmt.where(_has_files())
+    return int((await session.execute(stmt)).scalar_one())
+
+
+async def list_all_courses(session: AsyncSession, offset: int, limit: int, only_with_files: bool = False) -> list[Course]:
+    stmt = select(Course).order_by(Course.semester, Course.code, Course.name).offset(offset).limit(limit)
+    if only_with_files:
+        stmt = stmt.where(_has_files())
+    return list((await session.execute(stmt)).scalars())
+
+
 async def get_course(session: AsyncSession, course_id: int) -> Optional[Course]:
     return await session.get(Course, course_id)
 

app/services/analytics.py

@@ -0,0 +1,450 @@
+"""The analyzer's brain. Pure functions over plain data (no database, no Telegram), so every number
+is unit-tested. The loading of the data lives in app/db/repos/analytics.py.
+
+Honesty rules built in: small samples are labelled, a correlation is never called a cause, and every
+insight shows the numbers it is based on."""
+import html
+import re
+from collections import Counter
+from dataclasses import dataclass, field
+from datetime import date, datetime, timedelta, timezone
+from statistics import median
+from typing import Optional
+
+ET = timezone(timedelta(hours=3))
+SPARSE = 3  # a course with fewer files than this is "short of material"
+
+
+# ------------------------------------------------------------------ plain data
+@dataclass(frozen=True)
+class UserRow:
+    id: int
+    created: datetime  # UTC
+    joined: date  # Ethiopian date of joining
+    source: str
+    verified: bool
+    onboarded: bool
+    active_days: int
+    stream: Optional[str] = None
+    missing: Optional[str] = None  # join-gate places last missing, e.g. "dbucentralforum"
+
+
+@dataclass(frozen=True)
+class EventRow:
+    kind: str  # download | search | inline | find
+    ref: str
+    n: int
+    day: date
+    user_id: int
+
+
+@dataclass(frozen=True)
+class MaterialRow:
+    id: int
+    course_id: Optional[int]
+    title: str
+    downloads: int
+    status: str
+    kind: str
+
+
+@dataclass(frozen=True)
+class SubRow:
+    status: str
+    user_id: int
+    created: datetime
+    reviewed: Optional[datetime]
+
+
+@dataclass(frozen=True)
+class PostRow:
+    message_id: int
+    posted: datetime  # UTC
+    preview: str
+    tags: list[str]
+    reactions: Optional[int]
+
+
+@dataclass
+class Snapshot:
+    now: datetime
+    users: list[UserRow]
+    checkins: dict[int, set[date]]
+    events: list[EventRow]
+    materials: list[MaterialRow]
+    courses: dict[int, str]
+    subs: list[SubRow]
+    posts: list[PostRow] = field(default_factory=list)
+
+    @property
+    def today(self) -> date:
+        return self.now.astimezone(ET).date()
+
+
+# ------------------------------------------------------------------ small helpers
+def pct(part: int, whole: int) -> int:
+    return round(100 * part / whole) if whole else 0
+
+
+def bar(percent: float, width: int = 10) -> str:
+    filled = max(0, min(width, round(percent / 100 * width)))
+    return "▓" * filled + "░" * (width - filled)
+
+
+def esc(text: str) -> str:
+    return html.escape(text or "", quote=False)
+
+
+def extract_tags(text: str) -> list[str]:
+    """?start= tags inside a post: 'ad1', 'dorm5' (numbers are personal invite links, ignored)."""
+    tags = re.findall(r"[?&]start=([A-Za-z0-9_]{1,20})", text or "")
+    return sorted({t.lower() for t in tags if not t.isdigit()})
+
+
+# ------------------------------------------------------------------ growth + funnel
+def per_day(users: list[UserRow], today: date, days: int = 14) -> list[tuple[date, int]]:
+    counts = Counter(u.joined for u in users)
+    return [(today - timedelta(days=i), counts.get(today - timedelta(days=i), 0)) for i in range(days - 1, -1, -1)]
+
+
+def funnel(users: list[UserRow]) -> dict:
+    n = len(users)
+    verified = sum(u.verified for u in users)
+    onboarded = sum(u.onboarded for u in users)
+    active3 = sum(u.active_days >= 3 for u in users)
+    return dict(joined=n, verified=verified, onboarded=onboarded, active3=active3)
+
+
+def biggest_leak(f: dict) -> Optional[tuple[str, str, int, int]]:
+    """The stage with the largest loss between consecutive steps: (from, to, people lost, % lost)."""
+    steps = [("joined", "in channel + forum", f["joined"], f["verified"]),
+             ("in channel + forum", "finished setup", f["verified"], f["onboarded"])]
+    steps = [(a, b, x - y, pct(x - y, x)) for a, b, x, y in steps if x > 0]
+    return max(steps, key=lambda s: s[2]) if steps and max(s[2] for s in steps) > 0 else None
+
+
+def source_table(users: list[UserRow]) -> list[dict]:
+    groups: dict[str, list[UserRow]] = {}
+    for u in users:
+        groups.setdefault(u.source, []).append(u)
+    rows = []
+    for source, members in groups.items():
+        f = funnel(members)
+        rows.append(dict(source=source, **f, setup_pct=pct(f["onboarded"], f["joined"])))
+    return sorted(rows, key=lambda r: -r["joined"])
+
+
+def stuck_by_missing(users: list[UserRow]) -> list[tuple[str, int]]:
+    c = Counter((u.missing or "not recorded yet") for u in users if not u.verified)
+    return c.most_common()
+
+
+# ------------------------------------------------------------------ activity + retention
+def dau_wau(checkins: dict[int, set[date]], today: date) -> dict:
+    week = [today - timedelta(days=i) for i in range(6, -1, -1)]
+    dau = [(d, sum(1 for days in checkins.values() if d in days)) for d in week]
+    wau = sum(1 for days in checkins.values() if any(d in days for d in week))
+    last_full = dau[-2][1] if len(dau) >= 2 else 0  # yesterday is complete, today is not
+    return dict(dau=dau, wau=wau, stickiness=pct(last_full, wau))
+
+
+def retention(users: list[UserRow], checkins: dict[int, set[date]], today: date, cohorts: int = 7) -> dict:
+    """Share of each day's new students who were active exactly N days later. Check-ins only exist from the day
+    activity tracking started, so earlier cohorts are 'n/a' instead of a misleading 0. A day that is not over
+    yet (today) is never counted."""
+    all_days = [d for days in checkins.values() for d in days]
+    start = min(all_days) if all_days else None
+    rows = []
+    for offset in range(cohorts, 0, -1):
+        d = today - timedelta(days=offset)
+        members = [u for u in users if u.joined == d]
+        if not members:
+            continue
+        cells = {}
+        for n in (1, 3, 7):
+            target = d + timedelta(days=n)
+            if target >= today or start is None or target < start:
+                cells[n] = None  # not reached yet, or before we tracked activity
+            else:
+                cells[n] = pct(sum(1 for u in members if target in checkins.get(u.id, set())), len(members))
+        rows.append(dict(day=d, size=len(members), **{f"d{n}": v for n, v in cells.items()}))
+    eligible = [u for u in users if u.joined < today]
+    came_back = sum(1 for u in eligible if any(day > u.joined for day in checkins.get(u.id, set())))
+    return dict(rows=rows, tracking_start=start, came_back=came_back, eligible=len(eligible))
+
+
+# ------------------------------------------------------------------ library + searches
+def library(snap: Snapshot) -> dict:
+    ok = [m for m in snap.materials if m.status == "ok"]
+    per_course = Counter(m.course_id for m in ok if m.course_id)
+    empty = [name for cid, name in snap.courses.items() if per_course.get(cid, 0) == 0]
+    sparse = [name for cid, name in snap.courses.items() if 0 < per_course.get(cid, 0) < SPARSE]
+    week_start = snap.today - timedelta(days=6)
+    downloads_week = [e for e in snap.events if e.kind == "download" and e.day >= week_start]
+    since = snap.today - timedelta(days=13)
+    searches = [e for e in snap.events if e.kind in ("search", "inline", "find") and e.day >= since and len(e.ref) >= 2]
+    missed = Counter(e.ref for e in searches if e.n == 0)
+    found = Counter(e.ref for e in searches if e.n > 0)
+    top_files = sorted(ok, key=lambda m: -m.downloads)[:5]
+    return dict(
+        files=len(ok), kinds=Counter(m.kind for m in ok), courses=len(snap.courses), empty=empty, sparse=sparse,
+        downloads_total=sum(m.downloads for m in ok), downloads_week=len(downloads_week),
+        downloaders_week=len({e.user_id for e in downloads_week}), top_files=top_files,
+        by_course=[(snap.courses.get(cid, "?"), n) for cid, n in per_course.most_common(5)],
+        searches=len(searches), missed_total=sum(missed.values()), missed=missed.most_common(8), found=found.most_common(5),
+    )
+
+
+# ------------------------------------------------------------------ contributions + channel posts
+def contributions(snap: Snapshot) -> dict:
+    by = Counter(s.status for s in snap.subs)
+    done = [s for s in snap.subs if s.reviewed]
+    hours = [(s.reviewed - s.created).total_seconds() / 3600 for s in done]
+    pending = [s for s in snap.subs if s.status == "pending"]
+    oldest = max(((snap.now - s.created).total_seconds() / 3600 for s in pending), default=0.0)
+    top = Counter(s.user_id for s in snap.subs if s.status == "approved").most_common(3)
+    return dict(
+        pending=by["pending"], approved=by["approved"], rejected=by["rejected"],
+        approval=pct(by["approved"], by["approved"] + by["rejected"]),
+        avg_hours=round(sum(hours) / len(hours), 1) if hours else None, oldest_pending_hours=round(oldest, 1), top=top,
+    )
+
+
+def channel_posts(snap: Snapshot, window_hours: int = 6) -> list[dict]:
+    """For each post: students who joined the bot in the hours after it, and students who came through a tag
+    written in the post. NOTE: a correlation, not proof; close posts share the same joiners."""
+    rows = []
+    for p in snap.posts:
+        end = p.posted + timedelta(hours=window_hours)
+        soon = sum(1 for u in snap.users if p.posted <= u.created < end)
+        tagged = sum(1 for u in snap.users if u.source in p.tags)
+        rows.append(dict(message_id=p.message_id, posted=p.posted, preview=p.preview, tags=p.tags,
+                         reactions=p.reactions, soon=soon, tagged=tagged))
+    return sorted(rows, key=lambda r: (-r["tagged"], -r["soon"]))
+
+
+# ------------------------------------------------------------------ insights (rule based, numbers shown)
+@dataclass
+class Insight:
+    priority: int  # 1 = most important
+    title: str
+    evidence: str
+    action: str
+
+
+def insights(snap: Snapshot) -> list[Insight]:
+    out: list[Insight] = []
+    users, today = snap.users, snap.today
+    if not users:
+        return out
+    f = funnel(users)
+    leak = biggest_leak(f)
+
+    stuck = [u for u in users if not u.verified]
+    if f["joined"] and pct(len(stuck), f["joined"]) >= 15:
+        top = [(m, n) for m, n in stuck_by_missing(users)][:3]
+        detail = ", ".join(f"{n} missing {m}" if m != "not recorded yet" else f"{n} not recorded yet" for m, n in top)
+        out.append(Insight(1, "Students stop at the join step",
+                           f"{len(stuck)} of {f['joined']} ({pct(len(stuck), f['joined'])}%) never got through: {detail}.",
+                           "Check /gatecheck. If most miss the forum, consider making it optional or send one reminder."))
+    elif leak and leak[3] >= 10:
+        out.append(Insight(2, f"Biggest drop: {leak[0]} → {leak[1]}", f"{leak[2]} people ({leak[3]}%) are lost here.",
+                           "Make this step shorter or explain why it matters."))
+
+    verified_not_setup = f["verified"] - f["onboarded"]
+    if f["verified"] and pct(verified_not_setup, f["verified"]) >= 10:
+        out.append(Insight(2, "Some verified students did not finish setup",
+                           f"{verified_not_setup} of {f['verified']} ({pct(verified_not_setup, f['verified'])}%) stopped at stream/year.",
+                           "Check that the two setup questions are clear."))
+
+    ret = retention(users, snap.checkins, today)
+    d1 = [(r["d1"], r["size"]) for r in ret["rows"] if r["d1"] is not None]
+    if d1 and sum(s for _, s in d1) >= 20:
+        total = sum(s for _, s in d1)
+        avg = round(sum(v * s for v, s in d1) / total)
+        if avg < 20:
+            out.append(Insight(1, "Few students come back the next day", f"Day-1 return is {avg}% (over {total} students).",
+                               "Give a reason to return: new files weekly, a daily XP nudge, exam-week reminders."))
+    elif ret["tracking_start"] is None or snap.checkins and not d1:
+        since = ret["tracking_start"].isoformat() if ret["tracking_start"] else "now"
+        out.append(Insight(4, "Not enough days of data for retention yet",
+                           f"Activity has been tracked since {since}.", "Check again in a few days."))
+
+    lib = library(snap)
+    if lib["missed"] and lib["missed"][0][1] >= 2:
+        q, n = lib["missed"][0]
+        out.append(Insight(1, "Students search for things you do not have",
+                           f"{lib['missed_total']} searches in 14 days found nothing; top: “{q}” ({n} times).",
+                           "Add material for these words (see Library & searches), then they find it next time."))
+    if snap.courses and (len(lib["empty"]) / len(snap.courses)) >= 0.4:
+        out.append(Insight(2, "Many courses have no file", f"{len(lib['empty'])} of {len(snap.courses)} courses are empty.",
+                           "Use the 🎯 wanted-courses prompt in announcements; ask seniors for past exams."))
+
+    con = contributions(snap)
+    if con["pending"] and con["oldest_pending_hours"] >= 24:
+        out.append(Insight(2, "Student files are waiting", f"{con['pending']} waiting; the oldest {con['oldest_pending_hours']:.0f} h.",
+                           "Run /queue. Slow reviews discourage contributors."))
+
+    days = [n for _, n in per_day(users, today, 14)]
+    nonzero = [n for n in days if n]
+    if len(nonzero) >= 3 and max(days) >= 3 * max(1, median(days)):
+        out.append(Insight(3, "Growth comes from single pushes",
+                           f"Best day {max(days)} new students vs a typical day of {median(days):g}.",
+                           "Keep a steady posting rhythm; one-off pushes fade within 2-3 days."))
+
+    sources = [r for r in source_table(users) if r["joined"] >= 10 and r["source"] != "before tracking"]
+    if len(sources) >= 2:
+        best, worst = max(sources, key=lambda r: r["setup_pct"]), min(sources, key=lambda r: r["setup_pct"])
+        if best["setup_pct"] - worst["setup_pct"] >= 15:
+            out.append(Insight(3, "Sources differ in quality",
+                               f"“{best['source']}”: {best['setup_pct']}% finish setup vs “{worst['source']}”: {worst['setup_pct']}% "
+                               f"({best['joined']} and {worst['joined']} students, still small samples).",
+                               "Repeat what works in the better source."))
+    referral = sum(1 for u in users if u.source == "referral")
+    if len(users) >= 50 and pct(referral, len(users)) < 5:
+        out.append(Insight(3, "Few students join by invite", f"{referral} of {len(users)} ({pct(referral, len(users))}%) came from an invite.",
+                           "Mention “Invite friends” in channel posts; the share button is one tap."))
+
+    posts = channel_posts(snap)
+    if posts and (posts[0]["tagged"] or posts[0]["soon"]):
+        p = posts[0]
+        out.append(Insight(4, "Your best channel post so far",
+                           f"“{esc(p['preview'][:50])}” → {p['tagged']} via its tag, {p['soon']} new students within 6 h "
+                           "(a correlation, not proof).", "Write more posts like this one."))
+    return sorted(out, key=lambda i: i.priority)
+
+
+# ------------------------------------------------------------------ text for each screen
+def _fmt_day(d: date) -> str:
+    return d.strftime("%m-%d")
+
+
+def overview(snap: Snapshot) -> str:
+    users, today = snap.users, snap.today
+    f = funnel(users)
+    new_today = sum(1 for u in users if u.joined == today)
+    wau = dau_wau(snap.checkins, today)
+    ret = retention(users, snap.checkins, today)
+    lib = library(snap)
+    con = contributions(snap)
+    lines = [
+        f"📊 <b>Analytics</b> · {snap.now.astimezone(ET).strftime('%d %b %H:%M')} (Ethiopian time)",
+        f"👥 <b>{f['joined']}</b> students (+{new_today} today) · in channel+forum <b>{f['verified']}</b> ({pct(f['verified'], f['joined'])}%) "
+        f"· finished setup <b>{f['onboarded']}</b> ({pct(f['onboarded'], f['joined'])}%)",
+        f"🔁 Active this week <b>{wau['wau']}</b> · came back at least once <b>{ret['came_back']}</b> of {ret['eligible']} "
+        f"({pct(ret['came_back'], ret['eligible'])}%)",
+        f"📚 <b>{lib['files']}</b> files · {lib['downloads_total']} downloads ({lib['downloads_week']} this week) · "
+        f"{len(lib['empty'])} courses empty",
+        f"🔍 {lib['searches']} searches in 14 days · found nothing: {lib['missed_total']} ({pct(lib['missed_total'], lib['searches'])}%)",
+        f"📤 Student files: {con['pending']} waiting · {con['approved']} approved · {con['rejected']} rejected",
+    ]
+    ins = insights(snap)[:3]
+    if ins:
+        lines.append("\n💡 <b>What matters most now</b>")
+        lines += [f"{i}. <b>{esc(x.title)}</b>: {x.evidence}" for i, x in enumerate(ins, 1)]
+    lines.append("\nTap a button for details. Small samples (under ~30 people) are only hints.")
+    return "\n".join(lines)
+
+
+def section_growth(snap: Snapshot) -> str:
+    days = per_day(snap.users, snap.today, 14)
+    top = max((n for _, n in days), default=0) or 1
+    lines = ["👥 <b>Growth and sources</b>", "<b>New students per day</b>"]
+    lines += [f"{_fmt_day(d)}  {bar(100 * n / top, 12)} {n}" for d, n in days]
+    f = funnel(snap.users)
+    lines.append(f"\n<b>Funnel</b> (everyone ever)\njoined <b>{f['joined']}</b> → in channel + forum <b>{f['verified']}</b> → "
+                 f"finished setup <b>{f['onboarded']}</b> → used the bot on 3+ days <b>{f['active3']}</b>")
+    lines.append("\n<b>By source</b> (joined → in channel+forum → finished setup → 3+ days)")
+    for r in source_table(snap.users)[:10]:
+        small = " ·small sample" if r["joined"] < 10 else ""
+        lines.append(f"• <b>{esc(r['source'])}</b>: {r['joined']} → {r['verified']} → {r['onboarded']} → {r['active3']} "
+                     f"({r['setup_pct']}% setup){small}")
+    stuck = stuck_by_missing(snap.users)
+    if stuck:
+        lines.append("\n<b>Stuck at the join step</b>")
+        lines += [f"• {n}: {('missing ' + m) if m != 'not recorded yet' else m}" for m, n in stuck[:5]]
+    return "\n".join(lines)
+
+
+def section_retention(snap: Snapshot) -> str:
+    ret = retention(snap.users, snap.checkins, snap.today)
+    w = dau_wau(snap.checkins, snap.today)
+    lines = ["🔁 <b>Activity and retention</b>", "<b>Active students per day</b> (opened the bot)"]
+    peak = max((n for _, n in w["dau"]), default=0) or 1
+    lines += [f"{_fmt_day(d)}  {bar(100 * n / peak, 12)} {n}" for d, n in w["dau"]]
+    lines.append(f"\nWeekly active: <b>{w['wau']}</b> · yesterday's active ÷ weekly active: <b>{w['stickiness']}%</b>")
+    lines.append(f"Came back at least once: <b>{ret['came_back']}</b> of {ret['eligible']} ({pct(ret['came_back'], ret['eligible'])}%)")
+    lines.append("\n<b>Return rate by signup day</b> (share active again on day 1 / 3 / 7)")
+    cell = lambda v: "n/a" if v is None else f"{v}%"  # noqa: E731
+    if ret["rows"]:
+        lines += [f"{_fmt_day(r['day'])} (n={r['size']}): {cell(r['d1'])} / {cell(r['d3'])} / {cell(r['d7'])}" for r in ret["rows"]]
+    else:
+        lines.append("no signups in the last 7 days")
+    since = ret["tracking_start"].isoformat() if ret["tracking_start"] else "not yet"
+    lines.append(f"\n“n/a” = that day has not come yet, or it is before activity tracking began ({since}).")
+    return "\n".join(lines)
+
+
+def section_library(snap: Snapshot) -> str:
+    lib = library(snap)
+    lines = ["📚 <b>Library and searches</b>",
+             f"{lib['files']} files in {lib['courses']} courses · kinds: " + ", ".join(f"{k} {n}" for k, n in lib["kinds"].most_common()),
+             f"Downloads: {lib['downloads_total']} all time · {lib['downloads_week']} this week by {lib['downloaders_week']} students"]
+    if lib["top_files"]:
+        lines.append("\n<b>Most downloaded</b>")
+        lines += [f"• {esc(m.title[:45])} ({m.downloads})" for m in lib["top_files"] if m.downloads]
+    if lib["by_course"]:
+        lines.append("\n<b>Courses with the most files</b>: " + ", ".join(f"{esc(n[:25])} {c}" for n, c in lib["by_course"]))
+    lines.append(f"\n<b>No file at all</b> ({len(lib['empty'])}): " + (", ".join(esc(n[:28]) for n in lib["empty"][:8]) or "none")
+                 + ("…" if len(lib["empty"]) > 8 else ""))
+    if lib["sparse"]:
+        lines.append(f"<b>Short of files</b> (under {SPARSE}): " + ", ".join(esc(n[:28]) for n in lib["sparse"][:8]))
+    lines.append(f"\n🔍 <b>Searches (14 days)</b>: {lib['searches']} · found nothing {lib['missed_total']} ({pct(lib['missed_total'], lib['searches'])}%)")
+    if lib["missed"]:
+        lines.append("<b>Searched but NOT found</b> (add these!)")
+        lines += [f"• “{esc(q)}” × {n}" for q, n in lib["missed"]]
+    if lib["found"]:
+        lines.append("<b>Most searched and found</b>: " + ", ".join(f"“{esc(q)}” {n}" for q, n in lib["found"]))
+    return "\n".join(lines)
+
+
+def section_contrib(snap: Snapshot) -> str:
+    c = contributions(snap)
+    lines = ["📤 <b>Student contributions</b>",
+             f"Waiting <b>{c['pending']}</b> · approved <b>{c['approved']}</b> · rejected <b>{c['rejected']}</b> · approval rate <b>{c['approval']}%</b>"]
+    if c["avg_hours"] is not None:
+        lines.append(f"Average time to review: {c['avg_hours']} h")
+    if c["pending"]:
+        lines.append(f"Oldest waiting: {c['oldest_pending_hours']:.0f} h")
+    if c["top"]:
+        lines.append("Top contributors (user id · approved): " + ", ".join(f"{u} · {n}" for u, n in c["top"]))
+    if not snap.subs:
+        lines.append("No student files yet.")
+    return "\n".join(lines)
+
+
+def section_channel(snap: Snapshot) -> str:
+    rows = channel_posts(snap)
+    lines = ["📢 <b>Channel posts</b>"]
+    if not rows:
+        return "\n".join(lines + [
+            "No posts recorded yet. The bot records new posts of the channel by itself. To add an older post, "
+            "FORWARD it (text posts) to this bot as admin."])
+    lines.append("Sorted by students who came through the post's own link tag, then by new students within 6 hours.")
+    for r in rows[:8]:
+        react = "?" if r["reactions"] is None else r["reactions"]
+        tag = f" · tag {', '.join(r['tags'])}: <b>{r['tagged']}</b>" if r["tags"] else ""
+        lines.append(f"• {r['posted'].astimezone(ET).strftime('%m-%d %H:%M')} “{esc(r['preview'][:40])}”\n"
+                     f"   6h: <b>{r['soon']}</b> new{tag} · reactions: {react}")
+    lines.append("\nA post and a sign-up close in time is a hint, not proof. Reactions are only known for posts seen live.")
+    return "\n".join(lines)
+
+
+def section_insights(snap: Snapshot) -> str:
+    ins = insights(snap)
+    if not ins:
+        return "💡 <b>Insights</b>\nNothing stands out yet; there is not enough data."
+    lines = ["💡 <b>Insights and what to do</b> (from your numbers, most important first)"]
+    for i, x in enumerate(ins, 1):
+        lines.append(f"\n<b>{i}. {esc(x.title)}</b>\n{x.evidence}\n➡️ {esc(x.action)}")
+    return "\n".join(lines)

app/services/library.py

@@ -19,11 +19,19 @@ def tokenize(query: str, limit: int = 5) -> list[str]:
     return [t for t in _STRIP.sub(" ", query.lower()).split() if t][:limit]
 
 
-async def courses_page(session: AsyncSession, user: User, page: int) -> tuple[list[Course], int]:
+async def courses_page(session: AsyncSession, user: User, page: int, scope: str = "c") -> tuple[list[Course], int, bool]:
+    """(courses on this page, total, are there more courses in other streams?).
+    scope "c" = the student's own stream and year, "a" = every course that has files."""
     only = get_settings().hide_empty_courses
+    if scope == "a":
+        return (
+            await repo.list_all_courses(session, page * PAGE_SIZE, PAGE_SIZE, only),
+            await repo.count_all_courses(session, only),
+            False,
+        )
     total = await repo.count_courses(session, user.stream, user.year, only)
     courses = await repo.list_courses(session, user.stream, user.year, page * PAGE_SIZE, PAGE_SIZE, only)
-    return courses, total
+    return courses, total, await repo.count_all_courses(session, only) > total
 
 
 async def kinds_of(session: AsyncSession, course_id: int) -> tuple[Optional[Course], dict[str, int]]:

locales/en.json

@@ -47,7 +47,7 @@
   "season_win": "🏅 You finished #{rank} last week ({xp} XP)! Congratulations.",
   "season_win_prize": "\n💰 Prize: {birr} birr airtime card. The admin will contact you on Telegram.",
   "season_raffle": "🎟 You won the weekly raffle!",
-  "rules_text": "📜 <b>Rules</b>\n\n<b>How to earn XP</b>\n• Daily check-in: 3-7 XP (grows with your streak)\n• Downloading a file: +1 (up to 3 a day)\n• Inviting a friend: +5 when they join, +30 once they are an active student (3+ days), max 5 a week\n• The admin may grant extra XP for useful contributions\n• Automatic sources are capped at {cap} XP per day\n\n<b>Weekly contest</b>\nStarts Monday, ends Sunday (Ethiopian time).\nWhen prizes are on, to compete you need {days}+ active days that week, {xp}+ XP, to stay in the channel, and a healthy trust score.\n\n<b>Fairness</b>\n• Every XP is recorded in an unchangeable ledger; the same action is never paid twice.\n• Suspicious activity (fake accounts, unusual mass inviting) is reviewed by the admin; points may be removed or accounts banned.\n• Prizes may change; the admin's decision is final.\n• No payment is needed to take part.",
+  "rules_text": "📜 <b>Rules</b>\n\n<b>How to earn XP</b>\n• Daily check-in: 3-7 XP (grows with your streak)\n• Downloading a file: +1 (up to 3 a day)\n• Inviting a friend: +5 when they join, +30 once they are an active student (3+ days), max 5 a week\n• The admin may grant extra XP for useful contributions\n• Automatic sources are capped at {cap} XP per day\n\n<b>Weekly contest</b>\nStarts Monday, ends Sunday (Ethiopian time).\nWhen prizes are on, to compete you need {days}+ active days that week, {xp}+ XP, to stay in the channel, and a healthy trust score.\n\n<b>Fairness</b>\n• Every XP is recorded in an unchangeable ledger; the same action is never paid twice.\n• Suspicious activity (fake accounts, unusual mass inviting) is reviewed by the admin; points may be removed or accounts banned.\n• Prizes may change; the admin's decision is final.\n• We keep simple usage data (what is searched and downloaded) to decide which material to add. Only admins see it, and your messages are never shared.\n• No payment is needed to take part.",
   "level_1": "Rookie",
   "level_2": "Starter",
   "level_3": "Sharp",
@@ -105,5 +105,8 @@
   "badge_bronze": "🥉 Contributor",
   "badge_silver": "🥈 Silver contributor",
   "badge_gold": "🥇 Gold contributor",
-  "help_student": "📖 <b>How to use this bot</b>\n\n📚 <b>Library</b>: pick a course and get slides, modules and past exams. Or just type a course name to search.\n📤 <b>Contribute a file</b>: share your material; you earn XP when it is approved.\n👤 <b>Profile</b>: your level, XP, streak and badge.\n🏆 <b>Ranking</b>: the weekly leaderboard and prizes.\n👥 <b>Invite friends</b>: a personal link, one tap to share. You earn XP when they join and when they become active.\n🧮 <b>GPA calculator</b>: coming soon.\n\nAnywhere in Telegram you can also type @{bot} followed by a course name to share a file instantly.\nRules of XP and prizes: /rules"
+  "help_student": "📖 <b>How to use this bot</b>\n\n📚 <b>Library</b>: pick a course and get slides, modules and past exams. Or just type a course name to search.\n📤 <b>Contribute a file</b>: share your material; you earn XP when it is approved.\n👤 <b>Profile</b>: your level, XP, streak and badge.\n🏆 <b>Ranking</b>: the weekly leaderboard and prizes.\n👥 <b>Invite friends</b>: a personal link, one tap to share. You earn XP when they join and when they become active.\n🧮 <b>GPA calculator</b>: coming soon.\n\nAnywhere in Telegram you can also type @{bot} followed by a course name to share a file instantly.\nRules of XP and prizes: /rules",
+  "btn_more_courses": "🔀 Courses of other streams",
+  "btn_my_courses": "↩️ My courses",
+  "lib_courses_all": "📚 <b>All courses</b>\nEvery course that has files, from all streams 🔍"
 }

tests/test_logic.py

@@ -510,6 +510,130 @@ def test_members_who_never_saw_the_gate_still_count():
     assert "SET gate_passed = TRUE" in (root / "app/db/migrate.py").read_text(encoding="utf-8")
 
 
+def _snapshot():
+    from datetime import datetime, timedelta, timezone
+
+    from app.services import analytics as A
+
+    ET = A.ET
+    now = datetime(2026, 10, 22, 12, 0, tzinfo=timezone.utc)  # 15:00 Ethiopian time, 22 Oct
+    today = now.astimezone(ET).date()
+
+    def user(i, days_ago, source, verified, onboarded, active=0, missing=None):
+        created = now - timedelta(days=days_ago, hours=1)
+        return A.UserRow(id=i, created=created, joined=created.astimezone(ET).date(), source=source, verified=verified,
+                         onboarded=onboarded, active_days=active, missing=missing)
+
+    users = (
+        [user(i, 5, "organic", True, True, 3) for i in range(1, 7)]  # 6 joined 5 days ago, all fine
+        + [user(i, 5, "organic", False, False, 0, "dbucentralforum") for i in range(7, 13)]  # 6 stuck: forum missing
+        + [user(i, 1, "ad1", True, True, 1) for i in range(13, 19)]  # 6 joined yesterday
+        + [user(i, 0, "ad1", True, False) for i in range(19, 21)]  # 2 verified today, did not finish setup
+    )
+    d = lambda n: today - timedelta(days=n)  # noqa: E731
+    checkins = {i: {d(5), d(4)} for i in range(1, 4)}  # 3 of the first 12 came back the next day
+    checkins.update({i: {d(5)} for i in range(4, 7)})
+    checkins.update({i: {d(1), d(0)} for i in range(13, 17)})
+    events = (
+        [A.EventRow("search", "physics", 3, d(1), 1)] * 3
+        + [A.EventRow("search", "chemistry module", 0, d(1), 2)] * 4
+        + [A.EventRow("inline", "chemistry module", 0, d(0), 3)]
+        + [A.EventRow("search", "x", 0, d(0), 4)]  # too short to count
+        + [A.EventRow("download", "1", 1, d(0), 1), A.EventRow("download", "1", 1, d(2), 2)]
+    )
+    materials = [A.MaterialRow(1, 10, "Physics module", 7, "ok", "note"), A.MaterialRow(2, 10, "Physics exam", 2, "ok", "exam"),
+                 A.MaterialRow(3, 11, "Psych", 1, "ok", "note"), A.MaterialRow(4, 12, "Hidden", 9, "pending", "note")]
+    courses = {10: "General Physics", 11: "General Psychology", 12: "General Chemistry", 13: "Entrepreneurship"}
+    subs = [A.SubRow("approved", 5, now - timedelta(hours=30), now - timedelta(hours=28)),
+            A.SubRow("rejected", 6, now - timedelta(hours=10), now - timedelta(hours=6)),
+            A.SubRow("pending", 7, now - timedelta(hours=40), None)]
+    posts = [A.PostRow(1, now - timedelta(days=1, hours=3), "Big news t.me/Bot?start=ad1", ["ad1"], 12),
+             A.PostRow(2, now - timedelta(days=3), "meme", [], None)]
+    return A.Snapshot(now=now, users=users, checkins=checkins, events=events, materials=materials, courses=courses, subs=subs, posts=posts)
+
+
+def test_analytics_numbers():
+    from datetime import timedelta
+
+    from app.services import analytics as A
+
+    s = _snapshot()
+    f = A.funnel(s.users)
+    assert f == dict(joined=20, verified=14, onboarded=12, active3=6)
+    assert A.biggest_leak(f) == ("joined", "in channel + forum", 6, 30)
+    table = {r["source"]: r for r in A.source_table(s.users)}
+    assert table["organic"]["joined"] == 12 and table["organic"]["setup_pct"] == 50
+    assert table["ad1"]["joined"] == 8 and table["ad1"]["onboarded"] == 6 and table["ad1"]["setup_pct"] == 75
+    assert A.stuck_by_missing(s.users) == [("dbucentralforum", 6)]
+    days = A.per_day(s.users, s.today, 7)
+    assert len(days) == 7 and days[-1] == (s.today, 2) and days[-2] == (s.today - timedelta(days=1), 6)
+    assert sum(n for _, n in days) == 20
+
+
+def test_analytics_retention_and_activity():
+    from datetime import timedelta
+
+    from app.services import analytics as A
+
+    s = _snapshot()
+    ret = A.retention(s.users, s.checkins, s.today)
+    old = next(r for r in ret["rows"] if r["day"] == s.today - timedelta(days=5))
+    assert old["size"] == 12 and old["d1"] == 25  # 3 of the 12 were active the next day
+    assert old["d7"] is None  # day 7 has not come yet
+    yesterday = next(r for r in ret["rows"] if r["day"] == s.today - timedelta(days=1))
+    assert yesterday["d1"] is None  # "tomorrow" is today, which is not over: never counted, never guessed
+    assert ret["came_back"] == 7 and ret["eligible"] == 18
+    w = A.dau_wau(s.checkins, s.today)
+    assert w["wau"] == 10 and dict(w["dau"])[s.today] == 4
+    # before activity tracking began a cohort must be n/a, not 0%
+    ret2 = A.retention(s.users, {1: {s.today}}, s.today)
+    assert all(r["d1"] is None for r in ret2["rows"] if r["day"] < s.today - timedelta(days=2))
+
+
+def test_analytics_library_searches_contribs_channel():
+    from app.services import analytics as A
+
+    s = _snapshot()
+    lib = A.library(s)
+    assert lib["files"] == 3 and lib["empty"] == ["General Chemistry", "Entrepreneurship"]  # pending files do not count
+    assert lib["sparse"] == ["General Physics", "General Psychology"]  # 2 and 1 files: both under 3
+    assert lib["missed"][0] == ("chemistry module", 5)  # 4 text + 1 inline; the 1-letter search is ignored
+    assert lib["searches"] == 8 and lib["missed_total"] == 5
+    assert lib["downloads_week"] == 2 and lib["downloaders_week"] == 2 and lib["top_files"][0].title == "Physics module"
+    c = A.contributions(s)
+    assert (c["pending"], c["approved"], c["rejected"], c["approval"]) == (1, 1, 1, 50)
+    assert c["oldest_pending_hours"] == 40.0 and c["avg_hours"] == 3.0
+    ch = A.channel_posts(s)
+    assert ch[0]["message_id"] == 1 and ch[0]["tagged"] == 8  # the 8 students whose source is the post's tag "ad1"
+    assert A.extract_tags("go t.me/B?start=ad2 or t.me/B?start=123456 and ?start=Dorm5") == ["ad2", "dorm5"]
+
+
+def test_analytics_insights_and_screens():
+    import re
+
+    from app.services import analytics as A
+
+    s = _snapshot()
+    ins = A.insights(s)
+    titles = [i.title for i in ins]
+    assert "Students stop at the join step" in titles  # 6 of 20 = 30% (rule: 15% or more)
+    assert "Students search for things you do not have" in titles
+    assert "Many courses have no file" in titles  # 2 of 4 courses are empty
+    assert "Student files are waiting" in titles  # the oldest pending file is 40 h old
+    first = ins[0]
+    assert first.priority == 1 and "30%" in first.evidence and "dbucentralforum" in first.evidence
+    screens = (A.overview, A.section_growth, A.section_retention, A.section_library, A.section_contrib,
+               A.section_channel, A.section_insights)
+    for render in screens:
+        text = render(s)
+        assert 20 < len(text) < 4000, render.__name__
+        for tag in ("b", "i"):
+            assert len(re.findall(rf"<{tag}>", text)) == len(re.findall(rf"</{tag}>", text)), render.__name__
+    empty = A.Snapshot(now=s.now, users=[], checkins={}, events=[], materials=[], courses={}, subs=[], posts=[])
+    for render in screens:  # an empty project must not crash anything
+        assert render(empty)
+
+
 if __name__ == "__main__":
     for name, fn in list(globals().items()):
         if name.startswith("test_"):