ip map
#4
by SmileXing - opened
- .env.example +0 -4
- .github/workflows/ci.yml +29 -0
- .gitignore +0 -8
- README.md +0 -28
- app.py +2 -4
- pyproject.toml +0 -1
- requirements.txt +0 -1
- src/leaderboard_analytics/config.py +0 -6
- src/leaderboard_analytics/geoip_database.py +0 -36
- src/leaderboard_analytics/main.py +4 -32
- src/leaderboard_analytics/repositories.py +0 -17
- src/leaderboard_analytics/services.py +1 -207
- src/leaderboard_analytics/ui.py +1 -208
- tests/test_geoip_database.py +0 -29
- tests/test_repositories.py +0 -21
- tests/test_services.py +0 -75
.env.example
CHANGED
|
@@ -4,7 +4,3 @@ MONGO_COLLECTION=events
|
|
| 4 |
HOST=0.0.0.0
|
| 5 |
PORT=7860
|
| 6 |
GRADIO_SHARE=false
|
| 7 |
-
GRADIO_SSR_MODE=false
|
| 8 |
-
GEOIP_DATABASE_PATH=GeoLite2-Country.mmdb
|
| 9 |
-
GEOIP_DATABASE_URL=https://cdn.jsdelivr.net/npm/geolite2-country/GeoLite2-Country.mmdb.gz
|
| 10 |
-
GEOIP_AUTO_DOWNLOAD=true
|
|
|
|
| 4 |
HOST=0.0.0.0
|
| 5 |
PORT=7860
|
| 6 |
GRADIO_SHARE=false
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
pull_request:
|
| 6 |
+
|
| 7 |
+
jobs:
|
| 8 |
+
test:
|
| 9 |
+
runs-on: ubuntu-latest
|
| 10 |
+
steps:
|
| 11 |
+
- name: Check out repository
|
| 12 |
+
uses: actions/checkout@v4
|
| 13 |
+
|
| 14 |
+
- name: Set up Python
|
| 15 |
+
uses: actions/setup-python@v5
|
| 16 |
+
with:
|
| 17 |
+
python-version: "3.11"
|
| 18 |
+
|
| 19 |
+
- name: Install package
|
| 20 |
+
run: python -m pip install --upgrade pip && python -m pip install ".[dev]"
|
| 21 |
+
|
| 22 |
+
- name: Check formatting
|
| 23 |
+
run: ruff format --check .
|
| 24 |
+
|
| 25 |
+
- name: Lint
|
| 26 |
+
run: ruff check .
|
| 27 |
+
|
| 28 |
+
- name: Test
|
| 29 |
+
run: pytest
|
.gitignore
CHANGED
|
@@ -49,7 +49,6 @@ coverage.xml
|
|
| 49 |
*.py.cover
|
| 50 |
.hypothesis/
|
| 51 |
.pytest_cache/
|
| 52 |
-
.pytest_tmp/
|
| 53 |
cover/
|
| 54 |
|
| 55 |
# Translations
|
|
@@ -145,13 +144,6 @@ ENV/
|
|
| 145 |
env.bak/
|
| 146 |
venv.bak/
|
| 147 |
|
| 148 |
-
# Local GeoIP databases
|
| 149 |
-
*.mmdb
|
| 150 |
-
*.mmdb.gz
|
| 151 |
-
|
| 152 |
-
# Local analytics exports
|
| 153 |
-
visitor_ips*.csv
|
| 154 |
-
|
| 155 |
# Spyder project settings
|
| 156 |
.spyderproject
|
| 157 |
.spyproject
|
|
|
|
| 49 |
*.py.cover
|
| 50 |
.hypothesis/
|
| 51 |
.pytest_cache/
|
|
|
|
| 52 |
cover/
|
| 53 |
|
| 54 |
# Translations
|
|
|
|
| 144 |
env.bak/
|
| 145 |
venv.bak/
|
| 146 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
# Spyder project settings
|
| 148 |
.spyderproject
|
| 149 |
.spyproject
|
README.md
CHANGED
|
@@ -26,7 +26,6 @@ All analytics are based on the `events` collection and the following stable fiel
|
|
| 26 |
- Preferred event time: `ts` as a MongoDB Date
|
| 27 |
- Behavior context: `benchmark`, `filters`
|
| 28 |
- Visitor identity (approximate): `properties.visitor_id`
|
| 29 |
-
- Visitor IP for country analysis: `properties.ip`
|
| 30 |
- Change context: `properties.old_value`, `properties.new_value`, `properties.filter_name`
|
| 31 |
|
| 32 |
Important event names:
|
|
@@ -162,19 +161,6 @@ Recommended session-level funnel:
|
|
| 162 |
- If event date is later than first-seen date, classify as `returning`
|
| 163 |
- Count distinct `visitor_id` by period
|
| 164 |
|
| 165 |
-
### 13) Visitor Locations by Country
|
| 166 |
-
|
| 167 |
-
- **Definition**: Page view volume by visitor IP country/region.
|
| 168 |
-
- **Source fields**: `event_name`, `properties.ip`
|
| 169 |
-
- **Calculation**:
|
| 170 |
-
- Filter `event_name == "page_view"`
|
| 171 |
-
- Remove null/empty `properties.ip`
|
| 172 |
-
- Group page views by IP in MongoDB
|
| 173 |
-
- Resolve each IP to a country using the local MaxMind GeoLite2 Country database
|
| 174 |
-
- Group by `country_code` and `country_name`
|
| 175 |
-
- Map color = page view count (`pv`)
|
| 176 |
-
- Private, invalid, unresolved, or unconfigured IPs are grouped as `Unknown`
|
| 177 |
-
|
| 178 |
---
|
| 179 |
|
| 180 |
## Time Aggregation Rules
|
|
@@ -219,7 +205,6 @@ db.events.createIndex({ ts: 1, benchmark: 1 })
|
|
| 219 |
db.events.createIndex({ event_name: 1, ts: 1 })
|
| 220 |
db.events.createIndex({ session_id: 1, ts: 1 })
|
| 221 |
db.events.createIndex({ "properties.visitor_id": 1, ts: 1 })
|
| 222 |
-
db.events.createIndex({ event_name: 1, ts: 1, "properties.ip": 1 })
|
| 223 |
```
|
| 224 |
|
| 225 |
Legacy events with only `timestamp` remain supported, but backfilling `ts` is recommended before running this dashboard against large collections.
|
|
@@ -233,19 +218,6 @@ Only required runtime inputs:
|
|
| 233 |
- MongoDB connection URI (`MONGO_URI`)
|
| 234 |
- Mongo database/collection names (defaults supported)
|
| 235 |
|
| 236 |
-
Optional visitor location input:
|
| 237 |
-
|
| 238 |
-
- `GEOIP_DATABASE_PATH`: path to a local MaxMind `GeoLite2-Country.mmdb` file
|
| 239 |
-
- `GEOIP_DATABASE_URL`: URL for a gzipped GeoLite2 Country MMDB download
|
| 240 |
-
- `GEOIP_AUTO_DOWNLOAD`: whether to download and decompress the MMDB when missing
|
| 241 |
-
|
| 242 |
-
The dashboard does not call an external IP lookup API for visitor lookups. By default,
|
| 243 |
-
startup downloads `https://cdn.jsdelivr.net/npm/geolite2-country/GeoLite2-Country.mmdb.gz`
|
| 244 |
-
when `GEOIP_DATABASE_PATH` is missing, decompresses it, and uses the resulting MMDB file
|
| 245 |
-
locally. Set `GEOIP_AUTO_DOWNLOAD=false` if the runtime cannot access the network or if
|
| 246 |
-
you prefer to mount the MMDB yourself. If the database is unavailable, visitor location
|
| 247 |
-
rows are grouped as `Unknown`.
|
| 248 |
-
|
| 249 |
Local commands:
|
| 250 |
|
| 251 |
```bash
|
|
|
|
| 26 |
- Preferred event time: `ts` as a MongoDB Date
|
| 27 |
- Behavior context: `benchmark`, `filters`
|
| 28 |
- Visitor identity (approximate): `properties.visitor_id`
|
|
|
|
| 29 |
- Change context: `properties.old_value`, `properties.new_value`, `properties.filter_name`
|
| 30 |
|
| 31 |
Important event names:
|
|
|
|
| 161 |
- If event date is later than first-seen date, classify as `returning`
|
| 162 |
- Count distinct `visitor_id` by period
|
| 163 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
---
|
| 165 |
|
| 166 |
## Time Aggregation Rules
|
|
|
|
| 205 |
db.events.createIndex({ event_name: 1, ts: 1 })
|
| 206 |
db.events.createIndex({ session_id: 1, ts: 1 })
|
| 207 |
db.events.createIndex({ "properties.visitor_id": 1, ts: 1 })
|
|
|
|
| 208 |
```
|
| 209 |
|
| 210 |
Legacy events with only `timestamp` remain supported, but backfilling `ts` is recommended before running this dashboard against large collections.
|
|
|
|
| 218 |
- MongoDB connection URI (`MONGO_URI`)
|
| 219 |
- Mongo database/collection names (defaults supported)
|
| 220 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
Local commands:
|
| 222 |
|
| 223 |
```bash
|
app.py
CHANGED
|
@@ -7,9 +7,7 @@ SRC_DIR = ROOT_DIR / "src"
|
|
| 7 |
if str(SRC_DIR) not in sys.path:
|
| 8 |
sys.path.insert(0, str(SRC_DIR))
|
| 9 |
|
| 10 |
-
from leaderboard_analytics.main import
|
| 11 |
-
|
| 12 |
-
demo = create_demo()
|
| 13 |
|
| 14 |
if __name__ == "__main__":
|
| 15 |
-
|
|
|
|
| 7 |
if str(SRC_DIR) not in sys.path:
|
| 8 |
sys.path.insert(0, str(SRC_DIR))
|
| 9 |
|
| 10 |
+
from leaderboard_analytics.main import run # noqa: E402
|
|
|
|
|
|
|
| 11 |
|
| 12 |
if __name__ == "__main__":
|
| 13 |
+
run()
|
pyproject.toml
CHANGED
|
@@ -12,7 +12,6 @@ dependencies = [
|
|
| 12 |
"python-dotenv>=1.0.1",
|
| 13 |
"pandas>=2.2.3",
|
| 14 |
"plotly>=5.24.1",
|
| 15 |
-
"geoip2>=4.8.0",
|
| 16 |
]
|
| 17 |
|
| 18 |
[project.optional-dependencies]
|
|
|
|
| 12 |
"python-dotenv>=1.0.1",
|
| 13 |
"pandas>=2.2.3",
|
| 14 |
"plotly>=5.24.1",
|
|
|
|
| 15 |
]
|
| 16 |
|
| 17 |
[project.optional-dependencies]
|
requirements.txt
CHANGED
|
@@ -5,4 +5,3 @@ pydantic-settings>=2.6.0
|
|
| 5 |
python-dotenv>=1.0.1
|
| 6 |
pandas>=2.2.3
|
| 7 |
plotly>=5.24.1
|
| 8 |
-
geoip2>=4.8.0
|
|
|
|
| 5 |
python-dotenv>=1.0.1
|
| 6 |
pandas>=2.2.3
|
| 7 |
plotly>=5.24.1
|
|
|
src/leaderboard_analytics/config.py
CHANGED
|
@@ -12,12 +12,6 @@ class Settings(BaseSettings):
|
|
| 12 |
host: str = "0.0.0.0"
|
| 13 |
port: int = 7860
|
| 14 |
gradio_share: bool = False
|
| 15 |
-
gradio_ssr_mode: bool = False
|
| 16 |
-
geoip_database_path: str = "GeoLite2-Country.mmdb"
|
| 17 |
-
geoip_database_url: str = (
|
| 18 |
-
"https://cdn.jsdelivr.net/npm/geolite2-country/GeoLite2-Country.mmdb.gz"
|
| 19 |
-
)
|
| 20 |
-
geoip_auto_download: bool = True
|
| 21 |
|
| 22 |
|
| 23 |
@lru_cache(maxsize=1)
|
|
|
|
| 12 |
host: str = "0.0.0.0"
|
| 13 |
port: int = 7860
|
| 14 |
gradio_share: bool = False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
@lru_cache(maxsize=1)
|
src/leaderboard_analytics/geoip_database.py
DELETED
|
@@ -1,36 +0,0 @@
|
|
| 1 |
-
import gzip
|
| 2 |
-
import shutil
|
| 3 |
-
import tempfile
|
| 4 |
-
from pathlib import Path
|
| 5 |
-
from urllib.request import urlopen
|
| 6 |
-
|
| 7 |
-
DEFAULT_GEOIP_DATABASE_URL = (
|
| 8 |
-
"https://cdn.jsdelivr.net/npm/geolite2-country/GeoLite2-Country.mmdb.gz"
|
| 9 |
-
)
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
def ensure_geoip_database(
|
| 13 |
-
database_path: str | Path,
|
| 14 |
-
source_url: str = DEFAULT_GEOIP_DATABASE_URL,
|
| 15 |
-
*,
|
| 16 |
-
auto_download: bool = True,
|
| 17 |
-
timeout: float = 30.0,
|
| 18 |
-
) -> Path:
|
| 19 |
-
target_path = Path(database_path)
|
| 20 |
-
if target_path.exists() or not auto_download:
|
| 21 |
-
return target_path
|
| 22 |
-
|
| 23 |
-
target_path.parent.mkdir(parents=True, exist_ok=True)
|
| 24 |
-
with tempfile.NamedTemporaryFile(
|
| 25 |
-
prefix=f"{target_path.name}.",
|
| 26 |
-
suffix=".tmp",
|
| 27 |
-
dir=target_path.parent,
|
| 28 |
-
delete=False,
|
| 29 |
-
) as temp_file:
|
| 30 |
-
temp_path = Path(temp_file.name)
|
| 31 |
-
with urlopen(source_url, timeout=timeout) as response:
|
| 32 |
-
with gzip.GzipFile(fileobj=response) as gzip_file:
|
| 33 |
-
shutil.copyfileobj(gzip_file, temp_file)
|
| 34 |
-
|
| 35 |
-
temp_path.replace(target_path)
|
| 36 |
-
return target_path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/leaderboard_analytics/main.py
CHANGED
|
@@ -1,48 +1,20 @@
|
|
| 1 |
from leaderboard_analytics.config import get_settings
|
| 2 |
from leaderboard_analytics.db import get_database, get_events_collection, get_mongo_client
|
| 3 |
-
from leaderboard_analytics.geoip_database import ensure_geoip_database
|
| 4 |
from leaderboard_analytics.repositories import AnalyticsRepository
|
| 5 |
from leaderboard_analytics.services import AnalyticsService
|
| 6 |
from leaderboard_analytics.ui import build_dashboard
|
| 7 |
|
| 8 |
|
| 9 |
-
def
|
| 10 |
settings = get_settings()
|
| 11 |
client = get_mongo_client()
|
| 12 |
db = get_database(client)
|
| 13 |
events_collection = get_events_collection(db)
|
| 14 |
-
geoip_database_path = settings.geoip_database_path
|
| 15 |
-
try:
|
| 16 |
-
geoip_database_path = str(
|
| 17 |
-
ensure_geoip_database(
|
| 18 |
-
settings.geoip_database_path,
|
| 19 |
-
settings.geoip_database_url,
|
| 20 |
-
auto_download=settings.geoip_auto_download,
|
| 21 |
-
)
|
| 22 |
-
)
|
| 23 |
-
except Exception as exc:
|
| 24 |
-
print(f"GeoIP database download failed: {exc}")
|
| 25 |
|
| 26 |
repository = AnalyticsRepository(events_collection=events_collection)
|
| 27 |
-
service = AnalyticsService(
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
)
|
| 31 |
-
return build_dashboard(service=service)
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
def launch_demo(demo) -> None:
|
| 35 |
-
settings = get_settings()
|
| 36 |
-
demo.launch(
|
| 37 |
-
server_name=settings.host,
|
| 38 |
-
server_port=settings.port,
|
| 39 |
-
share=settings.gradio_share,
|
| 40 |
-
ssr_mode=settings.gradio_ssr_mode,
|
| 41 |
-
)
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
def run() -> None:
|
| 45 |
-
launch_demo(create_demo())
|
| 46 |
|
| 47 |
|
| 48 |
if __name__ == "__main__":
|
|
|
|
| 1 |
from leaderboard_analytics.config import get_settings
|
| 2 |
from leaderboard_analytics.db import get_database, get_events_collection, get_mongo_client
|
|
|
|
| 3 |
from leaderboard_analytics.repositories import AnalyticsRepository
|
| 4 |
from leaderboard_analytics.services import AnalyticsService
|
| 5 |
from leaderboard_analytics.ui import build_dashboard
|
| 6 |
|
| 7 |
|
| 8 |
+
def run() -> None:
|
| 9 |
settings = get_settings()
|
| 10 |
client = get_mongo_client()
|
| 11 |
db = get_database(client)
|
| 12 |
events_collection = get_events_collection(db)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
repository = AnalyticsRepository(events_collection=events_collection)
|
| 15 |
+
service = AnalyticsService(repository=repository)
|
| 16 |
+
demo = build_dashboard(service=service)
|
| 17 |
+
demo.launch(server_name=settings.host, server_port=settings.port, share=settings.gradio_share)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
if __name__ == "__main__":
|
src/leaderboard_analytics/repositories.py
CHANGED
|
@@ -419,23 +419,6 @@ class AnalyticsRepository:
|
|
| 419 |
]
|
| 420 |
return list(self.events_collection.aggregate(pipeline))
|
| 421 |
|
| 422 |
-
def visitor_ip_counts(self, filters: QueryFilters) -> list[dict]:
|
| 423 |
-
pipeline: list[dict] = [
|
| 424 |
-
{"$match": _indexed_time_prefilter(filters)},
|
| 425 |
-
_with_normalized_time(),
|
| 426 |
-
{
|
| 427 |
-
"$match": {
|
| 428 |
-
**_with_time_and_optional_benchmark(filters),
|
| 429 |
-
"event_name": "page_view",
|
| 430 |
-
"properties.ip": {"$nin": [None, ""]},
|
| 431 |
-
}
|
| 432 |
-
},
|
| 433 |
-
{"$group": {"_id": "$properties.ip", "pv": {"$sum": 1}}},
|
| 434 |
-
{"$project": {"_id": 0, "ip": "$_id", "pv": 1}},
|
| 435 |
-
{"$sort": {"pv": -1}},
|
| 436 |
-
]
|
| 437 |
-
return list(self.events_collection.aggregate(pipeline))
|
| 438 |
-
|
| 439 |
def available_benchmarks(
|
| 440 |
self, filters: QueryFilters | None = None, limit: int = 100
|
| 441 |
) -> list[str]:
|
|
|
|
| 419 |
]
|
| 420 |
return list(self.events_collection.aggregate(pipeline))
|
| 421 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 422 |
def available_benchmarks(
|
| 423 |
self, filters: QueryFilters | None = None, limit: int = 100
|
| 424 |
) -> list[str]:
|
src/leaderboard_analytics/services.py
CHANGED
|
@@ -1,162 +1,12 @@
|
|
| 1 |
-
import ipaddress
|
| 2 |
-
from pathlib import Path
|
| 3 |
-
from typing import Any, Protocol
|
| 4 |
-
|
| 5 |
import pandas as pd
|
| 6 |
|
| 7 |
from leaderboard_analytics.repositories import AnalyticsRepository
|
| 8 |
from leaderboard_analytics.schemas import QueryFilters
|
| 9 |
|
| 10 |
-
UNKNOWN_COUNTRY_CODE = "Unknown"
|
| 11 |
-
UNKNOWN_COUNTRY_NAME = "Unknown"
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
def _empty_ip_debug() -> dict[str, object]:
|
| 15 |
-
return {
|
| 16 |
-
"total_unique_ips": 0,
|
| 17 |
-
"total_ip_pv": 0,
|
| 18 |
-
"global_ips": 0,
|
| 19 |
-
"global_ip_pv": 0,
|
| 20 |
-
"private_ips": 0,
|
| 21 |
-
"private_ip_pv": 0,
|
| 22 |
-
"loopback_ips": 0,
|
| 23 |
-
"loopback_ip_pv": 0,
|
| 24 |
-
"reserved_ips": 0,
|
| 25 |
-
"reserved_ip_pv": 0,
|
| 26 |
-
"link_local_ips": 0,
|
| 27 |
-
"link_local_ip_pv": 0,
|
| 28 |
-
"multicast_ips": 0,
|
| 29 |
-
"multicast_ip_pv": 0,
|
| 30 |
-
"unspecified_ips": 0,
|
| 31 |
-
"unspecified_ip_pv": 0,
|
| 32 |
-
"invalid_ips": 0,
|
| 33 |
-
"invalid_ip_pv": 0,
|
| 34 |
-
"top_ip_pv_buckets": {
|
| 35 |
-
"1": 0,
|
| 36 |
-
"2-10": 0,
|
| 37 |
-
"11-100": 0,
|
| 38 |
-
"101-1000": 0,
|
| 39 |
-
">1000": 0,
|
| 40 |
-
},
|
| 41 |
-
}
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
def _ip_debug_category(ip_address: str) -> str:
|
| 45 |
-
try:
|
| 46 |
-
parsed_ip = ipaddress.ip_address(ip_address.strip())
|
| 47 |
-
except ValueError:
|
| 48 |
-
return "invalid"
|
| 49 |
-
|
| 50 |
-
if parsed_ip.is_global:
|
| 51 |
-
return "global"
|
| 52 |
-
if parsed_ip.is_loopback:
|
| 53 |
-
return "loopback"
|
| 54 |
-
if parsed_ip.is_private:
|
| 55 |
-
return "private"
|
| 56 |
-
if parsed_ip.is_reserved:
|
| 57 |
-
return "reserved"
|
| 58 |
-
if parsed_ip.is_link_local:
|
| 59 |
-
return "link_local"
|
| 60 |
-
if parsed_ip.is_multicast:
|
| 61 |
-
return "multicast"
|
| 62 |
-
if parsed_ip.is_unspecified:
|
| 63 |
-
return "unspecified"
|
| 64 |
-
return "reserved"
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
def _ip_pv_bucket(pv: int) -> str:
|
| 68 |
-
if pv <= 1:
|
| 69 |
-
return "1"
|
| 70 |
-
if pv <= 10:
|
| 71 |
-
return "2-10"
|
| 72 |
-
if pv <= 100:
|
| 73 |
-
return "11-100"
|
| 74 |
-
if pv <= 1000:
|
| 75 |
-
return "101-1000"
|
| 76 |
-
return ">1000"
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
class GeoIpCountryReader(Protocol):
|
| 80 |
-
def country(self, ip_address: str) -> Any: ...
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
class GeoIpResolver:
|
| 84 |
-
def __init__(
|
| 85 |
-
self,
|
| 86 |
-
database_path: str | Path | None = None,
|
| 87 |
-
reader: GeoIpCountryReader | None = None,
|
| 88 |
-
) -> None:
|
| 89 |
-
self.database_path = Path(database_path) if database_path else None
|
| 90 |
-
self._reader = reader
|
| 91 |
-
self._load_attempted = reader is not None
|
| 92 |
-
|
| 93 |
-
def resolve_country(self, ip_address: str) -> tuple[str, str]:
|
| 94 |
-
try:
|
| 95 |
-
parsed_ip = ipaddress.ip_address(ip_address.strip())
|
| 96 |
-
except ValueError:
|
| 97 |
-
return UNKNOWN_COUNTRY_CODE, UNKNOWN_COUNTRY_NAME
|
| 98 |
-
|
| 99 |
-
if not parsed_ip.is_global:
|
| 100 |
-
return UNKNOWN_COUNTRY_CODE, UNKNOWN_COUNTRY_NAME
|
| 101 |
-
|
| 102 |
-
reader = self._get_reader()
|
| 103 |
-
if reader is None:
|
| 104 |
-
return UNKNOWN_COUNTRY_CODE, UNKNOWN_COUNTRY_NAME
|
| 105 |
-
|
| 106 |
-
try:
|
| 107 |
-
response = reader.country(str(parsed_ip))
|
| 108 |
-
except Exception:
|
| 109 |
-
return UNKNOWN_COUNTRY_CODE, UNKNOWN_COUNTRY_NAME
|
| 110 |
-
|
| 111 |
-
country = response.country
|
| 112 |
-
if not getattr(country, "iso_code", None):
|
| 113 |
-
country = response.registered_country
|
| 114 |
-
|
| 115 |
-
code = getattr(country, "iso_code", None)
|
| 116 |
-
if not code:
|
| 117 |
-
return UNKNOWN_COUNTRY_CODE, UNKNOWN_COUNTRY_NAME
|
| 118 |
-
|
| 119 |
-
return code, getattr(country, "name", None) or code
|
| 120 |
-
|
| 121 |
-
def debug_status(self) -> dict[str, object]:
|
| 122 |
-
return {
|
| 123 |
-
"database_path": str(self.database_path) if self.database_path else "",
|
| 124 |
-
"database_configured": self.database_path is not None,
|
| 125 |
-
"database_exists": self.database_path.exists() if self.database_path else False,
|
| 126 |
-
"load_attempted": self._load_attempted,
|
| 127 |
-
"reader_loaded": self._reader is not None,
|
| 128 |
-
}
|
| 129 |
-
|
| 130 |
-
def _get_reader(self) -> GeoIpCountryReader | None:
|
| 131 |
-
if self._reader is not None:
|
| 132 |
-
return self._reader
|
| 133 |
-
|
| 134 |
-
if self._load_attempted:
|
| 135 |
-
return None
|
| 136 |
-
|
| 137 |
-
self._load_attempted = True
|
| 138 |
-
if self.database_path is None or not self.database_path.exists():
|
| 139 |
-
return None
|
| 140 |
-
|
| 141 |
-
try:
|
| 142 |
-
import geoip2.database
|
| 143 |
-
|
| 144 |
-
self._reader = geoip2.database.Reader(str(self.database_path))
|
| 145 |
-
except Exception:
|
| 146 |
-
return None
|
| 147 |
-
|
| 148 |
-
return self._reader
|
| 149 |
-
|
| 150 |
|
| 151 |
class AnalyticsService:
|
| 152 |
-
def __init__(
|
| 153 |
-
self,
|
| 154 |
-
repository: AnalyticsRepository,
|
| 155 |
-
geoip_database_path: str | Path | None = None,
|
| 156 |
-
geoip_resolver: GeoIpResolver | None = None,
|
| 157 |
-
) -> None:
|
| 158 |
self.repository = repository
|
| 159 |
-
self.geoip_resolver = geoip_resolver or GeoIpResolver(geoip_database_path)
|
| 160 |
|
| 161 |
def get_overview(self, filters: QueryFilters) -> tuple[pd.DataFrame, dict]:
|
| 162 |
rows = self.repository.overview_timeseries(filters)
|
|
@@ -204,61 +54,5 @@ class AnalyticsService:
|
|
| 204 |
frame["visitor_type"] = frame["is_new"].map({True: "new", False: "returning"})
|
| 205 |
return frame
|
| 206 |
|
| 207 |
-
def get_visitor_locations(self, filters: QueryFilters) -> pd.DataFrame:
|
| 208 |
-
frame, _debug = self.get_visitor_location_details(filters)
|
| 209 |
-
return frame
|
| 210 |
-
|
| 211 |
-
def get_visitor_location_details(self, filters: QueryFilters) -> tuple[pd.DataFrame, dict]:
|
| 212 |
-
locations: dict[tuple[str, str], dict[str, int | str]] = {}
|
| 213 |
-
ip_debug = _empty_ip_debug()
|
| 214 |
-
for row in self.repository.visitor_ip_counts(filters):
|
| 215 |
-
ip = str(row.get("ip", "")).strip()
|
| 216 |
-
if not ip:
|
| 217 |
-
continue
|
| 218 |
-
|
| 219 |
-
pv = int(row.get("pv", 0))
|
| 220 |
-
category = _ip_debug_category(ip)
|
| 221 |
-
ip_debug["total_unique_ips"] = int(ip_debug["total_unique_ips"]) + 1
|
| 222 |
-
ip_debug["total_ip_pv"] = int(ip_debug["total_ip_pv"]) + pv
|
| 223 |
-
ip_debug[f"{category}_ips"] = int(ip_debug[f"{category}_ips"]) + 1
|
| 224 |
-
ip_debug[f"{category}_ip_pv"] = int(ip_debug[f"{category}_ip_pv"]) + pv
|
| 225 |
-
ip_debug["top_ip_pv_buckets"][_ip_pv_bucket(pv)] += 1 # type: ignore[index]
|
| 226 |
-
|
| 227 |
-
code, name = self.geoip_resolver.resolve_country(ip)
|
| 228 |
-
key = (code, name)
|
| 229 |
-
if key not in locations:
|
| 230 |
-
locations[key] = {
|
| 231 |
-
"country_code": code,
|
| 232 |
-
"country_name": name,
|
| 233 |
-
"pv": 0,
|
| 234 |
-
"ip_count": 0,
|
| 235 |
-
}
|
| 236 |
-
|
| 237 |
-
locations[key]["pv"] = int(locations[key]["pv"]) + pv
|
| 238 |
-
locations[key]["ip_count"] = int(locations[key]["ip_count"]) + 1
|
| 239 |
-
|
| 240 |
-
frame = pd.DataFrame(
|
| 241 |
-
locations.values(),
|
| 242 |
-
columns=["country_code", "country_name", "pv", "ip_count"],
|
| 243 |
-
)
|
| 244 |
-
if frame.empty:
|
| 245 |
-
return frame, ip_debug
|
| 246 |
-
frame = frame.sort_values(["pv", "ip_count"], ascending=[False, False]).reset_index(
|
| 247 |
-
drop=True
|
| 248 |
-
)
|
| 249 |
-
return frame, ip_debug
|
| 250 |
-
|
| 251 |
-
def get_geoip_debug_info(self) -> dict[str, object]:
|
| 252 |
-
debug_status = getattr(self.geoip_resolver, "debug_status", None)
|
| 253 |
-
if debug_status is None:
|
| 254 |
-
return {
|
| 255 |
-
"database_path": "",
|
| 256 |
-
"database_configured": False,
|
| 257 |
-
"database_exists": False,
|
| 258 |
-
"load_attempted": False,
|
| 259 |
-
"reader_loaded": False,
|
| 260 |
-
}
|
| 261 |
-
return debug_status()
|
| 262 |
-
|
| 263 |
def get_available_benchmarks(self, filters: QueryFilters | None = None) -> list[str]:
|
| 264 |
return self.repository.available_benchmarks(filters)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import pandas as pd
|
| 2 |
|
| 3 |
from leaderboard_analytics.repositories import AnalyticsRepository
|
| 4 |
from leaderboard_analytics.schemas import QueryFilters
|
| 5 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
class AnalyticsService:
|
| 8 |
+
def __init__(self, repository: AnalyticsRepository) -> None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
self.repository = repository
|
|
|
|
| 10 |
|
| 11 |
def get_overview(self, filters: QueryFilters) -> tuple[pd.DataFrame, dict]:
|
| 12 |
rows = self.repository.overview_timeseries(filters)
|
|
|
|
| 54 |
frame["visitor_type"] = frame["is_new"].map({True: "new", False: "returning"})
|
| 55 |
return frame
|
| 56 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
def get_available_benchmarks(self, filters: QueryFilters | None = None) -> list[str]:
|
| 58 |
return self.repository.available_benchmarks(filters)
|
src/leaderboard_analytics/ui.py
CHANGED
|
@@ -8,7 +8,6 @@ from typing import Any
|
|
| 8 |
import gradio as gr
|
| 9 |
import pandas as pd
|
| 10 |
import plotly.express as px
|
| 11 |
-
import plotly.graph_objects as go
|
| 12 |
|
| 13 |
from leaderboard_analytics.schemas import Granularity, QueryFilters
|
| 14 |
from leaderboard_analytics.services import AnalyticsService
|
|
@@ -40,12 +39,6 @@ def _empty_plot(title: str):
|
|
| 40 |
return px.line(title=title)
|
| 41 |
|
| 42 |
|
| 43 |
-
def _empty_map(title: str):
|
| 44 |
-
figure = go.Figure()
|
| 45 |
-
_style_visitor_location_map(figure, title)
|
| 46 |
-
return figure
|
| 47 |
-
|
| 48 |
-
|
| 49 |
def _query_range_text(filters: QueryFilters) -> str:
|
| 50 |
return f"{filters.start_time.isoformat()} to {filters.end_time.isoformat()}"
|
| 51 |
|
|
@@ -64,160 +57,6 @@ def _write_csv_archive(tables: dict[str, pd.DataFrame]) -> str | None:
|
|
| 64 |
return archive.name
|
| 65 |
|
| 66 |
|
| 67 |
-
def _visitor_location_top_table(visitor_locations: pd.DataFrame) -> pd.DataFrame:
|
| 68 |
-
if visitor_locations.empty:
|
| 69 |
-
return pd.DataFrame(columns=["Region", "Users"])
|
| 70 |
-
|
| 71 |
-
return (
|
| 72 |
-
visitor_locations.sort_values(["ip_count", "pv"], ascending=[False, False])
|
| 73 |
-
.head(10)
|
| 74 |
-
.rename(columns={"country_name": "Region", "ip_count": "Users"})[["Region", "Users"]]
|
| 75 |
-
.reset_index(drop=True)
|
| 76 |
-
)
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
def _visitor_location_debug_text(
|
| 80 |
-
visitor_locations: pd.DataFrame,
|
| 81 |
-
geoip_debug: dict[str, object],
|
| 82 |
-
ip_debug: dict[str, object] | None = None,
|
| 83 |
-
) -> str:
|
| 84 |
-
if visitor_locations.empty:
|
| 85 |
-
total_pv = 0
|
| 86 |
-
total_users = 0
|
| 87 |
-
mapped_regions = 0
|
| 88 |
-
unknown_pv = 0
|
| 89 |
-
unknown_users = 0
|
| 90 |
-
else:
|
| 91 |
-
unknown_rows = visitor_locations[visitor_locations["country_code"] == "Unknown"]
|
| 92 |
-
mapped_rows = visitor_locations[visitor_locations["country_code"] != "Unknown"]
|
| 93 |
-
total_pv = int(visitor_locations["pv"].sum())
|
| 94 |
-
total_users = int(visitor_locations["ip_count"].sum())
|
| 95 |
-
mapped_regions = len(mapped_rows)
|
| 96 |
-
unknown_pv = int(unknown_rows["pv"].sum()) if not unknown_rows.empty else 0
|
| 97 |
-
unknown_users = int(unknown_rows["ip_count"].sum()) if not unknown_rows.empty else 0
|
| 98 |
-
|
| 99 |
-
configured = "yes" if geoip_debug.get("database_configured") else "no"
|
| 100 |
-
exists = "yes" if geoip_debug.get("database_exists") else "no"
|
| 101 |
-
loaded = "yes" if geoip_debug.get("reader_loaded") else "no"
|
| 102 |
-
attempted = "yes" if geoip_debug.get("load_attempted") else "no"
|
| 103 |
-
path = geoip_debug.get("database_path") or "(not configured)"
|
| 104 |
-
ip_debug = ip_debug or {}
|
| 105 |
-
global_ips = int(ip_debug.get("global_ips", 0))
|
| 106 |
-
global_pv = int(ip_debug.get("global_ip_pv", 0))
|
| 107 |
-
private_ips = int(ip_debug.get("private_ips", 0))
|
| 108 |
-
private_pv = int(ip_debug.get("private_ip_pv", 0))
|
| 109 |
-
loopback_ips = int(ip_debug.get("loopback_ips", 0))
|
| 110 |
-
loopback_pv = int(ip_debug.get("loopback_ip_pv", 0))
|
| 111 |
-
invalid_ips = int(ip_debug.get("invalid_ips", 0))
|
| 112 |
-
invalid_pv = int(ip_debug.get("invalid_ip_pv", 0))
|
| 113 |
-
buckets = ip_debug.get("top_ip_pv_buckets", {})
|
| 114 |
-
|
| 115 |
-
return (
|
| 116 |
-
f"GeoIP DB: configured={configured}, exists={exists}, loaded={loaded}, "
|
| 117 |
-
f"load_attempted={attempted} \n"
|
| 118 |
-
f"GeoIP path: `{path}` \n"
|
| 119 |
-
f"Total location PV: {total_pv} | Users/IPs: {total_users} | "
|
| 120 |
-
f"Mapped regions: {mapped_regions} \n"
|
| 121 |
-
f"Unknown PV: {unknown_pv} | Unknown users/IPs: {unknown_users} \n"
|
| 122 |
-
f"Public IPs: {global_ips} ({global_pv} PV) | Private IPs: {private_ips} "
|
| 123 |
-
f"({private_pv} PV) \n"
|
| 124 |
-
f"Loopback IPs: {loopback_ips} ({loopback_pv} PV) | Invalid IPs: {invalid_ips} "
|
| 125 |
-
f"({invalid_pv} PV) \n"
|
| 126 |
-
f"PV/IP buckets: {buckets}"
|
| 127 |
-
)
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
def _style_visitor_location_map(figure: go.Figure, title: str) -> None:
|
| 131 |
-
figure.update_geos(
|
| 132 |
-
projection_type="mercator",
|
| 133 |
-
showframe=False,
|
| 134 |
-
showcoastlines=True,
|
| 135 |
-
coastlinecolor="#cfd6df",
|
| 136 |
-
coastlinewidth=0.6,
|
| 137 |
-
showcountries=True,
|
| 138 |
-
countrycolor="#cfd6df",
|
| 139 |
-
countrywidth=0.7,
|
| 140 |
-
showland=True,
|
| 141 |
-
landcolor="#eef2f7",
|
| 142 |
-
showocean=True,
|
| 143 |
-
oceancolor="#f8fafc",
|
| 144 |
-
showlakes=True,
|
| 145 |
-
lakecolor="#f8fafc",
|
| 146 |
-
bgcolor="#ffffff",
|
| 147 |
-
lataxis_range=[-55, 75],
|
| 148 |
-
lonaxis_range=[-180, 180],
|
| 149 |
-
)
|
| 150 |
-
figure.update_layout(
|
| 151 |
-
title={"text": title, "x": 0.02, "xanchor": "left"},
|
| 152 |
-
height=560,
|
| 153 |
-
paper_bgcolor="#ffffff",
|
| 154 |
-
plot_bgcolor="#ffffff",
|
| 155 |
-
font={"color": "#1f2937"},
|
| 156 |
-
margin={"l": 0, "r": 0, "t": 52, "b": 0},
|
| 157 |
-
showlegend=False,
|
| 158 |
-
hoverlabel={
|
| 159 |
-
"bgcolor": "#ffffff",
|
| 160 |
-
"bordercolor": "#3b82f6",
|
| 161 |
-
"font_color": "#111827",
|
| 162 |
-
},
|
| 163 |
-
)
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
def _visitor_location_map(visitor_locations: pd.DataFrame, range_text: str) -> go.Figure:
|
| 167 |
-
map_df = (
|
| 168 |
-
visitor_locations[visitor_locations["country_code"] != "Unknown"].copy()
|
| 169 |
-
if not visitor_locations.empty
|
| 170 |
-
else visitor_locations.copy()
|
| 171 |
-
)
|
| 172 |
-
if map_df.empty:
|
| 173 |
-
return _empty_map(f"Visitor locations by country (no mapped data for {range_text})")
|
| 174 |
-
|
| 175 |
-
max_pv = max(int(map_df["pv"].max()), 1)
|
| 176 |
-
size_ref = 2.0 * max_pv / (52**2)
|
| 177 |
-
figure = go.Figure(
|
| 178 |
-
go.Scattergeo(
|
| 179 |
-
locationmode="country names",
|
| 180 |
-
locations=map_df["country_name"],
|
| 181 |
-
mode="markers",
|
| 182 |
-
text=map_df["country_name"],
|
| 183 |
-
customdata=map_df[["country_code", "pv", "ip_count"]],
|
| 184 |
-
hovertemplate=(
|
| 185 |
-
"<b>%{text}</b><br>"
|
| 186 |
-
"Country code: %{customdata[0]}<br>"
|
| 187 |
-
"PV: %{customdata[1]:,}<br>"
|
| 188 |
-
"Users/IPs: %{customdata[2]:,}<extra></extra>"
|
| 189 |
-
),
|
| 190 |
-
marker={
|
| 191 |
-
"size": map_df["pv"],
|
| 192 |
-
"sizemode": "area",
|
| 193 |
-
"sizeref": size_ref,
|
| 194 |
-
"sizemin": 8,
|
| 195 |
-
"color": "rgba(59, 130, 246, 0.55)",
|
| 196 |
-
"line": {"color": "rgba(37, 99, 235, 0.92)", "width": 1.2},
|
| 197 |
-
},
|
| 198 |
-
)
|
| 199 |
-
)
|
| 200 |
-
_style_visitor_location_map(figure, "Visitor locations by country")
|
| 201 |
-
figure.add_annotation(
|
| 202 |
-
x=0.02,
|
| 203 |
-
y=0.08,
|
| 204 |
-
xref="paper",
|
| 205 |
-
yref="paper",
|
| 206 |
-
text=(
|
| 207 |
-
f"Mapped regions: {len(map_df)}<br>"
|
| 208 |
-
f"Mapped PV: {int(map_df['pv'].sum()):,}<br>"
|
| 209 |
-
f"Users/IPs: {int(map_df['ip_count'].sum()):,}"
|
| 210 |
-
),
|
| 211 |
-
showarrow=False,
|
| 212 |
-
align="left",
|
| 213 |
-
bgcolor="rgba(255, 255, 255, 0.88)",
|
| 214 |
-
bordercolor="rgba(148, 163, 184, 0.55)",
|
| 215 |
-
borderwidth=1,
|
| 216 |
-
font={"color": "#1f2937", "size": 12},
|
| 217 |
-
)
|
| 218 |
-
return figure
|
| 219 |
-
|
| 220 |
-
|
| 221 |
def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
| 222 |
default_end = datetime.now(tz=UTC)
|
| 223 |
default_start = (default_end - timedelta(days=7)).replace(microsecond=0)
|
|
@@ -247,10 +86,6 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 247 |
object,
|
| 248 |
object,
|
| 249 |
object,
|
| 250 |
-
object,
|
| 251 |
-
object,
|
| 252 |
-
object,
|
| 253 |
-
object,
|
| 254 |
]:
|
| 255 |
try:
|
| 256 |
filters = QueryFilters(
|
|
@@ -264,22 +99,9 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 264 |
filter_df = service.get_filter_distribution(filters)
|
| 265 |
funnel_df = service.get_funnel(filters)
|
| 266 |
visitors_df = service.get_new_vs_returning(filters)
|
| 267 |
-
visitor_locations_df, ip_debug = service.get_visitor_location_details(filters)
|
| 268 |
-
visitor_locations_top_df = _visitor_location_top_table(visitor_locations_df)
|
| 269 |
-
visitor_locations_debug = _visitor_location_debug_text(
|
| 270 |
-
visitor_locations_df,
|
| 271 |
-
service.get_geoip_debug_info(),
|
| 272 |
-
ip_debug,
|
| 273 |
-
)
|
| 274 |
|
| 275 |
range_text = _query_range_text(filters)
|
| 276 |
-
if
|
| 277 |
-
overview_df.empty
|
| 278 |
-
and benchmark_df.empty
|
| 279 |
-
and filter_df.empty
|
| 280 |
-
and visitors_df.empty
|
| 281 |
-
and visitor_locations_df.empty
|
| 282 |
-
):
|
| 283 |
metrics = f"No data for {range_text}."
|
| 284 |
else:
|
| 285 |
metrics = (
|
|
@@ -322,7 +144,6 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 322 |
if not visitors_df.empty
|
| 323 |
else px.bar(title=f"New vs returning visitors (no data for {range_text})")
|
| 324 |
)
|
| 325 |
-
visitor_locations_plot = _visitor_location_map(visitor_locations_df, range_text)
|
| 326 |
csv_archive = _write_csv_archive(
|
| 327 |
{
|
| 328 |
"overview": overview_df,
|
|
@@ -330,7 +151,6 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 330 |
"filters": filter_df,
|
| 331 |
"funnel": funnel_df,
|
| 332 |
"visitors": visitors_df,
|
| 333 |
-
"visitor_locations": visitor_locations_df,
|
| 334 |
}
|
| 335 |
)
|
| 336 |
|
|
@@ -341,21 +161,16 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 341 |
filter_plot,
|
| 342 |
funnel_plot,
|
| 343 |
visitor_plot,
|
| 344 |
-
visitor_locations_plot,
|
| 345 |
-
visitor_locations_debug,
|
| 346 |
-
visitor_locations_top_df,
|
| 347 |
overview_df,
|
| 348 |
benchmark_df,
|
| 349 |
filter_df,
|
| 350 |
funnel_df,
|
| 351 |
visitors_df,
|
| 352 |
-
visitor_locations_df,
|
| 353 |
csv_archive,
|
| 354 |
)
|
| 355 |
except Exception as exc:
|
| 356 |
message = f"Query failed: {exc}"
|
| 357 |
empty = pd.DataFrame()
|
| 358 |
-
empty_top = pd.DataFrame(columns=["Region", "Users"])
|
| 359 |
return (
|
| 360 |
message,
|
| 361 |
_empty_plot(message),
|
|
@@ -368,10 +183,6 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 368 |
title=message,
|
| 369 |
),
|
| 370 |
px.bar(title=message),
|
| 371 |
-
_empty_map(message),
|
| 372 |
-
message,
|
| 373 |
-
empty_top,
|
| 374 |
-
empty,
|
| 375 |
empty,
|
| 376 |
empty,
|
| 377 |
empty,
|
|
@@ -422,19 +233,6 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 422 |
filter_plot = gr.Plot(label="Filter Behavior")
|
| 423 |
funnel_plot = gr.Plot(label="Funnel")
|
| 424 |
visitor_plot = gr.Plot(label="Visitor Segmentation")
|
| 425 |
-
with gr.Row():
|
| 426 |
-
with gr.Column(scale=2):
|
| 427 |
-
visitor_locations_plot = gr.Plot(label="Visitor Locations")
|
| 428 |
-
with gr.Column(scale=1):
|
| 429 |
-
visitor_locations_debug = gr.Markdown(
|
| 430 |
-
"GeoIP DB: not checked \n"
|
| 431 |
-
"Total location PV: 0 | Users/IPs: 0 | Mapped regions: 0"
|
| 432 |
-
)
|
| 433 |
-
visitor_locations_top_table = gr.DataFrame(
|
| 434 |
-
label="Top 10 Regions",
|
| 435 |
-
interactive=False,
|
| 436 |
-
wrap=True,
|
| 437 |
-
)
|
| 438 |
|
| 439 |
with gr.Accordion("Raw data", open=False):
|
| 440 |
csv_file = gr.File(label="CSV export")
|
|
@@ -443,7 +241,6 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 443 |
filter_table = gr.DataFrame(label="Filter Behavior")
|
| 444 |
funnel_table = gr.DataFrame(label="Funnel")
|
| 445 |
visitor_table = gr.DataFrame(label="Visitor Segmentation")
|
| 446 |
-
visitor_locations_table = gr.DataFrame(label="Visitor Locations")
|
| 447 |
|
| 448 |
outputs = [
|
| 449 |
metrics_text,
|
|
@@ -452,15 +249,11 @@ def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
|
| 452 |
filter_plot,
|
| 453 |
funnel_plot,
|
| 454 |
visitor_plot,
|
| 455 |
-
visitor_locations_plot,
|
| 456 |
-
visitor_locations_debug,
|
| 457 |
-
visitor_locations_top_table,
|
| 458 |
overview_table,
|
| 459 |
benchmark_table,
|
| 460 |
filter_table,
|
| 461 |
funnel_table,
|
| 462 |
visitor_table,
|
| 463 |
-
visitor_locations_table,
|
| 464 |
csv_file,
|
| 465 |
]
|
| 466 |
|
|
|
|
| 8 |
import gradio as gr
|
| 9 |
import pandas as pd
|
| 10 |
import plotly.express as px
|
|
|
|
| 11 |
|
| 12 |
from leaderboard_analytics.schemas import Granularity, QueryFilters
|
| 13 |
from leaderboard_analytics.services import AnalyticsService
|
|
|
|
| 39 |
return px.line(title=title)
|
| 40 |
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
def _query_range_text(filters: QueryFilters) -> str:
|
| 43 |
return f"{filters.start_time.isoformat()} to {filters.end_time.isoformat()}"
|
| 44 |
|
|
|
|
| 57 |
return archive.name
|
| 58 |
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
def build_dashboard(service: AnalyticsService) -> gr.Blocks:
|
| 61 |
default_end = datetime.now(tz=UTC)
|
| 62 |
default_start = (default_end - timedelta(days=7)).replace(microsecond=0)
|
|
|
|
| 86 |
object,
|
| 87 |
object,
|
| 88 |
object,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
]:
|
| 90 |
try:
|
| 91 |
filters = QueryFilters(
|
|
|
|
| 99 |
filter_df = service.get_filter_distribution(filters)
|
| 100 |
funnel_df = service.get_funnel(filters)
|
| 101 |
visitors_df = service.get_new_vs_returning(filters)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
range_text = _query_range_text(filters)
|
| 104 |
+
if overview_df.empty and benchmark_df.empty and filter_df.empty and visitors_df.empty:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
metrics = f"No data for {range_text}."
|
| 106 |
else:
|
| 107 |
metrics = (
|
|
|
|
| 144 |
if not visitors_df.empty
|
| 145 |
else px.bar(title=f"New vs returning visitors (no data for {range_text})")
|
| 146 |
)
|
|
|
|
| 147 |
csv_archive = _write_csv_archive(
|
| 148 |
{
|
| 149 |
"overview": overview_df,
|
|
|
|
| 151 |
"filters": filter_df,
|
| 152 |
"funnel": funnel_df,
|
| 153 |
"visitors": visitors_df,
|
|
|
|
| 154 |
}
|
| 155 |
)
|
| 156 |
|
|
|
|
| 161 |
filter_plot,
|
| 162 |
funnel_plot,
|
| 163 |
visitor_plot,
|
|
|
|
|
|
|
|
|
|
| 164 |
overview_df,
|
| 165 |
benchmark_df,
|
| 166 |
filter_df,
|
| 167 |
funnel_df,
|
| 168 |
visitors_df,
|
|
|
|
| 169 |
csv_archive,
|
| 170 |
)
|
| 171 |
except Exception as exc:
|
| 172 |
message = f"Query failed: {exc}"
|
| 173 |
empty = pd.DataFrame()
|
|
|
|
| 174 |
return (
|
| 175 |
message,
|
| 176 |
_empty_plot(message),
|
|
|
|
| 183 |
title=message,
|
| 184 |
),
|
| 185 |
px.bar(title=message),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
empty,
|
| 187 |
empty,
|
| 188 |
empty,
|
|
|
|
| 233 |
filter_plot = gr.Plot(label="Filter Behavior")
|
| 234 |
funnel_plot = gr.Plot(label="Funnel")
|
| 235 |
visitor_plot = gr.Plot(label="Visitor Segmentation")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
|
| 237 |
with gr.Accordion("Raw data", open=False):
|
| 238 |
csv_file = gr.File(label="CSV export")
|
|
|
|
| 241 |
filter_table = gr.DataFrame(label="Filter Behavior")
|
| 242 |
funnel_table = gr.DataFrame(label="Funnel")
|
| 243 |
visitor_table = gr.DataFrame(label="Visitor Segmentation")
|
|
|
|
| 244 |
|
| 245 |
outputs = [
|
| 246 |
metrics_text,
|
|
|
|
| 249 |
filter_plot,
|
| 250 |
funnel_plot,
|
| 251 |
visitor_plot,
|
|
|
|
|
|
|
|
|
|
| 252 |
overview_table,
|
| 253 |
benchmark_table,
|
| 254 |
filter_table,
|
| 255 |
funnel_table,
|
| 256 |
visitor_table,
|
|
|
|
| 257 |
csv_file,
|
| 258 |
]
|
| 259 |
|
tests/test_geoip_database.py
DELETED
|
@@ -1,29 +0,0 @@
|
|
| 1 |
-
import gzip
|
| 2 |
-
|
| 3 |
-
from leaderboard_analytics.geoip_database import ensure_geoip_database
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
def test_ensure_geoip_database_downloads_and_decompresses_gzip(tmp_path) -> None:
|
| 7 |
-
source = tmp_path / "GeoLite2-Country.mmdb.gz"
|
| 8 |
-
target = tmp_path / "GeoLite2-Country.mmdb"
|
| 9 |
-
expected_bytes = b"fake-mmdb-bytes"
|
| 10 |
-
|
| 11 |
-
with gzip.open(source, "wb") as gzip_file:
|
| 12 |
-
gzip_file.write(expected_bytes)
|
| 13 |
-
|
| 14 |
-
result = ensure_geoip_database(target, source.as_uri())
|
| 15 |
-
|
| 16 |
-
assert result == target
|
| 17 |
-
assert target.read_bytes() == expected_bytes
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
def test_ensure_geoip_database_keeps_existing_file(tmp_path) -> None:
|
| 21 |
-
source = tmp_path / "missing.mmdb.gz"
|
| 22 |
-
target = tmp_path / "GeoLite2-Country.mmdb"
|
| 23 |
-
expected_bytes = b"existing-mmdb-bytes"
|
| 24 |
-
target.write_bytes(expected_bytes)
|
| 25 |
-
|
| 26 |
-
result = ensure_geoip_database(target, source.as_uri())
|
| 27 |
-
|
| 28 |
-
assert result == target
|
| 29 |
-
assert target.read_bytes() == expected_bytes
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_repositories.py
CHANGED
|
@@ -72,24 +72,3 @@ def test_overview_totals_filters_empty_identifiers() -> None:
|
|
| 72 |
assert '"$visitors"' in pipeline_text or "'$visitors'" in pipeline_text
|
| 73 |
assert "$$s" in pipeline_text
|
| 74 |
assert "$$v" in pipeline_text
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
def test_visitor_ip_counts_groups_page_view_ips_with_existing_filters() -> None:
|
| 78 |
-
collection = CapturingCollection([{"ip": "8.8.8.8", "pv": 3}])
|
| 79 |
-
repository = AnalyticsRepository(collection) # type: ignore[arg-type]
|
| 80 |
-
filters = QueryFilters(
|
| 81 |
-
start_time=datetime(2026, 1, 1, tzinfo=UTC),
|
| 82 |
-
end_time=datetime(2026, 1, 31, tzinfo=UTC),
|
| 83 |
-
benchmark="MTEB",
|
| 84 |
-
)
|
| 85 |
-
|
| 86 |
-
rows = repository.visitor_ip_counts(filters)
|
| 87 |
-
|
| 88 |
-
assert rows == [{"ip": "8.8.8.8", "pv": 3}]
|
| 89 |
-
assert collection.pipeline is not None
|
| 90 |
-
pipeline_text = str(collection.pipeline)
|
| 91 |
-
assert "properties.ip" in pipeline_text
|
| 92 |
-
assert "page_view" in pipeline_text
|
| 93 |
-
assert "MTEB" in pipeline_text
|
| 94 |
-
assert "$nin" in pipeline_text
|
| 95 |
-
assert "$properties.ip" in pipeline_text
|
|
|
|
| 72 |
assert '"$visitors"' in pipeline_text or "'$visitors'" in pipeline_text
|
| 73 |
assert "$$s" in pipeline_text
|
| 74 |
assert "$$v" in pipeline_text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_services.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
| 1 |
from datetime import UTC, datetime
|
| 2 |
-
from pathlib import Path
|
| 3 |
|
| 4 |
from leaderboard_analytics.schemas import QueryFilters
|
| 5 |
from leaderboard_analytics.services import AnalyticsService
|
|
@@ -16,22 +15,6 @@ class FakeRepository:
|
|
| 16 |
return {"pv": 3, "uv": 1, "sessions": 1, "events": 5}
|
| 17 |
|
| 18 |
|
| 19 |
-
class LocationRepository:
|
| 20 |
-
def __init__(self, rows: list[dict]) -> None:
|
| 21 |
-
self.rows = rows
|
| 22 |
-
|
| 23 |
-
def visitor_ip_counts(self, filters: QueryFilters) -> list[dict]:
|
| 24 |
-
return self.rows
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
class FakeGeoIpResolver:
|
| 28 |
-
def __init__(self, countries: dict[str, tuple[str, str]]) -> None:
|
| 29 |
-
self.countries = countries
|
| 30 |
-
|
| 31 |
-
def resolve_country(self, ip_address: str) -> tuple[str, str]:
|
| 32 |
-
return self.countries[ip_address]
|
| 33 |
-
|
| 34 |
-
|
| 35 |
def test_overview_uses_full_range_distinct_totals() -> None:
|
| 36 |
service = AnalyticsService(FakeRepository()) # type: ignore[arg-type]
|
| 37 |
filters = QueryFilters(
|
|
@@ -50,61 +33,3 @@ def test_overview_uses_full_range_distinct_totals() -> None:
|
|
| 50 |
"events_per_session": 5.0,
|
| 51 |
"sessions_per_visitor": 1.0,
|
| 52 |
}
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def test_visitor_locations_groups_pv_and_ip_count_by_country() -> None:
|
| 56 |
-
repository = LocationRepository(
|
| 57 |
-
[
|
| 58 |
-
{"ip": "8.8.8.8", "pv": 3},
|
| 59 |
-
{"ip": "8.8.4.4", "pv": 2},
|
| 60 |
-
{"ip": "1.1.1.1", "pv": 4},
|
| 61 |
-
]
|
| 62 |
-
)
|
| 63 |
-
resolver = FakeGeoIpResolver(
|
| 64 |
-
{
|
| 65 |
-
"8.8.8.8": ("US", "United States"),
|
| 66 |
-
"8.8.4.4": ("US", "United States"),
|
| 67 |
-
"1.1.1.1": ("AU", "Australia"),
|
| 68 |
-
}
|
| 69 |
-
)
|
| 70 |
-
service = AnalyticsService(
|
| 71 |
-
repository, # type: ignore[arg-type]
|
| 72 |
-
geoip_resolver=resolver, # type: ignore[arg-type]
|
| 73 |
-
)
|
| 74 |
-
|
| 75 |
-
frame = service.get_visitor_locations(
|
| 76 |
-
QueryFilters(
|
| 77 |
-
start_time=datetime(2026, 1, 1, tzinfo=UTC),
|
| 78 |
-
end_time=datetime(2026, 1, 2, tzinfo=UTC),
|
| 79 |
-
)
|
| 80 |
-
)
|
| 81 |
-
|
| 82 |
-
assert frame.to_dict("records") == [
|
| 83 |
-
{"country_code": "US", "country_name": "United States", "pv": 5, "ip_count": 2},
|
| 84 |
-
{"country_code": "AU", "country_name": "Australia", "pv": 4, "ip_count": 1},
|
| 85 |
-
]
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
def test_visitor_locations_groups_unresolved_ips_as_unknown() -> None:
|
| 89 |
-
repository = LocationRepository(
|
| 90 |
-
[
|
| 91 |
-
{"ip": "10.0.0.1", "pv": 2},
|
| 92 |
-
{"ip": "not-an-ip", "pv": 1},
|
| 93 |
-
{"ip": "8.8.8.8", "pv": 3},
|
| 94 |
-
]
|
| 95 |
-
)
|
| 96 |
-
service = AnalyticsService(
|
| 97 |
-
repository, # type: ignore[arg-type]
|
| 98 |
-
geoip_database_path=Path("missing-geolite2-country.mmdb"),
|
| 99 |
-
)
|
| 100 |
-
|
| 101 |
-
frame = service.get_visitor_locations(
|
| 102 |
-
QueryFilters(
|
| 103 |
-
start_time=datetime(2026, 1, 1, tzinfo=UTC),
|
| 104 |
-
end_time=datetime(2026, 1, 2, tzinfo=UTC),
|
| 105 |
-
)
|
| 106 |
-
)
|
| 107 |
-
|
| 108 |
-
assert frame.to_dict("records") == [
|
| 109 |
-
{"country_code": "Unknown", "country_name": "Unknown", "pv": 6, "ip_count": 3}
|
| 110 |
-
]
|
|
|
|
| 1 |
from datetime import UTC, datetime
|
|
|
|
| 2 |
|
| 3 |
from leaderboard_analytics.schemas import QueryFilters
|
| 4 |
from leaderboard_analytics.services import AnalyticsService
|
|
|
|
| 15 |
return {"pv": 3, "uv": 1, "sessions": 1, "events": 5}
|
| 16 |
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
def test_overview_uses_full_range_distinct_totals() -> None:
|
| 19 |
service = AnalyticsService(FakeRepository()) # type: ignore[arg-type]
|
| 20 |
filters = QueryFilters(
|
|
|
|
| 33 |
"events_per_session": 5.0,
|
| 34 |
"sessions_per_visitor": 1.0,
|
| 35 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|