Upload hf_backend/config.py with huggingface_hub

hf_backend/config.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class AppConfig:
+    dataset_repo_id: str
+    hf_token: str
+    shared_token: str
+    resend_api_key: str
+    resend_api_base: str
+    from_email: str
+    kindle_email_default: str
+    translation_model: str
+    default_language: str
+    batch_size: int
+    epub_accumulated_num: int
+    epub_backmatter_skip_after_percent: int
+    epub_backmatter_titles: str
+    gemini_review_mode: str
+    gemini_review_min_chinese_ratio: float
+    gemini_review_length_ratio_min: float
+    gemini_review_length_ratio_max: float
+    api_base: str
+    ebook_convert_bin: str
+    custom_api: str
+    gemini_key: str
+    claude_key: str
+    deepl_key: str
+    caiyun_key: str
+    openai_key: str
+    manifest_path: str
+    inbox_prefix: str
+    done_prefix: str
+    work_dir: Path
+    max_jobs_per_run: int
+    lock_timeout_minutes: int
+    auto_resume_stale_minutes: int
+    max_auto_resume_attempts: int
+    epub_checkpoint_interval: int
+    weekly_url_template: str
+    weekly_filename_template: str
+    fetch_user_agent: str
+    fetch_timeout_seconds: int
+    standard_ebooks_search_url: str
+    project_gutenberg_search_url: str
+    internet_archive_advancedsearch_url: str
+    internet_archive_metadata_url_template: str
+    src_a_search_url: str
+    src_a_base_url: str
+    src_b_base_url: str
+
+    @property
+    def repo_root(self) -> Path:
+        return Path(__file__).resolve().parent.parent
+
+    @property
+    def make_book_script(self) -> Path:
+        return self.repo_root / "bilingual_book_maker" / "make_book.py"
+
+    def resolve_model(self, preferred: str | None = None) -> str:
+        model = (preferred or self.translation_model).strip()
+        if model == "google":
+            return "gemini"
+        return model
+
+    def resolve_language(self, preferred: str | None = None) -> str:
+        return (preferred or self.default_language).strip()
+
+
+def load_config() -> AppConfig:
+    dataset_repo_id = os.getenv("HF_DATASET_REPO_ID", "").strip()
+    hf_token = os.getenv("HF_TOKEN", "").strip()
+    shared_token = os.getenv("RUN_SHARED_TOKEN", "").strip()
+    if not dataset_repo_id:
+        raise RuntimeError("HF_DATASET_REPO_ID is required")
+    if not hf_token:
+        raise RuntimeError("HF_TOKEN is required")
+    if not shared_token:
+        raise RuntimeError("RUN_SHARED_TOKEN is required")
+
+    work_dir = Path(os.getenv("WORK_DIR", "/tmp/ebook-work")).resolve()
+    work_dir.mkdir(parents=True, exist_ok=True)
+
+    gemini_key = (
+        os.getenv("BBM_GEMINI_ROUTER_API_KEY", "").strip()
+        or os.getenv("BBM_GOOGLE_GEMINI_KEY", "").strip()
+    )
+
+    return AppConfig(
+        dataset_repo_id=dataset_repo_id,
+        hf_token=hf_token,
+        shared_token=shared_token,
+        resend_api_key=os.getenv("RESEND_API_KEY", "").strip(),
+        resend_api_base=os.getenv("RESEND_API_BASE", "https://api.resend.com").strip(),
+        from_email=os.getenv("FROM_EMAIL", "").strip(),
+        kindle_email_default=os.getenv("KINDLE_EMAIL_DEFAULT", "").strip(),
+        translation_model=os.getenv("TRANSLATION_MODEL", "gemini").strip(),
+        default_language=os.getenv("DEFAULT_LANGUAGE", "zh-hans").strip(),
+        batch_size=int(os.getenv("TRANSLATION_BATCH_SIZE", "10")),
+        epub_accumulated_num=int(os.getenv("EPUB_ACCUMULATED_NUM", "2200")),
+        epub_backmatter_skip_after_percent=int(
+            os.getenv("EPUB_BACKMATTER_SKIP_AFTER_PERCENT", "50")
+        ),
+        epub_backmatter_titles=os.getenv(
+            "EPUB_BACKMATTER_TITLES",
+            "about the author,notes about the author,notes on the author,biography,biographical note,credits,acknowledgments,acknowledgements,copyright,bibliography,index,endnotes,notes,further reading,also by the author,about this book,about this ebook",
+        ).strip(),
+        gemini_review_mode=os.getenv(
+            "GEMINI_REVIEW_MODE",
+            "suspicious_only",
+        ).strip(),
+        gemini_review_min_chinese_ratio=float(
+            os.getenv("GEMINI_REVIEW_MIN_CHINESE_RATIO", "0.2")
+        ),
+        gemini_review_length_ratio_min=float(
+            os.getenv("GEMINI_REVIEW_LENGTH_RATIO_MIN", "0.35")
+        ),
+        gemini_review_length_ratio_max=float(
+            os.getenv("GEMINI_REVIEW_LENGTH_RATIO_MAX", "2.5")
+        ),
+        api_base=os.getenv("TRANSLATION_API_BASE", "").strip(),
+        ebook_convert_bin=os.getenv("EBOOK_CONVERT_BIN", "ebook-convert").strip(),
+        custom_api=os.getenv("BBM_CUSTOM_API", "").strip(),
+        gemini_key=gemini_key,
+        claude_key=os.getenv("BBM_CLAUDE_API_KEY", "").strip(),
+        deepl_key=os.getenv("BBM_DEEPL_API_KEY", "").strip(),
+        caiyun_key=os.getenv("BBM_CAIYUN_API_KEY", "").strip(),
+        openai_key=os.getenv("BBM_OPENAI_API_KEY", "").strip(),
+        manifest_path=os.getenv("HF_MANIFEST_PATH", "jobs/index.json").strip(),
+        inbox_prefix=os.getenv("HF_INBOX_PREFIX", "inbox").strip(),
+        done_prefix=os.getenv("HF_DONE_PREFIX", "done").strip(),
+        work_dir=work_dir,
+        max_jobs_per_run=int(os.getenv("MAX_JOBS_PER_RUN", "1")),
+        lock_timeout_minutes=int(os.getenv("LOCK_TIMEOUT_MINUTES", "2400")),
+        auto_resume_stale_minutes=int(os.getenv("AUTO_RESUME_STALE_MINUTES", "20")),
+        max_auto_resume_attempts=int(os.getenv("MAX_AUTO_RESUME_ATTEMPTS", "30")),
+        epub_checkpoint_interval=int(os.getenv("EPUB_CHECKPOINT_INTERVAL", "50")),
+        weekly_url_template=os.getenv(
+            "WEEKLY_SOURCE_URL_TEMPLATE",
+            "https://github.com/hehonghui/awesome-english-ebooks/raw/refs/heads/master/01_economist/te_{date}/TheEconomist.{date}.epub",
+        ).strip(),
+        weekly_filename_template=os.getenv(
+            "WEEKLY_SOURCE_FILENAME_TEMPLATE",
+            "TheEconomist.{date}.epub",
+        ).strip(),
+        fetch_user_agent=os.getenv("FETCH_USER_AGENT", "ebook-fetcher/1.0").strip(),
+        fetch_timeout_seconds=int(os.getenv("FETCH_TIMEOUT_SECONDS", "30")),
+        standard_ebooks_search_url=os.getenv(
+            "STANDARD_EBOOKS_SEARCH_URL",
+            "https://standardebooks.org/ebooks",
+        ).strip(),
+        project_gutenberg_search_url=os.getenv(
+            "PROJECT_GUTENBERG_SEARCH_URL",
+            "https://www.gutenberg.org/ebooks/search/",
+        ).strip(),
+        internet_archive_advancedsearch_url=os.getenv(
+            "INTERNET_ARCHIVE_ADVANCEDSEARCH_URL",
+            "https://archive.org/advancedsearch.php",
+        ).strip(),
+        internet_archive_metadata_url_template=os.getenv(
+            "INTERNET_ARCHIVE_METADATA_URL_TEMPLATE",
+            "https://archive.org/metadata/{identifier}",
+        ).strip(),
+        src_a_search_url=os.getenv("SRC_A_SEARCH_URL", "").strip(),
+        src_a_base_url=os.getenv("SRC_A_BASE_URL", "").strip(),
+        src_b_base_url=os.getenv("SRC_B_BASE_URL", "").strip(),
+    )