| # Ginigen AI & K-AI Community - Global Robots.txt |
| # Used by: TeXray, VIDraft, FINAL Bench, AETHER, and all future projects |
| # Last Updated: 2026.03.01 |
|
|
| # Default rules for all bots |
| User-agent: * |
| Allow: / |
| Allow: /llms.txt |
| Disallow: /admin/ |
| Disallow: /private/ |
| Disallow: /temp/ |
| Disallow: /.git/ |
| Disallow: /.env |
| Crawl-delay: 1 |
|
|
| # Sitemap declaration |
| Sitemap: https://example.com/sitemap.xml |
| Sitemap: https://example.com/sitemap-projects.xml |
|
|
| # ============================================ |
| # AI Crawlers - Actively Welcome |
| # ============================================ |
|
|
| # OpenAI (ChatGPT, GPT-4, etc.) |
| User-agent: GPTBot |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: OpenAI GPT training crawler |
|
|
| # OpenAI ChatGPT Mobile App |
| User-agent: ChatGPT-User |
| Allow: / |
| Crawl-delay: 0 |
| Comment: ChatGPT mobile app crawler |
|
|
| # Google Extensions & AI Overviews |
| User-agent: Google-Extended |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Google AI Overview / SGE crawler |
|
|
| # Anthropic (Claude) |
| User-agent: Anthropic-ai |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Anthropic Claude training crawler |
|
|
| User-agent: ClaudeBot |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Claude AI assistant |
|
|
| # Perplexity AI |
| User-agent: PerplexityBot |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Perplexity AI search |
|
|
| # Apple Intelligence |
| User-agent: Applebot-Extended |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Apple AI training |
|
|
| # Cohere |
| User-agent: cohere-ai |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Cohere AI training |
|
|
| # Meta (LLaMA training) |
| User-agent: facebookexternalhit |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Meta AI training |
|
|
| # Mistral AI |
| User-agent: MistralBot |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Mistral AI training |
|
|
| # Hugging Face (Hub & Model Training) |
| User-agent: HuggingFaceBot |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: HuggingFace model discovery |
|
|
| # ============================================ |
| # Search Engines |
| # ============================================ |
|
|
| # Google |
| User-agent: Googlebot |
| Allow: / |
| Allow: /llms.txt |
| Crawl-delay: 0 |
| Comment: Google Search indexing |
|
|
| # Bing |
| User-agent: Bingbot |
| Allow: / |
| Crawl-delay: 1 |
| Comment: Bing Search indexing |
|
|
| # DuckDuckGo |
| User-agent: DuckDuckBot |
| Allow: / |
| Crawl-delay: 1 |
| Comment: DuckDuckGo Search indexing |
|
|
| # ============================================ |
| # Blocked / Rate-Limited Bots |
| # ============================================ |
|
|
| # Aggressive/spam bots |
| User-agent: AhrefsBot |
| User-agent: SemrushBot |
| User-agent: DotBot |
| User-agent: MJ12bot |
| Disallow: / |
| Comment: Blocked: Aggressive crawlers |
|
|
| # Bad actors |
| User-agent: BadBot |
| User-agent: SpoofBot |
| Disallow: / |
| Comment: Blocked: Known malicious crawlers |
|
|
| # ============================================ |
| # Analytics & Monitoring (Special Rules) |
| # ============================================ |
|
|
| # HuggingFace Analytics |
| User-agent: HFAnalytics |
| Allow: / |
| Crawl-delay: 0 |
|
|
| # Custom monitoring |
| User-agent: GiniginAI-Monitor |
| Allow: / |
| Crawl-delay: 0 |
|
|
| # ============================================ |
| # Notes for Future Implementation |
| # ============================================ |
| # 1. Replace "https://example.com" with actual domain |
| # 2. Add project-specific sitemaps as needed |
| # 3. Maintain AI crawler allowlist for optimal discoverability |
| # 4. Keep crawl-delay at 0 for important AI crawlers |
| # 5. Review and update quarterly as new AI services emerge |
|
|
