Add crawl delay for requests

Ref. https://projects.blender.org/Alaska/needs-information-reports/src/branch/main/needs_info_from_users.py

routers/utils_gitea.py

@@ -1,12 +1,35 @@
 # utils_gitea.py
 
 import json
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 BASE_API_URL = "https://projects.blender.org/api/v1"
+CRAWL_DELAY = 2
+last_request_time = None
+# Conform to Blenders crawl delay request:
+# https://projects.blender.org/robots.txt
+try:
+    projects = urllib.robotparser.RobotFileParser(
+        url="https://projects.blender.org/robots.txt")
+    projects.read()
+    projects_crawl_delay = projects.crawl_delay("*")
+    if projects_crawl_delay is not None:
+        CRAWL_DELAY = projects_crawl_delay
+except:
+    pass
+
+
+def wait_for_crawl_delay() -> None:
+    global last_request_time
+
+    # Respect the crawl delay
+    if last_request_time is not None:
+        time.sleep(max(CRAWL_DELAY - (time.time() - last_request_time), 0))
+    last_request_time = time.time()
 
 
 def url_json_get(url, data=None):
@@ -18,6 +41,7 @@ def url_json_get(url, data=None):
         else:
             request = urllib.request.Request(url)
 
+        wait_for_crawl_delay()
         response = urllib.request.urlopen(request)
         response_data = json.loads(response.read())
         return response_data