github-actions[bot] commited on
Commit
19fffbf
·
0 Parent(s):

deploy: cdbc8841 - 2026-10-10 20:06 UTC

Browse files
Files changed (5) hide show
  1. Dockerfile +27 -0
  2. LAST_COMMIT_DATE.txt +1 -0
  3. README.md +16 -0
  4. SOURCE_COMMIT_SHA.txt +1 -0
  5. start.sh +935 -0
Dockerfile ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.11-slim
2
+
3
+ WORKDIR /bootstrap
4
+
5
+ RUN apt-get update && apt-get install -y --no-install-recommends \
6
+ bash git ca-certificates curl openssh-client rsync unar \
7
+ && rm -rf /var/lib/apt/lists/* \
8
+ && (curl -fsSL -o /tmp/rar.tar.gz https://www.rarlab.com/rar/rarlinux-x64-712.tar.gz \
9
+ || curl -fsSL -o /tmp/rar.tar.gz https://www.rarlab.com/rar/rarlinux-x64-623.tar.gz) \
10
+ && tar -xzf /tmp/rar.tar.gz -C /tmp \
11
+ && install -m 0755 /tmp/rar/unrar /usr/local/bin/unrar \
12
+ && rm -rf /tmp/rar /tmp/rar.tar.gz \
13
+ && unrar | head -2 \
14
+ || echo "WARNING: unrar kurulamadi, rar acmada unar fallback kullanilacak"
15
+
16
+ ENV PYTHONUNBUFFERED=1
17
+ ENV PORT=7860
18
+
19
+ COPY start.sh /bootstrap/start.sh
20
+ RUN chmod +x /bootstrap/start.sh
21
+
22
+ EXPOSE 7860
23
+
24
+ HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
25
+ CMD python3 -c "import http.client; c=http.client.HTTPConnection('127.0.0.1',int(__import__('os').environ.get('PORT','7860')),timeout=5); c.request('GET','/api/health'); r=c.getresponse(); exit(0 if r.status==200 else 1)"
26
+
27
+ CMD ["/bootstrap/start.sh"]
LAST_COMMIT_DATE.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 2026-10-10T23:05:53+03:00
README.md ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Pulltime Storage Service
3
+ emoji: 💾
4
+ colorFrom: indigo
5
+ colorTo: blue
6
+ sdk: docker
7
+ app_port: 7860
8
+ pinned: false
9
+ ---
10
+
11
+ Bootstrap runtime for this private web-game infrastructure (inspired port).
12
+ Not affiliated with Rockstar Games. Contains no GTA5/brand assets.
13
+ Source code is fetched from private GitHub at container startup via GH_REPO_TOKEN.
14
+ Game asset folder (.mirror / mirror/) is NEVER committed. Only user-provided
15
+ legal files live in the Space Bucket mounted at /data and are copied to local
16
+ disk at boot. No torrent runs in the Space.
SOURCE_COMMIT_SHA.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ cdbc88410147cd95f43f5d4fa230dfe006db4e5b
start.sh ADDED
@@ -0,0 +1,935 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ # Bootstrap v4: private GitHub -> /runtime/app, bucket /data -> local disk, then serve.
5
+ #
6
+ # v3 -> v4 duzeltmeleri (GoFile HTTP 401 Unauthorized):
7
+ # 1) GoFile /contents artik HER istekte token istiyor. Tokensiz cagri = 401
8
+ # (status=error-token). v3 sadece "?token=" gonderiyor ve GOFILE_TOKEN
9
+ # bossa hicbir sey gondermiyordu -> hep 401.
10
+ # 2) v4 once GOFILE_TOKEN'i, yoksa gecici "guest" token uretiyor:
11
+ # POST https://api.gofile.io/accounts -> {"data":{"token":"..."}}
12
+ # ve istekleri "Authorization: Bearer <token>" header'i ile atiyor.
13
+ # 3) GoFile ayrica "website token" (wt) istiyor. v4 wt'yi
14
+ # https://gofile.io/dist/js/config.js icinden canli cekiyor,
15
+ # bulamazsa son bilinen degere (4fd6sg89d7s6) dusuyor ve
16
+ # /contents cagrisina ?wt= olarak ekliyor.
17
+ # 4) HF Spaces / VPS gibi DATACENTER IP'ler GoFile tarafindan listeleme
18
+ # seviyesinde engelleniyor (status=error-notPremium). Bu durumda API
19
+ # hicbir token ile calismaz. v4 bunu acikca loglar ve iki bypass sunar:
20
+ # ARCHIVE_URL = arsivin duz http(s) linki (onerilen: HF dataset,
21
+ # R2/S3 presigned URL). GoFile API hic kullanilmaz.
22
+ # GOFILE_DIRECT_URL = storeX.gofile.io/download/... direkt linki.
23
+ # /contents listesi atlanir, datacenterda da calisir.
24
+ # 5) v3, indirme BASARISIZKEN bile once bucket'taki eski veriyi siliyordu
25
+ # (clean_legacy_bucket) -> veri kaybi. v4 silmeyi SADECE yeni arsiv
26
+ # dogrulandiktan SONRA yapiyor.
27
+ #
28
+ # Secrets (Space Settings -> Variables and secrets):
29
+ # GH_REPO_TOKEN (required) fine-grained PAT, read-only to the private repo
30
+ # GAME_PASSWORD (required) login password for /
31
+ # GOFILE_TOKEN (onerilen) GoFile account token. Bossa guest token denenir ama
32
+ # datacenter IP'de (HF Spaces) listeleme engellidir.
33
+ # Variables (all optional):
34
+ # GH_REPO_SLUG default shs3131/playgta5-private
35
+ # GH_REPO_REF default main
36
+ # BUCKET_DIR default /data
37
+ # LOCAL_DATA_DIR default /runtime/game-data
38
+ # IO_WORKERS default 48 (parallel copy/extract threads)
39
+ # RANGE_MB default 64 (big files are split into ranges of this size)
40
+ # SHARDS default 8 (parallel tar shards for fast cold starts)
41
+ # DATA_VERSION default pA901JJr (bucket arsiv surum isareti)
42
+ # GOFILE_URL default https://gofile.io/d/pA901JJr
43
+ # ARCHIVE_NAME default mirror.rar
44
+ # ARCHIVE_URL default "" (duz http(s) link, GoFile API bypass; onerilen)
45
+ # GOFILE_DIRECT_URL default "" (storeX.gofile.io/download/... direkt link, liste atlanir)
46
+ # GOFILE_PASSWORD default "" (klasor sifreliyse duz parola)
47
+ #
48
+ # Bucket layout (any ONE of these works, fastest first):
49
+ # $BUCKET_DIR/harbor-heist.shardN.tar + .harbor-shards-ready (auto-built after first boot)
50
+ # $BUCKET_DIR/harbor-heist.rar (veya .zip/.tar.*) + .harbor-version (bucket arsivi)
51
+ # $BUCKET_DIR/mirror/newgame.zip (first boot, extracted straight to local disk)
52
+ # $BUCKET_DIR/harbor-heist/ + .harbor-ready (legacy folder)
53
+
54
+ if [ -z "${GH_REPO_TOKEN:-}" ]; then
55
+ echo "GH_REPO_TOKEN secret missing. Set it in Space Settings -> Variables and secrets." >&2
56
+ exit 1
57
+ fi
58
+ if [ -z "${GAME_PASSWORD:-}" ]; then
59
+ echo "GAME_PASSWORD secret missing. Set it in Space Settings -> Variables and secrets." >&2
60
+ exit 1
61
+ fi
62
+
63
+ REPO_SLUG="${GH_REPO_SLUG:-shs3131/playgta5-private}"
64
+ REPO_REF="${GH_REPO_REF:-main}"
65
+ APP_DIR="/runtime/app"
66
+ BUCKET_DIR="${BUCKET_DIR:-/data}"
67
+ LOCAL_DATA_DIR="${LOCAL_DATA_DIR:-/runtime/game-data}"
68
+ PORT="${PORT:-7860}"
69
+ IO_WORKERS="${IO_WORKERS:-48}"
70
+ RANGE_MB="${RANGE_MB:-64}"
71
+ SHARDS="${SHARDS:-8}"
72
+ export BIG_WORKERS="${BIG_WORKERS:-8}"
73
+ export COPY_CHUNK_MB="${COPY_CHUNK_MB:-4}"
74
+ BUILD_SHARDS="${BUILD_SHARDS:-0}" # 1 = pack tar shards into the bucket after load
75
+ # Ham Space degerleri (panel dosyasi bunlari ezemez, oncelik: Space > panel > default)
76
+ _RAW_DIRECT="${GOFILE_DIRECT_URL:-}"
77
+ _RAW_ARCHIVE="${ARCHIVE_URL:-}"
78
+ DATA_VERSION="${DATA_VERSION:-pA901JJr}"
79
+ GOFILE_URL="${GOFILE_URL:-https://gofile.io/d/pA901JJr}"
80
+ GOFILE_TOKEN="${GOFILE_TOKEN:-}"
81
+ GOFILE_PASSWORD="${GOFILE_PASSWORD:-}"
82
+ GOFILE_DIRECT_URL="${GOFILE_DIRECT_URL:-https://store3.gofile.io/download/web/07e3192f-cdef-47c1-a8ec-e79f3bfd6330/mirror.rar}"
83
+ ARCHIVE_URL="${ARCHIVE_URL:-}"
84
+ ARCHIVE_NAME="${ARCHIVE_NAME:-mirror.rar}"
85
+ VERSION_FILE="$BUCKET_DIR/.harbor-version"
86
+ ARCHIVE_BUCKET_PATH="$BUCKET_DIR/$ARCHIVE_NAME"
87
+
88
+ DST="$LOCAL_DATA_DIR/harbor-heist"
89
+ DONE_MARK="$DST/.local-complete"
90
+ MIRROR_DST="$LOCAL_DATA_DIR/mirror"
91
+ MIRROR_DONE="$MIRROR_DST/.local-complete"
92
+ TOOLS="/runtime/.tools"
93
+ T_START=$SECONDS
94
+ echo "[boot] bootstrap v4 (bearer+guest+wt gofile fix, safe bucket)"
95
+
96
+ mkdir -p /runtime "$TOOLS" "$LOCAL_DATA_DIR"
97
+
98
+ # Boot ciktisini hem konsola (HF Logs) hem dosyaya (panel log gorunumu) yaz.
99
+ LOG_FILE="${BOOT_LOG:-/runtime/boot.log}"
100
+ export BOOT_LOG="$LOG_FILE"
101
+ if ! exec > >(tee -a "$LOG_FILE") 2>&1; then echo "[boot] log tee kapali, konsola devam" >&2; fi
102
+
103
+ # ---------------------------------------------------------------------------
104
+ # 1) Clone in background (runs in parallel with the data load)
105
+ # ---------------------------------------------------------------------------
106
+ rm -rf "$APP_DIR"
107
+ (
108
+ echo "[clone] ${REPO_SLUG}@${REPO_REF} ..."
109
+ git clone --quiet --depth 1 --single-branch --branch "$REPO_REF" \
110
+ "https://x-access-token:${GH_REPO_TOKEN}@github.com/${REPO_SLUG}.git" "$APP_DIR" 2>&1 \
111
+ | sed "s/${GH_REPO_TOKEN}/***/g" || true
112
+ # do not leave the token in .git/config
113
+ git -C "$APP_DIR" remote set-url origin "https://github.com/${REPO_SLUG}.git"
114
+ echo "[clone] done"
115
+ ) &
116
+ CLONE_PID=$!
117
+
118
+ # ---------------------------------------------------------------------------
119
+ # 2) Helper tools (python, stdlib only)
120
+ # ---------------------------------------------------------------------------
121
+
122
+ # --- entry-point finder: lifts a single top-level dir, exit 0 if entry found
123
+ cat > "$TOOLS/find_entry.py" <<'PYEOF'
124
+ import os, shutil, sys
125
+ from pathlib import Path
126
+ dst = Path(sys.argv[1])
127
+ ENTRY = ('index.html', 'homepage.html')
128
+ if not dst.is_dir():
129
+ raise SystemExit(1)
130
+ if (dst / 'index.html').exists():
131
+ print("entry: index.html"); raise SystemExit(0)
132
+ for sub in [p for p in dst.iterdir() if p.is_dir()]:
133
+ if (sub / 'index.html').exists():
134
+ for item in list(sub.iterdir()):
135
+ shutil.move(str(item), str(dst / item.name))
136
+ try: sub.rmdir()
137
+ except OSError: pass
138
+ print(f"lifted single top-level dir: {sub.name}"); print("entry: index.html")
139
+ raise SystemExit(0)
140
+ best = None
141
+ for root, dirs, files in os.walk(dst):
142
+ depth = len(Path(root).relative_to(dst).parts)
143
+ if depth > 4:
144
+ dirs[:] = []; continue
145
+ dirs[:] = [d for d in dirs if d != '__MACOSX']
146
+ for fn in ENTRY:
147
+ if fn in files:
148
+ cand = Path(root) / fn
149
+ rel = cand.relative_to(dst).as_posix()
150
+ if 'mirror/playgta5.com' in rel or (Path(root) / 'loader.js').exists():
151
+ print(f"entry: {rel}"); raise SystemExit(0)
152
+ if best is None: best = cand
153
+ if best is not None:
154
+ print(f"entry: {best.relative_to(dst)}"); raise SystemExit(0)
155
+ print("WARNING: no index.html/homepage.html found"); raise SystemExit(1)
156
+ PYEOF
157
+
158
+ # --- parallel zip extract straight to local disk (no bucket write, no 2nd copy)
159
+ cat > "$TOOLS/unzip_par.py" <<'PYEOF'
160
+ import os, shutil, sys, threading, time, zipfile
161
+ from concurrent.futures import ThreadPoolExecutor
162
+ from pathlib import Path
163
+ src, dst, workers = sys.argv[1], Path(sys.argv[2]), int(sys.argv[3])
164
+ CHUNK = 16 * 1024 * 1024
165
+ dst.mkdir(parents=True, exist_ok=True)
166
+ tl = threading.local(); lock = threading.Lock()
167
+ done = n = errs = 0
168
+
169
+ def safe(name):
170
+ name = name.replace('\\', '/').lstrip('/')
171
+ if ':' in name: return None
172
+ parts = [p for p in name.split('/') if p not in ('', '.')]
173
+ if not parts or '..' in parts: return None
174
+ return parts
175
+
176
+ with zipfile.ZipFile(src) as z:
177
+ infos = [i for i in z.infolist()
178
+ if not (i.filename.startswith('__MACOSX/') or i.filename.endswith('.DS_Store'))]
179
+ jobs, parents = [], set()
180
+ for i in infos:
181
+ parts = safe(i.filename)
182
+ if parts is None: continue
183
+ t = dst.joinpath(*parts)
184
+ if i.is_dir(): parents.add(t)
185
+ else:
186
+ jobs.append((i, t)); parents.add(t.parent)
187
+ for p in parents: p.mkdir(parents=True, exist_ok=True)
188
+ total = sum(i.file_size for i, _ in jobs) or 1
189
+ jobs.sort(key=lambda j: j[0].file_size, reverse=True)
190
+
191
+ def work(job):
192
+ global done, n, errs
193
+ info, t = job
194
+ z = getattr(tl, 'z', None)
195
+ if z is None:
196
+ z = tl.z = zipfile.ZipFile(src)
197
+ try:
198
+ with z.open(info) as fi, open(t, 'wb') as fo:
199
+ while True:
200
+ b = fi.read(CHUNK)
201
+ if not b: break
202
+ fo.write(b)
203
+ with lock: done += len(b)
204
+ try:
205
+ ts = time.mktime(tuple(info.date_time) + (0, 0, -1)); os.utime(t, (ts, ts))
206
+ except Exception: pass
207
+ with lock: n += 1
208
+ except Exception as e:
209
+ with lock: errs += 1
210
+ print(f"UNZIP-ERR {info.filename}: {e}", flush=True)
211
+
212
+ stop = threading.Event(); t0 = time.monotonic()
213
+ def prog():
214
+ while not stop.wait(5):
215
+ with lock: d, c = done, n
216
+ el = time.monotonic() - t0
217
+ print(f"[unzip] {d/2**30:.2f}/{total/2**30:.2f} GiB ({100*d/total:.1f}%) files={c}/{len(jobs)} {d/2**20/max(el,1):.0f} MiB/s elapsed={el:.0f}s", flush=True)
218
+ threading.Thread(target=prog, daemon=True).start()
219
+ with ThreadPoolExecutor(max_workers=workers) as ex:
220
+ list(ex.map(work, jobs))
221
+ stop.set()
222
+ print(f"[unzip] done: {n}/{len(jobs)} files, {done/2**30:.2f} GiB, errs={errs} in {time.monotonic()-t0:.0f}s", flush=True)
223
+ raise SystemExit(0 if (errs == 0 and n > 0) else 1)
224
+ PYEOF
225
+
226
+ # --- parallel tree copy; big files are split into ranges so ONE huge file uses MANY streams
227
+ cat > "$TOOLS/copy_par.py" <<'PYEOF'
228
+ import os, sys, threading, time, faulthandler
229
+ faulthandler.dump_traceback_later(45, exit=False) # prints where every thread is blocked if no bytes moved
230
+ from concurrent.futures import ThreadPoolExecutor
231
+ from pathlib import Path
232
+ src, dst = Path(sys.argv[1]), Path(sys.argv[2])
233
+ workers, RANGE = int(sys.argv[3]), int(sys.argv[4]) * 1024 * 1024
234
+ BIG_WORKERS = int(os.environ.get("BIG_WORKERS", "8"))
235
+ CH = max(1, int(os.environ.get("COPY_CHUNK_MB", "4"))) * 1024 * 1024
236
+ lock = threading.Lock()
237
+ done = nfiles = errs = 0
238
+ files = []
239
+ for root, dirs, fs in os.walk(src):
240
+ for f in fs:
241
+ p = Path(root) / f
242
+ try: st = p.stat()
243
+ except OSError as e:
244
+ print(f"COPY-ERR stat {p}: {e}", flush=True); continue
245
+ files.append((p, st.st_size, st.st_mtime))
246
+ total = sum(s for _, s, _ in files) or 1
247
+ files.sort(key=lambda x: x[1], reverse=True)
248
+ remaining, failed = {}, set()
249
+ active = {}
250
+ FADV_DONTNEED = getattr(os, "POSIX_FADV_DONTNEED", None)
251
+
252
+ def finish(idx, p, t, mt):
253
+ global nfiles
254
+ if idx in failed:
255
+ try: t.unlink()
256
+ except OSError: pass
257
+ return
258
+ try: os.utime(t, (mt, mt))
259
+ except OSError: pass
260
+ with lock: nfiles += 1
261
+
262
+ def rng(idx, p, t, mt, off, ln):
263
+ global done, errs
264
+ err = None
265
+ active[threading.get_ident()] = (p.relative_to(src).as_posix(), off, ln)
266
+ for att in range(5):
267
+ fi = fo = None
268
+ got = 0
269
+ try:
270
+ fi = os.open(p, os.O_RDONLY); fo = os.open(t, os.O_WRONLY)
271
+ pos, end = off, off + ln
272
+ while pos < end:
273
+ b = os.pread(fi, min(CH, end - pos), pos)
274
+ if not b: raise OSError("short read")
275
+ w = 0
276
+ while w < len(b): w += os.pwrite(fo, b[w:], pos + w)
277
+ pos += len(b)
278
+ got += len(b)
279
+ with lock: done += len(b)
280
+ if FADV_DONTNEED is not None:
281
+ try: os.posix_fadvise(fi, off, ln, FADV_DONTNEED)
282
+ except OSError: pass
283
+ try: os.posix_fadvise(fo, off, ln, FADV_DONTNEED)
284
+ except OSError: pass
285
+ err = None; break
286
+ except OSError as e:
287
+ err = e
288
+ with lock: done -= got # retry must not double-count progress
289
+ time.sleep(1 + att)
290
+ finally:
291
+ for fd in (fi, fo):
292
+ if fd is not None:
293
+ try: os.close(fd)
294
+ except OSError: pass
295
+ active.pop(threading.get_ident(), None)
296
+ if err is not None:
297
+ with lock: failed.add(idx); errs += 1
298
+ print(f"COPY-ERR {p.relative_to(src)} @ {off}: {err}", flush=True)
299
+ with lock:
300
+ remaining[idx] -= 1; last = remaining[idx] == 0
301
+ if last: finish(idx, p, t, mt)
302
+
303
+ stop = threading.Event(); t0 = time.monotonic()
304
+ def prog():
305
+ while not stop.wait(5):
306
+ with lock: d, c = done, nfiles
307
+ if d > 0: faulthandler.cancel_dump_traceback_later()
308
+ el = time.monotonic() - t0
309
+ if d == 0 and el >= 30 and not getattr(prog, "shown", False):
310
+ prog.shown = True
311
+ print("[copy] STALLED, reads blocked on (file, offset, len):", flush=True)
312
+ for a in sorted(set(active.values()))[:20]: print(f" {a}", flush=True)
313
+ print(f"[copy] {d/2**30:.2f}/{total/2**30:.2f} GiB ({100*d/total:.1f}%) files={c}/{len(files)} {d/2**20/max(el,1):.0f} MiB/s elapsed={el:.0f}s", flush=True)
314
+ threading.Thread(target=prog, daemon=True).start()
315
+
316
+ # Two pools: small files can never be starved by slow big-file reads.
317
+ bigq, smallq, smallf = [], [], []
318
+ with ThreadPoolExecutor(max_workers=workers) as exs, ThreadPoolExecutor(max_workers=BIG_WORKERS) as exb:
319
+ for idx, (p, sz, mt) in enumerate(files):
320
+ t = dst / p.relative_to(src)
321
+ t.parent.mkdir(parents=True, exist_ok=True)
322
+ try:
323
+ ts = t.stat()
324
+ if ts.st_size == sz and int(ts.st_mtime) == int(mt):
325
+ with lock: done += sz; nfiles += 1
326
+ continue
327
+ except OSError: pass
328
+ with open(t, 'wb') as f:
329
+ if sz: f.truncate(sz)
330
+ if sz == 0:
331
+ finish(idx, p, t, mt); continue
332
+ nr = (sz + RANGE - 1) // RANGE
333
+ remaining[idx] = nr
334
+ for r in range(nr):
335
+ off = r * RANGE
336
+ if sz >= RANGE:
337
+ bigq.append((idx, p, t, mt, off, min(RANGE, sz - off)))
338
+ else:
339
+ smallq.append((idx, p, t, mt, off, min(RANGE, sz - off)))
340
+ # Phase 1: every small file first, SMALLEST first (this order is what worked on this mount)
341
+ for a in reversed(smallq): smallf.append(exs.submit(rng, *a))
342
+ for f in smallf: f.result()
343
+ print(f"[copy] phase 1 (small files) done in {time.monotonic()-t0:.0f}s; starting {len(bigq)} big-file ranges", flush=True)
344
+ # Phase 2: big files first to reduce tail latency, ranges ascending
345
+ bigq.sort(key=lambda a: (-files[a[0]][1], a[0], a[4]))
346
+ for a in bigq: exb.submit(rng, *a)
347
+ stop.set()
348
+ print(f"[copy] done: {nfiles}/{len(files)} files, {done/2**30:.2f} GiB, errs={errs} in {time.monotonic()-t0:.0f}s", flush=True)
349
+ raise SystemExit(0 if (errs == 0 and nfiles > 0) else 1)
350
+ PYEOF
351
+
352
+ # --- size-balanced shard lists (null separated) for parallel tar
353
+ cat > "$TOOLS/make_shards.py" <<'PYEOF'
354
+ import heapq, os, sys
355
+ from pathlib import Path
356
+ root, out, n = Path(sys.argv[1]), Path(sys.argv[2]), int(sys.argv[3])
357
+ out.mkdir(parents=True, exist_ok=True)
358
+ files = []
359
+ for r, d, fs in os.walk(root):
360
+ for f in fs:
361
+ p = Path(r) / f
362
+ if f == '.local-complete': continue
363
+ try: files.append((p.relative_to(root).as_posix(), p.stat().st_size))
364
+ except OSError: pass
365
+ files.sort(key=lambda x: x[1], reverse=True)
366
+ heap = [(0, i) for i in range(n)]; heapq.heapify(heap)
367
+ lists = [[] for _ in range(n)]
368
+ for name, sz in files:
369
+ load, i = heapq.heappop(heap)
370
+ lists[i].append(name); heapq.heappush(heap, (load + sz, i))
371
+ for i, l in enumerate(lists):
372
+ with open(out / f"shard_{i}.lst", 'wb') as f:
373
+ f.write(b'\0'.join(x.encode() for x in l))
374
+ PYEOF
375
+
376
+ # --- GoFile klasor/dosya linkinden indirme (en buyuk dosya). PC'de kanitli sema:
377
+ # Bearer + guest token + X-Website-Token header (sha256(UA::lang::token::t//14400::salt)),
378
+ # listeleyen token ile indirme (Cookie: accountToken). stdlib only.
379
+ cat > "$TOOLS/download_gofile.py" <<'PYEOF'
380
+ import hashlib, json, os, re, subprocess, sys, time, urllib.request, urllib.error
381
+ from pathlib import Path
382
+ from urllib.parse import quote, urlparse
383
+
384
+ UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
385
+ WT_LANG = "en-US"
386
+ WT_SALT = "12af056dacea0b"
387
+
388
+ url, out = sys.argv[1], Path(sys.argv[2])
389
+ tmp = out.parent / (out.name + ".part")
390
+ token_env = (os.environ.get("GOFILE_TOKEN") or "").strip()
391
+ password_plain = (os.environ.get("GOFILE_PASSWORD") or "").strip()
392
+
393
+ def req_json(u, token=None, method="GET", payload=None, headers=None):
394
+ data = json.dumps(payload).encode() if payload is not None else None
395
+ h = {"User-Agent": UA, "Accept": "application/json",
396
+ "Origin": "https://gofile.io", "Referer": "https://gofile.io/"}
397
+ if token:
398
+ h["Authorization"] = f"Bearer {token}"
399
+ if headers:
400
+ h.update(headers)
401
+ if payload is not None:
402
+ h["Content-Type"] = "application/json"
403
+ r = urllib.request.Request(u, data=data, headers=h, method=method)
404
+ try:
405
+ with urllib.request.urlopen(r, timeout=60) as resp:
406
+ return json.loads(resp.read().decode("utf-8", "replace"))
407
+ except urllib.error.HTTPError as e:
408
+ try:
409
+ body = e.read().decode("utf-8", "replace")[:500]
410
+ except Exception:
411
+ body = ""
412
+ raise SystemExit(f"GoFile API HTTP {e.code} for {u.split('?')[0]}: {body}")
413
+
414
+ def create_guest_token():
415
+ print("[gofile] no GOFILE_TOKEN, creating guest token ...", flush=True)
416
+ try:
417
+ d = req_json("https://api.gofile.io/accounts", method="POST", payload={})
418
+ except SystemExit as e:
419
+ raise SystemExit(f"{e}\n[gofile] guest token uretilemedi. Rate-limit ya da IP engeli olabilir.")
420
+ tok = (d.get("data") or {}).get("token") or ""
421
+ if d.get("status") != "ok" or not tok:
422
+ raise SystemExit(f"guest token failed: {d.get('status')} {str(d)[:300]}")
423
+ print("[gofile] guest token ok", flush=True)
424
+ return tok
425
+
426
+ def website_token(token):
427
+ raw = f"{UA}::{WT_LANG}::{token}::{int(time.time() / 14400)}::{WT_SALT}"
428
+ return hashlib.sha256(raw.encode()).hexdigest()
429
+
430
+ def resolve_id(u):
431
+ p = urlparse(u)
432
+ m = re.search(r"/d/([A-Za-z0-9_-]+)", p.path or "")
433
+ if m:
434
+ return m.group(1)
435
+ raise SystemExit(f"Invalid GoFile URL: {u}")
436
+
437
+ def collect_files(node, acc):
438
+ if not isinstance(node, dict):
439
+ return
440
+ if node.get("type") == "file" and node.get("link"):
441
+ acc.append(node)
442
+ kids = node.get("children")
443
+ if isinstance(kids, dict):
444
+ for ch in kids.values():
445
+ collect_files(ch, acc)
446
+
447
+ def download_stream(link, dest_tmp, token, referer):
448
+ h = {"User-Agent": UA, "Referer": referer, "Origin": "https://gofile.io"}
449
+ if token:
450
+ h["Authorization"] = f"Bearer {token}"
451
+ h["Cookie"] = f"accountToken={token}"
452
+ r = urllib.request.Request(link, headers=h)
453
+ dest_tmp.parent.mkdir(parents=True, exist_ok=True)
454
+ total = 0
455
+ try:
456
+ with urllib.request.urlopen(r, timeout=120) as resp, open(dest_tmp, "wb") as f:
457
+ while True:
458
+ b = resp.read(8 * 1024 * 1024)
459
+ if not b:
460
+ break
461
+ f.write(b)
462
+ total += len(b)
463
+ if total and (total // (512 * 1024 * 1024)) != ((total - len(b)) // (512 * 1024 * 1024)):
464
+ print(f"[gofile] ... {total/2**30:.2f} GiB", flush=True)
465
+ except urllib.error.HTTPError as e:
466
+ raise SystemExit(f"GoFile download HTTP {e.code}: link suresi dolmus ya da token/IP engeli olabilir.")
467
+ print(f"[gofile] stream done: {total/2**30:.2f} GiB", flush=True)
468
+
469
+ def main():
470
+ fid = resolve_id(url)
471
+ token = token_env or create_guest_token()
472
+ xwt = website_token(token)
473
+ api = (f"https://api.gofile.io/contents/{quote(fid)}"
474
+ f"?contentFilter=&page=1&pageSize=1000&sortField=name&sortDirection=1")
475
+ if password_plain:
476
+ api += f"&password={hashlib.sha256(password_plain.encode()).hexdigest()}"
477
+ data = req_json(api, token=token,
478
+ headers={"X-Website-Token": xwt, "X-BL": WT_LANG})
479
+ st = data.get("status")
480
+ if st != "ok":
481
+ if st in ("error-token", "error-wrongToken"):
482
+ raise SystemExit(
483
+ f"[gofile] {st}: token reddedildi. GOFILE_TOKEN kontrol et, restart edip tekrar dene.")
484
+ if st == "error-notPremium":
485
+ raise SystemExit(
486
+ "[gofile] error-notPremium: bu IP GoFile listeleme icin engelli "
487
+ "(HF Spaces/VPS datacenter IP). Cozum: ARCHIVE_URL secretina duz link koy, "
488
+ "ya da GOFILE_DIRECT_URL'e store linki koy, ya da /data'ya klasor yukle.")
489
+ if st == "error-notFound":
490
+ raise SystemExit(f"[gofile] {st}: link silinmis/suresi dolmus olabilir: {url}")
491
+ if st in ("error-owner", "error-notOwner"):
492
+ raise SystemExit(f"[gofile] {st}: icerik baska hesaba ait, o hesabin tokeni gerekli.")
493
+ raise SystemExit(f"GoFile API failed: {st} {str(data)[:500]}")
494
+ root = data.get("data") or {}
495
+ files = []
496
+ if root.get("type") == "file" and root.get("link"):
497
+ files = [root]
498
+ else:
499
+ collect_files(root, files)
500
+ files = [x for x in files if x.get("link")]
501
+ if not files:
502
+ raise SystemExit("No downloadable file found in GoFile folder (bos ya da sifreli olabilir).")
503
+ files.sort(key=lambda x: int(x.get("size") or 0), reverse=True)
504
+ print(f"[gofile] {len(files)} dosya, en buyuk indiriliyor:", flush=True)
505
+ for x in files[:5]:
506
+ print(f" - {x.get('name')} size={int(x.get('size') or 0)}", flush=True)
507
+ download_stream(files[0]["link"], tmp, token, referer=url)
508
+ tmp.replace(out)
509
+ print(f"[gofile] downloaded -> {out}", flush=True)
510
+ _v = subprocess.run([sys.executable, str(Path(__file__).with_name("validate_archive.py")), str(out)])
511
+ if _v.returncode != 0:
512
+ out.unlink(missing_ok=True)
513
+ raise SystemExit("[gofile] indirilen dosya gecerli arsiv degil, yukaridaki teshise bak.")
514
+
515
+ main()
516
+ PYEOF
517
+
518
+ # --- duz direkt indirme (ARCHIVE_URL / GOFILE_DIRECT_URL icin). gofile linklerinde
519
+ # token yoksa guest uretip Cookie ile gonderir (baglisiz direkt erisim cogunlukla
520
+ # HTML doner; validate yakalar). stdlib only.
521
+ cat > "$TOOLS/download_url.py" <<'PYEOF'
522
+ import json, os, subprocess, sys, urllib.request
523
+ UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
524
+ url, out = sys.argv[1], sys.argv[2]
525
+ tmp = out + ".part"
526
+ token = (os.environ.get("GOFILE_TOKEN") or "").strip()
527
+
528
+ def guest_token():
529
+ h = {"User-Agent": UA, "Accept": "application/json",
530
+ "Origin": "https://gofile.io", "Referer": "https://gofile.io/",
531
+ "Content-Type": "application/json"}
532
+ r = urllib.request.Request("https://api.gofile.io/accounts", data=b"{}",
533
+ headers=h, method="POST")
534
+ try:
535
+ with urllib.request.urlopen(r, timeout=60) as resp:
536
+ d = json.loads(resp.read().decode("utf-8", "replace"))
537
+ t = (d.get("data") or {}).get("token") or ""
538
+ return t if d.get("status") == "ok" and t else ""
539
+ except Exception as e:
540
+ print(f"[direct] guest token alinamadi: {e}", flush=True)
541
+ return ""
542
+
543
+ h = {"User-Agent": UA}
544
+ if "gofile.io" in url:
545
+ if not token:
546
+ token = guest_token()
547
+ if token:
548
+ h["Authorization"] = f"Bearer {token}"
549
+ h["Cookie"] = f"accountToken={token}"
550
+ h["Referer"] = os.environ.get("GOFILE_URL", "https://gofile.io/")
551
+ r = urllib.request.Request(url, headers=h)
552
+ d = os.path.dirname(out)
553
+ if d:
554
+ os.makedirs(d, exist_ok=True)
555
+ total = 0
556
+ with urllib.request.urlopen(r, timeout=120) as resp, open(tmp, "wb") as f:
557
+ while True:
558
+ b = resp.read(8 * 1024 * 1024)
559
+ if not b:
560
+ break
561
+ f.write(b)
562
+ total += len(b)
563
+ if total and (total // (512 * 1024 * 1024)) != ((total - len(b)) // (512 * 1024 * 1024)):
564
+ print(f"[direct] ... {total/2**30:.2f} GiB", flush=True)
565
+ os.replace(tmp, out)
566
+ print(f"[direct] downloaded {total/2**30:.2f} GiB -> {out}", flush=True)
567
+ _v = subprocess.run([sys.executable, os.path.join(os.path.dirname(os.path.abspath(__file__)), "validate_archive.py"), out])
568
+ if _v.returncode != 0:
569
+ try: os.remove(out)
570
+ except OSError: pass
571
+ raise SystemExit("[direct] indirilen dosya gecerli arsiv degil, yukaridaki teshise bak.")
572
+ PYEOF
573
+
574
+ # --- arsiv acma (rar icin unar, zip/tar stdlib). stdlib + unar.
575
+ cat > "$TOOLS/extract_archive.py" <<'PYEOF'
576
+ import shutil, subprocess, sys, tarfile, zipfile
577
+ from pathlib import Path
578
+
579
+ src, dst = Path(sys.argv[1]), Path(sys.argv[2])
580
+ name = src.name.lower()
581
+ dst.mkdir(parents=True, exist_ok=True)
582
+
583
+ if name.endswith(".zip"):
584
+ with zipfile.ZipFile(src) as z:
585
+ z.extractall(dst)
586
+ raise SystemExit(0)
587
+
588
+ if name.endswith((".tar", ".tar.gz", ".tgz", ".tar.xz", ".txz", ".tar.bz2", ".tbz2")):
589
+ with tarfile.open(src, "r:*") as t:
590
+ t.extractall(dst)
591
+ raise SystemExit(0)
592
+
593
+ if name.endswith(".rar"):
594
+ if shutil.which("unrar") is not None:
595
+ print("[extract] unrar ile aciliyor (RAR5 uyumlu) ...", flush=True)
596
+ r = subprocess.run(
597
+ ["unrar", "x", "-o+", "-y", str(src), str(dst) + "/"],
598
+ check=False,
599
+ )
600
+ raise SystemExit(r.returncode)
601
+ print("[extract] UYARI: unrar yok, unar deneniyor (RAR5 acamayabilir) ...", flush=True)
602
+ if shutil.which("unar") is None:
603
+ try:
604
+ subprocess.run(["apt-get", "update", "-qq"], check=False, timeout=120)
605
+ subprocess.run(["apt-get", "install", "-y", "-qq", "unar"], check=False, timeout=300)
606
+ except Exception:
607
+ pass
608
+ if shutil.which("unar") is None:
609
+ raise SystemExit("unrar/unar yok (.rar icin unrar gerekli). Dockerfile unrar kurulumuna bak.")
610
+ r = subprocess.run(
611
+ ["unar", "-quiet", "-force-overwrite", "-output-directory", str(dst), str(src)],
612
+ check=False,
613
+ )
614
+ raise SystemExit(r.returncode)
615
+
616
+ raise SystemExit(f"Unsupported archive type: {src.name}")
617
+ PYEOF
618
+
619
+ # --- indirilen arsiv saglam mi? (boyut + imza). Bozuk 3.5K HTML sayfalarini
620
+ # eler; teshisi loga basar. Cikis 0=saglam, 1=bozuk.
621
+ cat > "$TOOLS/validate_archive.py" <<'PYEOF'
622
+ import sys
623
+ from pathlib import Path
624
+
625
+ MIN_BYTES = 1 * 1024 * 1024
626
+ MAGIC = {
627
+ ".rar": (b"Rar!\x1a\x07", "RAR imzasi yok"),
628
+ ".zip": (b"PK", "ZIP imzasi yok (PK)"),
629
+ ".gz": (b"\x1f\x8b", "GZIP imzasi yok"),
630
+ ".tgz": (b"\x1f\x8b", "GZIP imzasi yok"),
631
+ }
632
+
633
+ p = Path(sys.argv[1])
634
+ try:
635
+ size = p.stat().st_size
636
+ except OSError as e:
637
+ print(f"[validate] yok/okunamiyor: {p}: {e}", flush=True)
638
+ raise SystemExit(1)
639
+ print(f"[validate] {p.name}: {size} bytes", flush=True)
640
+ if size < MIN_BYTES:
641
+ try:
642
+ preview = p.read_bytes()[:300].decode("utf-8", "replace")
643
+ except OSError:
644
+ preview = "?"
645
+ print(f"[validate] BOZUK: {size} bytes < {MIN_BYTES} (minik dosya = hata sayfasi?).", flush=True)
646
+ print(f"[validate] ilk 300 karakter: {preview}", flush=True)
647
+ raise SystemExit(1)
648
+ name = p.name.lower()
649
+ need = None
650
+ for ext, spec in MAGIC.items():
651
+ if name.endswith(ext):
652
+ need = spec
653
+ break
654
+ if need is not None:
655
+ with open(p, "rb") as f:
656
+ head = f.read(8)
657
+ if not head.startswith(need[0]):
658
+ print(f"[validate] BOZUK: {need[1]}. ilk baytlar: {head!r}", flush=True)
659
+ raise SystemExit(1)
660
+ print("[validate] saglam.", flush=True)
661
+ PYEOF
662
+
663
+ # --- TMP'deki acilmis agaci tek katman mirror/ duzenine tasir.
664
+ # mirror.rar icindeki mirror/ veya .mirror/ sarmalini seffaf gecer, boylece
665
+ # mirror/mirror/ nesting olmaz. Giris: mirror/playgta5.com/index.html (tercih)
666
+ # ya da mirror/index.html (duz arsiv). Cikis 0=tamam, 1=giris yok.
667
+ cat > "$TOOLS/layout_mirror.py" <<'PYEOF'
668
+ import shutil, sys
669
+ from pathlib import Path
670
+
671
+ tmp, dst = Path(sys.argv[1]), Path(sys.argv[2])
672
+ WRAP = ("mirror", ".mirror")
673
+ SKIP = ("__MACOSX",)
674
+
675
+ node = tmp
676
+ for _ in range(3):
677
+ try:
678
+ kids = [k for k in node.iterdir() if k.name not in SKIP]
679
+ except OSError as e:
680
+ print(f"[layout] okunamiyor: {node}: {e}", flush=True)
681
+ raise SystemExit(1)
682
+ if len(kids) == 1 and kids[0].is_dir() and kids[0].name in WRAP:
683
+ print(f"[layout] sarmal kaldirildi: {kids[0].name}/", flush=True)
684
+ node = kids[0]
685
+ else:
686
+ break
687
+
688
+ dst.mkdir(parents=True, exist_ok=True)
689
+ if any(dst.iterdir()):
690
+ print(f"[layout] BOZUK: hedef dolu, once temizlenmeli: {dst}", flush=True)
691
+ raise SystemExit(1)
692
+ moved = 0
693
+ for k in node.iterdir():
694
+ if k.name in SKIP or k.name.endswith(".DS_Store"):
695
+ continue
696
+ shutil.move(str(k), str(dst / k.name))
697
+ moved += 1
698
+ shutil.rmtree(tmp, ignore_errors=True)
699
+ print(f"[layout] {moved} oge -> {dst}", flush=True)
700
+
701
+ for cand in (dst / "playgta5.com" / "index.html", dst / "playgta5.com" / "homepage.html",
702
+ dst / "index.html", dst / "homepage.html"):
703
+ if cand.exists():
704
+ print(f"entry: {cand.relative_to(dst)}", flush=True)
705
+ raise SystemExit(0)
706
+ top = sorted(k.name for k in dst.iterdir())[:15]
707
+ print(f"[layout] BOZUK: giris (index.html) yok. ust duzey: {top}", flush=True)
708
+ raise SystemExit(1)
709
+ PYEOF
710
+
711
+ # ---------------------------------------------------------------------------
712
+ # 3) Data load: pick the fastest available source
713
+ # ---------------------------------------------------------------------------
714
+ mark_done() { date -u +%Y-%m-%dT%H:%M:%SZ > "$DONE_MARK"; }
715
+
716
+ # background: pack local tree into N tar shards in the bucket so next cold starts are fast
717
+ build_shards_bg() {
718
+ [ "$BUILD_SHARDS" = "1" ] || { echo "[shards] skipped (BUILD_SHARDS=0)"; return 0; }
719
+ (
720
+ set +e
721
+ nice -n 10 true
722
+ echo "[shards] building $SHARDS tar shards in bucket (background) ..."
723
+ L="/runtime/.shard-lists"; rm -rf "$L"
724
+ python3 "$TOOLS/make_shards.py" "$DST" "$L" "$SHARDS" || exit 0
725
+ rm -f "$BUCKET_DIR"/harbor-heist.shard*.tar "$BUCKET_DIR"/harbor-heist.shard*.tar.part "$BUCKET_DIR/.harbor-shards-ready"
726
+ seq 0 $((SHARDS-1)) | xargs -P "$SHARDS" -I{} sh -c \
727
+ "nice -n 10 tar -cf '$BUCKET_DIR/harbor-heist.shard{}.tar.part' -C '$DST' --null -T '$L/shard_{}.lst' \
728
+ && mv '$BUCKET_DIR/harbor-heist.shard{}.tar.part' '$BUCKET_DIR/harbor-heist.shard{}.tar'"
729
+ if [ "$(ls "$BUCKET_DIR"/harbor-heist.shard*.tar 2>/dev/null | wc -l)" -eq "$SHARDS" ]; then
730
+ echo "$SHARDS" > "$BUCKET_DIR/.harbor-shards-ready"
731
+ echo "[shards] ready ($SHARDS shards)."
732
+ else
733
+ echo "[shards] incomplete, will retry next boot." >&2
734
+ fi
735
+ ) &
736
+ }
737
+
738
+ # Eski bucket kalabaligini SADECE yeni arsiv dogrulandiktan sonra temizle.
739
+ # (Indirme basarisizken silmek veri kaybina yol acar.)
740
+ clean_legacy_bucket_after_success() {
741
+ echo "[bucket] yeni arsiv saglam, legacy kalabaligi temizliyorum ..."
742
+ rm -rf \
743
+ "$BUCKET_DIR/harbor-heist" \
744
+ "$BUCKET_DIR/mirror" \
745
+ "$BUCKET_DIR"/harbor-heist.shard*.tar \
746
+ "$BUCKET_DIR"/harbor-heist.shard*.tar.part \
747
+ "$BUCKET_DIR"/newgame.* \
748
+ "$BUCKET_DIR"/harbor-heist.zip \
749
+ "$BUCKET_DIR"/harbor-heist.tar* \
750
+ "$BUCKET_DIR"/harbor-heist.rar \
751
+ "$BUCKET_DIR/.harbor-ready" \
752
+ "$BUCKET_DIR/.mirror-ready" \
753
+ "$BUCKET_DIR/.harbor-shards-ready" \
754
+ "$BUCKET_DIR/.ingest-ready" || true
755
+ }
756
+
757
+ ensure_bucket_archive() {
758
+ mkdir -p "$BUCKET_DIR"
759
+ local marker=""
760
+ if [ -f "$VERSION_FILE" ]; then
761
+ marker="$(cat "$VERSION_FILE" 2>/dev/null || true)"
762
+ fi
763
+ if [ "$marker" = "$DATA_VERSION" ] && [ -s "$ARCHIVE_BUCKET_PATH" ]; then
764
+ return 0
765
+ fi
766
+ # 0) Duz direkt link: GoFile API'ye hic dokunmaz (datacenter dostu)
767
+ if [ -n "$ARCHIVE_URL" ]; then
768
+ echo "[bucket] ARCHIVE_URL'den cekiliyor ..."
769
+ if python3 "$TOOLS/download_url.py" "$ARCHIVE_URL" "$ARCHIVE_BUCKET_PATH"; then
770
+ echo "$DATA_VERSION" > "$VERSION_FILE"
771
+ date -u +%Y-%m-%dT%H:%M:%SZ > "$BUCKET_DIR/.ingest-ready"
772
+ clean_legacy_bucket_after_success
773
+ return 0
774
+ else
775
+ echo "[bucket] ARCHIVE_URL indirmesi basarisiz, siradaki kaynaga geciliyor." >&2
776
+ rm -f "$ARCHIVE_BUCKET_PATH.part"
777
+ fi
778
+ fi
779
+ # 1) GoFile direkt store linki: /contents listesi atlanir
780
+ if [ -n "$GOFILE_DIRECT_URL" ]; then
781
+ echo "[bucket] GOFILE_DIRECT_URL'den cekiliyor (liste atlandi) ..."
782
+ if python3 "$TOOLS/download_url.py" "$GOFILE_DIRECT_URL" "$ARCHIVE_BUCKET_PATH"; then
783
+ echo "$DATA_VERSION" > "$VERSION_FILE"
784
+ date -u +%Y-%m-%dT%H:%M:%SZ > "$BUCKET_DIR/.ingest-ready"
785
+ clean_legacy_bucket_after_success
786
+ return 0
787
+ else
788
+ echo "[bucket] GOFILE_DIRECT_URL basarisiz." >&2
789
+ rm -f "$ARCHIVE_BUCKET_PATH.part"
790
+ fi
791
+ fi
792
+ # 2) Klasik GoFile klasor linki (ev IP'sinde calisir, datacenterda error-notPremium verebilir)
793
+ echo "[bucket] fetching release from $GOFILE_URL ..."
794
+ if GOFILE_TOKEN="$GOFILE_TOKEN" GOFILE_PASSWORD="$GOFILE_PASSWORD" \
795
+ python3 "$TOOLS/download_gofile.py" "$GOFILE_URL" "$ARCHIVE_BUCKET_PATH"; then
796
+ echo "$DATA_VERSION" > "$VERSION_FILE"
797
+ date -u +%Y-%m-%dT%H:%M:%SZ > "$BUCKET_DIR/.ingest-ready"
798
+ clean_legacy_bucket_after_success
799
+ return 0
800
+ else
801
+ echo "[bucket] GoFile indirmesi basarisiz (yukaridaki [gofile] satirina bak)." >&2
802
+ rm -f "$ARCHIVE_BUCKET_PATH.part"
803
+ return 1
804
+ fi
805
+ }
806
+
807
+ SHARDS_READY=0
808
+ # Panelden kaydedilen linkler (yazan: /panel, okuyan: ensure_bucket_archive)
809
+ if [ -f "$BUCKET_DIR/.panel-env" ]; then
810
+ echo "[panel] kayitli linkler yukleniyor ..."
811
+ if [ -z "$_RAW_DIRECT" ]; then
812
+ eval "$(grep -E '^GOFILE_DIRECT_URL=https?://' "$BUCKET_DIR/.panel-env" 2>/dev/null || true)"
813
+ fi
814
+ if [ -z "$_RAW_ARCHIVE" ]; then
815
+ eval "$(grep -E '^ARCHIVE_URL=https?://' "$BUCKET_DIR/.panel-env" 2>/dev/null || true)"
816
+ fi
817
+ fi
818
+ export GOFILE_URL GOFILE_DIRECT_URL ARCHIVE_URL
819
+ if [ -f "$BUCKET_DIR/.harbor-shards-ready" ]; then
820
+ SHARDS_READY="$(cat "$BUCKET_DIR/.harbor-shards-ready" 2>/dev/null || echo 0)"
821
+ fi
822
+ LOADED=0
823
+ SOURCE=none
824
+ t0=$SECONDS
825
+
826
+ if ! ensure_bucket_archive; then
827
+ echo "[bucket] WARNING: archive not ready in bucket, fallback sources will be used." >&2
828
+ fi
829
+
830
+ if [ -f "$DONE_MARK" ] || [ -f "$MIRROR_DONE" ]; then
831
+ echo "[data] local copy already complete, skipping load."
832
+ LOADED=1; SOURCE=local-cache
833
+
834
+ elif [ "$SHARDS_READY" -gt 0 ] 2>/dev/null; then
835
+ echo "[data] source: tar shards ($SHARDS_READY parallel streams) ..."
836
+ mkdir -p "$DST"
837
+ ( while sleep 5; do echo "[tar] local size: $(du -sm "$DST" 2>/dev/null | cut -f1) MiB elapsed=$((SECONDS-t0))s"; done ) &
838
+ PROG_PID=$!
839
+ if ls "$BUCKET_DIR"/harbor-heist.shard*.tar | xargs -P "$SHARDS_READY" -I{} tar -xf {} -C "$DST" --no-same-owner; then
840
+ LOADED=1; SOURCE=tar-shards
841
+ else
842
+ echo "[data] shard extract failed, falling back." >&2
843
+ fi
844
+ kill "$PROG_PID" 2>/dev/null || true
845
+
846
+ fi
847
+
848
+ if [ "$LOADED" -eq 0 ] && [ -s "$ARCHIVE_BUCKET_PATH" ]; then
849
+ echo "[data] source: bucket archive ($ARCHIVE_NAME) -> validate + extract to mirror/ ..."
850
+ if ! python3 "$TOOLS/validate_archive.py" "$ARCHIVE_BUCKET_PATH"; then
851
+ echo "[data] bucket arsivi bozuk, silinip sonraki boot'ta yeniden indirilecek." >&2
852
+ rm -f "$ARCHIVE_BUCKET_PATH" "$VERSION_FILE"
853
+ else
854
+ rm -rf "$MIRROR_DST"; mkdir -p "$MIRROR_DST"
855
+ TMPX="$LOCAL_DATA_DIR/.mirror-tmp"; rm -rf "$TMPX"; mkdir -p "$TMPX"
856
+ LOCAL_ARCHIVE="$LOCAL_DATA_DIR/$ARCHIVE_NAME"
857
+ echo "[data] bucket->local kopyalaniyor: $(du -h "$ARCHIVE_BUCKET_PATH" 2>/dev/null | cut -f1) ..."
858
+ cp -f "$ARCHIVE_BUCKET_PATH" "$LOCAL_ARCHIVE"
859
+ if python3 "$TOOLS/extract_archive.py" "$LOCAL_ARCHIVE" "$TMPX"; then
860
+ rm -f "$LOCAL_ARCHIVE"
861
+ if python3 "$TOOLS/layout_mirror.py" "$TMPX" "$MIRROR_DST"; then
862
+ LOADED=1; SOURCE=mirror-archive
863
+ date -u +%Y-%m-%dT%H:%M:%SZ > "$MIRROR_DONE"
864
+ else
865
+ echo "Mirror duzeni kurulamadi. Arsiv ic yapisini kontrol et." >&2
866
+ rm -rf "$TMPX" "$MIRROR_DST"
867
+ fi
868
+ else
869
+ rm -f "$LOCAL_ARCHIVE"; rm -rf "$TMPX"
870
+ echo "Archive extract FAILED." >&2
871
+ fi
872
+ fi
873
+ fi
874
+
875
+ if [ "$LOADED" -eq 0 ] && [ -f "$BUCKET_DIR/mirror/newgame.zip" ]; then
876
+ echo "[data] source: newgame.zip -> parallel extract straight to local disk ($IO_WORKERS threads) ..."
877
+ rm -rf "$DST"; mkdir -p "$DST"
878
+ if python3 "$TOOLS/unzip_par.py" "$BUCKET_DIR/mirror/newgame.zip" "$DST" "$IO_WORKERS"; then
879
+ if python3 "$TOOLS/find_entry.py" "$DST"; then
880
+ LOADED=1; SOURCE=zip
881
+ build_shards_bg
882
+ else
883
+ echo "Giris dosyasi bulunamadi. Zip ic yapisini kontrol et." >&2
884
+ fi
885
+ else
886
+ echo "Zip extract FAILED. Yukaridaki UNZIP-ERR satirlarina bak." >&2
887
+ fi
888
+ fi
889
+
890
+ if [ "$LOADED" -eq 0 ] && [ -d "$BUCKET_DIR/harbor-heist" ] && [ -f "$BUCKET_DIR/.harbor-ready" ]; then
891
+ echo "[data] source: legacy bucket folder -> range-parallel copy ($IO_WORKERS threads, ${RANGE_MB}MB ranges) ..."
892
+ mkdir -p "$DST"
893
+ if python3 "$TOOLS/copy_par.py" "$BUCKET_DIR/harbor-heist" "$DST" "$IO_WORKERS" "$RANGE_MB"; then
894
+ LOADED=1; SOURCE=legacy-folder
895
+ build_shards_bg
896
+ else
897
+ echo "Bucket -> disk copy FAILED. COPY-ERR satirlarina bak." >&2
898
+ fi
899
+ fi
900
+
901
+ if [ "$LOADED" -eq 0 ] && [ -d "$BUCKET_DIR/mirror" ] && [ -f "$BUCKET_DIR/.mirror-ready" ]; then
902
+ echo "[data] source: legacy mirror/ -> range-parallel copy ..."
903
+ mkdir -p "$LOCAL_DATA_DIR/mirror"
904
+ if python3 "$TOOLS/copy_par.py" "$BUCKET_DIR/mirror" "$LOCAL_DATA_DIR/mirror" "$IO_WORKERS" "$RANGE_MB"; then
905
+ SOURCE=legacy-mirror
906
+ else
907
+ echo "WARNING: mirror copy failed, continuing without it." >&2
908
+ fi
909
+ fi
910
+
911
+ if [ "$LOADED" -eq 1 ] && [ "$SOURCE" != "local-cache" ]; then
912
+ mark_done
913
+ fi
914
+ if [ "$LOADED" -eq 0 ] && [ "$SOURCE" = "none" ]; then
915
+ echo "No usable game data in $BUCKET_DIR. Starting in SETUP mode." >&2
916
+ fi
917
+ echo "[data] source=$SOURCE loaded=$LOADED in $((SECONDS-t0))s"
918
+
919
+ # ---------------------------------------------------------------------------
920
+ # 4) Wait for clone, then serve
921
+ # ---------------------------------------------------------------------------
922
+ if ! wait "$CLONE_PID"; then
923
+ echo "git clone failed. Check GH_REPO_TOKEN / GH_REPO_SLUG / GH_REPO_REF." >&2
924
+ exit 1
925
+ fi
926
+
927
+ cd "$APP_DIR"
928
+ if [ ! -f space_server.py ]; then
929
+ echo "space_server.py not found in repo root. Check GH_REPO_SLUG/REF." >&2
930
+ exit 1
931
+ fi
932
+
933
+ echo "[boot] total bootstrap time: $((SECONDS-T_START))s -> starting server on :$PORT"
934
+ exec python3 space_server.py --port "$PORT" --app-dir "$APP_DIR" --local-data "$LOCAL_DATA_DIR" \
935
+ --bucket-dir "$BUCKET_DIR" --boot-log "$LOG_FILE" --tools-dir "$TOOLS" --data-version "$DATA_VERSION"