Document bucket files, improve upload & parsing

Enable DB startup and improve bucket handling, seed documentation, and ffmpeg progress parsing.

Changes:
- index.ts: await start_mongo_db() so DB initialization runs.
- lib/bucket.ts: upload_to_bucket now accepts bucket_path and optional REQ0, logs success/failure, notifies caller via REQ0, and uses bucket_path for stored object path.
- sub-router/showtime/helper.ts: stricter skip checks, added FileDownloadInfoOutput import, support document flag, and added document_seed_file to upsert bucket records into Mongo when a bucket file is used.
- sub-router/showtime/v2.ts: pass document: true to find_seed_file, remove stray logs, set fallback=false for proxy requests, unify redirect to proxy_url or original stream, update upload_to_bucket call to include bucket_path and REQ0, and fix progress parsing to handle size units (kB/MB/GB) converting to bytes.
- packages/lib/helpers/client/route.ts: F_is_url now accepts any input type (guards non-string).

These changes ensure uploaded files are stored at correct paths, callers can be notified after upload, bucket-sourced files get recorded in the DB, URL handling is more robust, and ffmpeg progress reporting computes byte sizes correctly.

src/index.ts

@@ -11,7 +11,7 @@ import { NS_runtime } from "../../../../packages/lib/env.runtime";
 import { NS_workers_proxy } from "../../../../packages/lib/proxy-server/workers-proxy";
 import { start_mongo_db } from "../../../../packages/db/mongo-db/init";
 
-// await start_mongo_db();
+await start_mongo_db();
 
 const { rewriteWorkersUrl, startWorkersProxy, rewriteWorkersUrl_t2, get_dns } =
   NS_workers_proxy;

src/lib/bucket.ts

@@ -232,12 +232,14 @@ async function get_latest_hf_bucket_file_by_name(arg0: {
 
 async function upload_to_bucket(arg0: {
   job_id: string;
+  bucket_path: string;
   file_with_path: string;
   skip?: {
     probe?: boolean;
   };
+  REQ0?: URL;
 }) {
-  const { job_id, file_with_path, skip } = arg0;
+  const { job_id, bucket_path, skip, file_with_path, REQ0 } = arg0;
 
   upload_jobs.set(job_id, {
     status: "in-progress",
@@ -272,20 +274,26 @@ async function upload_to_bucket(arg0: {
       accessToken: credentials.accessToken,
       files: [
         {
-          path: file_with_path,
+          path: bucket_path,
           content: blob
         }
       ]
     })
   });
 
-  console.log("\n", "🔥 UPLOAD BUCKET RESULTS: ", { E__ }, { R__ }, "\n");
-
   upload_jobs.delete(job_id);
 
   if (E__) {
+    console.log("\n", "❌ UPLOAD BUCKET FAILED", E__, "\n");
     return { success: false };
   }
 
+  console.log("\n", "🔥 UPLOAD BUCKET DONE", "\n");
+
+  if (REQ0) {
+    REQ0.searchParams.set("ops", "just_check");
+    fetch(REQ0.toString());
+  }
+
   return { success: true };
 }

src/sub-router/showtime/helper.ts

@@ -5,6 +5,7 @@ import { NS_bucket } from "../../lib/bucket";
 import { NS_promise } from "../../lib/promise";
 import node_fs from "node:fs";
 import { NS_ffmpeg_v2 } from "../../lib/ffmpeg-v2";
+import { FileDownloadInfoOutput } from "@huggingface/hub";
 
 export const NS_showtime_v2_helper = {
   find_seed_file
@@ -28,6 +29,7 @@ async function find_seed_file(arg0: {
     db?: boolean;
     bucket?: boolean;
   };
+  document?: boolean;
 }) {
   const {
     sbe_id,
@@ -37,7 +39,8 @@ async function find_seed_file(arg0: {
     job_id,
     S3Dir,
     cleanup_corruption,
-    skip
+    skip,
+    document
   } = arg0;
 
   let { picture_quality } = arg0;
@@ -49,7 +52,7 @@ async function find_seed_file(arg0: {
   // check database first
   const [_, db_file_info] = await NS_promise.F_catch_promise({
     promise: (async () => {
-      if (skip?.db) return;
+      if (skip?.db === true) return;
 
       const _ =
         await NS_mongo_db_record_factory.NS_record_factory_s3_bucket.findRecordsByFingerprint(
@@ -67,10 +70,10 @@ async function find_seed_file(arg0: {
     })()
   });
 
-  const in_db = db_file_info?.url && NS_routes_c.F_is_url(db_file_info.url);
+  const in_db = NS_routes_c.F_is_url(db_file_info?.url);
 
   if (in_db) {
-    const stream_link = db_file_info.url;
+    const stream_link = db_file_info!.url;
 
     const _u = new URL(stream_link);
     _u.searchParams.set("download", "true");
@@ -85,7 +88,7 @@ async function find_seed_file(arg0: {
   // check bucket
   const [__, bucket_file_info] = await NS_promise.F_catch_promise({
     promise: (async () => {
-      if (skip?.bucket) return;
+      if (skip?.bucket === true) return;
 
       return NS_bucket.get_latest_hf_bucket_file_by_name({
         folder_path: cache_folder,
@@ -95,16 +98,26 @@ async function find_seed_file(arg0: {
     })()
   });
 
-  console.log({ bucket_file_info }, "\n");
-
-  const in_bucket =
-    bucket_file_info?.url && NS_routes_c.F_is_url(bucket_file_info.url);
+  const in_bucket = NS_routes_c.F_is_url(bucket_file_info?.url);
 
   if (in_bucket) {
-    return bucket_file_info.url;
+    const stream_link = bucket_file_info!.url;
+
+    if (!in_db && skip?.db !== true && document === true) {
+      document_seed_file({
+        job_id,
+        sbe_id,
+        file_id,
+        stream_link,
+        name: output_destination.og,
+        file_info: bucket_file_info!
+      });
+    }
+
+    return stream_link;
   }
 
-  if (skip?.local) return;
+  if (skip?.local === true) return;
 
   // check local
   const file =
@@ -224,3 +237,36 @@ async function find_seed_file(arg0: {
 async function find_seed_local_file(arg0: {}) {
   const {} = arg0;
 }
+
+async function document_seed_file(arg0: {
+  job_id: string;
+  sbe_id: string;
+  file_id: string;
+  stream_link: string;
+  file_info: FileDownloadInfoOutput;
+  name: string;
+}) {
+  const { job_id, sbe_id, file_id, stream_link, file_info, name } = arg0;
+
+  const [E_, R_] = await NS_promise.F_catch_promise({
+    promise:
+      NS_mongo_db_record_factory.NS_record_factory_s3_bucket.upsertRecord(
+        { job_id, sbe_id, file_id },
+        {
+          provider: "hugging face",
+          job_id,
+          sbe_id,
+          file_id,
+          stream_link,
+          file_info,
+          name
+        }
+      )
+  });
+
+  if (E_) {
+    return E_;
+  }
+
+  return R_;
+}

src/sub-router/showtime/v2.ts

@@ -116,8 +116,6 @@ const sub_router = new Elysia({ prefix: "/showtimeV2" }).get(
 
         const filePath = path.join(S3Dir, file_id);
 
-        // console.log("\n", "FILE PATH:", filePath, "\n");
-
         return filePath;
       } catch (e) {}
     })();
@@ -254,7 +252,8 @@ const sub_router = new Elysia({ prefix: "/showtimeV2" }).get(
           file_id: clean_file_id,
           job_id,
           S3Dir,
-          cleanup_corruption: true
+          cleanup_corruption: true,
+          document: true
         })
       });
 
@@ -307,17 +306,16 @@ const sub_router = new Elysia({ prefix: "/showtimeV2" }).get(
                   use_h_version: true
                 },
                 env: "cloudflare",
-                fallback: true
+                fallback: false
               }
             })
             .json()
         });
 
-        const data = R_ as any;
+        const new_stream_link =
+          (R_ as any)?.proxy_url || stream_link.toString();
 
-        if (data?.proxy_url && NS_routes_c.F_is_url(data?.proxy_url)) {
-          return ctx.redirect(data?.proxy_url, 301);
-        }
+        return ctx.redirect(new_stream_link, 301);
       } else {
         // if seed is file set headers
         construct_response_headers({
@@ -463,9 +461,7 @@ const sub_router = new Elysia({ prefix: "/showtimeV2" }).get(
             });
 
             if (progress_data && Object.entries(progress_data).length) {
-              // seek_size_bytes =
-              //   (parseFloat(`${seek_size_bytes}`) || 0) +
-              //   (parseFloat(`${progress_data.size_bytes}`) || 0);
+              seek_size_bytes = parseFloat(`${progress_data.size_bytes}`) || 0;
 
               if (APP_ENV === "dev") console.log(progress_data, "\n");
 
@@ -523,7 +519,12 @@ const sub_router = new Elysia({ prefix: "/showtimeV2" }).get(
             });
 
             if (delivery_mode === "download" && file_with_path) {
-              NS_bucket.upload_to_bucket({ job_id, file_with_path });
+              NS_bucket.upload_to_bucket({
+                job_id,
+                bucket_path: `${cache_folder}/${file_id}`,
+                file_with_path,
+                REQ0
+              });
             }
           }
         } else {
@@ -644,17 +645,26 @@ function F_stderr_parse_progress(arg0: {
   const progress: Record<string, any> = {};
 
   const fps = stderr_line.match(/fps=\s*(\d+)/);
-  const size = stderr_line.match(/size=\s*(\d+)/);
+  const size = stderr_line.match(/size=\s*([\d.]+)([kMGT]?B)/);
   const time = stderr_line.match(/time=(\d{2}:\d{2}:\d{2}\.\d{2})/);
   const speed = stderr_line.match(/speed=\s*([\d.]+x)/);
   const bitrate = stderr_line.match(/bitrate=\s*([\d.]+kbits\/s)/);
 
-  if (!seek_size_bytes) {
-    seek_size_bytes = size ? parseFloat(`${size[1]}`) : 0;
-  }
-
   if (fps) progress.fps = parseInt(fps[1]);
   if (size) {
+    const value = parseFloat(size[1]);
+    const unit = size[2] || "B";
+
+    const multipliers: Record<string, number> = {
+      B: 1,
+      kB: 1024,
+      MB: 1024 ** 2,
+      GB: 1024 ** 3,
+      TB: 1024 ** 4
+    };
+
+    const seek_size_bytes = Math.round(value * (multipliers[unit] || 1));
+
     progress.size_bytes = seek_size_bytes;
     progress.size = NS_ffmpeg.F_format_size(seek_size_bytes);
   }