Upload 10 files

refresh_sources.py

@@ -22,13 +22,32 @@ CH_URL = "http://127.0.0.1:8123"
 
 def run_subprocess(cmd, cwd=None, env=None):
     try:
-        result = subprocess.run(
+        # We use Popen to stream stderr to the console for progress bars
+        # while still capturing stdout for return values if needed.
+        process = subprocess.Popen(
             cmd, cwd=cwd, env=env,
-            capture_output=True, text=True, check=True
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
         )
-        return result.stdout.strip(), True
-    except subprocess.CalledProcessError as e:
-        return e.output.strip() + "\n" + e.stderr.strip(), False
+        
+        # Stream stderr in real time
+        stderr_output = []
+        while True:
+            line = process.stderr.readline()
+            if not line and process.poll() is not None:
+                break
+            if line:
+                sys.stderr.write(line)
+                sys.stderr.flush()
+                stderr_output.append(line)
+        
+        stdout, _ = process.communicate()
+        
+        if process.returncode == 0:
+            return stdout.strip(), True
+        else:
+            return "".join(stderr_output).strip(), False
+    except Exception as e:
+        return str(e), False
 
 
 def execute_clickhouse_query(sql):

sources.yaml

@@ -28,20 +28,7 @@ sources:
       - view_name: hf_ohlc
         file_glob: "**/*.parquet"
         format: Parquet
-        description: "Full OHLCV data from HF dataset"
-        # columns:                             # explicit column list for the VIEW
-        #   - timestamp
-        #   - symbol
-        #   - isin
-        #   - series
-        #   - open
-        #   - high
-        #   - low
-        #   - close
-        #   - volume
-        #   - interval_minutes
-        #   - segment
-        #   - exchange
+        description: "Full OHLCV data from HF dataset, selects all 150+ columns automatically"
 
   # ── EXAMPLE: Adding a second data source ──────────────────────────────
   # Just uncomment and edit: