Update app.py

app.py

@@ -71,6 +71,12 @@ def fetch_and_upload_data():
                                 if not data or data[-1][0] == last_timestamp:
                                     break
                                     
+                                # Validate data structure
+                                if not all(len(candle) == 6 for candle in data):
+                                    msg = f"Invalid data structure for {symbol} {timeframe} {data_type}"
+                                    output_messages.append(msg)
+                                    break
+                                
                                 ohlcv.extend(data)
                                 last_timestamp = data[-1][0]
                                 current_since = data[-1][0] + 1
@@ -90,6 +96,13 @@ def fetch_and_upload_data():
 
                         # Convert to DataFrame and process
                         df = pd.DataFrame(ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
+                        
+                        # Validate data types and ranges
+                        if df['timestamp'].isna().any() or df['open'].isna().any():
+                            msg = f"Invalid data detected for {symbol} {timeframe} {data_type}"
+                            output_messages.append(msg)
+                            continue
+                            
                         df['datetime'] = pd.to_datetime(df['timestamp'], unit='ms')
                         df['symbol'] = symbol
                         df['market_type'] = data_type
@@ -99,11 +112,26 @@ def fetch_and_upload_data():
                         # Save to CSV
                         base_symbol = symbol.split('/')[0]
                         filename = f"{base_symbol}_USDT_Binance_{data_type}_{timeframe}.csv"
-                        subfolder = "data/futures" if data_type == "futures" else "data/spot"
+                        subfolder = f"data/{data_type}"
                         os.makedirs(subfolder, exist_ok=True)
                         filepath = os.path.join(subfolder, filename)
+                        
+                        # Verify timeframe consistency
+                        if timeframe == '1d':
+                            time_diff = df['timestamp'].diff().median() / (1000 * 60 * 60 * 24)
+                            if not (0.9 < time_diff < 1.1):  # Allow 10% deviation
+                                msg = f"Inconsistent daily intervals detected for {symbol} {timeframe} {data_type}"
+                                output_messages.append(msg)
+                                continue
+                        elif timeframe == '1h':
+                            time_diff = df['timestamp'].diff().median() / (1000 * 60 * 60)
+                            if not (0.9 < time_diff < 1.1):  # Allow 10% deviation
+                                msg = f"Inconsistent hourly intervals detected for {symbol} {timeframe} {data_type}"
+                                output_messages.append(msg)
+                                continue
+                        
                         df.to_csv(filepath, index=False)
-                            
+                        
                         # Upload to Hugging Face
                         api = HfApi(token=HF_TOKEN)
                         api.upload_file(
@@ -113,7 +141,7 @@ def fetch_and_upload_data():
                             repo_type="dataset",
                             commit_message=f"Update {filename}"
                         )
-                        msg = f"Successfully uploaded {filepath}"
+                        msg = f"Successfully uploaded {filepath} ({len(df)} records)"
                         output_messages.append(msg)
                         latest_output = output_messages.copy()