Create app.py

app.py

@@ -0,0 +1,79 @@
+import requests
+import json
+import time
+import csv
+from datetime import datetime, timedelta
+from datasets import Dataset, load_dataset
+import pandas as pd
+from huggingface_hub import HfApi
+
+def get_data(symbol, interval, start_time):
+    url = f"https://contract.mexc.com/api/v1/contract/kline/{symbol}?interval={interval}"
+    if start_time:
+        url += f"&start_time={start_time}"
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        print(f"Error during request: {e}")
+        return None
+
+def process_data(data):
+    if not data or not data.get('data'):
+        return []
+
+    processed_data = []
+    for item in data['data']:
+        timestamp, open_price, high_price, low_price, close_price, volume, amount = item
+        processed_data.append({
+            "time": int(timestamp),
+            "open": float(open_price),
+            "high": float(high_price),
+            "low": float(low_price),
+            "close": float(close_price),
+            "volume": float(volume),
+            "amount": float(amount)
+        })
+    return processed_data
+
+def create_huggingface_dataset(data, dataset_id):
+    df = pd.DataFrame(data)
+    dataset = Dataset.from_pandas(df)
+    dataset.push_to_hub(dataset_id)
+    return dataset
+
+
+def main():
+    symbol = "BTC_USDT"
+    interval = "Min1"
+    dataset_id = "ckcl/mexc_new"  # Replace with your Hugging Face Dataset ID
+    header = ["time", "open", "high", "low", "close", "volume", "amount"]
+    last_time = None
+    api = HfApi()
+
+
+    while True:
+        data = get_data(symbol, interval, last_time)
+        if data:
+            processed_data = process_data(data)
+
+            if processed_data:
+                try:
+                    dataset = load_dataset(dataset_id) # Load from the Hub if it exists
+                    df = pd.concat([pd.DataFrame(dataset['train']), pd.DataFrame(processed_data)], ignore_index=True)
+                    dataset = Dataset.from_pandas(df) # append new data
+                except Exception as e:
+                    print(f"Error loading dataset: {e} - create from new")
+                    dataset = create_huggingface_dataset(processed_data, dataset_id) # create new dataset if it doesnt exist
+                dataset.push_to_hub(dataset_id, private = True) # Overwrite the dataset on the hub
+                last_time =  processed_data[-1]['time']
+                print(f"Data saved to Huggingface Hub: {dataset_id}. last time {last_time}")
+            else:
+                print("No new data")
+        else:
+            print("Could not retrieve data")
+        time.sleep(60) # Check MEXC API for rate limits
+if __name__ == "__main__":
+    main()