add scheduler

.gitignore

@@ -1,5 +1,6 @@
 *.ipynb
 script/
+data/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

service/api.py

@@ -1,5 +1,7 @@
+import os
 import time
 import json
+import requests
 from typing import Any
 from .llm import search_bot
 from .vector_store import vector_store
@@ -47,4 +49,17 @@ class API():
             text += i
             yield i
 
+    def get_filecoin_price(self):
+        url="https://api.spacescope.io/v2/price/realtime"
+        headers = {
+        'authorization': f'Bearer {os.environ["SPACESCOPE_API_KEY"]}'
+        }
+        data = None
+        try:
+            response = requests.get(url, headers=headers)
+            data = json.loads(response.text)['data']['price']
+        except Exception as e:
+            print(e)
+        return data
+
 api = API()

service/dataset.py

@@ -0,0 +1,58 @@
+import os
+import json
+from typing import Any
+import pytz
+from pathlib import Path
+from huggingface_hub import CommitScheduler
+from datetime import datetime
+from datasets import load_dataset
+
+
+class Dataset:
+
+    DIR = os.path.join(os.path.dirname(__file__),"..","data")
+
+    def __init__(self):
+        self.scheduler = CommitScheduler(
+            repo_id="Jarvis-Log",
+            repo_type="dataset",
+            folder_path=self.DIR,
+            path_in_repo="data",
+            token=os.getenv("HF_TOKEN")
+        )
+
+    def current_time(self):
+        beijing_tz = pytz.timezone('Asia/Shanghai')
+        beijing_time = datetime.now(beijing_tz)
+        format_time = beijing_time.strftime('%Y-%m-%d %H:%M:%S')
+        return format_time
+    
+    def current_date(self):
+        beijing_tz = pytz.timezone('Asia/Shanghai')
+        beijing_time = datetime.now(beijing_tz)
+        format_date = beijing_time.strftime('%Y-%m-%d')
+        return format_date
+     
+    def _write_log(self, data, dir_name):
+        path = Path(self.DIR,dir_name) 
+        path.mkdir(parents=True, exist_ok=True)
+        path = path/f'{self.current_date()}.json'
+        data['datetime'] = self.current_time()
+        data = json.dumps(data)+"\n"
+        with self.scheduler.lock:
+            with path.open("a") as f:
+                f.write(data)
+    
+    def write_dataset(self,data):
+        self._write_log(data,"dataset")
+
+    def write_log(self,data):
+        self._write_log(data,"log")
+    
+    def __call__(self,data):
+        print(f"[{self.current_time()}]{data}")
+    
+    def load_dataset(self):
+        dataset = load_dataset("JR818/Jarvis-Log",data_dir="data/dataset",ignore_verifications=True)
+        return dataset.with_format("pandas").data
+dataset = Dataset()

service/scheduler.py

@@ -0,0 +1,27 @@
+import datetime
+import json
+import requests
+import os
+from apscheduler.schedulers.background import BackgroundScheduler
+from api import api
+from .vector_store import vector_store
+
+class Scheduler:
+
+    def __init__(self):
+        self.scheduler = BackgroundScheduler()
+        self.jobs = [self.update_filecoin_price]
+    
+    def run(self):
+        for job in self.jobs:
+            self.scheduler.add_job(job, 'interval', minutes=5)
+        self.scheduler.start()
+
+    def update_filecoin_price(self):
+        price = api.get_filecoin_price()
+        if price is None:
+            return
+        text  = "The price of filecoin(FIL) is currently ${} USD.".format(price)
+        vector_store.upsert(text,"dataset_1")
+        
+

service/vector_store.py

@@ -141,4 +141,9 @@ class VectorStore(Pinecone):
             for metadata in selected
         ]
 
+    def upsert(self,text,id):
+        embed = self.embeddings.embed_query(text)
+        vector_store._index.update(id=id,values=embed,set_metadata={"description":text,"text":text})
+        print(f"upsert text:{text} with id:{id}")
+
 vector_store = VectorStore()