8.7.20.54 added terminal print out function as a decorator to wrap up function and provide a terminal view for the code it executed.

cloud_db.py

@@ -15,6 +15,7 @@ dynamodb data operations
 '''
 
 # get the list of articles from articles table in dynamodb
+@terminal_print
 def get_table(table_name:str):
     result = db_client.scan(TableName = table_name)#,AttributesToGet = data_structure[table_name]["fields"])
     return [db_map_to_py_dict(r) for r in result["Items"]]
@@ -31,6 +32,7 @@ def post_item(table_name:str,item:dict):
     return res
 
 # update an article in table articles in dynamodb, return error if failed
+@terminal_print
 def put_item(table_name:str,item:dict):
     try:
         res = db_client.put_item(
@@ -42,6 +44,7 @@ def put_item(table_name:str,item:dict):
     return res
 
 # delete an article in table articles in dynamodb, return error if not found.
+@terminal_print
 def delete_item(table_name:str,key:dict):
     try:
         res = db_client.delete_item(
@@ -55,6 +58,7 @@ def delete_item(table_name:str,key:dict):
 
 '''
 '''
+@terminal_print
 def get_item(table_name:str,key:dict):
     try:
         res = db_client.get_item(
@@ -68,6 +72,7 @@ def get_item(table_name:str,key:dict):
 '''
 dynamodb structure management
 '''
+@terminal_print
 def get_structure(table_name:str):
     result = db_client.describe_table(TableName = table_name)
     return result["Table"]["AttributeDefinitions"]

cloud_storage.py

@@ -2,7 +2,7 @@ import boto3
 import os
 import tempfile
 
-from utility import aws_access_key_id, aws_secret_access_key
+from utility import aws_access_key_id, aws_secret_access_key,terminal_print
 
 s3 = boto3.client(
     's3', 
@@ -11,6 +11,7 @@ s3 = boto3.client(
     )
 
 # post or update a file to s3
+@terminal_print
 def upload_file(path, bucket, object_name=None):
     """Upload a file to an S3 bucket
 
@@ -33,6 +34,7 @@ def upload_file(path, bucket, object_name=None):
         return False
     return res
 
+@terminal_print
 def upload_fileobj(file_obj, bucket, object_name=None):
     '''
     Upload a file object to an S3 bucket
@@ -55,6 +57,7 @@ def upload_fileobj(file_obj, bucket, object_name=None):
     return res
 
 # get a file from s3
+@terminal_print
 def download_file(bucket, object_name, file_name=None):
     """Download a file from an S3 bucket
 
@@ -75,6 +78,7 @@ def download_file(bucket, object_name, file_name=None):
     return True
 
 # download a file object from s3
+@terminal_print
 def download_fileobj(bucket, object_name, temp_obj=None):
     '''
     Download a file object from an S3 bucket
@@ -97,6 +101,7 @@ def download_fileobj(bucket, object_name, temp_obj=None):
     return temp_obj
 
 # delete a file from s3
+@terminal_print
 def delete_file(bucket, object_name):
     """Delete a file from an S3 bucket
 
@@ -112,6 +117,7 @@ def delete_file(bucket, object_name):
     return response
 
 # list all files in a bucket
+@terminal_print
 def list_files(bucket):
     """List files in a bucket
 

supplier.py

@@ -2,11 +2,12 @@ import openai
 import tiktoken
 
 from application import *
+from utility import terminal_print
 
 openai.api_key = openai_api_key
 token_encoder = tiktoken.get_encoding("cl100k_base")
 
-
+@terminal_print
 def execute_prompt(prompt):
     '''
     execute_prompt function takes two arguments: text and prompt
@@ -28,22 +29,28 @@ def execute_prompt(prompt):
     )
     return res.choices[0]["text"] if res.choices else "<error> failed to generate text</error>"
 
+@terminal_print
 def format(**kwargs):
     if "format" in kwargs:
         return kwargs["format"]
     return kwargs
 
 
+@terminal_print
 def execute_instruction(article, instruction,model="gpt-3.5-turbo-16k",format="markdown"):
     '''
     execute_instruction function takes three arguments: article, instruction and model
     
+    Parameters
+    ----------
     article: the raw text from the article source
     instruction: the instruction for the rational execution it needs to complete
-    model: the model used for the rational execution, default to gpt-3.5-turbo-16k
-    format: the format of the table, default to markdown
+    model: the model to use for the rational execution
+    format: the format of the table to be formatted
 
     return: a string, the result of the rational execution
+    
+    
     '''
     msg_stream = [
         {

utility.py

@@ -6,6 +6,26 @@ from pdfminer.high_level import extract_text
 from pdfminer.pdfparser import PDFParser
 from pdfminer.pdfdocument import PDFDocument
 
+'''
+universal system functions
+'''
+
+def terminal_print(func):
+    from datetime import datetime
+    # import os
+    def wrapper(*args, **kwargs):
+        start = datetime.now()
+        print(f"{start.strftime('%y-%m-%d %H:%M:%S')} - executing: {func.__name__}")
+
+        result = func(*args, **kwargs)
+
+        end = datetime.now()
+        print(f"{end.strftime('%y-%m-%d %H:%M:%S')} - completed: {func.__name__}, runtime: {end-start} seconds")
+
+        return result
+
+    return wrapper
+
 
 '''
 following functions are for file manipulation
@@ -15,6 +35,7 @@ keyword_search = lambda kw, text: kw.lower() in text.lower()
 list_or = lambda l: sum(l)>0
 list_and = lambda l: sum(l)==len(l)
 
+@terminal_print
 def read_pdf(file_path):
     '''
     this function read the pdf file and return the text
@@ -86,6 +107,7 @@ def format_response(code,data):
 following functions are for string manipulation
 '''
 
+@terminal_print
 def format_text(text,remove_char_ls = ["\\n--\\n","\\n\\n","\n"]):
     '''
     this function format the text output by removing excessive characters
@@ -105,6 +127,7 @@ def format_text(text,remove_char_ls = ["\\n--\\n","\\n\\n","\n"]):
     
     return text
 
+@terminal_print
 def remove_symbols(text):
     '''
     this function remove symbols that are not in unicode
@@ -123,6 +146,7 @@ def remove_symbols(text):
     text = text.replace('-\n', '')
     return text
 
+@terminal_print
 def remove_citation(text):
     ''' 
     this function remove citation pattern in the text
@@ -155,6 +179,7 @@ def str_to_tuple(s):
     '''
     return tuple(s.replace("(","").replace(")","").split(","))
 
+@terminal_print
 def replace_symbols(s):
     '''
     this function replace symbols in the string to comply with file names
@@ -186,6 +211,7 @@ def replace_symbols(s):
 following functions are for dynamodb data manipulation
 '''
 
+@terminal_print
 def db_map_to_py_dict(db_map):
     '''
     this function convert dynamodb map data structure to python dictionary
@@ -222,6 +248,7 @@ def db_map_to_py_dict(db_map):
         
     return py_dict
 
+@terminal_print
 def py_dict_to_db_map(py_dict):
     '''
     this function convert python dictionary to dynamodb map data structure
@@ -255,6 +282,7 @@ def py_dict_to_db_map(py_dict):
             db_map[key] = {"NULL":True}
     return db_map
 
+@terminal_print
 def db_list_to_py_list(db_list):
     '''
     this function convert dynamodb list data structure to python list
@@ -283,6 +311,7 @@ def db_list_to_py_list(db_list):
                 
     return py_list
 
+@terminal_print
 def py_list_to_db_list(py_list):
     '''
     this function convert python list to dynamodb list data structure
@@ -321,10 +350,45 @@ def py_list_to_db_list(py_list):
         
     return db_list
 
+'''
+following functions are for markdown table creation
+'''
+
+@terminal_print
+def create_md_table(array):
+    '''
+    create markdown tables for an array.
+
+    Parameters
+    ----------
+    array: list
+        a table in the form of a list of lists
+
+    Returns
+    -------
+        md_table: str
+    '''
+    md_table = ""
+
+    for i,row in enumerate(array):
+        md_row = ""
+        for item in row:
+            md_item = f"| {item} "
+            md_row += md_item
+
+        md_row += " |\n"
+        md_table += md_row
+
+        if i == 0:
+            md_table += f"| {'|'.join(['---' for _ in range(len(row))])} |\n"
+
+    return md_table
+
 '''
 following functions are used for business logic. (to be moved to business logic layer)
 '''
 
+@terminal_print
 def est_cost(n_tokens,rate):
     '''
     this function calculate the estimated cost of the translation