pushing for deployment

.devcontainer/devcontainer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94803b7d77a0422321a2c120ea97516f8e818d261ba4c5c36b0802e1d74f7fa1
+size 968

.env.example

@@ -0,0 +1 @@
+OPENAI_API_KEY=""

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly

.gitignore

@@ -0,0 +1,83 @@
+# Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# IDE and Editor folders #
+##########################
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Node.js #
+###########
+/node_modules/
+/npm-debug.log
+/yarn-error.log
+
+# Python #
+##########
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+env/
+venv/
+pip-log.txt
+pip-delete-this-directory.txt
+.tox/
+.coverage
+.cache
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Visual Studio Code #
+######################
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+# Others #
+##########
+.env
+.secret
+lida_ko/web/ui

_pages/data_recommendation.py

@@ -0,0 +1,109 @@
+import streamlit as st
+from utils import load_datasets, get_chroma
+from streamlit_extras.grid import grid
+import pandas as pd
+from streamlit_extras.dataframe_explorer import dataframe_explorer
+import json
+import openai
+
+st.title("🤖 데이터 추천 받기 from AI")
+
+# ChromaDB 로드
+collection = get_chroma()
+
+# ID to metadata 딕셔너리 로드
+with open("./data/id_to_metadata.json", "r") as f:
+    id_to_metadata = json.load(f)
+
+# 초기 데이터프레임 로드
+# Section 1: 데이터 추천받기
+st.header("1️⃣ 데이터 추천받기")
+
+# 사용자 입력 처리
+dataframe = load_datasets()
+with st.container():
+    prompt = st.chat_input("무엇이 궁금하신가요? 또는 어떤 데이터를 찾고 있나요?", key='1')
+
+    if prompt:
+        st.session_state.messages.append({"role": "user", "content": prompt})
+
+        # 벡터 검색 수행
+        results = collection.query(query_texts=[prompt], n_results=10)
+        result_ids = results['ids'][0]
+        result_titles = [id_to_metadata[id]['title'] for id in result_ids if id in id_to_metadata]
+        # AI 응답 생성 (OpenAI API 사용)
+        with st.chat_message("assistant"):
+            message_placeholder = st.empty()
+            full_response = ""
+            with st.spinner("AI가 답변을 준비 중입니다..."):
+                response = openai.chat.completions.create(
+                    model="gpt-3.5-turbo",
+                    messages=[
+                        {"role": "system", "content": "You are a helpful assistant that recommends datasets based on user queries. MUST answer in KOREAN (한글)"},
+                        {"role": "user", "content": f"""이 쿼리에 의해서 '{prompt}', 나 이런 데이터베이스를 추천받았어: {', '.join(result_titles)}. 이 중에서 어떤게 내 질문에 가장 잘 대답할 수 있는지 순서대로 1,2,3,4.. 이런식으로 정렬해주고, 각각의 이유도 설명해줄래?
+                        만약 추천할만한 데이터가 없다고 생각한다면 절대 지어내지말고 그냥 '없어'라고 대답해줘."""},
+                    ],
+                    stream=True
+                )
+
+                for chunk in response:
+                    partial_response = chunk.choices[0].delta.content
+                    if partial_response:
+                        full_response += partial_response
+                    message_placeholder.markdown(full_response + "▌")
+                message_placeholder.markdown(full_response)
+
+        st.session_state.messages.append({"role": "assistant", "content": full_response})
+
+        # 데이터프레임 필터링
+        filtered_df = dataframe[dataframe.index.isin(result_ids)]
+    else:
+        filtered_df = dataframe
+
+    # 채팅 기록 표시
+    # for message in st.session_state.messages:
+    #     with st.chat_message(message["role"]):
+    #         st.markdown(message["content"])
+
+# 데이터프레임 전처리
+filtered_df['keywords'] = filtered_df['keywords'].apply(lambda x: ', '.join(x) if isinstance(x, list) else x)
+list_columns = [col for col in filtered_df.columns if filtered_df[col].apply(lambda x: isinstance(x, list)).all()]
+dict_columns = [col for col in filtered_df.columns if filtered_df[col].apply(lambda x: isinstance(x, dict)).all()]
+filtered_df = filtered_df.drop(columns=list_columns + dict_columns)
+
+st.divider()
+
+st.header("2️⃣ 추천 받은 데이터 중 선택하기")
+
+# TODO : bug here!!!!!
+if len(filtered_df) == 0:
+    st.warning("추천된 데이터가 없습니다. 다른 쿼리를 시도해주세요.")
+    st.stop()
+selectable_dataset_list = [None] + filtered_df['title'].values.tolist()
+
+# 데이터 선택 옵션
+selected_dataset = st.selectbox(
+    "🔍 분석할 데이터를 선택하세요", 
+    selectable_dataset_list,
+    index=selectable_dataset_list.index(st.session_state.get('selected_dataset')) if st.session_state.get('selected_dataset') in selectable_dataset_list else 0
+)
+st.session_state['selected_dataset'] = selected_dataset
+
+st.divider()
+
+# 다음 페이지로 이동
+def switch_to_summarization():
+    st.switch_page("_pages/summarization.py")
+
+st.header("3️⃣ 데이터 요약으로 넘어가기")
+next_page = st.button(
+    label=f"️✅ {st.session_state.selected_dataset}으로 요약하기" if st.session_state.selected_dataset else "⛔ 데이터를 선택해야 요약을 볼 수 있습니다.",
+    use_container_width=True,
+    type="secondary",
+    disabled=st.session_state.selected_dataset is None,
+)
+
+if next_page:
+    switch_to_summarization()
+
+st.divider()

_pages/data_selection.py

@@ -0,0 +1,48 @@
+import streamlit as st
+from utils import load_datasets, save_session_cache
+from streamlit_extras.grid import grid
+import pandas as pd
+from streamlit_extras.dataframe_explorer import dataframe_explorer
+
+st.title("💁 데이터 직접 선택하기")
+
+# Section 1: Data Selection
+st.header("1️⃣ 데이터 목록 필터링 하기")
+dataframe = load_datasets()
+
+dataframe['keywords'] = dataframe['keywords'].apply(lambda x: ', '.join(x) if isinstance(x, list) else x)
+list_columns = [col for col in dataframe.columns if dataframe[col].apply(lambda x: isinstance(x, list)).all()]
+dict_columns = [col for col in dataframe.columns if dataframe[col].apply(lambda x: isinstance(x, dict)).all()]
+    # list 타입의 컬럼을 드랍
+dataframe = dataframe.drop(columns=list_columns)
+dataframe = dataframe.drop(columns=dict_columns)
+filtered_df = dataframe_explorer(dataframe, case=False)
+st.dataframe(filtered_df, use_container_width=True)
+st.divider()
+
+st.header("2️⃣ 분석 데이터 선택하기")
+
+selectable_dataset_list = [None] + filtered_df['title'].values.tolist()
+
+# Select data for analysis
+selected_dataset = st.selectbox(
+    "🔍 분석할 데이터를 선택하세요", 
+    selectable_dataset_list,
+    index=selectable_dataset_list.index(st.session_state.selected_dataset) if st.session_state.selected_dataset in selectable_dataset_list else 0)
+st.session_state['selected_dataset'] = selected_dataset
+st.divider()
+
+# session_id=save_session_cache(st.session_state.to_dict())
+def switch_to_summarization():
+    st.switch_page(f"_pages/summarization.py")
+st.header("3️⃣ 데이터 요약으로 넘어가기")
+next_page = st.button(label=f"️✅ {st.session_state.selected_dataset}으로 요약하기" if st.session_state.selected_dataset else "⛔ 데이터를 선택해야 요약을 볼 수 있습니다.",
+          use_container_width=True,
+          type="secondary",
+          disabled=st.session_state.selected_dataset is None,
+          )
+if next_page:
+    switch_to_summarization()
+
+st.divider()
+

_pages/gallery.py

@@ -0,0 +1,5 @@
+import streamlit as st
+from streamlit_extras.grid import grid
+
+st.write("### 분석된 데이터와 시각화 결과")
+

_pages/goal.py

@@ -0,0 +1,77 @@
+import os
+
+import streamlit as st
+from streamlit_extras.grid import grid
+from streamlit_card import card
+
+from lida_ko import Manager, TextGenerationConfig
+from lida_ko.datamodel import Goal
+
+openai_api_key = os.environ["OPENAI_API_KEY"]
+selected_dataset = st.session_state.selected_dataset
+selected_method = st.session_state.selected_method
+selected_model = st.session_state.selected_model
+use_cache = st.session_state.use_cache
+temperature = st.session_state.temperature
+lida_manager: Manager = st.session_state.lida_manager
+summary = st.session_state.summary 
+num_goals = st.session_state.num_goals
+own_goal = st.session_state.own_goal
+
+st.title("🎯 분석 목표 설정하기")
+st.write("")
+st.empty()
+
+if not summary:
+    st.error("**ERROR**: 🚨 데이터 요약을 먼저 생성해주세요.")
+    st.stop()
+
+if summary:
+    textgen_config = TextGenerationConfig(
+        n=1,
+        temperature=temperature,
+        model=selected_model,
+        use_cache=use_cache)
+
+    # **** lida.goals *****
+    goals = lida_manager.goals(summary, n=num_goals, textgen_config=textgen_config)
+    st.write(f"## Goals ({len(goals)})")
+
+    default_goal = goals[0].question
+    goal_questions = [goal.question for goal in goals]
+
+    selected_goal = st.selectbox('Choose a generated goal', options=goal_questions, index=0)
+    selected_goal_index = goal_questions.index(selected_goal)
+    selected_goal_object = goals[selected_goal_index]
+    st.session_state.selected_goal_object = selected_goal_object
+
+    # st.markdown("### Selected Goal")
+    st.write("")
+    st.write("")
+    st.write("")
+    col1, col2, col3, = st.columns([1, 1, 1])
+    with col1:
+        st.write("### 1️⃣ 분석 목표")
+        st.write("")
+        st.write(f"- {selected_goal_object.question}")
+    with col2:
+        st.write("### 2️⃣시각화 방안")
+        st.write("")
+        st.write(f"- {selected_goal_object.visualization}")
+    with col3:
+        st.write("### 3️⃣ 인사이트")
+        st.write("")
+        st.write(f"- {selected_goal_object.rationale}")
+
+st.divider()
+
+st.header("✨ 시각화 만들기")
+
+next_page = st.button(label=f"️✅ {st.session_state.selected_goal_object.visualization}으로 시각화 생성!" if st.session_state.selected_goal_object else "⛔ 목표를 선택해야 시각화를 생성할 수 있습니다.",
+          use_container_width=True,
+          type="secondary",
+          disabled=st.session_state.selected_goal_object is None,
+          )
+
+if next_page:
+    st.switch_page(f"_pages/visualization.py")

_pages/home.py

@@ -0,0 +1,38 @@
+import streamlit as st
+
+
+st.title("Busan AI Data Navigator")
+st.caption("부산 공공데이터 추천, 분석, 시각화 서비스 with 생성형 인공지능")
+
+st.divider()
+col1, col2 = st.columns([4,2])
+with col1:
+    st.image(image='./static/cute_background.png', use_column_width=True)
+with col2:
+    st.markdown("""
+    <div style="background-color: #f0f8ff; padding: 20px; border-radius: 10px; text-align: center;">
+        <h2 style="color: #0056b3; margin-bottom: 5px;">🚀 데이터의 새로운 지평을 열다</h2>
+        <h3 style="color: #FF6347; font-size: 2.5em; margin-bottom: 15px;">Busan Data Navigator</h3>
+        <p style="color: #333; font-size: 18px; margin-bottom: 40px;">
+            🧠 생성형 AI &nbsp;&nbsp; 📊 원클릭 시각화 &nbsp;&nbsp; 💡 즉각적인 인사이트
+        </p>
+        <hr style="border: 0; height: 1px; background-image: linear-gradient(to right, #f0f8ff, #333, #f0f8ff); margin-bottom: 40px;">
+        <h3 style="color: #28a745; margin-bottom: 15px;">부산의 모든 데이터, 당신의 손끝에서</h3>
+        <p style="color: #333; font-size: 16px; margin-bottom: 20px;">
+            🌊 부산의 바다처럼 넓고 깊은 데이터의 세계로 빠져보세요
+        </p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.markdown("""
+    ---
+    ### 🌟 Busan Data Navigator로 무엇을 할 수 있나요?
+
+    - 🔍 **손쉬운 데이터 탐색**: 부산시의 방대한 데이터를 쉽게 찾고 이해하세요.
+    - 🤖 **AI 기반 인사이트**: 생성형 AI가 당신만의 데이터 스토리를 만들어냅니다.
+    - 📈 **클릭 한 번으로 시각화**: 복잡한 코딩 없이 데이터를 아름답게 표현하세요.
+    - 🌈 **창의적 발견**: 숨겨진 패턴과 트렌드를 발견하여 혁신을 이끌어내세요.
+    - 🚀 **의사결정 가속화**: 데이터 기반의 빠르고 정확한 결정을 내리세요.
+
+    > 전문가가 아니어도 괜찮아요. 누구나 쉽게 공공데이터를 활용할 수 있어요!
+    """)

_pages/my_account.py

@@ -0,0 +1,51 @@
+import streamlit as st
+
+st.title("마이 페이지")
+
+# User Information
+col1, col2, col3 = st.columns(3)
+
+with col1:
+    st.subheader("사용자 정보")
+    st.image("./static/boogi.png", caption="증명사진", width=100)
+    st.write("**성명:** 부기")
+    st.write("**소속 기관:** 부산광역시")
+    st.write("**소속 부서:** 디지털경제혁신실 시민행복팀")
+    st.write("**이메일:** boogi@korea.kr")
+    st.write("**직책:**  주무관")
+
+with col2:
+    st.subheader("최근 활동")
+    st.write("**마지막 로그인:** 2024-06-28 10:30 AM")
+    st.write("**로그인 IP:** 192.168.1.1")
+    st.write("**프리미엄 구독 상태:** :white_check_mark: 활성")
+    st.write("**구독 만료일:** 2025-06-28")
+
+    with st.container():
+        st.write("**로그인 기록**")
+        with st.popover("**확인**"):
+            recent_logins = ["2024-06-27 09:15 AM", "2024-06-26 08:45 AM", "2024-06-25 10:00 AM", "2024-06-24 11:30 AM", "2024-06-23 09:00 AM", "2024-06-22 08:30 AM"]
+            for elem in recent_logins:
+                st.write(f"{elem}")
+
+# Data Analysis History
+with col3:
+    st.subheader("데이터 분석 이력")
+    st.write("**분석 내역**")
+    with st.popover("**확인**"):
+        analyzed_datasets = [
+            "부산광역시_아동급식카드 가맹점",
+            "부산광역시_종합병원 현황",
+            "부산광역시 북구_주민등록 인구통계",
+            "법정동별연료별차종별_자동차등록대수",
+        ]
+        for elem in analyzed_datasets:
+            st.write(f"{elem}")
+    st.write("**시각화 내역**")
+    with st.popover("**확인**"):
+        build_visualizations = ['viz1.png', 'viz2.png', 'viz3.png', 'viz4.png']
+        build_visualizations = [f"./static/{elem}" for elem in build_visualizations]
+        for title, elem in zip(analyzed_datasets, build_visualizations):
+            st.subheader(f"{title}")
+            st.image(elem, use_column_width=True)
+            st.markdown("---")

_pages/report.py

@@ -0,0 +1,4 @@
+import streamlit as st
+
+st.write("### 데이터 선택")
+

_pages/summarization.py

@@ -0,0 +1,78 @@
+import os
+import pandas as pd
+import streamlit as st
+from lida_ko import Manager, TextGenerationConfig, llm
+from lida_ko.datamodel import Summary
+
+from utils import title_to_df, title_to_filename, title_to_id, id_to_metadata
+
+openai_api_key = os.environ["OPENAI_API_KEY"]
+selected_dataset = st.session_state.selected_dataset
+selected_method = st.session_state.selected_method
+selected_model = st.session_state.selected_model
+use_cache = st.session_state.use_cache
+temperature = st.session_state.temperature
+lida_manager: Manager = st.session_state.lida_manager
+
+st.title("🔎 요약 확인하기")
+st.write("")
+st.empty()
+
+if not selected_dataset and selected_method:
+    st.error("**ERROR**: 🚨 대상 데이터를 먼저 선택해주세요")
+    st.stop()
+
+if openai_api_key and selected_dataset and selected_method:
+    textgen_config = TextGenerationConfig(
+        n=1,
+        temperature=temperature,
+        model=selected_model,
+        use_cache=use_cache)
+
+    st.write("## Summary")
+    # **** lida.summarize *****
+    with st.spinner(f"{selected_dataset}에 대한 요약 생성 중..."):
+        df = title_to_df(selected_dataset)
+        filename = title_to_filename(selected_dataset)
+        metadata = id_to_metadata(title_to_id(selected_dataset))
+        selected_dataset = df
+        st.session_state.selected_dataframe = df
+
+        summary: Summary = lida_manager.summarize(
+            data=selected_dataset,
+            file_name=filename,
+            summary_method=selected_method,
+            textgen_config=textgen_config,
+            metadata=metadata)
+        st.write("### 1️⃣ 데이터 설명 및 요약")
+        if "dataset_description" in summary:
+            st.write(summary["dataset_description"])
+        st.divider()
+
+        st.write("### 2️⃣데이터 컬럼  확인")
+        if "fields" in summary:
+            fields = summary["fields"]
+            nfields = []
+            for field in fields:
+                flatted_fields = {}
+                flatted_fields["column"] = field["column"]
+                # flatted_fields["dtype"] = field["dtype"]
+                for row in field["properties"].keys():
+                    if row != "samples":
+                        flatted_fields[row] = field["properties"][row]
+                    else:
+                        flatted_fields[row] = str(field["properties"][row])
+                # flatted_fields = {**flatted_fields, **field["properties"]}
+                nfields.append(flatted_fields)
+            nfields_df = pd.DataFrame(nfields)
+            st.write(nfields_df)
+        else:
+            st.write(str(summary))
+        st.session_state.summary = summary
+        st.divider()
+
+        st.write("### 3️⃣ 데이터 원본 확인")
+        show_original_data = st.button("데이터 원본 보기", use_container_width=True)
+        if show_original_data:
+            with st.spinner("데이터 원본 로딩 중 (데이터가 많으면 오래 걸릴 수 있습니다.)"):
+                st.write(df)

_pages/visualization.py

@@ -0,0 +1,117 @@
+import os
+
+import pandas as pd
+import streamlit as st
+from streamlit_extras.grid import grid
+from streamlit_card import card
+
+from lida_ko import Manager, TextGenerationConfig
+from lida_ko.datamodel import Goal
+from lida_ko.utils import clean_code_snippet
+
+openai_api_key = os.environ["OPENAI_API_KEY"]
+selected_dataset = st.session_state.selected_dataset
+selected_method = st.session_state.selected_method
+selected_model = st.session_state.selected_model
+use_cache = st.session_state.use_cache
+temperature = st.session_state.temperature
+lida_manager: Manager = st.session_state.lida_manager
+summary = st.session_state.summary 
+selected_goal_object: Goal = st.session_state.selected_goal_object
+selected_dataframe: pd.DataFrame = st.session_state.selected_dataframe
+num_visualizations = st.session_state.num_visualizations
+
+def generate_visualizations(code = None, feedback=None):
+    if code and feedback:
+        visualizations = lida_manager.repair(
+            code=code,
+            goal=selected_goal_object,
+            summary=summary,
+            feedback=feedback,
+            textgen_config=textgen_config,
+            library=st.session_state.selected_library
+        )
+    else:
+        visualizations = lida_manager.visualize(
+                summary=summary,
+                goal=selected_goal_object,
+                textgen_config=textgen_config,
+                library=st.session_state.selected_library)
+    return visualizations
+
+st.title("📊 데이터 시각화 만들기")
+st.write("")
+st.empty()
+
+if not selected_goal_object:
+    st.error("**ERROR**: 🚨 데이터 분석 목표를 설정해주세요.")
+    st.stop()
+
+if selected_goal_object:
+    # Update the visualization generation call to use the selected library.
+
+    textgen_config = TextGenerationConfig(
+        n=num_visualizations, temperature=temperature,
+        model=selected_model,
+        use_cache=use_cache)
+
+    # **** lida.visualize *****
+    if not st.session_state.update_viz:
+        visualizations = generate_visualizations()
+        st.session_state.visualizations = visualizations
+    else:
+        st.session_state.update_viz = False
+
+    col1, col2, col3 = st.columns([5, 0.5, 5])
+    def render_visualization(idx, viz):
+        st.write(f'### 🌟 시각화 {idx + 1}')
+        if viz:
+            with st.spinner("인공지능이 시각화를 생성중입니다..."):
+                try:
+                    if st.session_state.selected_library == "plotly":
+                        data = st.session_state.selected_dataframe
+                        # extract the code from the generated responses and execute it
+                        temp_namespace = {
+                            'data': data,
+                        }
+                        exec(clean_code_snippet(viz['code']), temp_namespace)
+                        fig = st.plotly_chart(temp_namespace['chart'])
+                    else:
+                        from PIL import Image
+                        import io
+                        import base64
+
+                        imgdata = base64.b64decode(viz.raster)
+                        img = Image.open(io.BytesIO(imgdata))
+                        st.image(img, caption=f"Visualization {idx + 1}", use_column_width=True)
+                except Exception as e:
+                    st.error(f"Error loading visualization: {e}")
+        with st.popover("🧑‍💻 코드 확인하기", use_container_width=True):
+            if isinstance(viz, dict):
+                code_string = viz['code']
+            else:
+                code_string = viz.code
+            st.code(clean_code_snippet(code_string))
+        with st.popover("🗨️ 변경 요청하기", use_container_width=True):
+            chat_message = st.chat_input("(구현 중)변경하고 싶은 내용을 자연어 로 입력해주세요",key=f"chat_message_{idx}",
+                                         disabled=True)
+            if chat_message:
+                st.session_state.visualizations = generate_visualizations(viz['code'], chat_message)
+                st.session_state.update_viz = True
+                st.rerun()
+                render_visualization(idx, st.session_state.visualizations[idx])
+
+        return fig
+
+    with col1:
+        idx = 0
+        selected_viz = st.session_state.visualizations[idx]
+        render_visualization(idx, selected_viz)
+
+    with col2:
+        st.empty()
+        
+    with col3:
+        idx = 1
+        selected_viz = visualizations[idx]
+        render_visualization(idx, selected_viz)

commit_for_lfs.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# large_files.txt 파일이 존재하는지 확인
+if [ ! -f large_files.txt ]; then
+  echo "Error: large_files.txt file not found!"
+  exit 1
+fi
+
+# large_files.txt 파일의 각 줄을 읽어서 처리
+while IFS= read -r file; do
+  # 파일이 존재하는지 확인
+  if [ -f "$file" ]; then
+    echo "Processing $file"
+    git rm --cached "$file"
+    git add "$file"
+  else
+    echo "Warning: $file not found!"
+  fi
+done < large_files.txt
+
+# 변경 사항 커밋
+git commit -m "Move large files to Git LFS"

failure.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a44280b73d46fd1d9963549c1dc166719206a38822744ba34c8d057fda160f81
+size 30048

large_files.txt

@@ -0,0 +1,81 @@
+./lida_ko/web/ui/component---src-pages-demo-tsx-54fd6da10fa870d8d843.js.map
+./data/chroma_db/chroma.sqlite3
+./data/id_to_metadata_col.json
+./data/id_to_metadata_col_aug.json
+./_csv_data/부산광역시해운대구_재정정보공개시스템_세입자료수납내역표_20240115.JSON
+./_csv_data/부산광역시_도시공간정보시스템도로(그리드면)_20240524.JSON
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_계량기정보_20240106.JSON
+./_csv_data/부산광역시_교통시설물관리시스템_교통안전시설물정보(안전표지정보)_20220630.JSON
+./_csv_data/부산광역시_연제구_자료관도서목록_20200916.JSON
+./_csv_data/부산광역시_도로명주소정보_20240415.CSV
+./_csv_data/부산광역시영도구_영도도서관장서현황_20230816.CSV
+./_csv_data/부산광역시해운대구_재정정보공개시스템_세부사업별예산현액및지출액_20230113.JSON
+./_csv_data/부산시설공단_영락공원묘지사용현황_20201118.JSON
+./_csv_data/부산광역시연제구_자료관도서목록_20240414.JSON
+./_csv_data/부산교통공사_시간대별승하차인원_20231231.JSON
+./_csv_data/부산시설공단_영락공원묘지사용현황_20230125.JSON
+./_csv_data/부산광역시_지능형교통정보_RSE정보_20240430.JSON
+./_csv_data/부산광역시_부산도시공간정보시스템_도로상하수도기반시설물_하수맨홀_20231013.JSON
+./_csv_data/부산관광공사_부산관광기업지원센터홈페이지메타데이터개방_20230818.JSON
+./_csv_data/부산광역시_지능형교통정보_구간DSRC정보_20240430.CSV
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_계량기정보_20230126.JSON
+./_csv_data/부산광역시_버스노선별승하차정보_20230731.JSON
+./_csv_data/부산광역시사하구_불법주정차단속현황_20231130.JSON
+./_csv_data/부산광역시_열섬관측지점정보_20240430.JSON
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_수납정보_당월및체납수납처리정보_20240510.CSV
+./_csv_data/부산도시철도시간대별승하차현황_20161231.JSON
+./_csv_data/부산광역시_한국도로공사연계특별상황발생관리_20230828.JSON
+./_csv_data/부산광역시_지능형교통정보_구간DSRC정보_20240430.JSON
+./_csv_data/부산광역시_굴착사업예정지별규모정보_20230812.JSON
+./_csv_data/부산광역시_지역화폐(동백전)가맹점현황_20240305.JSON
+./_csv_data/부산광역시_제로페이가맹점현황_20230630.JSON
+./_csv_data/부산광역시_도시공간정보시스템기초구역및새주소_20220627.JSON
+./_csv_data/부산교통공사_시간대별승하차인원_20221231.JSON
+./_csv_data/부산광역시부산진구_불법주정차단속현황_20240325.JSON
+./_csv_data/부산교통공사_시간대별승하차인원_20201231.JSON
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_세입집계정보_세입일집계_20240106.CSV
+./_csv_data/부산광역시_동래구_지적정보_20240213.JSON
+./_csv_data/부산광역시사상구_일반건축물시가표준액_20211231.CSV
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_간단e납부시스템연계자료_일반수납내역분배자료_20240106.CSV
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_계량기정보_계량기변경이력정보_20220131.JSON
+./_csv_data/부산광역시_지능형교통정보_RSE정보_20240430.CSV
+./_csv_data/부산광역시_대기질진단평가대기질측정소기상정보_20230825.JSON
+./_csv_data/부산시설공단_영락공원봉안사용현황_20240125.JSON
+./_csv_data/부산광역시_부산도시공간정보시스템_도로상하수도기반시설물_부과정보_20231017.JSON
+./_csv_data/부산광역시_해운대구_재정정보공개시스템_세입자료_20231213.JSON
+./_csv_data/부산광역시_도로명주소정보_20240415.JSON
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_요금계산관련정보_감액관리대장_20240106.JSON
+./_csv_data/부산도시철도시간대별승하차현황_20191231.JSON
+./_csv_data/부산교통공사_시간대별승하차인원_20211231.JSON
+./_csv_data/부산광역시_지능형교통정보구간레벨패턴정보_20240531.CSV
+./_csv_data/부산광역시_교통시설물관리시스템_교통안전시설물정보(차선정보)_20220630.JSON
+./_csv_data/부산광역시사상구_일반건축물시가표준액_20211231.JSON
+./_csv_data/부산광역시_연제구_자료관도서목록_20200916.CSV
+./_csv_data/부산시설공단_영락공원묘지사용현황_20240125.JSON
+./_csv_data/부산광역시_한국도로공사연계특별상황발생관리_20230828.CSV
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_요금계산관련정보_감액관리대장_20230126.JSON
+./_csv_data/부산광역시_아동급식카드가맹점_20240320.JSON
+./_csv_data/부산시설공단_영락공원묘지사용현황_20220125.JSON
+./_csv_data/��산시설공단_영락공원봉안사용현황_20201022.JSON
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_수납정보_과오수납처리정보_20230126.JSON
+./_csv_data/부산광역시_강서구_불법주정차문자알림서비스_단속정보관리_20240517.JSON
+./_csv_data/부산광역시상수도사업본부_수용가정보시스템_수납정보_과오수납처리정보_20240106.JSON
+./_csv_data/부산도시철도시간대별승하차현황_20141231.JSON
+./_csv_data/부산광역시_도시공간정보시스템_도로(건축물)정보_20230717.JSON
+./_csv_data/부산교통공사_시간대별승하차인원_20181231.JSON
+./_csv_data/부산광역시_부산광역시_도시공간정보시스템_도로관리(가로수)_20230717.CSV
+./_csv_data/부산광역시부산진구_일반건축물시가표준액_20221003.JSON
+./_csv_data/부산광역시_동래구_지적정보_20230210.JSON
+./_csv_data/부산광역시_제로페이가맹점현황_20230630.CSV
+./_csv_data/부산광역시_도시공간정보시스템_도로(건축물)정보_20230717.CSV
+./_csv_data/부산시설공단_영락공원봉안사용현황_20220125.JSON
+./_csv_data/부산교통공사_시간대별승하차인원_20240430.JSON
+./_csv_data/부산광역시연제구_자료관도서목록_20240414.CSV
+./_csv_data/부산광역시_열섬관측지점정보_20240331.CSV
+./_csv_data/부산광역시_지능형교통정보구간레벨패턴정보_20240531.JSON
+./_csv_data/부산광역시_지역화폐(동백전)가맹점현황_20240305.CSV
+./_csv_data/부산도시철도시간대별승하차현황_20151231.JSON
+./_csv_data/부산광역시영도구_영도도서관장서현황_20230816.JSON
+./_csv_data/부산시설공단_영락공원봉안사용현황_20230125.JSON
+./_csv_data/부산광역시_한국도로공사연계고속도로정보_20230828.JSON
+./_csv_data/부산광역시_열섬관측지점정보_20240430.CSV

lida-0.0.14.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

lida-0.0.14.dist-info/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

lida-0.0.14.dist-info/METADATA

@@ -0,0 +1,288 @@
+Metadata-Version: 2.1
+Name: lida
+Version: 0.0.14
+Summary: LIDA: Automatic Generation of Visualizations from Data
+Author-email: Victor Dibia <victordibia@microsoft.com>
+License: MIT License
+        
+            Copyright (c) Microsoft Corporation.
+        
+            Permission is hereby granted, free of charge, to any person obtaining a copy
+            of this software and associated documentation files (the "Software"), to deal
+            in the Software without restriction, including without limitation the rights
+            to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+            copies of the Software, and to permit persons to whom the Software is
+            furnished to do so, subject to the following conditions:
+        
+            The above copyright notice and this permission notice shall be included in all
+            copies or substantial portions of the Software.
+        
+            THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+            IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+            FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+            AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+            LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+            OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+            SOFTWARE
+        
+Project-URL: Homepage, https://github.com/microsoft/lida
+Project-URL: Bug Tracker, https://github.com/microsoft/lida/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: llmx >=0.0.21a
+Requires-Dist: pydantic
+Requires-Dist: uvicorn
+Requires-Dist: typer
+Requires-Dist: fastapi
+Requires-Dist: python-multipart
+Requires-Dist: scipy
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: matplotlib
+Requires-Dist: altair
+Requires-Dist: seaborn
+Requires-Dist: plotly
+Requires-Dist: plotnine
+Requires-Dist: statsmodels
+Requires-Dist: networkx
+Requires-Dist: geopandas
+Requires-Dist: matplotlib-venn
+Requires-Dist: wordcloud
+Requires-Dist: kaleido !=0.2.1.post1,>=0.2.1
+Provides-Extra: infographics
+Requires-Dist: peacasso ; extra == 'infographics'
+Provides-Extra: tools
+Requires-Dist: geopy ; extra == 'tools'
+Requires-Dist: basemap ; extra == 'tools'
+Requires-Dist: basemap-data-hires ; extra == 'tools'
+Provides-Extra: transformers
+Requires-Dist: llmx[transformers] ; extra == 'transformers'
+Provides-Extra: web
+Requires-Dist: fastapi ; extra == 'web'
+Requires-Dist: uvicorn ; extra == 'web'
+
+# LIDA: Automatic Generation of Visualizations and Infographics using Large Language Models
+
+[![PyPI version](https://badge.fury.io/py/lida.svg)](https://badge.fury.io/py/lida)
+[![arXiv](https://img.shields.io/badge/arXiv-2303.02927-<COLOR>.svg)](https://arxiv.org/abs/2303.02927)
+![PyPI - Downloads](https://img.shields.io/pypi/dm/lida?label=pypi%20downloads)
+
+<a target="_blank" href="https://colab.research.google.com/github/microsoft/lida/blob/main/notebooks/tutorial.ipynb">
+<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+</a>
+
+<!-- <img src="docs/images/lidascreen.png" width="100%" /> -->
+
+LIDA is a library for generating data visualizations and data-faithful infographics. LIDA is grammar agnostic (will work with any programming language and visualization libraries e.g. matplotlib, seaborn, altair, d3 etc) and works with multiple large language model providers (OpenAI, Azure OpenAI, PaLM, Cohere, Huggingface). Details on the components of LIDA are described in the [paper here](https://arxiv.org/abs/2303.02927) and in this tutorial [notebook](notebooks/tutorial.ipynb). See the project page [here](https://microsoft.github.io/lida/) for updates!.
+
+> **Note on Code Execution:**
+> To create visualizations, LIDA _generates_ and _executes_ code.
+> Ensure that you run LIDA in a secure environment.
+
+## Features
+
+![lida components](https://github.com/microsoft/lida/blob/main/docs/images/lidamodules.jpg?raw=true)
+
+LIDA treats _**visualizations as code**_ and provides a clean api for generating, executing, editing, explaining, evaluating and repairing visualization code.
+
+- [x] Data Summarization
+- [x] Goal Generation
+- [x] Visualization Generation
+- [x] Visualization Editing
+- [x] Visualization Explanation
+- [x] Visualization Evaluation and Repair
+- [x] Visualization Recommendation
+- [x] Infographic Generation (beta) # pip install lida[infographics]
+
+```python
+
+from lida import Manager, llm
+
+lida = Manager(text_gen = llm("openai")) # palm, cohere ..
+summary = lida.summarize("data/cars.csv")
+goals = lida.goals(summary, n=2) # exploratory data analysis
+charts = lida.visualize(summary=summary, goal=goals[0]) # exploratory data analysis
+```
+
+## Getting Started
+
+Setup and verify that your python environment is **`python 3.10`** or higher (preferably, use [Conda](https://docs.conda.io/en/main/miniconda.html#installing)). Install the library via pip.
+
+```bash
+pip install lida
+```
+
+Once requirements are met, setup your api key. Learn more about setting up keys for other LLM providers [here](https://github.com/victordibia/llmx).
+
+```bash
+export OPENAI_API_KEY=<your key>
+```
+
+Alternatively you can install the library in dev model by cloning this repo and running `pip install -e .` in the repository root.
+
+## Web API and UI
+
+LIDA comes with an optional bundled ui and web api that you can explore by running the following command:
+
+```bash
+lida ui  --port=8080 --docs
+```
+
+Then navigate to http://localhost:8080/ in your browser. To view the web api specification, add the `--docs` option to the cli command, and navigate to `http://localhost:8080/api/docs` in your browser.
+
+The fastest and recommended way to get started after installation will be to try out the web ui above or run the [tutorial notebook](notebooks/tutorial.ipynb).
+
+## Building the Web API and UI with Docker
+
+The LIDA web api and ui can be setup using docker and the command below (ensure that you have docker installed, and you have set your `OPENAI_API_KEY` environment variable).
+
+```bash
+docker compose up
+```
+
+### Data Summarization
+
+Given a dataset, generate a compact summary of the data.
+
+```python
+from lida import Manager
+
+lida = Manager()
+summary = lida.summarize("data/cars.json") # generate data summary
+```
+
+### Goal Generation
+
+Generate a set of visualization goals given a data summary.
+
+```python
+goals = lida.goals(summary, n=5, persona="ceo with aerodynamics background") # generate goals
+```
+
+Add a `persona` parameter to generate goals based on that persona.
+
+### Visualization Generation
+
+Generate, refine, execute and filter visualization code given a data summary and visualization goal. Note that LIDA represents **visualizations as code**.
+
+```python
+# generate charts (generate and execute visualization code)
+charts = lida.visualize(summary=summary, goal=goals[0], library="matplotlib") # seaborn, ggplot ..
+```
+
+### Visualization Editing
+
+Given a visualization, edit the visualization using natural language.
+
+```python
+# modify chart using natural language
+instructions = ["convert this to a bar chart", "change the color to red", "change y axes label to Fuel Efficiency", "translate the title to french"]
+edited_charts = lida.edit(code=code,  summary=summary, instructions=instructions, library=library, textgen_config=textgen_config)
+
+```
+
+### Visualization Explanation
+
+Given a visualization, generate a natural language explanation of the visualization code (accessibility, data transformations applied, visualization code)
+
+```python
+# generate explanation for chart
+explanation = lida.explain(code=charts[0].code, summary=summary)
+```
+
+### Visualization Evaluation and Repair
+
+Given a visualization, evaluate to find repair instructions (which may be human authored, or generated), repair the visualization.
+
+```python
+evaluations = lida.evaluate(code=code,  goal=goals[i], library=library)
+```
+
+### Visualization Recommendation
+
+Given a dataset, generate a set of recommended visualizations.
+
+```python
+recommendations = lida.recommend(code=code, summary=summary, n=2,  textgen_config=textgen_config)
+```
+
+### Infographic Generation [WIP]
+
+Given a visualization, generate a data-faithful infographic. This methods should be considered experimental, and uses stable diffusion models from the [peacasso](https://github.com/victordibia/peacasso) library. You will need to run `pip install lida[infographics]` to install the required dependencies.
+
+```python
+infographics = lida.infographics(visualization = charts[0].raster, n=3, style_prompt="line art")
+```
+
+## Using LIDA with Locally Hosted LLMs (HuggingFace)
+
+LIDA uses the [llmx](https://github.com/victordibia/llmx) library as its interface for text generation. llmx supports multiple local models including HuggingFace models. You can use the huggingface models directly (assuming you have a gpu) or connect to an openai compatible local model endpoint e.g. using the excellent [vllm](https://vllm.readthedocs.io/en/latest/) library.
+
+#### Using HuggingFace Models Directly
+
+```python
+!pip3 install --upgrade llmx==0.0.17a0
+
+# Restart the colab session
+
+from lida import Manager
+from llmx import  llm
+text_gen = llm(provider="hf", model="uukuguy/speechless-llama2-hermes-orca-platypus-13b", device_map="auto")
+lida = Manager(text_gen=text_gen)
+# now you can call lida methods as above e.g.
+sumamry = lida.summarize("data/cars.csv") # ....
+```
+
+#### Using an OpenAI Compatible Endpoint e.g. [vllm server](https://vllm.readthedocs.io/en/latest/getting_started/quickstart.html#openai-compatible-server)
+
+```python
+from lida import Manager, TextGenerationConfig , llm
+
+model_name = "uukuguy/speechless-llama2-hermes-orca-platypus-13b"
+model_details = [{'name': model_name, 'max_tokens': 2596, 'model': {'provider': 'openai', 'parameters': {'model': model_name}}}]
+
+# assuming your vllm endpoint is running on localhost:8000
+text_gen = llm(provider="openai",  api_base="http://localhost:8000/v1", api_key="EMPTY", models=model_details)
+lida = Manager(text_gen = text_gen)
+```
+
+## Important Notes / Caveats / FAQs
+
+- LIDA generates and executes code based on provided input. Ensure that you run LIDA in a secure environment with appropriate permissions.
+- LIDA currently works best with datasets that have a small number of columns (<= 10). This is mainly due to the limited context size for most models. For larger datasets, consider preprocessing your dataset to use a subset of the columns.
+- LIDA assumes the dataset exists and is in a format that can be loaded into a pandas dataframe. For example, a csv file, or a json file with a list of objects. In practices the right dataset may need to be curated and preprocessed to ensure that it is suitable for the task at hand.
+- Smaller LLMs (e.g., OSS LLMs on Huggingface) have limited instruction following capabilities and may not work well with LIDA. LIDA works best with larger LLMs (e.g., OpenAI GPT 3.5, GPT 4).
+- How reliable is the LIDA approach? The LIDA [paper](https://aclanthology.org/2023.acl-demo.11/) describes experiments that evaluate the reliability of LIDA using a visualization error rate metric. With the current version of prompts, data summarization techniques, preprocessing/postprocessing logic and LLMs, LIDA has an error rate of < 3.5% on over 2200 visualizations generated (compared to a baseline of over 10% error rate). This area is work in progress.
+- Can I build my own apps with LIDA? Yes! You can either use the python api directly in your app or setup a web api endpoint and use the web api in your app. See the [web api](#web-api-and-ui) section for more details.
+- How is LIDA related to OpenAI Code Interpreter: LIDA shares several similarities with code interpreter in the sense that both involve writing and executing code to address user intent. LIDA differs in its focus on visualization, providing a modular api for developer reuse and providing evaluation metrics on the visualization use case.
+
+Naturally, some of the limitations above could be addressed by a much welcomed PR.
+
+## Community Examples Built with LIDA
+
+- LIDA + Streamlit: [lida-streamlit](https://github.com/lida-project/lida-streamlit),
+
+## Documentation and Citation
+
+A short paper describing LIDA (Accepted at ACL 2023 Conference) is available [here](https://arxiv.org/abs/2303.02927).
+
+```bibtex
+@inproceedings{dibia2023lida,
+    title = "{LIDA}: A Tool for Automatic Generation of Grammar-Agnostic Visualizations and Infographics using Large Language Models",
+    author = "Dibia, Victor",
+    booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations)",
+    month = jul,
+    year = "2023",
+    address = "Toronto, Canada",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2023.acl-demo.11",
+    doi = "10.18653/v1/2023.acl-demo.11",
+    pages = "113--126",
+}
+```
+
+LIDA builds on insights in automatic generation of visualization from an earlier paper - [Data2Vis: Automatic Generation of Data Visualizations Using Sequence to Sequence Recurrent Neural Networks](https://arxiv.org/abs/1804.03126).

lida-0.0.14.dist-info/RECORD

@@ -0,0 +1,121 @@
+../../../bin/lida,sha256=aNl2RQh-pBOgkJImOISZsOhtYHbW6PVS6yOqxzgGO5o,210
+lida-0.0.14.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+lida-0.0.14.dist-info/LICENSE,sha256=ws_MuBL-SCEBqPBFl9_FqZkaaydIJmxHrJG2parhU4M,1141
+lida-0.0.14.dist-info/METADATA,sha256=IAPteJfo0fZNZ2mpdZP39YNHxoR2pPT1gVPlJ5qd2Q8,13393
+lida-0.0.14.dist-info/RECORD,,
+lida-0.0.14.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lida-0.0.14.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+lida-0.0.14.dist-info/entry_points.txt,sha256=MEt9wqeDV8qbUj6Os23WEuAdal3m1iT5MEq5eDONYKE,38
+lida-0.0.14.dist-info/top_level.txt,sha256=I2cAWyFnAsrpZXthVWLiW313NQFig1tbewqtUHTeOQc,5
+lida/__init__.py,sha256=EZhgVFF1Gq8fl5l45uNpLwWduFy382T8PI6StlsJwFw,170
+lida/__pycache__/__init__.cpython-312.pyc,,
+lida/__pycache__/cli.cpython-312.pyc,,
+lida/__pycache__/datamodel.cpython-312.pyc,,
+lida/__pycache__/utils.cpython-312.pyc,,
+lida/__pycache__/version.cpython-312.pyc,,
+lida/cli.py,sha256=lDK4B4IarXpXzqr3fqZmF8oxVayV0iUgGr8ja6QPHXc,976
+lida/components/__init__.py,sha256=8TmnzYd8y9hWMk636q_Nh7FKGTKWugxxy6xT9NTPUnk,168
+lida/components/__pycache__/__init__.cpython-312.pyc,,
+lida/components/__pycache__/executor.cpython-312.pyc,,
+lida/components/__pycache__/goal.cpython-312.pyc,,
+lida/components/__pycache__/infographer.cpython-312.pyc,,
+lida/components/__pycache__/manager.cpython-312.pyc,,
+lida/components/__pycache__/persona.cpython-312.pyc,,
+lida/components/__pycache__/scaffold.cpython-312.pyc,,
+lida/components/__pycache__/summarizer.cpython-312.pyc,,
+lida/components/executor.py,sha256=D89sad5eGPYiA5bgXna7qab16_TwmHf-pbZLZYvzD_Y,10741
+lida/components/goal.py,sha256=clxYLLhnvD52OWYP86GGW3JlnTIDzt-0AwX13BiLW7w,3272
+lida/components/infographer.py,sha256=66N9V_wKO_BFy5tqKmx-zThDwyARx2NmQ4Ffl_QhV-0,2201
+lida/components/manager.py,sha256=zMZp9dE8PXDI0kwB0r5Lo9AEgJBHW2Ql93gtjk_ZM7w,13710
+lida/components/persona.py,sha256=9R0qG84UaaTyN77mYd_Tu5JRviEJcmeo0XlFOxRRuQw,2299
+lida/components/scaffold.py,sha256=DK-a6O9sjRHXVofyOSOzkAyTucVESxIwq4YrJDih2Xg,5090
+lida/components/summarizer.py,sha256=JZMFjgwxfJmre_0ZndEV-ffev9yScsd7vZO6DWRFpwM,6819
+lida/components/viz/__init__.py,sha256=9N76qTPXTP3rC4YEfZBYlRnyHCWmMiD3e5k4lKyLWcU,166
+lida/components/viz/__pycache__/__init__.cpython-312.pyc,,
+lida/components/viz/__pycache__/vizeditor.cpython-312.pyc,,
+lida/components/viz/__pycache__/vizevaluator.cpython-312.pyc,,
+lida/components/viz/__pycache__/vizexplainer.cpython-312.pyc,,
+lida/components/viz/__pycache__/vizgenerator.cpython-312.pyc,,
+lida/components/viz/__pycache__/vizrecommender.cpython-312.pyc,,
+lida/components/viz/__pycache__/vizrepairer.cpython-312.pyc,,
+lida/components/viz/vizeditor.py,sha256=YmtmMT-GG9h9-OATkJD9y6vsCpuHtSPotn-qCSiAZ8I,2246
+lida/components/viz/vizevaluator.py,sha256=hcWYD4iCw2JPe5uPXhuim1MZgqp6UXptlOmiUEQCaHM,3632
+lida/components/viz/vizexplainer.py,sha256=FNjJds68yfaosSoxzOTyFw_GQzZXnW9PHCiIeO8iq0c,3141
+lida/components/viz/vizgenerator.py,sha256=PqmdY1sRW7lRj518nnKjtiBNtBEErjghW4mtpX2eslc,2762
+lida/components/viz/vizrecommender.py,sha256=7YTnfjceT_78pZy2Mw8U-mxQr7vfhMw87HnDKcfXXBc,3225
+lida/components/viz/vizrepairer.py,sha256=4NE3dzZ0ZzVLikqL71PmzCqkpCYJDJVwV7xygdJoFHc,2056
+lida/datamodel.py,sha256=-8CqFd1yDcq8VWCMpVtchykb2oA-DxO_udD3Gb15agg,5066
+lida/utils.py,sha256=-hHNifyRQp4BI18mLJBblqS4_fi1JlQKODxgoFbKACc,7883
+lida/version.py,sha256=Re70LR9m7cAhH54rssYyZTF_NDTijR8Lo_1hWF3ofTI,19
+lida/web/.gitignore,sha256=XFjtcLQokSpzuLSJJA0krmuRnvuZnml0xOI1t6DfgB8,1951
+lida/web/__init__.py,sha256=41PQWCBiwBL4PvUq7LTosbpPKRjMhiO-_nw_a1_a9i0,57
+lida/web/__pycache__/__init__.cpython-312.pyc,,
+lida/web/__pycache__/app.cpython-312.pyc,,
+lida/web/app.py,sha256=oFIo74JKgStw_k07rDliS0ID4K5ZUnhCxZ3h06SeC6M,12076
+lida/web/ui/21614df092a0a42959abd49fa5ffec5702fa9463-ea1bf954f60f23d1e6a7.js,sha256=4aVDoO4R7GgvncjyOrYPQMm35-fkz9hAn_Ba4HcBA4k,75617
+lida/web/ui/21614df092a0a42959abd49fa5ffec5702fa9463-ea1bf954f60f23d1e6a7.js.map,sha256=Ie75dYmYDOdc2hLI3rBSNL7r2vwfGPgR3TxhvYqfMc4,232206
+lida/web/ui/404.html,sha256=WDZyjf50UKcX-JAm1E-2ccKZfFbhfWYdhSkLucKuLhc,33693
+lida/web/ui/404/index.html,sha256=VZT2ohc4teIE3xGwSfJYHK6gCb85KpLXPUAyDhvwD6M,33689
+lida/web/ui/81e257386408544e35976acc2a4075b730ed48a4-326068d326429305ec9e.js,sha256=AvN2-sKtJiA_A9MDyI8u0UbtMYIT4-awXL-scJmbTng,1023442
+lida/web/ui/81e257386408544e35976acc2a4075b730ed48a4-326068d326429305ec9e.js.LICENSE.txt,sha256=w99HJHjUmlrRrav8yVCgDpHvFi0mJwdB8NYFWSE9FBY,368
+lida/web/ui/81e257386408544e35976acc2a4075b730ed48a4-326068d326429305ec9e.js.map,sha256=2MvtKHv3u2EozXwFLN9ZzZtfKK-jvFR7ddNcE6BnwRw,2235246
+lida/web/ui/app-80dcacffbc4d71399cc7.js,sha256=UpI_z7ci8jkyQVPmv0-S1yd4PPOFSPz1QgN8lxj7oxo,92843
+lida/web/ui/app-80dcacffbc4d71399cc7.js.map,sha256=lpqocLT96d03A9iyXSZYJNFPKuIPMYZOdJSGVRqm8NA,366829
+lida/web/ui/chunk-map.json,sha256=NGlK5-KjsqlFr7uI2i-4r1Eop1Ofma1KchnM6GUbwBg,463
+lida/web/ui/component---src-pages-404-tsx-271998ff555bf33bd7ce.js,sha256=OsJgqFvtKgcd-fOt1uZrFT1UPpLIsoqpw5WWyZ-6DoU,796
+lida/web/ui/component---src-pages-404-tsx-271998ff555bf33bd7ce.js.map,sha256=uXI9Lodeu5ecit4MNgtOOriKL_CVHvoqaE1qOdE9aFM,1986
+lida/web/ui/component---src-pages-demo-tsx-54fd6da10fa870d8d843.js,sha256=vF-8PLKlISyekaiRteMe1aqLMylVti66TegM0yObxEM,2575854
+lida/web/ui/component---src-pages-demo-tsx-54fd6da10fa870d8d843.js.LICENSE.txt,sha256=OxpnaKJbsQKb59YXzfE7x9aF59tIhVexdZgCjJMqxvA,255
+lida/web/ui/component---src-pages-demo-tsx-54fd6da10fa870d8d843.js.map,sha256=n1L7Qfc7WOpV_Ul3JTENJPuk2gfWYvzHouSJorJ8fgY,11066782
+lida/web/ui/component---src-pages-index-tsx-36a4bd360cacad51120f.js,sha256=JoCdvxYcTxez9vGw13FpCgZYJThh265trVR3XoxmsIM,58467
+lida/web/ui/component---src-pages-index-tsx-36a4bd360cacad51120f.js.LICENSE.txt,sha256=2JfuP83fRW3Pz2a3fdlWh1bR2RT2rng4b4l3wyTq1QI,81
+lida/web/ui/component---src-pages-index-tsx-36a4bd360cacad51120f.js.map,sha256=mmUYQgNR-kNhgzhywSz-5HdebUNb-2B2ltIxu0vqP_I,144441
+lida/web/ui/component---src-pages-login-tsx-bbe1cb64566ee588bf48.js,sha256=EBMIINO9mLg5P8Buf9q_V1Vy2Jrt-FX7fS0vzecPJHs,334
+lida/web/ui/component---src-pages-login-tsx-bbe1cb64566ee588bf48.js.map,sha256=7R9pGDJ2sLjkRzbPW7FOcgso8-sTA8cA8_ZcZy9wZh8,834
+lida/web/ui/demo/index.html,sha256=INJepIamxOF4J532hQBiB_6LXoRV-qRpyCFJb2ZLi88,45396
+lida/web/ui/favicon-32x32.png,sha256=iVkmreRqCcMmvCfA8GG97KSXJ5VQgr2LSGvEy9dwqjM,1261
+lida/web/ui/files/infographics_small.jpg,sha256=gH3Et6K7K6Mc6nlYVRUU-r1OTcfTsbzAoSwcmYqy9XY,579528
+lida/web/ui/files/lidamodules.jpg,sha256=km8887P_cmVFEWfszKXjiXyJf1e96kQyYnhUCiDP9vc,444281
+lida/web/ui/framework-adcf09f5896f386aa38b.js,sha256=R1V2mLMyVcde9eXyWw0ISUWAQsw4CuzIXRtiCF4jZzs,129870
+lida/web/ui/framework-adcf09f5896f386aa38b.js.LICENSE.txt,sha256=rAVUv7o8YtMNUiCz6P4UPh956J0Bsf5kE7k45rmyTRU,736
+lida/web/ui/framework-adcf09f5896f386aa38b.js.map,sha256=MhhKN2GC_0Vh_WeH2Jf5fRJY8eqB29lzZ8e9F8Afz4I,325126
+lida/web/ui/icons/icon-144x144.png,sha256=Oe4BCNmLpmc7QHjSrJaKa35QNMLWP7D-PV6u3O8vakM,5193
+lida/web/ui/icons/icon-192x192.png,sha256=f1Qt7KzgXebMXRMLEpQ8HmwJ7iRWKD1utxMt0ks9-Ps,7253
+lida/web/ui/icons/icon-256x256.png,sha256=V_ya4RWfpMAm8bMX4jiyOH9CI0APPvYRuEeHf1QCmx4,10405
+lida/web/ui/icons/icon-384x384.png,sha256=kgccmWDwgsjEfdJU1Ql3S6eNHAezlFITs-YDpIbR6qA,16972
+lida/web/ui/icons/icon-48x48.png,sha256=b877kD-RF_LyQY9FQVOg8WfN8aOS77aSqBWe--dE7Rg,1554
+lida/web/ui/icons/icon-512x512.png,sha256=gG61aUWehlG1C5sDcHbLsAE8WqvtRudkQAlAtHpEjVY,17708
+lida/web/ui/icons/icon-72x72.png,sha256=M4vgzglAN18UyYfVQdLJXD5lH66LojMpTE2tMolj0wY,2301
+lida/web/ui/icons/icon-96x96.png,sha256=sGMi8cFXyKVL8nd0zr_UtpZWQ-DeKHhkoK8DvWwb1u0,3189
+lida/web/ui/images/default.png,sha256=8Uujd1ZsVvu2TipGh-IpL96VY-D4EzgT8dMwhRqYH-c,2181
+lida/web/ui/images/screen.png,sha256=32yA0XTIACR41rEOSbPRalcOflnbhtqQuedkKXxdP-0,859540
+lida/web/ui/images/videoscreen.png,sha256=JEKpRdZ6Hu05L90Z6a08VDzyeheQrDfjJQkPjb0ThFY,297503
+lida/web/ui/index.html,sha256=KVhiLpswj6qrXgXkD-QXQyac9xfcp4TN9N7L-pz37PI,70501
+lida/web/ui/login/index.html,sha256=fygLgpAVim7XQejshb5PVb6uJ23AS-KEC0wfkVrPdKs,38129
+lida/web/ui/manifest.webmanifest,sha256=I27E1YkJkNox5HH0KCrdeJGd0qJ5TBJ2Ik-CSfLboIo,839
+lida/web/ui/page-data/404.html/page-data.json,sha256=SNaqt-SfyIUEBDdfZOcKMwf6-5m2ABaAeyrWcVD5nDo,124
+lida/web/ui/page-data/404/page-data.json,sha256=WeQ4MtE5uTDhhcllrYfoasoa0MAV4JlQVQ7cbPJaNjM,120
+lida/web/ui/page-data/app-data.json,sha256=ezJ1Co9TkrHd2IgPGxSl1arWm-hPcRCTvUhalSSNvGA,50
+lida/web/ui/page-data/demo/page-data.json,sha256=w9CFbpbu7wwO7GSH_9OBLcXUQrgFUgKvnL4k26x2hdU,234
+lida/web/ui/page-data/index/page-data.json,sha256=WgJ2ybagiFRTVk336zFdCCoqTyKNaYETjbUDnJyaN8U,230
+lida/web/ui/page-data/login/page-data.json,sha256=olO33JwKWIwZ_lTwiOQoA9Rq19CrSTWvwEbXKz8Tcs0,136
+lida/web/ui/page-data/sq/d/1865044719.json,sha256=txg4OyXorH8gMXXj9hHOyDUuq7crmUkyfrInBKwWpdQ,204
+lida/web/ui/polyfill-9f027554f9c426b688ff.js,sha256=4Zs9J1BzMqSXPEFoTmn5Egi-w75lmBhIyEyZX6ohIw8,84996
+lida/web/ui/polyfill-9f027554f9c426b688ff.js.map,sha256=1LhDnOuoeo2PShM63PPP-_fa67tNskVFk_TyQohImOU,207967
+lida/web/ui/sitemap/sitemap-0.xml,sha256=wsMkW4LAkoHiSEltYHJVkWPW4glVfIwl0StUQRyIIo4,657
+lida/web/ui/sitemap/sitemap-index.xml,sha256=TAqF8h914e7WqqcADO52Lg5CJnSla6AMhUN4PhVCLoc,198
+lida/web/ui/styles.15bd7f1a07f6d77699dc.css,sha256=RqfsFlXsptr-TPoKmmwWaueDIF6wuKSnEzKJCtRhdDw,28926
+lida/web/ui/webpack-runtime-8958b079d5c4b6876c66.js,sha256=k_UYQlqXY1g5bCLkgL0rfXM9ISNIDZwRm4_1dmJOhJo,4031
+lida/web/ui/webpack-runtime-8958b079d5c4b6876c66.js.map,sha256=oBf39hpx6q_gE1pnzzq6jXfjtxZ91rnwKoSZCOMKwBk,18625
+lida/web/ui/webpack.stats.json,sha256=sYVq0gTItl0k683QH5E4zTogNUYvh_7sc5KS_rwr_3I,3151
+lida/web/ui/~partytown/debug/partytown-atomics.js,sha256=CCiVZvJlJJ-bpdlu3YZp56d9lwRXLBr3vwPnwFBqOgY,28104
+lida/web/ui/~partytown/debug/partytown-media.js,sha256=zxUyEHmEWOnQPbvASzdPsXb0OpQzuuSbbEBrYxuRxzo,17511
+lida/web/ui/~partytown/debug/partytown-sandbox-sw.js,sha256=mVf8LhtM14vhjQAYRv-jbS4h1Qy4yJQ6VatbJ0ETcR8,27197
+lida/web/ui/~partytown/debug/partytown-sw.js,sha256=PZMigbHnae4mx1-iGRMqmnjYyEa9jemjxN0mauWrX2E,1995
+lida/web/ui/~partytown/debug/partytown-ww-atomics.js,sha256=IcpXMfkSWv3bEsostzwb7Vc-VWUwjyPIq_C_OlvZAwU,82089
+lida/web/ui/~partytown/debug/partytown-ww-sw.js,sha256=G6aQu6TpZodxizX_-Y0oVgjr6cUYucD8Un3hoKSevTA,81649
+lida/web/ui/~partytown/debug/partytown.js,sha256=MHbZY3VWACUHpWJ8On1qh4a4TlU-lK_YEfIp6CioEDk,3341
+lida/web/ui/~partytown/partytown-atomics.js,sha256=reONw3Q-OMEL7dQBnsdRp5FBkTw7ScQC-ot7LPeKZ_4,28273
+lida/web/ui/~partytown/partytown-media.js,sha256=B-7OwpuPYwU90tdc5kT_yH77suMas2JfPaNh1k26o60,5689
+lida/web/ui/~partytown/partytown-sw.js,sha256=_Cwc2iY6AaE0R0ZBAf_nQlF6JHxIg3hKU1VTUhkyED0,29137
+lida/web/ui/~partytown/partytown.js,sha256=Zp1E4gz7Nbna2RFs1iRc1hH-4a7p6b717bwjLX-KYkc,1384

lida-0.0.14.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.42.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

lida-0.0.14.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+lida = lida.cli:run

lida-0.0.14.dist-info/top_level.txt

@@ -0,0 +1 @@
+lida

lida_ko/__init__.py

@@ -0,0 +1,5 @@
+from llmx import TextGenerationConfig, llm, TextGenerator
+from .components.manager import Manager
+
+
+__all__ = ["TextGenerationConfig", "llm", "TextGenerator", "Manager"]

lida_ko/cli.py

@@ -0,0 +1,47 @@
+import typer
+import uvicorn
+import os
+from typing_extensions import Annotated
+from llmx import providers
+
+# from lida.web.backend.app import launch
+
+app = typer.Typer()
+
+
+@app.command()
+def ui(host: str = "127.0.0.1",
+       port: int = 8081,
+       workers: int = 1,
+       reload: Annotated[bool, typer.Option("--reload")] = True,
+       docs: bool = False):
+    """
+    Launch the lida .Pass in parameters host, port, workers, and reload to override the default values.
+    """
+
+    os.environ["LIDA_API_DOCS"] = str(docs)
+
+    uvicorn.run(
+        "lida.web.app:app",
+        host=host,
+        port=port,
+        workers=workers,
+        reload=reload,
+    )
+
+
+@app.command()
+def models():
+    print("A list of supported providers:")
+    for provider in providers.items():
+        print(f"Provider: {provider[1]['name']}")
+        for model in provider[1]["models"]:
+            print(f"  - {model['name']}")
+
+
+def run():
+    app()
+
+
+if __name__ == "__main__":
+    app()

lida_ko/components/__init__.py

@@ -0,0 +1,7 @@
+from .summarizer import Summarizer
+from .viz import *
+from .goal import *
+from .scaffold import *
+from .executor import *
+from .manager import *
+from .persona import *

lida_ko/components/executor.py

@@ -0,0 +1,282 @@
+import ast
+import base64
+import importlib
+import io
+import os
+import re
+import traceback
+from typing import Any, List
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import plotly.io as pio
+
+from lida_ko.datamodel import ChartExecutorResponse, Summary
+
+
+def preprocess_code(code: str) -> str:
+    """Preprocess code to remove any preamble and explanation text"""
+
+    code = code.replace("<imports>", "")
+    code = code.replace("<stub>", "")
+    code = code.replace("<transforms>", "")
+
+    # remove all text after chart = plot(data)
+    if "chart = plot(data)" in code:
+        # print(code)
+        index = code.find("chart = plot(data)")
+        if index != -1:
+            code = code[: index + len("chart = plot(data)")]
+
+    if "```" in code:
+        pattern = r"```(?:\w+\n)?([\s\S]+?)```"
+        matches = re.findall(pattern, code)
+        if matches:
+            code = matches[0]
+        # code = code.replace("```", "")
+        # return code
+
+    if "import" in code:
+        # return only text after the first import statement
+        index = code.find("import")
+        if index != -1:
+            code = code[index:]
+
+    code = code.replace("```", "")
+    if "chart = plot(data)" not in code:
+        code = code + "\nchart = plot(data)"
+    return code
+
+
+def get_globals_dict(code_string, data):
+    # Parse the code string into an AST
+    tree = ast.parse(code_string)
+    # Extract the names of the imported modules and their aliases
+    imported_modules = []
+    for node in tree.body:
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                module = importlib.import_module(alias.name)
+                imported_modules.append((alias.name, alias.asname, module))
+        elif isinstance(node, ast.ImportFrom):
+            module = importlib.import_module(node.module)
+            for alias in node.names:
+                obj = getattr(module, alias.name)
+                imported_modules.append(
+                    (f"{node.module}.{alias.name}", alias.asname, obj)
+                )
+
+    # Import the required modules into a dictionary
+    globals_dict = {}
+    for module_name, alias, obj in imported_modules:
+        if alias:
+            globals_dict[alias] = obj
+        else:
+            globals_dict[module_name.split(".")[-1]] = obj
+
+    ex_dicts = {"pd": pd, "data": data, "plt": plt}
+    globals_dict.update(ex_dicts)
+    return globals_dict
+
+
+class ChartExecutor:
+    """Execute code and return chart object"""
+
+    def __init__(self) -> None:
+        pass
+
+    def execute(
+        self,
+        code_specs: List[str],
+        data: Any,
+        summary: Summary,
+        library="altair",
+        return_error: bool = False,
+    ) -> Any:
+        """Validate and convert code"""
+
+        # # check if user has given permission to execute code. if env variable
+        # # LIDA_ALLOW_CODE_EVAL is set to '1'. Else raise exception
+        # if os.environ.get("LIDA_ALLOW_CODE_EVAL") != '1':
+        #     raise Exception(
+        #         "Permission to execute code not granted. Please set the environment variable LIDA_ALLOW_CODE_EVAL to '1' to allow code execution.")
+
+        if isinstance(summary, dict):
+            summary = Summary(**summary)
+
+        charts = []
+        code_spec_copy = code_specs.copy()
+        code_specs = [preprocess_code(code) for code in code_specs]
+        if library == "altair":
+            for code in code_specs:
+                try:
+                    ex_locals = get_globals_dict(code, data)
+                    exec(code, ex_locals)
+                    chart = ex_locals["chart"]
+                    vega_spec = chart.to_dict()
+                    del vega_spec["data"]
+                    if "datasets" in vega_spec:
+                        del vega_spec["datasets"]
+
+                    vega_spec["data"] = {"url": f"/files/data/{summary.file_name}"}
+                    charts.append(
+                        ChartExecutorResponse(
+                            spec=vega_spec,
+                            status=True,
+                            raster=None,
+                            code=code,
+                            library=library,
+                        )
+                    )
+                except Exception as exception_error:
+                    print(code_spec_copy, "\n===========\n")
+                    print(exception_error)
+                    print(traceback.format_exc())
+                    if return_error:
+                        charts.append(
+                            ChartExecutorResponse(
+                                spec=None,
+                                status=False,
+                                raster=None,
+                                code=code,
+                                library=library,
+                                error={
+                                    "message": str(exception_error),
+                                    "traceback": traceback.format_exc(),
+                                },
+                            )
+                        )
+            return charts
+        elif library == "matplotlib" or library == "seaborn":
+            # print colum dtypes
+            for code in code_specs:
+                try:
+                    ex_locals = get_globals_dict(code, data)
+                    # print(ex_locals)
+                    exec(code, ex_locals)
+                    chart = ex_locals["chart"]
+                    if plt:
+                        buf = io.BytesIO()
+                        plt.box(False)
+                        plt.grid(color="lightgray", linestyle="dashed", zorder=-10)
+                        # try:
+                        #     plt.draw()
+                        #     # plt.tight_layout()
+                        # except AttributeError:
+                        #     print("Warning: tight_layout encountered an error. The layout may not be optimal.")
+                        #     pass
+
+                        plt.savefig(buf, format="png", dpi=100, pad_inches=0.2)
+                        buf.seek(0)
+                        plot_data = base64.b64encode(buf.read()).decode("ascii")
+                        plt.close()
+                    charts.append(
+                        ChartExecutorResponse(
+                            spec=None,
+                            status=True,
+                            raster=plot_data,
+                            code=code,
+                            library=library,
+                        )
+                    )
+                except Exception as exception_error:
+                    print(code_spec_copy[0])
+                    print("****\n", str(exception_error))
+                    # print(traceback.format_exc())
+                    if return_error:
+                        charts.append(
+                            ChartExecutorResponse(
+                                spec=None,
+                                status=False,
+                                raster=None,
+                                code=code,
+                                library=library,
+                                error={
+                                    "message": str(exception_error),
+                                    "traceback": traceback.format_exc(),
+                                },
+                            )
+                        )
+            return charts
+        elif library == "ggplot":
+            # print colum dtypes
+            for code in code_specs:
+                try:
+                    ex_locals = get_globals_dict(code, data)
+                    exec(code, ex_locals)
+                    chart = ex_locals["chart"]
+                    if plt:
+                        buf = io.BytesIO()
+                        chart.save(buf, format="png")
+                        plot_data = base64.b64encode(buf.getvalue()).decode("utf-8")
+                    charts.append(
+                        ChartExecutorResponse(
+                            spec=None,
+                            status=True,
+                            raster=plot_data,
+                            code=code,
+                            library=library,
+                        )
+                    )
+                except Exception as exception_error:
+                    print(code)
+                    print(traceback.format_exc())
+                    if return_error:
+                        charts.append(
+                            ChartExecutorResponse(
+                                spec=None,
+                                status=False,
+                                raster=None,
+                                code=code,
+                                library=library,
+                                error={
+                                    "message": str(exception_error),
+                                    "traceback": traceback.format_exc(),
+                                },
+                            )
+                        )
+            return charts
+
+        elif library == "plotly":
+            for code in code_specs:
+                try:
+                    ex_locals = get_globals_dict(code, data)
+                    exec(code, ex_locals)
+                    chart = ex_locals["chart"]
+
+                    if pio:
+                        chart_bytes = pio.to_image(chart, 'png')
+                        plot_data = base64.b64encode(chart_bytes).decode('utf-8')
+
+                        charts.append(
+                            ChartExecutorResponse(
+                                spec=None,
+                                status=True,
+                                raster=plot_data,
+                                code=code,
+                                library=library,
+                            )
+                        )
+                except Exception as exception_error:
+                    print(code)
+                    print(traceback.format_exc())
+                    if return_error:
+                        charts.append(
+                            ChartExecutorResponse(
+                                spec=None,
+                                status=False,
+                                raster=None,
+                                code=code,
+                                library=library,
+                                error={
+                                    "message": str(exception_error),
+                                    "traceback": traceback.format_exc(),
+                                },
+                            )
+                        )
+            return charts
+
+        else:
+            raise Exception(
+                f"Unsupported library. Supported libraries are altair, matplotlib, seaborn, ggplot, plotly. You provided {library}"
+            )

lida_ko/components/goal.py

@@ -0,0 +1,65 @@
+import json
+import logging
+from lida_ko.utils import clean_code_snippet
+from llmx import TextGenerator
+from lida_ko.datamodel import Goal, TextGenerationConfig, Persona
+
+
+SYSTEM_INSTRUCTIONS = """
+You are a an experienced data analyst who can generate a given number of insightful GOALS about data, when given a summary of the data, and a specified persona. The VISUALIZATIONS YOU RECOMMEND MUST FOLLOW VISUALIZATION BEST PRACTICES (e.g., must use bar charts instead of pie charts for comparing quantities) AND BE MEANINGFUL (e.g., plot longitude and latitude on maps where appropriate). They must also be relevant to the specified persona. Each goal must include a question, a visualization (THE VISUALIZATION MUST REFERENCE THE EXACT COLUMN FIELDS FROM THE SUMMARY), and a rationale (JUSTIFICATION FOR WHICH dataset FIELDS ARE USED and what we will learn from the visualization). Each goal MUST mention the exact fields from the dataset summary above. YOU MUST provide goals in KOREAN, 한글
+"""
+
+FORMAT_INSTRUCTIONS = """
+THE OUTPUT MUST BE A CODE SNIPPET OF A VALID LIST OF JSON OBJECTS. IT MUST USE THE FOLLOWING FORMAT:
+
+```[
+    { "index": 0,  "question": "What is the distribution of X", "visualization": "histogram of X", "rationale": "This tells about "} ..
+    ]
+```
+THE OUTPUT SHOULD ONLY USE THE JSON FORMAT ABOVE.
+"""
+
+logger = logging.getLogger("lida")
+
+
+class GoalExplorer():
+    """Generat goals given a summary of data"""
+
+    def __init__(self) -> None:
+        pass
+
+    def generate(self, summary: dict, textgen_config: TextGenerationConfig,
+                 text_gen: TextGenerator, n=5, persona: Persona = None) -> list[Goal]:
+        """Generate goals given a summary of data"""
+
+        user_prompt = f"""The number of GOALS to generate is {n}. The goals should be based on the data summary below, \n\n .
+        {summary} \n\n"""
+
+        if not persona:
+            persona = Persona(
+                persona="A highly skilled data analyst who can come up with complex, insightful goals about data",
+                rationale="")
+
+        user_prompt += f"""\n The generated goals SHOULD BE FOCUSED ON THE INTERESTS AND PERSPECTIVE of a '{persona.persona} persona, who is insterested in complex, insightful goals about the data. \n"""
+
+        messages = [
+            {"role": "system", "content": SYSTEM_INSTRUCTIONS},
+            {"role": "assistant",
+             "content":
+             f"{user_prompt}\n\n {FORMAT_INSTRUCTIONS} \n\n. The generated {n} goals are: \n "}]
+
+        result: list[Goal] = text_gen.generate(messages=messages, config=textgen_config)
+
+        try:
+            json_string = clean_code_snippet(result.text[0]["content"])
+            result = json.loads(json_string)
+            # cast each item in the list to a Goal object
+            if isinstance(result, dict):
+                result = [result]
+            result = [Goal(**x) for x in result]
+        except json.decoder.JSONDecodeError:
+            logger.info(f"Error decoding JSON: {result.text[0]['content']}")
+            print(f"Error decoding JSON: {result.text[0]['content']}")
+            raise ValueError(
+                "The model did not return a valid JSON object while attempting generate goals. Consider using a larger model or a model with higher max token length.")
+        return result

lida_ko/components/infographer.py

@@ -0,0 +1,62 @@
+
+import logging
+from typing import Any, List, Union
+import PIL
+from peacasso.generator import ImageGenerator
+from peacasso.datamodel import GeneratorConfig, ModelConfig
+from peacasso.utils import base64_to_pil, pil_to_base64
+import torch
+
+logger = logging.getLogger("lida")
+
+
+class Infographer():
+    """Generat infographics given a visualization and a summary of data"""
+
+    def __init__(self, model_config: ModelConfig = None) -> None:
+        self.model = None
+        self.model_config = model_config or ModelConfig(
+            device="cuda",
+            model="runwayml/stable-diffusion-v1-5",
+            revision="main"
+        )
+
+    def load_model(self) -> None:
+        """Load image generator model from config"""
+        self.model = ImageGenerator(model_config=self.model_config)
+
+    def generate(
+            self, visualization: Union[torch.FloatTensor, PIL.Image.Image, str],
+            n: int, style_prompt: Union[str, List[str]] = "line art pastel",
+            return_pil: bool = True
+    ) -> List[Any]:
+        """Generate a an infographic, given a visualization and style"""
+
+        if isinstance(visualization, str):
+            try:
+                visualization, _ = base64_to_pil(visualization)
+            except Exception as pil_exception:
+                logger.error(pil_exception)
+                raise ValueError(
+                    f'Could not convert provided visualization to PIL image, {str(pil_exception)}') from pil_exception
+            self.load_model()
+
+        gen_config = GeneratorConfig(
+            prompt=style_prompt,
+            num_images=n,
+            width=512,
+            height=512,
+            guidance_scale=7.5,
+            num_inference_steps=50,
+            init_image=visualization,
+            return_intermediates=False,
+            seed=2147483647,
+            use_prompt_weights=False,
+            negative_prompt="text, background shapes or lines, title, words, characters, titles, letters",
+            strength=0.6,
+            filter_nsfw=False)
+
+        result = self.model.generate(gen_config)
+        if not return_pil:
+            result["images"] = [pil_to_base64(img) for img in result["images"]]
+        return result

lida_ko/components/manager.py

@@ -0,0 +1,449 @@
+# Visualization manager class that handles the visualization of the data with the following methods
+
+# summarize data given a df
+# generate goals given a summary
+# generate generate visualization specifications given a summary and a goal
+# execute the specification given some data
+
+import os
+from typing import List, Union
+import logging
+
+import pandas as pd
+from llmx import llm, TextGenerator
+from lida_ko.datamodel import Goal, Summary, TextGenerationConfig, Persona
+from lida_ko.utils import read_dataframe
+from ..components.summarizer import Summarizer
+from ..components.goal import GoalExplorer
+from ..components.persona import PersonaExplorer
+from ..components.executor import ChartExecutor
+from ..components.viz import VizGenerator, VizEditor, VizExplainer, VizEvaluator, VizRepairer, VizRecommender
+
+import lida_ko.web as lida
+
+
+logger = logging.getLogger("lida")
+
+
+class Manager(object):
+    def __init__(self, text_gen: TextGenerator = None) -> None:
+        """
+        Initialize the Manager object.
+
+        Args:
+            text_gen (TextGenerator, optional): Text generator object. Defaults to None.
+        """
+
+        self.text_gen = text_gen or llm()
+
+        self.summarizer = Summarizer()
+        self.goal = GoalExplorer()
+        self.vizgen = VizGenerator()
+        self.vizeditor = VizEditor()
+        self.executor = ChartExecutor()
+        self.explainer = VizExplainer()
+        self.evaluator = VizEvaluator()
+        self.repairer = VizRepairer()
+        self.recommender = VizRecommender()
+        self.data = None
+        self.infographer = None
+        self.persona = PersonaExplorer()
+
+    def check_textgen(self, config: TextGenerationConfig):
+        """
+        Check if self.text_gen is the same as the config passed in. If not, update self.text_gen.
+
+        Args:
+            config (TextGenerationConfig): Text generation configuration.
+        """
+        if config.provider is None:
+            config.provider = self.text_gen.provider or "openai"
+            logger.info("Provider is not set, using default provider - %s", config.provider)
+            return
+
+        if self.text_gen.provider != config.provider:
+
+            logger.info(
+                "Switching Text Generator Provider from %s to %s",
+                self.text_gen.provider,
+                config.provider)
+            self.text_gen = llm(provider=config.provider)
+
+    def summarize(
+        self,
+        data: Union[pd.DataFrame, str],
+        file_name="",
+        n_samples: int = 3,
+        summary_method: str = "default",
+        textgen_config: TextGenerationConfig = TextGenerationConfig(n=1, temperature=0),
+        metadata: dict = None,
+    ) -> Summary:
+        """
+        Summarize data given a DataFrame or file path.
+
+        Args:
+            data (Union[pd.DataFrame, str]): Input data, either a DataFrame or file path.
+            file_name (str, optional): Name of the file if data is loaded from a file path. Defaults to "".
+            n_samples (int, optional): Number of summary samples to generate. Defaults to 3.
+            summary_method (str, optional): Summary method to use. Defaults to "default".
+            textgen_config (TextGenerationConfig, optional): Text generation configuration. Defaults to TextGenerationConfig(n=1, temperature=0).
+
+        Returns:
+            Summary: Summary object containing the generated summary.
+
+        Example of Summary:
+
+            {'name': 'cars.csv',
+            'file_name': 'cars.csv',
+            'dataset_description': '',
+            'fields': [{'column': 'Name',
+            'properties': {'dtype': 'string',
+                'samples': ['Nissan Altima S 4dr',
+                'Mercury Marauder 4dr',
+                'Toyota Prius 4dr (gas/electric)'],
+                'num_unique_values': 385,
+                'semantic_type': '',
+                'description': ''}},
+            {'column': 'Type',
+            'properties': {'dtype': 'category',
+                'samples': ['SUV', 'Minivan', 'Sports Car'],
+                'num_unique_values': 5,
+                'semantic_type': '',
+                'description': ''}},
+            {'column': 'AWD',
+            'properties': {'dtype': 'number',
+                'std': 0,
+                'min': 0,
+                'max': 1,
+                'samples': [1, 0],
+                'num_unique_values': 2,
+                'semantic_type': '',
+                'description': ''}},
+            }
+
+        """
+        self.check_textgen(config=textgen_config)
+
+        if isinstance(data, str):
+            file_name = data.split("/")[-1]
+            data = read_dataframe(data)
+
+        self.data = data
+        return self.summarizer.summarize(
+            data=self.data, text_gen=self.text_gen, file_name=file_name, n_samples=n_samples,
+            summary_method=summary_method, textgen_config=textgen_config,
+            metadata=metadata)
+
+    def goals(
+        self,
+        summary: Summary,
+        textgen_config: TextGenerationConfig = TextGenerationConfig(),
+        n: int = 5,
+        persona: Persona = None
+    ) -> List[Goal]:
+        """
+        Generate goals based on a summary.
+
+        Args:
+            summary (Summary): Input summary.
+            textgen_config (TextGenerationConfig, optional): Text generation configuration. Defaults to TextGenerationConfig().
+            n (int, optional): Number of goals to generate. Defaults to 5.
+            persona (Persona, str, dict, optional): Persona information. Defaults to None.
+
+        Returns:
+            List[Goal]: List of generated goals.
+
+        Example of list of goals:
+
+            Goal 0
+            Question: What is the distribution of Retail_Price?
+
+            Visualization: histogram of Retail_Price
+
+            Rationale: This tells about the spread of prices of cars in the dataset.
+
+            Goal 1
+            Question: What is the distribution of Horsepower_HP_?
+
+            Visualization: box plot of Horsepower_HP_
+
+            Rationale: This tells about the distribution of horsepower of cars in the dataset.
+        """
+        self.check_textgen(config=textgen_config)
+
+        if isinstance(persona, dict):
+            persona = Persona(**persona)
+        if isinstance(persona, str):
+            persona = Persona(persona=persona, rationale="")
+
+        return self.goal.generate(summary=summary, text_gen=self.text_gen,
+                                  textgen_config=textgen_config, n=n, persona=persona)
+
+    def personas(
+            self, summary, textgen_config: TextGenerationConfig = TextGenerationConfig(),
+            n=5):
+        self.check_textgen(config=textgen_config)
+
+        return self.persona.generate(summary=summary, text_gen=self.text_gen,
+                                     textgen_config=textgen_config, n=n)
+
+    def visualize(
+        self,
+        summary,
+        goal,
+        textgen_config: TextGenerationConfig = TextGenerationConfig(),
+        library="seaborn",
+        return_error: bool = False,
+    ):
+        if isinstance(goal, dict):
+            goal = Goal(**goal)
+        if isinstance(goal, str):
+            goal = Goal(question=goal, visualization=goal, rationale="")
+
+        self.check_textgen(config=textgen_config)
+        code_specs = self.vizgen.generate(
+            summary=summary, goal=goal, textgen_config=textgen_config, text_gen=self.text_gen,
+            library=library)
+        if library == "plotly":
+            # if library is plotly, return chart without executing
+            charts = [{
+                'code' : code,
+                'data' : self.data,
+                'summary' : summary,
+                'library' : library,
+            } for code in code_specs]
+        else:
+            charts = self.execute(
+                code_specs=code_specs,
+                data=self.data,
+                summary=summary,
+                library=library,
+                return_error=return_error,
+            )
+        return charts
+
+    def execute(
+        self,
+        code_specs,
+        data,
+        summary: Summary,
+        library: str = "seaborn",
+        return_error: bool = False,
+    ):
+
+        if data is None:
+            root_file_path = os.path.dirname(os.path.abspath(lida.__file__))
+            print(root_file_path)
+            data = read_dataframe(
+                os.path.join(root_file_path, "files/data", summary.file_name)
+            )
+
+        # col_properties = summary.properties
+
+        return self.executor.execute(
+            code_specs=code_specs,
+            data=data,
+            summary=summary,
+            library=library,
+            return_error=return_error,
+        )
+
+    def edit(
+        self,
+        code,
+        summary: Summary,
+        instructions: List[str],
+        textgen_config: TextGenerationConfig = TextGenerationConfig(),
+        library: str = "seaborn",
+        return_error: bool = False,
+    ):
+        """Edit a visualization code given a set of instructions
+
+        Args:
+            code (_type_): _description_
+            instructions (List[Dict]): A list of instructions
+
+        Returns:
+            _type_: _description_
+        """
+
+        self.check_textgen(config=textgen_config)
+
+        if isinstance(instructions, str):
+            instructions = [instructions]
+
+        code_specs = self.vizeditor.generate(
+            code=code,
+            summary=summary,
+            instructions=instructions,
+            textgen_config=textgen_config,
+            text_gen=self.text_gen,
+            library=library,
+        )
+
+        charts = self.execute(
+            code_specs=code_specs,
+            data=self.data,
+            summary=summary,
+            library=library,
+            return_error=return_error,
+        )
+        return charts
+
+    def repair(
+        self,
+        code,
+        goal: Goal,
+        summary: Summary,
+        feedback,
+        textgen_config: TextGenerationConfig = TextGenerationConfig(),
+        library: str = "seaborn",
+        return_error: bool = False,
+    ):
+        """ Repair a visulization given some feedback"""
+        self.check_textgen(config=textgen_config)
+        code_specs = self.repairer.generate(
+            code=code,
+            feedback=feedback,
+            goal=goal,
+            summary=summary,
+            textgen_config=textgen_config,
+            text_gen=self.text_gen,
+            library=library,
+        )
+        if library == "plotly":
+            # if library is plotly, return chart without executing
+            charts = [{
+                'code' : code,
+                'data' : self.data,
+                'summary' : summary,
+                'library' : library,
+            } for code in code_specs]
+        charts = self.execute(
+            code_specs=code_specs,
+            data=self.data,
+            summary=summary,
+            library=library,
+            return_error=return_error,
+        )
+        return charts
+
+    def explain(
+        self,
+        code,
+        textgen_config: TextGenerationConfig = TextGenerationConfig(),
+        library: str = "seaborn",
+    ):
+        """Explain a visualization code given a set of instructions
+
+        Args:
+            code (_type_): _description_
+            instructions (List[Dict]): A list of instructions
+
+        Returns:
+            _type_: _description_
+        """
+        self.check_textgen(config=textgen_config)
+        return self.explainer.generate(
+            code=code,
+            textgen_config=textgen_config,
+            text_gen=self.text_gen,
+            library=library,
+        )
+
+    def evaluate(
+        self,
+        code,
+        goal: Goal,
+        textgen_config: TextGenerationConfig = TextGenerationConfig(),
+        library: str = "seaborn",
+    ):
+        """Evaluate a visualization code given a goal
+
+        Args:
+            code (_type_): _description_
+            goal (Goal): A visualization goal
+
+        Returns:
+            _type_: _description_
+        """
+
+        self.check_textgen(config=textgen_config)
+
+        return self.evaluator.generate(
+            code=code,
+            goal=goal,
+            textgen_config=textgen_config,
+            text_gen=self.text_gen,
+            library=library,
+        )
+
+    def recommend(
+        self,
+        code,
+        summary: Summary,
+        n=4,
+        textgen_config: TextGenerationConfig = TextGenerationConfig(),
+        library: str = "seaborn",
+        return_error: bool = False,
+    ):
+        """Edit a visualization code given a set of instructions
+
+        Args:
+            code (_type_): _description_
+            instructions (List[Dict]): A list of instructions
+
+        Returns:
+            _type_: _description_
+        """
+
+        self.check_textgen(config=textgen_config)
+
+        code_specs = self.recommender.generate(
+            code=code,
+            summary=summary,
+            n=n,
+            textgen_config=textgen_config,
+            text_gen=self.text_gen,
+            library=library,
+        )
+        charts = self.execute(
+            code_specs=code_specs,
+            data=self.data,
+            summary=summary,
+            library=library,
+            return_error=return_error,
+        )
+        return charts
+
+    def infographics(self, visualization: str, n: int = 1,
+                     style_prompt: Union[str, List[str]] = "",
+                     return_pil: bool = False
+                     ):
+        """
+        Generate infographics using the peacasso package.
+
+        Args:
+            visualization (str): A visualization code
+            n (int, optional): The number of infographics to generate. Defaults to 1.
+            style_prompt (Union[str, List[str]], optional): A style prompt or list of style prompts. Defaults to "".
+
+        Raises:
+            ImportError: If the peacasso package is not installed.
+        """
+
+        try:
+            import peacasso
+
+        except ImportError as exc:
+            raise ImportError(
+                'Please install lida with infographics support. pip install lida[infographics]. You will also need a GPU runtime.'
+            ) from exc
+
+        from ..components.infographer import Infographer
+
+        if self.infographer is None:
+            logger.info("Initializing Infographer")
+            self.infographer = Infographer()
+        return self.infographer.generate(
+            visualization=visualization, n=n, style_prompt=style_prompt, return_pil=return_pil)

lida_ko/components/persona.py

@@ -0,0 +1,52 @@
+import json
+import logging
+from lida_ko.utils import clean_code_snippet
+from llmx import TextGenerator
+from lida_ko.datamodel import Persona, TextGenerationConfig
+
+
+system_prompt = """You are an experienced data analyst  who can take a dataset summary and generate a list of n personas (e.g., ceo or accountant for finance related data, economist for population or gdp related data, doctors for health data, or just users) that might be critical stakeholders in exploring some data and describe rationale for why they are critical. The personas should be prioritized based on their relevance to the data. Think step by step.
+
+Your response should be perfect JSON in the following format:
+```[{"persona": "persona1", "rationale": "..."},{"persona": "persona1", "rationale": "..."}]```
+"""
+
+logger = logging.getLogger("lida")
+
+
+class PersonaExplorer():
+    """Generat personas given a summary of data"""
+
+    def __init__(self) -> None:
+        pass
+
+    def generate(self, summary: dict, textgen_config: TextGenerationConfig,
+                 text_gen: TextGenerator, n=5) -> list[Persona]:
+        """Generate personas given a summary of data"""
+
+        user_prompt = f"""The number of PERSONAs to generate is {n}. Generate {n} personas in the right format given the data summary below,\n .
+        {summary} \n""" + """
+
+        .
+        """
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "assistant", "content": user_prompt},
+        ]
+
+        result = text_gen.generate(messages=messages, config=textgen_config)
+
+        try:
+            json_string = clean_code_snippet(result.text[0]["content"])
+            result = json.loads(json_string)
+            # cast each item in the list to a Goal object
+            if isinstance(result, dict):
+                result = [result]
+            result = [Persona(**x) for x in result]
+        except json.decoder.JSONDecodeError:
+            logger.info(f"Error decoding JSON: {result.text[0]['content']}")
+            print(f"Error decoding JSON: {result.text[0]['content']}")
+            raise ValueError(
+                "The model did not return a valid JSON object while attempting generate personas.  Consider using a larger model or a model with higher max token length.")
+        return result

lida_ko/components/recommender.py

@@ -0,0 +1,118 @@
+import tqdm
+import os
+import pickle
+import json
+import logging
+import pandas as pd
+from typing import List, Dict, Any
+
+from konlpy.tag import Okt
+from rank_bm25 import BM25Okapi
+
+from llmx import TextGenerator, TextGenerationConfig
+
+system_prompt = """
+"""
+
+class Recommender:
+    def __init__(self) -> None:
+        self.recommendation = None
+        self.vector_db = None
+        self.query = None
+        self.metadata_for_bm25 = None
+
+        with open('data/id_to_metadata.json', 'r') as f:
+            self.id_to_metadata = json.load(f)
+        self._set_metadata_for_bm25()
+        self.okt = None
+
+    def recommend(self,
+                  text_gen: TextGenerator,
+                  textgen_config: TextGenerationConfig,
+                  query: str) -> Dict[Any]:
+        self.query = query
+        self.recommend_by_bm25()
+        self.recoomend_by_questions()
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "assistant", "content": f"""
+        Annotate the dictionary below. Only return a JSON object.
+        {base_summary}
+        """},
+        ]
+        # result must contains title, reason, relavance
+        extracted_result = []
+        text_gen.generate(messages=messages, config=textgen_config)
+
+        return extracted_result
+
+    def _set_metadata_for_bm25(self, data: Dict) -> Dict:
+        cache_file = 'data/metadata_for_bm25.pkl'
+
+        if not os.path.exists(cache_file):
+            result = {}
+            for k, v in self.id_to_metadata.items():
+                result[k] = ''
+                result[k] += v['title'] + ' '
+                result[k] += v['description'] + ' '
+                result[k] += ', '.join(v['keywords']) + ' '
+                result[k] += v['organization'] + ' '
+                result[k] += v['department'] + ' '
+            with open(cache_file, 'wb') as f:
+                pickle.dump(result, f)
+            result = self._set_metadata_for_bm25
+            return self.metadata_for_bm25
+
+        with open(cache_file, 'rb') as f:
+            result = pickle.load(f)    
+        result = self._set_metadata_for_bm25
+        return self.metadata_for_bm25
+
+    def _set_tokenized_corpus(self) -> None:
+        self.okt = Okt()
+        cache_file = 'data/tokenized_corpus.pkl'
+
+        if os.path.exists(cache_file):
+            with open(cache_file, 'rb') as f:
+                tokenized_corpus = pickle.load(f)
+                self.tokenized_corpus = tokenized_corpus
+                return
+
+        tokenized_corpus = [self.okt.morphs(value) for value in tqdm(self.metadata_for_bm25.values(), desc='Tokenizing')]
+
+        with open(cache_file, 'wb') as f:
+            pickle.dump(tokenized_corpus, f)
+
+        self.tokenized_corpus =tokenized_corpus
+        
+    def _recommend_by_bm25(self, top_n=20) -> List[str]:
+        if not self.query:
+            raise ValueError("query must be provided")
+
+        tokenized_query = self.okt.morphs(self.query)
+
+        bm25 = BM25Okapi(self.tokenized_corpus)
+
+        # 쿼리와 각 설명 간의 BM25 점수 계산
+        scores = bm25.get_scores(tokenized_query)
+        print(scores)
+
+        # 점수와 인덱스를 함께 저장
+        scored_datasets = list(zip(scores, self.metadata_for_bm25))
+        # 점수에 따라 정렬 (내림차순)
+        scored_datasets.sort(key=lambda x: x[0], reverse=True)
+
+        # 상위 N개의 결과 추출
+        top_n = 20
+        top_n_datasets = scored_datasets[:top_n]
+
+        print(top_n_datasets)
+
+        # 결과 출력
+        for score, data_id in top_n_datasets:
+            print(f"Dataset: {self.id_to_metadata[data_id]['title']}, Score: {score:.4f}, Description: {self.id_to_metadata[data_id]['description']}")
+
+    def _recommend_by_questions(self) -> List[str]:
+        pass
+    

lida_ko/components/scaffold.py

@@ -0,0 +1,123 @@
+from dataclasses import asdict
+
+from lida_ko.datamodel import Goal
+
+
+# if len(plt.xticks()[0])) > 20 assuming plot is made with plt or
+# len(ax.get_xticks()) > 20 assuming plot is made with ax, set a max of 20
+# ticks on x axis, ticker.MaxNLocator(20)
+
+class ChartScaffold(object):
+    """Return code scaffold for charts in multiple visualization libraries"""
+
+    def __init__(
+        self,
+    ) -> None:
+
+        pass
+
+    def get_template(self, goal: Goal, library: str):
+
+        general_instructions = f"If the solution requires a single value (e.g. max, min, median, first, last etc), ALWAYS add a line (axvline or axhline) to the chart, ALWAYS with a legend containing the single value (formatted with 0.2F). If using a <field> where semantic_type=date, YOU MUST APPLY the following transform before using that column i) convert date fields to date types using data[''] = pd.to_datetime(data[<field>], errors='coerce'), ALWAYS use  errors='coerce' ii) drop the rows with NaT values data = data[pd.notna(data[<field>])] iii) convert field to right time format for plotting.  ALWAYS make sure the x-axis labels are legible (e.g., rotate when needed). Solve the task  carefully by completing ONLY the <imports> AND <stub> section. Given the dataset summary, the plot(data) method should generate a {library} chart ({goal.visualization}) that addresses this goal: {goal.question}. DO NOT WRITE ANY CODE TO LOAD THE DATA. The data is already loaded and available in the variable data."
+
+        matplotlib_instructions = f" {general_instructions} DO NOT include plt.show(). The plot method must return a matplotlib object (plt). Think step by step. \n"
+
+        if library == "matplotlib":
+            instructions = {
+                "role": "assistant",
+                "content": f"  {matplotlib_instructions}. Use BaseMap for charts that require a map. Scope of BaseMap Must be restricted within BUSAN(city of south korea) area."}
+            template = \
+                f"""
+import matplotlib.pyplot as plt
+import pandas as pd
+import matplotlib.font_manager as fm
+<imports>
+# plan -
+def plot(data: pd.DataFrame):
+    path = '/usr/share/fonts/truetype/nanum/NanumGothic.ttf'
+    fontprop = fm.FontProperties(fname=path, size=12)
+    plt.rc('font', family=fontprop.get_name())
+    <stub> # only modify this section
+    plt.title('{goal.question}', wrap=True)
+    return plt;
+
+chart = plot(data) # data already contains the data to be plotted. Always include this line. No additional code beyond this line."""
+        elif library == "seaborn":
+            instructions = {
+                "role": "assistant",
+                "content": f"{matplotlib_instructions}. Use BaseMap for charts that require a map. Scope of BaseMap Must be restricted within BUSAN(city of south korea) area."}
+
+            template = \
+                f"""
+import seaborn as sns
+import pandas as pd
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+<imports>
+# solution plan
+# i.  ..
+def plot(data: pd.DataFrame):
+    path = '/usr/share/fonts/truetype/nanum/NanumGothic.ttf'
+    fontprop = fm.FontProperties(fname=path, size=12)
+    plt.rc('font', family=fontprop.get_name())
+
+    <stub> # only modify this section
+    plt.title('{goal.question}', wrap=True)
+    return plt;
+
+chart = plot(data) # data already contains the data to be plotted. Always include this line. No additional code beyond this line."""
+
+        elif library == "ggplot":
+            instructions = {
+                "role": "assistant",
+                "content": f"{general_instructions}. The plot method must return a ggplot object (chart)`. Think step by step.p. \n",
+            }
+
+            template = \
+                f"""
+import plotnine as p9
+<imports>
+def plot(data: pd.DataFrame):
+    chart = <stub>
+
+    return chart;
+
+chart = plot(data) # data already contains the data to be plotted. Always include this line. No additional code beyond this line.. """
+
+        elif library == "altair":
+            instructions = {
+                "role": "system",
+                "content": f"{general_instructions}. Always add a type that is BASED on semantic_type to each field such as :Q, :O, :N, :T, :G. Use :T if semantic_type is year or date. The plot method must return an altair object (chart)`. Think step by step. \n",
+            }
+            template = \
+                """
+import altair as alt
+<imports>
+def plot(data: pd.DataFrame):
+    <stub> # only modify this section
+    return chart
+chart = plot(data) # data already contains the data to be plotted.  Always include this line. No additional code beyond this line..
+"""
+
+        elif library == "plotly":
+            instructions = {
+                "role": "system",
+                "content": f"{general_instructions} If calculating metrics such as mean, median, mode, etc. ALWAYS use the option 'numeric_only=True' when applicable and available, AVOID visualizations that require nbformat library. DO NOT inlcude fig.show(). The plot method must return an plotly figure object (fig)`. Think step by step. \n.",
+            }
+            template = \
+                """
+import plotly.express as px
+<imports>
+def plot(data: pd.DataFrame):
+    fig = <stub> # only modify this section
+
+    return chart
+chart = plot(data) # variable data already contains the data to be plotted and should not be loaded again.  Always include this line. No additional code beyond this line..
+"""
+
+        else:
+            raise ValueError(
+                "Unsupported library. Choose from 'matplotlib', 'seaborn', 'plotly', 'bokeh', 'ggplot', 'altair'."
+            )
+
+        return template, instructions

lida_ko/components/summarizer.py

@@ -0,0 +1,171 @@
+import math
+import json
+import logging
+from typing import Union
+import pandas as pd
+from lida_ko.utils import clean_code_snippet, read_dataframe
+from lida_ko.datamodel import TextGenerationConfig
+from llmx import TextGenerator
+import warnings
+
+system_prompt = """
+You are an experienced data analyst that can annotate datasets. Your instructions are as follows:
+i) ALWAYS generate the name of the dataset and the dataset_description
+ii) ALWAYS generate a field description.
+iii.) ALWAYS generate a semantic_type (a single word) for each field given its values e.g. company, city, number, supplier, location, gender, longitude, latitude, url, ip address, zip code, email, etc
+You must return an updated JSON dictionary without any preamble or explanation.
+"""
+
+logger = logging.getLogger("lida")
+
+
+class Summarizer():
+    def __init__(self) -> None:
+        self.summary = None
+
+    def check_type(self, dtype: str, value):
+        """Cast value to right type to ensure it is JSON serializable"""
+        if isinstance(value, float) and math.isnan(value):
+            return None
+        if "float" in str(dtype):
+            return float(value)
+        elif "int" in str(dtype):
+            return int(value)
+        else:
+            return value
+
+    def get_column_properties(self, df: pd.DataFrame, n_samples: int = 3) -> list[dict]:
+        """Get properties of each column in a pandas DataFrame"""
+        properties_list = []
+        for column in df.columns:
+            dtype = df[column].dtype
+            properties = {}
+            if dtype in [int, float, complex]:
+                properties["dtype"] = "number"
+                properties["std"] = self.check_type(dtype, df[column].std())
+                properties["min"] = self.check_type(dtype, df[column].min())
+                properties["max"] = self.check_type(dtype, df[column].max())
+
+            elif dtype == bool:
+                properties["dtype"] = "boolean"
+            elif dtype == object:
+                # Check if the string column can be cast to a valid datetime
+                try:
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("ignore")
+                        pd.to_datetime(df[column], errors='raise')
+                        properties["dtype"] = "date"
+                except ValueError:
+                    # Check if the string column has a limited number of values
+                    if df[column].nunique() / len(df[column]) < 0.5:
+                        properties["dtype"] = "category"
+                    else:
+                        properties["dtype"] = "string"
+            elif pd.api.types.is_categorical_dtype(df[column]):
+                properties["dtype"] = "category"
+            elif pd.api.types.is_datetime64_any_dtype(df[column]):
+                properties["dtype"] = "date"
+            else:
+                properties["dtype"] = str(dtype)
+
+            # add min max if dtype is date
+            if properties["dtype"] == "date":
+                try:
+                    properties["min"] = df[column].min()
+                    properties["max"] = df[column].max()
+                except TypeError:
+                    cast_date_col = pd.to_datetime(df[column], errors='coerce')
+                    properties["min"] = cast_date_col.min()
+                    properties["max"] = cast_date_col.max()
+            # Add additional properties to the output dictionary
+            nunique = df[column].nunique()
+            if "samples" not in properties:
+                non_null_values = df[column][df[column].notnull()].unique()
+                n_samples = min(n_samples, len(non_null_values))
+                samples = pd.Series(non_null_values).sample(
+                    n_samples, random_state=42).tolist()
+                properties["samples"] = samples
+            properties["num_unique_values"] = nunique
+            properties["semantic_type"] = ""
+            properties["description"] = ""
+            properties_list.append(
+                {"column": column, "properties": properties})
+
+        return properties_list
+
+    def enrich(self, base_summary: dict, text_gen: TextGenerator,
+               textgen_config: TextGenerationConfig) -> dict:
+        """Enrich the data summary with descriptions"""
+        logger.info(f"Enriching the data summary with descriptions")
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "assistant", "content": f"""
+        Annotate the dictionary below. Only return a JSON object.
+        {base_summary}
+        """},
+        ]
+
+        response = text_gen.generate(messages=messages, config=textgen_config)
+        enriched_summary = base_summary
+        try:
+            json_string = clean_code_snippet(response.text[0]["content"])
+            enriched_summary = json.loads(json_string)
+        except json.decoder.JSONDecodeError:
+            error_msg = f"The model did not return a valid JSON object while attempting to generate an enriched data summary. Consider using a default summary or  a larger model with higher max token length. | {response.text[0]['content']}"
+            logger.info(error_msg)
+            print(response.text[0]["content"])
+            response_usage = response.usage if response.usage else "Usage information is not available"
+            raise ValueError(error_msg + "" + response_usage.total_tokens)
+        return enriched_summary
+
+    def summarize(
+            self, data: Union[pd.DataFrame, str],
+            text_gen: TextGenerator, file_name="", n_samples: int = 3,
+            textgen_config=TextGenerationConfig(n=1),
+            summary_method: str = "default", encoding: str = 'utf-8',
+            metadata: dict = None) -> dict:
+        """Summarize data from a pandas DataFrame or a file location"""
+
+        # if data is a file path, read it into a pandas DataFrame, set file_name to the file name
+        if isinstance(data, str):
+            file_name = data.split("/")[-1]
+            # modified to include encoding
+            data = read_dataframe(data, encoding=encoding)
+        data_properties = self.get_column_properties(data, n_samples)
+
+        # default single stage summary construction
+        base_summary = {
+            "name": file_name,
+            "file_name": file_name,
+            "dataset_description": "",
+            "fields": data_properties,
+        }
+        if metadata is not None:
+            for k, v in metadata.items():
+                if k == 'file_data':
+                    continue
+                if isinstance(v, list):
+                    v = ', '.join(v)
+                base_summary[k] = v
+
+        data_summary = base_summary
+
+        if summary_method == "llm":
+            # two stage summarization with llm enrichment
+            data_summary = self.enrich(
+                base_summary,
+                text_gen=text_gen,
+                textgen_config=textgen_config)
+        elif summary_method == "columns":
+            # no enrichment, only column names
+            data_summary = {
+                "name": file_name,
+                "file_name": file_name,
+                "dataset_description": ""
+            }
+
+        data_summary["field_names"] = data.columns.tolist()
+        data_summary["file_name"] = file_name
+
+        return data_summary

lida_ko/components/viz/__init__.py

@@ -0,0 +1,6 @@
+from .vizeditor import *
+from .vizexplainer import *
+from .vizgenerator import *
+from .vizevaluator import *
+from .vizrepairer import *
+from .vizrecommender import *

lida_ko/components/viz/vizeditor.py

@@ -0,0 +1,44 @@
+from llmx import TextGenerator, TextGenerationConfig, TextGenerationResponse
+from ..scaffold import ChartScaffold
+from lida_ko.datamodel import Goal, Summary
+
+
+system_prompt = """
+You are a high skilled visualization assistant that can modify a provided visualization code based on a set of instructions. You MUST return a full program. DO NOT include any preamble text. Do not include explanations or prose.
+"""
+
+
+class VizEditor(object):
+    """Generate visualizations from prompt"""
+
+    def __init__(
+        self,
+    ) -> None:
+        self.scaffold = ChartScaffold()
+
+    def generate(
+            self, code: str, summary: Summary, instructions: list[str],
+            textgen_config: TextGenerationConfig, text_gen: TextGenerator, library='altair'):
+        """Edit a code spec based on instructions"""
+
+        instruction_string = ""
+        for i, instruction in enumerate(instructions):
+            instruction_string += f"{i+1}. {instruction} \n"
+
+        library_template, library_instructions = self.scaffold.get_template(Goal(
+            index=0,
+            question="",
+            visualization="",
+            rationale=""), library)
+        # print("instructions", instructions)
+
+        messages = [
+            {
+                "role": "system", "content": system_prompt}, {
+                "role": "system", "content": f"The dataset summary is : \n\n {summary} \n\n"}, {
+                "role": "system", "content": f"The modifications you make MUST BE CORRECT and  based on the '{library}' library and also follow these instructions instructions \n\n{library_instructions} \n\n. The resulting code MUST use the following template \n\n {library_template} \n\n "}, {
+                    "role": "user", "content": f"ALL ADDITIONAL LIBRARIES USED MUST BE IMPORTED.\n The code to be modified is: \n\n{code} \n\n. YOU MUST THINK STEP BY STEP, AND CAREFULLY MODIFY ONLY the content of the plot(..) method TO MEET EACH OF THE FOLLOWING INSTRUCTIONS: \n\n {instruction_string} \n\n. The completed modified code THAT FOLLOWS THE TEMPLATE above is. \n"}]
+
+        completions: TextGenerationResponse = text_gen.generate(
+            messages=messages, config=textgen_config)
+        return [x['content'] for x in completions.text]

lida_ko/components/viz/vizevaluator.py

@@ -0,0 +1,58 @@
+
+import json
+from ...utils import clean_code_snippet
+from llmx import TextGenerator, TextGenerationConfig, TextGenerationResponse
+
+from lida_ko.datamodel import Goal
+
+system_prompt = """
+You are a helpful assistant highly skilled in evaluating the quality of a given visualization code by providing a score from 1 (bad) - 10 (good) while providing clear rationale. YOU MUST CONSIDER VISUALIZATION BEST PRACTICES for each evaluation. Specifically, you can carefully evaluate the code across the following dimensions
+- bugs (bugs):  are there bugs, logic errors, syntax error or typos? Are there any reasons why the code may fail to compile? How should it be fixed? If ANY bug exists, the bug score MUST be less than 5.
+- Data transformation (transformation): Is the data transformed appropriately for the visualization type? E.g., is the dataset appropriated filtered, aggregated, or grouped  if needed?
+- Goal compliance (compliance): how well the code meets the specified visualization goals?
+- Visualization type (type): CONSIDERING BEST PRACTICES, is the visualization type appropriate for the data and intent? Is there a visualization type that would be more effective in conveying insights? If a different visualization type is more appropriate, the score MUST be less than 5.
+- Data encoding (encoding): Is the data encoded appropriately for the visualization type?
+- aesthetics (aesthetics): Are the aesthetics of the visualization appropriate for the visualization type and the data?
+
+You must provide a score for each of the above dimensions.  Assume that data in chart = plot(data) contains a valid dataframe for the dataset. The `plot` function returns a chart (e.g., matplotlib, seaborn etc object).
+
+Your OUTPUT MUST BE A VALID JSON LIST OF OBJECTS in the format:
+
+```[
+{ "dimension":  "bugs",  "score": x , "rationale": " .."}, { "dimension":  "transformation",  "score": x, "rationale": " .."}, { "dimension":  "compliance",  "score": x, "rationale": " .."},{ "dimension":  "type",  "score": x, "rationale": " .."}, { "dimension":  "encoding",  "score": x, "rationale": " .."}, { "dimension":  "aesthetics",  "score": x, "rationale": " .."}
+]
+```
+"""
+
+
+class VizEvaluator(object):
+    """Generate visualizations Explanations given some code"""
+
+    def __init__(
+        self,
+    ) -> None:
+        pass
+
+    def generate(self, code: str, goal: Goal,
+                 textgen_config: TextGenerationConfig, text_gen: TextGenerator, library='altair'):
+        """Generate a visualization explanation given some code"""
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "assistant",
+             "content": f"Generate an evaluation given the goal and code below in {library}. The specified goal is \n\n {goal.question} \n\n and the visualization code is \n\n {code} \n\n. Now, evaluate the code based on the 6 dimensions above. \n. THE SCORE YOU ASSIGN MUST BE MEANINGFUL AND BACKED BY CLEAR RATIONALE. A SCORE OF 1 IS POOR AND A SCORE OF 10 IS VERY GOOD. The structured evaluation is below ."},
+        ]
+
+        # print(messages)
+        completions: TextGenerationResponse = text_gen.generate(
+            messages=messages, config=textgen_config)
+
+        completions = [clean_code_snippet(x['content']) for x in completions.text]
+        evaluations = []
+        for completion in completions:
+            try:
+                evaluation = json.loads(completion)
+                evaluations.append(evaluation)
+            except Exception as json_error:
+                print("Error parsing evaluation data", completion, str(json_error))
+        return evaluations

lida_ko/components/viz/vizexplainer.py

@@ -0,0 +1,60 @@
+
+import json
+from lida_ko.utils import clean_code_snippet
+from llmx import TextGenerator, TextGenerationConfig, TextGenerationResponse
+from ..scaffold import ChartScaffold
+
+
+system_prompt = """
+You are a helpful assistant highly skilled in providing helpful, structured explanations of visualization of the plot(data: pd.DataFrame) method in the provided code. You divide the code into sections and provide a description of each section and an explanation. The first section should be named "accessibility" and describe the physical appearance of the chart (colors, chart type etc), the goal of the chart, as well the main insights from the chart.
+You can explain code across the following 3 dimensions:
+1. accessibility: the physical appearance of the chart (colors, chart type etc), the goal of the chart, as well the main insights from the chart.
+2. transformation: This should describe the section of the code that applies any kind of data transformation (filtering, aggregation, grouping, null value handling etc)
+3. visualization: step by step description of the code that creates or modifies the presented visualization.
+
+"""
+
+format_instructions = """
+Your output MUST be perfect JSON in THE FORM OF A VALID LIST of JSON OBJECTS WITH PROPERLY ESCAPED SPECIAL CHARACTERS e.g.,
+
+```[
+    {"section": "accessibility", "code": "None", "explanation": ".."}  , {"section": "transformation", "code": "..", "explanation": ".."}  ,  {"section": "visualization", "code": "..", "explanation": ".."}
+    ] ```
+
+The code part of the dictionary must come from the supplied code and should cover the explanation. The explanation part of the dictionary must be a string. The section part of the dictionary must be one of "accessibility", "transformation", "visualization" with no repetition. THE LIST MUST HAVE EXACTLY 3 JSON OBJECTS [{}, {}, {}].  THE GENERATED JSON  MUST BE A LIST IE START AND END WITH A SQUARE BRACKET.
+"""
+
+
+class VizExplainer(object):
+    """Generate visualizations Explanations given some code"""
+
+    def __init__(
+        self,
+    ) -> None:
+        self.scaffold = ChartScaffold()
+
+    def generate(
+            self, code: str,
+            textgen_config: TextGenerationConfig, text_gen: TextGenerator, library='seaborn'):
+        """Generate a visualization explanation given some code"""
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "assistant", "content": f"The code to be explained is {code}.\n=======\n"},
+            {"role": "user",
+             "content": f"{format_instructions}. \n\n. The structured explanation for the code above is \n\n"}
+        ]
+
+        completions: TextGenerationResponse = text_gen.generate(
+            messages=messages, config=textgen_config)
+
+        completions = [clean_code_snippet(x['content']) for x in completions.text]
+        explanations = []
+
+        for completion in completions:
+            try:
+                exp = json.loads(completion)
+                explanations.append(exp)
+            except Exception as e:
+                print("Error parsing completion", completion, str(e))
+        return explanations

lida_ko/components/viz/vizgenerator.py

@@ -0,0 +1,40 @@
+from dataclasses import asdict
+from typing import Dict
+from llmx import TextGenerator, TextGenerationConfig, TextGenerationResponse
+
+from ..scaffold import ChartScaffold
+from lida_ko.datamodel import Goal
+
+
+system_prompt = """
+You are a helpful assistant highly skilled in writing PERFECT code for visualizations. Given some code template, you complete the template to generate a visualization given the dataset and the goal described. The code you write MUST FOLLOW VISUALIZATION BEST PRACTICES ie. meet the specified goal, apply the right transformation, use the right visualization type, use the right data encoding, and use the right aesthetics (e.g., ensure axis are legible). The transformations you apply MUST be correct and the fields you use MUST be correct. The visualization CODE MUST BE CORRECT and MUST NOT CONTAIN ANY SYNTAX OR LOGIC ERRORS (e.g., it must consider the field types and use them correctly). You MUST first generate a brief plan for how you would solve the task e.g. what transformations you would apply e.g. if you need to construct a new column, what fields you would use, what visualization type you would use, what aesthetics you would use, etc. .
+"""
+
+
+class VizGenerator(object):
+    """Generate visualizations from prompt"""
+
+    def __init__(
+        self
+    ) -> None:
+
+        self.scaffold = ChartScaffold()
+
+    def generate(self, summary: Dict, goal: Goal,
+                 textgen_config: TextGenerationConfig, text_gen: TextGenerator, library='altair'):
+        """Generate visualization code given a summary and a goal"""
+
+        library_template, library_instructions = self.scaffold.get_template(goal, library)
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "system", "content": f"The dataset summary is : {summary} \n\n"},
+            library_instructions,
+            {"role": "user",
+             "content":
+             f"Always add a legend with various colors where appropriate. The visualization code MUST only use data fields that exist in the dataset (field_names) or fields that are transformations based on existing field_names). Only use variables that have been defined in the code or are in the dataset summary. You MUST return a FULL PYTHON PROGRAM ENCLOSED IN BACKTICKS ``` that starts with an import statement. DO NOT add any explanation. \n\n THE GENERATED CODE SOLUTION SHOULD BE CREATED BY MODIFYING THE SPECIFIED PARTS OF THE TEMPLATE BELOW \n\n {library_template} \n\n.The FINAL COMPLETED CODE BASED ON THE TEMPLATE above is ... \n\n"}]
+
+        completions: TextGenerationResponse = text_gen.generate(
+            messages=messages, config=textgen_config)
+        response = [x['content'] for x in completions.text]
+
+        return response

lida_ko/components/viz/vizrecommender.py

@@ -0,0 +1,86 @@
+import logging
+import json
+from lida_ko.utils import clean_code_snippet
+from ..scaffold import ChartScaffold
+from llmx import TextGenerator, TextGenerationConfig, TextGenerationResponse
+# from lida.modules.scaffold import ChartScaffold
+from lida_ko.datamodel import Goal, Summary
+
+
+system_prompt = """
+
+You are a helpful assistant highly skilled in recommending a DIVERSE set of visualization code. Your input is an example visualization code,  a summary of a dataset and an example visualization goal that the user has already seen. Given this input, your task is to recommend additional visualizations that a user may be interested. Your recommendation may consider different types of valid data aggregations, chart types, clearer ways of displaying information and uses different variables from the data summary. THE CODE YOU GENERATE MUST BE CORRECT (follow the language syntax and syntax of the visualization grammar) AND FOLLOW VISUALIZATION BEST PRACTICES.
+
+Your output MUST be a n code snippets separated by ******* (5 asterisks). Each snippet MUST BE AN independent code snippet (with one plot method) similar to the example code. For example
+
+```python
+# code snippet 1
+import  ...
+....
+```
+*****
+
+```python
+# code snippet 2
+import ...
+....
+```
+
+```python
+# code snippet n
+import ...
+....
+```
+
+
+"""
+
+logger = logging.getLogger("lida")
+
+
+class VizRecommender(object):
+    """Generate visualizations from prompt"""
+
+    def __init__(
+        self,
+    ) -> None:
+        self.scaffold = ChartScaffold()
+
+    def generate(
+            self, code: str, summary: Summary,
+            textgen_config: TextGenerationConfig,
+            text_gen: TextGenerator,
+            n=3,
+            library='seaborn'):
+        """Recommend a code spec based on existing visualization"""
+
+        library_template, library_instructions = self.scaffold.get_template(Goal(
+            index=0,
+            question="",
+            visualization="",
+            rationale=""), library)
+
+        structure_instruction = f"""
+        EACH CODE SNIPPET MUST BE A FULL PROGRAM (IT MUST IMPORT ALL THE LIBRARIES THAT ARE USED AND MUST CONTAIN plot(data) method). IT MUST FOLLOW THE STRUCTURE BELOW AND ONLY MODIFY THE INDICATED SECTIONS. \n\n {library_template} \n\n.
+        """
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "system", "content": structure_instruction},
+            {"role": "system", "content": f"The dataset summary is : \n\n {summary} \n\n"},
+            {"role": "system",
+             "content":
+             f"An example visualization code is: \n\n ```{code}``` \n\n. You MUST use only the {library} library. \n"},
+            {"role": "user", "content": f"Recommend {n} (n=({n})) visualizations in the format specified. \n."}]
+
+        textgen_config.messages = messages
+        result: TextGenerationResponse = text_gen.generate(
+            messages=messages, config=textgen_config)
+        output = []
+        snippets = result.text[0]["content"].split("*****")
+        for snippet in snippets:
+            cleaned_snippet = clean_code_snippet(snippet)
+            if len(cleaned_snippet) > 4:
+                output.append(cleaned_snippet)
+
+        return output

lida_ko/components/viz/vizrepairer.py

@@ -0,0 +1,43 @@
+from typing import Dict, List, Union
+from llmx import TextGenerator, TextGenerationConfig, TextGenerationResponse
+
+from ..scaffold import ChartScaffold
+from lida_ko.datamodel import Goal, Summary
+
+system_prompt = """
+You are a helpful assistant highly skilled in revising visualization code to improve the quality of the code and visualization based on feedback.  Assume that data in plot(data) contains a valid dataframe.
+You MUST return a full program. DO NOT include any preamble text. Do not include explanations or prose.
+"""
+
+
+class VizRepairer(object):
+    """Fix visualization code based on feedback"""
+
+    def __init__(
+        self,
+    ) -> None:
+        self.scaffold = ChartScaffold()
+
+    def generate(
+            self, code: str, feedback: Union[str, Dict, List[Dict]],
+            goal: Goal, summary: Summary, textgen_config: TextGenerationConfig,
+            text_gen: TextGenerator, library='altair',):
+        """Fix a code spec based on feedback"""
+        library_template, library_instructions = self.scaffold.get_template(Goal(
+            index=0,
+            question="",
+            visualization="",
+            rationale=""), library)
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "system", "content": f"The dataset summary is : {summary}. \n . The original goal was: {goal}."},
+            {"role": "system",
+             "content":
+             f"You MUST use only the {library}. The resulting code MUST use the following template {library_template}. Only use variables that have been defined in the code or are in the dataset summary"},
+            {"role": "user", "content": f"The existing code to be fixed is: {code}. \n Fix the code above to address the feedback: {feedback}. ONLY apply feedback that are CORRECT."}]
+
+        # library with the following instructions {library_instructions}
+
+        completions: TextGenerationResponse = text_gen.generate(
+            messages=messages, config=textgen_config)
+        return [x['content'] for x in completions.text]

lida_ko/datamodel.py

@@ -0,0 +1,280 @@
+# from dataclasses import dataclass
+import base64
+from dataclasses import field
+from typing import Any, Dict, List, Optional, Union
+
+from llmx import TextGenerationConfig
+from pydantic.dataclasses import dataclass
+
+
+@dataclass
+class VizGeneratorConfig:
+    """Configuration for a visualization generation"""
+
+    hypothesis: str
+    data_summary: Optional[str] = ""
+    data_filename: Optional[str] = "cars.csv"
+
+
+@dataclass
+class CompletionResult:
+    text: str
+    logprobs: Optional[List[float]]
+    prompt: str
+    suffix: str
+
+
+@dataclass
+class UploadUrl:
+    """Response from a text generation"""
+
+    url: str
+
+
+@dataclass
+class Goal:
+    """A visualization goal"""
+    question: str
+    visualization: str
+    rationale: str
+    index: Optional[int] = 0
+
+    def _repr_markdown_(self):
+        return f"""
+### Goal {self.index}
+---
+**Question:** {self.question}
+
+**Visualization:** `{self.visualization}`
+
+**Rationale:** {self.rationale}
+"""
+
+
+@dataclass
+class Summary:
+    """A summary of a dataset
+    Tweaked for lida_ko"""
+
+    name: str
+    file_name: str
+    dataset_description: str
+    field_names: List[Any]
+    fields: Optional[List[Any]] = None
+    title: Optional[str] = None
+    keywords: Optional[str] = None
+    organization: Optional[str] = None
+    department: Optional[str] = None
+    phone: Optional[str] = None
+    update_interval: Optional[str] = None
+    updated_at: Optional[str] = None
+    next_update_at: Optional[str] = None
+    cost: Optional[str] = None
+    serving_type: Optional[str] = None
+    download_count: Optional[str] = None
+    permission_scope: Optional[str] = None
+    timestamp: Optional[str] = None
+    augmentation: Optional[dict] = None
+
+    def _repr_markdown_(self):
+        field_lines = "\n".join([f"- **{name}:** {field}" for name,
+                                field in zip(self.field_names, self.fields)])
+        metadata_lines = ""
+        if self.title is not None:
+            metadata_lines += f"- **Title:** {self.title}\n"
+        if self.keywords is not None:
+            metadata_lines += f"- **Keywords:** {self.keywords}\n"
+        if self.organization is not None:
+            metadata_lines += f"- **Organization:** {self.organization}\n"
+        if self.department is not None:
+            metadata_lines += f"- **Department:** {self.department}\n"
+        if self.phone is not None:
+            metadata_lines += f"- **Phone:** {self.phone}\n"
+        if self.update_interval is not None:
+            metadata_lines += f"- **Update Interval:** {self.update_interval}\n"
+        if self.updated_at is not None:
+            metadata_lines += f"- **Updated At:** {self.updated_at}\n"
+        if self.next_update_at is not None:
+            metadata_lines += f"- **Next Update At:** {self.next_update_at}\n"
+        if self.cost is not None:
+            metadata_lines += f"- **Cost:** {self.cost}\n"
+        if self.serving_type is not None:
+            metadata_lines += f"- **Serving Type:** {self.serving_type}\n"
+        if self.download_count is not None:
+            metadata_lines += f"- **Download Count:** {self.download_count}\n"
+        if self.permission_scope is not None:
+            metadata_lines += f"- **Permission Scope:** {self.permission_scope}\n"
+        if self.timestamp is not None:
+            metadata_lines += f"- **Timestamp:** {self.timestamp}\n"
+        if self.augmentation is not None:
+            metadata_lines += f"- **Augmentation:** {self.augmentation}\n"
+        return f"""
+## Dataset Summary
+
+---
+
+**Name:** {self.name}
+
+**File Name:** {self.file_name}
+
+**Dataset Description:**
+
+{self.dataset_description}
+
+{metadata_lines}
+
+**Fields:**
+
+{field_lines}
+"""
+
+
+@dataclass
+class Persona:
+    """A persona"""
+    persona: str
+    rationale: str
+
+    def _repr_markdown_(self):
+        return f"""
+### Persona
+---
+
+**Persona:** {self.persona}
+
+**Rationale:** {self.rationale}
+"""
+
+
+@dataclass
+class GoalWebRequest:
+    """A Goal Web Request"""
+
+    summary: Summary
+    textgen_config: Optional[TextGenerationConfig] = field(
+        default_factory=TextGenerationConfig
+    )
+    n: int = 5
+
+
+@dataclass
+class VisualizeWebRequest:
+    """A Visualize Web Request"""
+
+    summary: Summary
+    goal: Goal
+    library: str = "seaborn"
+    textgen_config: Optional[TextGenerationConfig] = field(
+        default_factory=TextGenerationConfig
+    )
+
+
+@dataclass
+class VisualizeRecommendRequest:
+    """A Visualize Recommendation Request"""
+
+    summary: Summary
+    code: str
+    library: str = "seaborn"
+    textgen_config: Optional[TextGenerationConfig] = field(
+        default_factory=TextGenerationConfig
+    )
+
+
+@dataclass
+class VisualizeEditWebRequest:
+    """A Visualize Edit Web Request"""
+
+    summary: Summary
+    code: str
+    instructions: Union[str, List[str]]
+    library: str = "seaborn"
+    textgen_config: Optional[TextGenerationConfig] = field(
+        default_factory=TextGenerationConfig
+    )
+
+
+@dataclass
+class VisualizeRepairWebRequest:
+    """A Visualize Repair Web Request"""
+
+    feedback: Optional[Union[str, List[str], List[Dict]]]
+    code: str
+    goal: Goal
+    summary: Summary
+    library: str = "seaborn"
+    textgen_config: Optional[TextGenerationConfig] = field(
+        default_factory=TextGenerationConfig
+    )
+
+
+@dataclass
+class VisualizeExplainWebRequest:
+    """A Visualize Explain Web Request"""
+
+    code: str
+    library: str = "seaborn"
+    textgen_config: Optional[TextGenerationConfig] = field(
+        default_factory=TextGenerationConfig
+    )
+
+
+@dataclass
+class VisualizeEvalWebRequest:
+    """A Visualize Eval Web Request"""
+
+    code: str
+    goal: Goal
+    library: str = "seaborn"
+    textgen_config: Optional[TextGenerationConfig] = field(
+        default_factory=TextGenerationConfig
+    )
+
+
+@dataclass
+class ChartExecutorResponse:
+    """Response from a visualization execution"""
+
+    spec: Optional[Union[str, Dict]]  # interactive specification e.g. vegalite
+    status: bool  # True if successful
+    raster: Optional[str]  # base64 encoded image
+    code: str  # code used to generate the visualization
+    library: str  # library used to generate the visualization
+    error: Optional[Dict] = None  # error message if status is False
+
+    def _repr_mimebundle_(self, include=None, exclude=None):
+        bundle = {"text/plain": self.code}
+        if self.raster is not None:
+            bundle["image/png"] = self.raster
+        if self.spec is not None:
+            bundle["application/vnd.vegalite.v5+json"] = self.spec
+
+        return bundle
+
+    def savefig(self, path):
+        """Save the raster image to a specified path if it exists"""
+        if self.raster:
+            with open(path, 'wb') as f:
+                f.write(base64.b64decode(self.raster))
+        else:
+            raise FileNotFoundError("No raster image to save")
+
+
+@dataclass
+class SummaryUrlRequest:
+    """A request for generating a summary with file url"""
+
+    url: str
+    textgen_config: Optional[TextGenerationConfig] = field(
+        default_factory=TextGenerationConfig
+    )
+
+
+@dataclass
+class InfographicsRequest:
+    """A request for infographics generation"""
+
+    visualization: str
+    n: int = 1
+    style_prompt: Union[str, List[str]] = ""
+    # return_pil: bool = False

lida_ko/utils.py

@@ -0,0 +1,227 @@
+import base64
+import json
+import logging
+from typing import Any, List, Tuple, Union
+import os
+import io
+import numpy as np
+import pandas as pd
+import re
+import matplotlib.pyplot as plt
+import tiktoken
+from diskcache import Cache
+import hashlib
+import io
+
+logger = logging.getLogger("lida")
+
+
+def get_dirs(path: str) -> List[str]:
+    return next(os.walk(path))[1]
+
+
+def clean_column_name(col_name: str) -> str:
+    """
+    Clean a single column name by replacing special characters and spaces with underscores.
+
+    :param col_name: The name of the column to be cleaned.
+    :return: A sanitized string valid as a column name.
+    """
+    return re.sub(r'[^0-9a-zA-Z_]', '_', col_name)
+
+
+def clean_column_names(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Clean all column names in the given DataFrame.
+
+    :param df: The DataFrame with possibly dirty column names.
+    :return: A copy of the DataFrame with clean column names.
+    """
+    cleaned_df = df.copy()
+    cleaned_df.columns = [clean_column_name(col) for col in cleaned_df.columns]
+    return cleaned_df
+
+
+def read_dataframe(file_location: str, encoding: str = 'utf-8') -> pd.DataFrame:
+    """
+    Read a dataframe from a given file location and clean its column names.
+    It also samples down to 4500 rows if the data exceeds that limit.
+
+    :param file_location: The path to the file containing the data.
+    :param encoding: Encoding to use for the file reading.
+    :return: A cleaned DataFrame.
+    """
+    file_extension = file_location.split('.')[-1]
+
+    read_funcs = {
+        'json': lambda: pd.read_json(file_location, orient='records', encoding=encoding),
+        'csv': lambda: pd.read_csv(file_location, encoding=encoding),
+        'xls': lambda: pd.read_excel(file_location, encoding=encoding),
+        'xlsx': lambda: pd.read_excel(file_location, encoding=encoding),
+        'parquet': pd.read_parquet,
+        'feather': pd.read_feather,
+        'tsv': lambda: pd.read_csv(file_location, sep="\t", encoding=encoding)
+    }
+
+    if file_extension not in read_funcs:
+        raise ValueError('Unsupported file type')
+
+    try:
+        df = read_funcs[file_extension]()
+    except Exception as e:
+        logger.error(f"Failed to read file: {file_location}. Error: {e}")
+        raise
+
+    # Clean column names
+    cleaned_df = clean_column_names(df)
+
+    # Sample down to 4500 rows if necessary
+    if len(cleaned_df) > 4500:
+        logger.info(
+            "Dataframe has more than 4500 rows. We will sample 4500 rows.")
+        cleaned_df = cleaned_df.sample(4500)
+
+    if cleaned_df.columns.tolist() != df.columns.tolist():
+        write_funcs = {
+            'csv': lambda: cleaned_df.to_csv(file_location, index=False, encoding=encoding),
+            'xls': lambda: cleaned_df.to_excel(file_location, index=False),
+            'xlsx': lambda: cleaned_df.to_excel(file_location, index=False),
+            'parquet': lambda: cleaned_df.to_parquet(file_location, index=False),
+            'feather': lambda: cleaned_df.to_feather(file_location, index=False),
+            'json': lambda: cleaned_df.to_json(file_location, orient='records', index=False, default_handler=str),
+            'tsv': lambda: cleaned_df.to_csv(file_location, index=False, sep='\t', encoding=encoding)
+        }
+
+        if file_extension not in write_funcs:
+            raise ValueError('Unsupported file type')
+
+        try:
+            write_funcs[file_extension]()
+        except Exception as e:
+            logger.error(f"Failed to write file: {file_location}. Error: {e}")
+            raise
+
+    return cleaned_df
+
+
+def file_to_df(file_location: str):
+    """ Get summary of data from file location """
+    file_name = file_location.split("/")[-1]
+    df = None
+    if "csv" in file_name:
+        df = pd.read_csv(file_location)
+    elif "xlsx" in file_name:
+        df = pd.read_excel(file_location)
+    elif "json" in file_name:
+        df = pd.read_json(file_location, orient="records")
+    elif "parquet" in file_name:
+        df = pd.read_parquet(file_location)
+    elif "feather" in file_name:
+        df = pd.read_feather(file_location)
+
+    return df
+
+
+def plot_raster(rasters: Union[str, List[str]], figsize: Tuple[int, int] = (10, 10)):
+    """
+    Plot a series of base64-encoded raster images in a horizontal layout.
+
+    Args:
+        rasters: A single base64 string or a list of base64-encoded strings representing the images.
+        figsize: A tuple indicating the size of the figure to display.
+    """
+    plt.figure(figsize=figsize)
+
+    if isinstance(rasters, str):
+        rasters = [rasters]
+
+    images = []
+
+    # Find the max height for resizing
+    max_height = 0
+    for raster in rasters:
+        decoded_image = base64.b64decode(raster)
+        image = plt.imread(io.BytesIO(decoded_image), format='PNG')
+
+        max_height = max(max_height, image.shape[0])
+
+    # Resize images to max_height while preserving the aspect ratio and alpha channel if it exists
+    for raster in rasters:
+        decoded_image = base64.b64decode(raster)
+        image = plt.imread(io.BytesIO(decoded_image), format='PNG')
+
+        aspect_ratio = image.shape[1] / image.shape[0]
+        new_width = int(max_height * aspect_ratio)
+        image_resized = np.array([np.interp(np.linspace(
+            0, len(row), new_width), np.arange(0, len(row)), row) for row in image])
+
+        if image_resized.shape[2] == 4:  # If RGBA, preserve alpha channel
+            alpha_channel = image_resized[:, :, 3:]
+            # Drop the alpha for visualization
+            image_resized = image_resized[:, :, :3]
+            image_resized = np.clip(image_resized, 0, 1)
+            image_resized = np.concatenate(
+                (image_resized, alpha_channel), axis=2)
+
+        images.append(image_resized)
+
+    # Concatenate images along the width
+    concatenated_image = np.concatenate(images, axis=1)
+
+    plt.imshow(concatenated_image)
+    plt.axis('off')
+    plt.show()
+
+
+def num_tokens_from_messages(messages, model="gpt-3.5-turbo-0301"):
+    """Returns the number of tokens used by a list of messages."""
+    try:
+        encoding = tiktoken.encoding_for_model(model)
+    except KeyError:
+        encoding = tiktoken.get_encoding("cl100k_base")
+    if model == "gpt-3.5-turbo-0301":  # note: future models may deviate from this
+        num_tokens = 0
+        for message in messages:
+            # every message follows <im_start>{role/name}\n{content}<im_end>\n
+            num_tokens += 4
+            for key, value in message.items():
+                num_tokens += len(encoding.encode(value))
+                if key == "name":  # if there's a name, the role is omitted
+                    num_tokens += -1  # role is always required and always 1 token
+        num_tokens += 2  # every reply is primed with <im_start>assistant
+        return num_tokens
+    else:
+        raise NotImplementedError(
+            f"""num_tokens_from_messages() is not presently implemented for model {model}.""")
+
+
+def cache_request(cache: Cache, params: Any, values: Any = None) -> Any:
+    # Generate a unique key for the request
+
+    key = hashlib.md5(json.dumps(
+        params, sort_keys=True).encode("utf-8")).hexdigest()
+    # Check if the request is cached
+    if key in cache and values is None:
+        print("retrieving from cache")
+        return cache[key]
+
+    # Cache the provided values and return them
+    if values:
+        print("saving to cache")
+        cache[key] = values
+    return values
+
+
+def clean_code_snippet(code_string):
+    # Extract code snippet using regex
+    cleaned_snippet = re.search(r'```(?:\w+)?\s*([\s\S]*?)\s*```', code_string)
+
+    if cleaned_snippet:
+        cleaned_snippet = cleaned_snippet.group(1)
+    else:
+        cleaned_snippet = code_string
+
+    # remove non-printable characters
+    # cleaned_snippet = re.sub(r'[\x00-\x1F]+', ' ', cleaned_snippet)
+
+    return cleaned_snippet

lida_ko/version.py

@@ -0,0 +1 @@
+VERSION = "0.0.14"

lida_ko/web/.gitignore

@@ -0,0 +1,142 @@
+output/
+samples/
+lida/web/backend/files
+examples/experiment_results.json
+examples/data
+data  
+.DS_Store
+test.py
+experiments/data
+test.py
+.azure
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+# build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+.env
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

lida_ko/web/__init__.py

@@ -0,0 +1,5 @@
+# __init__.py
+
+"""Handle web backend API for lida.
+ """
+ 

lida_ko/web/app.py

@@ -0,0 +1,310 @@
+import json
+import os
+import logging
+import requests
+from fastapi import FastAPI, UploadFile
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+import traceback
+
+from llmx import llm, providers
+from ..datamodel import GoalWebRequest, SummaryUrlRequest, TextGenerationConfig, UploadUrl, VisualizeEditWebRequest, VisualizeEvalWebRequest, VisualizeExplainWebRequest, VisualizeRecommendRequest, VisualizeRepairWebRequest, VisualizeWebRequest, InfographicsRequest
+from ..components import Manager
+
+
+# instantiate model and generator
+textgen = llm()
+logger = logging.getLogger("lida")
+api_docs = os.environ.get("LIDA_API_DOCS", "False") == "True"
+
+
+lida = Manager(text_gen=textgen)
+app = FastAPI()
+# allow cross origin requests for testing on localhost:800* ports only
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:8000", "http://127.0.0.1:8000", "http://localhost:8001"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+api = FastAPI(root_path="/api", docs_url="/docs" if api_docs else None, redoc_url=None)
+app.mount("/api", api)
+
+
+root_file_path = os.path.dirname(os.path.abspath(__file__))
+static_folder_root = os.path.join(root_file_path, "ui")
+files_static_root = os.path.join(root_file_path, "files/")
+data_folder = os.path.join(root_file_path, "files/data")
+os.makedirs(data_folder, exist_ok=True)
+os.makedirs(files_static_root, exist_ok=True)
+os.makedirs(static_folder_root, exist_ok=True)
+
+
+# mount lida front end UI files
+app.mount("/", StaticFiles(directory=static_folder_root, html=True), name="ui")
+api.mount("/files", StaticFiles(directory=files_static_root, html=True), name="files")
+
+
+# def check_model
+
+@api.post("/visualize")
+async def visualize_data(req: VisualizeWebRequest) -> dict:
+    """Generate goals given a dataset summary"""
+    try:
+        # print(req.textgen_config)
+        charts = lida.visualize(
+            summary=req.summary,
+            goal=req.goal,
+            textgen_config=req.textgen_config if req.textgen_config else TextGenerationConfig(),
+            library=req.library, return_error=True)
+        print("found charts: ", len(charts), " for goal: ")
+        if len(charts) == 0:
+            return {"status": False, "message": "No charts generated"}
+        return {"status": True, "charts": charts,
+                "message": "Successfully generated charts."}
+
+    except Exception as exception_error:
+        logger.error(f"Error generating visualization goals: {str(exception_error)}")
+        return {"status": False,
+                "message": f"Error generating visualization goals. {str(exception_error)}"}
+
+
+@api.post("/visualize/edit")
+async def edit_visualization(req: VisualizeEditWebRequest) -> dict:
+    """Given a visualization code, and a goal, generate a new visualization"""
+    try:
+        textgen_config = req.textgen_config if req.textgen_config else TextGenerationConfig()
+        charts = lida.edit(
+            code=req.code,
+            summary=req.summary,
+            instructions=req.instructions,
+            textgen_config=textgen_config,
+            library=req.library, return_error=True)
+
+        # charts = [asdict(chart) for chart in charts]
+        if len(charts) == 0:
+            return {"status": False, "message": "No charts generated"}
+        return {"status": True, "charts": charts,
+                "message": f"Successfully edited charts."}
+
+    except Exception as exception_error:
+        logger.error(f"Error generating visualization edits: {str(exception_error)}")
+        print(traceback.print_exc())
+        return {"status": False,
+                "message": f"Error generating visualization edits."}
+
+
+@api.post("/visualize/repair")
+async def repair_visualization(req: VisualizeRepairWebRequest) -> dict:
+    """ Given a visualization goal and some feedback, generate a new visualization that addresses the feedback"""
+
+    try:
+
+        charts = lida.repair(
+            code=req.code,
+            feedback=req.feedback,
+            goal=req.goal,
+            summary=req.summary,
+            textgen_config=req.textgen_config if req.textgen_config else TextGenerationConfig(),
+            library=req.library,
+            return_error=True
+        )
+
+        if len(charts) == 0:
+            return {"status": False, "message": "No charts generated"}
+        return {"status": True, "charts": charts,
+                "message": "Successfully generated chart repairs"}
+
+    except Exception as exception_error:
+        logger.error(f"Error generating visualization repairs: {str(exception_error)}")
+        return {"status": False,
+                "message": f"Error generating visualization repairs."}
+
+
+@api.post("/visualize/explain")
+async def explain_visualization(req: VisualizeExplainWebRequest) -> dict:
+    """Given a visualization code, provide an explanation of the code"""
+    textgen_config = req.textgen_config if req.textgen_config else TextGenerationConfig(
+        n=1,
+        temperature=0)
+
+    try:
+        explanations = lida.explain(
+            code=req.code,
+            textgen_config=textgen_config,
+            library=req.library)
+        return {"status": True, "explanations": explanations[0],
+                "message": "Successfully generated explanations"}
+
+    except Exception as exception_error:
+        logger.error(f"Error generating visualization explanation: {str(exception_error)}")
+        return {"status": False,
+                "message": f"Error generating visualization explanation."}
+
+
+@api.post("/visualize/evaluate")
+async def evaluate_visualization(req: VisualizeEvalWebRequest) -> dict:
+    """Given a visualization code, provide an evaluation of the code"""
+
+    try:
+        evaluations = lida.evaluate(
+            code=req.code,
+            goal=req.goal,
+            textgen_config=req.textgen_config if req.textgen_config else TextGenerationConfig(
+                n=1,
+                temperature=0),
+            library=req.library)[0]
+        return {"status": True, "evaluations": evaluations,
+                "message": "Successfully generated evaluation"}
+
+    except Exception as exception_error:
+        logger.error(f"Error generating visualization evaluation: {str(exception_error)}")
+        return {"status": False,
+                "message": f"Error generating visualization evaluation. {str(exception_error)}"}
+
+
+@api.post("/visualize/recommend")
+async def recommend_visualization(req: VisualizeRecommendRequest) -> dict:
+    """Given a dataset summary, generate a visualization recommendations"""
+
+    try:
+        textgen_config = req.textgen_config if req.textgen_config else TextGenerationConfig()
+        charts = lida.recommend(
+            summary=req.summary,
+            code=req.code,
+            textgen_config=textgen_config,
+            library=req.library,
+            return_error=True)
+
+        if len(charts) == 0:
+            return {"status": False, "message": "No charts generated"}
+        return {"status": True, "charts": charts,
+                "message": "Successfully generated chart recommendation"}
+
+    except Exception as exception_error:
+        logger.error(f"Error generating visualization recommendation: {str(exception_error)}")
+        return {"status": False,
+                "message": f"Error generating visualization recommendation."}
+
+
+@api.post("/text/generate")
+async def generate_text(textgen_config: TextGenerationConfig) -> dict:
+    """Generate text given some prompt"""
+
+    try:
+        completions = textgen.generate(textgen_config)
+        return {"status": True, "completions": completions.text}
+    except Exception as exception_error:
+        logger.error(f"Error generating text: {str(exception_error)}")
+        return {"status": False, "message": f"Error generating text."}
+
+
+@api.post("/goal")
+async def generate_goal(req: GoalWebRequest) -> dict:
+    """Generate goals given a dataset summary"""
+    try:
+        textgen_config = req.textgen_config if req.textgen_config else TextGenerationConfig()
+        goals = lida.goals(req.summary, n=req.n, textgen_config=textgen_config)
+        return {"status": True, "data": goals,
+                "message": f"Successfully generated {len(goals)} goals"}
+    except Exception as exception_error:
+        logger.error(f"Error generating goals: {str(exception_error)}")
+        # Check for a specific error message related to context length
+        if "context length" in str(exception_error).lower():
+            return {
+                "status": False,
+                "message": "The dataset you uploaded has too many columns. Please upload a dataset with fewer columns and try again."
+            }
+
+        # For other exceptions
+        return {
+            "status": False,
+            "message": f"Error generating visualization goals. {exception_error}"
+        }
+
+
+@api.post("/summarize")
+async def upload_file(file: UploadFile):
+    """ Upload a file and return a summary of the data """
+    # allow csv, excel, json
+    allowed_types = ["text/csv", "application/vnd.ms-excel", "application/json"]
+
+    # print("file: ", file)
+    # check file type
+    if file.content_type not in allowed_types:
+        return {"status": False,
+                "message": f"Uploaded file type ({file.content_type}) not allowed. Allowed types are: csv, excel, json"}
+
+    try:
+
+        # save file to files folder
+        file_location = os.path.join(data_folder, file.filename)
+        # open file without deleting existing contents
+        with open(file_location, "wb+") as file_object:
+            file_object.write(file.file.read())
+
+        # summarize
+        textgen_config = TextGenerationConfig(n=1, temperature=0)
+        summary = lida.summarize(
+            data=file_location,
+            file_name=file.filename,
+            summary_method="llm",
+            textgen_config=textgen_config)
+        return {"status": True, "summary": summary, "data_filename": file.filename}
+    except Exception as exception_error:
+        logger.error(f"Error processing file: {str(exception_error)}")
+        return {"status": False, "message": f"Error processing file."}
+
+
+# upload via url
+@api.post("/summarize/url")
+async def upload_file_via_url(req: SummaryUrlRequest) -> dict:
+    """ Upload a file from a url and return a summary of the data """
+    url = req.url
+    textgen_config = req.textgen_config if req.textgen_config else TextGenerationConfig(
+        n=1, temperature=0)
+    file_name = url.split("/")[-1]
+    file_location = os.path.join(data_folder, file_name)
+
+    # download file
+    url_response = requests.get(url, allow_redirects=True, timeout=1000)
+    open(file_location, "wb").write(url_response.content)
+    try:
+
+        summary = lida.summarize(
+            data=file_location,
+            file_name=file_name,
+            summary_method="llm",
+            textgen_config=textgen_config)
+        return {"status": True, "summary": summary, "data_filename": file_name}
+    except Exception as exception_error:
+        # traceback.print_exc()
+        logger.error(f"Error processing file: {str(exception_error)}")
+        return {"status": False, "message": f"Error processing file."}
+
+# convert image to infographics
+
+
+@api.post("/infographer")
+async def generate_infographics(req: InfographicsRequest) -> dict:
+    """Generate infographics using the peacasso package"""
+    try:
+        result = lida.infographics(
+            visualization=req.visualization,
+            n=req.n,
+            style_prompt=req.style_prompt
+            # return_pil=req.return_pil
+        )
+        return {"status": True, "result": result, "message": "Successfully generated infographics"}
+    except Exception as exception_error:
+        logger.error(f"Error generating infographics: {str(exception_error)}")
+        return {"status": False,
+                "message": f"Error generating infographics. {str(exception_error)}"}
+
+# list supported models
+
+
+@api.get("/models")
+def list_models() -> dict:
+    return {"status": True, "data": providers, "message": "Successfully listed models"}

memo.md

@@ -0,0 +1,3 @@
+## data for demonstration
+아동급식카드 가맹점
+- geo plot implementation