Upload 8 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+local_model/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+recipes.csv filter=lfs diff=lfs merge=lfs -text

cooking.py

@@ -0,0 +1,161 @@
+import os
+import pandas as pd
+import faiss
+import torch
+from transformers import AutoModel, AutoTokenizer
+from langchain_community.vectorstores import FAISS
+from langchain.text_splitter import CharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain.prompts import PromptTemplate
+from langchain_community.chat_models import ChatOllama
+from langchain.schema import Document
+
+# GPU 설정
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+VECTOR_STORE_PATH = "./vectorstore"
+EMBEDDINGS = HuggingFaceEmbeddings(model_name="intfloat/multilingual-e5-small")
+
+# CSV 파일에서 데이터를 로드하고 Document 객체로 변환
+def load_csv_to_documents(csv_file):
+    df = pd.read_csv(csv_file)
+    documents = []
+
+    # 각 행을 Document 객체로 변환
+    for index, row in df.iterrows():
+        document_text = f"Name: {row['name']}\nIngredients: {row['recipeIngredient']}\nInstructions: {row['recipeInstructions']}"
+        documents.append(Document(page_content=document_text))  # Document 객체로 변환
+
+    return documents
+
+def create_vectorstore():
+    csv_file = '/home/sohee/pj/united/recipes.csv'  # CSV 파일 경로
+
+    # 벡터 저장소 캐시를 먼저 확인하고 로드
+    if os.path.exists(VECTOR_STORE_PATH):
+        print("Vector store already exists. Loading it from disk...")
+        return FAISS.load_local(VECTOR_STORE_PATH, EMBEDDINGS, allow_dangerous_deserialization=True)
+
+    print("Creating new vector store...")
+    text_splitter = CharacterTextSplitter(
+        separator="\n",
+        chunk_size=1000,
+        chunk_overlap=200,
+        length_function=len,
+        is_separator_regex=False,
+    )
+
+    # CSV 파일에서 문서 로드
+    documents = load_csv_to_documents(csv_file)
+
+    # 문서를 청크로 분할
+    chunked_documents = text_splitter.split_documents(documents)
+
+    # 벡터 저장소 생성 (GPU 가속화 사용)
+    index = faiss.IndexFlatL2(EMBEDDINGS.embed_dimension)
+    if torch.cuda.is_available():
+        print("Using GPU for FAISS...")
+        res = faiss.StandardGpuResources()
+        index = faiss.index_cpu_to_gpu(res, 0, index)
+
+    vectorstore = FAISS.from_documents(chunked_documents, EMBEDDINGS, index=index)
+    vectorstore.save_local(VECTOR_STORE_PATH)
+
+    return vectorstore
+
+# 프롬프트 템플릿 정의
+prompt = PromptTemplate(
+    input_variables=["context", "question"],
+    template="Context: {context}\nQuestion: {question}\nAnswer:"
+)
+
+# LLM 모델 및 토크나이저 로컬 저장
+def save_model_locally():
+    model_name = "intfloat/multilingual-e5-small"  # 모델 이름
+    save_directory = "/home/sohee/pj/local_model"  # 저장할 경로
+
+    # 모델과 토크나이저 로드
+    model = AutoModel.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    # 모델 및 토크나이저를 로컬에 저장
+    model.save_pretrained(save_directory)
+    tokenizer.save_pretrained(save_directory)
+    print(f"Model and tokenizer saved locally at {save_directory}")
+
+# GPU 지원 LLM 모델 초기화
+llm = ChatOllama(model="llama3.1:8b", device=device)
+
+# 메타 텐서 문제 처리
+def initialize_meta_tensor_model():
+    print("Handling meta tensor...")
+    model = AutoModel.from_pretrained("intfloat/multilingual-e5-small")
+    model = model.to_empty(device=device)  # Meta 텐서를 처리하기 위한 to_empty 사용
+    return model
+
+# Format the retrieved documents to filter out the recipes containing the ingredient
+def format_docs(docs, query):
+    results = []
+    unique_recipes = set()
+
+    for doc in docs:
+        content = doc.page_content
+
+        # Extract name, ingredients, and instructions
+        name_line = content.split("\n")[0].split(": ")[1].strip() if "Name:" in content else ""
+        ingredients_line = content.split("\n")[1].split(": ")[1].strip() if "Ingredients:" in content else ""
+        instructions_line = content.split("\nInstructions: ")[-1].strip() if "Instructions:" in content else ""
+
+        # 검색어가 요리 이름(name) 또는 재료(ingredients)에 포함되는지 확인
+        if (query.lower() in name_line.lower() or any(q.lower() in ingredients_line.lower() for q in query.split(","))) and name_line not in unique_recipes:
+            unique_recipes.add(name_line)
+
+            # 만드는 방법을 목록으로 변경
+            formatted_instructions = '\n'.join([f"- {step.strip()}" for step in instructions_line.split('.') if step])
+
+            results.append({
+                "name": name_line,
+                "ingredients": ingredients_line,
+                "instructions": formatted_instructions  # 목록 형식으로 변경된 instructions
+            })
+
+        # 최대 5개까지만 결과를 포함하도록 제한
+        if len(results) >= 5:
+            break
+
+    return results
+
+def ask_query(query):
+    # 벡터 저장소 생성 또는 로드
+    vectorstore = create_vectorstore()
+    retriever = vectorstore.as_retriever(search_kwargs={"k": 150000})  # 검색 결과 수를 15개로 늘림
+
+    # Use `invoke` method instead of the deprecated one
+    docs = retriever.invoke(query)
+
+    # Format the documents based on the query (ingredient or name)
+    formatted_results = format_docs(docs, query)
+
+    # 결과 출력
+    if not formatted_results:
+        print(f"'{query}'에 해당하는 요리 레시피를 찾을 수 없습니다.")
+    else:
+        print(f"'{query}'이(가) 포함된 모든 음식명 리스트:")
+        for i, result in enumerate(formatted_results, 1):
+            print(f"[{i}번째 레시피]\n   (1) 음식 이름: {result['name']}")
+            print(f"   (2) 재료: {result['ingredients']}")
+            print(f"   (3) 만드는 방법: \n{result['instructions']}")
+            print("--" * 100)
+            print()
+
+if __name__ == "__main__":
+    while True:
+        user_query = input("질문을 입력하세요 (예:다이어트(다이어트식) 또는  된장찌개 또는 재료명(돼지고기, 오이 등) 입력. 종료를 원할 시 'exit' 입력): ")  # 사용자에게 질문 입력 받기
+        
+        if user_query.lower() == 'exit':  # 사용자가 'exit'을 입력하면 프로그램 종료
+            print("프로그램을 종료합니다.")
+            save_model_locally()  # 프로그램 종료 시 모델 저장
+            break
+        
+        ask_query(user_query)  # 입력된 질문 처리 및 응답 출력
+

local_model/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "intfloat/multilingual-e5-small",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.43.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 250037
+}

local_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a77d5da5ee721c7c740e4082447d3026b6521d3eac5edb93edb6aa88f03b7d7
+size 470637416

local_model/sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

local_model/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

local_model/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd98e5698b201ba914efb8c18b6709fa8735ab71dcad8d2b431e52e8bf68d932
+size 17082800

local_model/tokenizer_config.json

@@ -0,0 +1,55 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}

recipes.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3f0ca7dc1c49c0b9969c85bb2c1debb77dfe18c9a3ba490bce07c80828c8d25
+size 139790399