summarize.py

"""
Data Scientist.: Dr.Eddy Giusepe Chirinos Isidro

Siga a seguinte ordem na qual executei os scripts .py:

* Primeiro --> summarize.py
* Segundo --> refine.py
* Terceiro --> map_reduce.py
"""
from langchain import OpenAI, PromptTemplate, LLMChain
from langchain.text_splitter import CharacterTextSplitter
from langchain.chains.mapreduce import MapReduceChain
from langchain.prompts import PromptTemplate
from langchain.docstore.document import Document
from langchain.chains.summarize import load_summarize_chain
from langchain.callbacks import get_openai_callback

import openai
import os
from dotenv import load_dotenv, find_dotenv
_ = load_dotenv(find_dotenv()) # read local .env file
openai.api_key  = os.environ['OPENAI_API_KEY']


llm = OpenAI(temperature=0)

text_splitter = CharacterTextSplitter()


# Transforme o documento em conjunto de caracteres ASCII e grave no arquivo para evitar erros de codificação:
with open("space.json", "r",encoding="utf-8") as f:
    space = f.read().encode("ascii", "ignore").decode("ascii")

with open("space_ascii.txt", "w") as f:
    f.write(space)



texts = text_splitter.split_text(space)

print("Comprimento da lista de textos: ", len(texts))

# Ao testar, limite o número de documentos a alguns para salvar textos de tokens[:3] por exemplo.
docs = [Document(page_content=t) for t in texts[:3]]
print("🤗", docs)
print("")
chain = load_summarize_chain(llm,
                             chain_type="map_reduce",
                             verbose=False
                            )

with get_openai_callback() as cb:
    result = chain.run(docs)
    print("🤗🤗")
    print(result)
    print("Tokens usados: ", cb.total_tokens)