|
|
|
|
|
import os, sys, yaml, json, subprocess, logging, traceback, time, tarfile, re |
|
|
from pathlib import Path |
|
|
from datetime import datetime |
|
|
|
|
|
logging.basicConfig( |
|
|
level=logging.INFO, |
|
|
format='%(asctime)s [%(levelname)s] %(message)s', |
|
|
handlers=[ |
|
|
logging.StreamHandler(sys.stdout), |
|
|
logging.FileHandler('/tmp/setup_debug.log', mode='w') |
|
|
] |
|
|
) |
|
|
logger = logging.getLogger(__name__) |
|
|
|
|
|
STATUS_FILE = Path('/tmp/setup_status.json') |
|
|
READY_FLAG = Path('/tmp/faiss_ready') |
|
|
|
|
|
def update_status(status, message, progress=0): |
|
|
data = {'status': status, 'message': message, 'progress': progress, 'timestamp': datetime.now().isoformat()} |
|
|
with open(STATUS_FILE, 'w') as f: |
|
|
json.dump(data, f) |
|
|
logger.info(f"STATUS [{progress}%]: {status} - {message}") |
|
|
|
|
|
def run_cmd(cmd, desc, check=True, timeout=300): |
|
|
logger.info("="*80) |
|
|
logger.info(f"🔧 {desc}") |
|
|
logger.info(f"📝 {cmd}") |
|
|
logger.info("-"*80) |
|
|
|
|
|
try: |
|
|
start = time.time() |
|
|
result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout, check=check) |
|
|
elapsed = time.time() - start |
|
|
|
|
|
logger.info(f"⏱️ {elapsed:.2f}s | Exit: {result.returncode}") |
|
|
|
|
|
if result.stdout and len(result.stdout.strip()) > 0: |
|
|
logger.info(f"STDOUT: {result.stdout[:500]}") |
|
|
if result.stderr and len(result.stderr.strip()) > 0: |
|
|
logger.warning(f"STDERR: {result.stderr[:500]}") |
|
|
|
|
|
if result.returncode == 0: |
|
|
logger.info(f"✅ {desc} - OK") |
|
|
|
|
|
return result.stdout |
|
|
|
|
|
except subprocess.CalledProcessError as e: |
|
|
logger.error(f"❌ FALHOU: {desc}") |
|
|
raise |
|
|
|
|
|
def clean_html_text(text): |
|
|
if not text or not isinstance(text, str): |
|
|
return "" |
|
|
text = re.sub(r'<[^>]+>', '', text) |
|
|
text = re.sub(r'&[a-zA-Z]+;', ' ', text) |
|
|
text = re.sub(r'\s+', ' ', text) |
|
|
return text.strip() |
|
|
|
|
|
def is_valid_value(value): |
|
|
if value is None: |
|
|
return False |
|
|
if isinstance(value, str): |
|
|
return bool(value.strip()) |
|
|
if isinstance(value, (int, float)): |
|
|
return True |
|
|
return True |
|
|
|
|
|
def filter_and_validate_record(record, fields_to_keep): |
|
|
filtered = {} |
|
|
|
|
|
for field in fields_to_keep: |
|
|
value = None |
|
|
|
|
|
if field in record: |
|
|
value = record[field] |
|
|
else: |
|
|
for variant in [field.lower(), field.capitalize(), field.upper()]: |
|
|
if variant in record: |
|
|
value = record[variant] |
|
|
break |
|
|
|
|
|
if value is None: |
|
|
return None, False |
|
|
|
|
|
if not is_valid_value(value): |
|
|
return None, False |
|
|
|
|
|
field_name = 'id' if field in ['Id', 'id', 'ID'] else field.lower() |
|
|
|
|
|
if field.lower() == 'ementa' and isinstance(value, str): |
|
|
cleaned_value = clean_html_text(value) |
|
|
if not cleaned_value or not cleaned_value.strip(): |
|
|
return None, False |
|
|
filtered[field_name] = cleaned_value |
|
|
else: |
|
|
filtered[field_name] = value |
|
|
|
|
|
return filtered, True |
|
|
|
|
|
def process_tar_gz(tar_path, output_jsonl, fields_to_keep): |
|
|
logger.info(f"📦 {tar_path.name}") |
|
|
|
|
|
stats = {'total': 0, 'validos': 0} |
|
|
|
|
|
try: |
|
|
with tarfile.open(tar_path, 'r:gz') as tar: |
|
|
for member in tar.getmembers(): |
|
|
if member.name.endswith('jurisprudencias.jsonl') and member.isfile(): |
|
|
logger.info(f" ✅ {member.name}") |
|
|
|
|
|
file_obj = tar.extractfile(member) |
|
|
content = file_obj.read().decode('utf-8') |
|
|
|
|
|
lines = content.strip().split('\n') |
|
|
stats['total'] = len(lines) |
|
|
|
|
|
with open(output_jsonl, 'a', encoding='utf-8') as out: |
|
|
for line in lines: |
|
|
if not line.strip(): |
|
|
continue |
|
|
|
|
|
try: |
|
|
record = json.loads(line) |
|
|
filtered, is_valid = filter_and_validate_record(record, fields_to_keep) |
|
|
|
|
|
if is_valid: |
|
|
out.write(json.dumps(filtered, ensure_ascii=False) + '\n') |
|
|
stats['validos'] += 1 |
|
|
except: |
|
|
pass |
|
|
|
|
|
logger.info(f" ✅ {stats['validos']}/{stats['total']}") |
|
|
return stats['validos'] |
|
|
return 0 |
|
|
except Exception as e: |
|
|
logger.error(f" ❌ {e}") |
|
|
raise |
|
|
|
|
|
def main(): |
|
|
try: |
|
|
logger.info("\n" + "="*80) |
|
|
logger.info("🚀 PARA.AI RAG SETUP v3.5") |
|
|
logger.info("="*80) |
|
|
|
|
|
if READY_FLAG.exists(): |
|
|
update_status('ready', 'Ready', 100) |
|
|
return |
|
|
|
|
|
with open('config.yaml') as f: |
|
|
config = yaml.safe_load(f) |
|
|
|
|
|
chunk_start = config['chunk_start'] |
|
|
chunk_end = config['chunk_end'] |
|
|
github_repo = config['github_repo'] |
|
|
campos_filter = config['campos_filter'] |
|
|
|
|
|
base_url = github_repo.replace('https://github.com/', 'https://raw.githubusercontent.com/') |
|
|
if base_url.endswith('.git'): |
|
|
base_url = base_url[:-4] |
|
|
base_url = f"{base_url}/main/chunks_dados" |
|
|
|
|
|
work_dir = Path('/tmp/work') |
|
|
work_dir.mkdir(exist_ok=True) |
|
|
|
|
|
output_jsonl = work_dir / 'all_filtered.jsonl' |
|
|
if output_jsonl.exists(): |
|
|
output_jsonl.unlink() |
|
|
|
|
|
logger.info("\n📥 Download") |
|
|
update_status('downloading', 'Downloading', 10) |
|
|
|
|
|
total_validos = 0 |
|
|
|
|
|
for chunk_num in range(chunk_start, chunk_end + 1): |
|
|
chunk_name = f"chunk_dados_{chunk_num:06d}.tar.gz" |
|
|
chunk_url = f"{base_url}/{chunk_name}" |
|
|
chunk_path = work_dir / chunk_name |
|
|
|
|
|
try: |
|
|
run_cmd(f"curl -L -f -o {chunk_path} {chunk_url}", f"Chunk {chunk_num}", timeout=300) |
|
|
|
|
|
if chunk_path.exists(): |
|
|
validos = process_tar_gz(chunk_path, output_jsonl, campos_filter) |
|
|
total_validos += validos |
|
|
chunk_path.unlink() |
|
|
|
|
|
except Exception as e: |
|
|
logger.error(f" ❌ {e}") |
|
|
if chunk_path.exists(): |
|
|
chunk_path.unlink() |
|
|
|
|
|
logger.info(f"\n✅ Total: {total_validos}") |
|
|
|
|
|
if total_validos == 0: |
|
|
raise Exception("Nenhum registro!") |
|
|
|
|
|
logger.info("\n🤖 Build FAISS") |
|
|
update_status('building', 'Building', 70) |
|
|
|
|
|
os.chdir('/home/user/app') |
|
|
|
|
|
result = subprocess.run( |
|
|
f"python3 rag_builder.py --input {output_jsonl}", |
|
|
shell=True, |
|
|
capture_output=True, |
|
|
text=True, |
|
|
timeout=900 |
|
|
) |
|
|
|
|
|
if result.stdout: |
|
|
for line in result.stdout.split('\n'): |
|
|
if line.strip(): |
|
|
logger.info(line) |
|
|
|
|
|
if result.stderr: |
|
|
for line in result.stderr.split('\n'): |
|
|
if line.strip(): |
|
|
logger.warning(line) |
|
|
|
|
|
if result.returncode != 0: |
|
|
raise Exception(f"Build falhou: exit {result.returncode}") |
|
|
|
|
|
logger.info("✅ OK!") |
|
|
run_cmd(f"rm -rf {work_dir}", "Cleanup", check=False) |
|
|
|
|
|
update_status('ready', f'{total_validos} docs', 100) |
|
|
READY_FLAG.touch() |
|
|
|
|
|
except Exception as e: |
|
|
logger.error(f"\n💥 {e}") |
|
|
logger.error(traceback.format_exc()) |
|
|
update_status('error', str(e), 0) |
|
|
sys.exit(1) |
|
|
|
|
|
if __name__ == "__main__": |
|
|
main() |
|
|
|
