Spaces:
Runtime error
Runtime error
File size: 2,792 Bytes
b0f2512 e86db37 b0f2512 e86db37 b0f2512 9f997ca b0f2512 28cddd4 b0f2512 9f997ca b0f2512 9f997ca 63f19a1 b0f2512 26aa964 b0f2512 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
import os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from tqdm import tqdm
from langchain.vectorstores import Chroma
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.schema import Document
from coachable_course_agent.load_data import load_courses
def clean_provider_name(provider):
"""
Clean provider name by removing duplicate first letters.
If the first two letters are the same and both uppercase, drop the first one.
E.g., "IIllinois Tech" -> "Illinois Tech", "IIBM" -> "IBM"
"""
if not provider or len(provider) < 2:
return provider
# Check if first two letters are the same and both uppercase
if provider[0] == provider[1] and provider[0].isupper() and provider[1].isupper():
return provider[1:] # Drop the first letter
return provider
# 1. Load course catalog (including ESCO-linked skills)
course_data = load_courses("data/course_catalog_esco.json")
# Handle both old format (direct array) and new format (with metadata)
if isinstance(course_data, dict) and 'courses' in course_data:
courses = course_data['courses']
print(f"📊 Loaded {len(courses)} courses from catalog with metadata")
elif isinstance(course_data, list):
courses = course_data
print(f"📊 Loaded {len(courses)} courses from legacy format")
else:
print("❌ Unexpected course data format")
sys.exit(1)
# 2. Initialize embedding model
embedding_model = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2", show_progress=True)
# 3. Convert to LangChain documents
course_docs = []
for course in tqdm(courses, desc="Processing courses"):
skill_names = ", ".join([s["name"] for s in course.get("skills", [])])
content = f"{course['title']}. Skills: {skill_names}"
# Clean the provider name during loading
cleaned_provider = clean_provider_name(course["provider"])
metadata = {
"id": course["id"],
"title": course["title"],
"provider": cleaned_provider, # Use cleaned provider name
"source_platform": course.get("source_platform", ""),
"duration_hours": course.get("duration_hours", 0),
"level": course.get("level", "Unknown"),
"format": course.get("format", "Unknown"),
"url": course.get("url", ""),
"skills": skill_names
}
course_docs.append(Document(page_content=content, metadata=metadata))
# 4. Save to persistent ChromaDB
persist_dir = "data/courses_chroma"
vectorstore = Chroma.from_documents(
documents=course_docs,
embedding=embedding_model,
persist_directory=persist_dir
)
vectorstore.persist()
print(f"✅ Stored {len(course_docs)} courses to ChromaDB at '{persist_dir}'")
|