Create config.py

config.py

@@ -0,0 +1,341 @@
+"""
+Cogni-Engine v1 — Configuration
+All system parameters centralized here.
+Every module imports from this file.
+"""
+
+import os
+import secrets
+import hashlib
+
+
+# ═══════════════════════════════════════════════════════════
+# API SERVER
+# ═══════════════════════════════════════════════════════════
+
+PORT = int(os.environ.get("PORT", 7860))
+API_KEY = os.environ.get("API_KEY", secrets.token_urlsafe(32))
+MAX_REQUEST_SIZE_MB = 50
+
+# Print generated key if not set (first run)
+if not os.environ.get("API_KEY"):
+    print(f"[CONFIG] No API_KEY set. Generated temporary key: {API_KEY}")
+    print(f"[CONFIG] Set API_KEY environment variable for persistent key.")
+
+
+# ═══════════════════════════════════════════════════════════
+# TiDB CONNECTION
+# ═══════════════════════════════════════════════════════════
+
+TIDB_HOST = os.environ.get("TIDB_HOST", "")
+TIDB_PORT = int(os.environ.get("TIDB_PORT", 4000))
+TIDB_USER = os.environ.get("TIDB_USER", "")
+TIDB_PASSWORD = os.environ.get("TIDB_PASSWORD", "")
+TIDB_DATABASE = os.environ.get("TIDB_DATABASE", "cogni_engine")
+TIDB_SSL = os.environ.get("TIDB_SSL", "true").lower() == "true"
+
+# Connection pool
+TIDB_POOL_SIZE = int(os.environ.get("TIDB_POOL_SIZE", 5))
+TIDB_CONNECT_TIMEOUT = 10
+TIDB_READ_TIMEOUT = 30
+TIDB_WRITE_TIMEOUT = 30
+TIDB_RETRY_ATTEMPTS = 3
+TIDB_RETRY_DELAY = 2  # seconds between retries
+
+
+# ═══════════════════════════════════════════════════════════
+# VECTOR & EMBEDDING
+# ═══════════════════════════════════════════════════════════
+
+VECTOR_DIM = 128                    # Dimensi vektor embedding per node
+NGRAM_SIZES = [3, 4]                # Character n-gram sizes for hashing
+HASH_BUCKETS = 8192                 # Hash bucket count for n-gram vectorization
+RANDOM_PROJECTION_SEED = 42        # Seed for reproducible random projection matrix
+
+
+# ═══════════════════════════════════════════════════════════
+# KNOWLEDGE GRAPH
+# ═══════════════════════════════════════════════════════════
+
+SIMILARITY_THRESHOLD = 0.65         # Minimum cosine similarity to auto-create edge
+MERGE_THRESHOLD = 0.95              # Similarity above this = redundant nodes, merge
+PRUNE_WEIGHT_THRESHOLD = 0.05      # Edges below this weight get pruned
+MAX_TRAVERSAL_DEPTH = 8            # Maximum hops in graph walk
+MAX_CHAINS_PER_RESPONSE = 7        # Maximum reasoning chains used per response
+MAX_NODES_PER_SEARCH = 20          # Top-K nodes returned by similarity search
+MIN_EDGE_CONFIDENCE = 0.05         # Below this, edge is candidate for deletion
+MAX_GRAPH_MEMORY_NODES = 500000    # Safety limit for in-memory nodes
+MAX_GRAPH_MEMORY_EDGES = 2000000   # Safety limit for in-memory edges
+
+
+# ═══════════════════════════════════════════════════════════
+# ABSTRACTION
+# ═══════════════════════════════════════════════════════════
+
+MAX_ABSTRACTION_DEPTH = 5           # Maximum recursive abstraction levels
+CLUSTER_MIN_SIZE = 3                # Minimum nodes to form an abstraction
+CLUSTER_MAX_SIZE = 50               # Maximum nodes per cluster
+CLUSTER_SIMILARITY_INTRA = 0.60    # Minimum intra-cluster similarity
+CLUSTER_ITERATIONS = 20            # K-means iterations per clustering run
+ABSTRACTION_MIN_CONFIDENCE = 0.50  # Minimum confidence for abstraction node
+
+
+# ═══════════════════════════════════════════════════════════
+# INFERENCE
+# ═══════════════════════════════════════════════════════════
+
+INFERENCE_CONFIDENCE_MIN = 0.30     # Below this, don't save inferred edge
+INFERENCE_DECAY = 0.85              # Decay per hop: conf(A→C) = conf(A→B) * conf(B→C) * decay
+INFERENCE_MAX_CHAIN_LENGTH = 5     # Maximum transitive hops for inference
+ANALOGICAL_SIMILARITY_MIN = 0.70   # Minimum similarity for analogical reasoning
+MAX_INFERENCES_PER_CYCLE = 100     # Limit inferences per thinking cycle (prevent explosion)
+
+
+# ═══════════════════════════════════════════════════════════
+# THINKING LOOP
+# ═══════════════════════════════════════════════════════════
+
+THINKING_INTERVAL_FAST = 2          # Seconds between cycles when active (new data)
+THINKING_INTERVAL_SLOW = 15         # Seconds between cycles when stable
+THINKING_STABILITY_THRESHOLD = 5   # Operations < this = "stable" → slow down
+THINKING_BATCH_SIZE = 50           # Nodes/edges processed per sub-phase
+SYNC_INTERVAL_CYCLES = 100         # Flush to TiDB every N cycles
+SYNC_INTERVAL_SECONDS = 60         # Flush to TiDB every N seconds (whichever first)
+SELF_QUESTION_INTERVAL = 50        # Run self-questioning every N cycles
+VALIDATE_INTERVAL = 25             # Run validation every N cycles
+COMPRESS_INTERVAL = 100            # Run compression every N cycles
+
+
+# ═══════════════════════════════════════════════════════════
+# WEIGHT DYNAMICS
+# ═══════════════════════════════════════════════════════════
+
+WEIGHT_REINFORCE = 1.05             # Multiplier when edge is used in response
+WEIGHT_DECAY_RATE = 0.98           # Multiplier per decay cycle for unused edges
+WEIGHT_DECAY_INTERVAL_CYCLES = 500 # Apply decay every N cycles
+WEIGHT_MAX = 10.0                   # Maximum edge/node weight (prevent overflow)
+WEIGHT_MIN = 0.01                   # Minimum before considered for pruning
+NODE_WEIGHT_CONNECTION_BONUS = 0.02 # Weight bonus per connection for nodes
+USER_KNOWLEDGE_CONFIDENCE = 0.60   # Default confidence for knowledge extracted from user chat
+DATA_KNOWLEDGE_CONFIDENCE = 0.90   # Default confidence for knowledge from JSONL files
+
+
+# ═══════════════════════════════════════════════════════════
+# LANGUAGE GENERATION
+# ═══════════════════════════════════════════════════════════
+
+DEFAULT_TEMPERATURE = 0.7           # Default response variation (0=deterministic, 1=max variety)
+DEFAULT_FORMALITY = 0.5             # 0=very casual, 1=very formal
+DEFAULT_LANGUAGE = "id"             # Default output language
+MAX_RESPONSE_SEGMENTS = 8          # Maximum segments in a response
+MIN_RESPONSE_SEGMENTS = 2          # Minimum segments (even for simple answers)
+CONFIDENCE_HIGH = 0.80              # Above: assertive language
+CONFIDENCE_MEDIUM = 0.50            # Above: qualified language
+CONFIDENCE_LOW = 0.30               # Above: tentative language
+                                    # Below 0.30: honest uncertainty
+
+# Segment types available for response construction
+SEGMENT_TYPES = [
+    "introduction",
+    "main_explanation",
+    "supporting_detail",
+    "inference",
+    "context",
+    "acknowledgment_of_uncertainty",
+    "suggestion",
+    "conclusion",
+    "example",
+    "comparison",
+    "elaboration"
+]
+
+# Intent types for query classification
+INTENT_TYPES = [
+    "explain",      # "Apa itu X?" / "Jelaskan X"
+    "relation",     # "Apa hubungan X dengan Y?"
+    "how_to",       # "Bagaimana cara X?"
+    "compare",      # "Bandingkan X dan Y"
+    "define",       # "Definisi X"
+    "list",         # "Sebutkan X"
+    "cause",        # "Mengapa X?"
+    "opinion",      # "Pendapat tentang X?"
+    "general",      # Catch-all
+    "greeting",     # "Halo" etc
+    "followup"      # Continues previous context
+]
+
+
+# ═══════════════════════════════════════════════════════════
+# CONVERSATION & SESSION
+# ═══════════════════════════════════════════════════════════
+
+CONTEXT_WINDOW_TURNS = 10           # Number of conversation turns kept in memory
+SESSION_TIMEOUT_MINUTES = 30       # Session expires after inactivity
+MAX_CONCURRENT_SESSIONS = 100     # Maximum simultaneous conversations
+SESSION_CLEANUP_INTERVAL = 300     # Seconds between session cleanup sweeps
+
+
+# ═══════════════════════════════════════════════════════════
+# KEEP-ALIVE
+# ════════════════════════��══════════════════════════════════
+
+KEEP_ALIVE_INTERVAL = 300           # Self-ping every 5 minutes
+KEEP_ALIVE_ENABLED = True           # Can disable for local development
+
+
+# ═══════════════════════════════════════════════════════════
+# DATA INPUT
+# ═══════════════════════════════════════════════════════════
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
+SUPPORTED_DATA_EXTENSIONS = [".jsonl"]
+FILE_SCAN_INTERVAL = 30             # Seconds between /data/ folder scans
+MAX_LINES_PER_INGEST = 10000      # Max lines processed per ingest cycle (prevent blocking)
+
+# Core data types
+CORE_DATA_TYPES = [
+    # Knowledge
+    "fact", "definition", "explanation", "description", "property",
+    "statistic", "measurement", "term", "abbreviation", "jargon",
+    "slang", "idiom", "synonym", "antonym", "quote", "rule",
+    "example", "analogy", "opinion", "paragraph",
+    # Relational
+    "relation", "cause_effect", "comparison", "hierarchy",
+    "composition", "dependency", "contradiction", "timeline",
+    # Structured
+    "process", "procedure", "event", "history", "change", "qa"
+]
+
+# Edge relation types
+EDGE_RELATION_TYPES = [
+    # From data
+    "is_a", "part_of", "has", "located_in", "created_by",
+    "used_for", "causes", "prevents", "requires", "contains",
+    "member_of", "opposite_of", "synonym_of", "defined_as",
+    "example_of", "follows", "precedes", "related_to",
+    # From inference
+    "similar_to", "inferred_relation", "instance_of", "analogous_to"
+]
+
+
+# ═══════════════════════════════════════════════════════════
+# NODE ID GENERATION
+# ═══════════════════════════════════════════════════════════
+
+def generate_node_id(content: str, node_type: str = "") -> str:
+    """Generate deterministic node ID from content."""
+    raw = f"{node_type}:{content}".strip().lower()
+    return "n_" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
+
+
+def generate_edge_id(from_id: str, to_id: str, relation: str) -> str:
+    """Generate deterministic edge ID from components."""
+    raw = f"{from_id}|{to_id}|{relation}"
+    return "e_" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
+
+
+def generate_chain_id(path: list) -> str:
+    """Generate deterministic chain ID from path."""
+    raw = "|".join(str(p) for p in path)
+    return "c_" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
+
+
+def generate_session_id() -> str:
+    """Generate random session ID."""
+    return "s_" + secrets.token_hex(12)
+
+
+# ═══════════════════════════════════════════════════════════
+# INTELLIGENCE SCORE WEIGHTS
+# ═══════════════════════════════════════════════════════════
+
+INTELLIGENCE_WEIGHTS = {
+    "log_nodes": 0.15,
+    "log_edges": 0.15,
+    "avg_connections": 0.15,
+    "max_abstraction_depth": 0.15,
+    "avg_chain_length": 0.15,
+    "inference_ratio": 0.10,
+    "avg_confidence": 0.15
+}
+
+
+# ═══════════════════════════════════════════════════════════
+# LOGGING
+# ═══════════════════════════════════════════════════════════
+
+LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
+LOG_THINKING_DETAILS = os.environ.get("LOG_THINKING", "false").lower() == "true"
+LOG_API_REQUESTS = os.environ.get("LOG_API", "true").lower() == "true"
+
+
+# ═══════════════════════════════════════════════════════════
+# STARTUP VALIDATION
+# ═══════════════════════════════════════════════════════════
+
+def validate_config():
+    """Validate critical configuration on startup."""
+    errors = []
+    warnings = []
+
+    # TiDB — required for persistence
+    if not TIDB_HOST:
+        errors.append("TIDB_HOST not set. Database persistence will not work.")
+    if not TIDB_USER:
+        errors.append("TIDB_USER not set.")
+    if not TIDB_PASSWORD:
+        errors.append("TIDB_PASSWORD not set.")
+
+    # Data directory
+    if not os.path.exists(DATA_DIR):
+        try:
+            os.makedirs(DATA_DIR, exist_ok=True)
+            warnings.append(f"Created data directory: {DATA_DIR}")
+        except OSError as e:
+            errors.append(f"Cannot create data directory {DATA_DIR}: {e}")
+
+    # Sanity checks on parameters
+    if VECTOR_DIM < 32 or VECTOR_DIM > 1024:
+        warnings.append(f"VECTOR_DIM={VECTOR_DIM} outside recommended range [32, 1024]")
+
+    if SIMILARITY_THRESHOLD < 0.3 or SIMILARITY_THRESHOLD > 0.95:
+        warnings.append(f"SIMILARITY_THRESHOLD={SIMILARITY_THRESHOLD} may cause too many/few connections")
+
+    if INFERENCE_DECAY < 0.5 or INFERENCE_DECAY > 0.99:
+        warnings.append(f"INFERENCE_DECAY={INFERENCE_DECAY} outside recommended range [0.5, 0.99]")
+
+    if MAX_ABSTRACTION_DEPTH > 10:
+        warnings.append(f"MAX_ABSTRACTION_DEPTH={MAX_ABSTRACTION_DEPTH} very high, may cause slow clustering")
+
+    # Report
+    for w in warnings:
+        print(f"[CONFIG WARNING] {w}")
+    for e in errors:
+        print(f"[CONFIG ERROR] {e}")
+
+    if errors:
+        print(f"[CONFIG] {len(errors)} error(s) found. System may not function correctly.")
+        return False
+
+    print(f"[CONFIG] Validation passed. {len(warnings)} warning(s).")
+    return True
+
+
+def print_config_summary():
+    """Print non-sensitive configuration summary."""
+    print("=" * 55)
+    print("  COGNI-ENGINE v1 — Configuration")
+    print("=" * 55)
+    print(f"  Port:              {PORT}")
+    print(f"  TiDB Host:         {'SET' if TIDB_HOST else 'NOT SET'}")
+    print(f"  TiDB Database:     {TIDB_DATABASE}")
+    print(f"  Vector Dim:        {VECTOR_DIM}")
+    print(f"  Data Dir:          {DATA_DIR}")
+    print(f"  Similarity Thresh: {SIMILARITY_THRESHOLD}")
+    print(f"  Max Traversal:     {MAX_TRAVERSAL_DEPTH}")
+    print(f"  Max Abstraction:   {MAX_ABSTRACTION_DEPTH}")
+    print(f"  Think Fast:        {THINKING_INTERVAL_FAST}s")
+    print(f"  Think Slow:        {THINKING_INTERVAL_SLOW}s")
+    print(f"  Keep-Alive:        {'ON' if KEEP_ALIVE_ENABLED else 'OFF'}")
+    print(f"  Log Level:         {LOG_LEVEL}")
+    print("=" * 55)