Upload knowledge_library.py with huggingface_hub
Browse files- knowledge_library.py +557 -0
knowledge_library.py
ADDED
|
@@ -0,0 +1,557 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
FSI_FELON Β· KNOWLEDGE LIBRARY
|
| 4 |
+
Lightweight RAG for software engineering knowledge.
|
| 5 |
+
No external deps β uses numpy for similarity + BM25-style keyword scoring.
|
| 6 |
+
The model queries this when it doesn't know something, learns from it,
|
| 7 |
+
and gradually internalizes the knowledge.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import json, os, re, math
|
| 11 |
+
from collections import Counter
|
| 12 |
+
|
| 13 |
+
KNOWLEDGE_PATH = os.path.join(os.path.dirname(__file__) or '.', 'knowledge_library.json')
|
| 14 |
+
|
| 15 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 16 |
+
# KNOWLEDGE ENTRIES β curated SE textbook content
|
| 17 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 18 |
+
|
| 19 |
+
KNOWLEDGE_ENTRIES = [
|
| 20 |
+
# ββ PYTHON BASICS ββ
|
| 21 |
+
{
|
| 22 |
+
"topic": "python_functions",
|
| 23 |
+
"tags": ["python", "functions", "def", "arguments", "parameters", "return"],
|
| 24 |
+
"content": """Python Functions:
|
| 25 |
+
- Define with `def name(params):`, body indented 4 spaces
|
| 26 |
+
- Parameters: positional args, keyword args (name=value), *args (variadic), **kwargs (keyword variadic)
|
| 27 |
+
- Default values: `def f(x=10)`. NEVER use mutable defaults like `def f(x=[])`.
|
| 28 |
+
- Return values: `return value`. No return = returns None.
|
| 29 |
+
- Type hints: `def add(a: int, b: int) -> int:`. Optional.
|
| 30 |
+
- Docstrings: triple-quoted string right after def line.
|
| 31 |
+
- Lambda: `lambda x: x*2` for inline anonymous functions."""
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"topic": "python_classes",
|
| 35 |
+
"tags": ["python", "classes", "oop", "self", "inheritance", "methods"],
|
| 36 |
+
"content": """Python Classes:
|
| 37 |
+
- Define with `class Name:`. Methods take `self` as first param.
|
| 38 |
+
- `__init__(self, ...)` is constructor.
|
| 39 |
+
- Instance variables: `self.var = value` inside __init__.
|
| 40 |
+
- Inheritance: `class Child(Parent):`. Call `super().__init__()` in child __init__.
|
| 41 |
+
- `@property` decorator for computed attributes.
|
| 42 |
+
- `@staticmethod` for methods that don't need self.
|
| 43 |
+
- `@classmethod` with `cls` as first param for factory methods.
|
| 44 |
+
- `__str__` for string representation, `__repr__` for debugging representation."""
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"topic": "python_imports",
|
| 48 |
+
"tags": ["python", "import", "modules", "packages", "from"],
|
| 49 |
+
"content": """Python Imports:
|
| 50 |
+
- `import module` imports the whole module. Access with `module.name`.
|
| 51 |
+
- `from module import name` imports specific name into current namespace.
|
| 52 |
+
- `from module import *` imports all public names (avoid this).
|
| 53 |
+
- `import module as alias` for shorter names.
|
| 54 |
+
- `from module import name as alias` for renaming.
|
| 55 |
+
- Packages: directory with `__init__.py` file. Example: `from package.module import name`.
|
| 56 |
+
- Common imports: `import os`, `import sys`, `import json`, `import re`, `import math`."""
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"topic": "python_exceptions",
|
| 60 |
+
"tags": ["python", "exceptions", "error", "try", "except", "finally", "raise"],
|
| 61 |
+
"content": """Python Error Handling:
|
| 62 |
+
- `try:` block contains code that might raise an exception.
|
| 63 |
+
- `except ExceptionType as e:` catches specific exception types.
|
| 64 |
+
- `except Exception as e:` catches (almost) all exceptions.
|
| 65 |
+
- `else:` runs if no exception occurred.
|
| 66 |
+
- `finally:` ALWAYS runs (cleanup code).
|
| 67 |
+
- `raise ValueError("message")` to raise an exception.
|
| 68 |
+
- Common exceptions: ValueError, TypeError, KeyError, IndexError, FileNotFoundError, ZeroDivisionError.
|
| 69 |
+
- Context managers: `with open(path) as f:` auto-closes file even on error."""
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
"topic": "python_strings",
|
| 73 |
+
"tags": ["python", "strings", "format", "f-strings", "methods", "split", "join"],
|
| 74 |
+
"content": """Python String Operations:
|
| 75 |
+
- f-strings: `f"Hello {name}"` for string formatting.
|
| 76 |
+
- `.format()`: `"Hello {}".format(name)`.
|
| 77 |
+
- `.split(sep)`: split into list by separator.
|
| 78 |
+
- `.join(iterable)`: join list into string with separator.
|
| 79 |
+
- `.strip()`, `.lstrip()`, `.rstrip()`: remove whitespace.
|
| 80 |
+
- `.lower()`, `.upper()`, `.title()`, `.capitalize()`: case changes.
|
| 81 |
+
- `.replace(old, new)`: replace substrings.
|
| 82 |
+
- `.startswith(prefix)`, `.endswith(suffix)`: check prefixes/suffixes.
|
| 83 |
+
- `.find(sub)`, `.index(sub)`: find substring position.
|
| 84 |
+
- Slice: `s[start:end:step]`. Reverse: `s[::-1]`."""
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"topic": "python_lists",
|
| 88 |
+
"tags": ["python", "lists", "list", "array", "comprehension", "append", "slice"],
|
| 89 |
+
"content": """Python Lists:
|
| 90 |
+
- Create: `[1, 2, 3]` or `list(range(5))`.
|
| 91 |
+
- `.append(item)`: add single item to end.
|
| 92 |
+
- `.extend(iterable)`: add multiple items.
|
| 93 |
+
- `.insert(index, item)`: insert at position.
|
| 94 |
+
- `.pop(index)`: remove and return item at index (last if no index).
|
| 95 |
+
- `.remove(value)`: remove first occurrence of value.
|
| 96 |
+
- `.sort()`: sort in place. `.sort(reverse=True)` for descending.
|
| 97 |
+
- `sorted(list)`: returns new sorted list.
|
| 98 |
+
- List comprehension: `[x*2 for x in range(10) if x % 2 == 0]`.
|
| 99 |
+
- Slice: `lst[start:end:step]`. Copy: `lst[:]`.
|
| 100 |
+
- `len(lst)`, `min(lst)`, `max(lst)`, `sum(lst)`."""
|
| 101 |
+
},
|
| 102 |
+
{
|
| 103 |
+
"topic": "python_dicts",
|
| 104 |
+
"tags": ["python", "dictionary", "dict", "hashmap", "keys", "values", "items"],
|
| 105 |
+
"content": """Python Dictionaries:
|
| 106 |
+
- Create: `{"key": "value"}` or `dict(a=1, b=2)`.
|
| 107 |
+
- Access: `d["key"]`. Use `d.get("key", default)` to avoid KeyError.
|
| 108 |
+
- Set: `d["key"] = value`.
|
| 109 |
+
- `.keys()`, `.values()`, `.items()` for views.
|
| 110 |
+
- `for k, v in d.items():` for iteration.
|
| 111 |
+
- `key in d` to check existence.
|
| 112 |
+
- `.pop(key)`, `.popitem()` to remove.
|
| 113 |
+
- `.update(other_dict)` to merge.
|
| 114 |
+
- Dict comprehension: `{x: x**2 for x in range(5)}`.
|
| 115 |
+
- `defaultdict` from collections for default values.
|
| 116 |
+
- `Counter` from collections for counting."""
|
| 117 |
+
},
|
| 118 |
+
|
| 119 |
+
# ββ ALGORITHMS ββ
|
| 120 |
+
{
|
| 121 |
+
"topic": "algorithm_sorting",
|
| 122 |
+
"tags": ["algorithm", "sort", "sorting", "bubble", "merge", "quick", "binary search"],
|
| 123 |
+
"content": """Common Sorting Algorithms:
|
| 124 |
+
- Built-in: `sorted(list)` or `list.sort()`. Uses Timsort (O(n log n)).
|
| 125 |
+
- `sorted(list, key=lambda x: x[1])` for custom sort key.
|
| 126 |
+
- `sorted(list, reverse=True)` for descending.
|
| 127 |
+
- Binary search: `bisect_left(list, x)` from bisect module.
|
| 128 |
+
- Sort by multiple criteria: `sorted(list, key=lambda x: (x.age, x.name))`.
|
| 129 |
+
- Partial sort: `heapq.nsmallest(k, list)` or `heapq.nlargest(k, list)`.
|
| 130 |
+
- Finding min/max with conditions: `min(list, key=func)`."""
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"topic": "algorithm_recursion",
|
| 134 |
+
"tags": ["algorithm", "recursion", "recursive", "base case", "stack", "memoization"],
|
| 135 |
+
"content": """Recursion:
|
| 136 |
+
- A function that calls itself with smaller inputs.
|
| 137 |
+
- Must have a BASE CASE to stop recursion (e.g., n <= 1).
|
| 138 |
+
- Must make PROGRESS toward base case each call.
|
| 139 |
+
- Stack limit: Python has ~1000 recursion depth limit.
|
| 140 |
+
- Memoization: cache results with `@functools.lru_cache(maxsize=None)`.
|
| 141 |
+
- Examples: factorial, fibonacci, tree traversal, divide-and-conquer.
|
| 142 |
+
- Iterative alternative: use a stack (list) to simulate recursion.
|
| 143 |
+
- Tail recursion: Python does NOT optimize tail recursion."""
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"topic": "algorithm_search",
|
| 147 |
+
"tags": ["algorithm", "search", "linear", "binary", "bfs", "dfs", "graph"],
|
| 148 |
+
"content": """Search Algorithms:
|
| 149 |
+
- Linear search: check each element O(n). Works on unsorted data.
|
| 150 |
+
- Binary search: divide-and-conquer on sorted data O(log n). Use bisect module.
|
| 151 |
+
- BFS (Breadth-First Search): uses queue, finds shortest path in unweighted graph.
|
| 152 |
+
- DFS (Depth-First Search): uses stack or recursion, good for exhaustive search.
|
| 153 |
+
- Dijkstra: shortest path in weighted graph. Use `heapq` for priority queue.
|
| 154 |
+
- A*: heuristic-guided search for pathfinding.
|
| 155 |
+
- Graph representation: adjacency list `{node: [neighbors]}` or matrix."""
|
| 156 |
+
},
|
| 157 |
+
|
| 158 |
+
# ββ TESTING ββ
|
| 159 |
+
{
|
| 160 |
+
"topic": "testing_basics",
|
| 161 |
+
"tags": ["testing", "pytest", "unittest", "assert", "test", "tdd"],
|
| 162 |
+
"content": """Python Testing:
|
| 163 |
+
- Use pytest: `pip install pytest`. Run with `pytest` command.
|
| 164 |
+
- Test files named `test_*.py` or `*_test.py`.
|
| 165 |
+
- Test functions named `test_*`.
|
| 166 |
+
- Use `assert` for checks: `assert add(2, 3) == 5`.
|
| 167 |
+
- pytest discovers tests automatically.
|
| 168 |
+
- Fixtures: use `@pytest.fixture` for setup/teardown.
|
| 169 |
+
- `pytest.raises(ExceptionType)` for testing exceptions.
|
| 170 |
+
- `pytest.mark.parametrize` for running same test with multiple inputs.
|
| 171 |
+
- Coverage: `pytest --cov=module` to check code coverage.
|
| 172 |
+
- Test categories: normal cases, edge cases, error cases, boundary values."""
|
| 173 |
+
},
|
| 174 |
+
{
|
| 175 |
+
"topic": "testing_patterns",
|
| 176 |
+
"tags": ["testing", "mock", "patch", "fixture", "integration", "unit"],
|
| 177 |
+
"content": """Testing Patterns:
|
| 178 |
+
- Unit test: tests a single function/method in isolation.
|
| 179 |
+
- Integration test: tests multiple components together.
|
| 180 |
+
- Mock: `unittest.mock.patch` or `pytest-mock` to replace dependencies.
|
| 181 |
+
- Arrange-Act-Assert: set up data, execute code, verify result.
|
| 182 |
+
- Given-When-Then: BDD style of describing tests.
|
| 183 |
+
- Test doubles: Dummy, Stub, Spy, Mock, Fake.
|
| 184 |
+
- Coverage metrics: line coverage, branch coverage, path coverage.
|
| 185 |
+
- Property-based testing: `hypothesis` library generates random inputs.
|
| 186 |
+
- Snapshot testing: compare output to saved reference."""
|
| 187 |
+
},
|
| 188 |
+
|
| 189 |
+
# ββ CODE QUALITY ββ
|
| 190 |
+
{
|
| 191 |
+
"topic": "code_quality_linting",
|
| 192 |
+
"tags": ["code quality", "lint", "linter", "pylint", "flake8", "formatting"],
|
| 193 |
+
"content": """Code Quality:
|
| 194 |
+
- Linting: automated checking for errors and style issues.
|
| 195 |
+
- flake8: checks PEP 8 style, syntax errors, complexity.
|
| 196 |
+
- pylint: more thorough checking, generates quality score.
|
| 197 |
+
- black: opinionated code formatter (auto-fixes formatting).
|
| 198 |
+
- isort: sorts imports alphabetically.
|
| 199 |
+
- mypy: static type checker for type hints.
|
| 200 |
+
- ruff: fast Rust-based linter and formatter (combines flake8+isort+more).
|
| 201 |
+
- pre-commit: runs checks before git commit.
|
| 202 |
+
- PEP 8: 79-char lines, 4-space indent, snake_case functions, CamelCase classes."""
|
| 203 |
+
},
|
| 204 |
+
|
| 205 |
+
# ββ GIT ββ
|
| 206 |
+
{
|
| 207 |
+
"topic": "git_basics",
|
| 208 |
+
"tags": ["git", "version control", "commit", "branch", "merge", "clone"],
|
| 209 |
+
"content": """Git Basics:
|
| 210 |
+
- `git init`: initialize new repository.
|
| 211 |
+
- `git clone <url>`: copy remote repository.
|
| 212 |
+
- `git add <file>`: stage changes. `git add .` for all changes.
|
| 213 |
+
- `git commit -m "message"`: save staged changes.
|
| 214 |
+
- `git status`: see current state of working directory.
|
| 215 |
+
- `git log`: view commit history. `git log --oneline` for compact view.
|
| 216 |
+
- `git diff`: see unstaged changes. `git diff --staged` for staged changes.
|
| 217 |
+
- `git branch <name>`: create branch. `git checkout <name>`: switch branch.
|
| 218 |
+
- `git merge <branch>`: merge branch into current branch.
|
| 219 |
+
- `git push`: upload commits to remote. `git pull`: download and merge."""
|
| 220 |
+
},
|
| 221 |
+
|
| 222 |
+
# ββ FILE I/O ββ
|
| 223 |
+
{
|
| 224 |
+
"topic": "file_io",
|
| 225 |
+
"tags": ["python", "file", "io", "read", "write", "csv", "json", "open"],
|
| 226 |
+
"content": """Python File I/O:
|
| 227 |
+
- Read: `with open(path) as f: content = f.read()`.
|
| 228 |
+
- Read lines: `with open(path) as f: lines = f.readlines()`.
|
| 229 |
+
- Write: `with open(path, 'w') as f: f.write(content)`.
|
| 230 |
+
- Append: `with open(path, 'a') as f: f.write(content)`.
|
| 231 |
+
- CSV: `import csv`. `csv.reader(f)` to read. `csv.writer(f)` to write.
|
| 232 |
+
- JSON: `json.load(f)` to read. `json.dump(data, f)` to write. `json.dumps(obj)` to string.
|
| 233 |
+
- Binary: `open(path, 'rb')` / `open(path, 'wb')`.
|
| 234 |
+
- Path: use `os.path.join(dir, file)` for cross-platform paths.
|
| 235 |
+
- Directory: `os.makedirs(path, exist_ok=True)`. `os.listdir(path)`.
|
| 236 |
+
- Check exists: `os.path.exists(path)`. `os.path.isfile(path)`."""
|
| 237 |
+
},
|
| 238 |
+
|
| 239 |
+
# ββ SECURITY ββ
|
| 240 |
+
{
|
| 241 |
+
"topic": "security_basics",
|
| 242 |
+
"tags": ["security", "injection", "validation", "xss", "sql", "sanitize"],
|
| 243 |
+
"content": """Security Best Practices:
|
| 244 |
+
- Input validation: NEVER trust user input. Validate type, length, format.
|
| 245 |
+
- SQL injection: use parameterized queries, never string concatenation.
|
| 246 |
+
- XSS (Cross-Site Scripting): escape output in HTML context.
|
| 247 |
+
- eval() is DANGEROUS: use `ast.literal_eval()` for safe evaluation.
|
| 248 |
+
- exec() is DANGEROUS: avoid entirely. Use restricted environments if necessary.
|
| 249 |
+
- Password hashing: use bcrypt, argon2, or hashlib with salt.
|
| 250 |
+
- HTTPS: always use for network communication.
|
| 251 |
+
- Secret management: use environment variables, never hardcode secrets.
|
| 252 |
+
- Dependency scanning: check for known vulnerabilities (Dependabot).
|
| 253 |
+
- Principle of least privilege: give minimum necessary permissions."""
|
| 254 |
+
},
|
| 255 |
+
{
|
| 256 |
+
"topic": "security_sql_injection",
|
| 257 |
+
"tags": ["security", "sql", "injection", "parameterized", "prepared"],
|
| 258 |
+
"content": """SQL Injection Prevention:
|
| 259 |
+
- BAD: `f"SELECT * FROM users WHERE name = '{name}'"` β allows injection.
|
| 260 |
+
- GOOD: use parameterized queries: `cursor.execute("SELECT * FROM users WHERE name = ?", (name,))`.
|
| 261 |
+
- ORMs (SQLAlchemy, Django) auto-parameterize queries.
|
| 262 |
+
- Escape special characters if you must build strings.
|
| 263 |
+
- Use stored procedures for complex operations.
|
| 264 |
+
- Validate input: reject unexpected characters.
|
| 265 |
+
- Limit database permissions per application user.
|
| 266 |
+
- Monitor for injection attempts in logs."""
|
| 267 |
+
},
|
| 268 |
+
|
| 269 |
+
# ββ DESIGN PATTERNS ββ
|
| 270 |
+
{
|
| 271 |
+
"topic": "design_patterns_creational",
|
| 272 |
+
"tags": ["design patterns", "singleton", "factory", "builder", "prototype"],
|
| 273 |
+
"content": """Creational Design Patterns:
|
| 274 |
+
- Singleton: ensure class has exactly one instance. Module-level variable in Python.
|
| 275 |
+
- Factory: factory function returns objects without specifying exact class.
|
| 276 |
+
- Factory Method: subclass decides which class to instantiate.
|
| 277 |
+
- Abstract Factory: create families of related objects.
|
| 278 |
+
- Builder: separate construction from representation. Build step-by-step.
|
| 279 |
+
- Prototype: clone existing objects instead of creating new ones.
|
| 280 |
+
- Python-specific: use `__new__` for singleton. Use classmethods as factories."""
|
| 281 |
+
},
|
| 282 |
+
|
| 283 |
+
# ββ CONCURRENCY ββ
|
| 284 |
+
{
|
| 285 |
+
"topic": "python_concurrency",
|
| 286 |
+
"tags": ["python", "concurrent", "threading", "async", "multiprocessing", "asyncio"],
|
| 287 |
+
"content": """Python Concurrency:
|
| 288 |
+
- Threading: `import threading`. Thread(target=func, args=(...)). GIL limits CPU parallelism.
|
| 289 |
+
- Multiprocessing: `import multiprocessing`. Process(target=func). True parallelism.
|
| 290 |
+
- asyncio: `async def`, `await`, `asyncio.run()`. Cooperative multitasking for I/O.
|
| 291 |
+
- concurrent.futures: ThreadPoolExecutor for threads, ProcessPoolExecutor for processes.
|
| 292 |
+
- Lock: prevent race conditions. `threading.Lock()`. Use `with lock:`.
|
| 293 |
+
- Queue: thread-safe queue. `queue.Queue()`. For producer-consumer patterns.
|
| 294 |
+
- Use asyncio for: web servers, API calls, file I/O operations.
|
| 295 |
+
- Use multiprocessing for: CPU-heavy computations, data processing."""
|
| 296 |
+
},
|
| 297 |
+
|
| 298 |
+
# ββ WEB ββ
|
| 299 |
+
{
|
| 300 |
+
"topic": "web_requests",
|
| 301 |
+
"tags": ["web", "http", "requests", "api", "rest", "get", "post"],
|
| 302 |
+
"content": """HTTP Requests in Python:
|
| 303 |
+
- `import requests`. Install: `pip install requests`.
|
| 304 |
+
- GET: `r = requests.get(url, params={"key": "value"})`.
|
| 305 |
+
- POST: `r = requests.post(url, json={"key": "value"})`.
|
| 306 |
+
- Response: `r.status_code`, `r.json()`, `r.text`, `r.headers`.
|
| 307 |
+
- Timeout: `requests.get(url, timeout=5)` to avoid hanging.
|
| 308 |
+
- Error handling: `r.raise_for_status()` or check `r.status_code`.
|
| 309 |
+
- Headers: `requests.get(url, headers={"Authorization": "Bearer token"})`.
|
| 310 |
+
- Session: `s = requests.Session()` for persistent connections and cookies.
|
| 311 |
+
- Authentication: `requests.get(url, auth=("user", "pass"))`."""
|
| 312 |
+
},
|
| 313 |
+
|
| 314 |
+
# ββ DATA STRUCTURES ββ
|
| 315 |
+
{
|
| 316 |
+
"topic": "data_structures_advanced",
|
| 317 |
+
"tags": ["data structures", "heap", "queue", "stack", "tree", "graph", "hash"],
|
| 318 |
+
"content": """Advanced Python Data Structures:
|
| 319 |
+
- Heap (min): `import heapq`. `heapq.heappush(heap, item)`. `heapq.heappop(heap)`.
|
| 320 |
+
- Heap (max): push negative values, or use custom key with wrapper.
|
| 321 |
+
- Queue (FIFO): `from collections import deque`. `d.append()` / `d.popleft()`.
|
| 322 |
+
- Stack (LIFO): use list. `.append()` / `.pop()`.
|
| 323 |
+
- Priority Queue: `from queue import PriorityQueue` (thread-safe) or heapq.
|
| 324 |
+
- Default dict: `from collections import defaultdict(lambda: 0)`.
|
| 325 |
+
- Ordered dict: `from collections import OrderedDict` (Python 3.7+ dicts are ordered).
|
| 326 |
+
- Named tuple: `from collections import namedtuple`. `Point = namedtuple('Point', ['x', 'y'])`.
|
| 327 |
+
- ChainMap: combine multiple dicts.
|
| 328 |
+
- Counter: `from collections import Counter`. `Counter(list).most_common(3)`."""
|
| 329 |
+
},
|
| 330 |
+
|
| 331 |
+
# ββ TOOL CALL FORMAT ββ
|
| 332 |
+
{
|
| 333 |
+
"topic": "tool_call_format",
|
| 334 |
+
"tags": ["tool", "json", "format", "nanobot", "api", "function calling"],
|
| 335 |
+
"content": """Tool Call JSON Format:
|
| 336 |
+
- Write file: {"tool": "fs", "action": "write", "path": "filename.py", "content": "code here"}
|
| 337 |
+
- Read file: {"tool": "fs", "action": "read", "path": "filename.py"}
|
| 338 |
+
- List dir: {"tool": "fs", "action": "list", "path": "directory"}
|
| 339 |
+
- Check exists: {"tool": "fs", "action": "exists", "path": "filename.py"}
|
| 340 |
+
- Delete: {"tool": "fs", "action": "delete", "path": "filename.py"}
|
| 341 |
+
- Create dir: {"tool": "fs", "action": "mkdir", "path": "directory"}
|
| 342 |
+
- Compile check: {"tool": "compile", "path": "filename.py"}
|
| 343 |
+
- Run tests: {"tool": "test", "path": "directory"}
|
| 344 |
+
- Search code: {"tool": "search", "query": "pattern", "path": ".", "ext": ".py"}
|
| 345 |
+
- Run command: {"tool": "shell", "cmd": "command string"}
|
| 346 |
+
- Git init: {"tool": "git", "action": "init", "path": "."}
|
| 347 |
+
- Git commit: {"tool": "git", "action": "commit", "message": "commit msg"}
|
| 348 |
+
- Done: {"tool": "done"}
|
| 349 |
+
|
| 350 |
+
Always emit JSON on a single line. After the tool call, include THOUGHT: and RESULT: on new lines."""
|
| 351 |
+
},
|
| 352 |
+
]
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 356 |
+
# KNOWLEDGE LIBRARY
|
| 357 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 358 |
+
|
| 359 |
+
class KnowledgeLibrary:
|
| 360 |
+
"""Lightweight RAG β BM25 + keyword scoring. No external deps."""
|
| 361 |
+
|
| 362 |
+
def __init__(self, path=KNOWLEDGE_PATH):
|
| 363 |
+
self.path = path
|
| 364 |
+
self.entries = []
|
| 365 |
+
self.k1 = 1.5 # BM25 parameter
|
| 366 |
+
self.b = 0.75 # BM25 parameter
|
| 367 |
+
self._loaded = False
|
| 368 |
+
|
| 369 |
+
def load(self):
|
| 370 |
+
if self._loaded:
|
| 371 |
+
return
|
| 372 |
+
if os.path.exists(self.path):
|
| 373 |
+
with open(self.path) as f:
|
| 374 |
+
self.entries = json.load(f)
|
| 375 |
+
else:
|
| 376 |
+
self.entries = KNOWLEDGE_ENTRIES
|
| 377 |
+
self.save()
|
| 378 |
+
self._loaded = True
|
| 379 |
+
# Build word frequency stats for BM25
|
| 380 |
+
self._build_stats()
|
| 381 |
+
|
| 382 |
+
def save(self):
|
| 383 |
+
with open(self.path, 'w') as f:
|
| 384 |
+
json.dump(self.entries, f, indent=2)
|
| 385 |
+
|
| 386 |
+
def _build_stats(self):
|
| 387 |
+
self.doc_lengths = [len(e["content"].split()) for e in self.entries]
|
| 388 |
+
self.avg_doc_len = sum(self.doc_lengths) / max(1, len(self.doc_lengths))
|
| 389 |
+
self.N = len(self.entries)
|
| 390 |
+
# Document frequency: how many docs contain each word
|
| 391 |
+
self.df = Counter()
|
| 392 |
+
for entry in self.entries:
|
| 393 |
+
words = set(self._tokenize(entry["content"]) + self._tokenize(entry["topic"]) + sum((self._tokenize(t) for t in entry.get("tags", [])), []))
|
| 394 |
+
for w in words:
|
| 395 |
+
self.df[w] += 1
|
| 396 |
+
|
| 397 |
+
def _tokenize(self, text):
|
| 398 |
+
return re.findall(r'[a-z0-9_]+', text.lower())
|
| 399 |
+
|
| 400 |
+
def _bm25_score(self, query_tokens, doc_idx):
|
| 401 |
+
"""Score a single document against query tokens using BM25."""
|
| 402 |
+
entry = self.entries[doc_idx]
|
| 403 |
+
doc_tokens = self._tokenize(entry["content"]) + self._tokenize(entry["topic"])
|
| 404 |
+
doc_len = self.doc_lengths[doc_idx]
|
| 405 |
+
doc_tf = Counter(doc_tokens)
|
| 406 |
+
|
| 407 |
+
score = 0.0
|
| 408 |
+
for token in query_tokens:
|
| 409 |
+
if token not in doc_tf:
|
| 410 |
+
continue
|
| 411 |
+
tf = doc_tf[token]
|
| 412 |
+
df = self.df.get(token, 0)
|
| 413 |
+
# IDF
|
| 414 |
+
idf = math.log((self.N - df + 0.5) / (df + 0.5) + 1.0)
|
| 415 |
+
# BM25 TF component
|
| 416 |
+
tf_component = (tf * (self.k1 + 1)) / (tf + self.k1 * (1 - self.b + self.b * doc_len / self.avg_doc_len))
|
| 417 |
+
score += idf * tf_component
|
| 418 |
+
|
| 419 |
+
# Bonus for tag matches
|
| 420 |
+
query_set = set(query_tokens)
|
| 421 |
+
tag_tokens = set()
|
| 422 |
+
for tag in entry.get("tags", []):
|
| 423 |
+
tag_tokens.update(self._tokenize(tag))
|
| 424 |
+
tag_match = len(query_set & tag_tokens)
|
| 425 |
+
score += tag_match * 2.0 # Tag matches are strong signals
|
| 426 |
+
|
| 427 |
+
# Bonus for topic name match
|
| 428 |
+
topic_tokens = set(self._tokenize(entry["topic"]))
|
| 429 |
+
topic_match = len(query_set & topic_tokens)
|
| 430 |
+
score += topic_match * 3.0
|
| 431 |
+
|
| 432 |
+
return score
|
| 433 |
+
|
| 434 |
+
def query(self, question, top_k=3):
|
| 435 |
+
"""Search the knowledge library for relevant entries."""
|
| 436 |
+
self.load()
|
| 437 |
+
query_tokens = self._tokenize(question)
|
| 438 |
+
|
| 439 |
+
if not query_tokens:
|
| 440 |
+
return []
|
| 441 |
+
|
| 442 |
+
scores = []
|
| 443 |
+
for i in range(len(self.entries)):
|
| 444 |
+
score = self._bm25_score(query_tokens, i)
|
| 445 |
+
if score > 0:
|
| 446 |
+
scores.append((score, i))
|
| 447 |
+
|
| 448 |
+
scores.sort(reverse=True)
|
| 449 |
+
results = []
|
| 450 |
+
for score, idx in scores[:top_k]:
|
| 451 |
+
entry = self.entries[idx]
|
| 452 |
+
results.append({
|
| 453 |
+
"topic": entry["topic"],
|
| 454 |
+
"score": round(score, 2),
|
| 455 |
+
"content": entry["content"],
|
| 456 |
+
"tags": entry.get("tags", []),
|
| 457 |
+
})
|
| 458 |
+
|
| 459 |
+
return results
|
| 460 |
+
|
| 461 |
+
def add_entry(self, topic, content, tags):
|
| 462 |
+
"""Add a new knowledge entry (e.g., from web search results)."""
|
| 463 |
+
self.load()
|
| 464 |
+
entry = {"topic": topic, "content": content, "tags": tags}
|
| 465 |
+
self.entries.append(entry)
|
| 466 |
+
self.save()
|
| 467 |
+
self._loaded = False # Rebuild stats on next query
|
| 468 |
+
return len(self.entries)
|
| 469 |
+
|
| 470 |
+
def list_topics(self):
|
| 471 |
+
self.load()
|
| 472 |
+
return [e["topic"] for e in self.entries]
|
| 473 |
+
|
| 474 |
+
def stats(self):
|
| 475 |
+
self.load()
|
| 476 |
+
return {
|
| 477 |
+
"total_entries": len(self.entries),
|
| 478 |
+
"avg_doc_len": round(self.avg_doc_len, 1),
|
| 479 |
+
"unique_terms": len(self.df),
|
| 480 |
+
}
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 484 |
+
# QUERY TRACKER β tracks what the model learned
|
| 485 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 486 |
+
|
| 487 |
+
class QueryTracker:
|
| 488 |
+
"""Tracks library queries β shows knowledge gaps and learning progress."""
|
| 489 |
+
|
| 490 |
+
def __init__(self, path="library_queries.jsonl"):
|
| 491 |
+
self.path = path
|
| 492 |
+
self.queries = self._load()
|
| 493 |
+
|
| 494 |
+
def _load(self):
|
| 495 |
+
if os.path.exists(self.path):
|
| 496 |
+
with open(self.path) as f:
|
| 497 |
+
return [json.loads(l) for l in f if l.strip()]
|
| 498 |
+
return []
|
| 499 |
+
|
| 500 |
+
def log(self, task, question, results_used, success):
|
| 501 |
+
entry = {
|
| 502 |
+
"task": task[:200],
|
| 503 |
+
"question": question[:200],
|
| 504 |
+
"topics_used": [r["topic"] for r in results_used],
|
| 505 |
+
"success": success,
|
| 506 |
+
}
|
| 507 |
+
self.queries.append(entry)
|
| 508 |
+
with open(self.path, 'a') as f:
|
| 509 |
+
f.write(json.dumps(entry) + "\n")
|
| 510 |
+
|
| 511 |
+
def gap_analysis(self):
|
| 512 |
+
"""Find which topics the model queries most often β those are gaps."""
|
| 513 |
+
topics = Counter()
|
| 514 |
+
for q in self.queries:
|
| 515 |
+
for t in q.get("topics_used", []):
|
| 516 |
+
topics[t] += 1
|
| 517 |
+
return topics.most_common()
|
| 518 |
+
|
| 519 |
+
def learning_curve(self, window=20):
|
| 520 |
+
"""Track if library dependency is decreasing over time."""
|
| 521 |
+
if len(self.queries) < window:
|
| 522 |
+
return None
|
| 523 |
+
recent = self.queries[-window:]
|
| 524 |
+
earlier = self.queries[-window*2:-window] if len(self.queries) >= window*2 else self.queries[:window]
|
| 525 |
+
recent_rate = sum(1 for q in recent if q.get("topics_used")) / len(recent)
|
| 526 |
+
earlier_rate = sum(1 for q in earlier if q.get("topics_used")) / max(1, len(earlier))
|
| 527 |
+
return {
|
| 528 |
+
"recent_query_rate": recent_rate,
|
| 529 |
+
"earlier_query_rate": earlier_rate,
|
| 530 |
+
"trend": "decreasing" if recent_rate < earlier_rate else "increasing" if recent_rate > earlier_rate else "stable",
|
| 531 |
+
}
|
| 532 |
+
|
| 533 |
+
|
| 534 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 535 |
+
# MAIN / TEST
|
| 536 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 537 |
+
|
| 538 |
+
if __name__ == "__main__":
|
| 539 |
+
lib = KnowledgeLibrary()
|
| 540 |
+
lib.load()
|
| 541 |
+
print(f"Knowledge Library: {lib.stats()}")
|
| 542 |
+
|
| 543 |
+
# Test queries
|
| 544 |
+
test_queries = [
|
| 545 |
+
"how do I handle file I/O errors in Python?",
|
| 546 |
+
"what is the format for writing a file with the fs tool?",
|
| 547 |
+
"how do I prevent SQL injection?",
|
| 548 |
+
"what is list comprehension syntax?",
|
| 549 |
+
"how does pytest discover tests?",
|
| 550 |
+
]
|
| 551 |
+
|
| 552 |
+
for q in test_queries:
|
| 553 |
+
print(f"\n Query: {q}")
|
| 554 |
+
results = lib.query(q, top_k=2)
|
| 555 |
+
for r in results:
|
| 556 |
+
content_preview = r["content"][:120].replace("\n", " ")
|
| 557 |
+
print(f" [{r['score']}] {r['topic']}: {content_preview}...")
|