Harvester / code_whitelist.py
Zoberzzz's picture
Upload Harvester files
9e3b4b9 verified
Raw
History Blame Contribute Delete
29.5 kB
"""
MULTI-LANGUAGE CODE WHITELIST
==============================
Comprehensive safe-patterns whitelist for code generation and validation.
Covers: Python, C#, C++, JavaScript/TypeScript, Java, Go, Rust.
Two layers:
1. BLOCKLIST β€” patterns that must NEVER appear in generated code
2. WHITELIST β€” known-safe standard library calls, patterns, and modules
Usage:
from code_whitelist import validate_code, get_safe_patterns, LANGUAGES
result = validate_code(code, language="python")
# result = {"safe": True/False, "blocked": [...], "warnings": [...]}
patterns = get_safe_patterns("csharp")
# returns dict of safe modules, functions, patterns for that language
"""
import re
from dataclasses import dataclass, field
from typing import Optional
# ══════════════════════════════════════════════════════════════════════════════
# SUPPORTED LANGUAGES
# ══════════════════════════════════════════════════════════════════════════════
LANGUAGES = ["python", "csharp", "cpp", "javascript", "typescript", "java", "go", "rust"]
# ══════════════════════════════════════════════════════════════════════════════
# UNIVERSAL BLOCKLIST β€” dangerous patterns across ALL languages
# ══════════════════════════════════════════════════════════════════════════════
UNIVERSAL_BLOCKS = {
"shell_injection": [
r'\bos\.system\b',
r'\bos\.popen\b',
r'\bsubprocess\.call\b(?!\(.*shell\s*=\s*False)',
r'Runtime\.getRuntime\(\)\.exec\b',
r'\bProcess\.Start\b',
r'\bsystem\s*\(',
r'\bexecvp?\b\s*\(',
r'\bpopen\s*\(',
r'child_process\.exec\b(?!File)',
r'\bexec\.Command\b', # Go β€” flag for review, not always bad
],
"code_injection": [
r'\beval\s*\(',
r'\bexec\s*\(',
r'__import__\s*\(',
r'\bFunction\s*\(', # JS new Function()
r'setTimeout\s*\(\s*["\']', # JS string-based setTimeout
r'setInterval\s*\(\s*["\']',
r'\bCompile\s*\(.*\)\.Run', # C# dynamic compilation
],
"file_destruction": [
r'shutil\.rmtree\s*\(',
r'\.unlink\s*\(',
r'\brm\s+-rf\b',
r'Directory\.Delete\s*\(.*true',
r'removeSync\s*\(',
r'os\.RemoveAll\s*\(',
r'std::filesystem::remove_all\b',
],
"sql_injection": [
r'f["\'].*SELECT.*FROM.*\{', # Python f-string SQL
r'f["\'].*INSERT.*INTO.*\{',
r'f["\'].*UPDATE.*SET.*\{',
r'f["\'].*DELETE.*FROM.*\{',
r'\+\s*["\'].*SELECT', # String concat SQL
r'format!\s*\(.*SELECT', # Rust format! SQL
r'fmt\.Sprintf\s*\(.*SELECT', # Go sprintf SQL
r'String\.Format\s*\(.*SELECT', # C# string.Format SQL
],
"xss_risk": [
r'innerHTML\s*=',
r'document\.write\s*\(',
r'\.outerHTML\s*=',
r'dangerouslySetInnerHTML',
],
"crypto_misuse": [
r'\bMD5\b',
r'\bSHA1\b(?!_)',
r'\bDES\b',
r'\bRC4\b',
r'random\.random\(\).*(?:password|key|token|secret)',
r'Math\.random\(\).*(?:password|key|token|secret)',
],
"hardcoded_secrets": [
r'(?:password|secret|api_key|token)\s*=\s*["\'][^"\']{8,}["\']',
],
}
# ══════════════════════════════════════════════════════════════════════════════
# PER-LANGUAGE BLOCKLISTS
# ══════════════════════════════════════════════════════════════════════════════
LANGUAGE_BLOCKS = {
"python": [
r'\bpickle\.loads?\b', # Arbitrary code execution via deserialization
r'\byaml\.load\b(?!\(.*Loader)', # Unsafe YAML load
r'\bmarshal\.loads?\b',
r'__builtins__',
r'\bglobals\s*\(\)',
r'\bcompile\s*\(',
],
"csharp": [
r'SqlCommand\s*\(.*\+', # String concat SQL
r'BinaryFormatter', # Insecure deserialization
r'Process\.Start\s*\(',
r'\bunsafe\b', # unsafe code blocks
r'DllImport', # P/Invoke β€” flag for review
r'Assembly\.Load\b',
],
"cpp": [
r'\bgets\s*\(', # Buffer overflow
r'\bstrcpy\s*\(', # No bounds checking
r'\bstrcat\s*\(', # No bounds checking
r'\bsprintf\s*\(', # Use snprintf instead
r'\bmalloc\b.*\bfree\b', # Flag raw malloc (prefer smart ptrs)
r'#pragma\s+warning\s*\(\s*disable',
r'\breinterpret_cast\b',
r'\bvoid\s*\*', # Void pointers β€” flag for review
],
"javascript": [
r'require\s*\(\s*["\']child_process',
r'\.call\s*\(.*arguments\b',
r'with\s*\(', # with statement
r'document\.cookie\b',
r'window\.location\s*=',
],
"typescript": [
r'as\s+any\b', # Type escape hatch β€” flag
r'@ts-ignore',
r'require\s*\(\s*["\']child_process',
r'document\.cookie\b',
],
"java": [
r'ObjectInputStream\b', # Deserialization risk
r'Runtime\.exec\b',
r'ProcessBuilder\b',
r'Class\.forName\s*\(', # Reflection β€” flag
r'\.getMethod\s*\(',
r'Statement\b.*execute\b', # Raw SQL statements
],
"go": [
r'os/exec\b', # Flag for review
r'unsafe\.Pointer\b',
r'reflect\.(?:Value|Type)', # Heavy reflection β€” flag
r'cgo\b', # CGo boundary
],
"rust": [
r'\bunsafe\s*\{',
r'std::mem::transmute\b',
r'from_raw_parts\b',
r'\b\.unwrap\(\)', # Panics in production β€” warn
],
}
# ══════════════════════════════════════════════════════════════════════════════
# PER-LANGUAGE WHITELISTS β€” safe standard library modules, functions, patterns
# ══════════════════════════════════════════════════════════════════════════════
SAFE_PATTERNS = {
"python": {
"safe_modules": [
"collections", "itertools", "functools", "operator",
"math", "statistics", "decimal", "fractions",
"datetime", "calendar", "time",
"json", "csv", "io", "os.path", "pathlib",
"re", "string", "textwrap", "unicodedata",
"typing", "dataclasses", "enum", "abc",
"hashlib", "hmac", "secrets",
"logging", "unittest", "pytest",
"copy", "pprint", "bisect", "heapq",
"contextlib", "weakref",
"argparse", "configparser",
"sqlite3", # with parameterized queries
"urllib.parse",
"base64", "binascii",
"struct", "array",
"queue", "threading.Lock", "threading.Event",
"asyncio", "concurrent.futures",
],
"safe_builtins": [
"len", "range", "enumerate", "zip", "map", "filter",
"sorted", "reversed", "min", "max", "sum", "abs",
"all", "any", "isinstance", "issubclass", "type",
"int", "float", "str", "bool", "list", "dict", "set", "tuple",
"frozenset", "bytes", "bytearray", "memoryview",
"print", "input", "repr", "format", "hash", "id",
"getattr", "setattr", "hasattr", "delattr",
"property", "staticmethod", "classmethod",
"super", "iter", "next", "chr", "ord",
"round", "divmod", "pow",
"open", # reading only β€” write mode blocked separately
],
"safe_patterns": [
"list comprehension",
"dict comprehension",
"generator expression",
"context manager (with statement)",
"dataclass",
"type hints / annotations",
"f-string formatting (non-SQL)",
"try/except/finally",
"decorator pattern",
"property getter/setter",
"@functools.lru_cache",
"@functools.wraps",
"collections.defaultdict",
"collections.Counter",
"collections.namedtuple",
"pathlib.Path operations",
"logging.getLogger",
"argparse.ArgumentParser",
"parameterized SQL queries (?, %s placeholders)",
],
"safe_frameworks": [
"flask", "fastapi", "django",
"requests", "httpx",
"sqlalchemy", # with ORM / parameterized
"pydantic",
"pytest", "unittest",
"numpy", "pandas",
"click", "typer",
],
},
"csharp": {
"safe_modules": [
"System", "System.Collections.Generic", "System.Linq",
"System.Text", "System.Text.RegularExpressions",
"System.IO", "System.IO.Path",
"System.Threading", "System.Threading.Tasks",
"System.Net.Http", "System.Net.Http.Json",
"System.Text.Json", "System.Text.Json.Serialization",
"System.Security.Cryptography",
"System.Diagnostics.Debug",
"System.ComponentModel.DataAnnotations",
"System.Globalization",
"System.Math",
"Microsoft.Extensions.Logging",
"Microsoft.Extensions.DependencyInjection",
"Microsoft.Extensions.Configuration",
"Microsoft.EntityFrameworkCore",
],
"safe_patterns": [
"using statement / IDisposable",
"async/await with Task",
"LINQ queries",
"record types",
"pattern matching (switch expressions)",
"nullable reference types",
"string interpolation (non-SQL)",
"dependency injection",
"ILogger<T>",
"parameterized SQL (@param)",
"Entity Framework LINQ queries",
"try/catch/finally",
"readonly / init-only properties",
"sealed classes",
"interfaces and abstract classes",
"generic constraints",
"ValueTask for hot paths",
"Span<T> / Memory<T>",
"IAsyncEnumerable<T>",
"CancellationToken",
],
"safe_frameworks": [
"ASP.NET Core", "Entity Framework Core",
"xUnit", "NUnit", "MSTest",
"Serilog", "MediatR",
"FluentValidation",
"AutoMapper",
"Polly (resilience)",
],
},
"cpp": {
"safe_modules": [
"<algorithm>", "<numeric>", "<functional>",
"<vector>", "<array>", "<string>", "<string_view>",
"<map>", "<unordered_map>", "<set>", "<unordered_set>",
"<queue>", "<stack>", "<deque>", "<list>",
"<memory>", # smart pointers
"<optional>", "<variant>", "<any>", "<tuple>",
"<chrono>", "<cmath>", "<cstdint>",
"<iostream>", "<sstream>", "<fstream>",
"<regex>", "<filesystem>",
"<thread>", "<mutex>", "<condition_variable>",
"<atomic>", "<future>",
"<stdexcept>", "<cassert>",
"<ranges>", "<concepts>",
"<format>", # C++20
"<span>", # C++20
],
"safe_patterns": [
"std::unique_ptr / std::shared_ptr",
"std::make_unique / std::make_shared",
"RAII (constructor/destructor pairs)",
"range-based for loops",
"auto type deduction",
"const references (const T&)",
"constexpr",
"structured bindings (auto [a, b] = ...)",
"std::optional for nullable values",
"std::string_view for non-owning strings",
"std::move semantics",
"static_cast (not reinterpret_cast)",
"try/catch with std::exception",
"std::algorithm (sort, find, transform)",
"lambda expressions",
"templates and concepts (C++20)",
"enum class (scoped enums)",
"snprintf instead of sprintf",
"std::array instead of C arrays",
"std::vector instead of raw new[]",
],
"safe_libraries": [
"Boost (selected modules)",
"fmt", "spdlog",
"Catch2", "Google Test",
"nlohmann/json",
"abseil-cpp",
],
},
"javascript": {
"safe_modules": [
"Array", "Object", "Map", "Set", "WeakMap", "WeakSet",
"Promise", "JSON", "Math", "Date", "RegExp",
"String", "Number", "Symbol", "BigInt",
"URL", "URLSearchParams",
"TextEncoder", "TextDecoder",
"structuredClone",
"console",
"fetch",
"AbortController",
"crypto.subtle", # Web Crypto API
"crypto.randomUUID",
],
"safe_node_modules": [
"path", "url", "util", "events",
"crypto", # Node crypto
"fs/promises", # async file ops
"stream", "buffer",
"assert", "test", # Node test runner
"zlib", "querystring",
],
"safe_patterns": [
"const / let (no var)",
"arrow functions",
"template literals (non-SQL)",
"destructuring (object / array)",
"spread / rest operators",
"Promise.all / Promise.allSettled",
"async/await",
"optional chaining (?.) ",
"nullish coalescing (??)",
"Array methods (map, filter, reduce, find, some, every)",
"Object.entries / Object.keys / Object.values",
"try/catch/finally",
"class syntax",
"modules (import/export)",
"for...of loops",
"Map/Set for collections",
"structuredClone for deep copy",
"fetch with AbortController",
"parameterized queries (prepared statements)",
],
"safe_frameworks": [
"React", "Vue", "Svelte",
"Express (with helmet, cors)",
"Fastify",
"Jest", "Vitest", "Mocha",
"Zod (validation)",
"Prisma (ORM)",
"TypeORM", "Knex",
],
},
"typescript": {
"inherits": "javascript",
"safe_patterns": [
"strict mode (strict: true in tsconfig)",
"interface definitions",
"type aliases",
"generic types",
"discriminated unions",
"type guards (is / in / typeof / instanceof)",
"readonly modifier",
"Record<K, V> / Partial<T> / Required<T>",
"Pick<T, K> / Omit<T, K>",
"unknown over any",
"satisfies operator",
"const assertions (as const)",
"enum (prefer const enum or union types)",
"Zod / io-ts for runtime validation",
],
},
"java": {
"safe_modules": [
"java.util.*",
"java.util.stream.*",
"java.util.concurrent.*",
"java.util.function.*",
"java.time.*",
"java.math.*",
"java.lang.Math",
"java.lang.String",
"java.lang.StringBuilder",
"java.io.BufferedReader", "java.io.BufferedWriter",
"java.nio.file.Path", "java.nio.file.Files",
"java.security.MessageDigest",
"java.security.SecureRandom",
"java.text.MessageFormat",
"java.util.logging.*",
"java.util.regex.*",
"java.net.URI", "java.net.http.HttpClient",
],
"safe_patterns": [
"try-with-resources",
"Optional<T>",
"Stream API (map, filter, collect, reduce)",
"records (Java 14+)",
"sealed classes (Java 17+)",
"pattern matching for instanceof (Java 16+)",
"switch expressions (Java 14+)",
"text blocks (Java 15+)",
"var (local variable type inference)",
"CompletableFuture",
"PreparedStatement (parameterized SQL)",
"Collections.unmodifiableList / List.of / Map.of",
"interface default methods",
"lambda expressions",
"method references (::)",
"enum with methods",
"builder pattern",
"dependency injection (@Inject)",
],
"safe_frameworks": [
"Spring Boot", "Spring Security",
"JUnit 5", "Mockito", "AssertJ",
"Jackson", "Gson",
"SLF4J / Logback",
"Hibernate / JPA",
"Lombok",
"MapStruct",
],
},
"go": {
"safe_modules": [
"fmt", "strings", "strconv", "unicode",
"math", "math/big", "math/rand",
"sort", "slices", # Go 1.21+
"maps", # Go 1.21+
"errors", "log", "log/slog",
"io", "bufio", "bytes",
"os", "path", "path/filepath",
"encoding/json", "encoding/csv", "encoding/base64",
"net/http", "net/url",
"context",
"sync", "sync/atomic",
"time",
"regexp",
"crypto/sha256", "crypto/hmac", "crypto/rand",
"testing",
"embed",
"database/sql", # with parameterized queries
],
"safe_patterns": [
"error handling (if err != nil)",
"defer for cleanup",
"goroutines with sync.WaitGroup",
"channels for communication",
"select statement",
"context.Context for cancellation",
"interfaces (implicit satisfaction)",
"struct embedding (composition)",
"table-driven tests",
"functional options pattern",
"type assertions with ok check",
"range loops",
"iota for enums",
"init() functions",
"database/sql with $1 or ? placeholders",
"http.HandlerFunc / middleware chain",
"slog structured logging",
"generics (Go 1.18+)",
],
"safe_frameworks": [
"Gin", "Chi", "Echo",
"GORM", "sqlx",
"testify",
"Wire (DI)",
"Cobra (CLI)",
"Viper (config)",
],
},
"rust": {
"safe_modules": [
"std::collections (HashMap, BTreeMap, Vec, VecDeque, HashSet)",
"std::string::String",
"std::vec::Vec",
"std::io (Read, Write, BufReader, BufWriter)",
"std::fs",
"std::path (Path, PathBuf)",
"std::fmt",
"std::iter",
"std::convert (From, Into, TryFrom, TryInto)",
"std::ops",
"std::cmp (Ordering, min, max)",
"std::time (Duration, Instant)",
"std::thread",
"std::sync (Arc, Mutex, RwLock, mpsc)",
"std::error::Error",
"std::result::Result",
"std::option::Option",
"std::num",
],
"safe_patterns": [
"ownership and borrowing",
"pattern matching (match)",
"Result<T, E> for error handling",
"Option<T> for nullable values",
"? operator for error propagation",
"impl blocks and traits",
"derive macros (#[derive(Debug, Clone, PartialEq)])",
"iterators (.map, .filter, .collect, .fold)",
"closures (|x| ...)",
"enum with data variants",
"struct with impl",
"lifetime annotations where needed",
"Arc<Mutex<T>> for shared state",
"async/await with tokio",
"type aliases",
"const generics",
"builder pattern",
"#[cfg(test)] mod tests",
],
"safe_crates": [
"serde / serde_json",
"tokio / async-std",
"reqwest",
"clap (CLI)",
"anyhow / thiserror",
"tracing / log",
"sqlx",
"axum / actix-web / warp",
"rand",
],
},
}
# ══════════════════════════════════════════════════════════════════════════════
# VALIDATION ENGINE
# ══════════════════════════════════════════════════════════════════════════════
@dataclass
class ValidationResult:
safe: bool
language: str
blocked: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
score: float = 100.0 # 100 = perfectly safe, 0 = dangerous
def _check_blocks(code: str, patterns: list, category: str,
severity: str = "block") -> tuple[list[str], list[str]]:
"""Check code against a list of regex patterns. Returns (blocks, warnings)."""
blocks = []
warnings = []
for pattern in patterns:
if re.search(pattern, code, re.IGNORECASE):
msg = f"[{category}] matched: {pattern}"
if severity == "block":
blocks.append(msg)
else:
warnings.append(msg)
return blocks, warnings
def validate_code(code: str, language: str = "python") -> ValidationResult:
"""
Validate code against blocklist and whitelist.
Returns a ValidationResult with safety assessment.
"""
language = language.lower().strip()
if language in ("ts", "tsx"):
language = "typescript"
elif language in ("js", "jsx"):
language = "javascript"
elif language in ("cs", "c#"):
language = "csharp"
elif language in ("c++", "cc", "cxx", "hpp"):
language = "cpp"
elif language in ("rs",):
language = "rust"
elif language in ("py",):
language = "python"
result = ValidationResult(safe=True, language=language)
# 1. Check universal blocklist
for category, patterns in UNIVERSAL_BLOCKS.items():
blocks, warnings = _check_blocks(code, patterns, category)
result.blocked.extend(blocks)
result.warnings.extend(warnings)
# 2. Check language-specific blocklist
lang_blocks = LANGUAGE_BLOCKS.get(language, [])
blocks, warnings = _check_blocks(code, lang_blocks, f"{language}_specific")
result.blocked.extend(blocks)
result.warnings.extend(warnings)
# 3. For TypeScript, also check JavaScript blocks
if language == "typescript":
js_blocks = LANGUAGE_BLOCKS.get("javascript", [])
blocks, warnings = _check_blocks(code, js_blocks, "javascript_specific")
result.blocked.extend(blocks)
result.warnings.extend(warnings)
# 4. Calculate safety score
result.score = max(0, 100 - (len(result.blocked) * 25) - (len(result.warnings) * 5))
result.safe = len(result.blocked) == 0
return result
def get_safe_patterns(language: str) -> dict:
"""Get the whitelist of safe patterns for a specific language."""
language = language.lower().strip()
alias_map = {
"ts": "typescript", "tsx": "typescript",
"js": "javascript", "jsx": "javascript",
"cs": "csharp", "c#": "csharp",
"c++": "cpp", "cc": "cpp", "cxx": "cpp",
"rs": "rust", "py": "python",
}
language = alias_map.get(language, language)
patterns = SAFE_PATTERNS.get(language, {})
# TypeScript inherits JavaScript patterns
if language == "typescript" and patterns.get("inherits") == "javascript":
js_patterns = SAFE_PATTERNS.get("javascript", {})
merged = {**js_patterns, **patterns}
del merged["inherits"]
return merged
return patterns
def get_blocklist(language: str) -> dict:
"""Get all blocked patterns (universal + language-specific) for a language."""
language = language.lower().strip()
alias_map = {
"ts": "typescript", "tsx": "typescript",
"js": "javascript", "jsx": "javascript",
"cs": "csharp", "c#": "csharp",
"c++": "cpp", "cc": "cpp", "cxx": "cpp",
"rs": "rust", "py": "python",
}
language = alias_map.get(language, language)
return {
"universal": UNIVERSAL_BLOCKS,
"language_specific": LANGUAGE_BLOCKS.get(language, []),
}
# ══════════════════════════════════════════════════════════════════════════════
# CLI β€” quick testing
# ══════════════════════════════════════════════════════════════════════════════
if __name__ == "__main__":
import sys
# Demo: validate some sample code
test_samples = {
"python_safe": (
"python",
'from collections import Counter\ndata = Counter([1,2,3,1,2,1])\nprint(data.most_common())'
),
"python_dangerous": (
"python",
'import os\nos.system("rm -rf /")\neval(input())'
),
"csharp_safe": (
"csharp",
'using System.Linq;\nvar nums = new[] {1,2,3};\nvar sum = nums.Sum();'
),
"csharp_dangerous": (
"csharp",
'var cmd = new SqlCommand("SELECT * FROM users WHERE id=" + userId);'
),
"cpp_safe": (
"cpp",
'#include <vector>\n#include <algorithm>\nauto v = std::vector{3,1,2};\nstd::sort(v.begin(), v.end());'
),
"cpp_dangerous": (
"cpp",
'char buf[10];\ngets(buf);\nstrcpy(dest, src);'
),
"js_safe": (
"javascript",
'const items = [1,2,3];\nconst doubled = items.map(x => x * 2);\nconsole.log(doubled);'
),
"js_dangerous": (
"javascript",
'const code = req.body.code;\neval(code);\ndocument.innerHTML = userInput;'
),
"sql_injection": (
"python",
'query = f"SELECT * FROM users WHERE name = \'{user_input}\'"'
),
}
print("=" * 70)
print("MULTI-LANGUAGE CODE WHITELIST β€” Validation Demo")
print("=" * 70)
for name, (lang, code) in test_samples.items():
result = validate_code(code, lang)
status = "SAFE" if result.safe else "BLOCKED"
print(f"\n[{status}] {name} ({lang}) β€” score: {result.score}")
if result.blocked:
for b in result.blocked:
print(f" BLOCK: {b}")
if result.warnings:
for w in result.warnings:
print(f" WARN: {w}")
print(f"\n{'=' * 70}")
print(f"Supported languages: {', '.join(LANGUAGES)}")
print(f"Universal block categories: {len(UNIVERSAL_BLOCKS)}")
total_blocks = sum(len(v) for v in LANGUAGE_BLOCKS.values())
print(f"Language-specific block patterns: {total_blocks}")
total_safe = sum(
sum(len(v) for v in lang.values() if isinstance(v, list))
for lang in SAFE_PATTERNS.values()
if isinstance(lang, dict)
)
print(f"Safe patterns/modules catalogued: {total_safe}")