Spaces:
Sleeping
Sleeping
File size: 1,953 Bytes
6960b79 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | import re
def clean_code(code):
"""
Basic code cleaning: removes empty lines and trailing whitespace.
"""
lines = code.split('\n')
cleaned_lines = [line.rstrip() for line in lines if line.strip()]
return '\n'.join(cleaned_lines)
def create_pairs(data):
"""
Extracts vulnerable and secure code pairs.
Returns a list of tuples: (vulnerable_code, secure_code)
"""
pairs = []
for item in data:
vuln = item.get('vulnerable_code', '')
secure = item.get('secure_code', '')
if vuln and secure:
pairs.append((vuln, secure))
return pairs
def prepare_training_data(data):
"""
Formats data for LLM Fine-tuning (Prompt-Response format).
"""
training_samples = []
for item in data:
vuln_code = item.get('vulnerable_code', '')
secure_code = item.get('secure_code', '')
prompt = f"### Instruction:\nFix the following vulnerable code securely:\n\n### Input:\n{vuln_code}\n\n### Response:\n"
completion = f"{secure_code}<|endoftext|>"
training_samples.append({
"prompt": prompt,
"completion": completion
})
return training_samples
import hashlib
def check_data_leakage(train_set, test_set):
"""
Checks if any code snippets from the train set appear in the test set.
"""
train_hashes = {hashlib.md5(item['code'].strip().encode()).hexdigest() for item in train_set}
leaked_count = 0
for item in test_set:
test_hash = hashlib.md5(item['code'].strip().encode()).hexdigest()
if test_hash in train_hashes:
leaked_count += 1
return leaked_count == 0, leaked_count
def tokenize_data(tokenizer, data_samples, max_length=512):
"""
Tokenizes data for model training.
(Requires a tokenizer from transformers)
"""
# Placeholder for actual tokenization logic
pass
|