|
|
from cipher_8bit import *
|
|
|
from french_dataset import get_full_dataset
|
|
|
|
|
|
def get_frequency_ranks(encodings, symbols, sequence_len):
|
|
|
freq_ranks_dict = [0] * len(symbols)
|
|
|
encodings = encodings[:sequence_len]
|
|
|
for encoding in encodings:
|
|
|
freq_ranks_dict[encoding] += 1
|
|
|
freq_ranks = [0] * (sequence_len)
|
|
|
|
|
|
for i in range(len(encodings)):
|
|
|
freq_ranks[i] = freq_ranks_dict[encodings[i]]
|
|
|
return freq_ranks
|
|
|
|
|
|
def get_proximity_array(encodings, sequence_len):
|
|
|
distances = [0] * (sequence_len)
|
|
|
encodings = encodings[:sequence_len]
|
|
|
for i, encoding in enumerate(encodings):
|
|
|
try:
|
|
|
last_idx = encodings.index(encoding, 0, i)
|
|
|
distances[i] = (i - last_idx)
|
|
|
except ValueError:
|
|
|
|
|
|
distances[i] = 0
|
|
|
return distances
|
|
|
|
|
|
def preprocess_text(sequence_len=256):
|
|
|
full_text = get_full_dataset()
|
|
|
symbols = get_symbols(full_text, 256)
|
|
|
symbols = load_or_save_symbols(symbols)
|
|
|
substitution_rule = substitution_cipher(symbols, 1337)
|
|
|
|
|
|
i = 0
|
|
|
raw_length = sequence_len * 2
|
|
|
processed_data = []
|
|
|
while i * raw_length < len(full_text) - raw_length:
|
|
|
i += 1
|
|
|
sample_text = full_text[(i - 1) * raw_length: i * raw_length - 1]
|
|
|
encodings_array, indices = encode_text_with_indices(substitution_rule, symbols, sample_text)
|
|
|
if len(encodings_array) > sequence_len: encodings_array = encodings_array[:sequence_len]
|
|
|
if len(indices) > sequence_len: indices = indices[:sequence_len]
|
|
|
|
|
|
ranks = get_frequency_ranks(encodings_array, symbols, sequence_len)
|
|
|
distances = get_proximity_array(encodings_array, sequence_len)
|
|
|
processed_data.append([encodings_array, distances, indices])
|
|
|
return processed_data
|
|
|
|
|
|
|
