dedupe_passwords.py

"""Remove duplicate passwords that either exist in the original password list, were not translated or were translated to the same password twice."""

import argparse
from tqdm import tqdm

def main():
    parser = argparse.ArgumentParser(description="Remove passwords that exist in the original list and remove deduplicates from the output one.")
    parser.add_argument("-i", "--input_file", required=True, help="Path to the original password list.")
    parser.add_argument("-t", "--translated_file", required=True, help="Path to the translated password list.")
    parser.add_argument("-o", "--output_file", required=True, help="Path to the output deduplicated password list.")

    args = parser.parse_args()

    try:
        with open(args.input_file, 'r', encoding='latin1') as file:
            original = set(file.readlines())
        with open(args.translated_file, 'r', encoding='latin1') as file:
            translated = file.readlines()

        final_list = []
        final_set = set()

        for newpa in tqdm(translated):
            if newpa in original:
                continue
            if newpa in final_set:
                continue
            final_list.append(newpa)
            final_set.add(newpa)

        with open(args.output_file, 'w', encoding='latin1') as file:
            file.writelines(final_list)

    except FileNotFoundError:
        print("Some file was not found.")

if __name__ == "__main__":
    main()