                    if not check_ngram_free:
                        ngram_free = False
                        break

                if not ngram_free:
                    break

        # texts are ngram free
        if ngram_free and not args.get_ngram_freq_only:
            text_buf_ngram_free.append(text)

    # check if the text has only been trimmed
    trimmed = 0
    if not args.get_ngram_freq_only and len(text_buf_ngram_free) == 1 and \
        len(text_buf_ngram_free[0]) < len(myjson[key]):
        trimmed = 1

    return text_buf_ngram_free, trimmed, myjson, local_ngram

# insert word sequence into dictionary
def insert_dict(words, ngrams, pos):
    seq = " ".join(words)
    if seq not in ngrams:
        ngrams[seq] = 0
        #ngrams[seq] = pos

# insert each ngram from text into the ngrams dictionary
def compute_ngrams_insert_dict(args, text, ngrams):
    words, positions = get_words(text)
    if len(words) < args.min_ngram_size:
        return

    if len(words) < args.max_ngram_size:
        insert_dict(words, ngrams, positions[0])

    for i in range(len(words) - args.max_ngram_size+1):
        insert_dict(words[i:i+args.max_ngram_size], ngrams, positions[i])

# Build ngrams for the lambada dataset
def process_task_lambda(args, task_file, ngrams):
    print(' reading from {} and computing ngrams'.format(task_file))
    with open(task_file, 'r') as f:
        for line in f:


        # texts are ngram free
        if ngram_free and not args.get_ngram_freq_only:
            text_buf_ngram_free.append(text)

    # check if the text has only been trimmed
    trimmed = 0
    if not args.get_ngram_freq_only and len(text_buf_ngram_free) == 1 and \
        len(text_buf_ngram_free[0]) < len(myjson[key]):
        trimmed = 1

    return text_buf_ngram_free, trimmed, myjson, local_ngram

# insert word sequence into dictionary
def insert_dict(words, ngrams, pos):
    seq = " ".join(words)
    if seq not in ngrams:
        ngrams[seq] = 0
        #ngrams[seq] = pos

# insert each ngram from text into the ngrams dictionary
def compute_ngrams_insert_dict(args, text, ngrams):
    words, positions = get_words(text)
    if len(words) < args.min_ngram_size:
        return

    if len(words) < args.max_ngram_size:
        insert_dict(words, ngrams, positions[0])

    for i in range(len(words) - args.max_ngram_size+1):
        insert_dict(words[i:i+args.max_ngram_size], ngrams, positions[i])

# Build ngrams for the lambada dataset
def process_task_lambda(args, task_file, ngrams):
    print(' reading from {} and computing ngrams'.format(task_file))
    with open(task_file, 'r') as f:
        for line in f:
            try:
                myjson = json.loads(line)
                text = myjson['text']
                compute_ngrams_insert_dict(args, text, ngrams)
            except Exception as e:
                print('Error:', e)
    print(" Entities in ngrams {}".format(len(ngrams)), flush=True)

# Build ngrams for the dataset of the given task
def process_task(args, task_name, ngrams):

    print(' reading from {} and computing ngrams'.format('import datasets'))


def insert_dict(words, ngrams, pos):
    seq = " ".join(words)
    if seq not in ngrams:
        ngrams[seq] = 0
        #ngrams[seq] = pos

# insert each ngram from text into the ngrams dictionary
def compute_ngrams_insert_dict(args, text, ngrams):
    words, positions = get_words(text)
    if len(words) < args.min_ngram_size:
        return

    if len(words) < args.max_ngram_size:
        insert_dict(words, ngrams, positions[0])

    for i in range(len(words) - args.max_ngram_size+1):
        insert_dict(words[i:i+args.max_ngram_size], ngrams, positions[i])

# Build ngrams for the lambada dataset
def process_task_lambda(args, task_file, ngrams):
    print(' reading from {} and computing ngrams'.format(task_file))
    with open(task_file, 'r') as f:
        for line in f:
            try:
                myjson = json.loads(line)
                text = myjson['text']
                compute_ngrams_insert_dict(args, text, ngrams)
            except Exception as e:
                print('Error:', e)
    print(" Entities in ngrams {}".format(len(ngrams)), flush=True)

# Build ngrams for the dataset of the given task
def process_task(args, task_name, ngrams):

    print(' reading from {} and computing ngrams'.format('import datasets'))
    print(" Current entities in ngrams {}".format(len(ngrams)), flush=True)
    # using validation/test data from datasets
    from datasets import load_dataset

    entities_in_ngrams = len(ngrams)

    # load the dataset
    if task_name == 'squad':
        dataset = load_dataset('squad_v2', split='validation')
    elif task_name == 'natural_questions':
        dataset = load_dataset('natural_questions', split='validation')
    elif task_name == 'triviaqa':
        dataset = load_dataset('trivia_qa', 'unfiltered', split='test')
    elif task_name == 'webqa':


    for i in range(len(words) - args.max_ngram_size+1):
        insert_dict(words[i:i+args.max_ngram_size], ngrams, positions[i])

# Build ngrams for the lambada dataset
def process_task_lambda(args, task_file, ngrams):
    print(' reading from {} and computing ngrams'.format(task_file))
    with open(task_file, 'r') as f:
        for line in f:
            try:
                myjson = json.loads(line)
                text = myjson['text']
                compute_ngrams_insert_dict(args, text, ngrams)
            except Exception as e:
                print('Error:', e)
    print(" Entities in ngrams {}".format(len(ngrams)), flush=True)

# Build ngrams for the dataset of the given task
def process_task(args, task_name, ngrams):

    print(' reading from {} and computing ngrams'.format('import datasets'))
    print(" Current entities in ngrams {}".format(len(ngrams)), flush=True)
    # using validation/test data from datasets
    from datasets import load_dataset

    entities_in_ngrams = len(ngrams)

    # load the dataset
    if task_name == 'squad':
        dataset = load_dataset('squad_v2', split='validation')
    elif task_name == 'natural_questions':
        dataset = load_dataset('natural_questions', split='validation')
    elif task_name == 'triviaqa':
        dataset = load_dataset('trivia_qa', 'unfiltered', split='test')
    elif task_name == 'webqa':
        dataset = load_dataset('web_questions', split='test')
    elif task_name == 'race':
        dataset = load_dataset('race', 'all', split='test')
    elif task_name == 'drop':
        dataset = load_dataset('drop', split='validation')
    elif task_name == 'coqa':
        dataset = load_dataset('coqa', split='validation')
    elif task_name == 'piqa':
        dataset = load_dataset('piqa', split='test')
    else:
        print("Invalid task name: {}".format(task_name), flush=True)
        return

    # read the dataset and add to ngrams
    for line in dataset:
        try:
            if task_name in ['squad', 'triviaqa', 'webqa', 'race', 'drop']:
                text = line['question']
                compute_ngrams_insert_dict(args, text, ngrams)
            elif task_name == 'natural_questions':
                text = line['question']['text']
                compute_ngrams_insert_dict(args, text, ngrams)
            elif task_name == 'coqa':
                all_questions = line['questions']
                for question in all_questions:
                    compute_ngrams_insert_dict(args, question, ngrams)
            elif task_name == 'piqa':
                text = line['goal']
                compute_ngrams_insert_dict(args, text, ngrams)
        except Exception as e:
            print('Error:', e)

    print(" After task {} entities in ngrams {}, added {}".format(task_name, \
            len(ngrams), len(ngrams) - entities_in_ngrams), flush=True)

def compute_tasks_ngrams(args, ngrams):
    start_time = time.time()
    for _, task_name in enumerate(args.tasks):
        print('Task: {}'.format(task_name), flush=True)
        if task_name == 'lambada':
            assert args.lambada_path is not None
            process_task_lambda(args, args.lambada_path, ngrams)
        else:
            process_task(args, task_name, ngrams)
    print(" Taken time to compute ngrams {:.2f}".format(time.time() - \
        start_time), flush=True)

def compute_ngram_freq_sorted(args, ngrams):
    ngrams_freq = {}
    for ngram_key in ngrams.keys():
        length = len(ngram_key.split())
        ngrams_freq[length] = ngrams_freq[length] + 1 if length in \
            ngrams_freq else 1


        try:
            if task_name in ['squad', 'triviaqa', 'webqa', 'race', 'drop']:
                text = line['question']
                compute_ngrams_insert_dict(args, text, ngrams)
            elif task_name == 'natural_questions':
                text = line['question']['text']
                compute_ngrams_insert_dict(args, text, ngrams)
            elif task_name == 'coqa':
                all_questions = line['questions']
                for question in all_questions:
                    compute_ngrams_insert_dict(args, question, ngrams)
            elif task_name == 'piqa':
                text = line['goal']
                compute_ngrams_insert_dict(args, text, ngrams)
        except Exception as e:
            print('Error:', e)

    print(" After task {} entities in ngrams {}, added {}".format(task_name, \
            len(ngrams), len(ngrams) - entities_in_ngrams), flush=True)

def compute_tasks_ngrams(args, ngrams):
    start_time = time.time()
    for _, task_name in enumerate(args.tasks):
        print('Task: {}'.format(task_name), flush=True)
        if task_name == 'lambada':
            assert args.lambada_path is not None
            process_task_lambda(args, args.lambada_path, ngrams)
        else:
            process_task(args, task_name, ngrams)
    print(" Taken time to compute ngrams {:.2f}".format(time.time() - \
        start_time), flush=True)

def compute_ngram_freq_sorted(args, ngrams):
    ngrams_freq = {}
    for ngram_key in ngrams.keys():
        length = len(ngram_key.split())
        ngrams_freq[length] = ngrams_freq[length] + 1 if length in \
            ngrams_freq else 1

    ngrams_freq_sorted = sorted(ngrams_freq.items(), key=lambda item: item[0])
    print(" Ngram frequencies: {}".format(ngrams_freq_sorted), flush=True)
    print(" Entities in ngrams {} min_ngram_size {} max_ngram_size {}".format(\
            len(ngrams), ngrams_freq_sorted[0][0], ngrams_freq_sorted[len(\
            ngrams_freq_sorted) -1 ][0]), flush=True)
    return ngrams_freq_sorted

def get_ngrams_below_threshold(args, ngrams, ngrams_below_threshold, \
    dedup_file, dedup_key, ngrams_freq_sorted):

    start_time = time.time()
    # get the ngrams frequency


                text = line['goal']
                compute_ngrams_insert_dict(args, text, ngrams)
        except Exception as e:
            print('Error:', e)

    print(" After task {} entities in ngrams {}, added {}".format(task_name, \
            len(ngrams), len(ngrams) - entities_in_ngrams), flush=True)

def compute_tasks_ngrams(args, ngrams):
    start_time = time.time()
    for _, task_name in enumerate(args.tasks):
        print('Task: {}'.format(task_name), flush=True)
        if task_name == 'lambada':
            assert args.lambada_path is not None
            process_task_lambda(args, args.lambada_path, ngrams)
        else:
            process_task(args, task_name, ngrams)
    print(" Taken time to compute ngrams {:.2f}".format(time.time() - \
        start_time), flush=True)

def compute_ngram_freq_sorted(args, ngrams):
    ngrams_freq = {}
    for ngram_key in ngrams.keys():
        length = len(ngram_key.split())
        ngrams_freq[length] = ngrams_freq[length] + 1 if length in \
            ngrams_freq else 1

    ngrams_freq_sorted = sorted(ngrams_freq.items(), key=lambda item: item[0])
    print(" Ngram frequencies: {}".format(ngrams_freq_sorted), flush=True)
    print(" Entities in ngrams {} min_ngram_size {} max_ngram_size {}".format(\
            len(ngrams), ngrams_freq_sorted[0][0], ngrams_freq_sorted[len(\
            ngrams_freq_sorted) -1 ][0]), flush=True)
    return ngrams_freq_sorted

def get_ngrams_below_threshold(args, ngrams, ngrams_below_threshold, \
    dedup_file, dedup_key, ngrams_freq_sorted):

    start_time = time.time()
    # get the ngrams frequency
    args.get_ngram_freq_only = True

    # Open the large file to process in parallel
    num_workers = args.num_threads
    pool = multiprocessing.Pool(num_workers)
    fin = open(dedup_file, 'r', encoding='utf-8')
    free_ngram_abt_partial=partial(free_ngram, args=args, key=dedup_key, \
        ngrams=ngrams, ngrams_freq_sorted=ngrams_freq_sorted)
    free_ngrams_abt = pool.imap(free_ngram_abt_partial, fin, 500)

    counter = 0
    for _, _, _, local_ngram in free_ngrams_abt:
        counter += 1
        if counter % 1000 == 0:


            process_task_lambda(args, args.lambada_path, ngrams)
        else:
            process_task(args, task_name, ngrams)
    print(" Taken time to compute ngrams {:.2f}".format(time.time() - \
        start_time), flush=True)

def compute_ngram_freq_sorted(args, ngrams):
    ngrams_freq = {}
    for ngram_key in ngrams.keys():
        length = len(ngram_key.split())
        ngrams_freq[length] = ngrams_freq[length] + 1 if length in \
            ngrams_freq else 1

    ngrams_freq_sorted = sorted(ngrams_freq.items(), key=lambda item: item[0])
    print(" Ngram frequencies: {}".format(ngrams_freq_sorted), flush=True)
    print(" Entities in ngrams {} min_ngram_size {} max_ngram_size {}".format(\
            len(ngrams), ngrams_freq_sorted[0][0], ngrams_freq_sorted[len(\
            ngrams_freq_sorted) -1 ][0]), flush=True)
    return ngrams_freq_sorted

def get_ngrams_below_threshold(args, ngrams, ngrams_below_threshold, \
    dedup_file, dedup_key, ngrams_freq_sorted):

    start_time = time.time()
    # get the ngrams frequency
    args.get_ngram_freq_only = True

    # Open the large file to process in parallel
    num_workers = args.num_threads
    pool = multiprocessing.Pool(num_workers)
    fin = open(dedup_file, 'r', encoding='utf-8')
    free_ngram_abt_partial=partial(free_ngram, args=args, key=dedup_key, \
        ngrams=ngrams, ngrams_freq_sorted=ngrams_freq_sorted)
    free_ngrams_abt = pool.imap(free_ngram_abt_partial, fin, 500)

    counter = 0
    for _, _, _, local_ngram in free_ngrams_abt:
        counter += 1
        if counter % 1000 == 0:
            print(' [compute_stat]> processed {} documents in {:.2f} seconds ...'.
                    format(counter, time.time() - start_time), flush=True)
        for local_key in local_ngram:
            if local_key in ngrams:
                ngrams[local_key] += 1
        local_ngram = {}

    print(' Time taken to compute statistics {:.2f} seconds'.format(time.time() - \
        start_time), flush=True)
    pool.close()
    pool.join()

    start_time = time.time()
    counter_threshold = 0
    # Get ngram below theadhold
    for local_key, local_val in ngrams.items():
        if ngrams[local_key] < args.key_threshold:
            print(" [threshold] {} {}".format(local_key, local_val), flush=True)
            counter_threshold += 1
            ngrams_below_threshold[local_key] = 1

    print(' Ngrams below threshold {}'.format(counter_threshold), flush=True)
    fin.close()

def clean_ngrams_below_threshold(args, ngrams_below_threshold, dedup_file, \
    dedup_key):

    start_time = time.time()
    # Now actually filter the dataset
    args.get_ngram_freq_only = False
    #id_prefix = '-'.join(args.tasks[::2])
    id_prefix = '-'.join(args.tasks[::1])

    # get the range of the size of the ngrams
    ngrams_freq_sorted = compute_ngram_freq_sorted(args, ngrams_below_threshold)

    # Open the large file to process in parallel
    counter = splitted = ignored = split_mt_thld = trimmed_count = 0
    num_workers = args.num_threads
    pool = multiprocessing.Pool(num_workers)
    fin = open(dedup_file, 'r', encoding='utf-8')
    free_ngram_clean_partial=partial(free_ngram, args=args, key=dedup_key, \
        ngrams=ngrams_below_threshold, ngrams_freq_sorted=ngrams_freq_sorted)


                ngrams[local_key] += 1
        local_ngram = {}

    print(' Time taken to compute statistics {:.2f} seconds'.format(time.time() - \
        start_time), flush=True)
    pool.close()
    pool.join()

    start_time = time.time()
    counter_threshold = 0
    # Get ngram below theadhold
    for local_key, local_val in ngrams.items():
        if ngrams[local_key] < args.key_threshold:
            print(" [threshold] {} {}".format(local_key, local_val), flush=True)
            counter_threshold += 1
            ngrams_below_threshold[local_key] = 1

    print(' Ngrams below threshold {}'.format(counter_threshold), flush=True)
    fin.close()

def clean_ngrams_below_threshold(args, ngrams_below_threshold, dedup_file, \
    dedup_key):

    start_time = time.time()
    # Now actually filter the dataset
    args.get_ngram_freq_only = False
    #id_prefix = '-'.join(args.tasks[::2])
    id_prefix = '-'.join(args.tasks[::1])

    # get the range of the size of the ngrams
    ngrams_freq_sorted = compute_ngram_freq_sorted(args, ngrams_below_threshold)

    # Open the large file to process in parallel
    counter = splitted = ignored = split_mt_thld = trimmed_count = 0
    num_workers = args.num_threads
    pool = multiprocessing.Pool(num_workers)
    fin = open(dedup_file, 'r', encoding='utf-8')
    free_ngram_clean_partial=partial(free_ngram, args=args, key=dedup_key, \
        ngrams=ngrams_below_threshold, ngrams_freq_sorted=ngrams_freq_sorted)
    free_ngrams_clean = pool.imap(free_ngram_clean_partial, fin, 500)

    out_f = open(args.output, 'wb')

    for text_buf_ngram_free, trimmed, myjson, _ in free_ngrams_clean:
        counter += 1
        try:

            trimmed_count += trimmed

            if len(text_buf_ngram_free) > 1:
                splitted += 1
            if len(text_buf_ngram_free) == 0:
                ignored += 1
            # more than 10 splits ignored
            if len(text_buf_ngram_free) > args.splits_count:
                text_buf_ngram_free = []
                split_mt_thld += 1

            if args.output is not None:
                if "split_id" in myjson:
                    use_prefix = myjson["split_id"] + "-"
                else:
                    use_prefix = ""

                for i in range(len(text_buf_ngram_free)):
                    split_id_string = id_prefix + '-{:010d}'.format(int(\
                        counter)) + '-{:04d}'.format(int(i))
                    myjson[dedup_key] = text_buf_ngram_free[i]
                    myjson["split_id"] = use_prefix + split_id_string
                    outjson = json.dumps(myjson, ensure_ascii=False)
                    #outjson = json.dumps({"text":text_buf_ngram_free[i],
                    #    id_prefix+"_split_id":split_id_string},
                    #    ensure_ascii=False)
                    out_f.write(outjson.encode('utf-8'))
                    out_f.write('\n'.encode('utf-8'))

            if counter % 1000 == 0:
                print(' [final]> processed {} documents in {:.2f} seconds ...'.
                    format(counter, time.time() - start_time), flush=True)
        except Exception as e:
            print('Error:', e)

    print(' [final]> processed {} documents in {:.2f} seconds ...'.
        format(counter, time.time() - start_time), flush=True)

    print(' Total docs {} splitted {} ignored {} splits > theshold {} trimmed'\
        ' {}'.format(counter, splitted, ignored, split_mt_thld, trimmed_count)\
        , flush=True)

    pool.close()
    pool.join()

    out_f.close()
    fin.close()

if __name__ == '__main__':

    # we use 13-grams, any text less than 200 characters got removed
    # any text splitted more than 10 got removed as well

    print('parsing the arguments ...')

    parser = argparse.ArgumentParser()
    parser.add_argument('--tasks', nargs = '*', required=True, default=None, \
                        help = 'Tasks to use for deduplication: currently '
                        ' suuport [lambada, squad, natural_questions,'
                        ' triviaqa, webqa, race, drop, coqa, and piqa]')
    parser.add_argument('--lambada-path', type=str, default=None,
                       help='Only Lambada task needs the path')
    parser.add_argument('--dedup-dataset', nargs = '*', default=None,
                       help='Dataset to deduplicate with the key to use'
                        ' e.g. cc.json text')
    parser.add_argument('--output', type=str, default=None,
                       help='Output file name to save dedup dataset')


# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.

"""
Filter and clean documents:
Capable to clean docs with less than 512 characters, less than
256 characters and contains javascript, fix text and dataset specific
cleaning like stories and realnews datasets.
Program arguments have the details.
"""

import argparse
from functools import partial
import glob
import ftfy
import json
from langdetect import detect
import multiprocessing
import os
from pathlib import Path
import re
import time

def process_doc(json_line, args):

    # Read the line.
    document = json.loads(json_line)
    text = document['text']

    output = {'remove_512': False, 'remove_256_javascript': False, \
        'remove_512_non_english': False, 'ftfy_fix_text': False, \
        'general_cleaning': False}

    try:
        # Reomove all docs with less than 512 characters
        if "remove_512" in args.tasks:
            if len(text) < 512:
                output['remove_512'] = True
                return output, text, document, True

        # Remove docs if less than 256 character length and contains Javascript
        if "remove_256_javascript" in args.tasks:
            if len(text) < 256 and 'javascript' in text.lower():
                output['remove_256_javascript'] = True
                return output, text, document, True

        # Remove docs < 512 and nonenglish
        if "remove_512_non_english" in args.tasks:
            if len(text) < 512 and detect(text) != 'en':
                output['remove_512_non_english'] = True
                return output, text, document, True

        # Fix the text using ftfy, don't remove the text, hence return False
        if "ftfy_fix_text" in args.tasks:
            fixed_text = ftfy.fix_text(text)
            output['ftfy_fix_text'] = True
            return output, fixed_text, document, False

        # Cleaning extra spaces and newlines
        if "general_cleaning" in args.tasks:
            cleaned_text = re.sub(r"  +|\b\n+ |\b\n+", " ", text)
            #cleaned_text = re.sub(r"\n\n+", "\n\n", text) # used this for Gutenberg dataset
            #cleaned_text = re.sub(r"\n", "\n\n", text) # Used this for realnews

            # stories datasets
            #cleaned_text = re.sub(r" \'", "'", text)
            #cleaned_text = re.sub(r" \!", "!", cleaned_text)
            #cleaned_text = re.sub(r" \.", ".", cleaned_text)
            #cleaned_text = re.sub(r" \?", "?", cleaned_text)
            #cleaned_text = re.sub(r" - ", "-", cleaned_text)
            ##cleaned_text = re.sub(r"\" ", "\"", cleaned_text)
            #cleaned_text = re.sub(r" @ ", "@", cleaned_text)

            output['general_cleaning'] = True
            return output, cleaned_text, document, False

    except Exception as e:
        print('Error: *************************\n{}\ntext: {}'.format(e, \
            text), flush=True)
        return output, text, document, True

    # don't remove
    return output, text, document, False

def process_set(args, input_file, output_f_cleaned, output_f_filtered):

    print(' > working on {} ...'.format(input_file), flush=True)

    num_docs = num_remove_512 = num_remove_java = num_remove_512_non_english \
        = num_ftfy_fix_text = num_general_cleaning = 0

    # Output file and counters.
    output_cleaned = open(output_f_cleaned, 'wb')
    output_filtered = open(output_f_filtered, 'wb')

    start_time = time.time()

    # Setup multi-processing.
    num_workers = 40
    fin = open(input_file, 'r', encoding='utf-8')
    pool = multiprocessing.Pool(num_workers)
    process_doc_partial = partial(process_doc, args=args)
    processed_docs = pool.imap(process_doc_partial, fin, 500)

    # Process documents.
    for output, text, document, to_filter in processed_docs:
        num_docs += 1

        num_remove_512 += 1 if output['remove_512'] else 0
        num_remove_java += 1 if output['remove_256_javascript'] else 0
        num_remove_512_non_english += 1 if output['remove_512_non_english'] \
            else 0
        num_ftfy_fix_text += 1 if output['ftfy_fix_text'] else 0
        num_general_cleaning += 1 if output['general_cleaning'] else 0

        document['text'] = text
        myjson = json.dumps(document, ensure_ascii=False)

        if to_filter:
            output_filtered.write(myjson.encode('utf-8'))
            output_filtered.write('\n'.encode('utf-8'))
        else:
            output_cleaned.write(myjson.encode('utf-8'))
            output_cleaned.write('\n'.encode('utf-8'))

        if num_docs % args.log_interval == 0:
            print('    processed {:9d} documents in {:.2f} seconds ...'.format(
                num_docs, time.time() - start_time), flush=True)

    # Close the file.
    output_cleaned.close()
    output_filtered.close()
    fin.close()

    # Print stats.
    print('  >> total docs: {} remove_512 {} remove_256_javascript {} '\
        'remove_512_non_english {} ftfy_fix_text {} general_cleaning {}'.\
        format(num_docs, num_remove_512, num_remove_java,\
        num_remove_512_non_english, num_ftfy_fix_text, \
        num_general_cleaning), flush=True)

if __name__ == '__main__':

    print('parsing the arguments ...')

    parser = argparse.ArgumentParser()
    parser.add_argument('--input-files', nargs = '*', required=True, default=\
                        None, help = 'Input json files that needs to be'\
                        ' cleaned')
    parser.add_argument('--tasks', nargs = '*', required=True, default=None,\
                        help = 'Tasks to perform on the input files, ' \
                        'such as remove_512, remove_256_javascript, ' \
                        'remove_512_non_english, ftfy_fix_text, and ' \
                        'general_cleaning. 256 or 512 means the number' \
                        ' of characters.')

    parser.add_argument('--output-path', type=str, default=None,
                       help='Directory where the output should go')
    parser.add_argument('--log-interval', type=int, default=100,
                       help='Log interval')

    args = parser.parse_args()

    print('cleanup dataset ...')

    for input_file in args.input_files:
        input_filename, input_filename_ext = os.path.splitext(Path(input_file)\
            .name)

        output_f_cleaned = os.path.join(args.output_path, input_filename + \
            "_cleaned" + input_filename_ext)
        output_f_filtered = os.path.join(args.output_path, input_filename + \
            "_filtered" + input_filename_ext)

        process_set(args, input_file, output_f_cleaned, output_f_filtered)

    print('done :-)', flush=True)


"""
Filter and clean documents:
Capable to clean docs with less than 512 characters, less than
256 characters and contains javascript, fix text and dataset specific
cleaning like stories and realnews datasets.
Program arguments have the details.
"""

import argparse
from functools import partial
import glob
import ftfy
import json
from langdetect import detect
import multiprocessing
import os
from pathlib import Path
import re
import time

def process_doc(json_line, args):

    # Read the line.
    document = json.loads(json_line)
    text = document['text']

    output = {'remove_512': False, 'remove_256_javascript': False, \
        'remove_512_non_english': False, 'ftfy_fix_text': False, \
        'general_cleaning': False}

    try:
        # Reomove all docs with less than 512 characters
        if "remove_512" in args.tasks:
            if len(text) < 512:
                output['remove_512'] = True
                return output, text, document, True

        # Remove docs if less than 256 character length and contains Javascript
        if "remove_256_javascript" in args.tasks:
            if len(text) < 256 and 'javascript' in text.lower():
                output['remove_256_javascript'] = True
                return output, text, document, True

        # Remove docs < 512 and nonenglish
        if "remove_512_non_english" in args.tasks:
            if len(text) < 512 and detect(text) != 'en':
                output['remove_512_non_english'] = True
                return output, text, document, True

        # Fix the text using ftfy, don't remove the text, hence return False
        if "ftfy_fix_text" in args.tasks:
            fixed_text = ftfy.fix_text(text)
            output['ftfy_fix_text'] = True
            return output, fixed_text, document, False

        # Cleaning extra spaces and newlines
        if "general_cleaning" in args.tasks:
            cleaned_text = re.sub(r"  +|\b\n+ |\b\n+", " ", text)
            #cleaned_text = re.sub(r"\n\n+", "\n\n", text) # used this for Gutenberg dataset
            #cleaned_text = re.sub(r"\n", "\n\n", text) # Used this for realnews

            # stories datasets
            #cleaned_text = re.sub(r" \'", "'", text)
            #cleaned_text = re.sub(r" \!", "!", cleaned_text)
            #cleaned_text = re.sub(r" \.", ".", cleaned_text)
            #cleaned_text = re.sub(r" \?", "?", cleaned_text)
            #cleaned_text = re.sub(r" - ", "-", cleaned_text)
            ##cleaned_text = re.sub(r"\" ", "\"", cleaned_text)
            #cleaned_text = re.sub(r" @ ", "@", cleaned_text)

            output['general_cleaning'] = True
            return output, cleaned_text, document, False

    except Exception as e:
        print('Error: *************************\n{}\ntext: {}'.format(e, \
            text), flush=True)
        return output, text, document, True

    # don't remove
    return output, text, document, False

def process_set(args, input_file, output_f_cleaned, output_f_filtered):

    print(' > working on {} ...'.format(input_file), flush=True)

    num_docs = num_remove_512 = num_remove_java = num_remove_512_non_english \
        = num_ftfy_fix_text = num_general_cleaning = 0

    # Output file and counters.
    output_cleaned = open(output_f_cleaned, 'wb')
    output_filtered = open(output_f_filtered, 'wb')

    start_time = time.time()

    # Setup multi-processing.
    num_workers = 40
    fin = open(input_file, 'r', encoding='utf-8')
    pool = multiprocessing.Pool(num_workers)
    process_doc_partial = partial(process_doc, args=args)


            #cleaned_text = re.sub(r" \'", "'", text)
            #cleaned_text = re.sub(r" \!", "!", cleaned_text)
            #cleaned_text = re.sub(r" \.", ".", cleaned_text)
            #cleaned_text = re.sub(r" \?", "?", cleaned_text)
            #cleaned_text = re.sub(r" - ", "-", cleaned_text)
            ##cleaned_text = re.sub(r"\" ", "\"", cleaned_text)
            #cleaned_text = re.sub(r" @ ", "@", cleaned_text)

            output['general_cleaning'] = True
            return output, cleaned_text, document, False

    except Exception as e:
        print('Error: *************************\n{}\ntext: {}'.format(e, \
            text), flush=True)
        return output, text, document, True

    # don't remove
    return output, text, document, False

def process_set(args, input_file, output_f_cleaned, output_f_filtered):

    print(' > working on {} ...'.format(input_file), flush=True)

    num_docs = num_remove_512 = num_remove_java = num_remove_512_non_english \
        = num_ftfy_fix_text = num_general_cleaning = 0

    # Output file and counters.
    output_cleaned = open(output_f_cleaned, 'wb')
    output_filtered = open(output_f_filtered, 'wb')

    start_time = time.time()

    # Setup multi-processing.
    num_workers = 40
    fin = open(input_file, 'r', encoding='utf-8')
    pool = multiprocessing.Pool(num_workers)
    process_doc_partial = partial(process_doc, args=args)
    processed_docs = pool.imap(process_doc_partial, fin, 500)

    # Process documents.
    for output, text, document, to_filter in processed_docs:
        num_docs += 1

        num_remove_512 += 1 if output['remove_512'] else 0
        num_remove_java += 1 if output['remove_256_javascript'] else 0
        num_remove_512_non_english += 1 if output['remove_512_non_english'] \
            else 0
        num_ftfy_fix_text += 1 if output['ftfy_fix_text'] else 0
        num_general_cleaning += 1 if output['general_cleaning'] else 0

        document['text'] = text
        myjson = json.dumps(document, ensure_ascii=False)

        if to_filter:
            output_filtered.write(myjson.encode('utf-8'))
            output_filtered.write('\n'.encode('utf-8'))
        else:
            output_cleaned.write(myjson.encode('utf-8'))
            output_cleaned.write('\n'.encode('utf-8'))

        if num_docs % args.log_interval == 0:
            print('    processed {:9d} documents in {:.2f} seconds ...'.format(
                num_docs, time.time() - start_time), flush=True)

    # Close the file.
    output_cleaned.close()
    output_filtered.close()
    fin.close()

    # Print stats.
    print('  >> total docs: {} remove_512 {} remove_256_javascript {} '\
        'remove_512_non_english {} ftfy_fix_text {} general_cleaning {}'.\
        format(num_docs, num_remove_512, num_remove_java,\
        num_remove_512_non_english, num_ftfy_fix_text, \
        num_general_cleaning), flush=True)

if __name__ == '__main__':

    print('parsing the arguments ...')

    parser = argparse.ArgumentParser()
    parser.add_argument('--input-files', nargs = '*', required=True, default=\
                        None, help = 'Input json files that needs to be'\
                        ' cleaned')
    parser.add_argument('--tasks', nargs = '*', required=True, default=None,\
                        help = 'Tasks to perform on the input files, ' \
                        'such as remove_512, remove_256_javascript, ' \
                        'remove_512_non_english, ftfy_fix_text, and ' \
                        'general_cleaning. 256 or 512 means the number' \
                        ' of characters.')

    parser.add_argument('--output-path', type=str, default=None,
                       help='Directory where the output should go')
    parser.add_argument('--log-interval', type=int, default=100,


# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.

# WARNING! This file contains a blacklist of known malicious sites and thus contains some NSFW language.

import glob
import re
import time
import tldextract
import sys

# List of the domains to blacklist.
domain_blacklist = set([
    '500px',
    'aapks',
    'akamaihd',
    'amazon',
    'apple',
    'artifactfire',
    'artstation',
    'awwni',
    'bandcamp',
    'battleforthenet',
    'coinscalendar',
    'dailymotion',
    'deviantart',
    'discord',
    'discordapp',
    'dlapkandroid',
    'dropbox',
    'e621',
    'ebay',
    'edealinfo',
    'erome',
    'eroshare',
    'explosm',
    'facebook',
    'fbcdn',
    'flickr',
    'furaffinity',
    'futhead',
    'gatopardo',
    'gfycat',
    'gifsound',
    'gifsoup',
    'giphy',
    'github',
    'google',
    'gunprime',
    'gyazo',
    'horsefucker',
    'hotdealstar',
    'imagefap',
    'imageshack',
    'imgflip',
    'imgur',
    'instagram',
    'karmadecay',
    'kryptocal',
    'kym-cdn',
    'liveleak',
    'livememe',
    'lmgtfy',
    'magaimg',
    'memegenerator',
    'minorplanetcenter',
    'minus',
    'mobafire',
    'morejpeg',
    'nocookie',
    'pcpartpicker',
    'photobucket',
    'pinimg',
    'pinterest',
    'pixiv',
    'pornhub',
    'prntscr',
    'puu',
    'qkme',
    'quickmeme',
    'radd',
    'redd',
    'reddit',
    'reddit-stream',
    'redditlog',
    'redditmedia',
    'reddituploads',
    'redtube',
    'reupp',
    'reverb',
    'roanoke',
    'rollingstone',
    'sli',
    'soundcloud',
    'soundgasm',
    'spankbang',
    'spotify',
    'strawpoll',
    'streamable',
    'timeanddate',
    'tinypic',
    'touhouradio',
    'tumblr',
    'twimg',
    'twitch',
    'twitter',
    'vid',
    'vimeo',
    'vine',
    'vkaao',
    'vocaroo',
    'voyagefusion',
    'walmart',
    'wciu',
    'wikimedia',
    'wikipedia',
    'xhamster',
    'xkcd',
    'xvideos',
    'youtu',
    'youtube',
    'youtubedoubler',
    'ytimg',
    'zillexplorer',
])

def domain_is_in_blacklist(url):
    domain = tldextract.extract(url).domain
    return domain in domain_blacklist

# List of extentions to blacklist.
extentions_blacklist = (
    '.3gp',
    '.7z'
    '.ai',
    '.aif',
    '.apk',
    '.app',
    '.avi',
    '.bin',
    '.bmp',
    '.bz2',
    '.css',
    '.csv',
    '.dat',
    '.deb',
    '.dmg',
    '.doc',
    '.docx',
    '.exe',
    '.gif',
    '.gifv',
    '.gz',
    '.iso',
    '.jar',
    '.jpeg',
    '.jpg',
    '.js',
    '.log',
    '.mid',
    '.midi',
    '.mkv',
    '.mov',
    '.mp3',
    '.mp4',
    '.mpeg',
    '.mpg',
    '.ogg',
    '.ogv',
    '.otf',
    '.pdf',
    '.pkg',
    '.png',
    '.pps',
    '.ppt',
    '.pptx',
    '.psd',
    '.py',
    '.qt',
    '.ram',
    '.rar',
    '.sql',
    '.svg',
    '.swf',
    '.tar.gz',
    '.tar',
    '.tgz',
    '.tiff',
    '.ttf',
    '.txt',
    '.wav',
    '.webm',
    '.wma',
    '.wmv',
    '.xls',
    '.xlsx',
    '.xml',
    '.xz',
    '.zip',
)

def extention_is_in_blacklist(url):
    if url.split('?')[0].lower().endswith(extentions_blacklist):
        return True
    return False

# Malformed urls.
# This function is adapted from:
#   https://stackoverflow.com/questions/7160737/python-how-to-validate-a-url-in-python-malformed-or-not
url_regex = re.compile(
    r'^(?:http)s?://' # http:// or https://
    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
    r'(?::\d+)?' # optional port
    r'(?:/?|[/?]\S+)$', re.IGNORECASE)
def url_is_malformed(url):
    return re.match(url_regex, url) is None

def print_progress(prefix, start_time, urls_counter,
                   domain_blacklist_counter,
                   extention_blacklist_counter,
                   short_url_counter, malformed_url_counter,
                   duplicate_url_counter):
    string = prefix + ' | '
    string += 'time elapsed (s): {:.2f} | '.format(time.time() - start_time)
    string += 'number of urls: {} | '.format(urls_counter)
    string += 'domain blacklisted: {} | '.format(domain_blacklist_counter)
    string += 'extention blacklisted: {} | '.format(extention_blacklist_counter)
    string += 'short urls (<=8): {} | '.format(short_url_counter)
    string += 'malformed urls: {} | '.format(malformed_url_counter)
    string += 'duplicate urls: {}'.format(duplicate_url_counter)
    print(string, flush=True)

if __name__ == '__main__':

    print('remove blacklisted urls ..')

    # Path to the url files.
    path = sys.argv[1]
    # Output url file.
    output = sys.argv[2]

    # Get the list of url files.
    files = glob.glob(path + '/*.txt')
    print('> found {} files'.format(len(files)))

    urls = set()
    urls_counter = 0
    domain_blacklist_counter = 0
    extention_blacklist_counter = 0
    short_url_counter = 0
    malformed_url_counter = 0
    duplicate_url_counter = 0
    start_time = time.time()
    for filename in files:
        with open(filename, 'r') as f:
            for line in f:
                url = line.strip()
                urls_counter += 1
                if domain_is_in_blacklist(url):
                    print('[DOMAIN BLACKLIST]: {}'.format(url), flush=True)
                    domain_blacklist_counter += 1
                elif extention_is_in_blacklist(url):
                    print('[EXTENTION BLACKLIST]: {}'.format(url), flush=True)
                    extention_blacklist_counter += 1
                elif len(url) <= 8:
                    print('[SHORT URL]: {}'.format(url), flush=True)
                    short_url_counter += 1
                elif url_is_malformed(url):
                    print('[MALFORMED URL]: {}'.format(url), flush=True)
                    malformed_url_counter += 1
                elif url in urls:
                    print('[DUPLICATE URL]: {}'.format(url), flush=True)
                    duplicate_url_counter += 1
                else:
                    urls.add(url)
                if urls_counter % 100000 == 0:
                    print_progress('PROGRESS', start_time, urls_counter,
                                   domain_blacklist_counter,
                                   extention_blacklist_counter,
                                   short_url_counter, malformed_url_counter,
                                   duplicate_url_counter)

    print_progress('FINAL', start_time, urls_counter,
                   domain_blacklist_counter,
                   extention_blacklist_counter,
                   short_url_counter, malformed_url_counter,
                   duplicate_url_counter)

    # Write the final set of urls.
    print('> writing cleaned up url list to {}'.format(output))
    with open(output, 'w') as f:
        for url in urls:
            f.write(url + '\n')

    print('done :-)')


    'vid',
    'vimeo',
    'vine',
    'vkaao',
    'vocaroo',
    'voyagefusion',
    'walmart',
    'wciu',
    'wikimedia',
    'wikipedia',
    'xhamster',
    'xkcd',
    'xvideos',
    'youtu',
    'youtube',
    'youtubedoubler',
    'ytimg',
    'zillexplorer',
])

def domain_is_in_blacklist(url):
    domain = tldextract.extract(url).domain
    return domain in domain_blacklist

# List of extentions to blacklist.
extentions_blacklist = (
    '.3gp',
    '.7z'
    '.ai',
    '.aif',
    '.apk',
    '.app',
    '.avi',
    '.bin',
    '.bmp',
    '.bz2',
    '.css',
    '.csv',
    '.dat',
    '.deb',
    '.dmg',
    '.doc',


    '.sql',
    '.svg',
    '.swf',
    '.tar.gz',
    '.tar',
    '.tgz',
    '.tiff',
    '.ttf',
    '.txt',
    '.wav',
    '.webm',
    '.wma',
    '.wmv',
    '.xls',
    '.xlsx',
    '.xml',
    '.xz',
    '.zip',
)

def extention_is_in_blacklist(url):
    if url.split('?')[0].lower().endswith(extentions_blacklist):
        return True
    return False

# Malformed urls.
# This function is adapted from:
#   https://stackoverflow.com/questions/7160737/python-how-to-validate-a-url-in-python-malformed-or-not
url_regex = re.compile(
    r'^(?:http)s?://' # http:// or https://
    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
    r'(?::\d+)?' # optional port
    r'(?:/?|[/?]\S+)$', re.IGNORECASE)
def url_is_malformed(url):
    return re.match(url_regex, url) is None

def print_progress(prefix, start_time, urls_counter,
                   domain_blacklist_counter,
                   extention_blacklist_counter,
                   short_url_counter, malformed_url_counter,
                   duplicate_url_counter):


    '.xml',
    '.xz',
    '.zip',
)

def extention_is_in_blacklist(url):
    if url.split('?')[0].lower().endswith(extentions_blacklist):
        return True
    return False

# Malformed urls.
# This function is adapted from:
#   https://stackoverflow.com/questions/7160737/python-how-to-validate-a-url-in-python-malformed-or-not
url_regex = re.compile(
    r'^(?:http)s?://' # http:// or https://
    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
    r'(?::\d+)?' # optional port
    r'(?:/?|[/?]\S+)$', re.IGNORECASE)
def url_is_malformed(url):
    return re.match(url_regex, url) is None

def print_progress(prefix, start_time, urls_counter,
                   domain_blacklist_counter,
                   extention_blacklist_counter,
                   short_url_counter, malformed_url_counter,
                   duplicate_url_counter):
    string = prefix + ' | '
    string += 'time elapsed (s): {:.2f} | '.format(time.time() - start_time)
    string += 'number of urls: {} | '.format(urls_counter)
    string += 'domain blacklisted: {} | '.format(domain_blacklist_counter)
    string += 'extention blacklisted: {} | '.format(extention_blacklist_counter)
    string += 'short urls (<=8): {} | '.format(short_url_counter)
    string += 'malformed urls: {} | '.format(malformed_url_counter)
    string += 'duplicate urls: {}'.format(duplicate_url_counter)
    print(string, flush=True)

if __name__ == '__main__':


def extention_is_in_blacklist(url):
    if url.split('?')[0].lower().endswith(extentions_blacklist):
        return True
    return False

# Malformed urls.
# This function is adapted from:
#   https://stackoverflow.com/questions/7160737/python-how-to-validate-a-url-in-python-malformed-or-not
url_regex = re.compile(
    r'^(?:http)s?://' # http:// or https://
    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
    r'(?::\d+)?' # optional port
    r'(?:/?|[/?]\S+)$', re.IGNORECASE)
def url_is_malformed(url):
    return re.match(url_regex, url) is None

def print_progress(prefix, start_time, urls_counter,
                   domain_blacklist_counter,
                   extention_blacklist_counter,
                   short_url_counter, malformed_url_counter,
                   duplicate_url_counter):
    string = prefix + ' | '
    string += 'time elapsed (s): {:.2f} | '.format(time.time() - start_time)
    string += 'number of urls: {} | '.format(urls_counter)
    string += 'domain blacklisted: {} | '.format(domain_blacklist_counter)
    string += 'extention blacklisted: {} | '.format(extention_blacklist_counter)
    string += 'short urls (<=8): {} | '.format(short_url_counter)
    string += 'malformed urls: {} | '.format(malformed_url_counter)
    string += 'duplicate urls: {}'.format(duplicate_url_counter)
    print(string, flush=True)

if __name__ == '__main__':

    print('remove blacklisted urls ..')

    # Path to the url files.
    path = sys.argv[1]
    # Output url file.
    output = sys.argv[2]

    # Get the list of url files.
    files = glob.glob(path + '/*.txt')
    print('> found {} files'.format(len(files)))

    urls = set()
    urls_counter = 0
    domain_blacklist_counter = 0
    extention_blacklist_counter = 0


# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.

import json
import time
import sys

if __name__ == '__main__':

    url_filename = sys.argv[1]
    data_filename = sys.argv[2]
    output_filename = sys.argv[3]

    urls = set()
    with open(url_filename, 'r') as f:
        for line in f:
            myjson = json.loads(line)
            for key in myjson:
                this_urls = myjson[key]
                for i in range(1, len(this_urls)):
                    urls.add(this_urls[i])
    print('will be removing {} urls'.format(len(urls)), flush=True)

    written_docs = 0
    removed_docs = 0
    removed_chars = 0
    start_time = time.time()
    with open(output_filename, 'wb') as fout:
        with open(data_filename, 'r') as fin:
            for line in fin:
                try:
                    myjson = json.loads(line)
                    url = myjson['url']
                    if url in urls:
                        print('removing', myjson)
                        removed_docs += 1
                        removed_chars += len(myjson['text'])
                        continue
                    myjson = json.dumps(myjson, ensure_ascii=False)
                    fout.write(myjson.encode('utf-8'))
                    fout.write('\n'.encode('utf-8'))
                    written_docs += 1
                    if written_docs % 10000 == 0:
                        print(' [PROCESSED] time (s): {:.2f} | written: {} '
                              '| removed: {} (char: {})'.format(
                                  time.time() - start_time,
                                  written_docs, removed_docs, removed_chars))
                except Exception as e:
                    print('[SKIPPING]', line, e)

    print(' [PROCESSED] time (s): {:.2f} | written: {} '
          '| removed: {} (char: {})'.format(
              time.time() - start_time,
              written_docs, removed_docs, removed_chars))
    print('done :-)')


# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.

import argparse
from functools import partial
import itertools
import json
from lsh import cache, minhash
import multiprocessing
import numpy as np
import time
import pickle
import sys
import os

# This function is adapted from:
#   https://github.com/mattilyra/LSH/blob/master/examples/Introduction.ipynb
def shingles(text, char_ngram=5):
    return set(text[head:head + char_ngram]
               for head in range(0, len(text) - char_ngram))

# This function is adapted from:
#  https://github.com/mattilyra/LSH/blob/master/examples/Introduction.ipynb
def jaccard(set_a, set_b, args):
    if len(set_a) < 1 or len(set_b) < 1:
        return 0.0

    intersection = set_a & set_b
    union = set_a | set_b

    if args.jaccard == 'min':
        return len(intersection) / min(len(set_a), len(set_b))
    elif args.jaccard == 'max':
        return len(intersection) / max(len(set_a), len(set_b))
    else:
        return len(intersection) / len(union)

def compute_fingerprint(line, key):
    try:
        myjson = json.loads(line)
        url = myjson[key]
        text = myjson['text']
        fingerprint = hasher.fingerprint(text)
    except Exception as e:
        print('Error:', e)
        return None, None, None, False

    return url, text, fingerprint, True

def url_pairs_to_remove(args, bucket_urls, url_doc):
    remove_urls_list = []
    deduped_local, counter_local = 0, 0
    iteration = 0
    while len(bucket_urls) > 1:
        if args.heuristic_iter != -1 and \
            iteration == args.heuristic_iter:
            break

        items = list(bucket_urls)
        remove_urls = []
        main_url = items[np.random.randint(0, len(items))]
        main_dhingles = shingles(url_doc[main_url])

        for i in range(0, len(items)):
            counter_local += 1
            other_url = items[i]
            if other_url == main_url:
                continue
            other_shingles = shingles(url_doc[other_url])
            try:
                jaccard_sim = jaccard(main_dhingles, other_shingles, args)
            except Exception as e:
                print('Error:', e)
                jaccard_sim = 0.0
            if jaccard_sim > 0.5:
                remove_urls.append({other_url: jaccard_sim})
                deduped_local += 1
                bucket_urls.remove(other_url)

        bucket_urls.remove(main_url)
        if len(remove_urls) > 0:
            remove_urls_list.append({main_url: remove_urls})
        iteration += 1
    return remove_urls_list, deduped_local, counter_local

def write_remove_urls_list(remove_urls_list, f_out):
    if len(remove_urls_list) > 0:
        for each_url_remove in remove_urls_list:
            myjson = json.dumps(each_url_remove, ensure_ascii=False)
            f_out.write(myjson.encode('utf-8'))
            f_out.write('\n'.encode('utf-8'))

def compute_jaccard(each_bin, num_bins, start_time_local):

    remove_urls_list = []
    deduped_local, counter_local, bucket_local = 0, 0, 0

    for bucket_id in each_bin:
        bucket_local += 1
        if os.getpid() % num_bins == 0 and bucket_local % 100000 == 0:
            print("Counter {}, progress {:.2f} time {:.2f}".\
                format(bucket_local, float(bucket_local)/float(len(each_bin)),\
                time.time() - start_time_local), flush=True)

        if len(each_bin[bucket_id]) <= 1:
            continue

        bucket_urls = each_bin[bucket_id].copy()
        remove_urls_list_sub, deduped_local_sub, counter_local_sub = \
            url_pairs_to_remove(args, bucket_urls, url_doc)

        deduped_local += deduped_local_sub
        counter_local += counter_local_sub
        if len(remove_urls_list_sub) > 0:
            remove_urls_list.extend(remove_urls_list_sub)

    return remove_urls_list, deduped_local, counter_local

def find_pair_urls_parallel(args, lshcache, url_doc):
    start_time = time.time()
    f_out = open(args.output, 'wb')
    deduped, counter = 0, 0

    # compute jaccards of buckets in bin in parallel (parallelism
    # limited to # of bins)
    num_bins = len(lshcache.bins)
    pool = multiprocessing.Pool(num_bins)
    compute_jaccard_partial = partial(compute_jaccard, num_bins=num_bins, \
        start_time_local=start_time)
    # don't need to pass args and url_doc as they are already shared
    compute_jaccard_iter = pool.imap(compute_jaccard_partial, lshcache.bins)

    print("multiprocessing init took {:.2f}".format(time.time() - start_time),\
        flush=True)
    for remove_urls_list, deduped_local, counter_local in compute_jaccard_iter:
        deduped += deduped_local
        counter += counter_local
        write_remove_urls_list(remove_urls_list, f_out)
        print(' [write]> processed {} documents in {:.2f} '
            'seoncds and deduped {} documents ...'.format(counter, time.time()\
            - start_time, deduped), flush=True)

    pool.close()
    pool.join()
    f_out.close()

    print(' Taken time for jaccard similariries {:.2f} seconds'.format(\
        time.time() - start_time), flush=True)

def find_pair_urls_sequential(args, lshcache, url_doc):
    start_time = time.time()
    f_out = open(args.output, 'wb')
    deduped, counter = 0, 0
    for b in lshcache.bins:
        for bucket_id in b:
            if len(b[bucket_id]) <= 1:
                continue

            bucket_urls = b[bucket_id].copy()
            remove_urls_list_sub, deduped_local_sub, counter_local_sub = \
                url_pairs_to_remove(args, bucket_urls, url_doc)

            deduped += deduped_local_sub
            counter += counter_local_sub
            write_remove_urls_list(remove_urls_list_sub, f_out)
            if counter % 10000 == 0:
                print(' [write]> processed {} documents in {:.2f} '
                    'seoncds and deduped {} documents ...'.
                    format(counter, time.time() - start_time,
                    deduped), flush=True)
    f_out.close()
    print(' [write]> processed {} documents in {:.2f} '
        'seoncds and deduped {} documents ...'.
        format(counter, time.time() - start_time,
        deduped), flush=True)

if __name__ == '__main__':

    print('parsing the arguments ...')

    parser = argparse.ArgumentParser()
    parser.add_argument('--seed', type=int, default=1234,
                       help='Random seed used for python, numpy')
    parser.add_argument('--inputs', nargs = '*', default=None, help = \
                        'Pairwise list of the input files and keys, '
                        'e.g. --inputs cc.json cc_id news.json news_id')
    parser.add_argument('--load-fingerprints', nargs = '*', default=None,
                       help='Load fingerprints from a list of pickle files,'
                        ' e.g. cc.pkl news.pkl')
    parser.add_argument('--save-fingerprints', type=str, default=None,
                       help='Save the fingerprints of the inputs.')
    parser.add_argument('--output', type=str, default=None,
                       help='Output file name that consists of all ids'
                        ' with matching similarities')
    parser.add_argument('--jaccard', type=str, default='union',
                        choices=['union', 'min', 'max'], help='Jaccard'\
                        ' similarity computation')
    parser.add_argument('--heuristic-iter', type=int, default=1,
                       help='Number of iterations to run the heuristics'
                        ': use -1 for exact')
    parser.add_argument('--num-bands', type=int, default=10,
                       help='Number of bands to use in cache')
    parser.add_argument('--num-seeds', type=int, default=100,
                       help='Number of seeds to use for minhash. Note that'
                        ' this value should be divisible by num-bands')
    parser.add_argument('--jaccard-parallel', action='store_true',
                       help='Use this to process large number of documents.')
    args = parser.parse_args()

    print('finding possible duplicate content ...')

    # set seed and get an array of seeds of 100 integers
    np.random.seed(args.seed)
    seeds = np.random.randint(0, 1e6, size=args.num_seeds)

    # initialize minhash and lsh cache
    hasher = minhash.MinHasher(seeds=seeds, char_ngram=5, hashbytes=4)
    lshcache = cache.Cache(num_bands=args.num_bands, hasher=hasher)

    url_doc = {}

    # load fingerprints from pickle file if needed
    if args.load_fingerprints is not None:
        for count_fp, fp_file_name in enumerate(args.load_fingerprints):
            print("Loading fingerprints from pickle file {}".format(
                fp_file_name), flush=True)
            fp = open(fp_file_name, "rb")
            if count_fp == 0:
                # assign directory for the first pkl
                lshcache = pickle.load(fp)
                url_doc = pickle.load(fp)
            else:
                # append these to lshcache and url_doc
                local_lshcache = pickle.load(fp)
                local_url_doc = pickle.load(fp)
                for url in local_lshcache.fingerprints.keys():
                    url_doc[url] = local_url_doc[url]
                    lshcache.add_fingerprint(local_lshcache.fingerprints[url], url)
            fp.close()

    counter = 0
    start_time = time.time()

    # compute finger prints of the inputs if any
    # input file and the key to use as id
    if args.inputs is not None:
        print("Computing fingerprints", flush=True)
        assert len(args.inputs) % 2 == 0
        for input_file, key in zip(args.inputs[::2], args.inputs[1::2]):
            print(' document processing {} with key {}'.format(input_file, key),
                flush=True)

            # compute fingerprints in parallel
            num_workers = 40
            pool = multiprocessing.Pool(num_workers)
            fin = open(input_file, 'r', encoding='utf-8')
            compute_fingerprint_partial = partial(compute_fingerprint, key=key)
            compute_fingerprint_iter = pool.imap(compute_fingerprint_partial,
                                                    fin, 512)
            # traverse all the texts and add fingerprints
            for url, text, fingerprint, flag in compute_fingerprint_iter:
                counter += 1
                if flag:
                    url_doc[url] = text
                    lshcache.add_fingerprint(fingerprint, url)
                if counter % 10000 == 0:
                    print(' [read]> processed {} documents in {:.2f} '
                        'seconds ...'.format(counter, time.time() - \
                        start_time), flush=True)

            fin.close()
            pool.close()
            pool.join()

    # Save the fingerprints if needed
    if args.save_fingerprints is not None:
        print("Saving fingerprints to pickle file {}".format(
            args.save_fingerprints), flush=True)
        with open(args.save_fingerprints, 'wb') as f_save:
            pickle.dump(lshcache, f_save)
            pickle.dump(url_doc, f_save)

    # compute jaccard index of the input texts and write to file if needed
    if args.output is not None:
        print("Compute jaccard similarity", flush=True)
        if args.jaccard_parallel:
            find_pair_urls_parallel(args, lshcache, url_doc)
        else:
            find_pair_urls_sequential(args, lshcache, url_doc)

    print('done :-)')


shingles

# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.

import argparse
from functools import partial
import itertools
import json
from lsh import cache, minhash
import multiprocessing
import numpy as np
import time
import pickle
import sys
import os

# This function is adapted from:
#   https://github.com/mattilyra/LSH/blob/master/examples/Introduction.ipynb
def shingles(text, char_ngram=5):
    return set(text[head:head + char_ngram]
               for head in range(0, len(text) - char_ngram))

# This function is adapted from:
#  https://github.com/mattilyra/LSH/blob/master/examples/Introduction.ipynb
def jaccard(set_a, set_b, args):
    if len(set_a) < 1 or len(set_b) < 1:
        return 0.0

    intersection = set_a & set_b
    union = set_a | set_b

    if args.jaccard == 'min':
        return len(intersection) / min(len(set_a), len(set_b))
    elif args.jaccard == 'max':
        return len(intersection) / max(len(set_a), len(set_b))
    else:
        return len(intersection) / len(union)

def compute_fingerprint(line, key):
    try:


jaccard

from functools import partial
import itertools
import json
from lsh import cache, minhash
import multiprocessing
import numpy as np
import time
import pickle
import sys
import os

# This function is adapted from:
#   https://github.com/mattilyra/LSH/blob/master/examples/Introduction.ipynb
def shingles(text, char_ngram=5):
    return set(text[head:head + char_ngram]
               for head in range(0, len(text) - char_ngram))

# This function is adapted from:
#  https://github.com/mattilyra/LSH/blob/master/examples/Introduction.ipynb
def jaccard(set_a, set_b, args):
    if len(set_a) < 1 or len(set_b) < 1:
        return 0.0

    intersection = set_a & set_b
    union = set_a | set_b

    if args.jaccard == 'min':
        return len(intersection) / min(len(set_a), len(set_b))
    elif args.jaccard == 'max':
        return len(intersection) / max(len(set_a), len(set_b))
    else:
        return len(intersection) / len(union)

def compute_fingerprint(line, key):
    try:
        myjson = json.loads(line)
        url = myjson[key]
        text = myjson['text']
        fingerprint = hasher.fingerprint(text)
    except Exception as e:
        print('Error:', e)
        return None, None, None, False

    return url, text, fingerprint, True

def url_pairs_to_remove(args, bucket_urls, url_doc):
    remove_urls_list = []
    deduped_local, counter_local = 0, 0
    iteration = 0
    while len(bucket_urls) > 1:
        if args.heuristic_iter != -1 and \
            iteration == args.heuristic_iter:


    return set(text[head:head + char_ngram]
               for head in range(0, len(text) - char_ngram))

# This function is adapted from:
#  https://github.com/mattilyra/LSH/blob/master/examples/Introduction.ipynb
def jaccard(set_a, set_b, args):
    if len(set_a) < 1 or len(set_b) < 1:
        return 0.0

    intersection = set_a & set_b
    union = set_a | set_b

    if args.jaccard == 'min':
        return len(intersection) / min(len(set_a), len(set_b))
    elif args.jaccard == 'max':
        return len(intersection) / max(len(set_a), len(set_b))
    else:
        return len(intersection) / len(union)

def compute_fingerprint(line, key):
    try:
        myjson = json.loads(line)
        url = myjson[key]
        text = myjson['text']
        fingerprint = hasher.fingerprint(text)
    except Exception as e:
        print('Error:', e)
        return None, None, None, False

    return url, text, fingerprint, True

def url_pairs_to_remove(args, bucket_urls, url_doc):
    remove_urls_list = []
    deduped_local, counter_local = 0, 0
    iteration = 0
    while len(bucket_urls) > 1:
        if args.heuristic_iter != -1 and \
            iteration == args.heuristic_iter:
            break

        items = list(bucket_urls)
        remove_urls = []
        main_url = items[np.random.randint(0, len(items))]
        main_dhingles = shingles(url_doc[main_url])

        for i in range(0, len(items)):
            counter_local += 1
            other_url = items[i]
            if other_url == main_url:
                continue


    if args.jaccard == 'min':
        return len(intersection) / min(len(set_a), len(set_b))
    elif args.jaccard == 'max':
        return len(intersection) / max(len(set_a), len(set_b))
    else:
        return len(intersection) / len(union)

def compute_fingerprint(line, key):
    try:
        myjson = json.loads(line)
        url = myjson[key]
        text = myjson['text']
        fingerprint = hasher.fingerprint(text)
    except Exception as e:
        print('Error:', e)
        return None, None, None, False

    return url, text, fingerprint, True

def url_pairs_to_remove(args, bucket_urls, url_doc):
    remove_urls_list = []
    deduped_local, counter_local = 0, 0
    iteration = 0
    while len(bucket_urls) > 1:
        if args.heuristic_iter != -1 and \
            iteration == args.heuristic_iter:
            break

        items = list(bucket_urls)
        remove_urls = []
        main_url = items[np.random.randint(0, len(items))]
        main_dhingles = shingles(url_doc[main_url])

        for i in range(0, len(items)):
            counter_local += 1
            other_url = items[i]
            if other_url == main_url:
                continue
            other_shingles = shingles(url_doc[other_url])
            try:
                jaccard_sim = jaccard(main_dhingles, other_shingles, args)
            except Exception as e:
                print('Error:', e)
                jaccard_sim = 0.0
            if jaccard_sim > 0.5:
                remove_urls.append({other_url: jaccard_sim})
                deduped_local += 1
                bucket_urls.remove(other_url)

        bucket_urls.remove(main_url)
        if len(remove_urls) > 0:
            remove_urls_list.append({main_url: remove_urls})
        iteration += 1
    return remove_urls_list, deduped_local, counter_local

def write_remove_urls_list(remove_urls_list, f_out):
    if len(remove_urls_list) > 0:
        for each_url_remove in remove_urls_list:
            myjson = json.dumps(each_url_remove, ensure_ascii=False)
            f_out.write(myjson.encode('utf-8'))
            f_out.write('\n'.encode('utf-8'))

def compute_jaccard(each_bin, num_bins, start_time_local):

    remove_urls_list = []
    deduped_local, counter_local, bucket_local = 0, 0, 0

    for bucket_id in each_bin:
        bucket_local += 1
        if os.getpid() % num_bins == 0 and bucket_local % 100000 == 0:
            print("Counter {}, progress {:.2f} time {:.2f}".\
                format(bucket_local, float(bucket_local)/float(len(each_bin)),\
                time.time() - start_time_local), flush=True)


            other_url = items[i]
            if other_url == main_url:
                continue
            other_shingles = shingles(url_doc[other_url])
            try:
                jaccard_sim = jaccard(main_dhingles, other_shingles, args)
            except Exception as e:
                print('Error:', e)
                jaccard_sim = 0.0
            if jaccard_sim > 0.5:
                remove_urls.append({other_url: jaccard_sim})
                deduped_local += 1
                bucket_urls.remove(other_url)

        bucket_urls.remove(main_url)
        if len(remove_urls) > 0:
            remove_urls_list.append({main_url: remove_urls})
        iteration += 1
    return remove_urls_list, deduped_local, counter_local

def write_remove_urls_list(remove_urls_list, f_out):
    if len(remove_urls_list) > 0:
        for each_url_remove in remove_urls_list:
            myjson = json.dumps(each_url_remove, ensure_ascii=False)
            f_out.write(myjson.encode('utf-8'))
            f_out.write('\n'.encode('utf-8'))

def compute_jaccard(each_bin, num_bins, start_time_local):

    remove_urls_list = []
    deduped_local, counter_local, bucket_local = 0, 0, 0

    for bucket_id in each_bin:
        bucket_local += 1
        if os.getpid() % num_bins == 0 and bucket_local % 100000 == 0:
            print("Counter {}, progress {:.2f} time {:.2f}".\
                format(bucket_local, float(bucket_local)/float(len(each_bin)),\
                time.time() - start_time_local), flush=True)

        if len(each_bin[bucket_id]) <= 1:
            continue

        bucket_urls = each_bin[bucket_id].copy()
        remove_urls_list_sub, deduped_local_sub, counter_local_sub = \
            url_pairs_to_remove(args, bucket_urls, url_doc)


                print('Error:', e)
                jaccard_sim = 0.0
            if jaccard_sim > 0.5:
                remove_urls.append({other_url: jaccard_sim})
                deduped_local += 1
                bucket_urls.remove(other_url)

        bucket_urls.remove(main_url)
        if len(remove_urls) > 0:
            remove_urls_list.append({main_url: remove_urls})
        iteration += 1
    return remove_urls_list, deduped_local, counter_local

def write_remove_urls_list(remove_urls_list, f_out):
    if len(remove_urls_list) > 0:
        for each_url_remove in remove_urls_list:
            myjson = json.dumps(each_url_remove, ensure_ascii=False)
            f_out.write(myjson.encode('utf-8'))
            f_out.write('\n'.encode('utf-8'))

def compute_jaccard(each_bin, num_bins, start_time_local):

    remove_urls_list = []
    deduped_local, counter_local, bucket_local = 0, 0, 0

    for bucket_id in each_bin:
        bucket_local += 1
        if os.getpid() % num_bins == 0 and bucket_local % 100000 == 0:
            print("Counter {}, progress {:.2f} time {:.2f}".\
                format(bucket_local, float(bucket_local)/float(len(each_bin)),\
                time.time() - start_time_local), flush=True)

        if len(each_bin[bucket_id]) <= 1:
            continue

        bucket_urls = each_bin[bucket_id].copy()
        remove_urls_list_sub, deduped_local_sub, counter_local_sub = \
            url_pairs_to_remove(args, bucket_urls, url_doc)

        deduped_local += deduped_local_sub
        counter_local += counter_local_sub
        if len(remove_urls_list_sub) > 0:
            remove_urls_list.extend(remove_urls_list_sub)

    return remove_urls_list, deduped_local, counter_local

def find_pair_urls_parallel(args, lshcache, url_doc):
    start_time = time.time()
    f_out = open(args.output, 'wb')
    deduped, counter = 0, 0

    # compute jaccards of buckets in bin in parallel (parallelism
    # limited to # of bins)
    num_bins = len(lshcache.bins)
    pool = multiprocessing.Pool(num_bins)
    compute_jaccard_partial = partial(compute_jaccard, num_bins=num_bins, \
        start_time_local=start_time)
    # don't need to pass args and url_doc as they are already shared
    compute_jaccard_iter = pool.imap(compute_jaccard_partial, lshcache.bins)

    print("multiprocessing init took {:.2f}".format(time.time() - start_time),\
        flush=True)
    for remove_urls_list, deduped_local, counter_local in compute_jaccard_iter:
        deduped += deduped_local
        counter += counter_local


        bucket_local += 1
        if os.getpid() % num_bins == 0 and bucket_local % 100000 == 0:
            print("Counter {}, progress {:.2f} time {:.2f}".\
                format(bucket_local, float(bucket_local)/float(len(each_bin)),\
                time.time() - start_time_local), flush=True)

        if len(each_bin[bucket_id]) <= 1:
            continue

        bucket_urls = each_bin[bucket_id].copy()
        remove_urls_list_sub, deduped_local_sub, counter_local_sub = \
            url_pairs_to_remove(args, bucket_urls, url_doc)

        deduped_local += deduped_local_sub
        counter_local += counter_local_sub
        if len(remove_urls_list_sub) > 0:
            remove_urls_list.extend(remove_urls_list_sub)

    return remove_urls_list, deduped_local, counter_local

def find_pair_urls_parallel(args, lshcache, url_doc):
    start_time = time.time()
    f_out = open(args.output, 'wb')
    deduped, counter = 0, 0

    # compute jaccards of buckets in bin in parallel (parallelism
    # limited to # of bins)
    num_bins = len(lshcache.bins)
    pool = multiprocessing.Pool(num_bins)
    compute_jaccard_partial = partial(compute_jaccard, num_bins=num_bins, \
        start_time_local=start_time)
    # don't need to pass args and url_doc as they are already shared
    compute_jaccard_iter = pool.imap(compute_jaccard_partial, lshcache.bins)

    print("multiprocessing init took {:.2f}".format(time.time() - start_time),\
        flush=True)
    for remove_urls_list, deduped_local, counter_local in compute_jaccard_iter:
        deduped += deduped_local
        counter += counter_local
        write_remove_urls_list(remove_urls_list, f_out)
        print(' [write]> processed {} documents in {:.2f} '
            'seoncds and deduped {} documents ...'.format(counter, time.time()\
            - start_time, deduped), flush=True)

    pool.close()
    pool.join()
    f_out.close()

    print(' Taken time for jaccard similariries {:.2f} seconds'.format(\
        time.time() - start_time), flush=True)

def find_pair_urls_sequential(args, lshcache, url_doc):
    start_time = time.time()
    f_out = open(args.output, 'wb')
    deduped, counter = 0, 0
    for b in lshcache.bins:
        for bucket_id in b:
            if len(b[bucket_id]) <= 1:
                continue

            bucket_urls = b[bucket_id].copy()
            remove_urls_list_sub, deduped_local_sub, counter_local_sub = \
                url_pairs_to_remove(args, bucket_urls, url_doc)

            deduped += deduped_local_sub
            counter += counter_local_sub
            write_remove_urls_list(remove_urls_list_sub, f_out)
            if counter % 10000 == 0:
                print(' [write]> processed {} documents in {:.2f} '
                    'seoncds and deduped {} documents ...'.


    # don't need to pass args and url_doc as they are already shared
    compute_jaccard_iter = pool.imap(compute_jaccard_partial, lshcache.bins)

    print("multiprocessing init took {:.2f}".format(time.time() - start_time),\
        flush=True)
    for remove_urls_list, deduped_local, counter_local in compute_jaccard_iter:
        deduped += deduped_local
        counter += counter_local
        write_remove_urls_list(remove_urls_list, f_out)
        print(' [write]> processed {} documents in {:.2f} '
            'seoncds and deduped {} documents ...'.format(counter, time.time()\
            - start_time, deduped), flush=True)

    pool.close()
    pool.join()
    f_out.close()

    print(' Taken time for jaccard similariries {:.2f} seconds'.format(\
        time.time() - start_time), flush=True)

def find_pair_urls_sequential(args, lshcache, url_doc):
    start_time = time.time()
    f_out = open(args.output, 'wb')
    deduped, counter = 0, 0
    for b in lshcache.bins:
        for bucket_id in b:
            if len(b[bucket_id]) <= 1:
                continue

            bucket_urls = b[bucket_id].copy()
            remove_urls_list_sub, deduped_local_sub, counter_local_sub = \
                url_pairs_to_remove(args, bucket_urls, url_doc)

            deduped += deduped_local_sub
            counter += counter_local_sub
            write_remove_urls_list(remove_urls_list_sub, f_out)
            if counter % 10000 == 0:
                print(' [write]> processed {} documents in {:.2f} '
                    'seoncds and deduped {} documents ...'.
                    format(counter, time.time() - start_time,
                    deduped), flush=True)
    f_out.close()
    print(' [write]> processed {} documents in {:.2f} '
        'seoncds and deduped {} documents ...'.
        format(counter, time.time() - start_time,
        deduped), flush=True)

if __name__ == '__main__':

    print('parsing the arguments ...')

    parser = argparse.ArgumentParser()
    parser.add_argument('--seed', type=int, default=1234,
                       help='Random seed used for python, numpy')
    parser.add_argument('--inputs', nargs = '*', default=None, help = \
                        'Pairwise list of the input files and keys, '
                        'e.g. --inputs cc.json cc_id news.json news_id')
    parser.add_argument('--load-fingerprints', nargs = '*', default=None,
                       help='Load fingerprints from a list of pickle files,'
                        ' e.g. cc.pkl news.pkl')
    parser.add_argument('--save-fingerprints', type=str, default=None,
                       help='Save the fingerprints of the inputs.')
    parser.add_argument('--output', type=str, default=None,
                       help='Output file name that consists of all ids'
                        ' with matching similarities')
    parser.add_argument('--jaccard', type=str, default='union',


# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.

import argparse
import json
import os
import time

"""
This code adds id to each json object in a json file. User can add prefix
to the ids.
"""

if __name__ == '__main__':

    print('parsing the arguments ...')

    parser = argparse.ArgumentParser()
    parser.add_argument('--input-file', type=str, default=None, help='Input'\
        ' json file where id needs to be added')
    parser.add_argument('--output-file', type=str, default=None, help=\
        'Output file name with id')
    parser.add_argument('--id-prefix', type=str, default=None, help=\
        'Id prefix')
    parser.add_argument('--log-interval', type=int, default=100,
                       help='Log interval')
    args = parser.parse_args()

    print('Adding ids to dataset ...')

    f_input = open(args.input_file, 'r', encoding='utf-8')
    f_output = open(args.output_file, 'wb')

    unique_ids = 1
    start_time = time.time()
    for row in f_input:
        each_row = json.loads(row)
        adlr_id_string = args.id_prefix + '-{:010d}'.format(int(unique_ids))
        each_row['adlr_id'] = adlr_id_string
        myjson = json.dumps(each_row, ensure_ascii=False)

        f_output.write(myjson.encode('utf-8'))
        f_output.write('\n'.encode('utf-8'))

        if unique_ids % args.log_interval == 0:
            print('    processed {:9d} documents in {:.2f} seconds ...'.format( \
                    unique_ids, time.time() - start_time), flush=True)

        unique_ids += 1

    # Close the file.
    f_input.close()
    f_output.close()

    print('done :-)', flush=True)


# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.

import glob
import sys
import json
import argparse

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument("--json_path", type=str, default=".",
        help="path where all the json files are located")

    parser.add_argument("--output_file", type=str, default="merged_output.json",
        help="filename where the merged json should go")

    args = parser.parse_args()

    json_path = args.json_path
    out_file = args.output_file

    json_files = glob.glob(json_path + '/*.json')

    counter = 0

    with open(out_file, 'w') as outfile:
        for fname in json_files:
            counter += 1

            if counter % 1024 == 0:
                print("Merging at ", counter, flush=True)

            with open(fname, 'r') as infile:
                for row in infile:
                    each_row = json.loads(row)
                    outfile.write(row)

    print("Merged file", out_file, flush=True)


# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.

import json
import os
import sys
import types

import torch

def add_arguments(parser):
    group = parser.add_argument_group(title='Megatron loader')

    group.add_argument('--true-vocab-size', type=int, default=None,
                       help='original size of vocab, if specified will trim padding from embedding table.')
    group.add_argument('--vocab-file', type=str, default=None,
                       help='Path to the vocab file. If specified will use this to get vocab size and '
                       'trim padding from the embedding table.')
    group.add_argument('--megatron-path', type=str, default=None,
                       help='Base directory of deepspeed repository')

def _load_checkpoint(queue, args):

    # Search in directory above this
    sys.path.append(os.path.abspath(
        os.path.join(os.path.dirname(__file__),
                     os.path.pardir)))
    if args.megatron_path is not None:
        sys.path.insert(0, args.megatron_path)

    try:
        from megatron.arguments import parse_args, validate_args
        from megatron.global_vars import set_args, set_global_variables
        from megatron.checkpointing import load_args_from_checkpoint, load_checkpoint
        from megatron.model import module
        from megatron.core import mpu
        from megatron.core.enums import ModelType
        from megatron import fused_kernels
    except ModuleNotFoundError:
        print("Unable to import Megatron, please specify the path to Megatron using --megatron-path. Exiting.")
        queue.put("exit")
        exit(1)

    # We want all arguments to come from us
    sys.argv = ['script.py',
                '--no-masked-softmax-fusion',
                '--no-bias-gelu-fusion',
                '--no-bias-dropout-fusion',
                '--no-async-tensor-model-parallel-allreduce',
                '--use-cpu-initialization',
                '--micro-batch-size', '1',
                '--no-load-optim',
                '--no-load-rng',
                '--no-save-optim',
                '--no-save-rng',
                '--no-initialization',
                '--load', args.load_dir,
                '--load-iteration', str(args.load_iteration)
                ]

    margs = parse_args()
    margs, checkpoint_args = load_args_from_checkpoint(margs)

    # Arguments do sanity checks on the world size, but we don't care,
    # so trick it into thinking we are plenty of processes
    margs.world_size = margs.tensor_model_parallel_size * margs.pipeline_model_parallel_size

    margs = validate_args(margs)

    def check_for_arg(arg_name, default=None):
        if getattr(margs, arg_name, None) is None:
            if default is not None:
                setattr(margs, arg_name, default)
            else:
                print(f"Checkpoint does not specify the argument {arg_name}. Exiting.")
                print(f"Arguments: {margs}")
                queue.put("exit")
                exit(1)

    check_for_arg('tensor_model_parallel_size')
    check_for_arg('pipeline_model_parallel_size')
    check_for_arg('num_layers')
    check_for_arg('hidden_size')
    check_for_arg('seq_length')
    check_for_arg('num_attention_heads')
    check_for_arg('max_position_embeddings')
    check_for_arg('position_embedding_type')
    check_for_arg('tokenizer_type')
    check_for_arg('iteration')
    check_for_arg('bert_binary_head')
    check_for_arg('disable_bias_linear', False)
    check_for_arg('params_dtype')
    check_for_arg('swiglu', False)

    # Determine how to make our models
    if args.model_type == 'GPT':
        from pretrain_gpt import model_provider
        margs.model_type = ModelType.encoder_or_decoder
    elif args.model_type == 'BERT':
        from pretrain_bert import model_provider
        margs.model_type = ModelType.encoder_or_decoder
    elif args.model_type == 'EarlyExitGPT':
        from pretrain_early_exit_gpt import model_provider
        margs.model_type = ModelType.encoder_or_decoder
    else:
        raise Exception(f'unrecognized model type: {args.model_type}')

    # supress warning about torch.distributed not being initialized
    module.MegatronModule.embedding_warning_printed = True

    consumed_train_samples = None
    consumed_valid_samples = None
    def get_models(count, dtype):
        nonlocal consumed_train_samples
        nonlocal consumed_valid_samples
        model_array_len = margs.virtual_pipeline_model_parallel_size
        if model_array_len is None:
            model_array_len = 1
        models = [[] for _ in range(model_array_len)]
        pre_process = mpu.is_pipeline_first_stage()
        post_process = mpu.is_pipeline_last_stage()
        for rank in range(count):
            mpu.set_tensor_model_parallel_rank(rank)
            if margs.virtual_pipeline_model_parallel_size is not None:
                model_ = []
                for i in range(margs.virtual_pipeline_model_parallel_size):
                    mpu.set_virtual_pipeline_model_parallel_rank(i)
                    # Set pre_process and post_process only after virtual rank is set.
                    pre_process = mpu.is_pipeline_first_stage()
                    post_process = mpu.is_pipeline_last_stage()
                    this_model = model_provider(
                        pre_process=pre_process,
                        post_process=post_process
                    ).to(dtype)
                    model_.append(this_model)
            else:
                pre_process = mpu.is_pipeline_first_stage()
                post_process = mpu.is_pipeline_last_stage()
                model_rank = 0
                model_ = [model_provider(pre_process, post_process).to(dtype)]
            margs.consumed_train_samples = 0
            margs.consumed_valid_samples = 0
            load_checkpoint(model_, None, None)

            if consumed_train_samples is not None:
                assert(margs.consumed_train_samples == consumed_train_samples)
            else:
                consumed_train_samples = margs.consumed_train_samples
            if consumed_valid_samples is not None:
                assert(margs.consumed_valid_samples == consumed_valid_samples)
            else:
                consumed_valid_samples = margs.consumed_valid_samples
            for vp_rank in range(model_array_len):
                models[vp_rank].append(model_[vp_rank])
        return models

    set_global_variables(margs, build_tokenizer=False)
    mpu.set_tensor_model_parallel_world_size(margs.tensor_model_parallel_size)
    mpu.set_pipeline_model_parallel_world_size(margs.pipeline_model_parallel_size)
    mpu.set_virtual_pipeline_model_parallel_world_size(margs.virtual_pipeline_model_parallel_size)
    fused_kernels.load(margs)

    # Get true (non-padded) vocab size
    if args.true_vocab_size is not None:
        true_vocab_size = args.true_vocab_size
    elif args.vocab_file is not None:
        vocab = json.load(open(args.vocab_file))
        true_vocab_size = len(vocab)
        if args.true_vocab_size is not None and true_vocab_size != args.true_vocab_size:
            print("Both --true-vocab-size and --vocab-file specified and the vocab size does not match, aborting.")
            queue.put("exit")
            exit(1)
    else:
        true_vocab_size = None

    # short aliases
    tp_size = margs.tensor_model_parallel_size
    pp_size = margs.pipeline_model_parallel_size
    vp_size = margs.virtual_pipeline_model_parallel_size
    if vp_size is None:
        vp_size = 1

    # Layernorm has bias; RMSNorm does not.
    if hasattr(checkpoint_args, 'normalization'):
        norm_has_bias = checkpoint_args.normalization == "LayerNorm"
    else:
        # older models only supported LayerNorm
        norm_has_bias = True

    # metadata
    md = types.SimpleNamespace()
    md.model_type = args.model_type
    md.num_layers = margs.num_layers
    md.hidden_size = margs.hidden_size
    md.seq_length = margs.seq_length
    md.num_attention_heads = margs.num_attention_heads
    md.max_position_embeddings = margs.max_position_embeddings
    md.tokenizer_type = margs.tokenizer_type
    md.iteration = margs.iteration
    md.params_dtype = margs.params_dtype
    md.bert_binary_head = margs.bert_binary_head
    md.output_layer = margs.untie_embeddings_and_output_weights
    md.position_embedding_type = margs.position_embedding_type
    md.linear_bias = margs.add_bias_linear
    md.norm_has_bias = norm_has_bias
    md.swiglu = margs.swiglu
    md.previous_tensor_parallel_size = margs.tensor_model_parallel_size
    md.previous_pipeline_parallel_size = margs.pipeline_model_parallel_size
    md.true_vocab_size = true_vocab_size
    md.make_vocab_size_divisible_by = margs.make_vocab_size_divisible_by
    md.exit_layer_nums = margs.exit_layer_nums if hasattr(margs, 'exit_layer_nums') else []
    md.exit_layer_weight = margs.exit_layer_weight if hasattr(margs, 'exit_layer_weight') else []
    md.use_exit_mlp = margs.use_exit_mlp if hasattr(margs, 'use_exit_mlp') else False
    md.use_exit_block = margs.use_exit_block if hasattr(margs, 'use_exit_block') else False
    md.use_exit_norm = margs.use_exit_norm if hasattr(margs, 'use_exit_norm') else False
    md.untie_exit_output_weights = margs.untie_exit_output_weights if hasattr(margs, 'untie_exit_output_weights') else False
    md.pre_exit = margs.pre_exit
    md.checkpoint_args = checkpoint_args

    # Get first pipe stage
    mpu.set_pipeline_model_parallel_rank(0)
    if len(md.exit_layer_nums) > 0:
        layer_per_stage = md.num_layers / margs.pipeline_model_parallel_size
        mpu.set_early_exit_layer_nums(list(filter(lambda x: 0 < x <= layer_per_stage, md.exit_layer_nums)))
        mpu.set_early_exit_stages(list(set(map(lambda layer_num:  int((layer_num - 1) // layer_per_stage), md.exit_layer_nums))))
    all_models = [get_models(tp_size, md.params_dtype)]
    models = all_models[0][0]

    md.consumed_train_samples = consumed_train_samples
    md.consumed_valid_samples = consumed_valid_samples
    queue.put(md)

    def queue_put(name, msg):
        print(f"sending {name}")
        msg["name"] = name
        queue.put(msg)

    # Send embeddings
    message = {
        "word embeddings": torch.cat(
            [models[tp_rank].language_model.embedding.word_embeddings.weight.data for tp_rank in range(tp_size)],
            dim = 0)
    }
    if md.position_embedding_type == 'learned_absolute':
        message["position embeddings"] = models[0].language_model.embedding.position_embeddings.weight.data
    else:
        assert not hasattr(models[0].language_model.embedding, 'position_embeddings')

    queue_put("embeddings", message)

    total_layer_num = 0
    for vp_rank in range(vp_size):
        mpu.set_virtual_pipeline_model_parallel_rank(vp_rank)
        for pp_rank in range(pp_size):
            if pp_rank > 0:
                mpu.set_pipeline_model_parallel_rank(pp_rank)
                if len(md.exit_layer_nums) > 0:
                    mpu.set_early_exit_layer_nums(list(filter(lambda x: (layer_per_stage * pp_rank) < x <= (layer_per_stage * (pp_rank + 1)), md.exit_layer_nums)))
                if vp_rank == 0:
                    all_models.append(get_models(tp_size, md.params_dtype))
            models = all_models[pp_rank][vp_rank]
            for layer_id in range(len(models[0].language_model.encoder.layers)):
                message = {}
                # Get non-parallel tensors from tp_rank 0
                layer = models[0].language_model.encoder.layers[layer_id]
                layer_num = layer.layer_number
                has_early_exit = layer_num in md.exit_layer_nums
                use_exit_mlp = has_early_exit and md.use_exit_mlp
                use_exit_block = has_early_exit and md.use_exit_block
                use_exit_norm = has_early_exit and md.use_exit_norm
                message["input norm weight"] = layer.input_norm.weight.data
                if norm_has_bias:
                    message["input norm bias"] = layer.input_norm.bias.data
                message["post norm weight"] = layer.post_attention_norm.weight.data
                if norm_has_bias:
                    message["post norm bias"] = layer.post_attention_norm.bias.data
                if use_exit_norm:
                    message["exit norm weight"] = layer.exit_norm.weight.data
                    if norm_has_bias:
                        message['exit norm bias'] = layer.exit_norm.bias.data
                if md.linear_bias:
                    message["dense bias"] = layer.self_attention.dense.bias.data
                    if use_exit_mlp:
                        message["mlp l1 bias"] = layer.mlp.trunk.dense_4h_to_h.bias.data
                        message["mlp l1 exit bias"] = layer.mlp.branch.dense_4h_to_h.bias.data
                    else:
                        message["mlp l1 bias"] = layer.mlp.dense_4h_to_h.bias.data
                if use_exit_block:
                    message["exit block input norm weight"] = layer.exit_block.input_norm.weight.data
                    message["exit block post norm weight"] = layer.exit_block.post_attention_norm.weight.data
                    if norm_has_bias:
                        message["exit block input norm bias"] = layer.exit_block.input_norm.bias.data
                        message["exit block post norm bias"] = layer.exit_block.post_attention_norm.bias.data
                    if md.linear_bias:
                        message["exit block dense bias"] = layer.exit_block.self_attention.dense.bias.data
                        message["exit block mlp l1 bias"] = layer.exit_block.mlp.dense_4h_to_h.bias.data

                # Grab all parallel tensors for this layer
                qkv_weight = []
                qkv_bias = []
                dense_weight = []
                mlp_l0_weight = []
                mlp_l0_bias = []
                mlp_l1_weight = []
                mlp_l0_exit_weight = []
                mlp_l0_exit_bias = []
                mlp_l1_exit_weight  = []
                exit_output_weight = []
                exit_block_qkv_weight = []
                exit_block_qkv_bias = []
                exit_block_dense_weight = []
                exit_block_mlp_l0_weight = []
                exit_block_mlp_l0_bias = []
                exit_block_mlp_l1_weight = []
                for tp_rank, model in enumerate(models):
                    layer = model.language_model.encoder.layers[layer_id]
                    qkv_weight.append(layer.self_attention.query_key_value.weight.data)
                    dense_weight.append(layer.self_attention.dense.weight.data)
                    if use_exit_mlp:
                        mlp_l0_weight.append(layer.mlp.trunk.dense_h_to_4h.weight.data)
                        mlp_l1_weight.append(layer.mlp.trunk.dense_4h_to_h.weight.data)
                        mlp_l0_exit_weight.append(layer.mlp.branch.dense_h_to_4h.weight.data)
                        mlp_l1_exit_weight.append(layer.mlp.branch.dense_4h_to_h.weight.data)
                    else:
                        mlp_l0_weight.append(layer.mlp.dense_h_to_4h.weight.data)
                        mlp_l1_weight.append(layer.mlp.dense_4h_to_h.weight.data)
                    if has_early_exit and md.untie_exit_output_weights:
                        exit_output_weight.append(model.language_model.encoder.exit_output_weights[layer_num].data)
                    if md.linear_bias:
                        qkv_bias.append(layer.self_attention.query_key_value.bias.data)
                        if use_exit_mlp:
                            mlp_l0_bias.append(layer.mlp.trunk.dense_h_to_4h.bias.data)
                            mlp_l0_exit_bias.append(layer.mlp.branch.dense_h_to_4h.bias.data)
                        else:
                            mlp_l0_bias.append(layer.mlp.dense_h_to_4h.bias.data)
                    if use_exit_block:
                        exit_block_qkv_weight.append(layer.exit_block.self_attention.query_key_value.weight.data)
                        exit_block_dense_weight.append(layer.exit_block.self_attention.dense.weight.data)
                        exit_block_mlp_l0_weight.append(layer.exit_block.mlp.dense_h_to_4h.weight.data)
                        exit_block_mlp_l1_weight.append(layer.exit_block.mlp.dense_4h_to_h.weight.data)
                        if md.linear_bias:
                            exit_block_qkv_bias.append(layer.exit_block.self_attention.query_key_value.bias.data)
                            exit_block_mlp_l0_bias.append(layer.exit_block.mlp.dense_h_to_4h.bias.data)

                # Handle gated linear units
                if md.swiglu:
                    # concat all the first halves ('W's) and all the second halves ('V's)
                    for tp_rank in range(tp_size):
                        mlp_l0_weight[tp_rank] = torch.chunk(mlp_l0_weight[tp_rank], 2, dim=0)
                        if use_exit_mlp:
                            mlp_l0_exit_weight[tp_rank] = torch.chunk(mlp_l0_exit_weight[tp_rank], 2, dim=0)
                    message["mlp l0 weight W"] = torch.cat([w[0] for w in mlp_l0_weight], dim=0)
                    message["mlp l0 weight V"] = torch.cat([w[1] for w in mlp_l0_weight], dim=0)
                    if use_exit_mlp:
                        message["mlp l0 exit weight W"] = torch.cat([w[0] for w in mlp_l0_exit_weight], dim=0)
                        message["mlp l0 exit weight V"] = torch.cat([w[1] for w in mlp_l0_exit_weight], dim=0)
                else:
                    message["mlp l0 weight"] = torch.cat(mlp_l0_weight, dim=0)
                    if use_exit_mlp:
                        message["mlp l0 exit weight"] = torch.cat(mlp_l0_exit_weight, dim=0)

                # simple concat of the rest
                message["qkv weight"] = torch.cat(qkv_weight, dim=0)
                message["dense weight"] = torch.cat(dense_weight, dim=1)
                message["mlp l1 weight"] = torch.cat(mlp_l1_weight, dim=1)
                if use_exit_mlp:
                    message["mlp l1 exit weight"] = torch.cat(mlp_l1_exit_weight, dim=1)
                if has_early_exit and md.untie_exit_output_weights:
                    message["exit output weight"] = torch.cat(exit_output_weight, dim=0)
                if md.linear_bias:
                    message["qkv bias"] = torch.cat(qkv_bias, dim=0)
                    if md.swiglu:
                        for tp_rank in range(tp_size):
                            mlp_l0_bias[tp_rank] = torch.chunk(mlp_l0_bias[tp_rank], 2, dim=0)
                            if use_exit_mlp:
                                mlp_l0_exit_bias[tp_rank] = torch.chunk(mlp_l0_exit_bias[tp_rank], 2, dim=0)
                        message["mlp l0 bias W"] = torch.cat([b[0] for b in mlp_l0_bias],dim=0)
                        message["mlp l0 bias V"] = torch.cat([b[1] for b in mlp_l0_bias],dim=0)
                        if use_exit_mlp:
                            message["mlp l0 exit bias W"] = torch.cat([b[0] for b in mlp_l0_exit_bias],dim=0)
                            message["mlp l0 exit bias V"] = torch.cat([b[1] for b in mlp_l0_exit_bias],dim=0)
                    else:
                        message["mlp l0 bias"] = torch.cat(mlp_l0_bias, dim=0)
                        if use_exit_mlp:
                            message["mlp l0 exit bias"] = torch.cat(mlp_l0_exit_bias, dim=0)
                if use_exit_block:
                    if md.swiglu:
                        for tp_rank in range(tp_size):
                            exit_block_mlp_l0_weight[tp_rank] = torch.chunk(exit_block_mlp_l0_weight[tp_rank], 2, dim=0)
                        message["exit block mlp l0 weight W"] = torch.cat([w[0] for w in exit_block_mlp_l0_weight], dim=0)
                        message["exit block mlp l0 weight V"] = torch.cat([w[1] for w in exit_block_mlp_l0_weigh
# ... truncated ...