Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+dir/outputs/train_t7978754709018/nnet_e0_i250.p filter=lfs diff=lfs merge=lfs -text

dir/bucket_by_conflicting_nodes_5.py

@@ -0,0 +1,73 @@
+import pickle
+import os
+import bz2
+
+
+def harmonic(P, R):
+    """For Calculation of F-Score since it is the HM of P and R"""
+    return(2 * P * R / float(P + R))
+# Test on a couple of files
+base_path_csv = '/home/rs/15CS91R05/gaurav/myTryouts/init_results/prediction_csvs/'
+base_path_bz2 = '/home/rs/15CS91R05/Bishal/NewData/skt_dcs_DS.bz2_1L_bigram_heldout_dev/'
+
+pred_csvs = os.listdir(base_path_csv)
+
+
+"""Task 5: See data from number of conflicts
+    Approach: Select a node from DCS, take count of conflicting nodes using the conflictsDict_correct
+"""
+
+# Function to open bz2 files (that contains both DCS & SKT info)
+
+
+def open_dsbz2(filename):
+    with bz2.BZ2File(filename, 'r') as f:
+        loader = pickle.load(f)
+
+    conflicts_Dict_correct = loader['conflicts_Dict_correct']
+    nodelist_to_correct_mapping = loader['nodelist_to_correct_mapping']
+    nodelist_correct = loader['nodelist_correct']
+    featVMat_correct = loader['featVMat_correct']
+    featVMat = loader['featVMat']
+    conflicts_Dict = loader['conflicts_Dict']
+    nodelist = loader['nodelist']
+
+    return (nodelist_correct, conflicts_Dict_correct, featVMat_correct, nodelist_to_correct_mapping,\
+            nodelist, conflicts_Dict, featVMat)
+
+bucket_by_conflicting_nodes = {}
+num_conflicting_nodes = set()
+csv = open(base_path_csv + pred_csvs[5], 'r').readlines()
+
+for line in range(0, len(csv), 6):
+    head_line = csv[line].strip().split(',')
+    fname = head_line[0]
+    print("Bz2 File number", fname, line / 6)
+
+    (nodelist_correct, conflicts_Dict_correct, featVMat_correct, nodelist_to_correct_mapping,\
+        nodelist, conflicts_Dict, featVMat) = open_dsbz2(base_path_bz2 + fname + '.ds.bz2')
+
+    assert len(nodelist_correct) == len(conflicts_Dict_correct)
+    for node in conflicts_Dict_correct:
+        lemma = nodelist_correct[node].lemma
+        conflicting_nodes_count = len(conflicts_Dict_correct[node])
+        if conflicting_nodes_count not in num_conflicting_nodes:
+            num_conflicting_nodes.add(conflicting_nodes_count)
+            bucket_by_conflicting_nodes[conflicting_nodes_count] = {'lemmas': set(), 'precision': [0, 0], 'recall': [0, 0]}
+
+        bucket_by_conflicting_nodes[conflicting_nodes_count]['lemmas'].add(lemma)
+
+    data = csv[line + 5].strip().split(',')
+    word_recall = float(data[1]) / float(data[3])
+    lemma_recall = float(data[2]) / float(data[3])
+    word_precision = float(data[1]) / float(data[4])
+    lemma_precision = float(data[2]) / float(data[4])
+
+    bucket_by_conflicting_nodes[conflicting_nodes_count]['recall'][0] += word_recall
+    bucket_by_conflicting_nodes[conflicting_nodes_count]['recall'][1] += lemma_recall
+    bucket_by_conflicting_nodes[conflicting_nodes_count]['precision'][0] += word_precision
+    bucket_by_conflicting_nodes[conflicting_nodes_count]['precision'][1] += lemma_precision
+
+
+with open('final_task_gaurav/bucket_by_conflicting_nodes_5.p', 'wb') as f:
+    pickle.dump(bucket_by_conflicting_nodes, f)

dir/outputs/nnet_LOGS.csv

@@ -0,0 +1,13 @@
+odir,p_name,hidden_layer_size,_edge_vector_dim
+outputs/train_t8910478283364,outputs/train_t8910478283364/nnet.p,1200,1500,wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t8910591553006,outputs/train_t8910591553006/nnet.p,1200,1500,wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t8911603194394,outputs/train_t8911603194394/nnet.p,1200,1500,wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t8936290095166,outputs/train_t8936290095166/nnet.p,1200,1500,wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t9360928477371,outputs/train_t9360928477371/nnet.p,1200,1500,../wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t9557657979896,outputs/train_t9557657979896/nnet.p,1200,1500,../wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t50078795761,outputs/train_t50078795761/nnet.p,1200,1500,../wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t3896665073989,outputs/train_t3896665073989/nnet.p,1200,1500,../wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t3896696274425,outputs/train_t3896696274425/nnet.p,1200,1500,../wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t3896890268250,outputs/train_t3896890268250/nnet.p,1200,1500,../wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t3896948866803,outputs/train_t3896948866803/nnet.p,1200,1500,../wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/
+outputs/train_t3896991835776,outputs/train_t3896991835776/nnet.p,1200,1500,../wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/

dir/outputs/train_t7978754709018/nnet_e0_i250.p

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b224a77e8418b829f233fe6080bd5f1d596f22e4a735f696d0e6df4032a9e7a5
+size 14419601

dir/sh_TestPool_MP.py

@@ -0,0 +1,168 @@
+import multiprocessing as mp
+import TestPool_Unit
+from shutil import copyfile
+import numpy as np
+import time
+import sys
+from optparse import OptionParser
+
+from collections import defaultdict
+
+def Evaluate(result_arr):
+    print('Files Processed: ', len(result_arr))
+    recalls = []
+    recalls_of_word = []
+    precisions = []
+    precisions_of_words = []
+    fully_Correct_l = 0
+    fully_Correct_w = 0
+    for entry in result_arr:
+        (word_match, lemma_match, n_dcsWords, n_output_nodes) = entry
+        recalls.append(lemma_match/n_dcsWords)
+        recalls_of_word.append(word_match/n_dcsWords)
+
+        precisions.append(lemma_match/n_output_nodes)
+        precisions_of_words.append(word_match/n_output_nodes)
+        if lemma_match == n_dcsWords:
+            fully_Correct_l += 1
+        if word_match == n_dcsWords:
+            fully_Correct_w += 1
+    print('Avg. Micro Recall of Lemmas: {}'.format(np.mean(np.array(recalls))))
+    print('Avg. Micro Recall of Words: {}'.format(np.mean(np.array(recalls_of_word))))
+    print('Avg. Micro Precision of Lemmas: {}'.format(np.mean(np.array(precisions))))
+    print('Avg. Micro Precision of Words: {}'.format(np.mean(np.array(precisions_of_words))))
+    rl = np.mean(np.array(recalls))
+    pl = np.mean(np.array(precisions))
+    print('F-Score of Lemmas: ', (2*pl*rl)/(pl+rl))
+    print('Fully Correct Lemmawise: {}'.format(fully_Correct_l/len(recalls_of_word)))
+    print('Fully Correct Wordwise: {}'.format(fully_Correct_w/len(recalls_of_word)))
+    print('[{:0.2f}, {:0.2f}, {:0.2f}, {:0.2f}, {:0.2f}, {:0.2f}, {:0.2f}]'.format(100*np.mean(np.array(recalls)), 100*np.mean(np.array(recalls_of_word)), 100*np.mean(np.array(precisions)), \
+           100*np.mean(np.array(precisions_of_words)), 100*(2*pl*rl)/(pl+rl), 100*fully_Correct_l/len(recalls_of_word),\
+           100*fully_Correct_w/len(recalls_of_word)))
+    sys.stdout.flush()
+    
+tag = None
+proc_count = 4
+
+def main():
+    global proc_count, tag
+    ho_folders = {
+        'PR2': 'skt_dcs_DS.bz2_4K_pmi_rfe_heldout',
+        'BR2': 'skt_dcs_DS.bz2_4K_bigram_rfe_heldout',
+        'PM2': 'skt_dcs_DS.bz2_4K_pmi_mir_heldout',
+        'BM2': 'skt_dcs_DS.bz2_4K_bigram_mir_heldout',
+        'PR3': 'skt_dcs_DS.bz2_1L_pmi_rfe_heldout',
+        'BR3': 'skt_dcs_DS.bz2_1L_bigram_rfe_heldout',
+        'PM3': 'skt_dcs_DS.bz2_1L_pmi_mir_heldout_again',
+        'BM3': 'skt_dcs_DS.bz2_1L_bigram_heldout'
+    }
+    modelList = {
+        'PR2': 'outputs/train_{}/nnet_e1_i400.p'.format('t2788294192566'),
+        'BR2': 'outputs/train_{}/nnet_e1_i400.p'.format('t2789415023871'),
+        'PM2': 'outputs/train_{}/nnet_e1_i400.p'.format('t2753954441900'),
+        'BM2': 'outputs/train_{}/nnet_e1_i400.p'.format('t3401216067518'),
+        'PR3': 'outputs/train_{}/nnet_e1_i400.p'.format('t2761370242287'),
+        'BR3': 'outputs/train_{}/nnet_e1_i400.p'.format('t2779114903467'),
+        'PM3': 'outputs/train_{}/nnet_e1_i400.p'.format('t2756013734745'),
+        'BM3': 'outputs/train_{}/nnet_e1_i400.p'.format('t3471903174862')
+    }
+    modelFile = modelList[tag]
+    print('Tag: {}, ModelFile: {}'.format(tag, modelFile))
+    print('ProcCount: {}'.format(proc_count))
+    _dump = True
+    if _dump:
+        _outFile = 'outputs/dump_predictions/{}_NLoss'.format(tag)
+    else:
+        _outFile = None
+    print('OutFile: ', _outFile)
+
+    # Backup the model file
+    copyfile(modelFile, modelFile + '.bk')
+
+    # Create Queue, Result array
+    queue = mp.Queue()
+    result_arr = []
+
+    print('Source: ', '../NewData/{}/'.format(ho_folders[tag]))
+    # Start 6 workers - 8 slows down the pc
+    # proc_count = 4
+    procs = [None]*proc_count
+    for i in range(proc_count):
+        vpid = i
+        procs[i] = mp.Process(target = TestPool_Unit.pooled_Test, args = \
+                              (modelFile, vpid, queue, '../NewData/{}/'.format(ho_folders[tag]), int(9600/proc_count), _dump, _outFile))
+    # Start Processes
+    for i in range(proc_count):
+        procs[i].start()
+        
+    # Fetch partial results
+    stillRunning = True
+    printer_timer = 100
+    while stillRunning:
+        stillRunning = False
+        for i in range(proc_count):
+            p = procs[i]
+            # print('Process with\t vpid: {}\t ->\t pid: {}\t ->\t running status: {}'.format(i, p.pid, p.is_alive()))
+            if p.is_alive():
+                stillRunning = True
+        
+        
+        if printer_timer == 0:
+            printer_timer = 100
+            while not queue.empty():
+                result_arr.append(queue.get())
+            # Evaluate results till now
+            if len(result_arr) > 0:
+                Evaluate(result_arr)
+
+        printer_timer -= 1
+        
+        time.sleep(1)
+    while not queue.empty():
+        result_arr.append(queue.get())
+    Evaluate(result_arr)
+    for i in range(proc_count):
+        procs[i].join()
+def setArgs(_tag, _pc):
+    global proc_count, tag
+    tag = _tag
+    proc_count = _pc
+    print('Tag, ProcCount: {}, {}'.format(tag, proc_count))
+    
+if __name__ == '__main__':
+    
+
+    #print('Number of arguments:', len(sys.argv), 'arguments.')
+    #print('Argument List:', str(sys.argv))
+    parser = OptionParser()
+    parser.add_option("-t", "--tag", dest="tag",
+                      help="Tag for feature set to use", metavar="TAG")
+    parser.add_option("-p", "--procs", dest="proc_count", default = 4,
+                      help="Number of child process", metavar="PROCS")
+
+    (options, args) = parser.parse_args()
+
+    options = vars(options)
+    _tag = options['tag']
+    if _tag is None:
+        raise Exception('None is tag')
+    pc = int(options['proc_count'])
+    setArgs(_tag, pc)
+    
+    main()
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/11095.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3c2170c1ba5c7b2a4f737ad4596a85b26612f432d814803145cdd22cc3999a2
+size 2500609

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/132845.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b360d0e798ebccb039dad4231de59be4dee87bc06476aced3518b91f602545d3
+size 959708

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/13718.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46da78eff48361f7e8c6ac3f89a9df7874bf6d7a9374188cad45e5413180e8a6
+size 2041406

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/13978.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24ad9067a6e3d3616cdfaefb504c9436c48fd719eb7e356c024e350eef7fee10
+size 1570927

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/150705.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9d37c3f9c39e8077373c5c6f1bc5b5ef9f41496e305bcac955eae6d91f6e313
+size 1918875

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/155358.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a163a51491c67ebe9332fe6b57087b12fcfd64b76cc2de7a5d7939dea851cabf
+size 2405589

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/160568.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c83e837985d3f6a0c5a1f239ed859512982beda9388d0dd92f098eec4afef046
+size 976570

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/1618.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f291f2149e787e524f8dc1cc4d19e4836755ca56364c559f4943348b08689aee
+size 246995

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/161938.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08f7bb19546410d8bcef7b19fd59b92a44ca879e30c2fadef6a561f3b8879337
+size 1187507

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/164353.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c85d3719a80acf4f041875682f48a358d888319f3ad9f0e285e2c04b8ba7289f
+size 835483

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/166502.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2fc184db4031295245293381ab99a0618b6f8da3c34a983946611b5768713c7
+size 705303

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/20753.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5a5757449519ac987c87037e9977ac7c70352811faf3eb1ee20a6670e77e183
+size 2319924

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/222129.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a955abc1af4e3ea82652b3100c6ffb682e4a6918d3fc668dfbc17380d59e205c
+size 1172882

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/224349.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fecac2300d4b5b80dedf8b5cf031b8682f44496da2b3d7543452d2ad1c83599
+size 5219326

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/231785.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fecc73f0a71f378582e493c72e11a3dc586294b12b195ee7fd05fd42d8293a8
+size 2326312

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/236526.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1db3aba248d9f6c4c9e8aac4d6a604b28b914b8f8733b7ddff91e944e6982d36
+size 5333884

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/24384.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac35fcdd721ce8734b1747ceebf3fa398eea3caa0cc79cd86ced4e19887a378e
+size 21812

wordsegmentation/skt_dcs_DS.bz2_4K_bigram_mir_10K/258696.ds.bz2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e02de9b1d92bade8bc49c2b8630c01f68579503757e436f0abea892765ac64ae
+size 1046840