Initial

Scripts/UnixCoder/model_gen.py

@@ -56,9 +56,6 @@ class Seq2Seq(nn.Module):
         mask = source_ids.ne(1)[:, None, :]*source_ids.ne(1)[:, :, None]
         encoder_output = self.encoder(
             source_ids, attention_mask=mask, use_cache=True)
-        # print("source_ids:", source_ids.size()) # torch.Size([56, 510])
-        # print("exist:", exist.size()) # torch.Size([56, 1])
-        # print("target_ids:", target_ids.size()) # torch.Size([56, 240])
         ids = torch.cat((source_ids, target_ids), -1)
 
         mask = self.bias[:,
@@ -68,33 +65,15 @@ class Seq2Seq(nn.Module):
         out = self.decoder(target_ids, attention_mask=mask,
                            past_key_values=encoder_output.past_key_values).last_hidden_state
 
-        # 先concat 再池化
-        # print("out:", out.size()) # torch.Size([56, 240, 768])
-
         lm_logits = self.lm_head(out[..., 1:, :])
-        # print("lm_logits:", lm_logits.size()) # torch.Size([56, 239, 51416])
-
         # Shift so that tokens < n predict n
         active_loss = target_ids[..., 2:].ne(1).view(-1)
-        # print("active_loss:", active_loss.size()) # torch.Size([13328])
         shift_logits = lm_logits[..., :-1, :].contiguous()
-        # print("shift_logits:", shift_logits.size()) # torch.Size([56, 238, 51416])
-
         shift_labels = target_ids[..., 2:].contiguous()
-        # print("shift_labels:", shift_labels.size()) # torch.Size([56, 238])
 
         exist_labels = exist.contiguous()
-        # print("exist_labels:", exist_labels.size()) # torch.Size([56, 1])
-
-        # print("shift_logits.size:", shift_logits.size(-1)) # 51416
-        # print("shift_logits.view(-1, shift_logits.size(-1)):", shift_logits.view(-1, shift_logits.size(-1))[active_loss].size()) # torch.Size([614, 51416])
-        # print("shift_labels.view(-1):", shift_labels.view(-1)[active_loss].size()) # torch.Size([614])
-
         pred_out = out[..., 0, :]
-        # print("pred_out:", pred_out.size()) # torch.Size([56, 768])
         pred_sigmoid = self.sigmoid(self.pred_dense(pred_out))
-        # print("pred_sigmoid:", pred_sigmoid.size()) # torch.Size([56, 1])
-
         # Flatten the tokens
         loss_fct_code = nn.CrossEntropyLoss(ignore_index=-1)
         loss_fct_pred = nn.MSELoss(reduction="mean")
@@ -103,8 +82,6 @@ class Seq2Seq(nn.Module):
 
         loss_pred = loss_fct_pred(pred_sigmoid, exist_labels)
         loss = loss_pred * self.mse_loss_weight + loss_code * self.ce_loss_weight
-        # loss = loss.to(torch.float32)
-        # loss = loss_pred
 
         outputs = loss, loss*active_loss.sum(), active_loss.sum(), loss_pred, loss_code
         return outputs
@@ -135,10 +112,7 @@ class Seq2Seq(nn.Module):
                 mask = mask & ids[:, None, :].ne(1)
                 out = self.decoder(input_ids, attention_mask=mask,
                                    past_key_values=context).last_hidden_state
-                # print("out:", out.size())
-                # concat 池化 out
                 hidden_states = out[:, -1, :]
-                # print("hidden_states:", hidden_states.size())
                 if out.size(1) == 1:
                     pred_sigmoid = self.sigmoid(self.pred_dense(
                         hidden_states.view(-1, 1, hidden_states.size(-1))))
@@ -155,14 +129,9 @@ class Seq2Seq(nn.Module):
             pred = [torch.cat([x.view(-1) for x in p] + [zero] *
                               (self.max_length-len(p))).view(1, -1) for p in pred]
             predicates.append(predicate[0][0])# ZM modified
-            #print("ZM-Model_Debug_P_Each_Itr: %d,  %d,  %d" % (len(predicate), len(predicate[0]), len(predicate[0][0])))
             preds.append(torch.cat(pred, 0).unsqueeze(0))
-        #print("ZM-Model_Debug_Predicate_Shape: %d" % (len(predicates)))
-        #print("ZM-Model_Debug_Codes_BeforeCat: %d,  %d,  %d,  %d" % (len(preds), len(preds[0]), len(preds[0][0]), len(preds[0][0][0])))
         preds = torch.cat(preds, 0)
         predicates = torch.tensor(predicates, device="cuda")# ZM modified
-        # predicates = torch.cat(predicates, 0).unsqueeze(0)
-        #print("ZM-Model_Debug_Codes_AfterCat: %d,  %d,  %d" % (len(preds), len(preds[0]), len(preds[0][0])))
         return preds, predicates
 
 

Scripts/UnixCoder/run_one_model.py

@@ -53,7 +53,6 @@ class Example(object):
                  vec,
                  exist,
                  module
-                 # propertyposition,
                  ):
         self.idx = idx
         self.source = source
@@ -77,8 +76,6 @@ def read_examples_no_bracket(filename, is_function_test):
                     break
             line = line.strip()
             js = json.loads(line)
-            if idx > 1000:
-                break
             if js["Stmt"].strip()[0] == "}":
                 continue
             if js["Value"].strip().lower() == "nothing" and '#' in js['FIR']:
@@ -119,11 +116,6 @@ def read_examples_no_bracket(filename, is_function_test):
             mod = ""
             if "Module" in js.keys():
                 mod = js["Module"]
-         #   propos = ' '.join(js['pp'])
-          #  propos = ' '.join(propos.strip().split(','))
-            # print(code)
-            # print(nl)
-            # print(pro)
             examples.append(
                 Example(
                     idx=idx,
@@ -152,8 +144,6 @@ def read_examples(filename, is_function_test):
                     break
             line = line.strip()
             js = json.loads(line)
-            if idx > 3000:
-                break
             if 'idx' not in js:
                 js['idx'] = idx
             code = ' '.join(js['FIR_token']).replace('\n', ' ')
@@ -188,11 +178,6 @@ def read_examples(filename, is_function_test):
             mod = ""
             if "Module" in js.keys():
                 mod = js["Module"]
-         #   propos = ' '.join(js['pp'])
-          #  propos = ' '.join(propos.strip().split(','))
-            # print(code)
-            # print(nl)
-            # print(pro)
             examples.append(
                 Example(
                     idx=idx,
@@ -233,7 +218,7 @@ def convert_examples_to_features(examples, tokenizer, args, stage=None):
         # source
         func_tokens = tokenizer.tokenize(example.funcname)
         source_tokens = tokenizer.tokenize(
-            example.source)  # [:args.max_source_length-5]
+            example.source)
         pro_tokens = tokenizer.tokenize(example.property)
         vec_tokens = example.vec
         source_tokens = [tokenizer.cls_token, "<encoder-decoder>", tokenizer.sep_token, "<mask0>"] + func_tokens + [tokenizer.cls_token] + \
@@ -243,8 +228,6 @@ def convert_examples_to_features(examples, tokenizer, args, stage=None):
         padding_length = args.max_source_length - len(source_ids)
         source_ids += [tokenizer.pad_token_id] * padding_length
 
-        # target
-        # if stage=="test":
         target_tokens = tokenizer.tokenize(example.target)
         exist = [example.exist]
         target_tokens = [tokenizer.cls_token, "<mask0>"] + \
@@ -252,13 +235,6 @@ def convert_examples_to_features(examples, tokenizer, args, stage=None):
         target_ids = tokenizer.convert_tokens_to_ids(target_tokens)
         padding_length = args.max_target_length - len(target_ids)
         target_ids += [tokenizer.pad_token_id] * padding_length
-        # else:
-        #     target_tokens = tokenizer.tokenize(example.target)
-        #     exist_tokens = tokenizer.tokenize(example.exist)
-        #     target_tokens = ["<mask0>"] + exist_tokens + [tokenizer.cls_token] + target_tokens + [tokenizer.sep_token]
-        #     target_ids = tokenizer.convert_tokens_to_ids(target_tokens)
-        #     padding_length = args.max_target_length - len(target_ids)
-        #     target_ids += [tokenizer.pad_token_id] * padding_length
 
         features.append(
             InputFeatures(
@@ -470,14 +446,7 @@ def vega_train_main():
         total_eval_all = len(eval_examples_all)
         patience, best_acc, losses, dev_dataset = 0, 0, [], {}
         for epoch in tqdm(range(args.num_train_epochs)):
-            # print(args.num_train_epochs)
-
             for idx, batch in enumerate(train_dataloader):
-                # print("##########Debug################")
-                # print(idx)
-                # print("###############Debug###########")
-                # if idx > 100:
-                #     break
                 batch = tuple(t.to(device) for t in batch)
                 source_ids, exist, target_ids = batch
                 loss, _, _, mse_loss, ce_loss = model(
@@ -572,9 +541,7 @@ def vega_train_main():
                         # convert ids to text
                         for pred, predicate in zip(preds, predicates):
                             t = pred[0].cpu().numpy()
-                            #p = predicate[0].cpu().numpy()
                             p = predicate.float().item()
-                            #print("ZM_Debug -- ppp:  " + str(p))
                             t = list(t)
                             #p = list(p)
                             tem_i = 0
@@ -608,7 +575,6 @@ def vega_train_main():
                     cnt_iteration += 1
                     pred = ref[0].strip()
                     predicate = ref[1]
-                    #print("ZM_Debug -- predicate:  " + str(predicate))
                     if gold.property.strip().lower() != "nothing":
                         predicate = 1.0
                     else:
@@ -626,7 +592,6 @@ def vega_train_main():
 
 
                     if pred == gt_pred and int(round(predicate)) == int(round(gt_predicate)):
-                        #print("Total correct, Inside this place")
                         EM = EM + 1.0
                         EM_V = EM_V + 1.0
                         EM_P = EM_P + 1.0
@@ -646,43 +611,16 @@ def vega_train_main():
 
                     model_predicate.append(predicate)
                     groundtruth_predicate.append(gt_predicate)
-                    # if len(pred.split(tokenizer.cls_token)) >= 2:
-                    #     if pred.split(tokenizer.cls_token)[0].strip() == gt_pred.split(tokenizer.cls_token)[0].strip():
-                    #         EM_P += 1
-                    #     if pred.split(tokenizer.cls_token)[1].strip() == gt_pred.split(tokenizer.cls_token)[1].strip():
-                    #         EM_V += 1
-                # MAE_P = mean_absolute_error(
-                #     np.array(model_predicate), np.array(groundtruth_predicate))
-                # MSE_P = mean_squared_error(
-                #     np.array(model_predicate), np.array(groundtruth_predicate))
-                # RMSE_P = np.sqrt(MSE_P)
                 dev_acc = round((100*EM/total), 2)
                 dev_acc_v = round((100*EM_V/total), 2)
                 dev_acc_p = round((100*EM_P/total), 2)
                 logger.info("  %s = %s " % ("Current Acc", str(dev_acc)))
-                #logger.info("  %s = %s "%("Current Edit sim",str(round(edit_sim/total, 2))))
                 logger.info("  "+"*"*20)
                 logger.info("  %s = %s " % ("Current Acc V", str(dev_acc_v)))
-                #logger.info("  %s = %s "%("Current Edit sim",str(round(edit_sim/total, 2))))
                 logger.info("  "+"*"*20)
                 logger.info("  %s = %s " % ("Current Acc P", str(dev_acc_p)))
-                #logger.info("  %s = %s "%("Current Edit sim",str(round(edit_sim/total, 2))))
                 logger.info("  "+"*"*20)
-                # logger.info("  %s = %s " %
-                #             ("Current MAE P", str(round(MAE_P, 2))))
-                # #logger.info("  %s = %s "%("Current Edit sim",str(round(edit_sim/total, 2))))
-                # logger.info("  "+"*"*20)
-                # logger.info("  %s = %s " %
-                #             ("Current MSE P", str(round(MSE_P, 2))))
-                # #logger.info("  %s = %s "%("Current Edit sim",str(round(edit_sim/total, 2))))
-                # logger.info("  "+"*"*20)
-                # logger.info("  %s = %s " %
-                #             ("Current RMSE P", str(round(RMSE_P, 2))))
-                # #logger.info("  %s = %s "%("Current Edit sim",str(round(edit_sim/total, 2))))
-                # logger.info("  "+"*"*20)
                 if dev_acc > best_acc:
-                    #logger.info("  Best acc:%s",dev_acc)
-                    #logger.info("  "+"*"*20)
                     best_acc = dev_acc
                     # Save best checkpoint for best bleu
                     output_dir = os.path.join(
@@ -694,15 +632,6 @@ def vega_train_main():
                     output_model_file = os.path.join(
                         output_dir, "pytorch_model.bin")
                     torch.save(model_to_save.state_dict(), output_model_file)
-                    # with open(args.output_dir+"/p_valid_wrong.csv", 'w', encoding='utf-8', newline="") as fcsv2:
-                    #     writer = csv.writer(fcsv2)
-                    #     for wl in p_wrong_list:
-                    #         writer.writerow(wl)
-                    # with open(args.output_dir+"/v_valid_wrong.csv", 'w', encoding='utf-8', newline="") as fcsv2:
-                    #     writer = csv.writer(fcsv2)
-                    #     for wl in v_wrong_list:
-                    #         writer.writerow(wl)
-                #print("ZM Debug--cnt_err_v:  " + str(cnt_v))
                 logger.info("  Best acc:%s", best_acc)
                 logger.info("  " + "*" * 20)
                     
@@ -753,9 +682,7 @@ def vega_train_main():
                 # convert ids to text
                 for pred, predicate in zip(preds, predicates):
                     t = pred[0].cpu().numpy()
-                    #p = predicate[0].cpu().numpy()
                     p = predicate.float().item()
-                    #print("ZM_Debug -- ppp:  " + str(p))
                     t = list(t)
                     tem_i = 0
                     if 0 in t:
@@ -802,7 +729,6 @@ def vega_train_main():
                     predicate = 0.0
                 if 1 in gold.vec[-97:]:
                     predicate = 1.0
-            #my_cls = tokenizer.decode([tokenizer.cls_token_id],clean_up_tokenization_spaces=False)
             gt_pred = gold.target.strip()
             gt_predicate = gold.exist
             is_re = False
@@ -840,30 +766,13 @@ def vega_train_main():
 
             if pred == gt_pred:
                 EM_V += 1
-            # else:
-            #     print("TEST Wrong    pred:", pred, "    gt_pred:", gt_pred)
             if round(predicate) == gt_predicate:
                 EM_P += 1
             model_predicate.append(predicate)
             groundtruth_predicate.append(gt_predicate)
-
-        # MAE_P = mean_absolute_error(
-        #     np.array(model_predicate), np.array(groundtruth_predicate))
-        # MSE_P = mean_squared_error(
-        #     np.array(model_predicate), np.array(groundtruth_predicate))
-        # RMSE_P = np.sqrt(MSE_P)
-
         dev_acc = round((100 * EM / total), 2)
         dev_acc_v = round((100 * EM_V / total), 2)
         dev_acc_p = round((100 * EM_P / total), 2)
-        # logger.info("  %s = %s " % ("Test Acc", str(dev_acc)))
-        # logger.info("  %s = %s " % ("Test Acc V", str(dev_acc_v)))
-        # logger.info("  %s = %s " % ("Test Acc P", str(dev_acc_p)))
-        # logger.info("  %s = %s "%("Test Edit sim",str(round(edit_sim/total, 2))))
-        # logger.info("  %s = %s " % ("Test MAE P", str(round(MAE_P, 2))))
-        # logger.info("  %s = %s " % ("Test MSE P", str(round(MSE_P, 2))))
-        # logger.info("  %s = %s " % ("Test RMSE P", str(round(RMSE_P, 2))))
-        # logger.info("  " + "*" * 20)
         predictions = []
         
 
@@ -897,15 +806,6 @@ def vega_train_main():
                 json.dump(dic, f2)
                 f2.write('\n')
 
-        
-        # with open(args.output_dir+"/p_wrong.csv", 'w', encoding='utf-8', newline="") as fcsv2:
-        #     writer = csv.writer(fcsv2)
-        #     for wl in p_wrong_list:
-        #         writer.writerow(wl)
-        # with open(args.output_dir+"/v_wrong.csv", 'w', encoding='utf-8', newline="") as fcsv2:
-        #     writer = csv.writer(fcsv2)
-        #     for wl in v_wrong_list:
-        #         writer.writerow(wl)
 
 
 if __name__ == "__main__":

run_fine_tuning.sh

@@ -10,6 +10,6 @@ python ./Scripts/UnixCoder/run_one_model.py \
   --train_batch_size 64 \
   --eval_batch_size 48 \
   --learning_rate 6e-5 \
-  --num_train_epochs 3 \
+  --num_train_epochs 50 \
   --mse_loss_weight 0.9 \
   --ce_loss_weight 0.1