Delete legacy

legacy/talkNet_multi_multicard.py

@@ -1,124 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-import sys, time, numpy, os, subprocess, pandas, tqdm
-
-from loss_multi import lossAV, lossA, lossV
-from model.talkNetModel import talkNetModel
-
-import pytorch_lightning as pl
-from torch import distributed as dist
-
-
-class talkNet(pl.LightningModule):
-
-    def __init__(self, cfg):
-        super(talkNet, self).__init__()
-        self.model = talkNetModel().cuda()
-        self.cfg = cfg
-        self.lossAV = lossAV().cuda()
-        self.lossA = lossA().cuda()
-        self.lossV = lossV().cuda()
-        print(
-            time.strftime("%m-%d %H:%M:%S") + " Model para number = %.2f" %
-            (sum(param.numel() for param in self.model.parameters()) / 1024 / 1024))
-
-    def configure_optimizers(self):
-        optimizer = torch.optim.Adam(self.parameters(), lr=self.cfg.SOLVER.BASE_LR)
-        scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
-                                                    step_size=1,
-                                                    gamma=self.cfg.SOLVER.SCHEDULER.GAMMA)
-        return {"optimizer": optimizer, "lr_scheduler": scheduler}
-
-    def training_step(self, batch, batch_idx):
-        audioFeature, visualFeature, labels, masks = batch
-        b, s, t = visualFeature.shape[0], visualFeature.shape[1], visualFeature.shape[2]
-        audioFeature = audioFeature.repeat(1, s, 1, 1)
-        audioFeature = audioFeature.view(b * s, *audioFeature.shape[2:])
-        visualFeature = visualFeature.view(b * s, *visualFeature.shape[2:])
-        labels = labels.view(b * s, *labels.shape[2:])
-        masks = masks.view(b * s, *masks.shape[2:])
-
-        audioEmbed = self.model.forward_audio_frontend(audioFeature)    # feedForward
-        visualEmbed = self.model.forward_visual_frontend(visualFeature)
-        audioEmbed, visualEmbed = self.model.forward_cross_attention(audioEmbed, visualEmbed)
-        outsAV = self.model.forward_audio_visual_backend(audioEmbed, visualEmbed)
-        outsA = self.model.forward_audio_backend(audioEmbed)
-        outsV = self.model.forward_visual_backend(visualEmbed)
-        labels = labels.reshape((-1))
-        nlossAV, _, _, prec = self.lossAV.forward(outsAV, labels, masks)
-        nlossA = self.lossA.forward(outsA, labels, masks)
-        nlossV = self.lossV.forward(outsV, labels, masks)
-        loss = nlossAV + 0.4 * nlossA + 0.4 * nlossV
-        self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)
-        return loss
-
-    def training_epoch_end(self, training_step_outputs):
-        self.saveParameters(
-            os.path.join(self.cfg.WORKSPACE, "model", "{}.pth".format(self.current_epoch)))
-
-    def evaluate_network(self, loader):
-        self.eval()
-        predScores = []
-        self.model = self.model.cuda()
-        self.lossAV = self.lossAV.cuda()
-        self.lossA = self.lossA.cuda()
-        self.lossV = self.lossV.cuda()
-        evalCsvSave = self.cfg.evalCsvSave
-        evalOrig = self.cfg.evalOrig
-        for audioFeature, visualFeature, labels, masks in tqdm.tqdm(loader):
-            with torch.no_grad():
-                b, s = visualFeature.shape[0], visualFeature.shape[1]
-                t = visualFeature.shape[2]
-                audioFeature = audioFeature.repeat(1, s, 1, 1)
-                audioFeature = audioFeature.view(b * s, *audioFeature.shape[2:])
-                visualFeature = visualFeature.view(b * s, *visualFeature.shape[2:])
-                labels = labels.view(b * s, *labels.shape[2:])
-                masks = masks.view(b * s, *masks.shape[2:])
-                audioEmbed = self.model.forward_audio_frontend(audioFeature.cuda())
-                visualEmbed = self.model.forward_visual_frontend(visualFeature.cuda())
-                audioEmbed, visualEmbed = self.model.forward_cross_attention(
-                    audioEmbed, visualEmbed)
-                outsAV = self.model.forward_audio_visual_backend(audioEmbed, visualEmbed)
-                labels = labels.reshape((-1)).cuda()
-                outsAV = outsAV.view(b, s, t, -1)[:, 0, :, :].view(b * t, -1)
-                labels = labels.view(b, s, t)[:, 0, :].view(b * t)
-                masks = masks.view(b, s, t)[:, 0, :].view(b * t)
-                _, predScore, _, _ = self.lossAV.forward(outsAV, labels, masks)
-                predScore = predScore.detach().cpu().numpy()
-                predScores.extend(predScore)
-        evalLines = open(evalOrig).read().splitlines()[1:]
-        labels = []
-        labels = pandas.Series(['SPEAKING_AUDIBLE' for line in evalLines])
-        scores = pandas.Series(predScores)
-        evalRes = pandas.read_csv(evalOrig)
-        evalRes['score'] = scores
-        evalRes['label'] = labels
-        evalRes.drop(['label_id'], axis=1, inplace=True)
-        evalRes.drop(['instance_id'], axis=1, inplace=True)
-        evalRes.to_csv(evalCsvSave, index=False)
-        cmd = "python -O utils/get_ava_active_speaker_performance.py -g %s -p %s " % (evalOrig,
-                                                                                      evalCsvSave)
-        mAP = float(
-            str(subprocess.run(cmd, shell=True, capture_output=True).stdout).split(' ')[2][:5])
-        return mAP
-
-    def saveParameters(self, path):
-        torch.save(self.state_dict(), path)
-
-    def loadParameters(self, path):
-        selfState = self.state_dict()
-        loadedState = torch.load(path)
-        for name, param in loadedState.items():
-            origName = name
-            if name not in selfState:
-                name = name.replace("module.", "")
-                if name not in selfState:
-                    print("%s is not in the model." % origName)
-                    continue
-            if selfState[name].size() != loadedState[origName].size():
-                sys.stderr.write("Wrong parameter length: %s, model: %s, loaded: %s" %
-                                 (origName, selfState[name].size(), loadedState[origName].size()))
-                continue
-            selfState[name].copy_(param)

legacy/talkNet_multicard.py

@@ -1,146 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-import sys, time, numpy, os, subprocess, pandas, tqdm
-
-from loss import lossAV, lossA, lossV
-from model.talkNetModel import talkNetModel
-
-import pytorch_lightning as pl
-from torch import distributed as dist
-
-
-class talkNet(pl.LightningModule):
-
-    def __init__(self, cfg):
-        super(talkNet, self).__init__()
-        self.cfg = cfg
-        self.model = talkNetModel()
-        self.lossAV = lossAV()
-        self.lossA = lossA()
-        self.lossV = lossV()
-        print(
-            time.strftime("%m-%d %H:%M:%S") + " Model para number = %.2f" %
-            (sum(param.numel() for param in self.model.parameters()) / 1024 / 1024))
-
-    def configure_optimizers(self):
-        optimizer = torch.optim.Adam(self.parameters(), lr=self.cfg.SOLVER.BASE_LR)
-        scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
-                                                    step_size=1,
-                                                    gamma=self.cfg.SOLVER.SCHEDULER.GAMMA)
-        return {"optimizer": optimizer, "lr_scheduler": scheduler}
-
-    def training_step(self, batch, batch_idx):
-        audioFeature, visualFeature, labels = batch
-        audioEmbed = self.model.forward_audio_frontend(audioFeature[0])    # feedForward
-        visualEmbed = self.model.forward_visual_frontend(visualFeature[0])
-        audioEmbed, visualEmbed = self.model.forward_cross_attention(audioEmbed, visualEmbed)
-        outsAV = self.model.forward_audio_visual_backend(audioEmbed, visualEmbed)
-        outsA = self.model.forward_audio_backend(audioEmbed)
-        outsV = self.model.forward_visual_backend(visualEmbed)
-        labels = labels[0].reshape((-1))
-        nlossAV, _, _, prec = self.lossAV.forward(outsAV, labels)
-        nlossA = self.lossA.forward(outsA, labels)
-        nlossV = self.lossV.forward(outsV, labels)
-        loss = nlossAV + 0.4 * nlossA + 0.4 * nlossV
-        self.log("train_loss", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)
-
-        return loss
-
-    def training_epoch_end(self, training_step_outputs):
-        self.saveParameters(
-            os.path.join(self.cfg.WORKSPACE, "model", "{}.pth".format(self.current_epoch)))
-
-    def validation_step(self, batch, batch_idx):
-        audioFeature, visualFeature, labels, indices = batch
-        audioEmbed = self.model.forward_audio_frontend(audioFeature[0])
-        visualEmbed = self.model.forward_visual_frontend(visualFeature[0])
-        audioEmbed, visualEmbed = self.model.forward_cross_attention(audioEmbed, visualEmbed)
-        outsAV = self.model.forward_audio_visual_backend(audioEmbed, visualEmbed)
-        labels = labels[0].reshape((-1))
-        loss, predScore, _, _ = self.lossAV.forward(outsAV, labels)
-        predScore = predScore[:, -1:].detach().cpu().numpy()
-        # self.log("val_loss", loss)
-
-        return predScore
-
-    def validation_epoch_end(self, validation_step_outputs):
-        evalCsvSave = self.cfg.evalCsvSave
-        evalOrig = self.cfg.evalOrig
-        predScores = []
-
-        for out in validation_step_outputs:    # batch size =1
-            predScores.extend(out)
-
-        evalLines = open(evalOrig).read().splitlines()[1:]
-        labels = []
-        labels = pandas.Series(['SPEAKING_AUDIBLE' for line in evalLines])
-        scores = pandas.Series(predScores)
-        evalRes = pandas.read_csv(evalOrig)
-        print(len(evalRes), len(predScores), len(evalLines))
-        evalRes['score'] = scores
-        evalRes['label'] = labels
-        evalRes.drop(['label_id'], axis=1, inplace=True)
-        evalRes.drop(['instance_id'], axis=1, inplace=True)
-        evalRes.to_csv(evalCsvSave, index=False)
-        cmd = "python -O utils/get_ava_active_speaker_performance.py -g %s -p %s " % (evalOrig,
-                                                                                      evalCsvSave)
-        mAP = float(
-            str(subprocess.run(cmd, shell=True, capture_output=True).stdout).split(' ')[2][:5])
-        print("validation mAP: {}".format(mAP))
-
-    def saveParameters(self, path):
-        torch.save(self.state_dict(), path)
-
-    def loadParameters(self, path):
-        selfState = self.state_dict()
-        loadedState = torch.load(path, map_location='cpu')
-        for name, param in loadedState.items():
-            origName = name
-            if name not in selfState:
-                name = name.replace("module.", "")
-                if name not in selfState:
-                    print("%s is not in the model." % origName)
-                    continue
-            if selfState[name].size() != loadedState[origName].size():
-                sys.stderr.write("Wrong parameter length: %s, model: %s, loaded: %s" %
-                                 (origName, selfState[name].size(), loadedState[origName].size()))
-                continue
-            selfState[name].copy_(param)
-
-    def evaluate_network(self, loader):
-        self.eval()
-        self.model = self.model.cuda()
-        self.lossAV = self.lossAV.cuda()
-        self.lossA = self.lossA.cuda()
-        self.lossV = self.lossV.cuda()
-        predScores = []
-        evalCsvSave = self.cfg.evalCsvSave
-        evalOrig = self.cfg.evalOrig
-        for audioFeature, visualFeature, labels in tqdm.tqdm(loader):
-            with torch.no_grad():
-                audioEmbed = self.model.forward_audio_frontend(audioFeature[0].cuda())
-                visualEmbed = self.model.forward_visual_frontend(visualFeature[0].cuda())
-                audioEmbed, visualEmbed = self.model.forward_cross_attention(
-                    audioEmbed, visualEmbed)
-                outsAV = self.model.forward_audio_visual_backend(audioEmbed, visualEmbed)
-                labels = labels[0].reshape((-1)).cuda()
-                _, predScore, _, _ = self.lossAV.forward(outsAV, labels)
-                predScore = predScore[:, 1].detach().cpu().numpy()
-                predScores.extend(predScore)
-        evalLines = open(evalOrig).read().splitlines()[1:]
-        labels = []
-        labels = pandas.Series(['SPEAKING_AUDIBLE' for line in evalLines])
-        scores = pandas.Series(predScores)
-        evalRes = pandas.read_csv(evalOrig)
-        evalRes['score'] = scores
-        evalRes['label'] = labels
-        evalRes.drop(['label_id'], axis=1, inplace=True)
-        evalRes.drop(['instance_id'], axis=1, inplace=True)
-        evalRes.to_csv(evalCsvSave, index=False)
-        cmd = "python -O utils/get_ava_active_speaker_performance.py -g %s -p %s " % (evalOrig,
-                                                                                      evalCsvSave)
-        mAP = float(
-            str(subprocess.run(cmd, shell=True, capture_output=True).stdout).split(' ')[2][:5])
-        return mAP

legacy/talkNet_orig.py

@@ -1,102 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-import sys, time, numpy, os, subprocess, pandas, tqdm
-
-from loss import lossAV, lossA, lossV
-from model.talkNetModel import talkNetModel
-
-
-class talkNet(nn.Module):
-
-    def __init__(self, lr=0.0001, lrDecay=0.95, **kwargs):
-        super(talkNet, self).__init__()
-        self.model = talkNetModel().cuda()
-        self.lossAV = lossAV().cuda()
-        self.lossA = lossA().cuda()
-        self.lossV = lossV().cuda()
-        self.optim = torch.optim.Adam(self.parameters(), lr=lr)
-        self.scheduler = torch.optim.lr_scheduler.StepLR(self.optim, step_size=1, gamma=lrDecay)
-        print(
-            time.strftime("%m-%d %H:%M:%S") + " Model para number = %.2f" %
-            (sum(param.numel() for param in self.model.parameters()) / 1024 / 1024))
-
-    def train_network(self, loader, epoch, **kwargs):
-        self.train()
-        self.scheduler.step(epoch - 1)
-        index, top1, loss = 0, 0, 0
-        lr = self.optim.param_groups[0]['lr']
-        for num, (audioFeature, visualFeature, labels) in enumerate(loader, start=1):
-            self.zero_grad()
-            audioEmbed = self.model.forward_audio_frontend(audioFeature[0].cuda())    # feedForward
-            visualEmbed = self.model.forward_visual_frontend(visualFeature[0].cuda())
-            audioEmbed, visualEmbed = self.model.forward_cross_attention(audioEmbed, visualEmbed)
-            outsAV = self.model.forward_audio_visual_backend(audioEmbed, visualEmbed)
-            outsA = self.model.forward_audio_backend(audioEmbed)
-            outsV = self.model.forward_visual_backend(visualEmbed)
-            labels = labels[0].reshape((-1)).cuda()    # Loss
-            nlossAV, _, _, prec = self.lossAV.forward(outsAV, labels)
-            nlossA = self.lossA.forward(outsA, labels)
-            nlossV = self.lossV.forward(outsV, labels)
-            nloss = nlossAV + 0.4 * nlossA + 0.4 * nlossV
-            loss += nloss.detach().cpu().numpy()
-            top1 += prec
-            nloss.backward()
-            self.optim.step()
-            index += len(labels)
-            sys.stderr.write(time.strftime("%m-%d %H:%M:%S") + \
-            " [%2d] Lr: %5f, Training: %.2f%%, "    %(epoch, lr, 100 * (num / loader.__len__())) + \
-            " Loss: %.5f, ACC: %2.2f%% \r"        %(loss/(num), 100 * (top1/index)))
-            sys.stderr.flush()
-        sys.stdout.write("\n")
-        return loss / num, lr
-
-    def evaluate_network(self, loader, evalCsvSave, evalOrig, **kwargs):
-        self.eval()
-        predScores = []
-        for audioFeature, visualFeature, labels in tqdm.tqdm(loader):
-            with torch.no_grad():
-                audioEmbed = self.model.forward_audio_frontend(audioFeature[0].cuda())
-                visualEmbed = self.model.forward_visual_frontend(visualFeature[0].cuda())
-                audioEmbed, visualEmbed = self.model.forward_cross_attention(
-                    audioEmbed, visualEmbed)
-                outsAV = self.model.forward_audio_visual_backend(audioEmbed, visualEmbed)
-                labels = labels[0].reshape((-1)).cuda()
-                _, predScore, _, _ = self.lossAV.forward(outsAV, labels)
-                predScore = predScore[:, 1].detach().cpu().numpy()
-                predScores.extend(predScore)
-        evalLines = open(evalOrig).read().splitlines()[1:]
-        labels = []
-        labels = pandas.Series(['SPEAKING_AUDIBLE' for line in evalLines])
-        scores = pandas.Series(predScores)
-        evalRes = pandas.read_csv(evalOrig)
-        evalRes['score'] = scores
-        evalRes['label'] = labels
-        evalRes.drop(['label_id'], axis=1, inplace=True)
-        evalRes.drop(['instance_id'], axis=1, inplace=True)
-        evalRes.to_csv(evalCsvSave, index=False)
-        cmd = "python -O utils/get_ava_active_speaker_performance.py -g %s -p %s " % (evalOrig,
-                                                                                      evalCsvSave)
-        mAP = float(
-            str(subprocess.run(cmd, shell=True, capture_output=True).stdout).split(' ')[2][:5])
-        return mAP
-
-    def saveParameters(self, path):
-        torch.save(self.state_dict(), path)
-
-    def loadParameters(self, path):
-        selfState = self.state_dict()
-        loadedState = torch.load(path)
-        for name, param in loadedState.items():
-            origName = name
-            if name not in selfState:
-                name = name.replace("module.", "")
-                if name not in selfState:
-                    print("%s is not in the model." % origName)
-                    continue
-            if selfState[name].size() != loadedState[origName].size():
-                sys.stderr.write("Wrong parameter length: %s, model: %s, loaded: %s" %
-                                 (origName, selfState[name].size(), loadedState[origName].size()))
-                continue
-            selfState[name].copy_(param)

legacy/trainTalkNet_multicard.py

@@ -1,171 +0,0 @@
-import time, os, torch, argparse, warnings, glob
-
-from utils.tools import *
-from dlhammer import bootstrap
-import pytorch_lightning as pl
-from pytorch_lightning import Trainer, seed_everything
-from pytorch_lightning.callbacks import ModelCheckpoint
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-
-
-class MyCollator(object):
-
-    def __init__(self, cfg):
-        self.cfg = cfg
-
-    def __call__(self, data):
-        audiofeatures = [item[0] for item in data]
-        visualfeatures = [item[1] for item in data]
-        labels = [item[2] for item in data]
-        masks = [item[3] for item in data]
-        cut_limit = self.cfg.MODEL.CLIP_LENGTH
-        # pad audio
-        lengths = torch.tensor([t.shape[1] for t in audiofeatures])
-        max_len = max(lengths)
-        padded_audio = torch.stack([
-            torch.cat([i, i.new_zeros((i.shape[0], max_len - i.shape[1], i.shape[2]))], 1)
-            for i in audiofeatures
-        ], 0)
-
-        if max_len > cut_limit * 4:
-            padded_audio = padded_audio[:, :, :cut_limit * 4, ...]
-
-        # pad video
-        lengths = torch.tensor([t.shape[1] for t in visualfeatures])
-        max_len = max(lengths)
-        padded_video = torch.stack([
-            torch.cat(
-                [i, i.new_zeros((i.shape[0], max_len - i.shape[1], i.shape[2], i.shape[3]))], 1)
-            for i in visualfeatures
-        ], 0)
-        padded_labels = torch.stack(
-            [torch.cat([i, i.new_zeros((i.shape[0], max_len - i.shape[1]))], 1) for i in labels], 0)
-        padded_masks = torch.stack(
-            [torch.cat([i, i.new_zeros((i.shape[0], max_len - i.shape[1]))], 1) for i in masks], 0)
-
-        if max_len > cut_limit:
-            padded_video = padded_video[:, :, :cut_limit, ...]
-            padded_labels = padded_labels[:, :, :cut_limit, ...]
-            padded_masks = padded_masks[:, :, :cut_limit, ...]
-        return padded_audio, padded_video, padded_labels, padded_masks
-
-
-class DataPrep(pl.LightningDataModule):
-
-    def __init__(self, cfg):
-        self.cfg = cfg
-
-    def train_dataloader(self):
-        cfg = self.cfg
-
-        if self.cfg.MODEL.NAME == "baseline":
-            from dataLoader import train_loader, val_loader
-            loader = train_loader(trialFileName = cfg.trainTrialAVA, \
-                              audioPath      = os.path.join(cfg.audioPathAVA , 'train'), \
-                              visualPath     = os.path.join(cfg.visualPathAVA, 'train'), \
-                              batchSize=2500
-                              )
-        elif self.cfg.MODEL.NAME == "multi":
-            from dataLoader_multiperson import train_loader, val_loader
-            loader = train_loader(trialFileName = cfg.trainTrialAVA, \
-                              audioPath      = os.path.join(cfg.audioPathAVA , 'train'), \
-                              visualPath     = os.path.join(cfg.visualPathAVA, 'train'), \
-                              num_speakers=cfg.MODEL.NUM_SPEAKERS,
-                              )
-        if cfg.MODEL.NAME == "baseline":
-            trainLoader = torch.utils.data.DataLoader(
-                loader,
-                batch_size=1,
-                shuffle=True,
-                num_workers=4,
-            )
-        elif cfg.MODEL.NAME == "multi":
-            collator = MyCollator(cfg)
-            trainLoader = torch.utils.data.DataLoader(loader,
-                                                      batch_size=1,
-                                                      shuffle=True,
-                                                      num_workers=4,
-                                                      collate_fn=collator)
-
-        return trainLoader
-
-    def val_dataloader(self):
-        cfg = self.cfg
-        loader = val_loader(trialFileName = cfg.evalTrialAVA, \
-                            audioPath     = os.path.join(cfg.audioPathAVA , cfg.evalDataType), \
-                            visualPath    = os.path.join(cfg.visualPathAVA, cfg.evalDataType), \
-                            )
-        valLoader = torch.utils.data.DataLoader(loader,
-                                                batch_size=cfg.VAL.BATCH_SIZE,
-                                                shuffle=False,
-                                                num_workers=16)
-        return valLoader
-
-
-def main():
-    # The structure of this code is learnt from https://github.com/clovaai/voxceleb_trainer
-    cfg = bootstrap(print_cfg=False)
-    print(cfg)
-
-    warnings.filterwarnings("ignore")
-    seed_everything(42, workers=True)
-
-    cfg = init_args(cfg)
-
-    # checkpoint_callback = ModelCheckpoint(dirpath=os.path.join(cfg.WORKSPACE, "model"),
-    #                                       save_top_k=-1,
-    #                                       filename='{epoch}')
-
-    data = DataPrep(cfg)
-
-    trainer = Trainer(
-        gpus=int(cfg.TRAIN.TRAINER_GPU),
-        precision=32,
-    # callbacks=[checkpoint_callback],
-        max_epochs=25,
-        replace_sampler_ddp=True)
-    # val_trainer = Trainer(deterministic=True, num_sanity_val_steps=-1, gpus=1)
-    if cfg.downloadAVA == True:
-        preprocess_AVA(cfg)
-        quit()
-
-    # if cfg.RESUME:
-    #     modelfiles = glob.glob('%s/model_0*.model' % cfg.modelSavePath)
-    #     modelfiles.sort()
-    #     if len(modelfiles) >= 1:
-    #         print("Model %s loaded from previous state!" % modelfiles[-1])
-    #         epoch = int(os.path.splitext(os.path.basename(modelfiles[-1]))[0][6:]) + 1
-    #         s = talkNet(cfg)
-    #         s.loadParameters(modelfiles[-1])
-    #     else:
-    #         epoch = 1
-    #         s = talkNet(cfg)
-    epoch = 1
-    if cfg.MODEL.NAME == "baseline":
-        from talkNet_multicard import talkNet
-    elif cfg.MODEL.NAME == "multi":
-        from talkNet_multi import talkNet
-
-    s = talkNet(cfg)
-
-    # scoreFile = open(cfg.scoreSavePath, "a+")
-
-    trainer.fit(s, train_dataloaders=data.train_dataloader())
-
-    modelfiles = glob.glob('%s/*.pth' % os.path.join(cfg.WORKSPACE, "model"))
-
-    modelfiles.sort()
-    for path in modelfiles:
-        s.loadParameters(path)
-        prec = trainer.validate(s, data.val_dataloader())
-
-    # if epoch % cfg.testInterval == 0:
-    # s.saveParameters(cfg.modelSavePath + "/model_%04d.model" % epoch)
-    # trainer.validate(dataloaders=valLoader)
-    # print(time.strftime("%Y-%m-%d %H:%M:%S"), "%d epoch, mAP %2.2f%%" % (epoch, mAPs[-1]))
-    # scoreFile.write("%d epoch, LOSS %f, mAP %2.2f%%\n" % (epoch, loss, mAPs[-1]))
-    # scoreFile.flush()
-
-
-if __name__ == '__main__':
-    main()

legacy/train_multi.py

@@ -1,156 +0,0 @@
-import time, os, torch, argparse, warnings, glob
-
-from dataLoader_multiperson import train_loader, val_loader
-from utils.tools import *
-from talkNet_multi import talkNet
-
-
-def collate_fn_padding(data):
-    audiofeatures = [item[0] for item in data]
-    visualfeatures = [item[1] for item in data]
-    labels = [item[2] for item in data]
-    masks = [item[3] for item in data]
-    cut_limit = 200
-    # pad audio
-    lengths = torch.tensor([t.shape[1] for t in audiofeatures])
-    max_len = max(lengths)
-    padded_audio = torch.stack([
-        torch.cat([i, i.new_zeros((i.shape[0], max_len - i.shape[1], i.shape[2]))], 1)
-        for i in audiofeatures
-    ], 0)
-
-    if max_len > cut_limit * 4:
-        padded_audio = padded_audio[:, :, :cut_limit * 4, ...]
-
-    # pad video
-    lengths = torch.tensor([t.shape[1] for t in visualfeatures])
-    max_len = max(lengths)
-    padded_video = torch.stack([
-        torch.cat([i, i.new_zeros((i.shape[0], max_len - i.shape[1], i.shape[2], i.shape[3]))], 1)
-        for i in visualfeatures
-    ], 0)
-    padded_labels = torch.stack(
-        [torch.cat([i, i.new_zeros((i.shape[0], max_len - i.shape[1]))], 1) for i in labels], 0)
-    padded_masks = torch.stack(
-        [torch.cat([i, i.new_zeros((i.shape[0], max_len - i.shape[1]))], 1) for i in masks], 0)
-
-    if max_len > cut_limit:
-        padded_video = padded_video[:, :, :cut_limit, ...]
-        padded_labels = padded_labels[:, :, :cut_limit, ...]
-        padded_masks = padded_masks[:, :, :cut_limit, ...]
-    # print(padded_audio.shape, padded_video.shape, padded_labels.shape, padded_masks.shape)
-    return padded_audio, padded_video, padded_labels, padded_masks
-
-
-def main():
-    # The structure of this code is learnt from https://github.com/clovaai/voxceleb_trainer
-    warnings.filterwarnings("ignore")
-
-    parser = argparse.ArgumentParser(description="TalkNet Training")
-    # Training details
-    parser.add_argument('--lr', type=float, default=0.0001, help='Learning rate')
-    parser.add_argument('--lrDecay', type=float, default=0.95, help='Learning rate decay rate')
-    parser.add_argument('--maxEpoch', type=int, default=25, help='Maximum number of epochs')
-    parser.add_argument('--testInterval',
-                        type=int,
-                        default=1,
-                        help='Test and save every [testInterval] epochs')
-    parser.add_argument(
-        '--batchSize',
-        type=int,
-        default=2500,
-        help=
-        'Dynamic batch size, default is 2500 frames, other batchsize (such as 1500) will not affect the performance'
-    )
-    parser.add_argument('--batch_size', type=int, default=1, help='batch_size')
-    parser.add_argument('--num_speakers', type=int, default=5, help='num_speakers')
-    parser.add_argument('--nDataLoaderThread', type=int, default=4, help='Number of loader threads')
-    # Data path
-    parser.add_argument('--dataPathAVA',
-                        type=str,
-                        default="/data08/AVA",
-                        help='Save path of AVA dataset')
-    parser.add_argument('--savePath', type=str, default="exps/exp1")
-    # Data selection
-    parser.add_argument('--evalDataType',
-                        type=str,
-                        default="val",
-                        help='Only for AVA, to choose the dataset for evaluation, val or test')
-    # For download dataset only, for evaluation only
-    parser.add_argument('--downloadAVA',
-                        dest='downloadAVA',
-                        action='store_true',
-                        help='Only download AVA dataset and do related preprocess')
-    parser.add_argument('--evaluation',
-                        dest='evaluation',
-                        action='store_true',
-                        help='Only do evaluation by using pretrained model [pretrain_AVA.model]')
-    args = parser.parse_args()
-    # Data loader
-    args = init_args(args)
-
-    if args.downloadAVA == True:
-        preprocess_AVA(args)
-        quit()
-
-    loader = train_loader(trialFileName = args.trainTrialAVA, \
-                          audioPath      = os.path.join(args.audioPathAVA , 'train'), \
-                          visualPath     = os.path.join(args.visualPathAVA, 'train'), \
-                          # num_speakers = args.num_speakers, \
-                          **vars(args))
-    trainLoader = torch.utils.data.DataLoader(loader,
-                                              batch_size=args.batch_size,
-                                              shuffle=True,
-                                              num_workers=args.nDataLoaderThread,
-                                              collate_fn=collate_fn_padding)
-
-    loader = val_loader(trialFileName = args.evalTrialAVA, \
-                        audioPath     = os.path.join(args.audioPathAVA , args.evalDataType), \
-                        visualPath    = os.path.join(args.visualPathAVA, args.evalDataType), \
-                        # num_speakers = args.num_speakers, \
-                        **vars(args))
-    valLoader = torch.utils.data.DataLoader(loader, batch_size=1, shuffle=False, num_workers=16)
-
-    if args.evaluation == True:
-        download_pretrain_model_AVA()
-        s = talkNet(**vars(args))
-        s.loadParameters('pretrain_AVA.model')
-        print("Model %s loaded from previous state!" % ('pretrain_AVA.model'))
-        mAP = s.evaluate_network(loader=valLoader, **vars(args))
-        print("mAP %2.2f%%" % (mAP))
-        quit()
-
-    modelfiles = glob.glob('%s/model_0*.model' % args.modelSavePath)
-    modelfiles.sort()
-    if len(modelfiles) >= 1:
-        print("Model %s loaded from previous state!" % modelfiles[-1])
-        epoch = int(os.path.splitext(os.path.basename(modelfiles[-1]))[0][6:]) + 1
-        s = talkNet(epoch=epoch, **vars(args))
-        s.loadParameters(modelfiles[-1])
-    else:
-        epoch = 1
-        s = talkNet(epoch=epoch, **vars(args))
-
-    mAPs = []
-    scoreFile = open(args.scoreSavePath, "a+")
-
-    while (1):
-        loss, lr = s.train_network(epoch=epoch, loader=trainLoader, **vars(args))
-
-        if epoch % args.testInterval == 0:
-            s.saveParameters(args.modelSavePath + "/model_%04d.model" % epoch)
-            mAPs.append(s.evaluate_network(epoch=epoch, loader=valLoader, **vars(args)))
-            print(time.strftime("%Y-%m-%d %H:%M:%S"),
-                  "%d epoch, mAP %2.2f%%, bestmAP %2.2f%%" % (epoch, mAPs[-1], max(mAPs)))
-            scoreFile.write("%d epoch, LR %f, LOSS %f, mAP %2.2f%%, bestmAP %2.2f%%\n" %
-                            (epoch, lr, loss, mAPs[-1], max(mAPs)))
-            scoreFile.flush()
-
-        if epoch >= args.maxEpoch:
-            quit()
-
-        epoch += 1
-
-
-if __name__ == '__main__':
-    main()