| | get.auc.by.epoch <- function(configs, base.line="uniprotID") { |
| | log.dir <- configs$log_dir |
| | data.train <- as.numeric(strsplit(system(paste0("wc -l ", configs$data_file_train), intern = T), split = " ")[[1]][1]) |
| | num_saved_batches <- floor(ceiling(data.train * configs$train_size / configs$ngpus / configs$batch_size) |
| | * configs$num_epochs / configs$num_save_batches) + 1 |
| | epochs <- c(1:(configs$num_epochs)) |
| | source('/share/pascal/Users/gz2294/Pipeline/AUROC.R') |
| | library(doParallel) |
| | cl <- makeCluster(72) |
| | registerDoParallel(cl) |
| | |
| | res <- foreach (i = 1:length(epochs), .combine=rbind) %dopar% { |
| | i <- epochs[i] |
| | source('~/Pipeline/AUROC.R') |
| | if (file.exists(paste0(log.dir, 'test_result.epoch.', i, '.csv'))) { |
| | test.result <- read.csv(paste0(log.dir, 'test_result.epoch.', i, '.csv')) |
| | if ('y.0' %in% colnames(test.result)) { |
| | if ('y.2' %in% colnames(test.result)) { |
| | |
| | test.result <- test.result[!is.na(test.result$y.0) & !is.na(test.result$y.1) & !is.na(test.result$y.2),] |
| | test.logits <- test.result[, c("y.0", "y.1", "y.2")] |
| | test.logits <- t(apply(as.matrix(test.logits), 1, soft.max)) |
| | |
| | if (-1 %in% test.result$score) { |
| | test.logits <- test.logits[,3] / (test.logits[,2] + test.logits[,3]) |
| | } else { |
| | test.logits <- 1 - test.logits[,1] |
| | } |
| | } else if ('y.1' %in% colnames(test.result)) { |
| | test.result <- test.result[!is.na(test.result$y.1),] |
| | test.logits <- test.result$y.1 |
| | } else { |
| | test.result <- test.result[!is.na(test.result$y.0),] |
| | test.logits <- test.result$y.0 |
| | } |
| | } else { |
| | test.result <- test.result[!is.na(test.result$y),] |
| | test.logits <- test.result$y |
| | } |
| | result <- plot.AUC(test.result$score, test.logits |
| | |
| | ) |
| | J_stats <- result$curve[,2] - result$curve[,1] |
| | optimal.cutoff <- result$curve[which(J_stats==max(J_stats))[1],3] |
| | } else { |
| | result <- list(auc=NA) |
| | optimal.cutoff <- NA |
| | } |
| | if (file.exists(paste0(log.dir, 'result_dict.epoch.', i-1, '.ddp_rank.', 0, '.json'))) { |
| | val_losses <- c() |
| | train_losses <- c() |
| | if (configs$ngpus > 1) { |
| | for (rank in 0:(configs$ngpus-1)) { |
| | val_dic <- jsonlite::read_json(paste0(log.dir, 'result_dict.epoch.', i-1, '.ddp_rank.', rank, '.json')) |
| | if (!is.null(val_dic$val_loss_y)) { |
| | val_losses <- c(val_losses, val_dic$val_loss_y) |
| | train_losses <- c(train_losses, val_dic$train_loss_y) |
| | } else { |
| | val_losses <- c(val_losses, val_dic$val_loss) |
| | train_losses <- c(train_losses, val_dic$train_loss) |
| | } |
| | } |
| | } else { |
| | rank <- configs$gpu_id |
| | if (is.null(rank)) { |
| | rank <- 0 |
| | } |
| | val_dic <- jsonlite::read_json(paste0(log.dir, 'result_dict.epoch.', i-1, '.ddp_rank.', rank, '.json')) |
| | if (!is.null(val_dic$val_loss_y)) { |
| | val_losses <- c(val_losses, val_dic$val_loss_y) |
| | train_losses <- c(train_losses, val_dic$train_loss_y) |
| | } else { |
| | val_losses <- c(val_losses, val_dic$val_loss) |
| | train_losses <- c(train_losses, val_dic$train_loss) |
| | } |
| | } |
| | } else { |
| | train_losses <- NA |
| | val_losses <- NA |
| | } |
| | if (file.exists(paste0(log.dir, 'test_result.epoch.', i, '.txt'))) { |
| | test_dic <- readLines(paste0(log.dir, 'test_result.epoch.', i, '.txt'), warn = F) |
| | test_dic <- gsub("'", '"', test_dic) |
| | test_dic <- jsonlite::fromJSON(test_dic) |
| | if (!is.null(test_dic$test_loss_y)) { |
| | test_losses <- test_dic$test_loss_y |
| | } else { |
| | test_losses <- test_dic$test_loss |
| | } |
| | } else { |
| | test_losses <- NA |
| | } |
| | res <- data.frame(train=mean(train_losses), |
| | val=mean(val_losses), |
| | test=mean(test_losses), |
| | aucs=result$auc, |
| | optimal.cutoffs=optimal.cutoff) |
| | res |
| | } |
| | stopCluster(cl) |
| | res$epochs <- epochs |
| | res <- res[!is.na(res$train),] |
| | val <- res$val |
| | aucs <- res$aucs |
| | optimal.cutoffs <- res$optimal.cutoffs |
| | epochs <- res$epochs |
| | train <- res$train |
| | to.plot <- data.frame(epoch=rep(epochs, 2), |
| | loss=c(train, val), |
| | auc=rep(aucs, 2), |
| | metric_name=c(rep("train_loss", length(epochs)), |
| | rep("val_loss", length(epochs)))) |
| | |
| | if (base.line == "uniprotID") { |
| | baseline.uniprotID <- system( |
| | paste0("/share/vault/Users/gz2294/miniconda3/envs/r4-base/bin/python ", |
| | "/share/pascal/Users/gz2294/PreMode.final/analysis/random.forest.process.classifier.py ", |
| | configs$data_file_train, " ", |
| | configs$data_file_test), intern = T, |
| | ) |
| | baseline.auc <- as.numeric(strsplit(baseline.uniprotID, ": ")[[1]][2]) |
| | if (dim(res)[1] > 0) { |
| | res$baseline.auc <- baseline.auc |
| | } |
| | } else if (base.line == "esm") { |
| | alphabet <- c('<cls>', '<pad>', '<eos>', '<unk>', |
| | 'L', 'A', 'G', 'V', 'S', 'E', 'R', 'T', 'I', 'D', |
| | 'P', 'K', 'Q', 'N', 'F', 'Y', 'M', 'H', 'W', 'C', |
| | 'X', 'B', 'U', 'Z', 'O', '.', '-', |
| | '<null_1>', '<mask>') |
| | data.file.name <- configs$data_type |
| | fold <- strsplit(configs$data_file_train, "pfams.0.8.seed.")[[1]][2] |
| | fold <- as.numeric(substr(fold, 1, 1)) |
| | if (is.na(fold)) { |
| | fold <- 0 |
| | } |
| | baseline.file <- paste0('/share/pascal/Users/gz2294/PreMode.final/analysis/esm2.inference/', |
| | data.file.name, "/testing.fold.", fold, ".logits.csv") |
| | test.result <- read.csv(configs$data_file_test, row.names = 1) |
| | if (file.exists(baseline.file)) { |
| | baseline.res <- read.csv(baseline.file) |
| | logits <- baseline.res[,2:34] |
| | colnames(logits) <- alphabet |
| | score <- c() |
| | for (k in 1:dim(logits)[1]) { |
| | score <- c(score, logits[k, test.result$alt[k]] - logits[k, test.result$ref[k]]) |
| | } |
| | result <- plot.AUC(test.result$score, score) |
| | if (dim(res)[1] > 0) { |
| | res$baseline.auc <- result$auc |
| | } |
| | } |
| | } |
| | library(ggplot2) |
| | if (is.na(to.plot$auc[1])) { |
| | p <- ggplot(to.plot, aes(x=epoch)) + |
| | geom_line(aes(y=loss, col=metric_name)) + |
| | theme_bw() + |
| | theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) |
| | } else { |
| | p <- ggplot(to.plot, aes(x=epoch)) + |
| | geom_line(aes(y=loss, col=metric_name)) + |
| | geom_line(aes(y=auc)) + |
| | scale_y_continuous( |
| | |
| | name = "Loss", |
| | breaks = seq(0, max(1.1, max(to.plot$loss)), by = 0.05), limits = c(0, max(1.1, max(to.plot$loss))), |
| | |
| | sec.axis = sec_axis(~ . , name="AUC", |
| | breaks = seq(0, max(1.1, max(to.plot$loss)), by = 0.05)) |
| | ) + |
| | theme_bw() + |
| | theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) |
| | } |
| | |
| | ggsave('Loss.AUC.by.epoch.pdf', p, width = 9, height = 6) |
| | |
| | print(paste0("min val epoch (", epochs[which(val==min(val))[1]], |
| | ") AUC: ", round(aucs[which(val==min(val))[1]], digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[which(val==min(val))[1]], digits = 2))) |
| | print(paste0("end epoch (", epochs[length(val)], |
| | ") AUC: ", round(aucs[length(aucs)], digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[length(aucs)], digits = 2))) |
| | print(paste0("max AUC epoch (", epochs[which(aucs==max(aucs))[1]], |
| | "): ", round(max(aucs), digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[which(aucs==max(aucs))[1]], digits = 2))) |
| | res |
| | } |
| |
|
| |
|
| | get.auc.by.step <- function(configs, base.line="uniprotID") { |
| | log.dir <- configs$log_dir |
| | data.train <- as.numeric(strsplit(system(paste0("wc -l ", configs$data_file_train), intern = T), split = " ")[[1]][1]) |
| | num_saved_batches = floor(ceiling(data.train * configs$train_size / configs$ngpus / configs$batch_size) |
| | * configs$num_epochs / configs$num_save_batches) + 1 |
| | steps <- c(1:(num_saved_batches-1))*configs$num_save_batches |
| | source('/share/pascal/Users/gz2294/Pipeline/AUROC.R') |
| | library(doParallel) |
| | cl <- makeCluster(72) |
| | registerDoParallel(cl) |
| | res <- foreach (i = 1:length(steps), .combine=rbind) %dopar% { |
| | |
| | source('~/Pipeline/AUROC.R') |
| | i <- steps[i] |
| | if (file.exists(paste0(log.dir, 'test_result.step.', i, '.csv'))) { |
| | test.result <- read.csv(paste0(log.dir, 'test_result.step.', i, '.csv')) |
| | if ('y.0' %in% colnames(test.result)) { |
| | if ('y.2' %in% colnames(test.result)) { |
| | |
| | test.result <- test.result[!is.na(test.result$y.0) & !is.na(test.result$y.1) & !is.na(test.result$y.2),] |
| | test.logits <- test.result[, c("y.0", "y.1", "y.2")] |
| | test.logits <- t(apply(as.matrix(test.logits), 1, soft.max)) |
| | |
| | if (-1 %in% test.result$score) { |
| | test.logits <- test.logits[,3] / (test.logits[,2] + test.logits[,3]) |
| | } else { |
| | test.logits <- 1 - test.logits[,1] |
| | } |
| | } else if ('y.1' %in% colnames(test.result)) { |
| | test.result <- test.result[!is.na(test.result$y.1),] |
| | test.logits <- test.result$y.1 |
| | } else { |
| | test.result <- test.result[!is.na(test.result$y.0),] |
| | test.logits <- test.result$y.0 |
| | } |
| | } else { |
| | test.result <- test.result[!is.na(test.result$y),] |
| | test.logits <- test.result$y |
| | } |
| | result <- plot.AUC(test.result$score, test.logits |
| | |
| | ) |
| | J_stats <- result$curve[,2] - result$curve[,1] |
| | optimal.cutoff <- result$curve[which(J_stats==max(J_stats))[1],3] |
| | } else { |
| | result <- list(auc=NA) |
| | optimal.cutoff <- NA |
| | } |
| | if (file.exists(paste0(log.dir, 'result_dict.batch.', i, '.ddp_rank.', 0, '.json'))) { |
| | val_losses <- c() |
| | train_losses <- c() |
| | if (configs$ngpus > 1) { |
| | for (rank in 0:(configs$ngpus-1)) { |
| | val_dic <- jsonlite::read_json(paste0(log.dir, 'result_dict.batch.', i, '.ddp_rank.', rank, '.json')) |
| | if (!is.null(val_dic$val_loss_y)) { |
| | val_losses <- c(val_losses, val_dic$val_loss_y) |
| | train_losses <- c(train_losses, val_dic$train_loss_y) |
| | } else { |
| | val_losses <- c(val_losses, val_dic$val_loss) |
| | train_losses <- c(train_losses, val_dic$train_loss) |
| | } |
| | } |
| | } else { |
| | rank <- configs$gpu_id |
| | if (is.null(rank)) { |
| | rank <- 0 |
| | } |
| | val_dic <- jsonlite::read_json(paste0(log.dir, 'result_dict.batch.', i, '.ddp_rank.', rank, '.json')) |
| | if (!is.null(val_dic$val_loss_y)) { |
| | val_losses <- c(val_losses, val_dic$val_loss_y) |
| | train_losses <- c(train_losses, val_dic$train_loss_y) |
| | } else { |
| | val_losses <- c(val_losses, val_dic$val_loss) |
| | train_losses <- c(train_losses, val_dic$train_loss) |
| | } |
| | } |
| | } else { |
| | val_losses <- NA |
| | train_losses <- NA |
| | } |
| | if (file.exists(paste0(log.dir, 'test_result.step.', i, '.txt'))) { |
| | test_dic <- readLines(paste0(log.dir, 'test_result.step.', i, '.txt'), warn = F) |
| | test_dic <- gsub("'", '"', test_dic) |
| | test_dic <- jsonlite::fromJSON(test_dic) |
| | if (!is.null(test_dic$test_loss_y)) { |
| | test_losses <- test_dic$test_loss_y |
| | } else { |
| | test_losses <- test_dic$test_loss |
| | } |
| | } else { |
| | test_losses <- NA |
| | } |
| | res <- data.frame(train=mean(train_losses), |
| | val=mean(val_losses), |
| | test=mean(test_losses), |
| | aucs=result$auc, |
| | optimal.cutoffs=optimal.cutoff) |
| | print(res) |
| | } |
| | stopCluster(cl) |
| | res$steps <- steps |
| | res <- res[!is.na(res$train),] |
| | val <- res$val |
| | aucs <- res$aucs |
| | optimal.cutoffs <- res$optimal.cutoffs |
| | steps <- res$steps |
| | train <- res$train |
| | to.plot <- data.frame(step=rep(steps, 2), |
| | loss=c(train, val), |
| | auc=rep(aucs, 2), |
| | metric_name=c(rep("train_loss", length(steps)), |
| | rep("val_loss", length(steps)))) |
| | |
| | if (base.line == "uniprotID") { |
| | baseline.uniprotID <- system( |
| | paste0("/share/vault/Users/gz2294/miniconda3/envs/r4-base/bin/python ", |
| | "/share/pascal/Users/gz2294/PreMode.final/analysis/random.forest.process.classifier.py ", |
| | configs$data_file_train, " ", |
| | configs$data_file_test), intern = T, |
| | ) |
| | baseline.auc <- as.numeric(strsplit(baseline.uniprotID, ": ")[[1]][2]) |
| | if (dim(res)[1] > 0) { |
| | res$baseline.auc <- baseline.auc |
| | } |
| | } else if (base.line == "esm") { |
| | alphabet <- c('<cls>', '<pad>', '<eos>', '<unk>', |
| | 'L', 'A', 'G', 'V', 'S', 'E', 'R', 'T', 'I', 'D', |
| | 'P', 'K', 'Q', 'N', 'F', 'Y', 'M', 'H', 'W', 'C', |
| | 'X', 'B', 'U', 'Z', 'O', '.', '-', |
| | '<null_1>', '<mask>') |
| | data.file.name <- configs$data_type |
| | fold <- strsplit(configs$data_file_train, "pfams.0.8.seed.")[[1]][2] |
| | fold <- as.numeric(substr(fold, 1, 1)) |
| | if (is.na(fold)) { |
| | fold <- 0 |
| | } |
| | baseline.file <- paste0('/share/pascal/Users/gz2294/PreMode.final/analysis/esm2.inference/', |
| | data.file.name, "/testing.fold.", fold, ".logits.csv") |
| | test.result <- read.csv(configs$data_file_test, row.names = 1) |
| | if (file.exists(baseline.file)) { |
| | baseline.res <- read.csv(baseline.file) |
| | logits <- baseline.res[,2:34] |
| | colnames(logits) <- alphabet |
| | score <- c() |
| | for (k in 1:dim(logits)[1]) { |
| | score <- c(score, logits[k, test.result$alt[k]] - logits[k, test.result$ref[k]]) |
| | } |
| | result <- plot.AUC(test.result$score, score) |
| | if (dim(res)[1] > 0) { |
| | res$baseline.auc <- result$auc |
| | } |
| | } |
| | } |
| | library(ggplot2) |
| | if (is.na(to.plot$auc[1])) { |
| | p <- ggplot(to.plot, aes(x=step)) + |
| | geom_line(aes(y=loss, col=metric_name)) + |
| | scale_x_continuous(breaks = |
| | seq(1*configs$num_save_batches, |
| | (num_saved_batches - 1)*configs$num_save_batches, |
| | by = configs$num_save_batches)) + |
| | theme_bw() + |
| | theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) |
| | } else { |
| | p <- ggplot(to.plot, aes(x=step)) + |
| | geom_line(aes(y=loss, col=metric_name)) + |
| | geom_line(aes(y=auc)) + |
| | scale_y_continuous( |
| | |
| | name = "Loss", |
| | breaks = seq(0, max(1.1, max(to.plot$loss)), by = 0.05), limits = c(0, max(1.1, max(to.plot$loss))), |
| | |
| | sec.axis = sec_axis(~ . , name="AUC", |
| | breaks = seq(0, max(1.1, max(to.plot$loss)), by = 0.05)) |
| | ) + |
| | scale_x_continuous(breaks = |
| | seq(1*configs$num_save_batches, |
| | (num_saved_batches - 1)*configs$num_save_batches, |
| | by = configs$num_save_batches)) + |
| | theme_bw() + |
| | theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) |
| | } |
| | ggsave('Loss.AUC.by.step.pdf', p, width = max(6, min(6 * length(steps) / 50, 20)), height = 4) |
| | |
| | print(paste0("min val step (", steps[which(val==min(val))[1]], |
| | ") AUC: ", round(aucs[which(val==min(val))[1]], digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[which(val==min(val))[1]], digits = 2))) |
| | print(paste0("end step (", steps[length(val)], |
| | ") AUC: ", round(aucs[length(aucs)], digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[length(aucs)], digits = 2))) |
| | print(paste0("max AUC step (", steps[which(aucs==max(aucs))[1]], |
| | "): ", round(max(aucs), digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[which(aucs==max(aucs))[1]], digits = 2))) |
| | res |
| | } |
| |
|
| |
|
| | get.auc.by.step.split.pLDDT <- function(configs, base.line="uniprotID") { |
| | log.dir <- configs$log_dir |
| | data.train <- as.numeric(strsplit(system(paste0("wc -l ", configs$data_file_train), intern = T), split = " ")[[1]][1]) |
| | num_saved_batches = floor(ceiling(data.train * configs$train_size / configs$ngpus / configs$batch_size) |
| | * configs$num_epochs / configs$num_save_batches) + 1 |
| | steps <- c(1:(num_saved_batches-1))*configs$num_save_batches |
| | source('~/Pipeline/uniprot.table.add.annotation.R') |
| | test.file <- read.csv(configs$data_file_test) |
| | test.file <- uniprot.table.add.annotation.parallel(test.file, 'pLDDT.region') |
| | source('/share/pascal/Users/gz2294/Pipeline/AUROC.R') |
| | library(doParallel) |
| | cl <- makeCluster(72) |
| | registerDoParallel(cl) |
| | res <- foreach (i = 1:length(steps), .combine=rbind) %dopar% { |
| | |
| | source('~/Pipeline/AUROC.R') |
| | i <- steps[i] |
| | if (file.exists(paste0(log.dir, 'test_result.step.', i, '.csv'))) { |
| | test.result <- read.csv(paste0(log.dir, 'test_result.step.', i, '.csv')) |
| | if ('y.0' %in% colnames(test.result)) { |
| | if ('y.2' %in% colnames(test.result)) { |
| | |
| | test.result <- test.result[!is.na(test.result$y.0) & !is.na(test.result$y.1) & !is.na(test.result$y.2),] |
| | test.logits <- test.result[, c("y.0", "y.1", "y.2")] |
| | test.logits <- t(apply(as.matrix(test.logits), 1, soft.max)) |
| | |
| | if (-1 %in% test.result$score) { |
| | test.logits <- test.logits[,3] / (test.logits[,2] + test.logits[,3]) |
| | } else { |
| | test.logits <- 1 - test.logits[,1] |
| | } |
| | } else if ('y.1' %in% colnames(test.result)) { |
| | test.result <- test.result[!is.na(test.result$y.1),] |
| | test.logits <- test.result$y.1 |
| | } else { |
| | test.result <- test.result[!is.na(test.result$y.0),] |
| | test.logits <- test.result$y.0 |
| | } |
| | } else { |
| | test.result <- test.result[!is.na(test.result$y),] |
| | test.logits <- test.result$y |
| | } |
| | result <- plot.AUC(test.result$score, test.logits) |
| | result.1 <- plot.AUC(test.result$score[test.file$pLDDT.region >= 70], |
| | test.logits[test.file$pLDDT.region >= 70]) |
| | result.2 <- plot.AUC(test.result$score[test.file$pLDDT.region < 70], |
| | test.logits[test.file$pLDDT.region < 70]) |
| | J_stats <- result$curve[,2] - result$curve[,1] |
| | optimal.cutoff <- result$curve[which(J_stats==max(J_stats))[1],3] |
| | } else { |
| | result <- list(auc=NA) |
| | result.1 <- list(auc=NA) |
| | result.2 <- list(auc=NA) |
| | optimal.cutoff <- NA |
| | } |
| | if (file.exists(paste0(log.dir, 'result_dict.batch.', i, '.ddp_rank.', 0, '.json'))) { |
| | val_losses <- c() |
| | train_losses <- c() |
| | if (configs$ngpus > 1) { |
| | for (rank in 0:(configs$ngpus-1)) { |
| | val_dic <- jsonlite::read_json(paste0(log.dir, 'result_dict.batch.', i, '.ddp_rank.', rank, '.json')) |
| | if (!is.null(val_dic$val_loss_y)) { |
| | val_losses <- c(val_losses, val_dic$val_loss_y) |
| | train_losses <- c(train_losses, val_dic$train_loss_y) |
| | } else { |
| | val_losses <- c(val_losses, val_dic$val_loss) |
| | train_losses <- c(train_losses, val_dic$train_loss) |
| | } |
| | } |
| | } else { |
| | rank <- configs$gpu_id |
| | if (is.null(rank)) { |
| | rank <- 0 |
| | } |
| | val_dic <- jsonlite::read_json(paste0(log.dir, 'result_dict.batch.', i, '.ddp_rank.', rank, '.json')) |
| | if (!is.null(val_dic$val_loss_y)) { |
| | val_losses <- c(val_losses, val_dic$val_loss_y) |
| | train_losses <- c(train_losses, val_dic$train_loss_y) |
| | } else { |
| | val_losses <- c(val_losses, val_dic$val_loss) |
| | train_losses <- c(train_losses, val_dic$train_loss) |
| | } |
| | } |
| | } else { |
| | val_losses <- NA |
| | train_losses <- NA |
| | } |
| | if (file.exists(paste0(log.dir, 'test_result.step.', i, '.txt'))) { |
| | test_dic <- readLines(paste0(log.dir, 'test_result.step.', i, '.txt'), warn = F) |
| | test_dic <- gsub("'", '"', test_dic) |
| | test_dic <- jsonlite::fromJSON(test_dic) |
| | if (!is.null(test_dic$test_loss_y)) { |
| | test_losses <- test_dic$test_loss_y |
| | } else { |
| | test_losses <- test_dic$test_loss |
| | } |
| | } else { |
| | test_losses <- NA |
| | } |
| | res <- data.frame(train=mean(train_losses), |
| | val=mean(val_losses), |
| | test=mean(test_losses), |
| | aucs=result$auc, |
| | aucs.low_pLDDT=result.2$auc, |
| | aucs.high_pLDDT=result.1$auc, |
| | optimal.cutoffs=optimal.cutoff) |
| | print(res) |
| | } |
| | stopCluster(cl) |
| | res$steps <- steps |
| | res <- res[!is.na(res$train),] |
| | val <- res$val |
| | aucs <- res$aucs |
| | optimal.cutoffs <- res$optimal.cutoffs |
| | steps <- res$steps |
| | train <- res$train |
| | result.1.neg <- sum(test.file$pLDDT.region >= 70 & test.file$score == -1) |
| | result.1.pos <- sum(test.file$pLDDT.region >= 70 & test.file$score == 1) |
| | result.2.neg <- sum(test.file$pLDDT.region < 70 & test.file$score == -1) |
| | result.2.pos <- sum(test.file$pLDDT.region < 70 & test.file$score == 1) |
| | to.plot <- data.frame(step=rep(steps, 2), |
| | loss=c(train, val), |
| | auc=rep(aucs, 2), |
| | auc.pLDDT=c(res$aucs.high_pLDDT, res$aucs.low_pLDDT), |
| | auc.name = c(rep(paste0("pLDDT >= 0.7 (", result.1.neg, "/", result.1.pos, ")"), |
| | length(steps)), |
| | rep(paste0("pLDDT < 0.7 (", result.2.neg, "/", result.2.pos, ")"), |
| | length(steps))), |
| | metric_name=c(rep("train_loss", length(steps)), |
| | rep("val_loss", length(steps)))) |
| | |
| | if (base.line == "uniprotID") { |
| | baseline.uniprotID <- system( |
| | paste0("/share/vault/Users/gz2294/miniconda3/envs/r4-base/bin/python ", |
| | "/share/pascal/Users/gz2294/PreMode.final/analysis/random.forest.process.classifier.py ", |
| | configs$data_file_train, " ", |
| | configs$data_file_test), intern = T, |
| | ) |
| | baseline.auc <- as.numeric(strsplit(baseline.uniprotID, ": ")[[1]][2]) |
| | if (dim(res)[1] > 0) { |
| | res$baseline.auc <- baseline.auc |
| | } |
| | } else if (base.line == "esm") { |
| | alphabet <- c('<cls>', '<pad>', '<eos>', '<unk>', |
| | 'L', 'A', 'G', 'V', 'S', 'E', 'R', 'T', 'I', 'D', |
| | 'P', 'K', 'Q', 'N', 'F', 'Y', 'M', 'H', 'W', 'C', |
| | 'X', 'B', 'U', 'Z', 'O', '.', '-', |
| | '<null_1>', '<mask>') |
| | data.file.name <- configs$data_type |
| | fold <- strsplit(configs$data_file_train, "pfams.0.8.seed.")[[1]][2] |
| | fold <- as.numeric(substr(fold, 1, 1)) |
| | if (is.na(fold)) { |
| | fold <- 0 |
| | } |
| | baseline.file <- paste0('/share/pascal/Users/gz2294/PreMode.final/analysis/esm2.inference/', |
| | data.file.name, "/testing.fold.", fold, ".logits.csv") |
| | test.result <- read.csv(configs$data_file_test, row.names = 1) |
| | if (file.exists(baseline.file)) { |
| | baseline.res <- read.csv(baseline.file) |
| | logits <- baseline.res[,2:34] |
| | colnames(logits) <- alphabet |
| | score <- c() |
| | for (k in 1:dim(logits)[1]) { |
| | score <- c(score, logits[k, test.result$alt[k]] - logits[k, test.result$ref[k]]) |
| | } |
| | result <- plot.AUC(test.result$score, score) |
| | if (dim(res)[1] > 0) { |
| | res$baseline.auc <- result$auc |
| | } |
| | } |
| | } |
| | library(ggplot2) |
| | if (is.na(to.plot$auc[1])) { |
| | p <- ggplot(to.plot, aes(x=step)) + |
| | geom_line(aes(y=loss, col=metric_name)) + |
| | scale_x_continuous(breaks = |
| | seq(1*configs$num_save_batches, |
| | (num_saved_batches - 1)*configs$num_save_batches, |
| | by = configs$num_save_batches)) + |
| | theme_bw() + |
| | theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) |
| | } else { |
| | p <- ggplot(to.plot, aes(x=step)) + |
| | geom_line(aes(y=loss, col=metric_name)) + |
| | geom_line(aes(y=auc)) + |
| | geom_line(aes(y=auc.pLDDT, col=auc.name)) + |
| | scale_y_continuous( |
| | |
| | name = "Loss", |
| | breaks = seq(0, max(1.1, max(to.plot$loss)), by = 0.05), limits = c(0, max(1.1, max(to.plot$loss))), |
| | |
| | sec.axis = sec_axis(~ . , name="AUC", |
| | breaks = seq(0, max(1.1, max(to.plot$loss)), by = 0.05)) |
| | ) + |
| | scale_x_continuous(breaks = |
| | seq(1*configs$num_save_batches, |
| | (num_saved_batches - 1)*configs$num_save_batches, |
| | by = configs$num_save_batches)) + |
| | theme_bw() + |
| | theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) |
| | } |
| | ggsave('Loss.AUC.by.step.pdf', p, width = max(9, min(9 * length(steps) / 50, 20)), height = 6) |
| | |
| | print(paste0("min val step (", steps[which(val==min(val))[1]], |
| | ") AUC: ", round(aucs[which(val==min(val))[1]], digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[which(val==min(val))[1]], digits = 2))) |
| | print(paste0("end step (", steps[length(val)], |
| | ") AUC: ", round(aucs[length(aucs)], digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[length(aucs)], digits = 2))) |
| | print(paste0("max AUC step (", steps[which(aucs==max(aucs))[1]], |
| | "): ", round(max(aucs), digits = 2), |
| | " Optimal cutoff: ", |
| | round(optimal.cutoffs[which(aucs==max(aucs))[1]], digits = 2))) |
| | res |
| | } |
| |
|
| |
|
| | get.R.by.epoch <- function(configs, bin=FALSE) { |
| | log.dir <- configs$log_dir |
| | epochs <- c(1:configs$num_epochs) |
| | library(doParallel) |
| | cl <- makeCluster(72) |
| | registerDoParallel(cl) |
| | res <- foreach (i = 1:length(epochs), .combine=dplyr::bind_rows) %dopar% { |
| | source('/share/pascal/Users/gz2294/Pipeline/AUROC.R') |
| | i <- epochs[i] |
| | if (file.exists(paste0(log.dir, 'test_result.epoch.', i, '.csv'))) { |
| | test.result <- read.csv(paste0(log.dir, 'test_result.epoch.', i, '.csv')) |
| | score.columns <- colnames(test.result)[grep("^score", colnames(test.result))] |
| | score.columns <- score.columns[order(score.columns)] |
| | result.columns <- colnames(test.result)[grep("^y.", colnames(test.result))] |
| | result.columns <- result.columns[order(result.columns)] |
| | test.result <- test.result[!is.na(test.result[,result.columns[1]]),] |
| | result <- plot.R2(test.result[,score.columns], test.result[,result.columns], |
| | bin=bin, |
| | filename=paste0(log.dir, 'test_result.epoch.', i, '.pdf')) |
| | |
| | |
| | rank <- 0 |
| | val_dic <- jsonlite::read_json(paste0(log.dir, 'result_dict.epoch.', i-1, '.ddp_rank.', rank, '.json')) |
| | |
| | |
| | res <- data.frame(epochs = i, |
| | train_loss = val_dic$val_loss, |
| | val_loss = val_dic$train_loss, |
| | R2s=t(as.data.frame(result$R2))) |
| | } else { |
| | res <- data.frame(epochs = NA, |
| | train_loss = NA, |
| | val_loss = NA) |
| | } |
| | res |
| | } |
| | stopCluster(cl) |
| | res <- res[!is.na(res$train_loss),] |
| | epochs <- res$epochs |
| | train <- res$train_loss |
| | val <- res$val_loss |
| | R2s <- as.data.frame(res[,startsWith(colnames(res), "R2s")]) |
| | if (dim(R2s)[1] > 0) { |
| | to.plot <- data.frame(epoch=rep(epochs, 2), |
| | loss=c(train, val), |
| | R2=c(rowMeans(R2s), rowMeans(R2s)), |
| | Rs=rbind(R2s, R2s), |
| | metric_name=c(rep("train_loss", length(epochs)), |
| | rep("val_loss", length(epochs)))) |
| | to.plot.2 <- data.frame() |
| | for (i in 1:dim(R2s)[2]) { |
| | to.plot.2 <- rbind(to.plot.2, |
| | data.frame(R2=R2s[,i], |
| | epoch=epochs, |
| | label=paste0('assay.', i))) |
| | } |
| | library(ggplot2) |
| | p <- ggplot(to.plot, aes(x=epoch)) + |
| | geom_line(aes(y=loss, col=metric_name)) + |
| | geom_line(data = to.plot.2, aes(y=R2, col=label)) + |
| | scale_y_continuous( |
| | |
| | name = "Loss", breaks = round(seq(min(to.plot$loss)-0.5, max(to.plot$loss)+0.5, by = 0.1), 1), |
| | |
| | sec.axis = sec_axis(~ . , name="R", breaks = round(seq(min(R2s), max(R2s), by = 0.1), 1)) |
| | ) + |
| | scale_x_continuous(breaks = seq(1, configs$num_epochs, by = 1)) + |
| | theme_bw() + |
| | theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) |
| | |
| | print(paste0("min val epoch R: ", round(R2s[which(val==min(val)),], digits = 100))) |
| | print(paste0("end epoch R: ", round(R2s[dim(R2s)[1],], digits = 100))) |
| | ggsave('Loss.R.by.epoch.pdf', p, width = min(49.9, 9 * length(epochs) / 10), height = 6) |
| | } |
| | res |
| | } |
| |
|
| | get.R.by.step <- function(configs, bin=FALSE) { |
| | log.dir <- configs$log_dir |
| | data.train <- as.numeric(strsplit(system(paste0("wc -l ", configs$data_file_train), intern = T), split = " ")[[1]][1]) |
| | num_saved_batches = floor(ceiling(floor(data.train * configs$train_size / configs$ngpus) / configs$batch_size) |
| | * configs$num_epochs / configs$num_save_batches) + 1 |
| | print(paste0("num batches: ", num_saved_batches)) |
| | steps <- c(1:(num_saved_batches-1)) * configs$num_save_batches |
| | R2s <- data.frame() |
| | train <- c() |
| | val <- c() |
| | |
| | library(doParallel) |
| | cl <- makeCluster(72) |
| | registerDoParallel(cl) |
| | |
| | res <- foreach (i = 1:length(steps), .combine = dplyr::bind_rows) %dopar% { |
| | source('/share/pascal/Users/gz2294/Pipeline/AUROC.R') |
| | i <- steps[i] |
| | if (file.exists(paste0(log.dir, 'test_result.step.', i, '.csv'))) { |
| | test.result <- read.csv(paste0(log.dir, 'test_result.step.', i, '.csv')) |
| | score.columns <- colnames(test.result)[grep("^score", colnames(test.result))] |
| | score.columns <- score.columns[order(score.columns)] |
| | result.columns <- colnames(test.result)[grep("^y.", colnames(test.result))] |
| | result.columns <- result.columns[order(result.columns)] |
| | test.result <- test.result[!is.na(test.result[,result.columns[1]]),] |
| | result <- plot.R2(test.result[,score.columns], test.result[,result.columns], |
| | bin=bin, |
| | filename=paste0(log.dir, 'test_result.step.', i, '.pdf')) |
| | val_losses <- c() |
| | train_losses <- c() |
| | rank <- 0 |
| | val_dic <- jsonlite::read_json(paste0(log.dir, 'result_dict.batch.', i, '.ddp_rank.', rank, '.json')) |
| | val_losses <- c(val_losses, val_dic$val_loss) |
| | train_losses <- c(train_losses, val_dic$train_loss) |
| | res <- data.frame(train=mean(train_losses), |
| | val=mean(val_losses), |
| | R2s=t(as.data.frame(result$R2))) |
| | } else { |
| | res <- data.frame(train=NA, |
| | val=NA, |
| | R2s=NA) |
| | } |
| | } |
| | stopCluster(cl) |
| | res$steps <- steps |
| | res <- res[!is.na(res$train),] |
| | train <- res$train |
| | val <- res$val |
| | steps <- res$steps |
| | R2s <- res[,colnames(res)[grep("^R2s", colnames(res))]] |
| | if (is.null(dim(R2s))) { |
| | R2s <- as.matrix(R2s) |
| | } |
| | to.plot <- data.frame(step=rep(steps, 2), |
| | loss=c(train, val), |
| | R2=c(rowMeans(R2s), rowMeans(R2s)), |
| | Rs=rbind(R2s, R2s), |
| | metric_name=c(rep("train_loss", length(steps)), |
| | rep("val_loss", length(steps)))) |
| | to.plot.2 <- data.frame() |
| | for (i in 1:dim(R2s)[2]) { |
| | to.plot.2 <- rbind(to.plot.2, |
| | data.frame(R2=R2s[,i], |
| | step=steps, |
| | label=paste0('assay.', i))) |
| | } |
| | |
| | library(ggplot2) |
| | p <- ggplot(to.plot, aes(x=step)) + |
| | geom_line(aes(y=loss, col=metric_name)) + |
| | geom_line(data = to.plot.2, aes(y=R2, col=label)) + |
| | scale_y_continuous( |
| | |
| | name = "Loss", breaks = round(seq(min(to.plot$loss)-0.5, max(to.plot$loss)+0.5, by = 0.1), 1), |
| | |
| | sec.axis = sec_axis(~ . , name="R", breaks = round(seq(min(R2s), max(R2s), by = 0.1), 1)) |
| | ) + |
| | scale_x_continuous(breaks = |
| | seq(1*configs$num_save_batches, |
| | (num_saved_batches - 1)*configs$num_save_batches, |
| | by = configs$num_save_batches)) + |
| | theme_bw() + |
| | theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) |
| | ggsave('Loss.R.by.step.pdf', p, width = min(max(9 * length(steps) / 100, 9), 49.9), height = 6) |
| | R2s <- as.data.frame(R2s) |
| | print(paste0("min val step (", steps[which(val==min(val))[1]], |
| | ") R: ", R2s[which(val==min(val))[1],])) |
| | print(paste0("end step (", steps[length(val)], |
| | ") R: ", R2s[length(steps),])) |
| | print(paste0("max R step (", steps[which(R2s==max(R2s))[1]], |
| | "): ", max(R2s))) |
| | |
| | |
| | |
| | |
| | |
| | |
| | res |
| | } |
| |
|
