Saving weights and logs of step 10000

.gitattributes

@@ -27,4 +27,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 chemT5_data.csv filter=lfs diff=lfs merge=lfs -text
-chemT5_data.tsv filter=lfs diff=lfs merge=lfs -text

chemT5_data.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:790657db4eff6c29407874fc4eb06ecfa134b91f924a44c215a0bf8b556ad307
-size 48054222
+oid sha256:7af01ac70fdcb2008a1bdc777d14181ab7f02ec48a55522710d2a7cd3f4e3952
+size 41116570

dataset-clean.py

@@ -13,13 +13,38 @@ input_sentence_size = None
 # Initialize a dataset
 #dataset = load_dataset('csv', data_files='/home/zoez/Chem-T5/train-file.csv',split="train")
 dataset = pd.read_csv('./chemT5_data.csv')#('/home/zoez/Chem-T5/train-file.csv')
+#dataset=pd.DataFrame(columns=['SMILES'],data=dataset)
+#dataset['SMILES']=dataset['SMILES'].str[2:]
+# for i, line in tqdm(enumerate(dataset['SMILES'])):
+#     print(line) 
+#     line = re.sub('\d+ ', '',line)
+#     #
+#     #newLine=line#atomwise_tokenizer(line)
+#     #print(newLine)
+#     #print(int(i/10))
+#     dataset.iloc[i]['SMILES']=line
+# print(dataset[0:5])
+# dataset.dropna()
+#dataset.to_csv('chemT5_data.csv',index=False)
+
 #print(dataset.iloc[0])
 
 dataset=pd.DataFrame(columns=['SMILES'],data=dataset)
-#dataset.drop('Unnamed: 0',1)
-#print(dataset.columns)
-dataset.columns=['SMILES']
+# print(dataset[0:5])
+# print(dataset.columns)
 
+# #dataset.drop('Unnamed: 0',1)
+# print(dataset.columns)
+# dataset.columns=['SMILES']
+# for i, line in tqdm(enumerate(dataset['SMILES'])):
+#     #line = re.sub('\d+ ', '',line)
+#     #print(line) 
+#     newLine=line#atomwise_tokenizer(line)
+#     #print(newLine)
+#     #print(int(i/10))
+#     dataset.iloc[i]['SMILES']=newLine
+# print(dataset['SMILES'][0:5])
 
-dataset.fillna('', inplace=True)
-dataset.to_csv('chemT5_data.csv',sep = ' ')
+dataset=dataset[~dataset.SMILES.str.contains("\"\"", regex=False,na=True)]
+#print(dataset[0:5])
+dataset.to_csv('chemT5_data.csv',index=False)

events.out.tfevents.1651650601.toxicgpu.cs.vt.edu.23181.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49e1c63d58bd954395e335ebfaa4e9dccfab105ab974c2f7da7eda1f8472a523
+size 40

events.out.tfevents.1651774324.toxicgpu.cs.vt.edu.2962.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de2218534456aa901282dfc1ace06039a8590cb8aac77177f745e1029c656314
+size 40

events.out.tfevents.1651774377.toxicgpu.cs.vt.edu.4116.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:911544eef0373a776c4d8a9035cae7a97d5c5a1fafa70b6ca2f59b3ac76a9609
+size 40

events.out.tfevents.1651774597.toxicgpu.cs.vt.edu.5771.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63b787fcb2b7e97f89d7be11ebb8859ab1ff98b882da0ff9fa1a7bb5a0abc8fe
+size 40

events.out.tfevents.1651774686.toxicgpu.cs.vt.edu.6128.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d8b9fb2ddaf0b2d87eb166d90dc0980e5386459906b5cf7b6135bd6ad5cf153
+size 40

events.out.tfevents.1651774751.toxicgpu.cs.vt.edu.7181.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:151565b73a0af3c7639f03dea6c25e2b440161f176f6661357a49e9122eb9a37
+size 40

events.out.tfevents.1651822478.toxicgpu.cs.vt.edu.31615.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e75d0d56e2064320d0238b790bc4c51826bea5c2d88d20ddec70797701248e2e
+size 40

events.out.tfevents.1651823225.toxicgpu.cs.vt.edu.32383.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f5c234e3e2b32fafd655d01824aaa400935ed79ce280c85a95b97f6805d0967
+size 40

events.out.tfevents.1651824342.toxicgpu.cs.vt.edu.2732.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:687c32f6139062f90c29c191620ff300b24cf3115e28c1abd8f548c58f8e31bd
+size 40

events.out.tfevents.1651824633.toxicgpu.cs.vt.edu.3509.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b77e5779e2dc13eeaa6bd2d850552966aae69aa2940d2ab67ee3f25a368dff
+size 40

events.out.tfevents.1651824828.toxicgpu.cs.vt.edu.3970.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8bec9562ab542ab8e34c2b3588e4b11fd53fa8d347624565638bc96eece2004
+size 40

events.out.tfevents.1651824941.toxicgpu.cs.vt.edu.4751.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c7156fe5e376bc60524bc25626cc042152247d31606161c575a2ec8c53a80d7
+size 1471867

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d99ed12fc3df890828fc608bde1949bb19fce1d45e4117685d366f0b31787a9
+oid sha256:568d3d6b8d71a100dda0d44d84d3e5704afd75c510ac8e4edd6e57c2ac2d0076
 size 990170015

pretrain_data.py

@@ -0,0 +1,27 @@
+import tensorflow as tf
+import torch as pt
+import pandas as pd
+import re
+from t5_tokenizer_model import SentencePieceUnigramTokenizer
+#from pretokenizer import atomwise_tokenizer
+from tqdm import tqdm
+from transformers import AutoTokenizer, T5Tokenizer, T5ForConditionalGeneration, T5Config
+from tokenizers import Tokenizer
+import numpy as np 
+
+
+tokenizer = AutoTokenizer.from_pretrained("./")
+dataset = pd.read_csv('./chemT5_data.csv')
+train=pd.DataFrame(data=dataset)
+
+for i, line in tqdm(enumerate(dataset['SMILES'])):
+    print(i," "+line)
+    line = tokenizer.encode(line)
+    #print(line) 
+    newLine=tokenizer.convert_ids_to_tokens(line)
+    #print(newLine)
+    #print(int(i/10))
+    train.iloc[i]['SMILES']=newLine
+
+#print(train[0:5])
+train.to_csv('pretrain.csv',index=False)

run_t5_mlm_flax.py

@@ -29,6 +29,7 @@ from typing import Dict, List, Optional
 
 import numpy as np
 from datasets import load_dataset
+from tokenizer import split_into_sentences
 from tqdm import tqdm
 
 import flax

tokenizer-trainer_uni.py

@@ -57,13 +57,21 @@ tokenizer.train_from_iterator(
 
 
 # Save files to disk
-tokenizer.save("/home/zoez/chemT5/uni-tokenizer.json")
+#tokenizer.save("/home/zoez/chemT5/uni-tokenizer.json")
 
 
 print(tokenizer.encode("O=[N+]([O-])c1ccc(Cl)cc1").tokens)
 
-from transformers import T5Config
+#from transformers import T5Config
+
+for i, line in tqdm(enumerate(dataset['SMILES'])):
+    #line = re.sub('\d+\t', '',line)
+    #print(line) 
+    newLine=tokenizer.encode(line).tokens#atomwise_tokenizer(line)
+    #print(newLine)
+    #print(int(i/10))
+    dataset.iloc[i]['SMILES']=newLine
 
 
-config = T5Config.from_pretrained("google/t5-v1_1-base", vocab_size=tokenizer.get_vocab_size())
-config.save_pretrained("./")
+#config = T5Config.from_pretrained("google/t5-v1_1-base", vocab_size=tokenizer.get_vocab_size())
+#config.save_pretrained("./")

train_scprit.sh

@@ -4,10 +4,10 @@ python run_t5_mlm_flax.py \
 	--model_type="t5" \
 	--config_name="./" \
 	--tokenizer_name="./" \
-	--train_file="chemT5_data.csv" \
+	--train_file="./chemT5_data.csv" \
 	--max_seq_length="256" \
-	--per_device_train_batch_size="8" \
-	--per_device_eval_batch_size="8" \
+	--per_device_train_batch_size="1" \
+	--per_device_eval_batch_size="1" \
 	--adafactor \
 	--learning_rate="0.005" \
 	--weight_decay="0.001" \
@@ -20,3 +20,4 @@ python run_t5_mlm_flax.py \
 	
 
 	
+~

try.py

@@ -13,30 +13,51 @@ import numpy as np
 
 
 
-#model = T5ForConditionalGeneration.from_pretrained(pretrained_model_name_or_path="/home/zoez/Chem-T5", from_flax=True)
-tokenizer = AutoTokenizer.from_pretrained("/home/zoez/chemT5")
-#tokenizer = Tokenizer.from_file("/home/zoez/Chem-T5/tokenizer.json")
+#model = T5ForConditionalGeneration.from_pretrained(pretrained_model_name_or_path="./", from_flax=True)
+tokenizer = AutoTokenizer.from_pretrained("./")
+#tokenizer = Tokenizer.from_file("/home/zoez/chemT5")
 #model = model.to(device)
 
-print(tokenizer.encode(atomwise_tokenizer("O=[N+]([O-])c1ccc(Cl)cc1O=[N+]([O-])c1ccc(Cl)cc1")).tokens)
+#print(tokenizer.encode("O=[N+]([O-])c1ccc(Cl)cc1").tokens)
 
 
-# # encode context the generation is conditioned on
-# input_ids1 = tokenizer.encode(": O[N+]([O-])c1ccc(Cl)cc1",return_tensors='pt')
+# # # encode context the generation is conditioned on
+# input_ids1 = tokenizer.encode("1",return_tensors='pt')
+# print(input_ids1)
 
-# # activate beam search and early_stopping
+# # # activate beam search and early_stopping
 # beam_output1 = model.generate(
 #     input_ids1, 
 #     max_length=50, 
 #     num_beams=5, 
 #     early_stopping=True
 # )
+encoding=tokenizer.encode("O=[N+]([O-])c1ccc(Cl)cc1")
+print(tokenizer.convert_ids_to_tokens(encoding))
 # #print(tokenizer.encode("O=[N+]([O-])c1ccc(Cl)cc1").tokens)
+
+# # set seed to reproduce results. Feel free to change the seed though to get different results
+# tf.random.set_seed(0)
+
+# # use temperature to decrease the sensitivity to low probability candidates
+# sample_output = model.generate(
+#     input_ids1, 
+#     do_sample=True, 
+#     max_length=50, 
+#     top_k=0, 
+#     temperature=0.7
+# )
+
+# print("Output:\n" + 100 * '-')
+# print(tokenizer.decode(sample_output[0], skip_special_tokens=True))
+
 # print("Output: 1\n" + 100 * '-')
 # print(tokenizer.decode(beam_output1[0], skip_special_tokens=True))
+# decoding=tokenizer.decode(beam_output1[0], skip_special_tokens=True)
+# print(tokenizer.convert_ids_to_tokens(decoding))
 
 # # encode context the generation is conditioned on
-# input_ids2 = tokenizer.encode("SMILES: ",return_tensors='pt')
+# input_ids2 = tokenizer.encode(": ",return_tensors='pt')
 
 # # activate beam search and early_stopping
 # beam_output2 = model.generate(
@@ -47,12 +68,12 @@ print(tokenizer.encode(atomwise_tokenizer("O=[N+]([O-])c1ccc(Cl)cc1O=[N+]([O-])c
 #     num_return_sequences=9, 
 #     early_stopping=True
 # )
-# #print(tokenizer.encode("O=[N+]([O-])c1ccc(Cl)cc1").tokens)
+# print(tokenizer.encode("O=[N+]([O-])c1ccc(Cl)cc1"))
 # print("Output: 2\n" + 100 * '-')
-# #print(tokenizer.decode(beam_output2[0], skip_special_tokens=True))
+# print(tokenizer.decode(beam_output2[0], skip_special_tokens=True))
 
-# #start = latent_to_string(latent0)
-# #destination = latent_to_string(latent1)
+# # #start = latent_to_string(latent0)
+# # #destination = latent_to_string(latent1)
 # mols1 = []
 # step = np.linspace(0,1,100)
 # invalid = 0