Rename model.py to training.py

model.py → training.py

@@ -1,11 +1,28 @@
-# loading model
+# loading model and Library
+
 from transformers import pipeline
 from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import DataCollatorWithPadding
 
 pipe = pipeline("SQL_Query_Generator", model="defog/sqlcoder-34b-alpha")
 
 tokenizer = AutoTokenizer.from_pretrained("defog/sqlcoder-34b-alpha")
 model = AutoModelForCausalLM.from_pretrained("defog/sqlcoder-34b-alpha")
+raw_dataset= load_datset('sql_train_dataset.json')
+
+#%% section 1 (preparing the dataset for fine tunning)
+
+def tokenize_func(df):
+    return tokenizer(df['question'],df['answer'],truncation=True)
+
+tokenize_dataset=raw_dataset.map(tokenize_func,batched=True)
 
-#%% section 1
+data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf")
 
+tf_train_dataset = tokenized_datasets["train"].to_tf_dataset(
+    columns=["attention_mask", "input_ids", "token_type_ids"],
+    label_cols=["labels"],
+    shuffle=True,
+    collate_fn=data_collator,
+    batch_size=8,
+)