aziz100 commited on
Commit
b7f5d93
·
verified ·
1 Parent(s): b827276

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +46 -0
app.py CHANGED
@@ -1,5 +1,51 @@
1
  import gradio as gr
2
  from huggingface_hub import InferenceClient
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
 
4
  """
5
  For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 
1
  import gradio as gr
2
  from huggingface_hub import InferenceClient
3
+ from datasets import load_dataset
4
+ from transformers import BartTokenizer, BartForSequenceClassification, Trainer, TrainingArguments
5
+
6
+ import pandas as pd
7
+ from datasets import load_dataset, DatasetDict
8
+
9
+ dataset = load_dataset("csv", data_files="/home/aziz/fine_tuning/FAQ_Appliance_Store_FR.csv")
10
+
11
+ split_dataset = dataset["train"].train_test_split(test_size=0.2)
12
+
13
+ dataset = DatasetDict({
14
+ "train": split_dataset["train"],
15
+ "test": split_dataset["test"]
16
+ })
17
+
18
+
19
+ # Load pretrained model and tokenizer
20
+ model = BartForSequenceClassification.from_pretrained("facebook/bart-large-mnli")
21
+ tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-mnli")
22
+
23
+ # Tokenize the dataset
24
+ def preprocess_function(examples):
25
+ return tokenizer(examples['question'], examples['answer'], truncation=True, padding="max_length")
26
+
27
+ tokenized_datasets = dataset.map(preprocess_function, batched=True)
28
+
29
+ # Define training arguments
30
+ training_args = TrainingArguments(
31
+ output_dir="./results",
32
+ evaluation_strategy="epoch",
33
+ save_strategy="epoch",
34
+ per_device_train_batch_size=8,
35
+ num_train_epochs=3,
36
+ )
37
+
38
+ trainer = Trainer(
39
+ model=model,
40
+ args=training_args,
41
+ train_dataset=tokenized_datasets["train"],
42
+ eval_dataset=tokenized_datasets["test"],
43
+ )
44
+
45
+ trainer.train()
46
+ model.save_pretrained("./my_model")
47
+ tokenizer.save_pretrained("./my_model")
48
+
49
 
50
  """
51
  For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference