Initial commit: Adding Iraqi dialect LLM model and code

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,49 @@
+# Ignore Python cache files
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+
+# Ignore Jupyter Notebook checkpoints
+.ipynb_checkpoints
+
+# Ignore virtual environments and dependencies
+venv/
+env/
+iraqi_arabic_llm_env/
+
+# Ignore system-specific files
+.DS_Store
+Thumbs.db
+
+# Ignore model training artifacts
+*.bin
+*.safetensors
+*.h5
+*.ckpt
+*.pt
+
+# Ignore dataset files if you don't want to push them
+data/
+datasets/
+logs/
+
+# Ignore Hugging Face cache files
+huggingface/
+.cache/
+hf_cache/
+
+# Ignore any temporary or debug files
+*.log
+*.tmp
+*.swp
+*.bak
+
+# Ignore environment variables or configuration files containing secrets
+.env
+config.json
+
+# Ignore Gradio or Streamlit deployment files (if using locally for testing)
+app.db
+__pycache__/
+static/

app.py

@@ -0,0 +1,44 @@
+# app.py
+import os
+from flask import Flask, request, jsonify
+from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer
+
+app = Flask(__name__)
+
+# Set up the model path as a raw string to avoid Windows path issues
+model_path = r"C:\Users\moham\iraqi_arabic_llm_project\models\text_generation_model"
+
+# Print model_path to confirm correctness
+print("Model path:", model_path)
+
+# Check if the model path exists
+if not os.path.exists(model_path):
+    raise ValueError(f"Model path does not exist: {model_path}")
+
+# Load the model and tokenizer directly
+model = AutoModelForCausalLM.from_pretrained(model_path)
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+text_generator = pipeline("text-generation", model=model, tokenizer=tokenizer)
+
+# Endpoint for text generation
+@app.route('/generate_text', methods=['POST'])
+def generate_text():
+    input_text = request.json.get("text")
+    generated_text = text_generator(input_text, max_length=50, num_return_sequences=1)
+    return jsonify(generated_text)
+
+# Endpoint for sentiment analysis (if you have a sentiment model saved similarly)
+# Uncomment and set up sentiment analyzer as needed
+# from transformers import AutoModelForSequenceClassification
+# sentiment_model_path = r"C:\Users\moham\iraqi_arabic_llm_project\models\sentiment_analysis_model"
+# sentiment_model = AutoModelForSequenceClassification.from_pretrained(sentiment_model_path)
+# sentiment_analyzer = pipeline("sentiment-analysis", model=sentiment_model, tokenizer=tokenizer)
+
+@app.route('/analyze_sentiment', methods=['POST'])
+def analyze_sentiment():
+    input_text = request.json.get("text")
+    sentiment = sentiment_analyzer(input_text)  # Uncomment once you have sentiment model loaded
+    return jsonify(sentiment)
+
+if __name__ == '__main__':
+    app.run(debug=True)

models/checkpoint-4341/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.46.1"
+}

models/checkpoint-4341/trainer_state.json

@@ -0,0 +1,89 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 4341,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.3455425017277125,
+      "grad_norm": 2.7930502891540527,
+      "learning_rate": 4.424095830453813e-05,
+      "loss": 1.8711,
+      "step": 500
+    },
+    {
+      "epoch": 0.691085003455425,
+      "grad_norm": 3.2233235836029053,
+      "learning_rate": 3.848191660907625e-05,
+      "loss": 1.2259,
+      "step": 1000
+    },
+    {
+      "epoch": 1.0366275051831375,
+      "grad_norm": 2.0669665336608887,
+      "learning_rate": 3.272287491361438e-05,
+      "loss": 1.1263,
+      "step": 1500
+    },
+    {
+      "epoch": 1.38217000691085,
+      "grad_norm": 1.8985625505447388,
+      "learning_rate": 2.69638332181525e-05,
+      "loss": 1.0713,
+      "step": 2000
+    },
+    {
+      "epoch": 1.7277125086385625,
+      "grad_norm": 1.88094961643219,
+      "learning_rate": 2.1204791522690624e-05,
+      "loss": 1.0282,
+      "step": 2500
+    },
+    {
+      "epoch": 2.073255010366275,
+      "grad_norm": 2.1321468353271484,
+      "learning_rate": 1.5445749827228752e-05,
+      "loss": 1.0091,
+      "step": 3000
+    },
+    {
+      "epoch": 2.4187975120939877,
+      "grad_norm": 2.1656711101531982,
+      "learning_rate": 9.686708131766875e-06,
+      "loss": 0.9872,
+      "step": 3500
+    },
+    {
+      "epoch": 2.7643400138217,
+      "grad_norm": 1.986304521560669,
+      "learning_rate": 3.927666436304999e-06,
+      "loss": 0.9691,
+      "step": 4000
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 4341,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 10000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 567134355456000.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

models/iraqi_proverb_model/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[PAD]": 64000
+}

models/iraqi_proverb_model/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.46.1"
+}

models/iraqi_proverb_model/special_tokens_map.json

@@ -0,0 +1,12 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": "<|endoftext|>"
+}

models/iraqi_proverb_model/tokenizer_config.json

@@ -0,0 +1,52 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

process_proverbs.py

@@ -0,0 +1,48 @@
+import re
+
+def process_proverbs(input_path, output_path):
+    try:
+        # Attempt to open the input file and read lines
+        with open(input_path, 'r', encoding='utf-8') as file:
+            lines = file.readlines()
+        print("Successfully read the input file.")
+    except FileNotFoundError:
+        print(f"File not found: {input_path}")
+        return
+
+    proverbs = []
+    current_proverb = ""
+    
+    for line in lines:
+        # Clean the line by stripping leading/trailing whitespace
+        line = line.strip()
+        print(f"Processing line: {line}")  # Debug output for each line
+
+        # Identify proverbs or meanings based on specific patterns, e.g., numbers at the start
+        if re.match(r'^\d+', line):  # This line has a number, possibly a new entry
+            if current_proverb:
+                # Save the accumulated proverb/meaning pair
+                proverbs.append(current_proverb.strip())
+            current_proverb = line
+        else:
+            # Continue accumulating lines for the current entry
+            current_proverb += " " + line
+
+    # Save any remaining proverb
+    if current_proverb:
+        proverbs.append(current_proverb.strip())
+
+    # Write formatted proverbs to the output file
+    try:
+        with open(output_path, 'w', encoding='utf-8') as f:
+            for entry in proverbs:
+                f.write(entry + '\n')
+                print(f"Writing line to file: {entry}")  # Debug print for each line written
+        print(f"Data successfully saved to {output_path}")
+    except Exception as e:
+        print(f"Error writing to file: {e}")
+
+# Example usage
+input_path = "iraqi_proverbs.txt"  # Replace with the path to your text file
+output_path = "data/raw_data/iraqi_data.txt"  # Adjust this path if necessary
+process_proverbs(input_path, output_path)

requirements.txt

@@ -0,0 +1,8 @@
+torch
+transformers
+datasets
+flask
+beautifulsoup4
+requests
+pandas
+scikit-learn

scripts/extract_from_pdf.py

@@ -0,0 +1,41 @@
+# scripts/extract_from_pdf.py
+import pdfplumber
+import os
+
+def extract_text_from_pdf(pdf_path, output_path):
+    # Open the file for writing in append mode to ensure each page's content is saved immediately
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    with open(output_path, 'w', encoding='utf-8') as f:
+        with pdfplumber.open(pdf_path) as pdf:
+            for page_num, page in enumerate(pdf.pages, start=1):
+                # Define bounding boxes for each column
+                right_column = (page.width * 0.7, 0, page.width, page.height)
+
+                # Extract text from the right column
+                right_text = page.within_bbox(right_column).extract_text(x_tolerance=3, y_tolerance=3, layout=True)
+                
+                # Debug output to check extracted text
+                print(f"\n--- Page {page_num} ---")
+                print("Right column (proverbs):")
+                print(right_text if right_text else "No text found in right column")
+
+                # Write each line from the right column directly to the file
+                if right_text:
+                    right_lines = right_text.split('\n')
+                    for line in right_lines:
+                        # Check and print each line before writing to verify it has content
+                        line = line.strip()
+                        if line:
+                            print(f"Writing line to file: {line}")
+                            f.write(line + '\n')
+
+    print(f"Data successfully saved to {output_path}")
+
+if __name__ == "__main__":
+    # Path to your PDF file
+    pdf_path = r"C:\Users\moham\iraqi_arabic_llm_project\iraqi_proverbs.pdf"
+    
+    # Output path for the extracted proverbs
+    output_path = "../data/raw_data/iraqi_data.txt"
+    
+    extract_text_from_pdf(pdf_path, output_path)

scripts/extract_from_pdf_with_ocr.py

@@ -0,0 +1,52 @@
+# scripts/extract_from_pdf_with_ocr_right_column.py
+import pytesseract
+from pdf2image import convert_from_path
+import os
+import re
+from PIL import Image
+
+# Update the Tesseract OCR path if necessary
+pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
+
+# Regular expression to match lines with Arabic characters
+arabic_char_pattern = re.compile(r'[\u0600-\u06FF]+')
+
+def extract_text_from_pdf_with_ocr(pdf_path, output_path):
+    # Open the output file in write mode
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    with open(output_path, 'w', encoding='utf-8') as f:
+        # Convert PDF pages to images
+        pages = convert_from_path(pdf_path, dpi=300)
+
+        for page_num, page_image in enumerate(pages, start=1):
+            # Define the bounding box for the right column (proverbs)
+            width, height = page_image.size
+            right_column = page_image.crop((width * 0.7, 0, width, height))
+
+            # Perform OCR on the right column with custom configuration
+            custom_config = r'--psm 6'  # Treat the right column as a single block of text
+            right_text = pytesseract.image_to_string(right_column, lang="ara", config=custom_config)
+
+            # Debug output for the right column
+            print(f"\n--- Page {page_num} --- Right column (proverbs) ---")
+            print(right_text if right_text else "No text found in right column")
+
+            # Process and save only lines with Arabic text
+            if right_text:
+                right_lines = right_text.split('\n')
+                for line in right_lines:
+                    line = line.strip()
+                    if arabic_char_pattern.search(line):  # Ensure line contains Arabic text
+                        print(f"Writing line to file: {line}")  # Debug output for each line
+                        f.write(line + '\n')
+
+    print(f"Data successfully saved to {output_path}")
+
+if __name__ == "__main__":
+    # Path to your PDF file
+    pdf_path = r"C:\Users\moham\iraqi_arabic_llm_project\iraqi_proverbs.pdf"
+    
+    # Output path for the extracted proverbs
+    output_path = "../data/raw_data/iraqi_data.txt"
+    
+    extract_text_from_pdf_with_ocr(pdf_path, output_path)

scripts/extract_from_pdf_with_ocr_right_column.py

@@ -0,0 +1,52 @@
+# scripts/extract_from_pdf_with_ocr_right_column.py
+import pytesseract
+from pdf2image import convert_from_path
+import os
+import re
+from PIL import Image
+
+# Update the Tesseract OCR path if necessary
+pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
+
+# Regular expression to match lines with Arabic characters
+arabic_char_pattern = re.compile(r'[\u0600-\u06FF]+')
+
+def extract_text_from_pdf_with_ocr(pdf_path, output_path):
+    # Open the output file in write mode
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    with open(output_path, 'w', encoding='utf-8') as f:
+        # Convert PDF pages to images
+        pages = convert_from_path(pdf_path, dpi=300)
+
+        for page_num, page_image in enumerate(pages, start=1):
+            # Define the bounding box for the right column (proverbs)
+            width, height = page_image.size
+            right_column = page_image.crop((width * 0.7, 0, width, height))
+
+            # Perform OCR on the right column with custom configuration
+            custom_config = r'--psm 6'  # Treat the right column as a single block of text
+            right_text = pytesseract.image_to_string(right_column, lang="ara", config=custom_config)
+
+            # Debug output for the right column
+            print(f"\n--- Page {page_num} --- Right column (proverbs) ---")
+            print(right_text if right_text else "No text found in right column")
+
+            # Process and save only lines with Arabic text
+            if right_text:
+                right_lines = right_text.split('\n')
+                for line in right_lines:
+                    line = line.strip()
+                    if arabic_char_pattern.search(line):  # Ensure line contains Arabic text
+                        print(f"Writing line to file: {line}")  # Debug output for each line
+                        f.write(line + '\n')
+
+    print(f"Data successfully saved to {output_path}")
+
+if __name__ == "__main__":
+    # Path to your PDF file
+    pdf_path = r"C:\Users\moham\iraqi_arabic_llm_project\iraqi_proverbs.pdf"
+    
+    # Output path for the extracted proverbs
+    output_path = "../data/raw_data/iraqi_data.txt"
+    
+    extract_text_from_pdf_with_ocr(pdf_path, output_path)

scripts/generate_text.py

@@ -0,0 +1,28 @@
+import sys
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+def generate_text(prompt, model_name="models/iraqi_proverb_model", max_length=50):
+    # Load the trained model and tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(model_name)
+
+    # Encode the prompt and generate text
+    inputs = tokenizer.encode(prompt, return_tensors="pt")
+    outputs = model.generate(
+        inputs,
+        max_length=max_length,
+        num_return_sequences=1,
+        do_sample=True,
+        top_k=50,
+        pad_token_id=tokenizer.eos_token_id  # Set pad_token_id to eos_token_id to avoid warnings
+    )
+
+    # Decode the output and print the generated text
+    generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return generated_text
+
+# Main script
+if __name__ == "__main__":
+    # Get prompt from command-line arguments, default to an example prompt if none provided
+    prompt = sys.argv[1] if len(sys.argv) > 1 else "في العراق، يقولون"
+    print("Generated text:", generate_text(prompt))

scripts/preprocess.py

@@ -0,0 +1,16 @@
+# scripts/preprocess.py
+import re
+
+def clean_text(text):
+    text = re.sub(r'[^\u0621-\u064A\s]', '', text)  # Arabic letters only
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+if __name__ == "__main__":
+    with open('../data/raw_data/iraqi_data.txt', 'r') as f:
+        raw_data = f.readlines()
+    
+    cleaned_data = [clean_text(line) for line in raw_data]
+    with open('../data/cleaned_data/iraqi_data_cleaned.txt', 'w') as f:
+        for line in cleaned_data:
+            f.write(line + '\n')

scripts/preprocess_data.py

@@ -0,0 +1,19 @@
+import pandas as pd
+
+def preprocess_data(input_file, output_file):
+    data = []
+    with open(input_file, "r", encoding="utf-8") as f:
+        for line in f:
+            if " - " in line:
+                proverb, meaning = line.split(" - ", 1)
+                data.append({"proverb": proverb.strip(), "meaning": meaning.strip()})
+
+    # Save as a DataFrame
+    df = pd.DataFrame(data)
+    df.to_csv(output_file, index=False)
+    print(f"Data saved to {output_file}")
+
+# Example usage
+input_file = "data/raw_data/iraqi_data.txt"
+output_file = "data/processed_data.csv"
+preprocess_data(input_file, output_file)

scripts/scrape_data.py

@@ -0,0 +1,43 @@
+# scripts/scrape_data.py
+import requests
+from bs4 import BeautifulSoup
+import os
+
+def scrape_iraqi_arabic_text(url):
+    response = requests.get(url)
+    response.raise_for_status()  # Check if the request was successful
+
+    soup = BeautifulSoup(response.content, "html.parser")
+
+    # Update these to match the actual tag and class on the website
+    proverbs = soup.find_all("p", class_="proverb-text")  # Replace with the correct tag and class
+
+    iraqi_text = []
+    for proverb in proverbs:
+        text = proverb.get_text().strip()
+        if text:  # Only add non-empty lines
+            iraqi_text.append(text)
+
+    if not iraqi_text:
+        print("No proverbs found. Check the tag and class names in the code.")
+    return iraqi_text
+
+def save_text_to_file(text_list, relative_filepath):
+    project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    absolute_filepath = os.path.join(project_root, relative_filepath)
+    os.makedirs(os.path.dirname(absolute_filepath), exist_ok=True)
+
+    with open(absolute_filepath, 'w', encoding='utf-8') as f:
+        for line in text_list:
+            f.write(line + '\n')
+    print(f"Data saved to {absolute_filepath}")
+
+if __name__ == "__main__":
+    url = "https://ar.mo3jam.com/dialect/Iraqi"
+    iraqi_data = scrape_iraqi_arabic_text(url)
+    
+    # Check if any data was scraped
+    if iraqi_data:
+        save_text_to_file(iraqi_data, "data/raw_data/iraqi_data.txt")
+    else:
+        print("No data found. Please check the scraping criteria.")

scripts/train_model.py

@@ -0,0 +1,56 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, TextDataset, DataCollatorForLanguageModeling
+import torch
+
+# Load the tokenizer and model
+model_name = "aubmindlab/aragpt2-base"  # Arabic GPT-2 model
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
+
+# Add a padding token if not present
+if tokenizer.pad_token is None:
+    tokenizer.add_special_tokens({'pad_token': '[PAD]'})  # Adds a [PAD] token if it's not defined
+    model.resize_token_embeddings(len(tokenizer))  # Resize model's embeddings to account for the new token
+
+# Load and prepare the data for training
+def load_dataset(file_path, tokenizer, block_size=128):
+    # Read the text data from the file
+    with open(file_path, "r", encoding="utf-8") as f:
+        text = f.read()
+    
+    # Tokenize the text data
+    inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True, max_length=block_size)
+    
+    # Return as a PyTorch dataset
+    dataset = TextDataset(tokenizer=tokenizer, file_path=file_path, block_size=block_size)
+    return dataset
+
+# Define the path to your text data file
+data_file = "data/raw_data/iraqi_data.txt"  # Update this path if necessary
+train_dataset = load_dataset(data_file, tokenizer)
+
+# Set up training arguments
+training_args = TrainingArguments(
+    output_dir="./models",
+    overwrite_output_dir=True,
+    num_train_epochs=3,
+    per_device_train_batch_size=2,
+    save_steps=10_000,
+    save_total_limit=2,
+)
+
+# Define the data collator for language modeling
+data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+# Initialize the Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    data_collator=data_collator,
+)
+
+# Save the trained model and tokenizer
+trainer.save_model("models/iraqi_proverb_model")
+tokenizer.save_pretrained("models/iraqi_proverb_model")
+print("Training complete. Model and tokenizer saved to 'models/iraqi_proverb_model'.")
+

scripts/train_sentiment_analysis.py

@@ -0,0 +1,32 @@
+# scripts/train_sentiment_analysis.py
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
+from datasets import load_dataset
+
+# Load the tokenizer and model
+tokenizer = AutoTokenizer.from_pretrained("CAMeL-Lab/bert-base-camelbert-mix")
+model = AutoModelForSequenceClassification.from_pretrained("CAMeL-Lab/bert-base-camelbert-mix", num_labels=3)
+
+# Load and tokenize dataset
+dataset = load_dataset("csv", data_files="../data/sentiment_data.csv")
+
+def preprocess_function(examples):
+    return tokenizer(examples["text"], truncation=True)
+
+tokenized_dataset = dataset.map(preprocess_function, batched=True)
+
+# Training arguments
+training_args = TrainingArguments(
+    output_dir="../models/sentiment_analysis_model",
+    evaluation_strategy="epoch",
+    per_device_train_batch_size=4,
+    num_train_epochs=3
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_dataset["train"],
+    eval_dataset=tokenized_dataset["test"]
+)
+
+trainer.train()

scripts/train_text_generation.py

@@ -0,0 +1,36 @@
+# scripts/train_text_generation.py
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments
+from datasets import load_dataset
+
+# Replace with a public model available on Hugging Face
+tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv2")
+model = AutoModelForCausalLM.from_pretrained("aubmindlab/bert-base-arabertv2")
+
+# Load and tokenize dataset
+dataset = load_dataset("text", data_files="../data/cleaned_data/iraqi_data_cleaned.txt")
+
+def tokenize_function(examples):
+    return tokenizer(examples["text"], return_tensors="pt", truncation=True)
+
+tokenized_dataset = dataset.map(tokenize_function, batched=True)
+
+# Training arguments
+training_args = TrainingArguments(
+    output_dir="../models/text_generation_model",
+    evaluation_strategy="epoch",
+    per_device_train_batch_size=2,
+    num_train_epochs=3
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_dataset['train']
+)
+
+# Train the model
+trainer.train()
+
+# Save the model after training
+trainer.save_model("../models/text_generation_model")
+tokenizer.save_pretrained("../models/text_generation_model")