File size: 1,901 Bytes
b6a6b92 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
train_files = ["train grupa 1.csv", "train grupa 2.csv", "train grupa 3.csv", "train grupa 4.csv"]
test_files = ["test grupa 1.csv", "test grupa 2.csv", "test grupa 3.csv", "test grupa 4.csv"]
VALID_LABELS = ["positive", "negative", "neutral", "mixed","sarcasm"]
label_map = {"negative": 0, "neutral": 1, "positive": 2,"mixed":3, "sarcasm":4}
def load_data(file):
df = pd.read_csv(file, sep=";")
df = df[["text", "label"]]
df = df.dropna()
# normalize label casing
df["label"] = df["label"].astype(str).str.lower()
df = df[df["label"].isin(VALID_LABELS)]
X = df["text"].astype(str).values
y = np.array([label_map[l] for l in df["label"]])
return X, y
def evaluate(y_true, y_pred):
return {
"accuracy": accuracy_score(y_true, y_pred),
"precision": precision_score(y_true, y_pred, average="weighted", zero_division=0),
"recall": recall_score(y_true, y_pred, average="weighted", zero_division=0),
"f1": f1_score(y_true, y_pred, average="weighted", zero_division=0),
}
vectorizer = TfidfVectorizer(stop_words="english", max_features=5000)
print("\n===== Logistic Regression: Individual Training =====")
for i in range(4):
X_train, y_train = load_data(train_files[i])
X_test, y_test = load_data(test_files[i])
X_train = vectorizer.fit_transform(X_train)
X_test = vectorizer.transform(X_test)
model = LogisticRegression(
max_iter=1000,
)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
print(f"\nDataset {i+1}")
print(evaluate(y_test, y_pred))
|