update model

.gitattributes copy

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,5 +1,5 @@
 .env
-models/__pycache__
+clustering/__pycache__
 utils/__pycache__
 dataset/*
 figure/*

app.py

@@ -1,39 +1,226 @@
 import gradio as gr
-from clustering.pure import run_kmeans, run_dbscan
-from clustering.pretrained import run_bert_clustering
-from utils.preprocess import load_csv, get_numeric, get_text_column
 import pandas as pd
+import numpy as np
+from clustering.pure import run_kmeans, run_dbscan, run_fuzzy_cmeans, run_som
+from clustering.pretrained import run_bert_clustering
+from utils.preprocess import (
+    load_csv,
+    get_numeric,
+    get_text_column,
+    normalize_data,
+    denormalize_data,
+)
+from utils.visualize import plot_embedding, plot_som
+from sklearn.metrics import silhouette_score
+import tempfile
+import matplotlib.pyplot as plt
+import os
+
+trained_model = {
+    "labels": None,
+    "X": None,
+    "model_type": None,
+    "algorithm": None,
+    "centroid": None,
+    "pretrained_model": None,
+    "scaler": None,
+    "model": None,
+    "normalizer": None,
+}
+
 
-def cluster(file, model_type, algorithm, n_clusters, eps, min_samples, pretrained_model):
+def cluster(file, model_type, algorithm, n_clusters, eps, min_samples, pretrained_model,
+            missing_strategy, dim_reduce_method, n_init, max_iter, normalize_method):
     df = load_csv(file.name)
-    
+
+    download_path = None
+    plot_path = None
+    embedding_path = None
+    som_plot_path = None
+
     if model_type == "Pure":
-        X = get_numeric(df)
+        X = get_numeric(df, strategy=missing_strategy)
+        trained_model["model_type"] = "Pure"
+        trained_model["algorithm"] = algorithm
+        trained_model["X"] = X
+
+        # Normalize before clustering
+        X_norm, norm_scaler = normalize_data(X, normalize_method)
+        trained_model["normalizer"] = norm_scaler
+
         if algorithm == "KMeans":
-            labels = run_kmeans(X, n_clusters)
+            model, labels, scaler, X_scaled = run_kmeans(
+                X_norm,
+                n_clusters,
+                init="k-means++",
+                n_init=n_init,
+                max_iter=max_iter,
+                random_state=42,
+                algorithm="lloyd",
+            )
+            trained_model["centroid"] = model.cluster_centers_
+
+        elif algorithm == "DBSCAN":
+            model, labels, scaler, X_scaled = run_dbscan(X_norm, eps, min_samples)
+            trained_model["centroid"] = None
+
+        elif algorithm == "Fuzzy C-Means":
+            model, labels, scaler, X_scaled = run_fuzzy_cmeans(X_norm, n_clusters)
+            trained_model["centroid"] = None
+
+        elif algorithm == "SOM":
+            model, labels, scaler, X_scaled = run_som(X_norm)
+            trained_model["centroid"] = None
+
+        else:
+            raise ValueError(f"Unknown algorithm: {algorithm}")
+
+        if algorithm == "DBSCAN":
+            labels = np.where(labels == -1, 0, labels + 1)
         else:
-            labels = run_dbscan(X, eps, min_samples)
+            labels = labels + 1
+
+        trained_model["model"] = model
+        trained_model["scaler"] = scaler
+        trained_model["labels"] = labels
+
+        df_export = df.copy()
+        df_export["cluster"] = labels
+        filename = f"cluster_{algorithm.replace(' ', '_').upper()}.csv"
+        download_path = os.path.join(tempfile.gettempdir(), filename)
+        df_export.to_csv(download_path, index=False)
+
+        if X_scaled.shape[1] >= 2:
+            X_plot = denormalize_data(pd.DataFrame(X_scaled, columns=X.columns), norm_scaler)
+            plt.figure()
+            plt.scatter(X_plot.iloc[:, 0], X_plot.iloc[:, 1], c=labels, cmap="tab10", s=30)
+            if algorithm == "KMeans" and trained_model["centroid"] is not None:
+                centroids = trained_model["centroid"]
+                centroids_denorm = denormalize_data(
+                    pd.DataFrame(centroids, columns=X.columns), norm_scaler
+                )
+                plt.scatter(
+                    centroids_denorm.iloc[:, 0],
+                    centroids_denorm.iloc[:, 1],
+                    c="red",
+                    marker="X",
+                    s=100,
+                    label="Centroids",
+                )
+                plt.legend()
+            plt.title(f"Cluster Scatter Plot (first 2 features) - {algorithm}")
+            plt.tight_layout()
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as img_tmp:
+                plt.savefig(img_tmp.name)
+                plot_path = img_tmp.name
+            plt.close()
+
+        if dim_reduce_method != "None" and algorithm != "SOM":
+            try:
+                embedding_path = plot_embedding(X_scaled, labels, method=dim_reduce_method)
+            except Exception as e:
+                print("UMAP/TSNE Failed:", e)
+
+        if algorithm == "SOM":
+            try:
+                som_plot_path = plot_som(model, X_scaled, labels)
+            except Exception as e:
+                print("SOM visualization failed:", e)
+
+        score = silhouette_score(X_scaled, labels) if len(set(labels)) > 1 else -1
+        df_out = pd.DataFrame({"Set": ["All Data"], "Silhouette Score": [score]})
+
+        return (
+            df_out,
+            download_path,
+            plot_path,
+            som_plot_path if algorithm == "SOM" else embedding_path,
+        )
+
     else:
+        trained_model["model_type"] = "Pretrained"
         texts = get_text_column(df)
+
+        if not texts or len(texts) == 0:
+            raise ValueError("No text data found for pretrained clustering. Check your CSV and preprocessing.")
+
         labels = run_bert_clustering(texts, n_clusters, pretrained_model)
-    
-    df["Cluster"] = labels
-    return df
-
-iface = gr.Interface(
-    fn=cluster,
-    inputs=[
-        gr.File(label="Upload CSV"),
-        gr.Radio(["Pure", "Pretrained"], label="Model Type"),
-        gr.Radio(["KMeans", "DBSCAN"], label="Algorithm"),
-        gr.Slider(2, 20, step=1, label="n_clusters"),
-        gr.Slider(0.1, 5.0, step=0.1, label="DBSCAN eps"),
-        gr.Slider(1, 20, step=1, label="DBSCAN min_samples"),
-        gr.Textbox(value="all-MiniLM-L6-v2", label="Pretrained Model Name")
-    ],
-    outputs=gr.Dataframe(label="Resulting Clusters"),
-    title="Unsupervised Clustering App"
-)
+
+        trained_model["labels"] = labels
+        trained_model["X"] = texts
+        trained_model["pretrained_model"] = pretrained_model
+
+        df_export = df.copy()
+        df_export["cluster"] = labels
+        filename = f"cluster_PRETRAINED.csv"
+        download_path = os.path.join(tempfile.gettempdir(), filename)
+        df_export.to_csv(download_path, index=False)
+
+        df_out = pd.DataFrame({"Set": ["All Data"], "Clusters": [len(set(labels))]})
+        return df_out, download_path, None, None
+
+
+with gr.Blocks() as iface:
+    gr.Markdown("## 🧠 Unsupervised Clustering App")
+
+    file = gr.File(label="Upload CSV")
+    model_type = gr.Radio(["Pure", "Pretrained"], label="Model Type", value="Pure")
+    algorithm = gr.Radio(
+        ["KMeans", "DBSCAN", "Fuzzy C-Means", "SOM"],
+        label="Algorithm",
+        value="KMeans",
+    )
+
+    n_clusters = gr.Slider(2, 20, step=1, label="Number of Clusters", visible=True)
+    n_init = gr.Slider(1, 50, step=1, label="Number of Initial Samples", value=30, visible=True)
+    max_iter = gr.Slider(100, 5000, step=100, label="Max Iteration", value=2000, visible=True)
+    eps = gr.Slider(0.1, 5.0, step=0.1, label="Epsilon", visible=False)
+    min_samples = gr.Slider(1, 20, step=1, label="Minimum Samples", visible=False)
+    pretrained_model = gr.Textbox(value="all-MiniLM-L6-v2", label="Pretrained Model Name", visible=False)
+    missing_strategy = gr.Dropdown(["Fill with Mean", "Fill with Zero", "Drop Rows"],
+                                   label="Missing Value Strategy", value="Drop Rows")
+    normalize_method = gr.Radio(["none", "mapminmax", "z-score"],
+                                label="Normalization Method", value="none")
+    dim_reduce_method = gr.Radio(["None", "UMAP", "TSNE"], label="Dimensionality Reduction", value="None", visible=False)
+
+    def update_fields(model_type_val, algorithm_val):
+        if model_type_val == "Pure":
+            return (
+                gr.update(visible=(algorithm_val in ["KMeans", "Fuzzy C-Means"])),  # n_clusters
+                gr.update(visible=(algorithm_val == "KMeans")),  # n_init
+                gr.update(visible=(algorithm_val == "KMeans")),  # max_iter
+                gr.update(visible=(algorithm_val == "DBSCAN")),  # eps
+                gr.update(visible=(algorithm_val == "DBSCAN")),  # min_samples
+                gr.update(visible=False),  # pretrained_model
+            )
+        else:
+            return (
+                gr.update(visible=True),  # n_clusters
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=True),
+            )
+
+    model_type.change(fn=update_fields, inputs=[model_type, algorithm],
+                      outputs=[n_clusters, n_init, max_iter, eps, min_samples, pretrained_model])
+    algorithm.change(fn=update_fields, inputs=[model_type, algorithm],
+                     outputs=[n_clusters, n_init, max_iter, eps, min_samples, pretrained_model])
+
+    btn = gr.Button("Run Clustering")
+    output = gr.Dataframe(label="Resulting Clusters")
+    download_csv = gr.File(label="Download Clustered CSV")
+    cluster_plot = gr.Image(label="2D Cluster Plot")
+    dim_plot = gr.Image(label="SOM Visualization")
+
+    btn.click(fn=cluster,
+              inputs=[
+                  file, model_type, algorithm, n_clusters, eps, min_samples,
+                  pretrained_model, missing_strategy, dim_reduce_method,
+                  n_init, max_iter, normalize_method
+              ],
+              outputs=[output, download_csv, cluster_plot, dim_plot])
 
 if __name__ == "__main__":
     iface.launch()

clustering/pretrained.py

@@ -4,6 +4,6 @@ from sentence_transformers import SentenceTransformer
 def run_bert_clustering(texts, n_clusters, model_name="all-MiniLM-L6-v2"):
     model = SentenceTransformer(model_name)
     embeddings = model.encode(texts, show_progress_bar=False)
-    km = KMeans(n_clusters=n_clusters, random_state=42)
+    km = KMeans(n_clusters=n_clusters, random_state=69)
     labels = km.fit_predict(embeddings)
     return labels

clustering/pure.py

@@ -1,16 +1,84 @@
 from sklearn.cluster import KMeans, DBSCAN
 from sklearn.preprocessing import StandardScaler
+import pandas as pd
+from minisom import MiniSom
+import numpy as np
 
-def run_kmeans(data, n_clusters):
+import numpy as np
+import skfuzzy as fuzz
+from sklearn.preprocessing import StandardScaler
+
+
+def run_kmeans(
+    data: pd.DataFrame,
+    n_clusters: int,
+    init="k-means++",
+    n_init=30,
+    max_iter=2000,
+    random_state=42,
+    algorithm="lloyd",
+):
     scaler = StandardScaler()
     X_scaled = scaler.fit_transform(data)
-    model = KMeans(n_clusters=n_clusters, random_state=42)
+
+    model = KMeans(
+        n_clusters=n_clusters,
+        init=init,
+        n_init=n_init,
+        max_iter=max_iter,
+        random_state=random_state,
+        algorithm=algorithm,
+    )
     labels = model.fit_predict(X_scaled)
-    return labels
 
-def run_dbscan(data, eps=0.5, min_samples=5):
+    return model, labels, scaler, X_scaled
+
+
+def run_dbscan(data: pd.DataFrame, eps: float = 0.5, min_samples: int = 5):
     scaler = StandardScaler()
     X_scaled = scaler.fit_transform(data)
+
     model = DBSCAN(eps=eps, min_samples=min_samples)
     labels = model.fit_predict(X_scaled)
-    return labels
+
+    return model, labels, scaler, X_scaled
+
+
+def run_fuzzy_cmeans(data, n_clusters):
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(data.T).T
+    X_T = X_scaled.T
+    cntr, u, u0, d, jm, p, fpc = fuzz.cluster.cmeans(
+        X_T, c=n_clusters, m=2, error=0.005, maxiter=3000, init=None, seed=42
+    )
+    labels = np.argmax(u, axis=0)
+
+    # labels เป็น 0-based
+    return cntr, labels, scaler, X_scaled
+
+
+def run_som(
+    data: pd.DataFrame,
+    x: int = 10,
+    y: int = 10,
+    sigma: float = 1.0,
+    learning_rate: float = 0.5,
+    num_iteration: int = 1000,
+):
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(data)
+
+    som = MiniSom(
+        x,
+        y,
+        X_scaled.shape[1],
+        sigma=sigma,
+        learning_rate=learning_rate,
+        random_seed=42,
+    )
+    som.random_weights_init(X_scaled)
+    som.train_random(X_scaled, num_iteration)
+    win_map = np.array([som.winner(xi) for xi in X_scaled])
+    labels = np.array([w[0] * y + w[1] for w in win_map])
+
+    return som, labels, scaler, X_scaled

requirements.txt

@@ -2,3 +2,7 @@ gradio
 pandas
 scikit-learn
 sentence-transformers
+umap-learn
+matplotlib
+minisom
+scikit-fuzzy

utils/preprocess.py

@@ -1,13 +1,40 @@
 import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
 
-def load_csv(file):
-    return pd.read_csv(file)
+def load_csv(file_path_or_obj):
+    return pd.read_csv(file_path_or_obj)
 
-def get_numeric(df):
-    return df.select_dtypes(include=['number'])
+def get_numeric(df: pd.DataFrame, strategy: str = "Fill with Mean") -> pd.DataFrame:
+    numeric_df = df.select_dtypes(include=['number'])
 
-def get_text_column(df):
-    for col in df.columns:
-        if df[col].dtype == 'object':
-            return df[col].dropna().astype(str).tolist()
+    if strategy == "Fill with Mean":
+        return numeric_df.fillna(numeric_df.mean(numeric_only=True))
+    elif strategy == "Fill with Zero":
+        return numeric_df.fillna(0)
+    elif strategy == "Drop Rows":
+        return numeric_df.dropna()
+    else:
+        return numeric_df
+
+def get_text_column(df: pd.DataFrame) -> list:
+    text_columns = df.select_dtypes(include=['object']).columns
+    if not text_columns.empty:
+        return df[text_columns[0]].dropna().astype(str).tolist()
     return []
+
+def normalize_data(data: pd.DataFrame, method: str):
+    if method == "z-score":
+        scaler = StandardScaler()
+    elif method == "mapminmax":
+        scaler = MinMaxScaler()
+    else:  # "none"
+        return data.copy(), None
+
+    scaled = scaler.fit_transform(data)
+    return pd.DataFrame(scaled, columns=data.columns), scaler
+
+def denormalize_data(scaled_data: pd.DataFrame, scaler):
+    if scaler is None:
+        return scaled_data
+    return pd.DataFrame(scaler.inverse_transform(scaled_data), columns=scaled_data.columns)

utils/visualize.py

@@ -0,0 +1,75 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import umap
+from sklearn.manifold import TSNE
+import tempfile
+
+def plot_embedding(X, labels, method="UMAP", title="Clustering Visualization") -> str:
+    if method.upper() == "NONE":
+        # ไม่ลดมิติ กูทำแค่ plot scatter ตามข้อมูลเดิม 2 มิติ
+        if X.shape[1] < 2:
+            raise ValueError("Data must have at least 2 features for plotting without dimensionality reduction.")
+        plt.figure(figsize=(8, 6))
+        scatter = plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='tab10', s=30)
+        plt.title(f"No Dimensionality Reduction - {title}")
+        plt.colorbar(scatter, label="Cluster ID")
+        plt.tight_layout()
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as tmp_img:
+            plt.savefig(tmp_img.name)
+            plt.close()
+            return tmp_img.name
+
+    elif method.upper() == "UMAP":
+        reducer = umap.UMAP(random_state=69)
+    elif method.upper() == "TSNE":
+        reducer = TSNE(random_state=69, perplexity=30, max_iter=1000)
+    else:
+        raise ValueError(f"Unknown method: {method}. Use 'UMAP', 'TSNE', or 'None'.")
+
+    X_embedded = reducer.fit_transform(X)
+
+    plt.figure(figsize=(8, 6))
+    scatter = plt.scatter(X_embedded[:, 0], X_embedded[:, 1], c=labels, cmap='tab10', s=30)
+    plt.title(f"{method.upper()} - {title}")
+    plt.colorbar(scatter, label="Cluster ID")
+    plt.tight_layout()
+
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as tmp_img:
+        plt.savefig(tmp_img.name)
+        plt.close()
+        return tmp_img.name
+
+def plot_som(som_model, X_scaled, labels):
+    """
+    Visualize SOM clustering result with U-Matrix + labeled points.
+    som_model: trained SOM object (เช่น MiniSom)
+    X_scaled: scaled data array
+    labels: cluster labels assigned for each point
+    """
+
+    plt.figure(figsize=(8, 8))
+
+    # วาด U-Matrix (distance map)
+    plt.pcolor(som_model.distance_map().T, cmap='bone_r')
+    plt.colorbar(label='Distance')
+
+    # วาดจุดข้อมูลบน SOM grid
+    markers = ['o', 's', 'D', '^', 'v', 'p', '*', 'h', 'x', '+']  # marker สำหรับ cluster สูงสุด 10 กลุ่ม
+    colors = plt.cm.tab10.colors
+
+    for cnt, x in enumerate(X_scaled):
+        w = som_model.winner(x)  # ตำแหน่ง node ที่ชนะ (winner neuron)
+        cluster_id = labels[cnt] - 1  # adjust label to zero-based index
+        plt.plot(w[0] + 0.5, w[1] + 0.5, markers[cluster_id % len(markers)],
+                 markerfacecolor=colors[cluster_id % len(colors)],
+                 markeredgecolor='k',
+                 markersize=12,
+                 markeredgewidth=1.5)
+
+    plt.title("SOM Clustering Visualization (U-Matrix + Clustered Data Points)")
+    plt.tight_layout()
+
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as tmp_img:
+        plt.savefig(tmp_img.name)
+        plt.close()
+        return tmp_img.name