diff --git a/FeatureSelect-Methods-Leaderboard/.gitattributes b/FeatureSelect-Methods-Leaderboard/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..dab9a4e17afd2ef39d90ccb0b40ef2786fe77422 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Dockerfile b/FeatureSelect-Methods-Leaderboard/AutoFS/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..0afef4f68592b71ef1378da974cacd3f886b2335 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.9 + +WORKDIR /code + +COPY ./requirements.txt /code/requirements.txt + +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt + +COPY . /code + +CMD ["python", "Webapp/app.py"] diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/README.md b/FeatureSelect-Methods-Leaderboard/AutoFS/README.md new file mode 100644 index 0000000000000000000000000000000000000000..42b5ff572a61d767eab5b733c4bc89a14b058c5d --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/README.md @@ -0,0 +1,33 @@ +--- +title: AutoFS Leaderboard +emoji: 📊 +colorFrom: blue +colorTo: green +sdk: docker +pinned: false +--- + +# AutoFS Leaderboard + +This is a visualization dashboard for AutoFS experiment results. + +## Deployment on Hugging Face Spaces + +1. Create a new Space on Hugging Face. +2. Select "Docker" as the SDK. +3. Upload all files from this repository. +4. The application will automatically build and run. + +## Local Development + +1. Install dependencies: + ```bash + pip install -r requirements.txt + ``` + +2. Run the application: + ```bash + python Webapp/app.py + ``` + +3. Open http://localhost:5000 in your browser. diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/__pycache__/app.cpython-39.pyc b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/__pycache__/app.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a148f5e38f15df5516a30b3ba0fde4bcc3f55751 Binary files /dev/null and b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/__pycache__/app.cpython-39.pyc differ diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app.py b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app.py new file mode 100644 index 0000000000000000000000000000000000000000..2f52f3c7e0086a1ae404d5d31f691ca0d0ee0c74 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app.py @@ -0,0 +1,174 @@ +import os +import sys +import pickle +import json +from flask import Flask, jsonify, request, render_template + +# Add project root to sys.path to import leaderboard +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +from leaderboard import rank_results +# =============================== +# 基本路径配置 +# =============================== +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +RESULT_DIR = os.path.join(PROJECT_ROOT, "results") +DATASET_DIR = os.path.join(PROJECT_ROOT, "datasets") + +os.makedirs(RESULT_DIR, exist_ok=True) +os.makedirs(DATASET_DIR, exist_ok=True) + +# =============================== +# Flask App +# =============================== +app = Flask(__name__) + +# =============================== +# 内存缓存 +# =============================== +RESULT_CACHE = {} # {dataset_name: results} + + +# =============================== +# PKL IO 工具 +# =============================== +def save_result_json(dataset, results): + path = os.path.join(RESULT_DIR, f"{dataset}.json") + with open(path, "w", encoding='utf-8') as f: + json.dump(results, f, indent=4) + + +def load_result_json(dataset): + path = os.path.join(RESULT_DIR, f"{dataset}.json") + if not os.path.exists(path): + return None + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def list_available_datasets(): + datasets = set() + + for f in os.listdir(RESULT_DIR): + if f.endswith(".json"): + datasets.add(f.replace(".json", "")) + + # 默认展示数据集 + datasets.add("Authorship") + + return sorted(datasets) + + +# =============================== +# ⚠️ 你自己的 Agent 入口 +# =============================== +def run_agent_for_dataset(dataset): + """ + ⚠️ 用你自己的 router / agent 替换这里 + 必须返回 List[Dict] + """ + # --------------------------- + # 示例(占位) + # --------------------------- + return [ + + ] +# { +# "algorithm": "shibaile", +# "num_features": 15, +# "mean_f1": 0.9233716475, +# "mean_auc": 0.9816098520, +# "time": 5.75, +# "score": 0.9408431088, +# }, +# { +# "algorithm": "JMchongxinzailai", +# "num_features": 15, +# "mean_f1": 0.918, +# "mean_auc": 0.979, +# "time": 7.32, +# "score": 0.932, +# }, + +# =============================== +# 页面 +# =============================== +@app.route("/") +def index(): + return render_template( + "index.html", + datasets=list_available_datasets(), + default_dataset="Authorship", + ) + + +# =============================== +# API:获取结果 +# =============================== +@app.route("/api/results") +def get_results(): + dataset = request.args.get("dataset", "Authorship") + + # ① 内存缓存 (Disabled for debugging) + # if dataset in RESULT_CACHE: + # print("------------------------------------------------------------------zoude cache") + # return jsonify(RESULT_CACHE[dataset]) + + # ② 磁盘 pkl/json + results = load_result_json(dataset) + print(111,results) + if results is not None: + print("------------------------------------------------------------------zoude json\n",results) + RESULT_CACHE[dataset] = results + leaderboard = rank_results(results) + return jsonify(leaderboard) + + # ③ Agent 实时运行 + results = run_agent_for_dataset(dataset) + print(222,results) + + # ④ 存储 + if results and len(results) > 0: + save_result_json(dataset, results) + RESULT_CACHE[dataset] = results + + print("------------------------------------------------------------------zoude agent") + leaderboard = rank_results(results) + # + return jsonify(leaderboard) + # print(333,leaderboard) +# @app.route("/api/results") +# def get_results(): + dataset = request.args.get("dataset", "Authorship") + + print(f"[DEBUG] request dataset = {dataset}") + + if dataset in RESULT_CACHE: + print("[DEBUG] hit memory cache") + return jsonify(RESULT_CACHE[dataset]) + + results = load_result_pkl(dataset) + if results is not None: + print("[DEBUG] hit pkl cache") + RESULT_CACHE[dataset] = results + return jsonify(results) + + print("[DEBUG] run agent") + results = run_agent_for_dataset(dataset) + print("[DEBUG] agent results =", results) + + save_result_pkl(dataset, results) + RESULT_CACHE[dataset] = results + return jsonify(results) + + +# =============================== +# API:数据集列表 +# =============================== +@app.route("/api/datasets") +def api_datasets(): + return jsonify(list_available_datasets()) + + +if __name__ == "__main__": + port = int(os.environ.get("PORT", 5000)) + app.run(host="0.0.0.0", port=port, debug=True) diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app1.py b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app1.py new file mode 100644 index 0000000000000000000000000000000000000000..196a1d9d1473b7b09006c0cc17992eda1ddcc95f --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app1.py @@ -0,0 +1,160 @@ +import os +import pickle +from flask import Flask, jsonify, request, render_template + +# =============================== +# 基本路径配置 +# =============================== +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +RESULT_DIR = os.path.join(PROJECT_ROOT, "results") +DATASET_DIR = os.path.join(PROJECT_ROOT, "datasets") + +os.makedirs(RESULT_DIR, exist_ok=True) +os.makedirs(DATASET_DIR, exist_ok=True) + +# =============================== +# Flask App +# =============================== +app = Flask(__name__) + +# =============================== +# 内存缓存 +# =============================== +RESULT_CACHE = {} # {dataset_name: results} + + +# =============================== +# PKL IO 工具 +# =============================== +def save_result_pkl(dataset, results): + path = os.path.join(RESULT_DIR, f"{dataset}.pkl") + with open(path, "wb") as f: + pickle.dump(results, f) + + +def load_result_pkl(dataset): + path = os.path.join(RESULT_DIR, f"{dataset}.pkl") + if not os.path.exists(path): + return None + with open(path, "rb") as f: + return pickle.load(f) + + +def list_available_datasets(): + datasets = set() + + for f in os.listdir(RESULT_DIR): + if f.endswith(".pkl"): + datasets.add(f.replace(".pkl", "")) + + # 默认展示数据集 + datasets.add("Authorship") + + return sorted(datasets) + + +# =============================== +# ⚠️ 你自己的 Agent 入口 +# =============================== +def run_agent_for_dataset(dataset): + """ + ⚠️ 用你自己的 router / agent 替换这里 + 必须返回 List[Dict] + """ + # --------------------------- + # 示例(占位) + # --------------------------- + return [ + { + "algorithm": "UCRFS", + "num_features": 15, + "mean_f1": 0.9233716475, + "mean_auc": 0.9816098520, + "time": 5.75, + "score": 0.9408431088, + }, + { + "algorithm": "JMIM", + "num_features": 15, + "mean_f1": 0.918, + "mean_auc": 0.979, + "time": 7.32, + "score": 0.932, + }, + ] + + +# =============================== +# 页面 +# =============================== +@app.route("/") +def index(): + return render_template( + "index.html", + datasets=list_available_datasets(), + default_dataset="Authorship", + ) + + +# =============================== +# API:获取结果 +# =============================== +@app.route("/api/results") +def get_results(): + dataset = request.args.get("dataset", "Authorship") + + # ① 内存缓存 + if dataset in RESULT_CACHE: + rank_results = jsonify(RESULT_CACHE[dataset]) + + # ② 磁盘 pkl + results = load_result_pkl(dataset) + if results is not None: + RESULT_CACHE[dataset] = results + rank_results = jsonify(results) + + # ③ Agent 实时运行 + results = run_agent_for_dataset(dataset) + + # ④ 存储 + save_result_pkl(dataset, results) + RESULT_CACHE[dataset] = results + + rank_results = jsonify(results) + leaderboard = rank_results(rank_results) + return leaderboard +# @app.route("/api/results") +# def get_results(): + dataset = request.args.get("dataset", "Authorship") + + print(f"[DEBUG] request dataset = {dataset}") + + if dataset in RESULT_CACHE: + print("[DEBUG] hit memory cache") + return jsonify(RESULT_CACHE[dataset]) + + results = load_result_pkl(dataset) + if results is not None: + print("[DEBUG] hit pkl cache") + RESULT_CACHE[dataset] = results + return jsonify(results) + + print("[DEBUG] run agent") + results = run_agent_for_dataset(dataset) + print("[DEBUG] agent results =", results) + + save_result_pkl(dataset, results) + RESULT_CACHE[dataset] = results + return jsonify(results) + + +# =============================== +# API:数据集列表 +# =============================== +@app.route("/api/datasets") +def api_datasets(): + return jsonify(list_available_datasets()) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app11.py b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app11.py new file mode 100644 index 0000000000000000000000000000000000000000..0f788467ac45d220359d30d98817b5c7ff2b899e --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/app11.py @@ -0,0 +1,27 @@ +import pickle +from flask import Flask, render_template +from leaderboard import rank_results + +app = Flask(__name__) + +# 🚀 这里直接接你 Agent 跑完的 results +def get_results(dataname): + # with open("/home/fangsensen/AutoFS/results/"+ dataname +".pkl", "rb") as f: + # results = pickle.load(f) + # print(1111111111,results) + results = [{'selected_features': [59, 50, 56, 4, 38, 9, 29, 23, 0, 20, 34, 36, 24, 26, 28], 'num_features': 15, 'metrics': {'nb': {'f1': 0.9181133571145461, 'auc': 0.9807805770573524}, 'svm': {'f1': 0.9282600079270711, 'auc': 0.980695564275392}, 'rf': {'f1': 0.9219976218787156, 'auc': 0.9768409098650539}}, 'time': 9.468129634857178, 'algorithm': 'JMIM'}, {'selected_features': [59, 50, 56, 4, 38, 0, 9, 29, 23, 20, 36, 34, 24, 28, 26], 'num_features': 15, 'metrics': {'nb': {'f1': 0.9163694015061433, 'auc': 0.9805189493459717}, 'svm': {'f1': 0.9265953230281413, 'auc': 0.98064247666047}, 'rf': {'f1': 0.9189853349187476, 'auc': 0.9769432925613145}}, 'time': 1.5439717769622803, 'algorithm': 'CFR'}, {'selected_features': [59, 64, 63, 22, 26, 11, 49, 7, 18, 24, 28, 12, 0, 8, 45], 'num_features': 15, 'metrics': {'nb': {'f1': 0.8498612762584224, 'auc': 0.9612941645198875}, 'svm': {'f1': 0.8672215616329766, 'auc': 0.9669919810144432}, 'rf': {'f1': 0.8516052318668254, 'auc': 0.9579325242146627}}, 'time': 3.4254932403564453, 'algorithm': 'DCSF'}, {'selected_features': [69, 59, 9, 4, 38, 24, 0, 49, 26, 18, 28, 11, 66, 12, 7], 'num_features': 15, 'metrics': {'nb': {'f1': 0.8747522790328972, 'auc': 0.968331958034509}, 'svm': {'f1': 0.8916369401506141, 'auc': 0.9765525653706246}, 'rf': {'f1': 0.9151010701545778, 'auc': 0.9804838761712887}}, 'time': 2.531461477279663, 'algorithm': 'IWFS'}, {'selected_features': [59, 50, 4, 38, 24, 0, 56, 26, 29, 49, 28, 23, 34, 36, 20], 'num_features': 15, 'metrics': {'nb': {'f1': 0.8806183115338884, 'auc': 0.973024320439098}, 'svm': {'f1': 0.9082837891399126, 'auc': 0.9784503098286724}, 'rf': {'f1': 0.897661514070551, 'auc': 0.9735557096666029}}, 'time': 2.793144941329956, 'algorithm': 'MRI'}, {'selected_features': [59, 69, 9, 5, 10, 31, 36, 20, 33, 47, 22, 29, 44, 56, 8], 'num_features': 15, 'metrics': {'nb': {'f1': 0.911375346809354, 'auc': 0.979648928949016}, 'svm': {'f1': 0.9064605628220372, 'auc': 0.9782951525850493}, 'rf': {'f1': 0.9252477209671027, 'auc': 0.9822236522028844}}, 'time': 2.9142298698425293, 'algorithm': 'MRMD'}, {'selected_features': [59, 69, 9, 56, 29, 50, 36, 4, 38, 0, 20, 24, 23, 28, 34], 'num_features': 15, 'metrics': {'nb': {'f1': 0.9177962742766549, 'auc': 0.9819010381640604}, 'svm': {'f1': 0.9178755449861277, 'auc': 0.980385760789456}, 'rf': {'f1': 0.9344431232659534, 'auc': 0.9825427569391104}}, 'time': 5.751329660415649, 'algorithm': 'UCRFS'}, {'selected_features': [[23, 15, 69, 43, 9, 52, 33, 8, 5, 3, 59, 47, 34, 55, 36], [50, 16, 31, 44, 47, 9, 69, 42, 33, 36, 63, 65, 23, 20, 22], [29, 13, 38, 3, 28, 59, 56, 69, 26, 20, 34, 50, 14, 49, 36], [59, 19, 20, 36, 24, 29, 9, 10, 23, 28, 22, 8, 56, 0, 60]], 'num_features': [15, 15, 15, 15], 'union_num_features': 4, 'metrics': {'nb': {'f1': 0.879587792310741, 'auc': 0.9680606961937624}, 'svm': {'f1': 0.8917162108600871, 'auc': 0.9710497573464302}, 'rf': {'f1': 0.8789536266349584, 'auc': 0.9655313327795009}}, 'time': 14.973412275314331, 'algorithm': 'CSMDCCMR'}] + leaderboard = rank_results(results) + # print(222222222222222,leaderboard) + return leaderboard + + +@app.route("/") +def index(): + dataname = 'Authorship' + results = get_results(dataname) + leaderboard = rank_results(results) + return render_template("index.html", leaderboard=leaderboard) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index.html b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index.html new file mode 100644 index 0000000000000000000000000000000000000000..614e29d7ea6c432394946ce1740443a48478d474 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index.html @@ -0,0 +1,603 @@ + + + + + + AutoFS Leaderboard + + + + + +
+
+

🏆 AutoFS Leaderboard

+
+ + +
+
+ + + +
+ + + + + + + + + +
+ +
+
+ +
+
+ +
+
+ + + + + + + + +
+
+ + + + + + + + diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index1.html b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index1.html new file mode 100644 index 0000000000000000000000000000000000000000..41e01d7b8ee8c1674d3b78819ba5ed399d2685ef --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index1.html @@ -0,0 +1,212 @@ + + + + + AutoFS Leaderboard + + + + + + + +

Feature Selection Leaderboard

+ + + + + + + + + + + + + +
RankAlgorithm #Features Mean F1 Mean AUC Time (s)
+ + +
+
+ +
+
+ +
+
+ + +
+
+ +
+
+ + + + + diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index11.html b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index11.html new file mode 100644 index 0000000000000000000000000000000000000000..f12091e72dbf58a422a104221c3a9717b6cdb141 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/index11.html @@ -0,0 +1,136 @@ + + + + + Feature Selection Leaderboard + + + + +

Feature Selection Leaderboard

+ + + + + + + + + + + + + + + + + + +
Rank ↑↓Algorithm ↑↓Num Features ↑↓Mean F1 ↑↓Mean AUC ↑↓Time ↑↓
+ + + + + diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/indexa.html b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/indexa.html new file mode 100644 index 0000000000000000000000000000000000000000..deffce1addf6c807def30618293446d717353cf2 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/Webapp/templates/indexa.html @@ -0,0 +1,434 @@ + + + + + FeatureSelect Leaderboard + + + + + + + + + + + + + + +
+
+ +

FeatureSelect + + Leaderboard +

+
+

An Automatic Evaluator for FeatureSelect Methods

+ + + + + + Length-controlled (LC) win rates alleviate length biases of GPT-4, but it may favor models finetuned on its outputs. + +
+ + GitHub logo + +
+ + + +
+ + Version: +
+ + + + +
+ + Filter: +
+ + + + + + +
+ + + +
+ +
+ + Baseline: GPT-4 Preview (11/06)   |   Auto-annotator: GPT-4 Preview (11/06) + +
+ + + + + + + + + + + + +
RankAlgorithm #Features Mean F1 Mean AUC Time (s)
+ + +
+
+ + Github + +
+
+

About AlpacaEval

+

+ AlpacaEval + an LLM-based automatic evaluation that is fast, cheap, and reliable. + It is based on the + AlpacaFarm + evaluation set, + which tests the ability of models to follow general user instructions. + These responses are then compared to reference responses (Davinci003 for AlpacaEval, GPT-4 Preview for AlpacaEval 2.0) by + the provided GPT-4 based auto-annotators, + which results in the win rates presented above. + AlpacaEval displays a high agreement rate with ground truth human annotations, + and leaderboard rankings on AlpacaEval are very correlated with leaderboard rankings + based on human annotators. + Please see our + documentation + for more details on our analysis. +

+

Adding new models

+

+ We welcome new model contributions to the leaderboard from the community! + To do so, please follow the steps in the + contributions + section. + Specifically, you'll need to run the model on the evaluation set, + auto-annotate the outputs, and submit a PR with the model config and leaderboard results. + We've also set up a + Discord + for community support and discussion. +

+

Adding new evaluators or eval sets

+

+ We also welcome contributions for new evaluators or new eval sets! + For making new evaluators, we release our ground-truth + human annotations + and comparison + metrics. + We also release a + rough guide + to follow for making new eval sets. + We specifically encourage contributions for harder instructions distributions and for safety testing of + LLMs. +

+

AlpacaEval limitations

+

+ 这里是简介 +

+
+ +
+ + + + + + + \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/__pycache__/leaderboard.cpython-37.pyc b/FeatureSelect-Methods-Leaderboard/AutoFS/__pycache__/leaderboard.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..154de261b411802dbb59ae9d481c68b8022197d0 Binary files /dev/null and b/FeatureSelect-Methods-Leaderboard/AutoFS/__pycache__/leaderboard.cpython-37.pyc differ diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/debug_data.py b/FeatureSelect-Methods-Leaderboard/AutoFS/debug_data.py new file mode 100644 index 0000000000000000000000000000000000000000..f105f3bc479595365887adc1ce3714205b1a6295 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/debug_data.py @@ -0,0 +1,36 @@ +import os +import json +import sys +import traceback + +# Mock paths +PROJECT_ROOT = os.path.abspath(os.getcwd()) +RESULT_DIR = os.path.join(PROJECT_ROOT, "results") +print(f"Result Dir: {RESULT_DIR}") + +dataset = "Authorship" +path = os.path.join(RESULT_DIR, f"{dataset}.json") +print(f"Path: {path}") +print(f"Exists: {os.path.exists(path)}") + +if os.path.exists(path): + try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + print(f"Data loaded, length: {len(data)}") + + # Try ranking + sys.path.append(PROJECT_ROOT) + try: + from leaderboard import rank_results + ranked = rank_results(data) + print(f"Ranked data length: {len(ranked)}") + if len(ranked) > 0: + print("First item:", ranked[0]) + except Exception as e: + print(f"Ranking failed: {e}") + traceback.print_exc() + except Exception as e: + print(f"Failed to read/parse json: {e}") +else: + print("File not found!") diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/leaderboard.py b/FeatureSelect-Methods-Leaderboard/AutoFS/leaderboard.py new file mode 100644 index 0000000000000000000000000000000000000000..f9825d6b5f764e52b81f0d0affd3a1107a18a4e1 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/leaderboard.py @@ -0,0 +1,124 @@ +import numpy as np +import pandas as pd +# def rank_results( +# results, +# metric="f1", +# clf_average="mean", +# weights=None +# ): +# """ +# 对 FSExecutor 输出结果进行排行榜排序 + +# Parameters +# ---------- +# results : list of dict +# 每个 dict 是一个算法在一个数据集上的结果 +# metric : str +# 使用的指标: 'f1' or 'auc' +# clf_average : str +# 'mean' 或 'max',表示跨分类器如何聚合 +# weights : dict or None +# 多指标加权,例如 {'f1':0.5, 'auc':0.5} + +# Returns +# ------- +# ranked_df : pd.DataFrame +# """ + +# rows = [] + +# for res in results: +# algo = res["algorithm"] +# metrics = res["metrics"] + +# # --------- 单指标 ---------- +# if weights is None: +# vals = [] +# for clf, m in metrics.items(): +# if metric in m: +# vals.append(m[metric]) + +# if not vals: +# raise ValueError(f"No metric {metric} for {algo}") + +# score = np.mean(vals) if clf_average == "mean" else np.max(vals) + +# # --------- 多指标加权 ---------- +# else: +# score = 0.0 +# for m_name, w in weights.items(): +# vals = [ +# metrics[clf][m_name] +# for clf in metrics +# if m_name in metrics[clf] +# ] +# score += w * np.mean(vals) + +# rows.append({ +# "algorithm": algo, +# "score": score, +# "num_features": res["num_features"], +# "time": res.get("time", None) +# }) + +# df = pd.DataFrame(rows) + +# # --------- 按 score 排序 ---------- +# df = df.sort_values( +# by="score", +# ascending=False +# ).reset_index(drop=True) + +# df["rank"] = df.index + 1 + +# return df + +def aggregate_metrics(metrics, w_f1=0.7, w_auc=0.3): + """ + metrics: + { + "nb": {"f1": x, "auc": y}, + "svm": {"f1": x, "auc": y}, + "rf": {"f1": x, "auc": y}, + } + """ + f1s = [m["f1"] for m in metrics.values()] + aucs = [m["auc"] for m in metrics.values()] + + mean_f1 = sum(f1s) / len(f1s) + mean_auc = sum(aucs) / len(aucs) + + return w_f1 * mean_f1 + w_auc * mean_auc,mean_f1,mean_auc + +def rank_results( + results, +): + """ + results: list[dict] + return: 排序后的 list[dict](每个 dict 会新增 score 字段) + """ + + ranked = [] + + for r in results: + # 1. 性能融合 + perf_score,mean_f1,mean_auc = aggregate_metrics(r["metrics"]) + + # 2. 惩罚项 + # feature_penalty = alpha * r["num_features"] + # time_penalty = beta * r["time"] + + # final_score = perf_score - feature_penalty - time_penalty + final_score = perf_score + ranked.append({ + **r, + "mean_f1":mean_f1, + "mean_auc":mean_auc, + "score": final_score, + "perf_score": perf_score + }) + + # 3. 排序(score 越大越好) + ranked.sort(key=lambda x: x["score"], reverse=True) + + return ranked diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/requirements.txt b/FeatureSelect-Methods-Leaderboard/AutoFS/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75c86c5629e86dacce75d317796702739c16b39f --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/requirements.txt @@ -0,0 +1,3 @@ +Flask +pandas +numpy diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/results/Authorship.json b/FeatureSelect-Methods-Leaderboard/AutoFS/results/Authorship.json new file mode 100644 index 0000000000000000000000000000000000000000..400159640036ea1f2ccb74fb31bcf430caef7b52 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/results/Authorship.json @@ -0,0 +1,349 @@ +[ + { + "selected_features": [ + 59, + 50, + 56, + 4, + 38, + 9, + 29, + 23, + 0, + 20, + 34, + 36, + 24, + 26, + 28 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9181133571145461, + "auc": 0.9807805770573524 + }, + "svm": { + "f1": 0.9282600079270711, + "auc": 0.980695564275392 + }, + "rf": { + "f1": 0.9219976218787156, + "auc": 0.9768411621948705 + } + }, + "time": 7.003696441650391, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 59, + 50, + 56, + 4, + 38, + 0, + 9, + 29, + 23, + 20, + 36, + 34, + 24, + 28, + 26 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9163694015061433, + "auc": 0.9805189493459717 + }, + "svm": { + "f1": 0.9265953230281413, + "auc": 0.98064247666047 + }, + "rf": { + "f1": 0.9189853349187476, + "auc": 0.97694404479886 + } + }, + "time": 2.083444595336914, + "algorithm": "CFR" + }, + { + "selected_features": [ + 59, + 64, + 63, + 22, + 26, + 11, + 49, + 7, + 18, + 24, + 28, + 12, + 0, + 8, + 45 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8498612762584224, + "auc": 0.9612941645198875 + }, + "svm": { + "f1": 0.8672215616329766, + "auc": 0.9669919810144432 + }, + "rf": { + "f1": 0.8516052318668254, + "auc": 0.9579321358773162 + } + }, + "time": 3.310762882232666, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 69, + 59, + 9, + 4, + 38, + 24, + 0, + 49, + 26, + 18, + 28, + 11, + 66, + 12, + 7 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8747522790328972, + "auc": 0.968331958034509 + }, + "svm": { + "f1": 0.8916369401506141, + "auc": 0.9765525653706246 + }, + "rf": { + "f1": 0.9151010701545778, + "auc": 0.9804839794856123 + } + }, + "time": 2.473106622695923, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 59, + 50, + 4, + 38, + 24, + 0, + 56, + 26, + 29, + 49, + 28, + 23, + 34, + 36, + 20 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8806183115338884, + "auc": 0.973024320439098 + }, + "svm": { + "f1": 0.9082837891399126, + "auc": 0.9784503098286724 + }, + "rf": { + "f1": 0.897661514070551, + "auc": 0.973555585899326 + } + }, + "time": 2.8017048835754395, + "algorithm": "MRI" + }, + { + "selected_features": [ + 59, + 69, + 9, + 5, + 10, + 31, + 36, + 20, + 33, + 47, + 22, + 29, + 44, + 56, + 8 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.911375346809354, + "auc": 0.979648928949016 + }, + "svm": { + "f1": 0.9064605628220372, + "auc": 0.9782951525850493 + }, + "rf": { + "f1": 0.9252477209671027, + "auc": 0.9822235518665571 + } + }, + "time": 1.9699223041534424, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 59, + 69, + 9, + 56, + 29, + 50, + 36, + 4, + 38, + 0, + 20, + 24, + 23, + 28, + 34 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9177962742766549, + "auc": 0.9819010381640604 + }, + "svm": { + "f1": 0.9178755449861277, + "auc": 0.980385760789456 + }, + "rf": { + "f1": 0.9344431232659534, + "auc": 0.9825426064346192 + } + }, + "time": 4.2307515144348145, + "algorithm": "UCRFS" + }, + { + "selected_features": [ + [ + 23, + 15, + 69, + 43, + 9, + 52, + 33, + 8, + 5, + 3, + 59, + 47, + 34, + 55, + 36 + ], + [ + 50, + 16, + 31, + 44, + 47, + 9, + 69, + 42, + 33, + 36, + 63, + 65, + 23, + 20, + 22 + ], + [ + 29, + 13, + 38, + 3, + 28, + 59, + 56, + 69, + 26, + 20, + 34, + 50, + 14, + 49, + 36 + ], + [ + 59, + 19, + 20, + 36, + 24, + 29, + 9, + 10, + 23, + 28, + 22, + 8, + 56, + 0, + 60 + ] + ], + "num_features": [ + 15, + 15, + 15, + 15 + ], + "union_num_features": 4, + "metrics": { + "nb": { + "f1": 0.879587792310741, + "auc": 0.9680606961937624 + }, + "svm": { + "f1": 0.8917162108600871, + "auc": 0.9710497573464302 + }, + "rf": { + "f1": 0.8789536266349584, + "auc": 0.9655310038725752 + } + }, + "time": 12.251755952835083, + "algorithm": "CSMDCCMR" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/results/Factors.json b/FeatureSelect-Methods-Leaderboard/AutoFS/results/Factors.json new file mode 100644 index 0000000000000000000000000000000000000000..e8f2214aaa4956e77dac26385b1a672b6f56d3d6 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/results/Factors.json @@ -0,0 +1,457 @@ +[ + { + "selected_features": [ + 180, + 63, + 110, + 194, + 193, + 184, + 197, + 25, + 147, + 36, + 96, + 170, + 181, + 0, + 37 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7585666666666667, + "auc": 0.964653722222222 + }, + "svm": { + "f1": 0.7981999999999999, + "auc": 0.9716779444444446 + }, + "rf": { + "f1": 0.7977666666666666, + "auc": 0.963972537037037 + } + }, + "time": 66.49715518951416, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 180, + 63, + 110, + 193, + 194, + 65, + 147, + 25, + 64, + 38, + 37, + 96, + 36, + 39, + 182 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7479, + "auc": 0.9626535185185184 + }, + "svm": { + "f1": 0.7959666666666666, + "auc": 0.9715684444444446 + }, + "rf": { + "f1": 0.7907666666666667, + "auc": 0.9631266018518517 + } + }, + "time": 16.008829355239868, + "algorithm": "CFR" + }, + { + "selected_features": [ + 180, + 213, + 130, + 79, + 174, + 188, + 179, + 16, + 67, + 116, + 210, + 190, + 55, + 141, + 4 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7210333333333333, + "auc": 0.9526347777777778 + }, + "svm": { + "f1": 0.5048333333333334, + "auc": 0.8947246296296296 + }, + "rf": { + "f1": 0.7523333333333333, + "auc": 0.9497905462962962 + } + }, + "time": 22.65188217163086, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 180, + 119, + 63, + 67, + 188, + 16, + 162, + 79, + 196, + 24, + 115, + 174, + 120, + 106, + 175 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7468333333333333, + "auc": 0.9592038333333333 + }, + "svm": { + "f1": 0.5647, + "auc": 0.9142652037037037 + }, + "rf": { + "f1": 0.7945666666666668, + "auc": 0.962168101851852 + } + }, + "time": 16.929046392440796, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 180, + 63, + 16, + 19, + 24, + 120, + 4, + 72, + 188, + 60, + 121, + 27, + 26, + 162, + 196 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6856333333333334, + "auc": 0.9451895555555555 + }, + "svm": { + "f1": 0.6991666666666667, + "auc": 0.948404962962963 + }, + "rf": { + "f1": 0.7519333333333333, + "auc": 0.9530803055555556 + } + }, + "time": 20.568500757217407, + "algorithm": "MRI" + }, + { + "selected_features": [ + 180, + 11, + 189, + 54, + 32, + 211, + 118, + 71, + 93, + 18, + 0, + 139, + 107, + 164, + 102 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7785666666666669, + "auc": 0.9682892037037037 + }, + "svm": { + "f1": 0.6084999999999999, + "auc": 0.931185111111111 + }, + "rf": { + "f1": 0.8092, + "auc": 0.9640595 + } + }, + "time": 14.148123025894165, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 180, + 11, + 189, + 54, + 32, + 96, + 64, + 36, + 98, + 63, + 29, + 0, + 156, + 14, + 207 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7952333333333332, + "auc": 0.9707861666666667 + }, + "svm": { + "f1": 0.6813666666666668, + "auc": 0.9460895925925925 + }, + "rf": { + "f1": 0.8157333333333334, + "auc": 0.9653315277777779 + } + }, + "time": 29.169645071029663, + "algorithm": "UCRFS" + }, + { + "selected_features": [ + [ + 211, + 177, + 198, + 92, + 178, + 215, + 7, + 111, + 118, + 93, + 139, + 107, + 4, + 67, + 102 + ], + [ + 85, + 214, + 151, + 131, + 104, + 90, + 141, + 111, + 102, + 139, + 178, + 128, + 47, + 199, + 42 + ], + [ + 36, + 59, + 46, + 33, + 212, + 179, + 54, + 144, + 127, + 140, + 30, + 71, + 116, + 199, + 132 + ], + [ + 110, + 214, + 189, + 163, + 90, + 176, + 191, + 65, + 151, + 102, + 208, + 116, + 30, + 69, + 58 + ], + [ + 185, + 177, + 200, + 139, + 95, + 30, + 178, + 153, + 120, + 212, + 72, + 42, + 107, + 19, + 105 + ], + [ + 86, + 214, + 119, + 6, + 92, + 100, + 93, + 31, + 102, + 4, + 54, + 68, + 187, + 208, + 35 + ], + [ + 108, + 213, + 32, + 179, + 139, + 54, + 105, + 204, + 10, + 116, + 23, + 26, + 201, + 188, + 113 + ], + [ + 5, + 59, + 70, + 176, + 66, + 45, + 7, + 196, + 93, + 118, + 98, + 131, + 152, + 67, + 84 + ], + [ + 0, + 214, + 35, + 199, + 111, + 105, + 66, + 4, + 116, + 79, + 107, + 29, + 34, + 48, + 9 + ], + [ + 96, + 59, + 33, + 198, + 80, + 199, + 100, + 102, + 34, + 132, + 119, + 8, + 111, + 93, + 174 + ] + ], + "num_features": [ + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15 + ], + "union_num_features": 10, + "metrics": { + "nb": { + "f1": 0.7188666666666665, + "auc": 0.9467155555555556 + }, + "svm": { + "f1": 0.5817666666666667, + "auc": 0.9037647222222223 + }, + "rf": { + "f1": 0.7609666666666667, + "auc": 0.9450812870370371 + } + }, + "time": 192.19890308380127, + "algorithm": "CSMDCCMR" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/results/dna.json b/FeatureSelect-Methods-Leaderboard/AutoFS/results/dna.json new file mode 100644 index 0000000000000000000000000000000000000000..f729c3fe94fc4b6d879a2434445e42f6ddc2594e --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/results/dna.json @@ -0,0 +1,331 @@ +[ + { + "selected_features": [ + 89, + 92, + 84, + 104, + 82, + 99, + 88, + 87, + 90, + 91, + 85, + 95, + 83, + 93, + 81 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7640929064657878, + "auc": 0.9133179854605366 + }, + "svm": { + "f1": 0.8536304666248171, + "auc": 0.9352907039838904 + }, + "rf": { + "f1": 0.8522494245658089, + "auc": 0.9412461781596505 + } + }, + "time": 61.692588090896606, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 82, + 99, + 93, + 88, + 87, + 90, + 95, + 94, + 85, + 83, + 86 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8071772337309061, + "auc": 0.9251172965675879 + }, + "svm": { + "f1": 0.8628164888051894, + "auc": 0.9403608959817996 + }, + "rf": { + "f1": 0.86185394433982, + "auc": 0.9447127585949784 + } + }, + "time": 14.362110137939453, + "algorithm": "CFR" + }, + { + "selected_features": [ + 89, + 104, + 92, + 84, + 81, + 99, + 93, + 83, + 95, + 94, + 82, + 97, + 90, + 87, + 88 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8425402803933877, + "auc": 0.9345225210498646 + }, + "svm": { + "f1": 0.881586105879891, + "auc": 0.9410499099603411 + }, + "rf": { + "f1": 0.8811048336472066, + "auc": 0.9480086728315744 + } + }, + "time": 23.570918560028076, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 99, + 93, + 95, + 94, + 83, + 81, + 97, + 74, + 72, + 71, + 62 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8436283741368488, + "auc": 0.9371476543435738 + }, + "svm": { + "f1": 0.8793262188742416, + "auc": 0.9464104127302048 + }, + "rf": { + "f1": 0.8789077212806028, + "auc": 0.9477423771202302 + } + }, + "time": 17.612692832946777, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 82, + 99, + 93, + 95, + 94, + 88, + 87, + 90, + 83, + 81, + 85 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8277045406988911, + "auc": 0.9322267536115253 + }, + "svm": { + "f1": 0.8711027411592386, + "auc": 0.9431894900660284 + }, + "rf": { + "f1": 0.8701820464532329, + "auc": 0.9464250615396989 + } + }, + "time": 18.142696142196655, + "algorithm": "MRI" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 82, + 99, + 93, + 88, + 95, + 87, + 90, + 94, + 83, + 85, + 86 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8157145846411384, + "auc": 0.9275568626302196 + }, + "svm": { + "f1": 0.8656204226825694, + "auc": 0.941107038573083 + }, + "rf": { + "f1": 0.8650554509311572, + "auc": 0.9451365316009367 + } + }, + "time": 14.290248394012451, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 99, + 82, + 93, + 95, + 94, + 97, + 83, + 81, + 90, + 87, + 88 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8353211969031178, + "auc": 0.9344102233712435 + }, + "svm": { + "f1": 0.8754760410127641, + "auc": 0.9449093560542415 + }, + "rf": { + "f1": 0.8743460975099393, + "auc": 0.9470115615925281 + } + }, + "time": 28.11224675178528, + "algorithm": "UCRFS" + }, + { + "selected_features": [ + [ + 92, + 104, + 93, + 95, + 94, + 99, + 90, + 91, + 97, + 89, + 103, + 102, + 96, + 88, + 98 + ], + [ + 84, + 81, + 89, + 83, + 104, + 82, + 85, + 86, + 87, + 88, + 74, + 72, + 66, + 71, + 57 + ], + [ + 89, + 84, + 92, + 104, + 82, + 99, + 87, + 93, + 88, + 83, + 85, + 91, + 86, + 74, + 94 + ] + ], + "num_features": [ + 15, + 15, + 15 + ], + "union_num_features": 3, + "metrics": { + "nb": { + "f1": 0.7881146683406571, + "auc": 0.9145317738835733 + }, + "svm": { + "f1": 0.8584641138313454, + "auc": 0.9310307930574658 + }, + "rf": { + "f1": 0.857187696170747, + "auc": 0.9399791620380172 + } + }, + "time": 96.02905464172363, + "algorithm": "CSMDCCMR" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/AutoFS/verify_backend.py b/FeatureSelect-Methods-Leaderboard/AutoFS/verify_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..b709a5aa0fa6bea050b92ee1155d28d687be9242 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/AutoFS/verify_backend.py @@ -0,0 +1,41 @@ +import json +import os +import sys + +# Mock paths +PROJECT_ROOT = os.path.abspath(os.getcwd()) +sys.path.append(PROJECT_ROOT) + +from leaderboard import rank_results + +def test_dataset(name): + path = os.path.join(PROJECT_ROOT, "results", f"{name}.json") + if not os.path.exists(path): + print(f"[ERROR] {name} not found") + return + + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + + print(f"--- Testing {name} ---") + try: + ranked = rank_results(data) + if len(ranked) > 0: + first = ranked[0] + print("Keys in first item:", first.keys()) + # Check for critical keys + for key in ['mean_f1', 'mean_auc', 'time']: + if key not in first: + print(f"[FAIL] Missing key: {key}") + elif first[key] is None: + print(f"[FAIL] Key is None: {key}") + else: + print(f"[OK] {key}: {first[key]} (type: {type(first[key])})") + else: + print("[WARN] Ranked list is empty") + except Exception as e: + print(f"[ERROR] Ranking failed: {e}") + +test_dataset("Authorship") +test_dataset("Factors") +test_dataset("dna") diff --git a/FeatureSelect-Methods-Leaderboard/Dockerfile b/FeatureSelect-Methods-Leaderboard/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..5c29eab169bbc67f62d9bc809bb5228889c3699d --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.9 + +WORKDIR /code + +COPY ./requirements.txt /code/requirements.txt + +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt + +COPY . /code + +# Fix permissions for Hugging Face Spaces (user 1000) +RUN chmod -R 777 /code + +EXPOSE 7860 + +CMD ["python", "Webapp/app.py"] diff --git a/FeatureSelect-Methods-Leaderboard/README.md b/FeatureSelect-Methods-Leaderboard/README.md new file mode 100644 index 0000000000000000000000000000000000000000..42b5ff572a61d767eab5b733c4bc89a14b058c5d --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/README.md @@ -0,0 +1,33 @@ +--- +title: AutoFS Leaderboard +emoji: 📊 +colorFrom: blue +colorTo: green +sdk: docker +pinned: false +--- + +# AutoFS Leaderboard + +This is a visualization dashboard for AutoFS experiment results. + +## Deployment on Hugging Face Spaces + +1. Create a new Space on Hugging Face. +2. Select "Docker" as the SDK. +3. Upload all files from this repository. +4. The application will automatically build and run. + +## Local Development + +1. Install dependencies: + ```bash + pip install -r requirements.txt + ``` + +2. Run the application: + ```bash + python Webapp/app.py + ``` + +3. Open http://localhost:5000 in your browser. diff --git a/FeatureSelect-Methods-Leaderboard/Webapp/__pycache__/app.cpython-39.pyc b/FeatureSelect-Methods-Leaderboard/Webapp/__pycache__/app.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a148f5e38f15df5516a30b3ba0fde4bcc3f55751 Binary files /dev/null and b/FeatureSelect-Methods-Leaderboard/Webapp/__pycache__/app.cpython-39.pyc differ diff --git a/FeatureSelect-Methods-Leaderboard/Webapp/app.py b/FeatureSelect-Methods-Leaderboard/Webapp/app.py new file mode 100644 index 0000000000000000000000000000000000000000..c0af9351ae88565e59d9485c7fe18ccf2f310369 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/Webapp/app.py @@ -0,0 +1,174 @@ +import os +import sys +import pickle +import json +from flask import Flask, jsonify, request, render_template + +# Add project root to sys.path to import leaderboard +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +from leaderboard import rank_results +# =============================== +# 基本路径配置 +# =============================== +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +RESULT_DIR = os.path.join(PROJECT_ROOT, "results") +DATASET_DIR = os.path.join(PROJECT_ROOT, "datasets") + +os.makedirs(RESULT_DIR, exist_ok=True) +os.makedirs(DATASET_DIR, exist_ok=True) + +# =============================== +# Flask App +# =============================== +app = Flask(__name__) + +# =============================== +# 内存缓存 +# =============================== +RESULT_CACHE = {} # {dataset_name: results} + + +# =============================== +# PKL IO 工具 +# =============================== +def save_result_json(dataset, results): + path = os.path.join(RESULT_DIR, f"{dataset}.json") + with open(path, "w", encoding='utf-8') as f: + json.dump(results, f, indent=4) + + +def load_result_json(dataset): + path = os.path.join(RESULT_DIR, f"{dataset}.json") + if not os.path.exists(path): + return None + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def list_available_datasets(): + datasets = set() + + for f in os.listdir(RESULT_DIR): + if f.endswith(".json"): + datasets.add(f.replace(".json", "")) + + # 默认展示数据集 + datasets.add("Authorship") + + return sorted(datasets) + + +# =============================== +# ⚠️ 你自己的 Agent 入口 +# =============================== +def run_agent_for_dataset(dataset): + """ + ⚠️ 用你自己的 router / agent 替换这里 + 必须返回 List[Dict] + """ + # --------------------------- + # 示例(占位) + # --------------------------- + return [ + + ] +# { +# "algorithm": "shibaile", +# "num_features": 15, +# "mean_f1": 0.9233716475, +# "mean_auc": 0.9816098520, +# "time": 5.75, +# "score": 0.9408431088, +# }, +# { +# "algorithm": "JMchongxinzailai", +# "num_features": 15, +# "mean_f1": 0.918, +# "mean_auc": 0.979, +# "time": 7.32, +# "score": 0.932, +# }, + +# =============================== +# 页面 +# =============================== +@app.route("/") +def index(): + return render_template( + "index.html", + datasets=list_available_datasets(), + default_dataset="Authorship", + ) + + +# =============================== +# API:获取结果 +# =============================== +@app.route("/api/results") +def get_results(): + dataset = request.args.get("dataset", "Authorship") + + # ① 内存缓存 (Disabled for debugging) + # if dataset in RESULT_CACHE: + # print("------------------------------------------------------------------zoude cache") + # return jsonify(RESULT_CACHE[dataset]) + + # ② 磁盘 pkl/json + results = load_result_json(dataset) + print(111,results) + if results is not None: + print("------------------------------------------------------------------zoude json\n",results) + RESULT_CACHE[dataset] = results + leaderboard = rank_results(results) + return jsonify(leaderboard) + + # ③ Agent 实时运行 + results = run_agent_for_dataset(dataset) + print(222,results) + + # ④ 存储 + if results and len(results) > 0: + save_result_json(dataset, results) + RESULT_CACHE[dataset] = results + + print("------------------------------------------------------------------zoude agent") + leaderboard = rank_results(results) + # + return jsonify(leaderboard) + # print(333,leaderboard) +# @app.route("/api/results") +# def get_results(): + dataset = request.args.get("dataset", "Authorship") + + print(f"[DEBUG] request dataset = {dataset}") + + if dataset in RESULT_CACHE: + print("[DEBUG] hit memory cache") + return jsonify(RESULT_CACHE[dataset]) + + results = load_result_pkl(dataset) + if results is not None: + print("[DEBUG] hit pkl cache") + RESULT_CACHE[dataset] = results + return jsonify(results) + + print("[DEBUG] run agent") + results = run_agent_for_dataset(dataset) + print("[DEBUG] agent results =", results) + + save_result_pkl(dataset, results) + RESULT_CACHE[dataset] = results + return jsonify(results) + + +# =============================== +# API:数据集列表 +# =============================== +@app.route("/api/datasets") +def api_datasets(): + return jsonify(list_available_datasets()) + + +if __name__ == "__main__": + port = int(os.environ.get("PORT", 7860)) + app.run(host="0.0.0.0", port=port, debug=False) diff --git a/FeatureSelect-Methods-Leaderboard/Webapp/app1.py b/FeatureSelect-Methods-Leaderboard/Webapp/app1.py new file mode 100644 index 0000000000000000000000000000000000000000..196a1d9d1473b7b09006c0cc17992eda1ddcc95f --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/Webapp/app1.py @@ -0,0 +1,160 @@ +import os +import pickle +from flask import Flask, jsonify, request, render_template + +# =============================== +# 基本路径配置 +# =============================== +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +RESULT_DIR = os.path.join(PROJECT_ROOT, "results") +DATASET_DIR = os.path.join(PROJECT_ROOT, "datasets") + +os.makedirs(RESULT_DIR, exist_ok=True) +os.makedirs(DATASET_DIR, exist_ok=True) + +# =============================== +# Flask App +# =============================== +app = Flask(__name__) + +# =============================== +# 内存缓存 +# =============================== +RESULT_CACHE = {} # {dataset_name: results} + + +# =============================== +# PKL IO 工具 +# =============================== +def save_result_pkl(dataset, results): + path = os.path.join(RESULT_DIR, f"{dataset}.pkl") + with open(path, "wb") as f: + pickle.dump(results, f) + + +def load_result_pkl(dataset): + path = os.path.join(RESULT_DIR, f"{dataset}.pkl") + if not os.path.exists(path): + return None + with open(path, "rb") as f: + return pickle.load(f) + + +def list_available_datasets(): + datasets = set() + + for f in os.listdir(RESULT_DIR): + if f.endswith(".pkl"): + datasets.add(f.replace(".pkl", "")) + + # 默认展示数据集 + datasets.add("Authorship") + + return sorted(datasets) + + +# =============================== +# ⚠️ 你自己的 Agent 入口 +# =============================== +def run_agent_for_dataset(dataset): + """ + ⚠️ 用你自己的 router / agent 替换这里 + 必须返回 List[Dict] + """ + # --------------------------- + # 示例(占位) + # --------------------------- + return [ + { + "algorithm": "UCRFS", + "num_features": 15, + "mean_f1": 0.9233716475, + "mean_auc": 0.9816098520, + "time": 5.75, + "score": 0.9408431088, + }, + { + "algorithm": "JMIM", + "num_features": 15, + "mean_f1": 0.918, + "mean_auc": 0.979, + "time": 7.32, + "score": 0.932, + }, + ] + + +# =============================== +# 页面 +# =============================== +@app.route("/") +def index(): + return render_template( + "index.html", + datasets=list_available_datasets(), + default_dataset="Authorship", + ) + + +# =============================== +# API:获取结果 +# =============================== +@app.route("/api/results") +def get_results(): + dataset = request.args.get("dataset", "Authorship") + + # ① 内存缓存 + if dataset in RESULT_CACHE: + rank_results = jsonify(RESULT_CACHE[dataset]) + + # ② 磁盘 pkl + results = load_result_pkl(dataset) + if results is not None: + RESULT_CACHE[dataset] = results + rank_results = jsonify(results) + + # ③ Agent 实时运行 + results = run_agent_for_dataset(dataset) + + # ④ 存储 + save_result_pkl(dataset, results) + RESULT_CACHE[dataset] = results + + rank_results = jsonify(results) + leaderboard = rank_results(rank_results) + return leaderboard +# @app.route("/api/results") +# def get_results(): + dataset = request.args.get("dataset", "Authorship") + + print(f"[DEBUG] request dataset = {dataset}") + + if dataset in RESULT_CACHE: + print("[DEBUG] hit memory cache") + return jsonify(RESULT_CACHE[dataset]) + + results = load_result_pkl(dataset) + if results is not None: + print("[DEBUG] hit pkl cache") + RESULT_CACHE[dataset] = results + return jsonify(results) + + print("[DEBUG] run agent") + results = run_agent_for_dataset(dataset) + print("[DEBUG] agent results =", results) + + save_result_pkl(dataset, results) + RESULT_CACHE[dataset] = results + return jsonify(results) + + +# =============================== +# API:数据集列表 +# =============================== +@app.route("/api/datasets") +def api_datasets(): + return jsonify(list_available_datasets()) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/FeatureSelect-Methods-Leaderboard/Webapp/app11.py b/FeatureSelect-Methods-Leaderboard/Webapp/app11.py new file mode 100644 index 0000000000000000000000000000000000000000..0f788467ac45d220359d30d98817b5c7ff2b899e --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/Webapp/app11.py @@ -0,0 +1,27 @@ +import pickle +from flask import Flask, render_template +from leaderboard import rank_results + +app = Flask(__name__) + +# 🚀 这里直接接你 Agent 跑完的 results +def get_results(dataname): + # with open("/home/fangsensen/AutoFS/results/"+ dataname +".pkl", "rb") as f: + # results = pickle.load(f) + # print(1111111111,results) + results = [{'selected_features': [59, 50, 56, 4, 38, 9, 29, 23, 0, 20, 34, 36, 24, 26, 28], 'num_features': 15, 'metrics': {'nb': {'f1': 0.9181133571145461, 'auc': 0.9807805770573524}, 'svm': {'f1': 0.9282600079270711, 'auc': 0.980695564275392}, 'rf': {'f1': 0.9219976218787156, 'auc': 0.9768409098650539}}, 'time': 9.468129634857178, 'algorithm': 'JMIM'}, {'selected_features': [59, 50, 56, 4, 38, 0, 9, 29, 23, 20, 36, 34, 24, 28, 26], 'num_features': 15, 'metrics': {'nb': {'f1': 0.9163694015061433, 'auc': 0.9805189493459717}, 'svm': {'f1': 0.9265953230281413, 'auc': 0.98064247666047}, 'rf': {'f1': 0.9189853349187476, 'auc': 0.9769432925613145}}, 'time': 1.5439717769622803, 'algorithm': 'CFR'}, {'selected_features': [59, 64, 63, 22, 26, 11, 49, 7, 18, 24, 28, 12, 0, 8, 45], 'num_features': 15, 'metrics': {'nb': {'f1': 0.8498612762584224, 'auc': 0.9612941645198875}, 'svm': {'f1': 0.8672215616329766, 'auc': 0.9669919810144432}, 'rf': {'f1': 0.8516052318668254, 'auc': 0.9579325242146627}}, 'time': 3.4254932403564453, 'algorithm': 'DCSF'}, {'selected_features': [69, 59, 9, 4, 38, 24, 0, 49, 26, 18, 28, 11, 66, 12, 7], 'num_features': 15, 'metrics': {'nb': {'f1': 0.8747522790328972, 'auc': 0.968331958034509}, 'svm': {'f1': 0.8916369401506141, 'auc': 0.9765525653706246}, 'rf': {'f1': 0.9151010701545778, 'auc': 0.9804838761712887}}, 'time': 2.531461477279663, 'algorithm': 'IWFS'}, {'selected_features': [59, 50, 4, 38, 24, 0, 56, 26, 29, 49, 28, 23, 34, 36, 20], 'num_features': 15, 'metrics': {'nb': {'f1': 0.8806183115338884, 'auc': 0.973024320439098}, 'svm': {'f1': 0.9082837891399126, 'auc': 0.9784503098286724}, 'rf': {'f1': 0.897661514070551, 'auc': 0.9735557096666029}}, 'time': 2.793144941329956, 'algorithm': 'MRI'}, {'selected_features': [59, 69, 9, 5, 10, 31, 36, 20, 33, 47, 22, 29, 44, 56, 8], 'num_features': 15, 'metrics': {'nb': {'f1': 0.911375346809354, 'auc': 0.979648928949016}, 'svm': {'f1': 0.9064605628220372, 'auc': 0.9782951525850493}, 'rf': {'f1': 0.9252477209671027, 'auc': 0.9822236522028844}}, 'time': 2.9142298698425293, 'algorithm': 'MRMD'}, {'selected_features': [59, 69, 9, 56, 29, 50, 36, 4, 38, 0, 20, 24, 23, 28, 34], 'num_features': 15, 'metrics': {'nb': {'f1': 0.9177962742766549, 'auc': 0.9819010381640604}, 'svm': {'f1': 0.9178755449861277, 'auc': 0.980385760789456}, 'rf': {'f1': 0.9344431232659534, 'auc': 0.9825427569391104}}, 'time': 5.751329660415649, 'algorithm': 'UCRFS'}, {'selected_features': [[23, 15, 69, 43, 9, 52, 33, 8, 5, 3, 59, 47, 34, 55, 36], [50, 16, 31, 44, 47, 9, 69, 42, 33, 36, 63, 65, 23, 20, 22], [29, 13, 38, 3, 28, 59, 56, 69, 26, 20, 34, 50, 14, 49, 36], [59, 19, 20, 36, 24, 29, 9, 10, 23, 28, 22, 8, 56, 0, 60]], 'num_features': [15, 15, 15, 15], 'union_num_features': 4, 'metrics': {'nb': {'f1': 0.879587792310741, 'auc': 0.9680606961937624}, 'svm': {'f1': 0.8917162108600871, 'auc': 0.9710497573464302}, 'rf': {'f1': 0.8789536266349584, 'auc': 0.9655313327795009}}, 'time': 14.973412275314331, 'algorithm': 'CSMDCCMR'}] + leaderboard = rank_results(results) + # print(222222222222222,leaderboard) + return leaderboard + + +@app.route("/") +def index(): + dataname = 'Authorship' + results = get_results(dataname) + leaderboard = rank_results(results) + return render_template("index.html", leaderboard=leaderboard) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/FeatureSelect-Methods-Leaderboard/Webapp/templates/index.html b/FeatureSelect-Methods-Leaderboard/Webapp/templates/index.html new file mode 100644 index 0000000000000000000000000000000000000000..614e29d7ea6c432394946ce1740443a48478d474 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/Webapp/templates/index.html @@ -0,0 +1,603 @@ + + + + + + AutoFS Leaderboard + + + + + +
+
+

🏆 AutoFS Leaderboard

+
+ + +
+
+ + + +
+ + + + + + + + + +
+ +
+
+ +
+
+ +
+
+ + + + + + + + +
+
+ + + + + + + + diff --git a/FeatureSelect-Methods-Leaderboard/Webapp/templates/index1.html b/FeatureSelect-Methods-Leaderboard/Webapp/templates/index1.html new file mode 100644 index 0000000000000000000000000000000000000000..41e01d7b8ee8c1674d3b78819ba5ed399d2685ef --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/Webapp/templates/index1.html @@ -0,0 +1,212 @@ + + + + + AutoFS Leaderboard + + + + + + + +

Feature Selection Leaderboard

+ + + + + + + + + + + + + +
RankAlgorithm #Features Mean F1 Mean AUC Time (s)
+ + +
+
+ +
+
+ +
+
+ + +
+
+ +
+
+ + + + + diff --git a/FeatureSelect-Methods-Leaderboard/Webapp/templates/index11.html b/FeatureSelect-Methods-Leaderboard/Webapp/templates/index11.html new file mode 100644 index 0000000000000000000000000000000000000000..f12091e72dbf58a422a104221c3a9717b6cdb141 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/Webapp/templates/index11.html @@ -0,0 +1,136 @@ + + + + + Feature Selection Leaderboard + + + + +

Feature Selection Leaderboard

+ + + + + + + + + + + + + + + + + + +
Rank ↑↓Algorithm ↑↓Num Features ↑↓Mean F1 ↑↓Mean AUC ↑↓Time ↑↓
+ + + + + diff --git a/FeatureSelect-Methods-Leaderboard/Webapp/templates/indexa.html b/FeatureSelect-Methods-Leaderboard/Webapp/templates/indexa.html new file mode 100644 index 0000000000000000000000000000000000000000..deffce1addf6c807def30618293446d717353cf2 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/Webapp/templates/indexa.html @@ -0,0 +1,434 @@ + + + + + FeatureSelect Leaderboard + + + + + + + + + + + + + + +
+
+ +

FeatureSelect + + Leaderboard +

+
+

An Automatic Evaluator for FeatureSelect Methods

+ + + + + + Length-controlled (LC) win rates alleviate length biases of GPT-4, but it may favor models finetuned on its outputs. + +
+ + GitHub logo + +
+ + + +
+ + Version: +
+ + + + +
+ + Filter: +
+ + + + + + +
+ + + +
+ +
+ + Baseline: GPT-4 Preview (11/06)   |   Auto-annotator: GPT-4 Preview (11/06) + +
+ + + + + + + + + + + + +
RankAlgorithm #Features Mean F1 Mean AUC Time (s)
+ + +
+
+ + Github + +
+
+

About AlpacaEval

+

+ AlpacaEval + an LLM-based automatic evaluation that is fast, cheap, and reliable. + It is based on the + AlpacaFarm + evaluation set, + which tests the ability of models to follow general user instructions. + These responses are then compared to reference responses (Davinci003 for AlpacaEval, GPT-4 Preview for AlpacaEval 2.0) by + the provided GPT-4 based auto-annotators, + which results in the win rates presented above. + AlpacaEval displays a high agreement rate with ground truth human annotations, + and leaderboard rankings on AlpacaEval are very correlated with leaderboard rankings + based on human annotators. + Please see our + documentation + for more details on our analysis. +

+

Adding new models

+

+ We welcome new model contributions to the leaderboard from the community! + To do so, please follow the steps in the + contributions + section. + Specifically, you'll need to run the model on the evaluation set, + auto-annotate the outputs, and submit a PR with the model config and leaderboard results. + We've also set up a + Discord + for community support and discussion. +

+

Adding new evaluators or eval sets

+

+ We also welcome contributions for new evaluators or new eval sets! + For making new evaluators, we release our ground-truth + human annotations + and comparison + metrics. + We also release a + rough guide + to follow for making new eval sets. + We specifically encourage contributions for harder instructions distributions and for safety testing of + LLMs. +

+

AlpacaEval limitations

+

+ 这里是简介 +

+
+ +
+ + + + + + + \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/complex_com.py b/FeatureSelect-Methods-Leaderboard/complex_com.py new file mode 100644 index 0000000000000000000000000000000000000000..84674fddc2d9d58211e5a4075a4e743d68ae93a3 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/complex_com.py @@ -0,0 +1,141 @@ +import math +from typing import Dict, Optional + + +class ComplexityScorer: + """ + Complexity scoring for feature selection algorithms + based on instantiated time and space complexity. + """ + + def __init__(self, alpha: float = 0.7, log_base: float = math.e): + """ + Parameters + ---------- + alpha : float + Weight for time complexity in the final score. + Typical values: 0.6 ~ 0.8 + log_base : float + Base of logarithm (default: natural log). + """ + self.alpha = alpha + self.log_base = log_base + + # ------------------------------ + # Core scoring functions + # ------------------------------ + + def _log(self, x: float) -> float: + """Logarithm with configurable base.""" + if self.log_base == math.e: + return math.log(x) + return math.log(x, self.log_base) + + def complexity_score(self, value: float, min_value: float) -> float: + """ + Generic complexity-to-score mapping. + + Parameters + ---------- + value : float + Instantiated complexity value of an algorithm. + min_value : float + Minimum complexity among all compared algorithms. + + Returns + ------- + score : float + Normalized score in (0, 1]. + """ + if value <= 0 or min_value <= 0: + raise ValueError("Complexity values must be positive.") + + ratio = value / min_value + return 1.0 / (1.0 + self._log(ratio)) + + def time_score(self, f_t: float, f_t_min: float) -> float: + """Time complexity score.""" + return self.complexity_score(f_t, f_t_min) + + def space_score(self, f_s: float, f_s_min: float) -> float: + """Space complexity score.""" + return self.complexity_score(f_s, f_s_min) + + def total_score( + self, + f_t: float, + f_t_min: float, + f_s: float, + f_s_min: float, + ) -> float: + """Combined complexity score.""" + s_t = self.time_score(f_t, f_t_min) + s_s = self.space_score(f_s, f_s_min) + return self.alpha * s_t + (1.0 - self.alpha) * s_s + + +# -------------------------------------- +# Utility: instantiate complexity formula +# -------------------------------------- + +def instantiate_complexity( + formula: str, + n: int, + d: int, + k: Optional[int] = None +) -> float: + """ + Instantiate asymptotic complexity formula. + + Supported variables: n, d, k + + Examples + -------- + "n * d**2" + "n * d * k" + "d**2" + "d + k" + """ + local_vars = {"n": n, "d": d} + if k is not None: + local_vars["k"] = k + + try: + return float(eval(formula, {"__builtins__": {}}, local_vars)) + except Exception as e: + raise ValueError(f"Invalid complexity formula: {formula}") from e + + +n, d, k = 1000, 50, 10 + +algorithms = { + "mRMR": {"time": "n * d**2", "space": "d**2"}, + "JMIM": {"time": "n * d * k", "space": "d * k"}, + "CFR": {"time": "n * d * k","space": "d + k"}, + "DCSF": {"time": "n * d * k", "space": "d + k"}, + "IWFS": {"time": "n * d * k", "space": "d + k"}, + "MRI": {"time": "n * d * k", "space": "d + k"}, + "MRMD": {"time": "n * d**2", "space": "d**2"}, + "UCRFS": {"time": "n * d + n**2", "space": "n**2"}, + + +} + +# Instantiate complexities +time_vals = [] +space_vals = {} + +for name, comp in algorithms.items(): + f_t = instantiate_complexity(comp["time"], n, d, k) + f_s = instantiate_complexity(comp["space"], n, d, k) + time_vals.append(f_t) + space_vals[name] = (f_t, f_s) + +f_t_min = min(time_vals) +f_s_min = min(v[1] for v in space_vals.values()) + +scorer = ComplexityScorer(alpha=0.7) + +for name, (f_t, f_s) in space_vals.items(): + score = scorer.total_score(f_t, f_t_min, f_s, f_s_min) + print(f"{name}: complexity score = {score:.4f}") diff --git a/FeatureSelect-Methods-Leaderboard/debug_data.py b/FeatureSelect-Methods-Leaderboard/debug_data.py new file mode 100644 index 0000000000000000000000000000000000000000..f105f3bc479595365887adc1ce3714205b1a6295 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/debug_data.py @@ -0,0 +1,36 @@ +import os +import json +import sys +import traceback + +# Mock paths +PROJECT_ROOT = os.path.abspath(os.getcwd()) +RESULT_DIR = os.path.join(PROJECT_ROOT, "results") +print(f"Result Dir: {RESULT_DIR}") + +dataset = "Authorship" +path = os.path.join(RESULT_DIR, f"{dataset}.json") +print(f"Path: {path}") +print(f"Exists: {os.path.exists(path)}") + +if os.path.exists(path): + try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + print(f"Data loaded, length: {len(data)}") + + # Try ranking + sys.path.append(PROJECT_ROOT) + try: + from leaderboard import rank_results + ranked = rank_results(data) + print(f"Ranked data length: {len(ranked)}") + if len(ranked) > 0: + print("First item:", ranked[0]) + except Exception as e: + print(f"Ranking failed: {e}") + traceback.print_exc() + except Exception as e: + print(f"Failed to read/parse json: {e}") +else: + print("File not found!") diff --git a/FeatureSelect-Methods-Leaderboard/leaderboard.py b/FeatureSelect-Methods-Leaderboard/leaderboard.py new file mode 100644 index 0000000000000000000000000000000000000000..f9825d6b5f764e52b81f0d0affd3a1107a18a4e1 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/leaderboard.py @@ -0,0 +1,124 @@ +import numpy as np +import pandas as pd +# def rank_results( +# results, +# metric="f1", +# clf_average="mean", +# weights=None +# ): +# """ +# 对 FSExecutor 输出结果进行排行榜排序 + +# Parameters +# ---------- +# results : list of dict +# 每个 dict 是一个算法在一个数据集上的结果 +# metric : str +# 使用的指标: 'f1' or 'auc' +# clf_average : str +# 'mean' 或 'max',表示跨分类器如何聚合 +# weights : dict or None +# 多指标加权,例如 {'f1':0.5, 'auc':0.5} + +# Returns +# ------- +# ranked_df : pd.DataFrame +# """ + +# rows = [] + +# for res in results: +# algo = res["algorithm"] +# metrics = res["metrics"] + +# # --------- 单指标 ---------- +# if weights is None: +# vals = [] +# for clf, m in metrics.items(): +# if metric in m: +# vals.append(m[metric]) + +# if not vals: +# raise ValueError(f"No metric {metric} for {algo}") + +# score = np.mean(vals) if clf_average == "mean" else np.max(vals) + +# # --------- 多指标加权 ---------- +# else: +# score = 0.0 +# for m_name, w in weights.items(): +# vals = [ +# metrics[clf][m_name] +# for clf in metrics +# if m_name in metrics[clf] +# ] +# score += w * np.mean(vals) + +# rows.append({ +# "algorithm": algo, +# "score": score, +# "num_features": res["num_features"], +# "time": res.get("time", None) +# }) + +# df = pd.DataFrame(rows) + +# # --------- 按 score 排序 ---------- +# df = df.sort_values( +# by="score", +# ascending=False +# ).reset_index(drop=True) + +# df["rank"] = df.index + 1 + +# return df + +def aggregate_metrics(metrics, w_f1=0.7, w_auc=0.3): + """ + metrics: + { + "nb": {"f1": x, "auc": y}, + "svm": {"f1": x, "auc": y}, + "rf": {"f1": x, "auc": y}, + } + """ + f1s = [m["f1"] for m in metrics.values()] + aucs = [m["auc"] for m in metrics.values()] + + mean_f1 = sum(f1s) / len(f1s) + mean_auc = sum(aucs) / len(aucs) + + return w_f1 * mean_f1 + w_auc * mean_auc,mean_f1,mean_auc + +def rank_results( + results, +): + """ + results: list[dict] + return: 排序后的 list[dict](每个 dict 会新增 score 字段) + """ + + ranked = [] + + for r in results: + # 1. 性能融合 + perf_score,mean_f1,mean_auc = aggregate_metrics(r["metrics"]) + + # 2. 惩罚项 + # feature_penalty = alpha * r["num_features"] + # time_penalty = beta * r["time"] + + # final_score = perf_score - feature_penalty - time_penalty + final_score = perf_score + ranked.append({ + **r, + "mean_f1":mean_f1, + "mean_auc":mean_auc, + "score": final_score, + "perf_score": perf_score + }) + + # 3. 排序(score 越大越好) + ranked.sort(key=lambda x: x["score"], reverse=True) + + return ranked diff --git a/FeatureSelect-Methods-Leaderboard/requirements.txt b/FeatureSelect-Methods-Leaderboard/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..75c86c5629e86dacce75d317796702739c16b39f --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/requirements.txt @@ -0,0 +1,3 @@ +Flask +pandas +numpy diff --git a/FeatureSelect-Methods-Leaderboard/results/Authorship.json b/FeatureSelect-Methods-Leaderboard/results/Authorship.json new file mode 100644 index 0000000000000000000000000000000000000000..400159640036ea1f2ccb74fb31bcf430caef7b52 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Authorship.json @@ -0,0 +1,349 @@ +[ + { + "selected_features": [ + 59, + 50, + 56, + 4, + 38, + 9, + 29, + 23, + 0, + 20, + 34, + 36, + 24, + 26, + 28 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9181133571145461, + "auc": 0.9807805770573524 + }, + "svm": { + "f1": 0.9282600079270711, + "auc": 0.980695564275392 + }, + "rf": { + "f1": 0.9219976218787156, + "auc": 0.9768411621948705 + } + }, + "time": 7.003696441650391, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 59, + 50, + 56, + 4, + 38, + 0, + 9, + 29, + 23, + 20, + 36, + 34, + 24, + 28, + 26 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9163694015061433, + "auc": 0.9805189493459717 + }, + "svm": { + "f1": 0.9265953230281413, + "auc": 0.98064247666047 + }, + "rf": { + "f1": 0.9189853349187476, + "auc": 0.97694404479886 + } + }, + "time": 2.083444595336914, + "algorithm": "CFR" + }, + { + "selected_features": [ + 59, + 64, + 63, + 22, + 26, + 11, + 49, + 7, + 18, + 24, + 28, + 12, + 0, + 8, + 45 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8498612762584224, + "auc": 0.9612941645198875 + }, + "svm": { + "f1": 0.8672215616329766, + "auc": 0.9669919810144432 + }, + "rf": { + "f1": 0.8516052318668254, + "auc": 0.9579321358773162 + } + }, + "time": 3.310762882232666, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 69, + 59, + 9, + 4, + 38, + 24, + 0, + 49, + 26, + 18, + 28, + 11, + 66, + 12, + 7 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8747522790328972, + "auc": 0.968331958034509 + }, + "svm": { + "f1": 0.8916369401506141, + "auc": 0.9765525653706246 + }, + "rf": { + "f1": 0.9151010701545778, + "auc": 0.9804839794856123 + } + }, + "time": 2.473106622695923, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 59, + 50, + 4, + 38, + 24, + 0, + 56, + 26, + 29, + 49, + 28, + 23, + 34, + 36, + 20 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8806183115338884, + "auc": 0.973024320439098 + }, + "svm": { + "f1": 0.9082837891399126, + "auc": 0.9784503098286724 + }, + "rf": { + "f1": 0.897661514070551, + "auc": 0.973555585899326 + } + }, + "time": 2.8017048835754395, + "algorithm": "MRI" + }, + { + "selected_features": [ + 59, + 69, + 9, + 5, + 10, + 31, + 36, + 20, + 33, + 47, + 22, + 29, + 44, + 56, + 8 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.911375346809354, + "auc": 0.979648928949016 + }, + "svm": { + "f1": 0.9064605628220372, + "auc": 0.9782951525850493 + }, + "rf": { + "f1": 0.9252477209671027, + "auc": 0.9822235518665571 + } + }, + "time": 1.9699223041534424, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 59, + 69, + 9, + 56, + 29, + 50, + 36, + 4, + 38, + 0, + 20, + 24, + 23, + 28, + 34 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9177962742766549, + "auc": 0.9819010381640604 + }, + "svm": { + "f1": 0.9178755449861277, + "auc": 0.980385760789456 + }, + "rf": { + "f1": 0.9344431232659534, + "auc": 0.9825426064346192 + } + }, + "time": 4.2307515144348145, + "algorithm": "UCRFS" + }, + { + "selected_features": [ + [ + 23, + 15, + 69, + 43, + 9, + 52, + 33, + 8, + 5, + 3, + 59, + 47, + 34, + 55, + 36 + ], + [ + 50, + 16, + 31, + 44, + 47, + 9, + 69, + 42, + 33, + 36, + 63, + 65, + 23, + 20, + 22 + ], + [ + 29, + 13, + 38, + 3, + 28, + 59, + 56, + 69, + 26, + 20, + 34, + 50, + 14, + 49, + 36 + ], + [ + 59, + 19, + 20, + 36, + 24, + 29, + 9, + 10, + 23, + 28, + 22, + 8, + 56, + 0, + 60 + ] + ], + "num_features": [ + 15, + 15, + 15, + 15 + ], + "union_num_features": 4, + "metrics": { + "nb": { + "f1": 0.879587792310741, + "auc": 0.9680606961937624 + }, + "svm": { + "f1": 0.8917162108600871, + "auc": 0.9710497573464302 + }, + "rf": { + "f1": 0.8789536266349584, + "auc": 0.9655310038725752 + } + }, + "time": 12.251755952835083, + "algorithm": "CSMDCCMR" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Colon.json b/FeatureSelect-Methods-Leaderboard/results/Colon.json new file mode 100644 index 0000000000000000000000000000000000000000..1a56c145ad16d6b5bc9e5445b503e7033961c5ee --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Colon.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5655913978494624, + "auc": 0.45946969696969703 + }, + "svm": { + "f1": 0.6182795698924732, + "auc": 0.3025757575757576 + }, + "rf": { + "f1": 0.5763440860215054, + "auc": 0.4905681818181818 + } + }, + "time": 30.261980533599854, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5655913978494624, + "auc": 0.45946969696969703 + }, + "svm": { + "f1": 0.6182795698924732, + "auc": 0.3025757575757576 + }, + "rf": { + "f1": 0.5763440860215054, + "auc": 0.4905681818181818 + } + }, + "time": 6.928465843200684, + "algorithm": "CFR" + }, + { + "selected_features": [ + 0, + 359, + 1052, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6268817204301076, + "auc": 0.6325757575757576 + }, + "svm": { + "f1": 0.6118279569892474, + "auc": 0.528030303030303 + }, + "rf": { + "f1": 0.6, + "auc": 0.5643560606060606 + } + }, + "time": 13.373862266540527, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 1052, + 0, + 1, + 359, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5989247311827958, + "auc": 0.6288636363636364 + }, + "svm": { + "f1": 0.6150537634408603, + "auc": 0.5270454545454546 + }, + "rf": { + "f1": 0.5795698924731184, + "auc": 0.5540530303030303 + } + }, + "time": 8.310370683670044, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 0, + 1, + 359, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5956989247311829, + "auc": 0.5593181818181818 + }, + "svm": { + "f1": 0.5860215053763441, + "auc": 0.5273106060606061 + }, + "rf": { + "f1": 0.5881720430107527, + "auc": 0.56 + } + }, + "time": 8.845045328140259, + "algorithm": "MRI" + }, + { + "selected_features": [ + 0, + 1052, + 1, + 2, + 3, + 4, + 359, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5978494623655913, + "auc": 0.5975 + }, + "svm": { + "f1": 0.6096774193548388, + "auc": 0.5052272727272727 + }, + "rf": { + "f1": 0.5827956989247312, + "auc": 0.5484469696969697 + } + }, + "time": 7.724148273468018, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 0, + 359, + 359, + 359, + 359, + 359, + 359, + 359, + 359, + 359, + 359, + 359, + 359, + 359, + 359 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6333333333333333, + "auc": 0.6225 + }, + "svm": { + "f1": 0.4806451612903224, + "auc": 0.32189393939393945 + }, + "rf": { + "f1": 0.6075268817204301, + "auc": 0.6012121212121211 + } + }, + "time": 14.179365396499634, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Factors.json b/FeatureSelect-Methods-Leaderboard/results/Factors.json new file mode 100644 index 0000000000000000000000000000000000000000..e8f2214aaa4956e77dac26385b1a672b6f56d3d6 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Factors.json @@ -0,0 +1,457 @@ +[ + { + "selected_features": [ + 180, + 63, + 110, + 194, + 193, + 184, + 197, + 25, + 147, + 36, + 96, + 170, + 181, + 0, + 37 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7585666666666667, + "auc": 0.964653722222222 + }, + "svm": { + "f1": 0.7981999999999999, + "auc": 0.9716779444444446 + }, + "rf": { + "f1": 0.7977666666666666, + "auc": 0.963972537037037 + } + }, + "time": 66.49715518951416, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 180, + 63, + 110, + 193, + 194, + 65, + 147, + 25, + 64, + 38, + 37, + 96, + 36, + 39, + 182 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7479, + "auc": 0.9626535185185184 + }, + "svm": { + "f1": 0.7959666666666666, + "auc": 0.9715684444444446 + }, + "rf": { + "f1": 0.7907666666666667, + "auc": 0.9631266018518517 + } + }, + "time": 16.008829355239868, + "algorithm": "CFR" + }, + { + "selected_features": [ + 180, + 213, + 130, + 79, + 174, + 188, + 179, + 16, + 67, + 116, + 210, + 190, + 55, + 141, + 4 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7210333333333333, + "auc": 0.9526347777777778 + }, + "svm": { + "f1": 0.5048333333333334, + "auc": 0.8947246296296296 + }, + "rf": { + "f1": 0.7523333333333333, + "auc": 0.9497905462962962 + } + }, + "time": 22.65188217163086, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 180, + 119, + 63, + 67, + 188, + 16, + 162, + 79, + 196, + 24, + 115, + 174, + 120, + 106, + 175 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7468333333333333, + "auc": 0.9592038333333333 + }, + "svm": { + "f1": 0.5647, + "auc": 0.9142652037037037 + }, + "rf": { + "f1": 0.7945666666666668, + "auc": 0.962168101851852 + } + }, + "time": 16.929046392440796, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 180, + 63, + 16, + 19, + 24, + 120, + 4, + 72, + 188, + 60, + 121, + 27, + 26, + 162, + 196 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6856333333333334, + "auc": 0.9451895555555555 + }, + "svm": { + "f1": 0.6991666666666667, + "auc": 0.948404962962963 + }, + "rf": { + "f1": 0.7519333333333333, + "auc": 0.9530803055555556 + } + }, + "time": 20.568500757217407, + "algorithm": "MRI" + }, + { + "selected_features": [ + 180, + 11, + 189, + 54, + 32, + 211, + 118, + 71, + 93, + 18, + 0, + 139, + 107, + 164, + 102 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7785666666666669, + "auc": 0.9682892037037037 + }, + "svm": { + "f1": 0.6084999999999999, + "auc": 0.931185111111111 + }, + "rf": { + "f1": 0.8092, + "auc": 0.9640595 + } + }, + "time": 14.148123025894165, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 180, + 11, + 189, + 54, + 32, + 96, + 64, + 36, + 98, + 63, + 29, + 0, + 156, + 14, + 207 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7952333333333332, + "auc": 0.9707861666666667 + }, + "svm": { + "f1": 0.6813666666666668, + "auc": 0.9460895925925925 + }, + "rf": { + "f1": 0.8157333333333334, + "auc": 0.9653315277777779 + } + }, + "time": 29.169645071029663, + "algorithm": "UCRFS" + }, + { + "selected_features": [ + [ + 211, + 177, + 198, + 92, + 178, + 215, + 7, + 111, + 118, + 93, + 139, + 107, + 4, + 67, + 102 + ], + [ + 85, + 214, + 151, + 131, + 104, + 90, + 141, + 111, + 102, + 139, + 178, + 128, + 47, + 199, + 42 + ], + [ + 36, + 59, + 46, + 33, + 212, + 179, + 54, + 144, + 127, + 140, + 30, + 71, + 116, + 199, + 132 + ], + [ + 110, + 214, + 189, + 163, + 90, + 176, + 191, + 65, + 151, + 102, + 208, + 116, + 30, + 69, + 58 + ], + [ + 185, + 177, + 200, + 139, + 95, + 30, + 178, + 153, + 120, + 212, + 72, + 42, + 107, + 19, + 105 + ], + [ + 86, + 214, + 119, + 6, + 92, + 100, + 93, + 31, + 102, + 4, + 54, + 68, + 187, + 208, + 35 + ], + [ + 108, + 213, + 32, + 179, + 139, + 54, + 105, + 204, + 10, + 116, + 23, + 26, + 201, + 188, + 113 + ], + [ + 5, + 59, + 70, + 176, + 66, + 45, + 7, + 196, + 93, + 118, + 98, + 131, + 152, + 67, + 84 + ], + [ + 0, + 214, + 35, + 199, + 111, + 105, + 66, + 4, + 116, + 79, + 107, + 29, + 34, + 48, + 9 + ], + [ + 96, + 59, + 33, + 198, + 80, + 199, + 100, + 102, + 34, + 132, + 119, + 8, + 111, + 93, + 174 + ] + ], + "num_features": [ + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15 + ], + "union_num_features": 10, + "metrics": { + "nb": { + "f1": 0.7188666666666665, + "auc": 0.9467155555555556 + }, + "svm": { + "f1": 0.5817666666666667, + "auc": 0.9037647222222223 + }, + "rf": { + "f1": 0.7609666666666667, + "auc": 0.9450812870370371 + } + }, + "time": 192.19890308380127, + "algorithm": "CSMDCCMR" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Isolet.json b/FeatureSelect-Methods-Leaderboard/results/Isolet.json new file mode 100644 index 0000000000000000000000000000000000000000..01d2531d9bc50c0776488cc4502cadc297373d2e --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Isolet.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 137, + 8, + 11, + 14, + 27, + 44, + 77, + 97, + 114, + 190, + 208, + 340, + 368, + 370, + 371 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.3708547008547008, + "auc": 0.8954675783475783 + }, + "svm": { + "f1": 0.41158119658119663, + "auc": 0.914048717948718 + }, + "rf": { + "f1": 0.3894017094017093, + "auc": 0.8827338034188036 + } + }, + "time": 203.20726919174194, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 137, + 0, + 35, + 304, + 541, + 14, + 202, + 209, + 336, + 563, + 4, + 13, + 41, + 92, + 43 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.38029914529914527, + "auc": 0.9058362108262108 + }, + "svm": { + "f1": 0.40782051282051285, + "auc": 0.9145582051282051 + }, + "rf": { + "f1": 0.3804700854700855, + "auc": 0.8804024643874644 + } + }, + "time": 45.961186170578, + "algorithm": "CFR" + }, + { + "selected_features": [ + 137, + 584, + 578, + 577, + 581, + 418, + 461, + 579, + 430, + 425, + 447, + 269, + 580, + 426, + 436 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.27162393162393167, + "auc": 0.8570815954415955 + }, + "svm": { + "f1": 0.3754273504273504, + "auc": 0.9026182051282051 + }, + "rf": { + "f1": 0.379059829059829, + "auc": 0.838709131054131 + } + }, + "time": 64.61027503013611, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 7, + 461, + 385, + 425, + 433, + 472, + 438, + 437, + 431, + 442, + 432, + 444, + 443, + 439, + 436 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.2845299145299146, + "auc": 0.8909103988603988 + }, + "svm": { + "f1": 0.3404273504273504, + "auc": 0.9177178347578347 + }, + "rf": { + "f1": 0.3977777777777778, + "auc": 0.8997356267806269 + } + }, + "time": 51.454981565475464, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 137, + 8, + 582, + 472, + 473, + 438, + 431, + 441, + 439, + 437, + 442, + 440, + 408, + 432, + 443 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.34867521367521365, + "auc": 0.8874809686609686 + }, + "svm": { + "f1": 0.36051282051282046, + "auc": 0.9044447863247863 + }, + "rf": { + "f1": 0.3655982905982905, + "auc": 0.8753007122507125 + } + }, + "time": 51.73934268951416, + "algorithm": "MRI" + }, + { + "selected_features": [ + 137, + 578, + 7, + 418, + 461, + 471, + 417, + 138, + 358, + 179, + 584, + 9, + 452, + 416, + 147 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.32572649572649576, + "auc": 0.8903427492877491 + }, + "svm": { + "f1": 0.44679487179487176, + "auc": 0.9263994871794874 + }, + "rf": { + "f1": 0.4795726495726496, + "auc": 0.8983328632478632 + } + }, + "time": 44.15886330604553, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 137, + 578, + 418, + 461, + 202, + 584, + 201, + 577, + 370, + 419, + 139, + 190, + 579, + 189, + 155 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.29585470085470084, + "auc": 0.8736537321937322 + }, + "svm": { + "f1": 0.38653846153846155, + "auc": 0.9141048717948717 + }, + "rf": { + "f1": 0.4051709401709402, + "auc": 0.8745863675213675 + } + }, + "time": 82.97627234458923, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Movement_libras.json b/FeatureSelect-Methods-Leaderboard/results/Movement_libras.json new file mode 100644 index 0000000000000000000000000000000000000000..74660173f6d1319e10599ecdafc15a05347b465c --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Movement_libras.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 88, + 7, + 21, + 4, + 34, + 49, + 64, + 67, + 70, + 31, + 54, + 26, + 81, + 18, + 50 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.47092592592592586, + "auc": 0.8778323412698413 + }, + "svm": { + "f1": 0.6162962962962963, + "auc": 0.9317675264550264 + }, + "rf": { + "f1": 0.69, + "auc": 0.9437158289241623 + } + }, + "time": 7.365044355392456, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 88, + 1, + 47, + 66, + 26, + 84, + 40, + 14, + 50, + 76, + 58, + 29, + 22, + 87, + 53 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.42925925925925934, + "auc": 0.8547172619047618 + }, + "svm": { + "f1": 0.5624074074074074, + "auc": 0.9296626984126984 + }, + "rf": { + "f1": 0.6751851851851852, + "auc": 0.9402772266313935 + } + }, + "time": 1.6786932945251465, + "algorithm": "CFR" + }, + { + "selected_features": [ + 88, + 47, + 43, + 61, + 35, + 53, + 41, + 67, + 33, + 45, + 51, + 37, + 55, + 69, + 49 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.3466666666666667, + "auc": 0.8160896164021165 + }, + "svm": { + "f1": 0.4440740740740742, + "auc": 0.8974200837742505 + }, + "rf": { + "f1": 0.5687037037037038, + "auc": 0.9113059413580248 + } + }, + "time": 2.884277820587158, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 88, + 2, + 47, + 43, + 61, + 35, + 63, + 53, + 41, + 45, + 67, + 33, + 37, + 51, + 65 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.38333333333333325, + "auc": 0.8307671957671959 + }, + "svm": { + "f1": 0.5483333333333335, + "auc": 0.9154117063492062 + }, + "rf": { + "f1": 0.6148148148148148, + "auc": 0.923960262345679 + } + }, + "time": 2.2524569034576416, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 88, + 7, + 47, + 43, + 61, + 53, + 35, + 63, + 41, + 67, + 45, + 33, + 37, + 51, + 65 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.3892592592592592, + "auc": 0.8504034391534391 + }, + "svm": { + "f1": 0.5098148148148148, + "auc": 0.9198627645502646 + }, + "rf": { + "f1": 0.6250000000000001, + "auc": 0.9307310956790124 + } + }, + "time": 2.1902174949645996, + "algorithm": "MRI" + }, + { + "selected_features": [ + 88, + 25, + 55, + 31, + 85, + 9, + 49, + 69, + 19, + 77, + 57, + 17, + 71, + 10, + 13 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.4546296296296296, + "auc": 0.8831509038800706 + }, + "svm": { + "f1": 0.5607407407407408, + "auc": 0.9241936728395062 + }, + "rf": { + "f1": 0.6553703703703704, + "auc": 0.9352295524691357 + } + }, + "time": 1.8538167476654053, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 88, + 31, + 55, + 3, + 12, + 79, + 73, + 25, + 47, + 58, + 19, + 69, + 52, + 40, + 78 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.4848148148148148, + "auc": 0.8921395502645502 + }, + "svm": { + "f1": 0.6138888888888888, + "auc": 0.932921626984127 + }, + "rf": { + "f1": 0.6985185185185185, + "auc": 0.9435047398589067 + } + }, + "time": 3.534160852432251, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Musk1.json b/FeatureSelect-Methods-Leaderboard/results/Musk1.json new file mode 100644 index 0000000000000000000000000000000000000000..7b114f488f73ef75e87d201df132dedb2914342b --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Musk1.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 107, + 5, + 157, + 77, + 52, + 33, + 95, + 53, + 87, + 9, + 7, + 105, + 133, + 20, + 126 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5802521008403361, + "auc": 0.6220743614149142 + }, + "svm": { + "f1": 0.7096638655462184, + "auc": 0.7637501571395221 + }, + "rf": { + "f1": 0.777310924369748, + "auc": 0.8512089267220995 + } + }, + "time": 14.140826940536499, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 107, + 5, + 157, + 93, + 87, + 105, + 95, + 9, + 133, + 53, + 80, + 83, + 33, + 19, + 126 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5938375350140055, + "auc": 0.6475842417494267 + }, + "svm": { + "f1": 0.6942577030812324, + "auc": 0.7576441642871253 + }, + "rf": { + "f1": 0.7778711484593839, + "auc": 0.8511981514405953 + } + }, + "time": 3.776477813720703, + "algorithm": "CFR" + }, + { + "selected_features": [ + 107, + 144, + 66, + 4, + 30, + 145, + 146, + 75, + 163, + 36, + 156, + 98, + 0, + 15, + 21 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5836134453781512, + "auc": 0.6617031529670936 + }, + "svm": { + "f1": 0.6200280112044818, + "auc": 0.6323048925764296 + }, + "rf": { + "f1": 0.7459383753501402, + "auc": 0.8110476566755861 + } + }, + "time": 5.198076963424683, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 107, + 54, + 133, + 145, + 156, + 15, + 0, + 14, + 51, + 163, + 146, + 52, + 75, + 43, + 44 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5505602240896358, + "auc": 0.6076456608540011 + }, + "svm": { + "f1": 0.646358543417367, + "auc": 0.6842285796383097 + }, + "rf": { + "f1": 0.7876750700280113, + "auc": 0.8650958700740501 + } + }, + "time": 4.191560983657837, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 107, + 5, + 15, + 52, + 0, + 12, + 14, + 44, + 27, + 156, + 17, + 77, + 106, + 163, + 85 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5837535014005601, + "auc": 0.6581709558273321 + }, + "svm": { + "f1": 0.6777310924369748, + "auc": 0.7323509868361976 + }, + "rf": { + "f1": 0.7659663865546219, + "auc": 0.844170872019587 + } + }, + "time": 4.924739360809326, + "algorithm": "MRI" + }, + { + "selected_features": [ + 107, + 144, + 4, + 66, + 30, + 145, + 146, + 75, + 36, + 152, + 156, + 98, + 163, + 21, + 90 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5864145658263304, + "auc": 0.6611494232231261 + }, + "svm": { + "f1": 0.6242296918767508, + "auc": 0.6377086962508005 + }, + "rf": { + "f1": 0.7400560224089635, + "auc": 0.8113520583780807 + } + }, + "time": 3.45463490486145, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 107, + 144, + 66, + 4, + 30, + 145, + 157, + 94, + 42, + 93, + 133, + 14, + 19, + 95, + 87 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5749299719887955, + "auc": 0.6599485779621548 + }, + "svm": { + "f1": 0.6365546218487396, + "auc": 0.6478560182940335 + }, + "rf": { + "f1": 0.7829131652661065, + "auc": 0.8436542571341342 + } + }, + "time": 6.485081195831299, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Pixel.json b/FeatureSelect-Methods-Leaderboard/results/Pixel.json new file mode 100644 index 0000000000000000000000000000000000000000..b1c53c870f97e1a04b718218fe5eb6cd8f5c59fd --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Pixel.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 152, + 56, + 214, + 108, + 82, + 127, + 182, + 137, + 85, + 153, + 62, + 178, + 167, + 123, + 95 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6568, + "auc": 0.9283085740740741 + }, + "svm": { + "f1": 0.7232333333333333, + "auc": 0.9434225925925924 + }, + "rf": { + "f1": 0.7277333333333333, + "auc": 0.9370961944444446 + } + }, + "time": 59.567777156829834, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 152, + 56, + 214, + 137, + 57, + 167, + 95, + 71, + 122, + 168, + 67, + 153, + 72, + 213, + 138 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5832, + "auc": 0.9104329629629628 + }, + "svm": { + "f1": 0.6564000000000001, + "auc": 0.9294791296296299 + }, + "rf": { + "f1": 0.6597666666666666, + "auc": 0.9203344722222222 + } + }, + "time": 12.102723360061646, + "algorithm": "CFR" + }, + { + "selected_features": [ + 152, + 57, + 214, + 156, + 95, + 68, + 113, + 122, + 76, + 71, + 168, + 3, + 142, + 198, + 82 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6910000000000001, + "auc": 0.9329236851851853 + }, + "svm": { + "f1": 0.7335333333333334, + "auc": 0.9446338703703704 + }, + "rf": { + "f1": 0.7320333333333333, + "auc": 0.9390299722222221 + } + }, + "time": 19.944496870040894, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 152, + 56, + 214, + 156, + 82, + 126, + 112, + 170, + 111, + 68, + 130, + 125, + 97, + 145, + 116 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6307666666666666, + "auc": 0.9211500555555555 + }, + "svm": { + "f1": 0.7021999999999999, + "auc": 0.9406485370370371 + }, + "rf": { + "f1": 0.7043666666666667, + "auc": 0.9336996111111111 + } + }, + "time": 16.28889012336731, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 152, + 56, + 214, + 67, + 137, + 156, + 71, + 95, + 168, + 127, + 198, + 122, + 57, + 82, + 153 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6387333333333333, + "auc": 0.9242300648148146 + }, + "svm": { + "f1": 0.7015, + "auc": 0.9395794074074073 + }, + "rf": { + "f1": 0.7029, + "auc": 0.9331555740740742 + } + }, + "time": 16.43316411972046, + "algorithm": "MRI" + }, + { + "selected_features": [ + 152, + 56, + 214, + 122, + 95, + 167, + 57, + 137, + 71, + 168, + 67, + 213, + 153, + 72, + 162 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6046999999999999, + "auc": 0.9170256851851851 + }, + "svm": { + "f1": 0.6860333333333333, + "auc": 0.9358115185185185 + }, + "rf": { + "f1": 0.6875333333333333, + "auc": 0.9254897037037038 + } + }, + "time": 13.104116916656494, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 152, + 56, + 214, + 95, + 122, + 142, + 168, + 67, + 72, + 113, + 76, + 198, + 3, + 156, + 137 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6798000000000002, + "auc": 0.9326571296296294 + }, + "svm": { + "f1": 0.7314333333333334, + "auc": 0.9445292592592592 + }, + "rf": { + "f1": 0.7298999999999999, + "auc": 0.9367795925925927 + } + }, + "time": 25.448102474212646, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/SRBCT.json b/FeatureSelect-Methods-Leaderboard/results/SRBCT.json new file mode 100644 index 0000000000000000000000000000000000000000..578a6d3ed50933ccfc9686efd394687c982a8281 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/SRBCT.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 0, + 75, + 298, + 607, + 764, + 766, + 856, + 1164, + 1195, + 1655, + 1740, + 2022, + 2051, + 2075, + 2290 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6273092369477912, + "auc": 0.84528950423778 + }, + "svm": { + "f1": 0.6040160642570281, + "auc": 0.8457038160859999 + }, + "rf": { + "f1": 0.6409638554216865, + "auc": 0.861319492076676 + } + }, + "time": 44.54918718338013, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7333333333333333, + "auc": 0.9146822982254018 + }, + "svm": { + "f1": 0.5140562248995983, + "auc": 0.7891250777745031 + }, + "rf": { + "f1": 0.7959839357429719, + "auc": 0.9421002152381465 + } + }, + "time": 9.953648090362549, + "algorithm": "CFR" + }, + { + "selected_features": [ + 0, + 2075, + 115, + 223, + 702, + 2022, + 189, + 476, + 564, + 1050, + 1437, + 2244, + 122, + 268, + 355 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5228915662650603, + "auc": 0.7724913257240844 + }, + "svm": { + "f1": 0.5534136546184739, + "auc": 0.8172622251099263 + }, + "rf": { + "f1": 0.6024096385542168, + "auc": 0.8235372472886842 + } + }, + "time": 16.482293844223022, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 483, + 75, + 2075, + 115, + 702, + 189, + 476, + 564, + 1050, + 1437, + 2244, + 223, + 1744, + 131, + 222 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5116465863453816, + "auc": 0.7474302827866046 + }, + "svm": { + "f1": 0.4208835341365462, + "auc": 0.6690086579023361 + }, + "rf": { + "f1": 0.5582329317269076, + "auc": 0.7982210890960891 + } + }, + "time": 13.982263326644897, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 0, + 75, + 115, + 2075, + 702, + 189, + 223, + 476, + 564, + 1050, + 1437, + 2244, + 1744, + 131, + 2022 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5076305220883534, + "auc": 0.778806246222913 + }, + "svm": { + "f1": 0.5068273092369477, + "auc": 0.8021679572024399 + }, + "rf": { + "f1": 0.5831325301204819, + "auc": 0.8168465341597526 + } + }, + "time": 12.532864093780518, + "algorithm": "MRI" + }, + { + "selected_features": [ + 0, + 483, + 75, + 607, + 2022, + 298, + 764, + 766, + 856, + 1164, + 1195, + 1655, + 1740, + 2051, + 2290 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6369477911646586, + "auc": 0.8532426601708211 + }, + "svm": { + "f1": 0.5951807228915663, + "auc": 0.841985242939266 + }, + "rf": { + "f1": 0.7092369477911646, + "auc": 0.8828265386699296 + } + }, + "time": 10.10467267036438, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 0, + 75, + 298, + 764, + 766, + 856, + 1164, + 1195, + 1655, + 1740, + 2051, + 2290, + 607, + 2022, + 2075 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6530120481927711, + "auc": 0.8531778885227161 + }, + "svm": { + "f1": 0.6184738955823295, + "auc": 0.8518449206179091 + }, + "rf": { + "f1": 0.6626506024096385, + "auc": 0.8717842672971984 + } + }, + "time": 20.301495790481567, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Sorlie.json b/FeatureSelect-Methods-Leaderboard/results/Sorlie.json new file mode 100644 index 0000000000000000000000000000000000000000..74e10d488f4ecb3ff0d8bdf5bc5055033b2bf935 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Sorlie.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 60, + 3, + 22, + 53, + 82, + 86, + 125, + 129, + 232, + 241, + 247, + 267, + 269, + 271, + 272 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.4752941176470588, + "auc": 0.7383005314004781 + }, + "svm": { + "f1": 0.4501960784313726, + "auc": 0.719930238313071 + }, + "rf": { + "f1": 0.5137254901960783, + "auc": 0.7685755407565001 + } + }, + "time": 9.430098295211792, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 60, + 147, + 55, + 120, + 231, + 285, + 81, + 94, + 268, + 112, + 0, + 95, + 181, + 8, + 1 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.580392156862745, + "auc": 0.8259310342185974 + }, + "svm": { + "f1": 0.5450980392156861, + "auc": 0.7832676526453034 + }, + "rf": { + "f1": 0.571764705882353, + "auc": 0.8264908030361856 + } + }, + "time": 2.447948455810547, + "algorithm": "CFR" + }, + { + "selected_features": [ + 60, + 268, + 429, + 257, + 211, + 398, + 24, + 23, + 36, + 89, + 8, + 14, + 377, + 318, + 12 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.4901960784313725, + "auc": 0.7364359745318555 + }, + "svm": { + "f1": 0.49098039215686273, + "auc": 0.7474049834427194 + }, + "rf": { + "f1": 0.4572549019607843, + "auc": 0.7337190552013433 + } + }, + "time": 4.25197172164917, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 305, + 248, + 268, + 429, + 211, + 257, + 318, + 398, + 26, + 23, + 40, + 389, + 8, + 36, + 14 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.49176470588235294, + "auc": 0.7177767143645034 + }, + "svm": { + "f1": 0.47137254901960784, + "auc": 0.67139044814377 + }, + "rf": { + "f1": 0.4956862745098039, + "auc": 0.7380827467118791 + } + }, + "time": 3.3783950805664062, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 60, + 3, + 268, + 429, + 211, + 257, + 318, + 398, + 23, + 26, + 36, + 8, + 14, + 40, + 24 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5003921568627451, + "auc": 0.7537137604444675 + }, + "svm": { + "f1": 0.4933333333333333, + "auc": 0.7564098314917077 + }, + "rf": { + "f1": 0.45647058823529413, + "auc": 0.7478239708442473 + } + }, + "time": 3.1828556060791016, + "algorithm": "MRI" + }, + { + "selected_features": [ + 60, + 415, + 15, + 248, + 21, + 69, + 113, + 217, + 299, + 231, + 18, + 32, + 253, + 121, + 34 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5207843137254901, + "auc": 0.7697925347459229 + }, + "svm": { + "f1": 0.5011764705882352, + "auc": 0.7349398020317366 + }, + "rf": { + "f1": 0.5301960784313725, + "auc": 0.791257665977318 + } + }, + "time": 2.5893442630767822, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 60, + 415, + 15, + 248, + 21, + 69, + 217, + 299, + 231, + 18, + 32, + 38, + 121, + 62, + 24 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5537254901960782, + "auc": 0.783051026532318 + }, + "svm": { + "f1": 0.5294117647058822, + "auc": 0.7468455769431984 + }, + "rf": { + "f1": 0.5474509803921569, + "auc": 0.8006576602412572 + } + }, + "time": 4.744657039642334, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Su.json b/FeatureSelect-Methods-Leaderboard/results/Su.json new file mode 100644 index 0000000000000000000000000000000000000000..e406f121cc232158858c118d289ae301e10d8888 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Su.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 0, + 2, + 15, + 43, + 446, + 645, + 860, + 1071, + 1188, + 1339, + 1549, + 1562, + 1804, + 2317, + 2536 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7379084967320264, + "auc": 0.851318793004184 + }, + "svm": { + "f1": 0.7209150326797386, + "auc": 0.8556527695476803 + }, + "rf": { + "f1": 0.7333333333333333, + "auc": 0.8668251601894715 + } + }, + "time": 128.23602986335754, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 0, + 1, + 667, + 53, + 1195, + 4404, + 4768, + 4582, + 538, + 4587, + 111, + 1791, + 1188, + 1947, + 894 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.662091503267974, + "auc": 0.842329921193972 + }, + "svm": { + "f1": 0.6509803921568629, + "auc": 0.8373450093959319 + }, + "rf": { + "f1": 0.6601307189542485, + "auc": 0.8539824699531019 + } + }, + "time": 27.811873197555542, + "algorithm": "CFR" + }, + { + "selected_features": [ + 0, + 2963, + 4662, + 4496, + 632, + 1157, + 3937, + 4664, + 2703, + 2722, + 5332, + 5516, + 4264, + 2690, + 5037 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5712418300653594, + "auc": 0.7830041194634437 + }, + "svm": { + "f1": 0.6620915032679738, + "auc": 0.7894087583871353 + }, + "rf": { + "f1": 0.6614379084967321, + "auc": 0.8169645642938338 + } + }, + "time": 41.62198877334595, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 4768, + 1157, + 3937, + 788, + 1767, + 3023, + 5384, + 3343, + 3385, + 2757, + 2647, + 4227, + 5037, + 4680, + 4001 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6470588235294116, + "auc": 0.8736547692478523 + }, + "svm": { + "f1": 0.8163398692810458, + "auc": 0.9220838360321494 + }, + "rf": { + "f1": 0.8000000000000002, + "auc": 0.9279837696103991 + } + }, + "time": 32.915005922317505, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 0, + 2, + 577, + 4147, + 3378, + 2325, + 2224, + 44, + 4978, + 1140, + 5539, + 3404, + 3759, + 2896, + 1055 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.3679738562091503, + "auc": 0.660592445974586 + }, + "svm": { + "f1": 0.5183006535947712, + "auc": 0.7396761431234029 + }, + "rf": { + "f1": 0.5418300653594772, + "auc": 0.7635141644785362 + } + }, + "time": 33.437132120132446, + "algorithm": "MRI" + }, + { + "selected_features": [ + 0, + 2963, + 1157, + 5285, + 3937, + 3577, + 4768, + 4664, + 3582, + 2717, + 4264, + 817, + 5189, + 4624, + 2722 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.681045751633987, + "auc": 0.8327988542523527 + }, + "svm": { + "f1": 0.7418300653594773, + "auc": 0.8546314727030482 + }, + "rf": { + "f1": 0.7555555555555556, + "auc": 0.8830532177445504 + } + }, + "time": 27.609816789627075, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 0, + 149, + 847, + 937, + 1188, + 1545, + 1707, + 1773, + 1804, + 2308, + 2500, + 2780, + 4582, + 5398, + 5561 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7529411764705883, + "auc": 0.9061295316176409 + }, + "svm": { + "f1": 0.7601307189542484, + "auc": 0.8986443092438516 + }, + "rf": { + "f1": 0.7366013071895424, + "auc": 0.898045812102995 + } + }, + "time": 58.16722297668457, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Synthetic_control.json b/FeatureSelect-Methods-Leaderboard/results/Synthetic_control.json new file mode 100644 index 0000000000000000000000000000000000000000..c19ceaf0f94167cb764bf2b53ceb7217455c096d --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Synthetic_control.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 0, + 2, + 3, + 5, + 6, + 7, + 9, + 10, + 11, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5191111111111111, + "auc": 0.8141028888888889 + }, + "svm": { + "f1": 0.519888888888889, + "auc": 0.8078855555555555 + }, + "rf": { + "f1": 0.5615555555555556, + "auc": 0.8295456666666666 + } + }, + "time": 7.448830842971802, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.4860000000000001, + "auc": 0.7895188888888889 + }, + "svm": { + "f1": 0.4796666666666667, + "auc": 0.7772808888888889 + }, + "rf": { + "f1": 0.5314444444444445, + "auc": 0.8097131111111113 + } + }, + "time": 1.9478082656860352, + "algorithm": "CFR" + }, + { + "selected_features": [ + 0, + 19, + 5, + 13, + 20, + 38, + 4, + 28, + 8, + 21, + 3, + 17, + 12, + 25, + 2 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7551111111111112, + "auc": 0.91871 + }, + "svm": { + "f1": 0.7633333333333332, + "auc": 0.9208675555555553 + }, + "rf": { + "f1": 0.7752222222222221, + "auc": 0.9235653333333335 + } + }, + "time": 3.3787379264831543, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 45, + 1, + 19, + 5, + 28, + 13, + 20, + 38, + 4, + 21, + 2, + 6, + 18, + 3, + 17 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8502222222222222, + "auc": 0.9751326666666668 + }, + "svm": { + "f1": 0.8503333333333334, + "auc": 0.9751284444444445 + }, + "rf": { + "f1": 0.8646666666666666, + "auc": 0.9733658888888889 + } + }, + "time": 2.6698920726776123, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 0, + 2, + 5, + 19, + 28, + 13, + 20, + 38, + 4, + 21, + 3, + 6, + 17, + 18, + 8 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7327777777777779, + "auc": 0.9056362222222222 + }, + "svm": { + "f1": 0.7334444444444445, + "auc": 0.9042895555555557 + }, + "rf": { + "f1": 0.7465555555555556, + "auc": 0.9092383333333333 + } + }, + "time": 2.4554781913757324, + "algorithm": "MRI" + }, + { + "selected_features": [ + 0, + 25, + 8, + 45, + 1, + 4, + 12, + 19, + 21, + 3, + 17, + 23, + 26, + 38, + 58 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.789222222222222, + "auc": 0.9422586666666668 + }, + "svm": { + "f1": 0.8034444444444444, + "auc": 0.9417746666666668 + }, + "rf": { + "f1": 0.815, + "auc": 0.9433906666666666 + } + }, + "time": 2.1384832859039307, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 0, + 25, + 8, + 19, + 13, + 20, + 21, + 3, + 17, + 23, + 4, + 38, + 12, + 5, + 28 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7076666666666667, + "auc": 0.9054357777777777 + }, + "svm": { + "f1": 0.7214444444444446, + "auc": 0.9109595555555554 + }, + "rf": { + "f1": 0.7141111111111111, + "auc": 0.9101680000000001 + } + }, + "time": 3.9587652683258057, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Waveform.json b/FeatureSelect-Methods-Leaderboard/results/Waveform.json new file mode 100644 index 0000000000000000000000000000000000000000..9f825b9bf96943f9ceee255bfdf75eaeb9554d62 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Waveform.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 6, + 9, + 10, + 16, + 14, + 12, + 13, + 8, + 15, + 5, + 11, + 4, + 7, + 18, + 17 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7635333333333332, + "auc": 0.9233307156020445 + }, + "svm": { + "f1": 0.7988000000000001, + "auc": 0.9325793477864597 + }, + "rf": { + "f1": 0.7800933333333333, + "auc": 0.9222583517995379 + } + }, + "time": 29.83793306350708, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 6, + 9, + 10, + 5, + 14, + 13, + 8, + 12, + 15, + 16, + 11, + 4, + 7, + 17, + 3 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7625866666666666, + "auc": 0.9241304592736382 + }, + "svm": { + "f1": 0.8009866666666666, + "auc": 0.9332884156760273 + }, + "rf": { + "f1": 0.7845466666666667, + "auc": 0.923271131368285 + } + }, + "time": 8.064061164855957, + "algorithm": "CFR" + }, + { + "selected_features": [ + 6, + 25, + 22, + 26, + 29, + 36, + 39, + 35, + 28, + 31, + 27, + 23, + 37, + 21, + 30 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5633466666666667, + "auc": 0.7408205822808953 + }, + "svm": { + "f1": 0.5674266666666666, + "auc": 0.7377706524582895 + }, + "rf": { + "f1": 0.5421600000000001, + "auc": 0.7271244002612218 + } + }, + "time": 12.278924226760864, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 6, + 14, + 5, + 9, + 29, + 39, + 31, + 23, + 27, + 28, + 33, + 36, + 25, + 35, + 22 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7036666666666667, + "auc": 0.8631543579532172 + }, + "svm": { + "f1": 0.7296133333333332, + "auc": 0.8894676161767011 + }, + "rf": { + "f1": 0.70776, + "auc": 0.8761294222116304 + } + }, + "time": 9.183784246444702, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 6, + 9, + 31, + 39, + 23, + 27, + 28, + 29, + 33, + 36, + 20, + 35, + 22, + 25, + 0 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.67628, + "auc": 0.8566477925629319 + }, + "svm": { + "f1": 0.6818933333333332, + "auc": 0.8609748484290338 + }, + "rf": { + "f1": 0.6667466666666667, + "auc": 0.8494535822802284 + } + }, + "time": 10.132760763168335, + "algorithm": "MRI" + }, + { + "selected_features": [ + 6, + 26, + 37, + 22, + 25, + 36, + 30, + 29, + 21, + 35, + 32, + 0, + 24, + 38, + 28 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5630266666666667, + "auc": 0.7398066954457843 + }, + "svm": { + "f1": 0.5679333333333333, + "auc": 0.7401273783888129 + }, + "rf": { + "f1": 0.5437333333333334, + "auc": 0.7308291475265188 + } + }, + "time": 8.501197814941406, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 6, + 26, + 25, + 22, + 37, + 32, + 21, + 38, + 34, + 0, + 20, + 30, + 36, + 39, + 29 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5626933333333334, + "auc": 0.7390766265969386 + }, + "svm": { + "f1": 0.5661466666666667, + "auc": 0.7378147908439789 + }, + "rf": { + "f1": 0.5399066666666666, + "auc": 0.7257480455333741 + } + }, + "time": 14.312170267105103, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/Wdbc.json b/FeatureSelect-Methods-Leaderboard/results/Wdbc.json new file mode 100644 index 0000000000000000000000000000000000000000..35d37f7241ce939c4fa35fc93df422214d2180e3 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/Wdbc.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 7, + 0, + 8, + 12, + 15, + 21, + 25, + 29, + 2, + 4, + 5, + 11, + 13, + 16, + 17 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.914118336262449, + "auc": 0.9743952927082781 + }, + "svm": { + "f1": 0.9041593438781489, + "auc": 0.9592507443228865 + }, + "rf": { + "f1": 0.9330990041007615, + "auc": 0.9788392086394305 + } + }, + "time": 3.3751213550567627, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 7, + 18, + 6, + 0, + 1, + 2, + 12, + 4, + 5, + 8, + 9, + 10, + 11, + 13, + 14 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9104862331575863, + "auc": 0.9708309109807445 + }, + "svm": { + "f1": 0.9006444053895724, + "auc": 0.9526593027147967 + }, + "rf": { + "f1": 0.9261862917398945, + "auc": 0.9753378080792067 + } + }, + "time": 0.8123750686645508, + "algorithm": "CFR" + }, + { + "selected_features": [ + 7, + 24, + 8, + 4, + 0, + 20, + 1, + 18, + 9, + 28, + 27, + 17, + 21, + 11, + 22 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9287639132981839, + "auc": 0.9748665503937425 + }, + "svm": { + "f1": 0.9158758055067369, + "auc": 0.9677884361291685 + }, + "rf": { + "f1": 0.9404803749267722, + "auc": 0.9751770519528566 + } + }, + "time": 1.3419904708862305, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 7, + 6, + 20, + 24, + 8, + 4, + 18, + 9, + 1, + 28, + 17, + 21, + 11, + 0, + 29 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9274751025190393, + "auc": 0.9737897045610696 + }, + "svm": { + "f1": 0.9168131224370238, + "auc": 0.9695629195074257 + }, + "rf": { + "f1": 0.943526654950205, + "auc": 0.9798341349118262 + } + }, + "time": 1.1033227443695068, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 7, + 0, + 24, + 8, + 4, + 18, + 1, + 9, + 28, + 17, + 21, + 11, + 29, + 22, + 20 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9234915055653193, + "auc": 0.9762512552190689 + }, + "svm": { + "f1": 0.895840656121851, + "auc": 0.9518220495745469 + }, + "rf": { + "f1": 0.9386057410661978, + "auc": 0.9779755298345753 + } + }, + "time": 1.009852647781372, + "algorithm": "MRI" + }, + { + "selected_features": [ + 7, + 20, + 24, + 0, + 8, + 27, + 1, + 4, + 17, + 2, + 22, + 28, + 9, + 18, + 16 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9359109548916226, + "auc": 0.9844299984144603 + }, + "svm": { + "f1": 0.9170474516695957, + "auc": 0.9693334566531014 + }, + "rf": { + "f1": 0.9430579964850616, + "auc": 0.9801838345401054 + } + }, + "time": 0.9091165065765381, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 7, + 20, + 24, + 0, + 8, + 27, + 1, + 4, + 17, + 2, + 22, + 28, + 9, + 18, + 16 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.9359109548916226, + "auc": 0.9844299984144603 + }, + "svm": { + "f1": 0.9170474516695957, + "auc": 0.9693334566531014 + }, + "rf": { + "f1": 0.9430579964850616, + "auc": 0.9801838345401054 + } + }, + "time": 1.6847598552703857, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/dna.json b/FeatureSelect-Methods-Leaderboard/results/dna.json new file mode 100644 index 0000000000000000000000000000000000000000..f729c3fe94fc4b6d879a2434445e42f6ddc2594e --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/dna.json @@ -0,0 +1,331 @@ +[ + { + "selected_features": [ + 89, + 92, + 84, + 104, + 82, + 99, + 88, + 87, + 90, + 91, + 85, + 95, + 83, + 93, + 81 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7640929064657878, + "auc": 0.9133179854605366 + }, + "svm": { + "f1": 0.8536304666248171, + "auc": 0.9352907039838904 + }, + "rf": { + "f1": 0.8522494245658089, + "auc": 0.9412461781596505 + } + }, + "time": 61.692588090896606, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 82, + 99, + 93, + 88, + 87, + 90, + 95, + 94, + 85, + 83, + 86 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8071772337309061, + "auc": 0.9251172965675879 + }, + "svm": { + "f1": 0.8628164888051894, + "auc": 0.9403608959817996 + }, + "rf": { + "f1": 0.86185394433982, + "auc": 0.9447127585949784 + } + }, + "time": 14.362110137939453, + "algorithm": "CFR" + }, + { + "selected_features": [ + 89, + 104, + 92, + 84, + 81, + 99, + 93, + 83, + 95, + 94, + 82, + 97, + 90, + 87, + 88 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8425402803933877, + "auc": 0.9345225210498646 + }, + "svm": { + "f1": 0.881586105879891, + "auc": 0.9410499099603411 + }, + "rf": { + "f1": 0.8811048336472066, + "auc": 0.9480086728315744 + } + }, + "time": 23.570918560028076, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 99, + 93, + 95, + 94, + 83, + 81, + 97, + 74, + 72, + 71, + 62 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8436283741368488, + "auc": 0.9371476543435738 + }, + "svm": { + "f1": 0.8793262188742416, + "auc": 0.9464104127302048 + }, + "rf": { + "f1": 0.8789077212806028, + "auc": 0.9477423771202302 + } + }, + "time": 17.612692832946777, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 82, + 99, + 93, + 95, + 94, + 88, + 87, + 90, + 83, + 81, + 85 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8277045406988911, + "auc": 0.9322267536115253 + }, + "svm": { + "f1": 0.8711027411592386, + "auc": 0.9431894900660284 + }, + "rf": { + "f1": 0.8701820464532329, + "auc": 0.9464250615396989 + } + }, + "time": 18.142696142196655, + "algorithm": "MRI" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 82, + 99, + 93, + 88, + 95, + 87, + 90, + 94, + 83, + 85, + 86 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8157145846411384, + "auc": 0.9275568626302196 + }, + "svm": { + "f1": 0.8656204226825694, + "auc": 0.941107038573083 + }, + "rf": { + "f1": 0.8650554509311572, + "auc": 0.9451365316009367 + } + }, + "time": 14.290248394012451, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 89, + 92, + 84, + 104, + 99, + 82, + 93, + 95, + 94, + 97, + 83, + 81, + 90, + 87, + 88 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8353211969031178, + "auc": 0.9344102233712435 + }, + "svm": { + "f1": 0.8754760410127641, + "auc": 0.9449093560542415 + }, + "rf": { + "f1": 0.8743460975099393, + "auc": 0.9470115615925281 + } + }, + "time": 28.11224675178528, + "algorithm": "UCRFS" + }, + { + "selected_features": [ + [ + 92, + 104, + 93, + 95, + 94, + 99, + 90, + 91, + 97, + 89, + 103, + 102, + 96, + 88, + 98 + ], + [ + 84, + 81, + 89, + 83, + 104, + 82, + 85, + 86, + 87, + 88, + 74, + 72, + 66, + 71, + 57 + ], + [ + 89, + 84, + 92, + 104, + 82, + 99, + 87, + 93, + 88, + 83, + 85, + 91, + 86, + 74, + 94 + ] + ], + "num_features": [ + 15, + 15, + 15 + ], + "union_num_features": 3, + "metrics": { + "nb": { + "f1": 0.7881146683406571, + "auc": 0.9145317738835733 + }, + "svm": { + "f1": 0.8584641138313454, + "auc": 0.9310307930574658 + }, + "rf": { + "f1": 0.857187696170747, + "auc": 0.9399791620380172 + } + }, + "time": 96.02905464172363, + "algorithm": "CSMDCCMR" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/madelon.json b/FeatureSelect-Methods-Leaderboard/results/madelon.json new file mode 100644 index 0000000000000000000000000000000000000000..327d0707bf7d6bac3d2a46cf7ca56d43ed6e5a09 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/madelon.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 105, + 281, + 453, + 336, + 475, + 442, + 338, + 378, + 455, + 356, + 70, + 170, + 411, + 35, + 398 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5867333333333333, + "auc": 0.6312839999999998 + }, + "svm": { + "f1": 0.7571, + "auc": 0.8319489000000001 + }, + "rf": { + "f1": 0.7899333333333334, + "auc": 0.8591658333333334 + } + }, + "time": 148.53723216056824, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 105, + 453, + 336, + 338, + 433, + 442, + 475, + 455, + 378, + 493, + 188, + 221, + 397, + 153, + 64 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5947666666666669, + "auc": 0.6383539999999999 + }, + "svm": { + "f1": 0.7640999999999999, + "auc": 0.8396392333333335 + }, + "rf": { + "f1": 0.7969666666666666, + "auc": 0.8623019833333333 + } + }, + "time": 32.84015369415283, + "algorithm": "CFR" + }, + { + "selected_features": [ + 105, + 423, + 228, + 90, + 404, + 276, + 332, + 205, + 402, + 7, + 154, + 198, + 283, + 445, + 473 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5394, + "auc": 0.5626276666666666 + }, + "svm": { + "f1": 0.5538333333333334, + "auc": 0.5995931999999999 + }, + "rf": { + "f1": 0.5449, + "auc": 0.5655548499999999 + } + }, + "time": 44.06774830818176, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 105, + 493, + 338, + 36, + 269, + 348, + 190, + 434, + 98, + 52, + 137, + 159, + 132, + 234, + 106 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6007666666666668, + "auc": 0.626716 + }, + "svm": { + "f1": 0.6625333333333334, + "auc": 0.7259910333333333 + }, + "rf": { + "f1": 0.6556333333333332, + "auc": 0.71134805 + } + }, + "time": 38.110742807388306, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 105, + 453, + 281, + 175, + 462, + 478, + 467, + 31, + 221, + 378, + 211, + 84, + 318, + 171, + 418 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5558333333333335, + "auc": 0.5935540666666667 + }, + "svm": { + "f1": 0.6892666666666666, + "auc": 0.7560348999999998 + }, + "rf": { + "f1": 0.6908333333333332, + "auc": 0.7600351166666667 + } + }, + "time": 42.77273678779602, + "algorithm": "MRI" + }, + { + "selected_features": [ + 105, + 90, + 423, + 276, + 404, + 228, + 332, + 205, + 402, + 173, + 168, + 280, + 283, + 7, + 154 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5406333333333333, + "auc": 0.5662360666666667 + }, + "svm": { + "f1": 0.5534333333333332, + "auc": 0.5995369 + }, + "rf": { + "f1": 0.5396000000000001, + "auc": 0.5609596 + } + }, + "time": 27.384498119354248, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 105, + 90, + 423, + 473, + 400, + 336, + 86, + 338, + 475, + 378, + 153, + 455, + 453, + 442, + 318 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5707000000000001, + "auc": 0.6100783 + }, + "svm": { + "f1": 0.6832666666666666, + "auc": 0.7472135333333334 + }, + "rf": { + "f1": 0.6797, + "auc": 0.7333997666666668 + } + }, + "time": 62.44458293914795, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/spambase.json b/FeatureSelect-Methods-Leaderboard/results/spambase.json new file mode 100644 index 0000000000000000000000000000000000000000..b264f8c5ab3d8e303df776d943c78edb9d0ba976 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/spambase.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 54, + 18, + 56, + 55, + 51, + 49, + 20, + 11, + 52, + 4, + 2, + 15, + 6, + 24, + 16 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7093240599869592, + "auc": 0.8214716341381643 + }, + "svm": { + "f1": 0.7149460262261828, + "auc": 0.798182727276804 + }, + "rf": { + "f1": 0.8845758168514091, + "auc": 0.9343369615479679 + } + }, + "time": 29.13853359222412, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 54, + 18, + 56, + 55, + 51, + 20, + 49, + 11, + 4, + 52, + 2, + 15, + 9, + 6, + 24 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7010070274577989, + "auc": 0.8188125513883866 + }, + "svm": { + "f1": 0.7149025574150547, + "auc": 0.7980935023976631 + }, + "rf": { + "f1": 0.8831848148953126, + "auc": 0.9338097203284742 + } + }, + "time": 8.306282997131348, + "algorithm": "CFR" + }, + { + "selected_features": [ + 54, + 3, + 46, + 37, + 21, + 40, + 31, + 47, + 14, + 6, + 41, + 43, + 13, + 33, + 23 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5618923422444396, + "auc": 0.7098339929247902 + }, + "svm": { + "f1": 0.733579656596392, + "auc": 0.8098006770275678 + }, + "rf": { + "f1": 0.7706875316960082, + "auc": 0.819905424265949 + } + }, + "time": 11.151320219039917, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 6, + 52, + 51, + 15, + 23, + 49, + 18, + 11, + 4, + 2, + 55, + 20, + 9, + 44, + 48 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.7968847352024924, + "auc": 0.8877001228968843 + }, + "svm": { + "f1": 0.8230964283126856, + "auc": 0.8763540815139506 + }, + "rf": { + "f1": 0.9002245888574948, + "auc": 0.9261076315826264 + } + }, + "time": 9.526047468185425, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 54, + 18, + 55, + 56, + 49, + 20, + 11, + 51, + 2, + 4, + 15, + 9, + 52, + 44, + 17 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6868217054263565, + "auc": 0.8046844182630204 + }, + "svm": { + "f1": 0.717481706875317, + "auc": 0.8013414726470681 + }, + "rf": { + "f1": 0.8785916105194522, + "auc": 0.9304267751926082 + } + }, + "time": 10.040618658065796, + "algorithm": "MRI" + }, + { + "selected_features": [ + 54, + 3, + 46, + 37, + 6, + 21, + 26, + 23, + 40, + 22, + 41, + 15, + 24, + 14, + 47 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.6735926972397305, + "auc": 0.79750374441141 + }, + "svm": { + "f1": 0.7622256031297543, + "auc": 0.8482402651238478 + }, + "rf": { + "f1": 0.8200391219300153, + "auc": 0.8701970768531541 + } + }, + "time": 7.383306980133057, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 54, + 3, + 46, + 37, + 21, + 40, + 31, + 6, + 47, + 22, + 43, + 51, + 41, + 32, + 55 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.5605303194957618, + "auc": 0.7395238879994978 + }, + "svm": { + "f1": 0.74435992175614, + "auc": 0.8212578241580086 + }, + "rf": { + "f1": 0.7960008693762227, + "auc": 0.8420140666681966 + } + }, + "time": 14.38956618309021, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/results/splice.json b/FeatureSelect-Methods-Leaderboard/results/splice.json new file mode 100644 index 0000000000000000000000000000000000000000..ad9572ecfed2e0d4e39880db1df98f9d594507f8 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/results/splice.json @@ -0,0 +1,254 @@ +[ + { + "selected_features": [ + 29, + 31, + 30, + 28, + 27, + 34, + 32, + 24, + 33, + 21, + 25, + 22, + 23, + 19, + 18 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8561546499477533, + "auc": 0.9363994601913161 + }, + "svm": { + "f1": 0.8655381400208986, + "auc": 0.9467355486431462 + }, + "rf": { + "f1": 0.8959456635318707, + "auc": 0.9576438061027346 + } + }, + "time": 19.900165557861328, + "algorithm": "JMIM" + }, + { + "selected_features": [ + 29, + 31, + 28, + 30, + 34, + 27, + 32, + 33, + 24, + 25, + 22, + 23, + 19, + 20, + 18 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8519749216300939, + "auc": 0.9367564358158572 + }, + "svm": { + "f1": 0.8640961337513062, + "auc": 0.9481007116756216 + }, + "rf": { + "f1": 0.8954649947753397, + "auc": 0.9580926551210168 + } + }, + "time": 5.530740737915039, + "algorithm": "CFR" + }, + { + "selected_features": [ + 29, + 31, + 27, + 30, + 28, + 34, + 32, + 24, + 33, + 20, + 22, + 18, + 25, + 17, + 21 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8418390804597701, + "auc": 0.9314439352840653 + }, + "svm": { + "f1": 0.8611494252873564, + "auc": 0.9460598992209758 + }, + "rf": { + "f1": 0.8930198537095089, + "auc": 0.9569600860853185 + } + }, + "time": 8.032433986663818, + "algorithm": "DCSF" + }, + { + "selected_features": [ + 29, + 31, + 28, + 30, + 34, + 27, + 32, + 24, + 33, + 20, + 22, + 25, + 21, + 17, + 16 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8521212121212121, + "auc": 0.9369547183169493 + }, + "svm": { + "f1": 0.8657053291536051, + "auc": 0.9482749813421181 + }, + "rf": { + "f1": 0.8959874608150471, + "auc": 0.9580451731800338 + } + }, + "time": 6.510812044143677, + "algorithm": "IWFS" + }, + { + "selected_features": [ + 29, + 31, + 28, + 30, + 34, + 27, + 32, + 33, + 24, + 22, + 25, + 20, + 23, + 18, + 19 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8518704284221524, + "auc": 0.9369295156316697 + }, + "svm": { + "f1": 0.8639080459770115, + "auc": 0.9480263928402802 + }, + "rf": { + "f1": 0.895548589341693, + "auc": 0.9580104358448902 + } + }, + "time": 6.646439790725708, + "algorithm": "MRI" + }, + { + "selected_features": [ + 29, + 31, + 28, + 30, + 34, + 27, + 32, + 33, + 24, + 22, + 25, + 23, + 19, + 18, + 20 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8518286311389758, + "auc": 0.9367810897897958 + }, + "svm": { + "f1": 0.8638035527690701, + "auc": 0.9479579218043312 + }, + "rf": { + "f1": 0.8951515151515153, + "auc": 0.9578920591082477 + } + }, + "time": 5.573030471801758, + "algorithm": "MRMD" + }, + { + "selected_features": [ + 29, + 31, + 28, + 30, + 34, + 27, + 32, + 24, + 33, + 20, + 22, + 18, + 25, + 17, + 21 + ], + "num_features": 15, + "metrics": { + "nb": { + "f1": 0.8519958202716823, + "auc": 0.9372189491106175 + }, + "svm": { + "f1": 0.8651828631138977, + "auc": 0.948149039497789 + }, + "rf": { + "f1": 0.8961128526645771, + "auc": 0.9580818778448604 + } + }, + "time": 9.888350248336792, + "algorithm": "UCRFS" + } +] \ No newline at end of file diff --git a/FeatureSelect-Methods-Leaderboard/verify_backend.py b/FeatureSelect-Methods-Leaderboard/verify_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..b709a5aa0fa6bea050b92ee1155d28d687be9242 --- /dev/null +++ b/FeatureSelect-Methods-Leaderboard/verify_backend.py @@ -0,0 +1,41 @@ +import json +import os +import sys + +# Mock paths +PROJECT_ROOT = os.path.abspath(os.getcwd()) +sys.path.append(PROJECT_ROOT) + +from leaderboard import rank_results + +def test_dataset(name): + path = os.path.join(PROJECT_ROOT, "results", f"{name}.json") + if not os.path.exists(path): + print(f"[ERROR] {name} not found") + return + + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + + print(f"--- Testing {name} ---") + try: + ranked = rank_results(data) + if len(ranked) > 0: + first = ranked[0] + print("Keys in first item:", first.keys()) + # Check for critical keys + for key in ['mean_f1', 'mean_auc', 'time']: + if key not in first: + print(f"[FAIL] Missing key: {key}") + elif first[key] is None: + print(f"[FAIL] Key is None: {key}") + else: + print(f"[OK] {key}: {first[key]} (type: {type(first[key])})") + else: + print("[WARN] Ranked list is empty") + except Exception as e: + print(f"[ERROR] Ranking failed: {e}") + +test_dataset("Authorship") +test_dataset("Factors") +test_dataset("dna") diff --git a/Webapp/app.py b/Webapp/app.py index 507a61be631827896cc788de1b5bfa84678a5200..b09037bdae5957bb3f6e67d04ff5ac7a0caf0900 100644 --- a/Webapp/app.py +++ b/Webapp/app.py @@ -101,6 +101,11 @@ def index(): ) +@app.route("/global") +def global_view(): + return render_template("global.html") + + # =============================== # API:获取结果 # =============================== @@ -133,7 +138,15 @@ def get_results(): print("------------------------------------------------------------------zoude agent") leaderboard = rank_results(results) - # + + # Ensure leaderboard is a list + if not isinstance(leaderboard, list): + print(f"[WARNING] rank_results returned {type(leaderboard)}, converting to list") + if hasattr(leaderboard, 'to_dict'): # Pandas DataFrame + leaderboard = leaderboard.to_dict(orient='records') + else: + leaderboard = list(leaderboard) + return jsonify(leaderboard) # print(333,leaderboard) # @app.route("/api/results") @@ -169,6 +182,80 @@ def api_datasets(): return jsonify(list_available_datasets()) +@app.route("/api/global_stats") +def get_global_stats(): + # 1. Load all datasets + all_datasets = list_available_datasets() + + # 2. Aggregate stats by algorithm + # Structure: { "AlgoName": { "f1_sum": 0, "auc_sum": 0, "time_sum": 0, "count": 0 } } + agg_stats = {} + + for ds_name in all_datasets: + results = load_result_json(ds_name) + if not results: + continue + + # Ensure results is a list (reuse logic from get_results) + if not isinstance(results, list): + if hasattr(results, 'to_dict'): + results = results.to_dict(orient='records') + elif isinstance(results, dict) and 'data' in results: + results = results['data'] + elif isinstance(results, dict) and 'results' in results: + results = results['results'] + elif isinstance(results, dict) and 'algorithm' in results: + results = [results] + else: + continue # Skip invalid data + + for res in results: + algo = res.get("algorithm", "Unknown") + + # Extract metrics + # Assuming rank_results logic: mean_f1, mean_auc might be pre-calculated in JSON + # or we calculate them on the fly if missing + metrics = res.get("metrics", {}) + + mean_f1 = res.get("mean_f1") + mean_auc = res.get("mean_auc") + + # If mean values missing, calculate from metrics + if mean_f1 is None and metrics: + f1s = [m.get("f1", 0) for m in metrics.values() if isinstance(m, dict)] + mean_f1 = sum(f1s) / len(f1s) if f1s else 0 + + if mean_auc is None and metrics: + aucs = [m.get("auc", 0) for m in metrics.values() if isinstance(m, dict)] + mean_auc = sum(aucs) / len(aucs) if aucs else 0 + + time_val = res.get("time", 0) + + if algo not in agg_stats: + agg_stats[algo] = {"f1_sum": 0.0, "auc_sum": 0.0, "time_sum": 0.0, "count": 0} + + agg_stats[algo]["f1_sum"] += float(mean_f1 or 0) + agg_stats[algo]["auc_sum"] += float(mean_auc or 0) + agg_stats[algo]["time_sum"] += float(time_val or 0) + agg_stats[algo]["count"] += 1 + + # 3. Calculate averages + final_list = [] + for algo, stats in agg_stats.items(): + count = stats["count"] + if count == 0: continue + + final_list.append({ + "algorithm": algo, + "mean_f1_global": stats["f1_sum"] / count, + "mean_auc_global": stats["auc_sum"] / count, + "mean_time_global": stats["time_sum"] / count, + "datasets_count": count + }) + + return jsonify(final_list) + + if __name__ == "__main__": port = int(os.environ.get("PORT", 7860)) app.run(host="0.0.0.0", port=port, debug=False) diff --git a/Webapp/templates/global.html b/Webapp/templates/global.html new file mode 100644 index 0000000000000000000000000000000000000000..98c4e7f8f8822f4ac5c21a01e90ec0a47adcb457 --- /dev/null +++ b/Webapp/templates/global.html @@ -0,0 +1,388 @@ + + + + + + Global Leaderboard + + + + + +
+
+

🌍 Global Algorithm Rankings

+ ← Back to Dataset View +
+ +
+

🏆 Scoring Formula: S = a·F1 + b·AUC + c·TimeScore

+

+ Note: TimeScore is calculated as normalized efficiency (1 = fastest, 0 = slowest). +
Constraint: a + b + c = 1. +

+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+
⚠️ Weights must sum to 1.0
+
+ +
Loading global stats...
+ + + + + + + + + + + + + + + +
#Algorithm Global F1 Global AUC Avg Time (s) Final Score
+
+ + + + + \ No newline at end of file diff --git a/Webapp/templates/index.html b/Webapp/templates/index.html index a333b0b39cc81de159977397963870affd9f7e6b..ef85055e1f8ba5136a0e7529e0385290a9866d9b 100644 --- a/Webapp/templates/index.html +++ b/Webapp/templates/index.html @@ -178,11 +178,14 @@

🏆 AutoFS Leaderboard

-
- - +
+ 🌍 Global Rankings +
+ + +
@@ -451,10 +454,29 @@ function updateTable(results) { tableBody.innerHTML = ""; - // Safety check - if (!Array.isArray(results)) { - tableBody.innerHTML = 'Error: Invalid data format'; - return; + // Robust data handling + if (!results) { + results = []; + } else if (!Array.isArray(results)) { + // Try to handle wrapped data or single object + if (results.data && Array.isArray(results.data)) { + results = results.data; + } else if (results.results && Array.isArray(results.results)) { + results = results.results; + } else if (typeof results === 'object') { + // Assume it's a single record? Or convert object values to array? + // For now, wrap in array if it looks like a record (has algorithm) + if (results.algorithm) { + results = [results]; + } else { + console.error("Invalid data format:", results); + tableBody.innerHTML = 'Error: Invalid data format. Check console for details.'; + return; + } + } else { + tableBody.innerHTML = 'Error: Invalid data format'; + return; + } } if (results.length === 0) { diff --git a/check_api.py b/check_api.py new file mode 100644 index 0000000000000000000000000000000000000000..fecff3751b72b944847320137883d15c697bfb62 --- /dev/null +++ b/check_api.py @@ -0,0 +1,18 @@ +import urllib.request +import json + +try: + url = "http://127.0.0.1:7860/api/results?dataset=Authorship" + with urllib.request.urlopen(url) as response: + data = json.loads(response.read().decode()) + print(f"Status Code: {response.getcode()}") + print(f"Type: {type(data)}") + if isinstance(data, list): + print(f"Length: {len(data)}") + if len(data) > 0: + print(f"First item keys: {data[0].keys()}") + else: + print("Data is NOT a list!") + print(data) +except Exception as e: + print(f"Error: {e}") diff --git a/check_global_api.py b/check_global_api.py new file mode 100644 index 0000000000000000000000000000000000000000..8d41d420a5069789a7e50132bd7e9240f004752d --- /dev/null +++ b/check_global_api.py @@ -0,0 +1,19 @@ +import urllib.request +import json + +try: + url = "http://127.0.0.1:7860/api/global_stats" + with urllib.request.urlopen(url) as response: + data = json.loads(response.read().decode()) + print(f"Status Code: {response.getcode()}") + print(f"Type: {type(data)}") + if isinstance(data, list): + print(f"Length: {len(data)}") + if len(data) > 0: + print(f"First item keys: {data[0].keys()}") + print(f"First item: {data[0]}") + else: + print("Data is NOT a list!") + print(data) +except Exception as e: + print(f"Error: {e}")