| { |
| "overall_3class_comparison": { |
| "Baseline_1_TFIDF_LogReg": { |
| "macro_f1": 0.6065885032485002, |
| "accuracy": 0.8164444444444444 |
| }, |
| "Baseline_2_BERT_overall_3class": { |
| "macro_f1": 0.7179307477990319, |
| "accuracy": 0.9002222222222223 |
| }, |
| "Baseline_3_BERT_ACSA_no_meta__aggregated_to_overall": { |
| "macro_f1": 0.4793355643956521, |
| "accuracy": 0.6584444444444445 |
| }, |
| "Proposed_BERT_Meta_Fusion__overall_head": { |
| "macro_f1": 0.697132916417921, |
| "accuracy": 0.8935555555555555 |
| }, |
| "Proposed_BERT_Meta_Fusion__aggregated_to_overall_(reference)": { |
| "macro_f1": 0.5042876450582995, |
| "accuracy": 0.6951111111111111 |
| } |
| }, |
| "proposed_per_aspect": { |
| "per_aspect": { |
| "SIZE": { |
| "macro_f1": 0.7995265557571497, |
| "accuracy": 0.8084444444444444, |
| "confusion_matrix": [ |
| [ |
| 1048, |
| 138, |
| 72 |
| ], |
| [ |
| 285, |
| 1854, |
| 254 |
| ], |
| [ |
| 41, |
| 72, |
| 736 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.7627365356622998, |
| "recall": 0.8330683624801272, |
| "f1-score": 0.7963525835866262, |
| "support": 1258.0 |
| }, |
| "Positive": { |
| "precision": 0.8982558139534884, |
| "recall": 0.7747597158378604, |
| "f1-score": 0.8319497419789096, |
| "support": 2393.0 |
| }, |
| "Negative": { |
| "precision": 0.6930320150659134, |
| "recall": 0.866902237926973, |
| "f1-score": 0.7702773417059131, |
| "support": 849.0 |
| }, |
| "accuracy": 0.8084444444444444, |
| "macro avg": { |
| "precision": 0.7846747882272339, |
| "recall": 0.8249101054149869, |
| "f1-score": 0.7995265557571497, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8216517567655182, |
| "recall": 0.8084444444444444, |
| "f1-score": 0.810362832403517, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "MATERIAL": { |
| "macro_f1": 0.7375608350812924, |
| "accuracy": 0.7748888888888888, |
| "confusion_matrix": [ |
| [ |
| 885, |
| 196, |
| 99 |
| ], |
| [ |
| 307, |
| 2119, |
| 271 |
| ], |
| [ |
| 74, |
| 66, |
| 483 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.6990521327014217, |
| "recall": 0.75, |
| "f1-score": 0.723630417007359, |
| "support": 1180.0 |
| }, |
| "Positive": { |
| "precision": 0.8899622007559849, |
| "recall": 0.78568780126066, |
| "f1-score": 0.8345805435210712, |
| "support": 2697.0 |
| }, |
| "Negative": { |
| "precision": 0.5662368112543963, |
| "recall": 0.7752808988764045, |
| "f1-score": 0.6544715447154471, |
| "support": 623.0 |
| }, |
| "accuracy": 0.7748888888888888, |
| "macro avg": { |
| "precision": 0.7184170482372677, |
| "recall": 0.770322900045688, |
| "f1-score": 0.7375608350812924, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7950833567640129, |
| "recall": 0.7748888888888888, |
| "f1-score": 0.7805518645117191, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "QUALITY": { |
| "macro_f1": 0.7225683708431877, |
| "accuracy": 0.7782222222222223, |
| "confusion_matrix": [ |
| [ |
| 1939, |
| 419, |
| 163 |
| ], |
| [ |
| 290, |
| 1360, |
| 59 |
| ], |
| [ |
| 42, |
| 25, |
| 203 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.8538088947600176, |
| "recall": 0.7691392304641016, |
| "f1-score": 0.8092654424040067, |
| "support": 2521.0 |
| }, |
| "Positive": { |
| "precision": 0.753880266075388, |
| "recall": 0.7957870099473376, |
| "f1-score": 0.7742670082550527, |
| "support": 1709.0 |
| }, |
| "Negative": { |
| "precision": 0.4776470588235294, |
| "recall": 0.7518518518518519, |
| "f1-score": 0.5841726618705037, |
| "support": 270.0 |
| }, |
| "accuracy": 0.7782222222222223, |
| "macro avg": { |
| "precision": 0.695112073219645, |
| "recall": 0.7722593640877636, |
| "f1-score": 0.7225683708431877, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.793288512065599, |
| "recall": 0.7782222222222223, |
| "f1-score": 0.7824682480252049, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "APPEARANCE": { |
| "macro_f1": 0.686600260772051, |
| "accuracy": 0.8044444444444444, |
| "confusion_matrix": [ |
| [ |
| 2098, |
| 260, |
| 145 |
| ], |
| [ |
| 273, |
| 1404, |
| 131 |
| ], |
| [ |
| 31, |
| 40, |
| 118 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.8734388009991674, |
| "recall": 0.8381941669996005, |
| "f1-score": 0.855453618756371, |
| "support": 2503.0 |
| }, |
| "Positive": { |
| "precision": 0.823943661971831, |
| "recall": 0.7765486725663717, |
| "f1-score": 0.7995444191343963, |
| "support": 1808.0 |
| }, |
| "Negative": { |
| "precision": 0.29949238578680204, |
| "recall": 0.6243386243386243, |
| "f1-score": 0.40480274442538594, |
| "support": 189.0 |
| }, |
| "accuracy": 0.8044444444444444, |
| "macro avg": { |
| "precision": 0.6656249495859335, |
| "recall": 0.7463604879681989, |
| "f1-score": 0.686600260772051, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8294470045910427, |
| "recall": 0.8044444444444444, |
| "f1-score": 0.8140632080530185, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "STYLE": { |
| "macro_f1": 0.7160335416761091, |
| "accuracy": 0.806, |
| "confusion_matrix": [ |
| [ |
| 1880, |
| 242, |
| 118 |
| ], |
| [ |
| 290, |
| 1585, |
| 150 |
| ], |
| [ |
| 39, |
| 34, |
| 162 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.851063829787234, |
| "recall": 0.8392857142857143, |
| "f1-score": 0.8451337379186334, |
| "support": 2240.0 |
| }, |
| "Positive": { |
| "precision": 0.8516926383664697, |
| "recall": 0.782716049382716, |
| "f1-score": 0.815748841996912, |
| "support": 2025.0 |
| }, |
| "Negative": { |
| "precision": 0.3767441860465116, |
| "recall": 0.6893617021276596, |
| "f1-score": 0.48721804511278194, |
| "support": 235.0 |
| }, |
| "accuracy": 0.806, |
| "macro avg": { |
| "precision": 0.693166884733405, |
| "recall": 0.7704544885986966, |
| "f1-score": 0.7160335416761091, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8265767678080969, |
| "recall": 0.806, |
| "f1-score": 0.813219381907331, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "VALUE": { |
| "macro_f1": 0.6463455740801278, |
| "accuracy": 0.814, |
| "confusion_matrix": [ |
| [ |
| 2944, |
| 324, |
| 172 |
| ], |
| [ |
| 231, |
| 630, |
| 46 |
| ], |
| [ |
| 35, |
| 29, |
| 89 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.9171339563862928, |
| "recall": 0.8558139534883721, |
| "f1-score": 0.8854135338345864, |
| "support": 3440.0 |
| }, |
| "Positive": { |
| "precision": 0.6408952187182095, |
| "recall": 0.6945975744211687, |
| "f1-score": 0.6666666666666666, |
| "support": 907.0 |
| }, |
| "Negative": { |
| "precision": 0.2899022801302932, |
| "recall": 0.5816993464052288, |
| "f1-score": 0.3869565217391304, |
| "support": 153.0 |
| }, |
| "accuracy": 0.814, |
| "macro avg": { |
| "precision": 0.6159771517449318, |
| "recall": 0.7107036247715898, |
| "f1-score": 0.6463455740801278, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.840130627156933, |
| "recall": 0.814, |
| "f1-score": 0.8243763490852736, |
| "support": 4500.0 |
| } |
| } |
| } |
| }, |
| "overall": { |
| "mean_macro_f1": 0.7181058563683197, |
| "mean_accuracy": 0.7976666666666666 |
| } |
| }, |
| "acsa_no_meta_per_aspect": { |
| "per_aspect": { |
| "SIZE": { |
| "macro_f1": 0.723496978308766, |
| "accuracy": 0.7164444444444444, |
| "confusion_matrix": [ |
| [ |
| 1035, |
| 152, |
| 71 |
| ], |
| [ |
| 686, |
| 1445, |
| 262 |
| ], |
| [ |
| 49, |
| 56, |
| 744 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.5847457627118644, |
| "recall": 0.8227344992050875, |
| "f1-score": 0.6836195508586526, |
| "support": 1258.0 |
| }, |
| "Positive": { |
| "precision": 0.8741681790683605, |
| "recall": 0.6038445465942331, |
| "f1-score": 0.7142857142857143, |
| "support": 2393.0 |
| }, |
| "Negative": { |
| "precision": 0.6908077994428969, |
| "recall": 0.8763250883392226, |
| "f1-score": 0.7725856697819314, |
| "support": 849.0 |
| }, |
| "accuracy": 0.7164444444444444, |
| "macro avg": { |
| "precision": 0.7165739137410405, |
| "recall": 0.7676347113795144, |
| "f1-score": 0.723496978308766, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7586645430509182, |
| "recall": 0.7164444444444444, |
| "f1-score": 0.7167120762023909, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "MATERIAL": { |
| "macro_f1": 0.6869602136807388, |
| "accuracy": 0.7111111111111111, |
| "confusion_matrix": [ |
| [ |
| 819, |
| 273, |
| 88 |
| ], |
| [ |
| 535, |
| 1896, |
| 266 |
| ], |
| [ |
| 72, |
| 66, |
| 485 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.5743338008415148, |
| "recall": 0.6940677966101695, |
| "f1-score": 0.6285495011511896, |
| "support": 1180.0 |
| }, |
| "Positive": { |
| "precision": 0.8483221476510067, |
| "recall": 0.7030033370411568, |
| "f1-score": 0.7688564476885644, |
| "support": 2697.0 |
| }, |
| "Negative": { |
| "precision": 0.5780691299165673, |
| "recall": 0.7784911717495987, |
| "f1-score": 0.6634746922024624, |
| "support": 623.0 |
| }, |
| "accuracy": 0.7111111111111111, |
| "macro avg": { |
| "precision": 0.6669083594696962, |
| "recall": 0.7251874351336417, |
| "f1-score": 0.6869602136807388, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7390612855879498, |
| "recall": 0.7111111111111111, |
| "f1-score": 0.7174753297814658, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "QUALITY": { |
| "macro_f1": 0.6921391557031549, |
| "accuracy": 0.7277777777777777, |
| "confusion_matrix": [ |
| [ |
| 1961, |
| 448, |
| 112 |
| ], |
| [ |
| 536, |
| 1112, |
| 61 |
| ], |
| [ |
| 42, |
| 26, |
| 202 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.7723513194170933, |
| "recall": 0.777865926219754, |
| "f1-score": 0.775098814229249, |
| "support": 2521.0 |
| }, |
| "Positive": { |
| "precision": 0.7011349306431274, |
| "recall": 0.6506729081334114, |
| "f1-score": 0.6749620637329287, |
| "support": 1709.0 |
| }, |
| "Negative": { |
| "precision": 0.5386666666666666, |
| "recall": 0.7481481481481481, |
| "f1-score": 0.6263565891472869, |
| "support": 270.0 |
| }, |
| "accuracy": 0.7277777777777777, |
| "macro avg": { |
| "precision": 0.6707176389089625, |
| "recall": 0.7255623275004378, |
| "f1-score": 0.6921391557031549, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7312838383821326, |
| "recall": 0.7277777777777777, |
| "f1-score": 0.7281445681469509, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "APPEARANCE": { |
| "macro_f1": 0.6134243985372673, |
| "accuracy": 0.7042222222222222, |
| "confusion_matrix": [ |
| [ |
| 1902, |
| 472, |
| 129 |
| ], |
| [ |
| 558, |
| 1161, |
| 89 |
| ], |
| [ |
| 45, |
| 38, |
| 106 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.7592814371257485, |
| "recall": 0.7598881342389133, |
| "f1-score": 0.7595846645367412, |
| "support": 2503.0 |
| }, |
| "Positive": { |
| "precision": 0.6947935368043088, |
| "recall": 0.6421460176991151, |
| "f1-score": 0.6674331704512791, |
| "support": 1808.0 |
| }, |
| "Negative": { |
| "precision": 0.3271604938271605, |
| "recall": 0.5608465608465608, |
| "f1-score": 0.41325536062378165, |
| "support": 189.0 |
| }, |
| "accuracy": 0.7042222222222222, |
| "macro avg": { |
| "precision": 0.5937451559190726, |
| "recall": 0.6542935709281964, |
| "f1-score": 0.6134243985372673, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.715222552222505, |
| "recall": 0.7042222222222222, |
| "f1-score": 0.7080144112598379, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "STYLE": { |
| "macro_f1": 0.651480797158933, |
| "accuracy": 0.7168888888888889, |
| "confusion_matrix": [ |
| [ |
| 1722, |
| 406, |
| 112 |
| ], |
| [ |
| 563, |
| 1356, |
| 106 |
| ], |
| [ |
| 47, |
| 40, |
| 148 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.7384219554030875, |
| "recall": 0.76875, |
| "f1-score": 0.7532808398950132, |
| "support": 2240.0 |
| }, |
| "Positive": { |
| "precision": 0.7524972253052165, |
| "recall": 0.6696296296296296, |
| "f1-score": 0.7086490723804547, |
| "support": 2025.0 |
| }, |
| "Negative": { |
| "precision": 0.40437158469945356, |
| "recall": 0.6297872340425532, |
| "f1-score": 0.4925124792013311, |
| "support": 235.0 |
| }, |
| "accuracy": 0.7168888888888889, |
| "macro avg": { |
| "precision": 0.6317635884692525, |
| "recall": 0.6893889545573942, |
| "f1-score": 0.651480797158933, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.7273109741667446, |
| "recall": 0.7168888888888889, |
| "f1-score": 0.7195786412327918, |
| "support": 4500.0 |
| } |
| } |
| }, |
| "VALUE": { |
| "macro_f1": 0.6489411814960196, |
| "accuracy": 0.8326666666666667, |
| "confusion_matrix": [ |
| [ |
| 3129, |
| 155, |
| 156 |
| ], |
| [ |
| 331, |
| 534, |
| 42 |
| ], |
| [ |
| 46, |
| 23, |
| 84 |
| ] |
| ], |
| "report": { |
| "Not_Mentioned": { |
| "precision": 0.892470051340559, |
| "recall": 0.9095930232558139, |
| "f1-score": 0.9009501871580766, |
| "support": 3440.0 |
| }, |
| "Positive": { |
| "precision": 0.75, |
| "recall": 0.5887541345093715, |
| "f1-score": 0.6596664607782582, |
| "support": 907.0 |
| }, |
| "Negative": { |
| "precision": 0.2978723404255319, |
| "recall": 0.5490196078431373, |
| "f1-score": 0.38620689655172413, |
| "support": 153.0 |
| }, |
| "accuracy": 0.8326666666666667, |
| "macro avg": { |
| "precision": 0.6467807972553636, |
| "recall": 0.6824555885361075, |
| "f1-score": 0.6489411814960196, |
| "support": 4500.0 |
| }, |
| "weighted avg": { |
| "precision": 0.8435380988214732, |
| "recall": 0.8326666666666667, |
| "f1-score": 0.8348168397604616, |
| "support": 4500.0 |
| } |
| } |
| } |
| }, |
| "overall": { |
| "mean_macro_f1": 0.6694071208141467, |
| "mean_accuracy": 0.7348518518518518 |
| } |
| }, |
| "note": "The Proposed model jointly trains per-aspect heads and an overall sentiment head on the shared fused representation. The overall_head result is the primary overall metric. The aggregated_to_overall result (voting from per-aspect predictions) is included for reference. Baseline 3 vs Proposed isolates the marginal value of metadata cross-attention fusion." |
| } |
