Repository navigation
Expand file tree
/
Copy pathcompare_patch_optimization.py
More file actions
95 lines (81 loc) · 2.98 KB
/
Copy pathcompare_patch_optimization.py
File metadata and controls
95 lines (81 loc) · 2.98 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
import csv
import json
from pathlib import Path
import matplotlib.pyplot as plt
import numpy as np
OUTPUT_ROOT = Path(__file__).resolve().parent / "outputs"
EXPERIMENTS = [
"patch_transformer",
"patch_opt_g2_cls_global",
"patch_opt_g3_cls_global",
"patch_opt_g3_cls_noglobal",
"patch_opt_frequency_concat",
"patch_opt_frequency_gated",
"patch_opt_frequency_film",
"patch_opt_frequency_crossattn",
]
METRICS = [
"multiclass_accuracy",
"multiclass_macro_f1",
"binary_accuracy",
"binary_f1_fake",
"binary_recall_fake",
"binary_auc_fake",
]
def load_metrics(exp):
path = OUTPUT_ROOT / exp / "best_eval" / "metrics.json"
if not path.exists():
return None
return json.loads(path.read_text(encoding="utf-8"))
def main():
rows = []
for exp in EXPERIMENTS:
metrics = load_metrics(exp)
if metrics is None:
print(f"skip missing {exp}", flush=True)
continue
row = {"experiment": exp}
for key in METRICS:
row[key] = metrics.get(key)
rows.append(row)
out_dir = OUTPUT_ROOT / "patch_optimization_analysis"
out_dir.mkdir(parents=True, exist_ok=True)
with (out_dir / "patch_optimization_comparison.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=["experiment", *METRICS])
writer.writeheader()
writer.writerows(rows)
(out_dir / "patch_optimization_comparison.json").write_text(json.dumps(rows, indent=2), encoding="utf-8")
if not rows:
return
labels = [r["experiment"].replace("patch_opt_", "").replace("patch_transformer", "baseline") for r in rows]
x = np.arange(len(rows))
width = 0.36
fig, ax = plt.subplots(figsize=(14, 6))
ax.bar(x - width / 2, [r["multiclass_macro_f1"] for r in rows], width, label="Macro-F1")
ax.bar(x + width / 2, [r["binary_f1_fake"] for r in rows], width, label="Fake-F1")
ax.set_ylim(0, 1)
ax.set_ylabel("Score")
ax.set_title("Patch Transformer Optimization and Frequency Fusion")
ax.set_xticks(x)
ax.set_xticklabels(labels, rotation=30, ha="right")
ax.grid(axis="y", alpha=0.25)
ax.legend()
fig.tight_layout()
fig.savefig(out_dir / "patch_optimization_metric_bars.png", dpi=220)
plt.close(fig)
fig, ax = plt.subplots(figsize=(13, 6))
ax.plot(labels, [r["multiclass_macro_f1"] for r in rows], marker="o", label="Macro-F1")
ax.plot(labels, [r["binary_recall_fake"] for r in rows], marker="o", label="Fake Recall")
ax.plot(labels, [r["binary_auc_fake"] for r in rows], marker="o", label="AUC")
ax.set_ylim(0, 1)
ax.set_ylabel("Score")
ax.set_title("Patch/Frequency Fusion Metric Trend")
ax.tick_params(axis="x", rotation=30)
ax.grid(axis="y", alpha=0.25)
ax.legend()
fig.tight_layout()
fig.savefig(out_dir / "patch_optimization_metric_lines.png", dpi=220)
plt.close(fig)
print(json.dumps(rows, indent=2), flush=True)
if __name__ == "__main__":
main()