import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
from sklearn.metrics import (
f1_score, precision_score, recall_score,
accuracy_score, roc_auc_score, confusion_matrix, classification_report
)Each CSV should have columns: prob_neg, prob_pos, y_true
experiments = {
"CNN10 + synth / 13b": "../output/results/predictions/cnn10/all/13b_cnn10eval_file_probs.csv",
"CNN10 + synth / 14a": "../output/results/predictions/cnn10/all/14a_cnn10eval_file_probs.csv",
"CNN10 no synth / 13b": "../output/results/predictions/cnn10/mefou/13b_cnn10eval_file_probs.csv",
"CNN10 no synth / 14a": "../output/results/predictions/cnn10/mefou/14a_cnn10eval_file_probs.csv",
"CNN12 + synth / 13b": "../output/results/predictions/cnn12/all/13b_cnn12eval_file_probs.csv",
"CNN12 + synth / 14a": "../output/results/predictions/cnn12/all/14a_cnn12eval_file_probs.csv",
"CNN12 no synth / 13b": "../output/results/predictions/cnn12/mefou/13b_cnn12eval_file_probs.csv",
"CNN12 no synth / 14a": "../output/results/predictions/cnn12/mefou/14a_cnn12eval_file_probs.csv",
"SVM + synth / 13b": [
"../output/results/predictions/svm/all/sample_13b_1_predictions.txt",
"../output/results/predictions/svm/all/sample_13b_1_y_test.csv",
"../output/results/predictions/svm/all/13b_1_frame-info.csv",
"../output/results/predictions/svm/all/sample_13b_2_predictions.txt",
"../output/results/predictions/svm/all/sample_13b_2_y_test.csv",
"../output/results/predictions/svm/all/13b_2_frame-info.csv",
],
"SVM + synth / 14a": [
"../output/results/predictions/svm/all/sample_14a_predictions.txt",
"../output/results/predictions/svm/all/sample_14a_y_test.csv",
"../output/results/predictions/svm/all/14a_frame-info.csv",
],
"SVM no synth / 13b": [
"../output/results/predictions/svm/mefou/sample_13b_1_predictions.txt",
"../output/results/predictions/svm/mefou/sample_13b_1_y_test.csv",
"../output/results/predictions/svm/mefou/13b_1_frame-info.csv",
"../output/results/predictions/svm/mefou/sample_13b_2_predictions.txt",
"../output/results/predictions/svm/mefou/sample_13b_2_y_test.csv",
"../output/results/predictions/svm/mefou/13b_2_frame-info.csv",
],
"SVM no synth / 14a": [
"../output/results/predictions/svm/mefou/sample_14a_predictions.txt",
"../output/results/predictions/svm/mefou/sample_14a_y_test.csv",
"../output/results/predictions/svm/mefou/14a_frame-info.csv",
],
}experiments['SVM + synth / 13b']['../output/results/predictions/svm/all/sample_13b_1_predictions.txt',
'../output/results/predictions/svm/all/sample_13b_1_y_test.csv',
'../output/results/predictions/svm/all/13b_1_frame-info.csv',
'../output/results/predictions/svm/all/sample_13b_2_predictions.txt',
'../output/results/predictions/svm/all/sample_13b_2_y_test.csv',
'../output/results/predictions/svm/all/13b_2_frame-info.csv']from numpy import loadtxt
def read_preds(pred_file):
preds = pd.read_csv(pred_file, header=None, names=["pred_label"])
#print(
# f"Loaded {len(preds)} predictions from {pred_file} "
# f"of type {type(preds)} and shape {preds.shape}"
#)
#print(f"Missing values in preds: {preds.isna().sum()}")
return preds
def read_true(true_file):
y_true = pd.read_csv(true_file, header=0)
#print(
# f"Loaded {len(y_true)} true labels from {true_file} "
# f"of type {type(y_true)} and shape {y_true.shape}"
#)
#print(f"Missing values in y_true: {y_true.isna().sum()}")
#print(f"First value in y_true: {y_true.iloc[0]}")
return y_true
def read_frame_info(frame_info_file):
frame_info = pd.read_csv(frame_info_file)
#print(
# f"Loaded {len(frame_info)} frame rows from {frame_info_file} "
# f"with {frame_info['file_path'].nunique()} unique audio files"
#)
return frame_info
def load_svm_predictions(paths, file_threshold):
"""Aggregate SVM chunk predictions to one row per audio file."""
if len(paths) == 3:
pred_file, true_file, frame_info_file = paths
preds = read_preds(pred_file)
y_true = read_true(true_file)
frame_info = read_frame_info(frame_info_file)
else:
pred_file_1, true_file_1, frame_info_file_1, pred_file_2, true_file_2, frame_info_file_2 = paths
preds_1 = read_preds(pred_file_1)
y_true_1 = read_true(true_file_1)
frame_info_1 = read_frame_info(frame_info_file_1)
preds_2 = read_preds(pred_file_2)
y_true_2 = read_true(true_file_2)
frame_info_2 = read_frame_info(frame_info_file_2)
preds = pd.concat([preds_1, preds_2], axis=0)
y_true = pd.concat([y_true_1, y_true_2], axis=0)
frame_info = pd.concat([frame_info_1, frame_info_2], axis=0)
if not (len(preds) == len(y_true) == len(frame_info)):
raise ValueError(
"SVM predictions, labels, and frame info must have the same number of rows. "
f"Got {len(preds)=}, {len(y_true)=}, {len(frame_info)=}"
)
label_map = {"background": 0, "chimpanze": 1}
chunk_df = frame_info[["file_path"]].copy()
chunk_df["preds"] = preds["pred_label"].map(label_map)
chunk_df["y_true"] = y_true.iloc[:, 0].map(label_map)
if chunk_df[["preds", "y_true"]].isna().any().any():
raise ValueError("Unexpected SVM label encountered while mapping predictions.")
file_df = (
chunk_df.groupby("file_path", as_index=False)
.agg(
prob_pos=("preds", "mean"),
true_pos_rate=("y_true", "mean"),
n_chunks=("preds", "size"),
)
)
file_df["prob_neg"] = 1 - file_df["prob_pos"]
file_df["preds"] = (file_df["prob_pos"] >= file_threshold).astype(int)
file_df["y_true"] = (file_df["true_pos_rate"] >= file_threshold).astype(int)
return file_dfFILE_THRESHOLD = 0.5
results = []
raw_data = {}
for name, path in experiments.items():
if isinstance(path, (list, tuple)):
df = load_svm_predictions(path, FILE_THRESHOLD)
else:
df = pd.read_csv(path).rename(columns={"predicts": "preds"})
if "prob_pos" in df.columns:
df["preds"] = (df["prob_pos"] >= FILE_THRESHOLD).astype(int)
parts = name.split(" / ")
config = parts[0]
test_set = parts[1]
model = "CNN10" if "CNN10" in config else "CNN12" if "CNN12" in config else "SVM"
synthetic = "Yes" if "+ synth" in config else "No"
row = {
"experiment": name,
"model": model,
"synthetic": synthetic,
"test_set": test_set,
"n_files": len(df),
"n_positive": int((df["y_true"] == 1).sum()),
"n_negative": int((df["y_true"] == 0).sum()),
"precision": precision_score(df["y_true"], df["preds"], pos_label=1, zero_division=0),
"recall": recall_score(df["y_true"], df["preds"], pos_label=1, zero_division=0),
"f1": f1_score(df["y_true"], df["preds"], pos_label=1, zero_division=0),
"precision-macro": precision_score(
df["y_true"], df["preds"], pos_label=1, zero_division=0, average="macro"
),
"recall-macro": recall_score(
df["y_true"], df["preds"], pos_label=1, zero_division=0, average="macro"
),
"f1-macro": f1_score(
df["y_true"], df["preds"], pos_label=1, zero_division=0, average="macro"
),
"accuracy": accuracy_score(df["y_true"], df["preds"]),
}
if "prob_pos" in df.columns:
row["auc"] = roc_auc_score(df["y_true"], df["prob_pos"]) if len(np.unique(df["y_true"])) > 1 else np.nan
else:
row["auc"] = np.nan
results.append(row)
print(
f"Loaded {name} with {len(df)} audio files, "
f"{row['n_positive']} positive and {row['n_negative']} negative samples"
)
print(row)
raw_data[name] = {
"preds": df["preds"].values,
"y_true": df["y_true"].values,
"prob_pos": df["prob_pos"].values if "prob_pos" in df.columns else df["preds"].values,
}
results_df = pd.DataFrame(results)
print(f"Loaded {len(results)} experiments")Loaded CNN10 + synth / 13b with 1317 audio files, 41 positive and 1276 negative samples
{'experiment': 'CNN10 + synth / 13b', 'model': 'CNN10', 'synthetic': 'Yes', 'test_set': '13b', 'n_files': 1317, 'n_positive': 41, 'n_negative': 1276, 'precision': 0.603448275862069, 'recall': 0.8536585365853658, 'f1': 0.7070707070707071, 'precision-macro': 0.799341294404426, 'recall-macro': 0.9178167291077299, 'f1-macro': 0.847815432430817, 'accuracy': 0.9779802581624905, 'auc': 0.9911117057879042}
Loaded CNN10 + synth / 14a with 968 audio files, 8 positive and 960 negative samples
{'experiment': 'CNN10 + synth / 14a', 'model': 'CNN10', 'synthetic': 'Yes', 'test_set': '14a', 'n_files': 968, 'n_positive': 8, 'n_negative': 960, 'precision': 0.5384615384615384, 'recall': 0.875, 'f1': 0.6666666666666666, 'precision-macro': 0.7687072090213452, 'recall-macro': 0.934375, 'f1-macro': 0.8315056570931245, 'accuracy': 0.9927685950413223, 'auc': 0.9947916666666666}
Loaded CNN10 no synth / 13b with 1317 audio files, 41 positive and 1276 negative samples
{'experiment': 'CNN10 no synth / 13b', 'model': 'CNN10', 'synthetic': 'No', 'test_set': '13b', 'n_files': 1317, 'n_positive': 41, 'n_negative': 1276, 'precision': 0.7435897435897436, 'recall': 0.7073170731707317, 'f1': 0.725, 'precision-macro': 0.8671000361141206, 'recall-macro': 0.8497400412875602, 'f1-macro': 0.8581930305403289, 'accuracy': 0.9832953682611997, 'auc': 0.9926791039070264}
Loaded CNN10 no synth / 14a with 968 audio files, 8 positive and 960 negative samples
{'experiment': 'CNN10 no synth / 14a', 'model': 'CNN10', 'synthetic': 'No', 'test_set': '14a', 'n_files': 968, 'n_positive': 8, 'n_negative': 960, 'precision': 0.7142857142857143, 'recall': 0.625, 'f1': 0.6666666666666666, 'precision-macro': 0.8555819830533671, 'recall-macro': 0.8114583333333334, 'f1-macro': 0.8320319278153739, 'accuracy': 0.9948347107438017, 'auc': 0.9947916666666667}
Loaded CNN12 + synth / 13b with 1317 audio files, 41 positive and 1276 negative samples
{'experiment': 'CNN12 + synth / 13b', 'model': 'CNN12', 'synthetic': 'Yes', 'test_set': '13b', 'n_files': 1317, 'n_positive': 41, 'n_negative': 1276, 'precision': 0.6829268292682927, 'recall': 0.6829268292682927, 'f1': 0.6829268292682927, 'precision-macro': 0.836369370746999, 'recall-macro': 0.836369370746999, 'f1-macro': 0.836369370746999, 'accuracy': 0.9802581624905087, 'auc': 0.9904618090068048}
Loaded CNN12 + synth / 14a with 968 audio files, 8 positive and 960 negative samples
{'experiment': 'CNN12 + synth / 14a', 'model': 'CNN12', 'synthetic': 'Yes', 'test_set': '14a', 'n_files': 968, 'n_positive': 8, 'n_negative': 960, 'precision': 0.6666666666666666, 'recall': 0.75, 'f1': 0.7058823529411765, 'precision-macro': 0.832290580465763, 'recall-macro': 0.8734375, 'f1-macro': 0.8516384146154554, 'accuracy': 0.9948347107438017, 'auc': 0.9916666666666667}
Loaded CNN12 no synth / 13b with 1317 audio files, 41 positive and 1276 negative samples
{'experiment': 'CNN12 no synth / 13b', 'model': 'CNN12', 'synthetic': 'No', 'test_set': '13b', 'n_files': 1317, 'n_positive': 41, 'n_negative': 1276, 'precision': 0.7435897435897436, 'recall': 0.7073170731707317, 'f1': 0.725, 'precision-macro': 0.8671000361141206, 'recall-macro': 0.8497400412875602, 'f1-macro': 0.8581930305403289, 'accuracy': 0.9832953682611997, 'auc': 0.990136860616255}
Loaded CNN12 no synth / 14a with 968 audio files, 8 positive and 960 negative samples
{'experiment': 'CNN12 no synth / 14a', 'model': 'CNN12', 'synthetic': 'No', 'test_set': '14a', 'n_files': 968, 'n_positive': 8, 'n_negative': 960, 'precision': 0.7142857142857143, 'recall': 0.625, 'f1': 0.6666666666666666, 'precision-macro': 0.8555819830533671, 'recall-macro': 0.8114583333333334, 'f1-macro': 0.8320319278153739, 'accuracy': 0.9948347107438017, 'auc': 0.996875}
Loaded SVM + synth / 13b with 1317 audio files, 41 positive and 1276 negative samples
{'experiment': 'SVM + synth / 13b', 'model': 'SVM', 'synthetic': 'Yes', 'test_set': '13b', 'n_files': 1317, 'n_positive': 41, 'n_negative': 1276, 'precision': 0.09090909090909091, 'recall': 0.7317073170731707, 'f1': 0.16172506738544473, 'precision-macro': 0.5398821037118909, 'recall-macro': 0.7482987996024161, 'f1-macro': 0.5121484373604202, 'accuracy': 0.7638572513287776, 'auc': 0.7668782016973773}
Loaded SVM + synth / 14a with 968 audio files, 8 positive and 960 negative samples
{'experiment': 'SVM + synth / 14a', 'model': 'SVM', 'synthetic': 'Yes', 'test_set': '14a', 'n_files': 968, 'n_positive': 8, 'n_negative': 960, 'precision': 1.0, 'recall': 0.5, 'f1': 0.6666666666666666, 'precision-macro': 0.9979253112033195, 'recall-macro': 0.75, 'f1-macro': 0.8322938322938322, 'accuracy': 0.9958677685950413, 'auc': 0.99453125}
Loaded SVM no synth / 13b with 1317 audio files, 41 positive and 1276 negative samples
{'experiment': 'SVM no synth / 13b', 'model': 'SVM', 'synthetic': 'No', 'test_set': '13b', 'n_files': 1317, 'n_positive': 41, 'n_negative': 1276, 'precision': 0.03644314868804665, 'recall': 0.6097560975609756, 'f1': 0.0687757909215956, 'precision-macro': 0.5055432859129615, 'recall-macro': 0.5458655095955348, 'f1-macro': 0.35688396258193045, 'accuracy': 0.4859529233105543, 'auc': 0.5123384815352856}
Loaded SVM no synth / 14a with 968 audio files, 8 positive and 960 negative samples
{'experiment': 'SVM no synth / 14a', 'model': 'SVM', 'synthetic': 'No', 'test_set': '14a', 'n_files': 968, 'n_positive': 8, 'n_negative': 960, 'precision': 0.02112676056338028, 'recall': 0.375, 'f1': 0.04, 'precision-macro': 0.5075367458991236, 'recall-macro': 0.6151041666666667, 'f1-macro': 0.4796864501679731, 'accuracy': 0.8512396694214877, 'auc': 0.7221354166666667}
Loaded 12 experiments
def get_confusion_matrix(y_test,preds, file=None):
cm = confusion_matrix(y_test,preds, normalize='true')
return cmdisplay_cols = [
"model", "synthetic", "test_set",
"f1", "precision", "recall",
"f1-macro", "precision-macro", "recall-macro",
"auc",
# "best_f1", "best_threshold"
]
summary = results_df[display_cols].sort_values(
["test_set", "f1"], ascending=[True, False]
)
print(summary.to_string(index=False, float_format="%.4f"))model synthetic test_set f1 precision recall f1-macro precision-macro recall-macro auc
CNN10 No 13b 0.7250 0.7436 0.7073 0.8582 0.8671 0.8497 0.9927
CNN12 No 13b 0.7250 0.7436 0.7073 0.8582 0.8671 0.8497 0.9901
CNN10 Yes 13b 0.7071 0.6034 0.8537 0.8478 0.7993 0.9178 0.9911
CNN12 Yes 13b 0.6829 0.6829 0.6829 0.8364 0.8364 0.8364 0.9905
SVM Yes 13b 0.1617 0.0909 0.7317 0.5121 0.5399 0.7483 0.7669
SVM No 13b 0.0688 0.0364 0.6098 0.3569 0.5055 0.5459 0.5123
CNN12 Yes 14a 0.7059 0.6667 0.7500 0.8516 0.8323 0.8734 0.9917
CNN10 Yes 14a 0.6667 0.5385 0.8750 0.8315 0.7687 0.9344 0.9948
CNN10 No 14a 0.6667 0.7143 0.6250 0.8320 0.8556 0.8115 0.9948
CNN12 No 14a 0.6667 0.7143 0.6250 0.8320 0.8556 0.8115 0.9969
SVM Yes 14a 0.6667 1.0000 0.5000 0.8323 0.9979 0.7500 0.9945
SVM No 14a 0.0400 0.0211 0.3750 0.4797 0.5075 0.6151 0.7221fig, ax = plt.subplots(figsize=(8, 6))
markers = {"CNN10": "o", "CNN12": "s", "SVM": "^"}
colors = {"Yes": "#4C72B0", "No": "#DD8452"}
edge_colors = {"13b": "black", "14a": "red"}
for _, row in results_df.iterrows():
ax.scatter(
row["recall-macro"], row["precision-macro"],
marker=markers[row["model"]],
c=colors[row["synthetic"]],
edgecolors=edge_colors[row["test_set"]],
s=150, linewidths=2, zorder=5
)
ax.annotate(
f"{row['model']}\n{row['test_set']}",
(row["recall-macro"], row["precision-macro"]),
textcoords="offset points",
xytext=(10, 5), fontsize=8
)
legend_elements = [
Line2D([0], [0], marker="o", color="w",
markerfacecolor="gray", markersize=10, label="CNN10"),
Line2D([0], [0], marker="s", color="w",
markerfacecolor="gray", markersize=10, label="CNN12"),
Line2D([0], [0], marker="^", color="w",
markerfacecolor="gray", markersize=10, label="SVM"),
Line2D([0], [0], marker="o", color="w",
markerfacecolor="#4C72B0", markersize=10, label="+ synthetic"),
Line2D([0], [0], marker="o", color="w",
markerfacecolor="#DD8452", markersize=10, label="no synthetic"),
Line2D([0], [0], marker="o", color="w",
markeredgecolor="black", markersize=10,
markeredgewidth=2, label="13b"),
Line2D([0], [0], marker="o", color="w",
markeredgecolor="red", markersize=10,
markeredgewidth=2, label="14a"),
]
ax.legend(handles=legend_elements, loc="lower left")
ax.set_xlabel("Recall (chimp)")
ax.set_ylabel("Precision (chimp)")
ax.set_title(f"Precision vs Recall @ file threshold {FILE_THRESHOLD:.2f}")
ax.set_xlim(-0.05, 1.05)
ax.set_ylim(-0.05, 1.05)
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
n_cols = 4
n_rows = int(np.ceil(len(raw_data) / n_cols))
fig, axes = plt.subplots(n_rows, n_cols, figsize=(20, 4 * n_rows))
axes = np.atleast_1d(axes).reshape(n_rows, n_cols)
for idx, (name, data) in enumerate(raw_data.items()):
row = idx // n_cols
col = idx % n_cols
ax = axes[row][col]
prob_pos = data["prob_pos"]
y_true = data["y_true"]
ax.hist(
prob_pos[y_true == 0], bins=30, alpha=0.6,
label="Background", density=True, color="#4C72B0"
)
ax.hist(
prob_pos[y_true == 1], bins=30, alpha=0.6,
label="Chimp", density=True, color="#DD8452"
)
ax.axvline(FILE_THRESHOLD, color="red", linestyle=":", linewidth=1.5)
ax.set_title(name, fontsize=9)
ax.set_xlabel("File score")
if col == 0:
ax.set_ylabel("Density")
if idx == 0:
ax.legend(fontsize=8)
for idx in range(len(raw_data), n_rows * n_cols):
axes[idx // n_cols][idx % n_cols].axis("off")
plt.suptitle("File Score Distributions by Experiment", fontsize=14, y=1.02)
plt.tight_layout()
plt.show()
n_cols = 4
n_rows = int(np.ceil(len(raw_data) / n_cols))
fig, axes = plt.subplots(n_rows, n_cols, figsize=(20, 4 * n_rows))
axes = np.atleast_1d(axes).reshape(n_rows, n_cols)
for idx, (name, data) in enumerate(raw_data.items()):
row = idx // n_cols
col = idx % n_cols
ax = axes[row][col]
preds = data["preds"]
y_true = data["y_true"]
cm = confusion_matrix(y_true, preds, normalize="true")
# Normalize by row (true label) → each row sums to 1
cm_norm = cm.astype(float) / cm.sum(axis=1, keepdims=True)
im = ax.imshow(cm_norm, cmap="Blues", aspect="auto", vmin=0, vmax=1)
ax.set_xticks([0, 1])
ax.set_yticks([0, 1])
ax.set_xticklabels(["bg", "chimp"])
ax.set_yticklabels(["bg", "chimp"])
ax.set_xlabel("Predicted", fontsize=8)
ax.set_ylabel("True", fontsize=8)
ax.set_title(name, fontsize=9)
for i in range(2):
for j in range(2):
val = cm_norm[i, j]
raw = cm[i, j]
color = "white" if val > 0.5 else "black"
ax.text(
j, i, "{:0.4f}".format(val),
ha="center", va="center",
color=color, fontsize=10
)
for idx in range(len(raw_data), n_rows * n_cols):
axes[idx // n_cols][idx % n_cols].axis("off")
plt.suptitle(
f"File-level Confusion Matrices @ threshold {FILE_THRESHOLD:.2f}",
fontsize=14,
y=1.02,
)
plt.tight_layout()
plt.show()