{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "intro",
   "metadata": {},
   "source": [
    "# 02 - Model comparison\n",
    "\n",
    "Train all four MaldiDeepKit classifiers on the **MALDI-Kleb-AI** Amikacin task and compare AUROC. See notebook 01 for the dataset cache; first run downloads 370 MB once.\n",
    "\n",
    "Each classifier exposes a `from_spectrum(bin_width, input_dim, **overrides)` factory that scales architectural defaults (kernel sizes, etc.) for a given binning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "load",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-04-28T09:51:49.087891Z",
     "iopub.status.busy": "2026-04-28T09:51:49.087791Z",
     "iopub.status.idle": "2026-04-28T09:52:08.991917Z",
     "shell.execute_reply": "2026-04-28T09:52:08.991134Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a629caebd7ba4e74a29c562dd01d9bbd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Processing spectra:   0%|          | 0/743 [00:00<?, ?spectrum/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X: (741, 6000) | prevalence(R): 49.80%\n"
     ]
    }
   ],
   "source": [
    "import sys, pathlib\n",
    "sys.path.insert(0, str(pathlib.Path.cwd().parent))\n",
    "from notebooks._demo import binary_labels, load_maldi_kleb_ai\n",
    "\n",
    "demo = load_maldi_kleb_ai(antibiotic='Amikacin', verbose=True)\n",
    "X, y = binary_labels(demo)\n",
    "print(f'X: {X.shape} | prevalence(R): {y.mean():.2%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "split",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-04-28T09:52:08.993231Z",
     "iopub.status.busy": "2026-04-28T09:52:08.993093Z",
     "iopub.status.idle": "2026-04-28T09:52:09.171736Z",
     "shell.execute_reply": "2026-04-28T09:52:09.171026Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.25, stratify=y, random_state=0)\n",
    "input_dim = X.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fit",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-04-28T09:52:09.173361Z",
     "iopub.status.busy": "2026-04-28T09:52:09.173106Z",
     "iopub.status.idle": "2026-04-28T09:53:10.856070Z",
     "shell.execute_reply": "2026-04-28T09:53:10.855114Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        MLP  acc=0.747  auroc=0.826\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        CNN  acc=0.715  auroc=0.781\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     ResNet  acc=0.500  auroc=0.472\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformer  acc=0.500  auroc=0.499\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from maldideepkit import (\n",
    "    MaldiCNNClassifier,\n",
    "    MaldiMLPClassifier,\n",
    "    MaldiResNetClassifier,\n",
    "    MaldiTransformerClassifier,\n",
    ")\n",
    "\n",
    "common = dict(epochs=30, random_state=0)\n",
    "\n",
    "classifiers = {\n",
    "    'MLP':         MaldiMLPClassifier.from_spectrum(bin_width=3, input_dim=input_dim, **common),\n",
    "    'CNN':         MaldiCNNClassifier.from_spectrum(bin_width=3, input_dim=input_dim, **common),\n",
    "    'ResNet':      MaldiResNetClassifier.from_spectrum(bin_width=3, input_dim=input_dim, **common),\n",
    "    'Transformer': MaldiTransformerClassifier.from_spectrum(bin_width=3, input_dim=input_dim, **common),\n",
    "}\n",
    "\n",
    "results = {}\n",
    "for name, clf in classifiers.items():\n",
    "    clf.fit(X_tr, y_tr)\n",
    "    proba = clf.predict_proba(X_te)\n",
    "    results[name] = {\n",
    "        'accuracy': clf.score(X_te, y_te),\n",
    "        'auroc': roc_auc_score(y_te, proba[:, 1]),\n",
    "    }\n",
    "    print(f'{name:>11s}  acc={results[name][\"accuracy\"]:.3f}  auroc={results[name][\"auroc\"]:.3f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ensemble-md",
   "metadata": {},
   "source": [
    "> Note: with only ~555 training spectra the Transformer is comfortably below its convergence regime in 30 epochs. Its training recipe (`lr=3e-4`, 5-epoch warmup, weight-decay 0.05) pays off on larger datasets and longer schedules.\n",
    "\n",
    "## Mean-of-probabilities ensemble\n",
    "\n",
    "`SpectralEnsemble` averages `predict_proba` across fitted members. The shallow MLP / CNN / ResNet families have different inductive biases and often decorrelate enough for the ensemble to beat any individual."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ensemble",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-04-28T09:53:10.857673Z",
     "iopub.status.busy": "2026-04-28T09:53:10.857425Z",
     "iopub.status.idle": "2026-04-28T09:53:41.481430Z",
     "shell.execute_reply": "2026-04-28T09:53:41.480684Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Ensemble  acc=0.710  auroc=0.785\n"
     ]
    }
   ],
   "source": [
    "from maldideepkit.utils import SpectralEnsemble\n",
    "\n",
    "ensemble = SpectralEnsemble([\n",
    "    MaldiMLPClassifier(input_dim=input_dim, epochs=30, random_state=0),\n",
    "    MaldiCNNClassifier(input_dim=input_dim, epochs=30, random_state=1),\n",
    "    MaldiResNetClassifier(input_dim=input_dim, epochs=30, random_state=2),\n",
    "]).fit(X_tr, y_tr)\n",
    "\n",
    "proba = ensemble.predict_proba(X_te)\n",
    "print(f'   Ensemble  acc={(proba.argmax(axis=1) == y_te).mean():.3f}  auroc={roc_auc_score(y_te, proba[:, 1]):.3f}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "maldideepkit (3.10.12)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "2d77e91ea54d49c4a86358fa249d9fe8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "3b6ee40d478040a984b470bf809a33c0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_c83209418a80414a84b3369d6b67a8c7",
       "placeholder": "",
       "style": "IPY_MODEL_c94dcd366cb2415e9a17ed552cb58a3e",
       "tabbable": null,
       "tooltip": null,
       "value": "Processing spectra: 100%"
      }
     },
     "944e7109c7c24dc9b5296e66c9e57980": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_df8ca16c4fbd4e28a601177a145619b5",
       "placeholder": "",
       "style": "IPY_MODEL_2d77e91ea54d49c4a86358fa249d9fe8",
       "tabbable": null,
       "tooltip": null,
       "value": " 743/743 [00:00&lt;00:00, 3916.70spectrum/s]"
      }
     },
     "94e28165034045f3996446e100d00687": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "9e31fbbf46374c11893ade17fac3ecc4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a629caebd7ba4e74a29c562dd01d9bbd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_3b6ee40d478040a984b470bf809a33c0",
        "IPY_MODEL_fe99c304f1d24c9d8747392de6c3154c",
        "IPY_MODEL_944e7109c7c24dc9b5296e66c9e57980"
       ],
       "layout": "IPY_MODEL_e269de41173241d4b2037e6d546e7216",
       "tabbable": null,
       "tooltip": null
      }
     },
     "c83209418a80414a84b3369d6b67a8c7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c94dcd366cb2415e9a17ed552cb58a3e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "df8ca16c4fbd4e28a601177a145619b5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e269de41173241d4b2037e6d546e7216": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fe99c304f1d24c9d8747392de6c3154c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_9e31fbbf46374c11893ade17fac3ecc4",
       "max": 743,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_94e28165034045f3996446e100d00687",
       "tabbable": null,
       "tooltip": null,
       "value": 743
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}