feat/add metrics for model training

kaggle/run-evaluation.ipynb

@@ -0,0 +1,151 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "99852947",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!git clone https://github.com/CfM47/ML-Project.git"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "89e0509f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2ba6b9c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!ls /kaggle/input/training-notebook-output"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05fff357",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "segmentation_root = Path('/kaggle/input/segmentations-images-automl/pictures')\n",
+    "\n",
+    "TRAIN_LABELED = segmentation_root / 'vega_3_tescan_labeled_images'\n",
+    "TRAIN_UNLABELED = segmentation_root / 'vega_3_tescan_unlabeled_images'\n",
+    "TEST_UNLABELED = segmentation_root / 'sampled_unlabeled'\n",
+    "TEST_LABELED = segmentation_root / 'sampled_labeled'\n",
+    "\n",
+    "# Pretrained model from previous training run\n",
+    "PRETRAINED_MODEL = Path('/kaggle/input/training-notebook-output/swin_results/model.pt')\n",
+    "\n",
+    "WORKING_DIR = Path('/kaggle/working')\n",
+    "PROJECT_ROOT = WORKING_DIR / 'ML-Project'\n",
+    "OUTPUT_DIR = WORKING_DIR / 'swin_results'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bbcc6efc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if str(PROJECT_ROOT) not in sys.path:\n",
+    "    sys.path.insert(0, str(PROJECT_ROOT))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bd42e98a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from model.swin.config import SwinTrainingConfig\n",
+    "\n",
+    "config = SwinTrainingConfig(\n",
+    "    output_dir=OUTPUT_DIR,\n",
+    "    device='auto',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39900be6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from model.swin.train import run_evaluation_only\n",
+    "\n",
+    "result = run_evaluation_only(\n",
+    "    TRAIN_UNLABELED,\n",
+    "    TRAIN_LABELED,\n",
+    "    TEST_UNLABELED,\n",
+    "    TEST_LABELED,\n",
+    "    PRETRAINED_MODEL,\n",
+    "    config=config,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3d554aad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result.predictions_figure"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3a5d834b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"Full Test Set Metrics:\")\n",
+    "for metric_name, value in result.test_metrics.full_metrics.items():\n",
+    "    print(f\"  {metric_name}: {value:.4f}\")\n",
+    "\n",
+    "print(\"\\nLeave-Two-Out Summary:\")\n",
+    "print(f\"  Subsets: {result.test_metrics.num_subsets} of size {result.test_metrics.subset_size}\")\n",
+    "for metric_name in sorted(result.test_metrics.subset_means.keys()):\n",
+    "    mean_val = result.test_metrics.subset_means[metric_name]\n",
+    "    std_val = result.test_metrics.subset_stds[metric_name]\n",
+    "    print(f\"  {metric_name}: {mean_val:.4f} ± {std_val:.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e5f6g7h8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Display all histograms\n",
+    "for metric_name, fig in sorted(result.histograms.items()):\n",
+    "    print(f'\\n{metric_name}:')\n",
+    "    display(fig)"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

kaggle/run-training.ipynb

@@ -85,7 +85,7 @@
    "source": [
     "from model.swin.train import run_final_training\n",
      "\n",
-     "model, test_metrics, mask_pairs, predictions_fig, loss_curves_fig = run_final_training(\n",
+     "result = run_final_training(\n",
      "    TRAIN_UNLABELED,\n",
      "    TRAIN_LABELED,\n",
      "    TEST_UNLABELED,\n",
@@ -101,7 +101,7 @@
     "metadata": {},
     "outputs": [],
     "source": [
-     "predictions_fig"
+     "result.predictions_figure"
     ]
    },
    {
@@ -111,8 +111,8 @@
     "metadata": {},
     "outputs": [],
     "source": [
-     "if loss_curves_fig is not None:\n",
-     "    display(loss_curves_fig)\n",
+     "if result.loss_curves_figure is not None:\n",
+     "    display(result.loss_curves_figure)\n",
      "else:\n",
      "    print('Loss curves not available (training history validation failed)')"
     ]
@@ -124,9 +124,29 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "print(\"Test Metrics:\")\n",
-    "for metric_name, value in test_metrics.items():\n",
-    "    print(f\"  {metric_name}: {value:.4f}\")"
+    "print(\"Full Test Set Metrics:\")\n",
+    "for metric_name, value in result.test_metrics.full_metrics.items():\n",
+    "    print(f\"  {metric_name}: {value:.4f}\")\n",
+    "\n",
+    "print(\"\\nLeave-Two-Out Summary:\")\n",
+    "print(f\"  Subsets: {result.test_metrics.num_subsets} of size {result.test_metrics.subset_size}\")\n",
+    "for metric_name in sorted(result.test_metrics.subset_means.keys()):\n",
+    "    mean_val = result.test_metrics.subset_means[metric_name]\n",
+    "    std_val = result.test_metrics.subset_stds[metric_name]\n",
+    "    print(f\"  {metric_name}: {mean_val:.4f} ± {std_val:.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e5f6g7h8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Display all histograms\n",
+    "for metric_name, fig in sorted(result.histograms.items()):\n",
+    "    print(f'\\n{metric_name}:')\n",
+    "    display(fig)"
    ]
   }
  ],

model/swin/__init__.py

@@ -1,10 +1,15 @@
 """Swin model training and validation submodule."""
 
 from model.swin.config import SwinTrainingConfig
-from model.swin.train import run_final_training, run_percentage_validation
+from model.swin.train import (
+    run_evaluation_only,
+    run_final_training,
+    run_percentage_validation,
+)
 
 __all__ = [
     "SwinTrainingConfig",
     "run_percentage_validation",
     "run_final_training",
+    "run_evaluation_only",
 ]