diff --git a/docs/api/models.rst b/docs/api/models.rst index 4187c123b..5d0c77eba 100644 --- a/docs/api/models.rst +++ b/docs/api/models.rst @@ -201,7 +201,6 @@ API Reference models/pyhealth.models.TCN models/pyhealth.models.TFMTokenizer models/pyhealth.models.GAN - models/pyhealth.models.VAE models/pyhealth.models.HALO models/pyhealth.models.GPT2 models/pyhealth.models.PromptEHR diff --git a/docs/api/models/pyhealth.models.VAE.rst b/docs/api/models/pyhealth.models.VAE.rst deleted file mode 100644 index 28822e069..000000000 --- a/docs/api/models/pyhealth.models.VAE.rst +++ /dev/null @@ -1,9 +0,0 @@ -pyhealth.models.VAE -=================================== - -The VAE model (treated as a regression task). - -.. autoclass:: pyhealth.models.VAE - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 9193c86c0..c04081a7b 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -214,8 +214,6 @@ These examples are located in ``examples/cxr/``. - Multi-label classification on ChestX-ray14 dataset (notebook) * - ``cxr/ChestXrayClassificationWithSaliency.ipynb`` - Chest X-ray classification with saliency maps (notebook) - * - ``cxr/chextXray_image_generation_VAE.py`` - - VAE for chest X-ray image generation * - ``cxr/ChestXray-image-generation-GAN.ipynb`` - GAN for chest X-ray image generation (notebook) @@ -328,8 +326,6 @@ Notebooks (Interactive) - Language model embeddings with OpenAI * - ``prepare_mapping.ipynb`` - Data preprocessing and mapping utilities - * - ``graph_torchvision_model.ipynb`` - - Using Torchvision models with graph data ---------- diff --git a/examples/cxr/chextXray_image_generation_VAE.py b/examples/cxr/chextXray_image_generation_VAE.py deleted file mode 100644 index 1d816dc1e..000000000 --- a/examples/cxr/chextXray_image_generation_VAE.py +++ /dev/null @@ -1,100 +0,0 @@ -from pyhealth.datasets import split_by_visit, get_dataloader -from pyhealth.trainer import Trainer -from pyhealth.datasets import COVID19CXRDataset -from pyhealth.models import VAE -from torchvision import transforms - -import torch -import numpy as np - -# step 1: load signal data -root = "/srv/local/data/COVID-19_Radiography_Dataset" -base_dataset = COVID19CXRDataset(root) - -# step 2: set task -sample_dataset = base_dataset.set_task() - -# the transformation automatically normalize the pixel intensity into [0, 1] -transform = transforms.Compose([ - transforms.Lambda(lambda x: x if x.shape[0] == 3 else x.repeat(3, 1, 1)), # only use the first channel - transforms.Resize((128, 128)), -]) - -def encode(sample): - sample["path"] = transform(sample["path"]) - return sample - -sample_dataset.set_transform(encode) - - -# split dataset -train_dataset, val_dataset, test_dataset = split_by_visit( - sample_dataset, [0.6, 0.2, 0.2] -) -train_dataloader = get_dataloader(train_dataset, batch_size=256, shuffle=True) -val_dataloader = get_dataloader(val_dataset, batch_size=256, shuffle=False) -test_dataloader = get_dataloader(test_dataset, batch_size=256, shuffle=False) - -data = next(iter(train_dataloader)) -print (data) - -print (data["path"][0].shape) - -print( - "loader size: train/val/test", - len(train_dataset), - len(val_dataset), - len(test_dataset), -) - -# STEP 3: define model -model = VAE( - dataset=sample_dataset, - input_channel=3, - input_size=128, - feature_keys=["path"], - label_key="path", - mode="regression", - hidden_dim = 128, -) - -# STEP 4: define trainer -trainer = Trainer(model=model, device="cuda:4", metrics=["kl_divergence", "mse", "mae"]) -trainer.train( - train_dataloader=train_dataloader, - val_dataloader=val_dataloader, - epochs=10, - monitor="kl_divergence", - monitor_criterion="min", - optimizer_params={"lr": 1e-3}, -) - -# # STEP 5: evaluate -# print(trainer.evaluate(test_dataloader)) - - -import matplotlib.pyplot as plt - -# EXP 1: check the real chestxray image and the reconstructed image -X, X_rec, _ = trainer.inference(test_dataloader) - -plt.figure() -plt.subplot(1, 2, 1) -plt.imshow(X[0].reshape(128, 128), cmap="gray") -plt.subplot(1, 2, 2) -plt.imshow(X_rec[0].reshape(128, 128), cmap="gray") -plt.savefig("chestxray_vae_comparison.png") - -# EXP 2: random images -model = trainer.model - -model.eval() -with torch.no_grad(): - x = np.random.normal(0, 1, 128) - x = x.astype(np.float32) - x = torch.from_numpy(x).to(trainer.device) - rec = model.decoder(x).detach().cpu().numpy() - rec = rec.reshape((128, 128)) - plt.figure() - plt.imshow(rec, cmap="gray") - plt.savefig("chestxray_vae_synthetic.png") \ No newline at end of file diff --git a/examples/graph_torchvision_model.ipynb b/examples/graph_torchvision_model.ipynb deleted file mode 100644 index faaf7cf9a..000000000 --- a/examples/graph_torchvision_model.ipynb +++ /dev/null @@ -1,472 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "21aab8ac", - "metadata": {}, - "outputs": [], - "source": [ - "\"\"\"\n", - "The following env works\n", - " - torch: 1.9.1\n", - " - torchvision: 0.10.0+cu102\n", - " - torch_sparse: 0.6.12\n", - " \n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "3cc9dcc6-9d43-47e9-a4d3-0ba29beb425e", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "PATH = '/home/chaoqiy2/github/PyHealth'\n", - "os.chdir(PATH)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "2e5c4ec0-0887-48c7-8f97-1ccafb401f2d", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/chaoqiy2/miniconda3/envs/moltext/lib/python3.7/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "from pyhealth.sampler import NeighborSampler\n", - "from pyhealth.models import Graph_TorchvisionModel\n", - "from pyhealth.models import GCN\n", - "from torchvision import transforms\n", - "from pyhealth.datasets import COVID19CXRDataset" - ] - }, - { - "cell_type": "markdown", - "id": "a80901ce-1077-4272-a087-f728683bcdad", - "metadata": {}, - "source": [ - "## Load Dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "e1c54991-ed56-400f-a82e-3609ef0f953c", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from pyhealth.datasets import COVID19CXRDataset\n", - "\n", - "root = \"/srv/local/data/COVID-19_Radiography_Dataset\"\n", - "base_dataset = COVID19CXRDataset(root)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "46475e1d-e31a-48f1-848d-457a9c2eee48", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "COVID19CXRClassification(task_name='COVID19CXRClassification', input_schema={'path': 'image'}, output_schema={'label': 'label'})" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "base_dataset.default_task" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "ad7bc53d-66db-499d-bddd-97c3e6184ad7", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Generating samples for COVID19CXRClassification: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 21165/21165 [00:00<00:00, 1282116.21it/s]\n" - ] - } - ], - "source": [ - "sample_dataset = base_dataset.set_task()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "b6caf164-9875-448a-938f-bae758b439d9", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from torchvision import transforms\n", - "\n", - "\n", - "transform = transforms.Compose([\n", - " transforms.Lambda(lambda x: x if x.shape[0] == 3 else x.repeat(3, 1, 1)),\n", - " transforms.Resize((224, 224)),\n", - " transforms.Normalize(mean=[0.5862785803043838], std=[0.27950088968644304])\n", - "])\n", - "\n", - "\n", - "def encode(sample):\n", - " sample[\"path\"] = transform(sample[\"path\"])\n", - " return sample\n", - "\n", - "\n", - "sample_dataset.set_transform(encode)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "6c46515c-9ff3-4ff5-b942-cf43666d3a2d", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from pyhealth.datasets import split_by_sample\n", - "\n", - "# Get Index of train, valid, test set\n", - "train_index, val_index, test_index = split_by_sample(\n", - " dataset=sample_dataset,\n", - " ratios=[0.7, 0.1, 0.2],\n", - " get_index = True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "a5c1e65c-ac81-421b-b265-7404fcab404c", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "| Uniform Initialization\n", - "| Uniform Initialization\n" - ] - } - ], - "source": [ - "import ssl\n", - "ssl._create_default_https_context = ssl._create_unverified_context\n", - "\n", - "model = Graph_TorchvisionModel(\n", - " dataset=sample_dataset,\n", - " feature_keys=[\"path\"],\n", - " label_key=\"label\",\n", - " mode=\"multiclass\",\n", - " model_name=\"resnet18\",\n", - " model_config={},\n", - " gnn_config={\"input_dim\": 256, \"hidden_dim\": 128},\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "c1640ebc-9fcb-49e1-9f2c-e7113ca85c62", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Build graph\n", - "# Set random = True will build random graph data\n", - "graph = model.build_graph(sample_dataset, random = True)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "567e6091-87c7-4b37-a78d-402bc7c96939", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Define Sampler as Dataloader\n", - "train_dataloader = NeighborSampler(sample_dataset, graph[\"edge_index\"], node_idx=train_index, sizes=[15, 10], batch_size=64, shuffle=True, num_workers=12)\n", - "\n", - "# We sample all edges connected to target node for validation and test (Sizes = [-1, -1])\n", - "valid_dataloader = NeighborSampler(sample_dataset, graph[\"edge_index\"], node_idx=val_index, sizes=[-1, -1], batch_size=64, shuffle=False, num_workers=12)\n", - "test_dataloader = NeighborSampler(sample_dataset, graph[\"edge_index\"], node_idx=test_index, sizes=[-1, -1], batch_size=64, shuffle=False, num_workers=12)" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "c54b30ec-e7fc-402d-aba5-7c361e95cd60", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Graph_TorchvisionModel(\n", - " (model): ResNet(\n", - " (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n", - " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n", - " (layer1): Sequential(\n", - " (0): BasicBlock(\n", - " (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " (1): BasicBlock(\n", - " (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (layer2): Sequential(\n", - " (0): BasicBlock(\n", - " (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", - " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (downsample): Sequential(\n", - " (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", - " (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (1): BasicBlock(\n", - " (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (layer3): Sequential(\n", - " (0): BasicBlock(\n", - " (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", - " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (downsample): Sequential(\n", - " (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", - " (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (1): BasicBlock(\n", - " (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (layer4): Sequential(\n", - " (0): BasicBlock(\n", - " (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n", - " (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (downsample): Sequential(\n", - " (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n", - " (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (1): BasicBlock(\n", - " (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " (relu): ReLU(inplace=True)\n", - " (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n", - " (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n", - " )\n", - " )\n", - " (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n", - " (fc): Linear(in_features=512, out_features=256, bias=True)\n", - " )\n", - " (gnn): GCN(\n", - " (gc1): GraphConvolution (256 -> 128)\n", - " (gc2): GraphConvolution (128 -> 4)\n", - " )\n", - ")\n", - "Metrics: None\n", - "Device: cpu\n", - "\n", - "Training:\n", - "Batch size: 64\n", - "Optimizer: \n", - "Optimizer params: {'lr': 0.001}\n", - "Weight decay: 0.0\n", - "Max grad norm: None\n", - "Val dataloader: NeighborSampler(sizes=[-1, -1])\n", - "Monitor: accuracy\n", - "Monitor criterion: max\n", - "Epochs: 1\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Epoch 0 / 1: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 232/232 [18:34<00:00, 4.80s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--- Train epoch-0, step-232 ---\n", - "loss: 1.3025\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "Evaluation: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 34/34 [01:19<00:00, 2.34s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--- Eval epoch-0, step-232 ---\n", - "accuracy: 0.4872\n", - "f1_macro: 0.1643\n", - "f1_micro: 0.4872\n", - "loss: 1.2535\n", - "New best accuracy score (0.4872) at epoch-0, step-232\n", - "Loaded best model\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "from pyhealth.trainer import Trainer\n", - "\n", - "resnet_trainer = Trainer(model=model, device=\"cpu\")\n", - "resnet_trainer.train(\n", - " train_dataloader=train_dataloader,\n", - " val_dataloader=valid_dataloader,\n", - " epochs=1,\n", - " monitor=\"accuracy\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "e1d2e71f-2a33-4187-b145-2161172821a9", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluation: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 67/67 [02:41<00:00, 2.42s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'accuracy': 0.4786590097780537, 'f1_macro': 0.1618557783142647, 'f1_micro': 0.4786590097780537, 'loss': 1.256981566770753}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "print(resnet_trainer.evaluate(test_dataloader))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26253fef", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.16" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/pyhealth/models/__init__.py b/pyhealth/models/__init__.py index 2f30ae673..1d8a800a0 100644 --- a/pyhealth/models/__init__.py +++ b/pyhealth/models/__init__.py @@ -13,7 +13,6 @@ from .logistic_regression import LogisticRegression from .gan import GAN from .gnn import GAT, GCN -from .graph_torchvision_model import Graph_TorchvisionModel from .graphcare import GraphCare from .grasp import GRASP, GRASPLayer from .medfuse import MedFuse, MedFuseLayer @@ -41,7 +40,6 @@ from .transformers_model import TransformersModel from .ehrmamba import EHRMamba, MambaBlock from .ehrmamba_cehr import EHRMambaCEHR -from .vae import VAE from .vision_embedding import VisionEmbeddingModel from .text_embedding import TextEmbedding from .sdoh import SdohClassifier diff --git a/pyhealth/models/graph_torchvision_model.py b/pyhealth/models/graph_torchvision_model.py deleted file mode 100644 index f91350cee..000000000 --- a/pyhealth/models/graph_torchvision_model.py +++ /dev/null @@ -1,351 +0,0 @@ -""" -Recommended: - - torch: 1.9.1 - - torchvision: 0.10.0+cu102 - - torch_sparse: 0.6.12 -""" - -import math -import sys -from typing import Dict, List - -import torch -import torch.nn as nn -import torch.nn.functional as F -import torchvision - -sys.path.append('.') - -from pyhealth.datasets import SampleDataset -from pyhealth.models import BaseModel -from pyhealth.sampler import NeighborSampler - -SUPPORTED_MODELS = [ - "resnet18", - "resnet34", - "resnet50", - "resnet101", - "resnet152", - "densenet121", - "densenet161", - "densenet169", - "densenet201", - "vit_b_16", - "vit_b_32", - "vit_l_16", - "vit_l_32", - "vit_h_14", - "swin_t", - "swin_s", - "swin_b", -] - -SUPPORTED_MODELS_FINAL_LAYER = {} -for model in SUPPORTED_MODELS: - if "resnet" in model: - SUPPORTED_MODELS_FINAL_LAYER[model] = "fc" - elif "densenet" in model: - SUPPORTED_MODELS_FINAL_LAYER[model] = "classifier" - elif "vit" in model: - SUPPORTED_MODELS_FINAL_LAYER[model] = "heads.head" - elif "swin" in model: - SUPPORTED_MODELS_FINAL_LAYER[model] = "head" - else: - raise NotImplementedError - - -class GraphConvolution(nn.Module): - """ - Simple GCN layer, similar to https://arxiv.org/abs/1609.02907 - """ - - def __init__(self, in_features, out_features, bias=True, init='xavier'): - super(GraphConvolution, self).__init__() - self.in_features = in_features - self.out_features = out_features - self.weight = nn.Parameter(torch.FloatTensor(in_features, out_features)) - if bias: - self.bias = nn.Parameter(torch.FloatTensor(out_features)) - else: - self.register_parameter('bias', None) - if init == 'uniform': - print("| Uniform Initialization") - self.reset_parameters_uniform() - elif init == 'xavier': - print("| Xavier Initialization") - self.reset_parameters_xavier() - elif init == 'kaiming': - print("| Kaiming Initialization") - self.reset_parameters_kaiming() - else: - raise NotImplementedError - - def reset_parameters_uniform(self): - stdv = 1. / math.sqrt(self.weight.size(1)) - self.weight.data.uniform_(-stdv, stdv) - if self.bias is not None: - self.bias.data.uniform_(-stdv, stdv) - - def reset_parameters_xavier(self): - nn.init.xavier_normal_(self.weight.data, gain=0.02) # Implement Xavier Uniform - if self.bias is not None: - nn.init.constant_(self.bias.data, 0.0) - - def reset_parameters_kaiming(self): - nn.init.kaiming_normal_(self.weight.data, a=0, mode='fan_in') - if self.bias is not None: - nn.init.constant_(self.bias.data, 0.0) - - def forward(self, input, adj): - support = torch.mm(input, self.weight) - # print("adj", adj.dtype, "support", support.dtype) - output = torch.spmm(adj, support) - if self.bias is not None: - return output + self.bias - else: - return output - - def __repr__(self): - return self.__class__.__name__ + ' (' \ - + str(self.in_features) + ' -> ' \ - + str(self.out_features) + ')' - - -class GCN(nn.Module): - def __init__(self, nfeat, nhid, nclass, dropout, init): - super(GCN, self).__init__() - - self.gc1 = GraphConvolution(nfeat, nhid, init=init) - self.gc2 = GraphConvolution(nhid, nclass, init=init) - self.dropout = dropout - - def bottleneck(self, path1, path2, path3, adj, in_x): - return F.relu(path3(F.relu(path2(F.relu(path1(in_x, adj)), adj)), adj)) - - def to_sparse_adj(self, adj, size): - return torch.sparse_coo_tensor(adj.edge_index, torch.ones_like(adj.edge_index[0]), size = size, dtype = torch.float32) - - def forward(self, x, adjs): - - temp = self.to_sparse_adj(adjs[0], size = (adjs[0].size[0], adjs[0].size[0])) - x = F.dropout(F.relu(self.gc1(x, temp)), self.dropout, training=self.training) - temp = self.to_sparse_adj(adjs[1], size = (adjs[0].size[0], adjs[0].size[0])) - x = self.gc2(x, temp) - - return F.log_softmax(x, dim=1) - - -class Graph_TorchvisionModel(BaseModel): - """Models from PyTorch's torchvision package. - - This class is a wrapper for models from torchvision. It will automatically load - the corresponding model and weights from torchvision. The final layer will be - replaced with a linear layer with the correct output size. - - -----------------------------------ResNet------------------------------------------ - Paper: Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. Deep Residual Learning - for Image Recognition. CVPR 2016. - -----------------------------------DenseNet---------------------------------------- - Paper: Gao Huang, Zhuang Liu, Laurens van der Maaten, Kilian Q. Weinberger. - Densely Connected Convolutional Networks. CVPR 2017. - ----------------------------Vision Transformer (ViT)------------------------------- - Paper: Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, - Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, - Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby. An Image is Worth - 16x16 Words: Transformers for Image Recognition at Scale. ICLR 2021. - ----------------------------Swin Transformer (and V2)------------------------------ - Paper: Ze Liu, Yutong Lin, Yue Cao, Han Hu, Yixuan Wei, Zheng Zhang, Stephen Lin, - Baining Guo. Swin Transformer: Hierarchical Vision Transformer Using Shifted - Windows. ICCV 2021. - - Paper: Ze Liu, Han Hu, Yutong Lin, Zhuliang Yao, Zhenda Xie, Yixuan Wei, Jia Ning, - Yue Cao, Zheng Zhang, Li Dong, Furu Wei, Baining Guo. Swin Transformer V2: Scaling - Up Capacity and Resolution. CVPR 2022. - ----------------------------------------------------------------------------------- - ----------------------- Graph Convolutional Networks (GCN)------------------------- - Paper: Thomas N. Kipf, Max Welling. - Semi-Supervised Classification with Graph Convolutional Networks. ICLR 2017. - ----------------------------------------------------------------------------------- - - Args: - dataset: the dataset to train the model. It is used to query certain - information such as the set of all tokens. - feature_keys: list of keys in samples to use as features, e.g., ["image"]. - Only one feature is supported. - label_key: key in samples to use as label, e.g., "drugs". - mode: one of "binary", "multiclass", or "multilabel". - model_name: str, name of the model to use, e.g., "resnet18". - See SUPPORTED_MODELS in the source code for the full list. - model_config: dict, kwargs to pass to the model constructor, - e.g., {"weights": "DEFAULT"}. See the torchvision documentation for the - set of supported kwargs for each model. - ----------------------------------------------------------------------------------- - """ - - def __init__( - self, - dataset: SampleDataset, - feature_keys: List[str], - label_key: str, - mode: str, - model_name: str, - model_config: dict, - gnn_config: dict, - ): - super(Graph_TorchvisionModel, self).__init__( - dataset=dataset, - feature_keys=feature_keys, - label_key=label_key, - mode=mode, - ) - - self.model_name = model_name - self.model_config = model_config - self.gnn_config = gnn_config - - assert len(feature_keys) == 1, "Only one feature is supported!" - assert model_name in SUPPORTED_MODELS_FINAL_LAYER.keys(), \ - f"PyHealth does not currently include {model_name} model!" - - # for torchvision 0.10.0 - self.model = torchvision.models.__dict__[model_name](**model_config) - final_layer_name = SUPPORTED_MODELS_FINAL_LAYER[model_name] - final_layer = self.model - for name in final_layer_name.split("."): - final_layer = getattr(final_layer, name) - hidden_dim = final_layer.in_features - - # Graph Model Configs - gnn_input_dim = gnn_config["input_dim"] - gnn_hidden_dim = gnn_config["hidden_dim"] - - self.label_tokenizer = self.get_label_tokenizer() - output_size = self.get_output_size(self.label_tokenizer) - self.gnn = GCN(nfeat=gnn_input_dim, nhid=gnn_hidden_dim, nclass=output_size, dropout=0.5, init='uniform') - - setattr(self.model, final_layer_name.split(".")[0], nn.Linear(hidden_dim, gnn_input_dim)) - - - def build_graph(self, data, random = False) -> Dict[str, torch.Tensor]: - """This module generate edge index of graph structure based on given data. - Currently, we do not have multi-modal data, so this module randomly generate edge index""" - - if random: - edge_index = torch.randint(len(data), size = (2, int(1.0 * len(data)))) - - return { - 'edge_index': edge_index - } - - - def forward(self, **kwargs) -> Dict[str, torch.Tensor]: - """Forward propagation.""" - # concat the info within one batch (batch, channel, length) - x = kwargs["image"] - x = torch.stack(x, dim=0).to(self.device) - if x.shape[1] == 1: - x = x.repeat((1, 3, 1, 1)) - img_embs = self.model(x) - logits = self.gnn(img_embs, kwargs["adjacencies"]) - y_true = self.prepare_labels(kwargs[self.label_key], self.label_tokenizer) - loss = self.get_loss_function()(logits, y_true) - y_prob = self.prepare_y_prob(logits) - return { - "loss": loss, - "y_prob": y_prob, - "y_true": y_true, - } - - -if __name__ == "__main__": - - from torchvision import transforms - - from pyhealth.datasets import COVID19CXRDataset, split_by_sample - - base_dataset = COVID19CXRDataset( - root="/srv/local/data/COVID-19_Radiography_Dataset", - ) - - sample_dataset = base_dataset.set_task() - - transform = transforms.Compose([ - # transforms.Grayscale(), - transforms.Lambda(lambda x: x if x.shape[0] == 3 else x.repeat(3, 1, 1)), - transforms.Resize((224, 224)), - transforms.Normalize(mean=[0.5862785803043838], std=[0.27950088968644304]) - ]) - - - def encode(sample): - sample["path"] = transform(sample["path"]) - return sample - - sample_dataset.set_transform(encode) - - - # Get Index of train, valid, test set - train_index, val_index, test_index = split_by_sample( - dataset=sample_dataset, - ratios=[0.7, 0.1, 0.2], - get_index = True - ) - - model = Graph_TorchvisionModel( - dataset=sample_dataset, - feature_keys=["path"], - label_key="label", - mode="multiclass", - model_name="resnet18", - # model_config={"weights": "DEFAULT"}, - model_config={}, - gnn_config={"input_dim": 256, "hidden_dim": 128}, - ) - - graph = model.build_graph(sample_dataset, random = True) - - from pyhealth.datasets import split_by_sample - - # Get Index of train, valid, test set - train_index, val_index, test_index = split_by_sample( - dataset=sample_dataset, - ratios=[0.7, 0.1, 0.2], - get_index = True - ) - - # Define Sampler as Dataloader - train_dataloader = NeighborSampler(sample_dataset, graph["edge_index"], node_idx=train_index, sizes=[15, 10], batch_size=64, shuffle=True, num_workers=12) - # We sample all edges connected to target node for validation and test (Sizes = [-1, -1]) - valid_dataloader = NeighborSampler(sample_dataset, graph["edge_index"], node_idx=val_index, sizes=[-1, -1], batch_size=64, shuffle=False, num_workers=12) - test_dataloader = NeighborSampler(sample_dataset, graph["edge_index"], node_idx=test_index, sizes=[-1, -1], batch_size=64, shuffle=False, num_workers=12) - - - # train_dataloader = NeighborSampler(sample_dataset, graph["edge_index"], node_idx=train_index, sizes=[15, 10], batch_size=64, shuffle=True, num_workers=12) - - # data_graph_batch = next(iter(train_dataloader)) - - # # try the model - # ret = model(**data_graph_batch) - # print(ret) - - # # try loss backward - # ret["loss"].backward() - - from pyhealth.trainer import Trainer - resnet_trainer = Trainer(model=model, device="cpu") - resnet_trainer.train( - train_dataloader=train_dataloader, - val_dataloader=valid_dataloader, - epochs=1, - monitor="accuracy", - ) - - print(resnet_trainer.evaluate(test_dataloader)) - - - resnet_trainer.train( - train_dataloader=train_dataloader, - val_dataloader=valid_dataloader, - epochs=1, - monitor="accuracy", - ) diff --git a/pyhealth/models/vae.py b/pyhealth/models/vae.py deleted file mode 100644 index e46c395e1..000000000 --- a/pyhealth/models/vae.py +++ /dev/null @@ -1,207 +0,0 @@ -import functools -from typing import Dict, List, Optional, Tuple -import pickle -import numpy as np - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from pyhealth.datasets import BaseSignalDataset -from pyhealth.models import BaseModel, ResBlock2D - - -class VAE(BaseModel): - """VAE model (take 128x128 or 64x64 or 32x32 images) - - Kingma, Diederik P., and Max Welling. "Auto-encoding variational bayes." - - Note: - We use CNN models as the encoder and decoder layers for now. - - Args: - dataset: the dataset to train the model. It is used to query certain - information such as the set of all tokens. - feature_keys: list of keys in samples to use as features, - e.g. ["conditions", "procedures"]. - label_key: key in samples to use as label (e.g., "drugs"). - mode: one of "binary", "multiclass", or "multilabel". - embedding_dim: the embedding dimension. Default is 128. - hidden_dim: the hidden dimension. Default is 128. - **kwargs: other parameters for the Deepr layer. - - Examples: - """ - - def __init__( - self, - dataset: BaseSignalDataset, - feature_keys: List[str], - label_key: str, - input_channel: int, - input_size: int, - mode: str, - hidden_dim: int = 128, - **kwargs, - ): - super(VAE, self).__init__( - dataset=dataset, - feature_keys=feature_keys, - label_key=label_key, - mode=mode, - ) - self.hidden_dim = hidden_dim - - # encoder part - if input_size == 128: - self.encoder1 = nn.Sequential( - ResBlock2D(input_channel, 16, 2, True, True), - ResBlock2D(16, 64, 2, True, True), - ResBlock2D(64, 256, 2, True, True), - ) - self.mu = nn.Linear(256 * 2 * 2, self.hidden_dim) # for mu - self.log_std2 = nn.Linear(256 * 2 * 2, self.hidden_dim) # for log (sigma^2) - - self.decoder1 = nn.Sequential( - nn.ConvTranspose2d(self.hidden_dim, 256, kernel_size=5, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(256, 128, kernel_size=5, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(128, 64, kernel_size=5, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(64, 32, kernel_size=6, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(32, input_channel, kernel_size=6, stride=2), - nn.Sigmoid(), - ) - - elif input_size == 64: - self.encoder1 = nn.Sequential( - ResBlock2D(input_channel, 16, 2, True, True), - ResBlock2D(16, 64, 2, True, True), - ResBlock2D(64, 256, 2, True, True), - ) - self.mu = nn.Linear(256, self.hidden_dim) # for mu - self.log_std2 = nn.Linear(256, self.hidden_dim) # for log (sigma^2) - - self.decoder1 = nn.Sequential( - nn.ConvTranspose2d(self.hidden_dim, 128, kernel_size=5, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(128, 64, kernel_size=5, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(64, 32, kernel_size=6, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(32, input_channel, kernel_size=6, stride=2), - nn.Sigmoid(), - ) - - elif input_size == 32: - self.encoder1 = nn.Sequential( - ResBlock2D(input_channel, 16, 2, True, True), - ResBlock2D(16, 64, 2, True, True), - # ResBlock2D(64, 256, 2, True, True), - ) - self.mu = nn.Linear(64 * 2 * 2, self.hidden_dim) # for mu - self.log_std2 = nn.Linear(64 * 2 * 2, self.hidden_dim) # for log (sigma^2) - - self.decoder1 = nn.Sequential( - nn.ConvTranspose2d(self.hidden_dim, 64, kernel_size=5, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(64, 32, kernel_size=6, stride=2), - nn.ReLU(), - nn.ConvTranspose2d(32, input_channel, kernel_size=6, stride=2), - nn.Sigmoid(), - ) - - def encoder(self, x) -> Tuple[torch.Tensor, torch.Tensor]: - h = self.encoder1(x) - batch_size = h.shape[0] - h = h.view(batch_size, -1) - return self.mu(h), torch.sqrt(torch.exp(self.log_std2(h))) - - def sampling(self, mu, std) -> torch.Tensor: # reparameterization trick - eps = torch.randn_like(std) - return mu + eps * std - - def decoder(self, z) -> torch.Tensor: - x_hat = self.decoder1(z) - return x_hat - - @staticmethod - def loss_function(y, x, mu, std): - ERR = F.binary_cross_entropy(y, x, reduction='sum') - KLD = -0.5 * torch.sum(1 + torch.log(std**2) - mu**2 - std**2) - return ERR + KLD - - def forward(self, **kwargs) -> Dict[str, torch.Tensor]: - - # concat the info within one batch (batch, channel, height, width) - # if the input is a list of numpy array, we need to convert it to tensor - if isinstance(kwargs[self.feature_keys[0]][0], np.ndarray): - x = torch.tensor( - np.array(kwargs[self.feature_keys[0]]).astype("float16"), device=self.device - ).float() - else: - x = torch.stack(kwargs[self.feature_keys[0]], dim=0).to(self.device) - - mu, std = self.encoder(x) - z = self.sampling(mu, std) - z = z.unsqueeze(2).unsqueeze(3) - x_rec = self.decoder(z) - - loss = self.loss_function(x_rec, x, mu, std) - results = { - "loss": loss, - "y_prob": x_rec, - "y_true": x, - } - return results - - -if __name__ == "__main__": - from pyhealth.datasets import SampleSignalDataset, get_dataloader - from pyhealth.datasets import COVID19CXRDataset - from torchvision import transforms - - root = "/srv/local/data/COVID-19_Radiography_Dataset" - base_dataset = COVID19CXRDataset(root, dev=True, refresh_cache=False) - - sample_dataset = base_dataset.set_task() - - # the transformation automatically normalize the pixel intensity into [0, 1] - transform = transforms.Compose([ - transforms.Lambda(lambda x: x if x.shape[0] == 3 else x.repeat(3, 1, 1)), # only use the first channel - transforms.Resize((128, 128)), - ]) - - def encode(sample): - sample["path"] = transform(sample["path"]) - return sample - - sample_dataset.set_transform(encode) - - # data loader - from pyhealth.datasets import get_dataloader - - train_loader = get_dataloader(sample_dataset, batch_size=2, shuffle=True) - - # model - model = VAE( - dataset=sample_dataset, - input_channel=3, - input_size=128, - feature_keys=["path"], - label_key="path", - mode="regression", - hidden_dim = 256, - ).to("cuda") - - # data batch - data_batch = next(iter(train_loader)) - - # try the model - ret = model(**data_batch) - print(ret) - - # try loss backward - ret["loss"].backward() \ No newline at end of file