{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8f764c6-97a3-4af9-a2f5-249964ca062a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# PyTorch Example: Convolutional Autoencoder -> Representation Learning\n",
    "# Introduces:\n",
    "# - Tensor conventions (N, C, H, W)\n",
    "# - Datasets & DataLoaders\n",
    "# - Writing models as `nn.Module`\n",
    "# - Training / validation loops\n",
    "# - Optimizers & schedulers\n",
    "# - Checkpointing & loading models\n",
    "# - Visualization of training dynamics\n",
    "# - Using a trained encoder for downstream tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1577fef5-8b84-4bcd-8b99-130bdaed68e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "# Device\n",
    "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "print(\"Using device:\", DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aed56fd3-4fe8-42c2-abdd-e978a6efa1a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# PyTorch image tensors have dimensions (N, C, H, W)\n",
    "# - N: batch size\n",
    "# - C: channels (1 for MNIST)\n",
    "# - H, W: spatial dimensions\n",
    "\n",
    "x = torch.randn(4, 1, 28, 28)\n",
    "print(x.shape)  # (N, C, H, W)\n",
    "\n",
    "# Loading image data\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "train_dataset = datasets.MNIST(root=\"./data\", train=True, download=True, transform=transform)\n",
    "val_dataset = datasets.MNIST(root=\"./data\", train=False, download=True, transform=transform)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=128, shuffle=True) # smaller batch size because storing activations for backprop\n",
    "val_loader = DataLoader(val_dataset, batch_size=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34ac1d8d-db05-4a69-abdc-aa617b57c713",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convolutional Autoencoder class\n",
    "\n",
    "class ConvAutoencoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        # Encoder\n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Conv2d(1, 16, 3, stride=2, padding=1),  # (16, 14, 14)\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(16, 32, 3, stride=2, padding=1), # (32, 7, 7)\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        \n",
    "        # Bottleneck dimension = 32 * 7 * 7\n",
    "        \n",
    "        # Decoder\n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.ConvTranspose2d(32, 16, 3, stride=2, padding=1, output_padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.ConvTranspose2d(16, 1, 3, stride=2, padding=1, output_padding=1),\n",
    "            nn.Sigmoid(),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        z = self.encoder(x)\n",
    "        x_hat = self.decoder(z)\n",
    "        return x_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82742446-e7f2-407c-ae48-83e36eaba743",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create model and move to GPU potentially\n",
    "model = ConvAutoencoder().to(DEVICE)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70fe6b2e-4359-4fc5-b90e-7e84503bd44e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Link an optimizer to the parameters of the model, and control its step size schedule with a learning rate scheduler\n",
    "# Pick the per-example loss function for the training\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-3)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "# Training and validation functions (train one epoch, evaluate validation error at the end)\n",
    "\n",
    "def train_epoch(model, loader):\n",
    "    model.train()\n",
    "    total_loss = 0\n",
    "\n",
    "    for x, _ in loader:\n",
    "        x = x.to(DEVICE)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        x_hat = model(x)\n",
    "        loss = criterion(x_hat, x)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        total_loss += loss.item() * x.size(0)\n",
    "\n",
    "    return total_loss / len(loader.dataset)\n",
    "\n",
    "\n",
    "def eval_epoch(model, loader):\n",
    "    model.eval()\n",
    "    total_loss = 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for x, _ in loader:\n",
    "            x = x.to(DEVICE)\n",
    "            x_hat = model(x)\n",
    "            loss = criterion(x_hat, x)\n",
    "            total_loss += loss.item() * x.size(0)\n",
    "\n",
    "    return total_loss / len(loader.dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cea4d395-b79e-4808-8606-335fdcc65569",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train while checkpointing the best model after each epoch\n",
    "\n",
    "num_epochs = 10\n",
    "train_losses, val_losses = [], []\n",
    "\n",
    "best_val_loss = float(\"inf\")\n",
    "checkpoint_path = \"best_autoencoder.pt\"\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    train_loss = train_epoch(model, train_loader)\n",
    "    val_loss = eval_epoch(model, val_loader)\n",
    "\n",
    "    scheduler.step()\n",
    "\n",
    "    train_losses.append(train_loss)\n",
    "    val_losses.append(val_loss)\n",
    "\n",
    "    print(f\"Epoch {epoch+1}: train={train_loss:.4f}, val={val_loss:.4f}\")\n",
    "\n",
    "    if val_loss < best_val_loss:\n",
    "        best_val_loss = val_loss\n",
    "        torch.save(model.state_dict(), checkpoint_path)\n",
    "        print(\"  -> Saved best model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98679a55-19bf-4f75-8ba1-57e7a0b3db7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot convergence diagnostics\n",
    "\n",
    "plt.plot(train_losses, label=\"train\")\n",
    "plt.plot(val_losses, label=\"val\")\n",
    "plt.legend()\n",
    "plt.title(\"Loss curves\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "445bc178-e3fc-49b6-b8f7-24e8546e1bef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize sample reconstructions\n",
    "model.load_state_dict(torch.load(checkpoint_path))\n",
    "model.eval()\n",
    "\n",
    "x, _ = next(iter(val_loader))\n",
    "x = x.to(DEVICE)\n",
    "\n",
    "with torch.no_grad():\n",
    "    x_hat = model(x)\n",
    "\n",
    "# Move the model to the CPU\n",
    "x = x.cpu()\n",
    "x_hat = x_hat.cpu()\n",
    "\n",
    "fig, axes = plt.subplots(2, 8, figsize=(12, 3))\n",
    "for i in range(8):\n",
    "    axes[0, i].imshow(x[i].squeeze(), cmap=\"gray\")\n",
    "    axes[1, i].imshow(x_hat[i].squeeze(), cmap=\"gray\")\n",
    "    axes[0, i].axis(\"off\")\n",
    "    axes[1, i].axis(\"off\")\n",
    "\n",
    "axes[0,0].set_title(\"Original\")\n",
    "axes[1,0].set_title(\"Reconstruction\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7c3e1ec-3b88-4f7a-b064-91ed4360a260",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the features learned by the autoencoder (32 x 7 x 7) for downstream classification task\n",
    "\n",
    "class EncoderClassifier(nn.Module):\n",
    "    def __init__(self, encoder):\n",
    "        super().__init__()\n",
    "        self.encoder = encoder\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            nn.Linear(32*7*7, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 10)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        with torch.no_grad():\n",
    "            z = self.encoder(x)\n",
    "        return self.classifier(z)\n",
    "\n",
    "classifier = EncoderClassifier(model.encoder).to(DEVICE)\n",
    "\n",
    "optimizer_cls = optim.Adam(classifier.parameters(), lr=1e-3)\n",
    "criterion_cls = nn.CrossEntropyLoss() # takes LOGITS as inputs\n",
    "\n",
    "def train_classifier_epoch(model, loader):\n",
    "    model.train()\n",
    "    total_loss, correct = 0, 0\n",
    "\n",
    "    for x, y in loader:\n",
    "        x, y = x.to(DEVICE), y.to(DEVICE)\n",
    "\n",
    "        optimizer_cls.zero_grad()\n",
    "        logits = model(x)\n",
    "        loss = criterion_cls(logits, y)\n",
    "        loss.backward()\n",
    "        optimizer_cls.step()\n",
    "\n",
    "        total_loss += loss.item() * x.size(0)\n",
    "        correct += (logits.argmax(1) == y).sum().item()\n",
    "\n",
    "    return total_loss / len(loader.dataset), correct / len(loader.dataset)\n",
    "\n",
    "\n",
    "def eval_classifier_epoch(model, loader):\n",
    "    model.eval()\n",
    "    total_loss, correct = 0, 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for x, y in loader:\n",
    "            x, y = x.to(DEVICE), y.to(DEVICE)\n",
    "            logits = model(x)\n",
    "            loss = criterion_cls(logits, y)\n",
    "\n",
    "            total_loss += loss.item() * x.size(0)\n",
    "            correct += (logits.argmax(1) == y).sum().item()\n",
    "\n",
    "    return total_loss / len(loader.dataset), correct / len(loader.dataset)\n",
    "\n",
    "for epoch in range(5):\n",
    "    train_loss, train_acc = train_classifier_epoch(classifier, train_loader)\n",
    "    val_loss, val_acc = eval_classifier_epoch(classifier, val_loader)\n",
    "\n",
    "    print(f\"[CLS] Epoch {epoch+1}: train_acc={train_acc:.3f}, val_acc={val_acc:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52f75038-3b3f-4df7-9f7a-e2c7efb2d83e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
