refactor: minor changes in probabilistic.py

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.0
+current_version = 0.2.0
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)

.travis.yml

@@ -1,11 +1,7 @@
 dist: bionic
 sudo: false
 language: python
-python:
-  - 3.9
-  - 3.8
-  - 3.7
-  - 3.6
+python: 3.9
 cache:
   directories:
   - "$HOME/.cache/pip"
@@ -19,26 +15,11 @@ script:
 - ./tests/test_examples.sh examples/
 after_success:
 - bash <(curl -s https://codecov.io/bash)
-
-# Publish on PyPI
-jobs:
-  include:
-    - stage: build
-      python: 3.9
-      script: echo "Starting Pypi build"
-      deploy:
+deploy:
   provider: pypi
   username: __token__
-        distributions: "sdist bdist_wheel"
   password:
     secure: PDoASdYdVlt1aIROYilAsCW6XpBs/TDel0CSptDzX0CI7i4+ksEW6Jk0JyL58bQt7V4F8PeGty4A8SODzAUIk2d8sty5RI4VJjvXZFCXlUsW+JGUN3EvWNqJLnwN8TDxgu2ENao37GUh0dC6pL8b6bVDGeOLaY1E/YR1jimmTJuxxjKjBIU8ByqTNBnC3rzybMTPU3nRoOM/WMQUyReHrPoUJj685sLqrLruhAqhiYsPbotP8xY6i8+KBbhp5vgiARV2+LkbeGcYZwozCzrEqPKY7YIfVPh895cw0v4NRyFwK1P2jyyIt22Z9Ni0Uy1J5/Qp9Sv6mBPeGjm3pnpDCQyS+2bNIDaj08KUYTIo1mC/Jcu4jQgppZEF+oey9q1tgGo+/JhsTeERKV9BoPF5HDiRArU1s5aWJjFnCsHfu+W1XqX8bwN3aTYsEIaApT3/irc6XyFJIfMN82+z+lUcZ4Y1yAHT3nH1Vif+pZYZB0UOSGrHwuI/UayjKzbCzHMuHWylWB/9ehd4o4YVp6iubVHc7Sj0KQkwBgwgl6TvwNcUuFsplFabCxmX0mVcavXsWiOBc+ivPmU6574zGj0JcEk5ghVgnKH+QS96aVrKOzegwbl4O13jY8dJp+/zgXl0gJOvRKr4BhuBJKcBaMQHdSKUChVsJJtqDyt59GvWcbg=
   on:
     tags: true
     skip_existing: true
-
-# The password is encrypted with:
-# `cd prototorch && travis encrypt your-pypi-api-token --add deploy.password`
-# See https://docs.travis-ci.com/user/deployment/pypi and
-# https://github.com/travis-ci/travis.rb#installation
-# for more details
-# Note: The encrypt command does not work well in ZSH.

docs/source/conf.py

@@ -23,7 +23,7 @@ author = "Jensun Ravichandran"
 
 # The full version, including alpha/beta/rc tags
 #
-release = "0.3.0"
+release = "0.2.0"
 
 # -- General configuration ---------------------------------------------------
 

examples/binnam_tecator.py

@@ -0,0 +1,81 @@
+"""Neural Additive Model (NAM) example for binary classification."""
+
+import argparse
+
+import prototorch as pt
+import pytorch_lightning as pl
+import torch
+from matplotlib import pyplot as plt
+
+if __name__ == "__main__":
+    # Command-line arguments
+    parser = argparse.ArgumentParser()
+    parser = pl.Trainer.add_argparse_args(parser)
+    args = parser.parse_args()
+
+    # Dataset
+    train_ds = pt.datasets.Tecator("~/datasets")
+
+    # Dataloaders
+    train_loader = torch.utils.data.DataLoader(train_ds, batch_size=64)
+
+    # Hyperparameters
+    hparams = dict(lr=0.1)
+
+    # Define the feature extractor
+    class FE(torch.nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.modules_list = torch.nn.ModuleList([
+                torch.nn.Linear(1, 3),
+                torch.nn.Sigmoid(),
+                torch.nn.Linear(3, 1),
+                torch.nn.Sigmoid(),
+            ])
+
+        def forward(self, x):
+            for m in self.modules_list:
+                x = m(x)
+            return x
+
+    # Initialize the model
+    model = pt.models.BinaryNAM(
+        hparams,
+        extractors=torch.nn.ModuleList([FE() for _ in range(100)]),
+    )
+
+    # Compute intermediate input and output sizes
+    model.example_input_array = torch.zeros(4, 100)
+
+    # Callbacks
+    es = pl.callbacks.EarlyStopping(
+        monitor="train_loss",
+        min_delta=0.001,
+        patience=20,
+        mode="min",
+        verbose=True,
+        check_on_train_epoch_end=True,
+    )
+
+    # Setup trainer
+    trainer = pl.Trainer.from_argparse_args(
+        args,
+        callbacks=[
+            es,
+        ],
+        terminate_on_nan=True,
+        weights_summary=None,
+        accelerator="ddp",
+    )
+
+    # Training loop
+    trainer.fit(model, train_loader)
+
+    # Visualize extractor shape functions
+    fig, axes = plt.subplots(10, 10)
+    for i, ax in enumerate(axes.flat):
+        x = torch.linspace(-2, 2, 100)  # TODO use min/max from data
+        y = model.extractors[i](x.view(100, 1)).squeeze().detach()
+        ax.plot(x, y)
+        ax.set(title=f"Feature {i + 1}", xticklabels=[], yticklabels=[])
+    plt.show()

examples/binnam_xor.py

@@ -0,0 +1,86 @@
+"""Neural Additive Model (NAM) example for binary classification."""
+
+import argparse
+
+import prototorch as pt
+import pytorch_lightning as pl
+import torch
+from matplotlib import pyplot as plt
+
+if __name__ == "__main__":
+    # Command-line arguments
+    parser = argparse.ArgumentParser()
+    parser = pl.Trainer.add_argparse_args(parser)
+    args = parser.parse_args()
+
+    # Dataset
+    train_ds = pt.datasets.XOR()
+
+    # Dataloaders
+    train_loader = torch.utils.data.DataLoader(train_ds, batch_size=256)
+
+    # Hyperparameters
+    hparams = dict(lr=0.001)
+
+    # Define the feature extractor
+    class FE(torch.nn.Module):
+        def __init__(self, hidden_size=10):
+            super().__init__()
+            self.modules_list = torch.nn.ModuleList([
+                torch.nn.Linear(1, hidden_size),
+                torch.nn.ReLU(),
+                torch.nn.Linear(hidden_size, 1),
+                torch.nn.ReLU(),
+            ])
+
+        def forward(self, x):
+            for m in self.modules_list:
+                x = m(x)
+            return x
+
+    # Initialize the model
+    model = pt.models.BinaryNAM(
+        hparams,
+        extractors=torch.nn.ModuleList([FE(20) for _ in range(2)]),
+    )
+
+    # Compute intermediate input and output sizes
+    model.example_input_array = torch.zeros(4, 2)
+
+    # Summary
+    print(model)
+
+    # Callbacks
+    vis = pt.models.Vis2D(data=train_ds)
+    es = pl.callbacks.EarlyStopping(
+        monitor="train_loss",
+        min_delta=0.001,
+        patience=50,
+        mode="min",
+        verbose=False,
+        check_on_train_epoch_end=True,
+    )
+
+    # Setup trainer
+    trainer = pl.Trainer.from_argparse_args(
+        args,
+        callbacks=[
+            vis,
+            es,
+        ],
+        terminate_on_nan=True,
+        weights_summary="full",
+        accelerator="ddp",
+    )
+
+    # Training loop
+    trainer.fit(model, train_loader)
+
+    # Visualize extractor shape functions
+    fig, axes = plt.subplots(2)
+    for i, ax in enumerate(axes.flat):
+        x = torch.linspace(0, 1, 100)  # TODO use min/max from data
+        y = model.extractors[i](x.view(100, 1)).squeeze().detach()
+        ax.plot(x, y)
+        ax.set(title=f"Feature {i + 1}")
+    plt.show()

prototorch/models/__init__.py

@@ -1,5 +1,7 @@
 """`models` plugin for the `prototorch` package."""
 
+from importlib.metadata import PackageNotFoundError, version
+
 from .callbacks import PrototypeConvergence, PruneLoserPrototypes
 from .cbc import CBC, ImageCBC
 from .glvq import (
@@ -17,8 +19,9 @@ from .glvq import (
 )
 from .knn import KNN
 from .lvq import LVQ1, LVQ21, MedianLVQ
+from .nam import BinaryNAM
 from .probabilistic import CELVQ, PLVQ, RSLVQ, SLVQ
 from .unsupervised import GrowingNeuralGas, HeskesSOM, KohonenSOM, NeuralGas
 from .vis import *
 
-__version__ = "0.3.0"
+__version__ = "0.2.0"

prototorch/models/abstract.py

@@ -1,5 +1,7 @@
 """Abstract classes to be inherited by prototorch models."""
 
+from typing import Final, final
+
 import pytorch_lightning as pl
 import torch
 import torchmetrics
@@ -41,6 +43,7 @@ class ProtoTorchBolt(pl.LightningModule):
         else:
             return optimizer
 
+    @final
     def reconfigure_optimizers(self):
         self.trainer.accelerator.setup_optimizers(self.trainer)
 
@@ -93,7 +96,7 @@ class UnsupervisedPrototypeModel(PrototypeModel):
             )
 
     def compute_distances(self, x):
-        protos = self.proto_layer().type_as(x)
+        protos = self.proto_layer()
         distances = self.distance_layer(x, protos)
         return distances
 
@@ -133,14 +136,14 @@ class SupervisedPrototypeModel(PrototypeModel):
 
     def forward(self, x):
         distances = self.compute_distances(x)
-        _, plabels = self.proto_layer()
+        plabels = self.proto_layer.labels
         winning = stratified_min_pooling(distances, plabels)
         y_pred = torch.nn.functional.softmin(winning)
         return y_pred
 
     def predict_from_distances(self, distances):
         with torch.no_grad():
-            _, plabels = self.proto_layer()
+            plabels = self.proto_layer.labels
             y_pred = self.competition_layer(distances, plabels)
         return y_pred
 
@@ -172,7 +175,7 @@ class NonGradientMixin(ProtoTorchMixin):
     """Mixin for custom non-gradient optimization."""
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.automatic_optimization = False
+        self.automatic_optimization: Final = False
 
     def training_step(self, train_batch, batch_idx, optimizer_idx=None):
         raise NotImplementedError
@@ -180,6 +183,7 @@ class NonGradientMixin(ProtoTorchMixin):
 
 class ImagePrototypesMixin(ProtoTorchMixin):
     """Mixin for models with image prototypes."""
+    @final
     def on_train_batch_end(self, outputs, batch, batch_idx, dataloader_idx):
         """Constrain the components to the range [0, 1] by clamping after updates."""
         self.proto_layer.components.data.clamp_(0.0, 1.0)

prototorch/models/callbacks.py

@@ -55,7 +55,7 @@ class PruneLoserPrototypes(pl.Callback):
                 distribution = dict(zip(labels.tolist(), counts.tolist()))
                 if self.verbose:
                     print(f"Re-adding pruned prototypes...")
-                    print(f"distribution={distribution}")
+                    print(f"{distribution=}")
                 pl_module.add_prototypes(
                     distribution=distribution,
                     components_initializer=self.prototypes_initializer)
@@ -134,4 +134,4 @@ class GNGCallback(pl.Callback):
             pl_module.errors[
                 worst_neighbor] = errors[worst_neighbor] * self.reduction
 
-            trainer.accelerator.setup_optimizers(trainer)
+            trainer.accelerator_backend.setup_optimizers(trainer)

prototorch/models/glvq.py

@@ -55,7 +55,7 @@ class GLVQ(SupervisedPrototypeModel):
     def shared_step(self, batch, batch_idx, optimizer_idx=None):
         x, y = batch
         out = self.compute_distances(x)
-        _, plabels = self.proto_layer()
+        plabels = self.proto_layer.labels
         loss = self.loss(out, y, plabels)
         return out, loss
 
@@ -112,8 +112,7 @@ class SiameseGLVQ(GLVQ):
         proto_opt = self.optimizer(self.proto_layer.parameters(),
                                    lr=self.hparams.proto_lr)
         # Only add a backbone optimizer if backbone has trainable parameters
-        bb_params = list(self.backbone.parameters())
-        if (bb_params):
+        if (bb_params := list(self.backbone.parameters())):
             bb_opt = self.optimizer(bb_params, lr=self.hparams.bb_lr)
             optimizers = [proto_opt, bb_opt]
         else:

prototorch/models/lvq.py

@@ -10,7 +10,9 @@ from .glvq import GLVQ
 class LVQ1(NonGradientMixin, GLVQ):
     """Learning Vector Quantization 1."""
     def training_step(self, train_batch, batch_idx, optimizer_idx=None):
-        protos, plables = self.proto_layer()
+        protos = self.proto_layer.components
+        plabels = self.proto_layer.labels
+
         x, y = train_batch
         dis = self.compute_distances(x)
         # TODO Vectorized implementation
@@ -28,8 +30,8 @@ class LVQ1(NonGradientMixin, GLVQ):
             self.proto_layer.load_state_dict({"_components": updated_protos},
                                              strict=False)
 
-        print(f"dis={dis}")
-        print(f"y={y}")
+        print(f"{dis=}")
+        print(f"{y=}")
         # Logging
         self.log_acc(dis, y, tag="train_acc")
 
@@ -39,7 +41,8 @@ class LVQ1(NonGradientMixin, GLVQ):
 class LVQ21(NonGradientMixin, GLVQ):
     """Learning Vector Quantization 2.1."""
     def training_step(self, train_batch, batch_idx, optimizer_idx=None):
-        protos, plabels = self.proto_layer()
+        protos = self.proto_layer.components
+        plabels = self.proto_layer.labels
 
         x, y = train_batch
         dis = self.compute_distances(x)
@@ -96,7 +99,8 @@ class MedianLVQ(NonGradientMixin, GLVQ):
         return lower_bound
 
     def training_step(self, train_batch, batch_idx, optimizer_idx=None):
-        protos, plabels = self.proto_layer()
+        protos = self.proto_layer.components
+        plabels = self.proto_layer.labels
 
         x, y = train_batch
         dis = self.compute_distances(x)

prototorch/models/nam.py

@@ -0,0 +1,58 @@
+"""ProtoTorch Neural Additive Model."""
+
+import torch
+import torchmetrics
+
+from .abstract import ProtoTorchBolt
+
+
+class BinaryNAM(ProtoTorchBolt):
+    """Neural Additive Model for binary classification.
+
+    Paper: https://arxiv.org/abs/2004.13912
+    Official implementation: https://github.com/google-research/google-research/tree/master/neural_additive_models
+
+    """
+    def __init__(self, hparams: dict, extractors: torch.nn.ModuleList,
+                 **kwargs):
+        super().__init__(hparams, **kwargs)
+
+        # Default hparams
+        self.hparams.setdefault("threshold", 0.5)
+
+        self.extractors = extractors
+        self.linear = torch.nn.Linear(in_features=len(extractors),
+                                      out_features=1,
+                                      bias=True)
+
+    def extract(self, x):
+        """Apply the local extractors batch-wise on features."""
+        out = torch.zeros_like(x)
+        for j in range(x.shape[1]):
+            out[:, j] = self.extractors[j](x[:, j].unsqueeze(1)).squeeze()
+        return out
+
+    def forward(self, x):
+        x = self.extract(x)
+        x = self.linear(x)
+        return torch.sigmoid(x)
+
+    def training_step(self, batch, batch_idx, optimizer_idx=None):
+        x, y = batch
+        preds = self(x).squeeze()
+        train_loss = torch.nn.functional.binary_cross_entropy(preds, y.float())
+        self.log("train_loss", train_loss)
+        accuracy = torchmetrics.functional.accuracy(preds.int(), y.int())
+        self.log("train_acc",
+                 accuracy,
+                 on_step=False,
+                 on_epoch=True,
+                 prog_bar=True,
+                 logger=True)
+        return train_loss
+
+    def predict(self, x):
+        out = self(x)
+        pred = torch.zeros_like(out, device=self.device)
+        pred[out > self.hparams.threshold] = 1
+        return pred

prototorch/models/probabilistic.py

@@ -1,5 +1,4 @@
 """Probabilistic GLVQ methods"""
-
 import torch
 
 from ..core.losses import nllr_loss, rslvq_loss
@@ -20,7 +19,7 @@ class CELVQ(GLVQ):
     def shared_step(self, batch, batch_idx, optimizer_idx=None):
         x, y = batch
         out = self.compute_distances(x)  # [None, num_protos]
-        _, plabels = self.proto_layer()
+        plabels = self.proto_layer.labels
         winning = stratified_min_pooling(out, plabels)  # [None, num_classes]
         probs = -1.0 * winning
         batch_loss = self.loss(probs, y.long())
@@ -32,7 +31,7 @@ class ProbabilisticLVQ(GLVQ):
     def __init__(self, hparams, rejection_confidence=0.0, **kwargs):
         super().__init__(hparams, **kwargs)
 
-        self.conditional_distribution = None
+        self.conditional_distribution = GaussianPrior(self.hparams.variance)
         self.rejection_confidence = rejection_confidence
 
     def forward(self, x):
@@ -54,10 +53,11 @@ class ProbabilisticLVQ(GLVQ):
     def training_step(self, batch, batch_idx, optimizer_idx=None):
         x, y = batch
         out = self.forward(x)
-        _, plabels = self.proto_layer()
+        plabels = self.proto_layer.labels
         batch_loss = self.loss(out, y, plabels)
-        loss = batch_loss.sum()
-        return loss
+        train_loss = batch_loss.sum()
+        self.log("train_loss", train_loss)
+        return train_loss
 
 
 class SLVQ(ProbabilisticLVQ):
@@ -65,7 +65,6 @@ class SLVQ(ProbabilisticLVQ):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.loss = LossLayer(nllr_loss)
-        self.conditional_distribution = GaussianPrior(self.hparams.variance)
 
 
 class RSLVQ(ProbabilisticLVQ):
@@ -73,7 +72,6 @@ class RSLVQ(ProbabilisticLVQ):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.loss = LossLayer(rslvq_loss)
-        self.conditional_distribution = GaussianPrior(self.hparams.variance)
 
 
 class PLVQ(ProbabilisticLVQ, SiameseGMLVQ):

prototorch/models/unsupervised.py

@@ -53,7 +53,7 @@ class KohonenSOM(NonGradientMixin, UnsupervisedPrototypeModel):
         grid = self._grid.view(-1, 2)
         gd = squared_euclidean_distance(wp, grid)
         nh = torch.exp(-gd / self._sigma**2)
-        protos = self.proto_layer()
+        protos = self.proto_layer.components
         diff = x.unsqueeze(dim=1) - protos
         delta = self._lr * self.hparams.alpha * nh.unsqueeze(-1) * diff
         updated_protos = protos + delta.sum(dim=0)

prototorch/models/vis.py

@@ -117,6 +117,24 @@ class Vis2DAbstract(pl.Callback):
         plt.close()
 
 
+class Vis2D(Vis2DAbstract):
+    def on_epoch_end(self, trainer, pl_module):
+        if not self.precheck(trainer):
+            return True
+
+        x_train, y_train = self.x_train, self.y_train
+        ax = self.setup_ax(xlabel="Data dimension 1",
+                           ylabel="Data dimension 2")
+        self.plot_data(ax, x_train, y_train)
+        mesh_input, xx, yy = mesh2d(x_train, self.border, self.resolution)
+        mesh_input = torch.from_numpy(mesh_input).type_as(x_train)
+        y_pred = pl_module.predict(mesh_input)
+        y_pred = y_pred.cpu().reshape(xx.shape)
+        ax.contourf(xx, yy, y_pred, cmap=self.cmap, alpha=0.35)
+
+        self.log_and_display(trainer, pl_module)
+
+
 class VisGLVQ2D(Vis2DAbstract):
     def on_epoch_end(self, trainer, pl_module):
         if not self.precheck(trainer):
@@ -251,6 +269,8 @@ class VisImgComp(Vis2DAbstract):
                                    size=self.embedding_data,
                                    replace=False)
             data = self.x_train[ind]
+            # print(f"{data.shape=}")
+            # print(f"{self.y_train[ind].shape=}")
             tb.add_embedding(data.view(len(ind), -1),
                              label_img=data,
                              global_step=None,

setup.py

@@ -22,7 +22,7 @@ with open("README.md", "r") as fh:
     long_description = fh.read()
 
 INSTALL_REQUIRES = [
-    "prototorch>=0.7.0",
+    "prototorch>=0.6.0",
     "pytorch_lightning>=1.3.5",
     "torchmetrics",
 ]
@@ -53,7 +53,7 @@ ALL = CLI + DEV + DOCS + EXAMPLES + TESTS
 
 setup(
     name=safe_name("prototorch_" + PLUGIN_NAME),
-    version="0.3.0",
+    version="0.2.0",
     description="Pre-packaged prototype-based "
     "machine learning models using ProtoTorch and PyTorch-Lightning.",
     long_description=long_description,
@@ -63,7 +63,7 @@ setup(
     url=PROJECT_URL,
     download_url=DOWNLOAD_URL,
     license="MIT",
-    python_requires=">=3.6",
+    python_requires=">=3.9",
     install_requires=INSTALL_REQUIRES,
     extras_require={
         "dev": DEV,
@@ -80,9 +80,6 @@ setup(
         "License :: OSI Approved :: MIT License",
         "Natural Language :: English",
         "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.6",
         "Operating System :: OS Independent",
         "Topic :: Scientific/Engineering :: Artificial Intelligence",
         "Topic :: Software Development :: Libraries",

tests/test_examples.sh

@@ -1,27 +1,11 @@
 #! /bin/bash
 
-
-# Read Flags
-gpu=0
-while [ -n "$1" ]; do
-	case "$1" in
-	    --gpu) gpu=1;;
-	    -g) gpu=1;;
-        *) path=$1;;
-	esac
-	shift
-done
-
-python --version
-echo "Using GPU: " $gpu
-
-# Loop
 failed=0
 
-for example in $(find $path -maxdepth 1 -name "*.py")
+for example in $(find $1 -maxdepth 1 -name "*.py")
 do
     echo  -n "$x" $example '... '
-    export DISPLAY= && python $example --fast_dev_run 1 --gpus $gpu &> run_log.txt
+    export DISPLAY= && python $example --fast_dev_run 1 &> run_log.txt
     if [[ $? -ne 0 ]]; then
         echo "FAILED!!"
         cat run_log.txt