Merge branch 'master' into kernel_distances

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.0
+current_version = 0.4.2
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Install Prototorch by running '...'
+2. Run script '...'
+3. See errors
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. Ubuntu 20.10]
+ - Prototorch Version: [e.g. v0.4.0]
+ - Python Version: [e.g. 3.9.5]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/pythonapp.yml

@@ -23,10 +23,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install .
+        pip install .[all]
-    - name: Install extras
-      run: |
-        pip install -r requirements.txt
     - name: Lint with flake8
       run: |
         pip install flake8

.travis.yml

@@ -5,10 +5,8 @@ python: 3.8
 cache:
   directories:
   - "./tests/artifacts"
-  # - "$HOME/.prototorch/datasets"
 install:
-- pip install . --progress-bar off
+- pip install .[all] --progress-bar off
-- pip install -r requirements.txt
 
 # Generate code coverage report
 script:
@@ -25,8 +23,8 @@ deploy:
   password:
     secure: rVQNCxKIuiEtMz4zLSsjdt6spG7cf3miKN5eqjxZfcELALHxAV4w/+CideQObOn3u9emmxb87R9XWKcogqK2MXqnuIcY4mWg7HUqaip1bhz/4YiVXjFILcG6itjX9IUF1DrtjKKRk6xryucSZcEB7yTcXz1hQTb768KWlLlKOVTRNwr7j07eyeafexz/L2ANQCqfOZgS4b0k2AMeDBRPykPULtyeneEFlb6MJZ2MxeqtTNVK4b/6VsQSZwQ9jGJNGWonn5Y287gHmzvEcymSJogTe2taxGBWawPnOsibws9v88DEAHdsEvYdnqEE3hFl0R5La2Lkjd8CjNUYegxioQ57i3WNS3iksq10ZLMCbH29lb9YPG7r6Y8z9H85735kV2gKLdf+o7SPS03TRgjSZKN6pn4pLG0VWkxC6l8VfLuJnRNTHX4g6oLQwOWIBbxybn9Zw/yLjAXAJNgBHt5v86H6Jfi1Va4AhEV6itkoH9IM3/uDhrE/mmorqyVled/CPNtBWNTyoDevLNxMUDnbuhH0JzLki+VOjKnTxEfq12JB8X9faFG5BjvU9oGjPPewrp5DGGzg6KDra7dikciWUxE1eTFFDhMyG1CFGcjKlDvlAGHyI6Kih35egGUeq+N/pitr2330ftM9Dm4rWpOTxPyCI89bXKssx/MgmLG7kSM=
   on:
-  tags: true
+    tags: true
-  skip_existing: true
+    skip_existing: true
 
 # The password is encrypted with:
 # `cd prototorch && travis encrypt your-pypi-api-token --add deploy.password`

docs/source/conf.py

@@ -23,7 +23,7 @@ author = "Jensun Ravichandran"
 
 # The full version, including alpha/beta/rc tags
 #
-release = "0.4.0"
+release = "0.4.2"
 
 # -- General configuration ---------------------------------------------------
 

prototorch/__init__.py

@@ -1,7 +1,7 @@
 """ProtoTorch package."""
 
 # Core Setup
-__version__ = "0.4.0"
+__version__ = "0.4.2"
 
 __all_core__ = [
     "datasets",

prototorch/components/components.py

@@ -4,8 +4,10 @@ import warnings
 from typing import Tuple
 
 import torch
-from prototorch.components.initializers import (ComponentsInitializer,
+from prototorch.components.initializers import (ClassAwareInitializer,
-                                                EqualLabelInitializer,
+                                                ComponentsInitializer,
+                                                EqualLabelsInitializer,
+                                                UnequalLabelsInitializer,
                                                 ZeroReasoningsInitializer)
 from prototorch.functions.initializers import get_initializer
 from torch.nn.parameter import Parameter
@@ -30,12 +32,15 @@ class Components(torch.nn.Module):
         else:
             self._initialize_components(number_of_components, initializer)
 
-    def _initialize_components(self, number_of_components, initializer):
+    def _precheck_initializer(self, initializer):
         if not isinstance(initializer, ComponentsInitializer):
             emsg = f"`initializer` has to be some subtype of " \
                 f"{ComponentsInitializer}. " \
                 f"You have provided: {initializer=} instead."
             raise TypeError(emsg)
+
+    def _initialize_components(self, number_of_components, initializer):
+        self._precheck_initializer(initializer)
         self._components = Parameter(
             initializer.generate(number_of_components))
 
@@ -57,7 +62,7 @@ class LabeledComponents(Components):
     Every Component has a label assigned.
     """
     def __init__(self,
-                 labels=None,
+                 distribution=None,
                  initializer=None,
                  *,
                  initialized_components=None):
@@ -65,15 +70,27 @@ class LabeledComponents(Components):
             super().__init__(initialized_components=initialized_components[0])
             self._labels = initialized_components[1]
         else:
-            self._initialize_labels(labels)
+            self._initialize_labels(distribution)
             super().__init__(number_of_components=len(self._labels),
                              initializer=initializer)
 
-    def _initialize_labels(self, labels):
+    def _initialize_components(self, number_of_components, initializer):
-        if type(labels) == tuple:
+        if isinstance(initializer, ClassAwareInitializer):
-            num_classes, prototypes_per_class = labels
+            self._precheck_initializer(initializer)
-            labels = EqualLabelInitializer(num_classes, prototypes_per_class)
+            self._components = Parameter(
+                initializer.generate(number_of_components, self.distribution))
+        else:
+            super()._initialize_components(self, number_of_components,
+                                           initializer)
 
+    def _initialize_labels(self, distribution):
+        if type(distribution) == tuple:
+            num_classes, prototypes_per_class = distribution
+            labels = EqualLabelsInitializer(num_classes, prototypes_per_class)
+        elif type(distribution) == list:
+            labels = UnequalLabelsInitializer(distribution)
+
+        self.distribution = labels.distribution
         self._labels = labels.generate()
 
     @property

prototorch/components/initializers.py

@@ -1,6 +1,7 @@
 """ProtoTroch Initializers."""
 import warnings
 from collections.abc import Iterable
+from itertools import chain
 
 import torch
 from torch.utils.data import DataLoader, Dataset
@@ -91,6 +92,15 @@ class ClassAwareInitializer(ComponentsInitializer):
         self.clabels = torch.unique(self.labels)
         self.num_classes = len(self.clabels)
 
+    def _get_samples_from_initializer(self, length, dist):
+        if not dist:
+            per_class = length // self.num_classes
+            dist = self.num_classes * [per_class]
+        samples_list = [
+            init.generate(n) for init, n in zip(self.initializers, dist)
+        ]
+        return torch.vstack(samples_list)
+
 
 class StratifiedMeanInitializer(ClassAwareInitializer):
     def __init__(self, arg):
@@ -102,10 +112,9 @@ class StratifiedMeanInitializer(ClassAwareInitializer):
             class_initializer = MeanInitializer(class_data)
             self.initializers.append(class_initializer)
 
-    def generate(self, length):
+    def generate(self, length, dist=[]):
-        per_class = length // self.num_classes
+        samples = self._get_samples_from_initializer(length, dist)
-        samples_list = [init.generate(per_class) for init in self.initializers]
+        return samples
-        return torch.vstack(samples_list)
 
 
 class StratifiedSelectionInitializer(ClassAwareInitializer):
@@ -126,10 +135,8 @@ class StratifiedSelectionInitializer(ClassAwareInitializer):
         mask = torch.bernoulli(n1) - torch.bernoulli(n2)
         return x + (self.noise * mask)
 
-    def generate(self, length):
+    def generate(self, length, dist=[]):
-        per_class = length // self.num_classes
+        samples = self._get_samples_from_initializer(length, dist)
-        samples_list = [init.generate(per_class) for init in self.initializers]
-        samples = torch.vstack(samples_list)
         if self.noise is not None:
             # samples = self.add_noise(samples)
             samples = samples + self.noise
@@ -142,11 +149,29 @@ class LabelsInitializer:
         raise NotImplementedError("Subclasses should implement this!")
 
 
-class EqualLabelInitializer(LabelsInitializer):
+class UnequalLabelsInitializer(LabelsInitializer):
+    def __init__(self, dist):
+        self.dist = dist
+
+    @property
+    def distribution(self):
+        return self.dist
+
+    def generate(self):
+        clabels = range(len(self.dist))
+        labels = list(chain(*[[i] * n for i, n in zip(clabels, self.dist)]))
+        return torch.tensor(labels)
+
+
+class EqualLabelsInitializer(LabelsInitializer):
     def __init__(self, classes, per_class):
         self.classes = classes
         self.per_class = per_class
 
+    @property
+    def distribution(self):
+        return self.classes * [self.per_class]
+
     def generate(self):
         return torch.arange(self.classes).repeat(self.per_class, 1).T.flatten()
 

prototorch/functions/distances.py

@@ -3,8 +3,11 @@
 import numpy as np
 import torch
 
-from prototorch.functions.helper import (_check_shapes, _int_and_mixed_shape,
+from prototorch.functions.helper import (
-                                         equal_int_shape)
+    _check_shapes,
+    _int_and_mixed_shape,
+    equal_int_shape,
+)
 
 
 def squared_euclidean_distance(x, y):
@@ -261,5 +264,86 @@ def tangent_distance(signals, protos, subspaces, squared=False, epsilon=1e-10):
             return diss.permute([1, 0, 2]).squeeze(-1)
 
 
+class KernelDistance:
+    r"""Kernel Distance
+
+    Distance based on a kernel function.
+    """
+    def __init__(self, kernel_fn):
+        self.kernel_fn = kernel_fn
+
+    def __call__(self, x_batch: torch.Tensor, y_batch: torch.Tensor):
+        return self._single_call(x_batch, y_batch)
+
+    def _single_call(self, x, y):
+        remove_dims = []
+        if len(x.shape) == 1:
+            x = x.unsqueeze(0)
+            remove_dims.append(0)
+        if len(y.shape) == 1:
+            y = y.unsqueeze(0)
+            remove_dims.append(-1)
+
+        output = self.kernel_fn(x, x).diag().unsqueeze(1) - 2 * self.kernel_fn(
+            x, y) + self.kernel_fn(y, y).diag()
+
+        for dim in remove_dims:
+            output.squeeze_(dim)
+
+        return torch.sqrt(output)
+
+
+class BatchKernelDistance:
+    r"""Kernel Distance
+
+    Distance based on a kernel function.
+    """
+    def __init__(self, kernel_fn):
+        self.kernel_fn = kernel_fn
+
+    def __call__(self, x_batch: torch.Tensor, y_batch: torch.Tensor):
+        remove_dims = 0
+        # Extend Single inputs
+        if len(x_batch.shape) == 1:
+            x_batch = x_batch.unsqueeze(0)
+            remove_dims += 1
+        if len(y_batch.shape) == 1:
+            y_batch = y_batch.unsqueeze(0)
+            remove_dims += 1
+
+        # Loop over batches
+        output = torch.FloatTensor(len(x_batch), len(y_batch))
+        for i, x in enumerate(x_batch):
+            for j, y in enumerate(y_batch):
+                output[i][j] = self._single_call(x, y)
+
+        for _ in range(remove_dims):
+            output.squeeze_(0)
+
+        return output
+
+    def _single_call(self, x, y):
+        kappa_xx = self.kernel_fn(x, x)
+        kappa_xy = self.kernel_fn(x, y)
+        kappa_yy = self.kernel_fn(y, y)
+
+        squared_distance = kappa_xx - 2 * kappa_xy + kappa_yy
+
+        return torch.sqrt(squared_distance)
+
+
+class SquaredKernelDistance(KernelDistance):
+    r"""Squared Kernel Distance
+
+    Kernel distance without final squareroot.
+    """
+    def single_call(self, x, y):
+        kappa_xx = self.kernel_fn(x, x)
+        kappa_xy = self.kernel_fn(x, y)
+        kappa_yy = self.kernel_fn(y, y)
+
+        return kappa_xx - 2 * kappa_xy + kappa_yy
+
+
 # Aliases
-sed = squared_euclidean_distance
+sed = squared_euclidean_distance

prototorch/functions/kernels.py

@@ -0,0 +1,28 @@
+"""
+Experimental Kernels
+"""
+
+import torch
+
+
+class ExplicitKernel:
+    def __init__(self, projection=torch.nn.Identity()):
+        self.projection = projection
+
+    def __call__(self, x, y):
+        return self.projection(x) @ self.projection(y).T
+
+
+class RadialBasisFunctionKernel:
+    def __init__(self, sigma) -> None:
+        self.s2 = sigma * sigma
+
+    def __call__(self, x, y):
+        remove_dim = False
+        if len(x.shape) > 1:
+            x = x.unsqueeze(1)
+            remove_dim = True
+        output = torch.exp(-torch.sum((x - y)**2, dim=-1) / (2 * self.s2))
+        if remove_dim:
+            output = output.squeeze(1)
+        return output

prototorch/functions/losses.py

@@ -31,3 +31,26 @@ def glvq_loss(distances, target_labels, prototype_labels):
     dp, dm = _get_dp_dm(distances, target_labels, prototype_labels)
     mu = (dp - dm) / (dp + dm)
     return mu
+
+
+def lvq1_loss(distances, target_labels, prototype_labels):
+    """LVQ1 loss function with support for one-hot labels.
+
+    See Section 4 [Sado&Yamada]
+    https://papers.nips.cc/paper/1995/file/9c3b1830513cc3b8fc4b76635d32e692-Paper.pdf
+    """
+    dp, dm = _get_dp_dm(distances, target_labels, prototype_labels)
+    mu = dp
+    mu[dp > dm] = -dm[dp > dm]
+    return mu
+
+
+def lvq21_loss(distances, target_labels, prototype_labels):
+    """LVQ2.1 loss function with support for one-hot labels.
+    
+    See Section 4 [Sado&Yamada]
+    https://papers.nips.cc/paper/1995/file/9c3b1830513cc3b8fc4b76635d32e692-Paper.pdf
+    """
+    dp, dm = _get_dp_dm(distances, target_labels, prototype_labels)
+    mu = dp - dm
+    return mu

prototorch/modules/models.py

@@ -1,8 +1,7 @@
 import torch
 from torch import nn
 
-from prototorch.functions.distances import (euclidean_distance_matrix,
+from prototorch.functions.distances import euclidean_distance_matrix, tangent_distance
-                                            tangent_distance)
 from prototorch.functions.helper import _check_shapes, _int_and_mixed_shape
 from prototorch.functions.normalization import orthogonalization
 from prototorch.modules.prototypes import Prototypes1D

requirements.txt

@@ -1,5 +0,0 @@
-matplotlib==3.1.2
-pytest==5.3.4
-requests==2.22.0
-codecov==2.0.22
-tqdm==4.44.1

setup.py

@@ -1,8 +1,8 @@
 """
-  _____           _     _______             _ 
+  _____           _     _______             _
- |  __ \         | |   |__   __|           | |  
+ |  __ \         | |   |__   __|           | |
  | |__) | __ ___ | |_ ___ | | ___  _ __ ___| |__
- |  ___/ '__/ _ \| __/ _ \| |/ _ \| '__/ __| '_ \ 
+ |  ___/ '__/ _ \| __/ _ \| |/ _ \| '__/ __| '_ \
  | |   | | | (_) | || (_) | | (_) | | | (__| | | |
  |_|   |_|  \___/ \__\___/|_|\___/|_|  \___|_| |_|
 
@@ -21,27 +21,28 @@ INSTALL_REQUIRES = [
     "torchvision>=0.5.0",
     "numpy>=1.9.1",
 ]
+DATASETS = [
+    "requests",
+    "tqdm",
+]
+DEV = ["bumpversion"]
 DOCS = [
     "recommonmark",
     "sphinx",
     "sphinx_rtd_theme",
     "sphinxcontrib-katex",
 ]
-DATASETS = [
-    "requests",
-    "tqdm",
-]
 EXAMPLES = [
     "sklearn",
     "matplotlib",
     "torchinfo",
 ]
-TESTS = ["pytest"]
+TESTS = ["codecov", "pytest"]
-ALL = DOCS + DATASETS + EXAMPLES + TESTS
+ALL = DATASETS + DEV + DOCS + EXAMPLES + TESTS
 
 setup(
     name="prototorch",
-    version="0.4.0",
+    version="0.4.2",
     description="Highly extensible, GPU-supported "
     "Learning Vector Quantization (LVQ) toolbox "
     "built using PyTorch and its nn API.",
@@ -71,6 +72,7 @@ setup(
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Operating System :: OS Independent",
         "Topic :: Scientific/Engineering :: Artificial Intelligence",
         "Topic :: Software Development :: Libraries",

tests/test_functions.py

@@ -5,8 +5,13 @@ import unittest
 import numpy as np
 import torch
 
-from prototorch.functions import (activations, competitions, distances,
+from prototorch.functions import (
-                                  initializers, losses)
+    activations,
+    competitions,
+    distances,
+    initializers,
+    losses,
+)
 
 
 class TestActivations(unittest.TestCase):

tests/test_kernels.py

@@ -0,0 +1,98 @@
+"""ProtoTorch kernels test suite."""
+
+import unittest
+
+import numpy as np
+import torch
+
+from prototorch.functions.distances import KernelDistance
+from prototorch.functions.kernels import ExplicitKernel, RadialBasisFunctionKernel
+
+
+class TestExplicitKernel(unittest.TestCase):
+    def setUp(self):
+        self.single_x = torch.randn(1024)
+        self.single_y = torch.randn(1024)
+
+        self.batch_x = torch.randn(32, 1024)
+        self.batch_y = torch.randn(32, 1024)
+
+    def test_single_values(self):
+        kernel = ExplicitKernel()
+        self.assertEqual(
+            kernel(self.single_x, self.single_y).shape, torch.Size([]))
+
+    def test_single_batch(self):
+        kernel = ExplicitKernel()
+        self.assertEqual(
+            kernel(self.single_x, self.batch_y).shape, torch.Size([32]))
+
+    def test_batch_single(self):
+        kernel = ExplicitKernel()
+        self.assertEqual(
+            kernel(self.batch_x, self.single_y).shape, torch.Size([32]))
+
+    def test_batch_values(self):
+        kernel = ExplicitKernel()
+        self.assertEqual(
+            kernel(self.batch_x, self.batch_y).shape, torch.Size([32, 32]))
+
+
+class TestRadialBasisFunctionKernel(unittest.TestCase):
+    def setUp(self):
+        self.single_x = torch.randn(1024)
+        self.single_y = torch.randn(1024)
+
+        self.batch_x = torch.randn(32, 1024)
+        self.batch_y = torch.randn(32, 1024)
+
+    def test_single_values(self):
+        kernel = RadialBasisFunctionKernel(1)
+        self.assertEqual(
+            kernel(self.single_x, self.single_y).shape, torch.Size([]))
+
+    def test_single_batch(self):
+        kernel = RadialBasisFunctionKernel(1)
+        self.assertEqual(
+            kernel(self.single_x, self.batch_y).shape, torch.Size([32]))
+
+    def test_batch_single(self):
+        kernel = RadialBasisFunctionKernel(1)
+        self.assertEqual(
+            kernel(self.batch_x, self.single_y).shape, torch.Size([32]))
+
+    def test_batch_values(self):
+        kernel = RadialBasisFunctionKernel(1)
+        self.assertEqual(
+            kernel(self.batch_x, self.batch_y).shape, torch.Size([32, 32]))
+
+
+class TestKernelDistance(unittest.TestCase):
+    def setUp(self):
+        self.single_x = torch.randn(1024)
+        self.single_y = torch.randn(1024)
+
+        self.batch_x = torch.randn(32, 1024)
+        self.batch_y = torch.randn(32, 1024)
+
+        self.kernel = ExplicitKernel()
+
+    def test_single_values(self):
+        distance = KernelDistance(self.kernel)
+        self.assertEqual(
+            distance(self.single_x, self.single_y).shape, torch.Size([]))
+
+    def test_single_batch(self):
+        distance = KernelDistance(self.kernel)
+        self.assertEqual(
+            distance(self.single_x, self.batch_y).shape, torch.Size([32]))
+
+    def test_batch_single(self):
+        distance = KernelDistance(self.kernel)
+        self.assertEqual(
+            distance(self.batch_x, self.single_y).shape, torch.Size([32]))
+
+    def test_batch_values(self):
+        distance = KernelDistance(self.kernel)
+        self.assertEqual(
+            distance(self.batch_x, self.batch_y).shape, torch.Size([32, 32]))

tox.ini

@@ -1,15 +0,0 @@
-# tox (https://tox.readthedocs.io/) is a tool for running tests
-# in multiple virtualenvs. This configuration file will run the
-# test suite on all supported python versions. To use it, "pip install tox"
-# and then run "tox" from this directory.
-
-[tox]
-envlist = py36,py37,py38
-
-[testenv]
-deps =
-    pytest
-    coverage
-commands =
-    pip install -e .
-    coverage run -m pytest