Merge branch 'master' into kernel_distances

GLVQ training is unstable when prototypes are initialized exactly to datapoints
without small shifts. Perhaps because of zero distances?

.bumpversion.cfg

@@ -1,21 +1,13 @@
 [bumpversion]
-current_version = 0.1.1-dev0
+current_version = 0.4.2
 commit = True
 tag = True
-parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
 serialize = 
-	{major}.{minor}.{patch}-{release}{build}
 	{major}.{minor}.{patch}
 
-[bumpversion:part:release]
-optional_value = prod
-first_value = dev
-values = 
-	dev
-	rc
-	prod
-
 [bumpversion:file:setup.py]
 
 [bumpversion:file:./prototorch/__init__.py]
 
+[bumpversion:file:./docs/source/conf.py]

.codacy.yml

@@ -0,0 +1,15 @@
+# To validate the contents of your configuration file
+# run the following command in the folder where the configuration file is located:
+# codacy-analysis-cli validate-configuration --directory `pwd`
+# To analyse, run:
+# codacy-analysis-cli analyse --tool remark-lint --directory `pwd`
+---
+engines:
+  pylintpython3:
+    exclude_paths:
+      - config/engines.yml
+  remark-lint:
+    exclude_paths:
+      - config/engines.yml
+exclude_paths:
+  - 'tests/**'

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Install Prototorch by running '...'
+2. Run script '...'
+3. See errors
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. Ubuntu 20.10]
+ - Prototorch Version: [e.g. v0.4.0]
+ - Python Version: [e.g. 3.9.5]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/pythonapp.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install .
+        pip install .[all]
     - name: Lint with flake8
       run: |
         pip install flake8

.readthedocs.yml

@@ -0,0 +1,27 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+  fail_on_warning: true
+
+# Build documentation with MkDocs
+# mkdocs:
+#   configuration: mkdocs.yml
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.8
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - all

.travis.yml

@@ -4,16 +4,31 @@ language: python
 python: 3.8
 cache:
   directories:
-    - ./tests/artifacts
+  - "./tests/artifacts"
-
 install:
-  - pip install . --progress-bar off
+- pip install .[all] --progress-bar off
-  - pip install codecov
-  - pip install pytest
 
+# Generate code coverage report
 script:
-  - coverage run -m pytest
+- coverage run -m pytest
 
 # Push the results to codecov
 after_success:
-  - codecov
+- bash <(curl -s https://codecov.io/bash)
+
+# Publish on PyPI
+deploy:
+  provider: pypi
+  username: __token__
+  password:
+    secure: rVQNCxKIuiEtMz4zLSsjdt6spG7cf3miKN5eqjxZfcELALHxAV4w/+CideQObOn3u9emmxb87R9XWKcogqK2MXqnuIcY4mWg7HUqaip1bhz/4YiVXjFILcG6itjX9IUF1DrtjKKRk6xryucSZcEB7yTcXz1hQTb768KWlLlKOVTRNwr7j07eyeafexz/L2ANQCqfOZgS4b0k2AMeDBRPykPULtyeneEFlb6MJZ2MxeqtTNVK4b/6VsQSZwQ9jGJNGWonn5Y287gHmzvEcymSJogTe2taxGBWawPnOsibws9v88DEAHdsEvYdnqEE3hFl0R5La2Lkjd8CjNUYegxioQ57i3WNS3iksq10ZLMCbH29lb9YPG7r6Y8z9H85735kV2gKLdf+o7SPS03TRgjSZKN6pn4pLG0VWkxC6l8VfLuJnRNTHX4g6oLQwOWIBbxybn9Zw/yLjAXAJNgBHt5v86H6Jfi1Va4AhEV6itkoH9IM3/uDhrE/mmorqyVled/CPNtBWNTyoDevLNxMUDnbuhH0JzLki+VOjKnTxEfq12JB8X9faFG5BjvU9oGjPPewrp5DGGzg6KDra7dikciWUxE1eTFFDhMyG1CFGcjKlDvlAGHyI6Kih35egGUeq+N/pitr2330ftM9Dm4rWpOTxPyCI89bXKssx/MgmLG7kSM=
+  on:
+    tags: true
+    skip_existing: true
+
+# The password is encrypted with:
+# `cd prototorch && travis encrypt your-pypi-api-token --add deploy.password`
+# See https://docs.travis-ci.com/user/deployment/pypi and
+# https://github.com/travis-ci/travis.rb#installation
+# for more details
+# Note: The encrypt command does not work well in ZSH.

MANIFEST.in

@@ -1,6 +1,8 @@
 include .bumpversion.cfg
 include LICENSE
 include tox.ini
+include *.md
+include *.txt
 include *.yml
 recursive-include docs *.bat
 recursive-include docs *.png

README.md

@@ -1,63 +1,58 @@
-# ProtoTorch
+# ProtoTorch: Prototype Learning in PyTorch
 
-ProtoTorch is a PyTorch-based Python toolbox for bleeding-edge research in
+![ProtoTorch Logo](https://prototorch.readthedocs.io/en/latest/_static/horizontal-lockup.png)
-prototype-based machine learning algorithms.
 
 [![Build Status](https://travis-ci.org/si-cim/prototorch.svg?branch=master)](https://travis-ci.org/si-cim/prototorch)
-[![GitHub version](https://badge.fury.io/gh/si-cim%2Fprototorch.svg)](https://badge.fury.io/gh/si-cim%2Fprototorch)
-[![PyPI version](https://badge.fury.io/py/prototorch.svg)](https://badge.fury.io/py/prototorch)
 ![tests](https://github.com/si-cim/prototorch/workflows/tests/badge.svg)
+[![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/si-cim/prototorch?color=yellow&label=version)](https://github.com/si-cim/prototorch/releases)
+[![PyPI](https://img.shields.io/pypi/v/prototorch)](https://pypi.org/project/prototorch/)
 [![codecov](https://codecov.io/gh/si-cim/prototorch/branch/master/graph/badge.svg)](https://codecov.io/gh/si-cim/prototorch)
-[![Downloads](https://pepy.tech/badge/prototorch)](https://pepy.tech/project/prototorch)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/76273904bf9343f0a8b29cd8aca242e7)](https://www.codacy.com/gh/si-cim/prototorch?utm_source=github.com&utm_medium=referral&utm_content=si-cim/prototorch&utm_campaign=Badge_Grade)
+![PyPI - Downloads](https://img.shields.io/pypi/dm/prototorch?color=blue)
 [![GitHub license](https://img.shields.io/github/license/si-cim/prototorch)](https://github.com/si-cim/prototorch/blob/master/LICENSE)
 
+*Tensorflow users, see:* [ProtoFlow](https://github.com/si-cim/protoflow)
+
 ## Description
 
 This is a Python toolbox brewed at the Mittweida University of Applied Sciences
-in Germany for bleeding-edge research in Learning Vector Quantization (LVQ)
+in Germany for bleeding-edge research in Prototype-based Machine Learning
-and potentially other prototype-based methods. Although, there are
+methods and other interpretable models. The focus of ProtoTorch is ease-of-use,
-other (perhaps more extensive) LVQ toolboxes available out there, the focus of
+extensibility and speed.
-ProtoTorch is ease-of-use, extensibility and speed.
-
-Many popular prototype-based Machine Learning (ML) algorithms like K-Nearest
-Neighbors (KNN), Generalized Learning Vector Quantization (GLVQ) and Generalized
-Matrix Learning Vector Quantization (GMLVQ) are implemented using the "nn" API
-provided by PyTorch.
 
 ## Installation
 
 ProtoTorch can be installed using `pip`.
+```bash
+pip install -U prototorch
 ```
-pip install prototorch
+To also install the extras, use
+```bash
+pip install -U prototorch[all]
 ```
 
+*Note: If you're using [ZSH](https://www.zsh.org/) (which is also the default
+shell on MacOS now), the square brackets `[ ]` have to be escaped like so:
+`\[\]`, making the install command `pip install -U prototorch\[all\]`.*
+
 To install the bleeding-edge features and improvements:
-```
+```bash
 git clone https://github.com/si-cim/prototorch.git
-git checkout dev
 cd prototorch
-pip install -e .
+git checkout dev
+pip install -e .[all]
 ```
 
-## Usage
+## Documentation
 
-ProtoTorch is modular. It is very easy to use the modular pieces provided by
+The documentation is available at <https://www.prototorch.ml/en/latest/>. Should
-ProtoTorch, like the layers, losses, callbacks and metrics to build your own
+that link not work try <https://prototorch.readthedocs.io/en/latest/>.
-prototype-based(instance-based) models. These pieces blend-in seamlessly with
-numpy and PyTorch to allow you mix and match the modules from ProtoTorch with
-other PyTorch modules.
-
-ProtoTorch comes prepackaged with many popular LVQ algorithms in a convenient
-API, with more algorithms and techniques coming soon. If you would simply like
-to be able to use those algorithms to train large ML models on a GPU, ProtoTorch
-lets you do this without requiring a black-belt in high-performance Tensor
-computation.
 
 ## Bibtex
 
 If you would like to cite the package, please use this:
 ```bibtex
-@misc{Ravichandran2020,
+@misc{Ravichandran2020b,
   author = {Ravichandran, J},
   title = {ProtoTorch},
   year = {2020},

RELEASE.md

@@ -1,3 +1,16 @@
-# Release 0.1.0-dev0
+# ProtoTorch Releases
+
+## Release 0.2.0
+
+### Includes
+- Fixes in example scripts.
+
+## Release 0.1.1-dev0
+
+### Includes
+- Minor bugfixes.
+- 100% line coverage.
+
+## Release 0.1.0-dev0
 
 Initial public release of ProtoTorch.

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= python3 -m sphinx
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+  set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+  echo.
+  echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+  echo.installed, then set the SPHINXBUILD environment variable to point
+  echo.to the full path of the 'sphinx-build' executable. Alternatively you
+  echo.may add the Sphinx directory to PATH.
+  echo.
+  echo.If you don't have Sphinx installed, grab it from
+  echo.http://sphinx-doc.org/
+  exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/requirements.txt

@@ -0,0 +1,4 @@
+torch==1.6.0
+matplotlib==3.1.2
+sphinx_rtd_theme==0.5.0
+sphinxcontrib-katex==0.6.1

docs/source/api.rst

@@ -0,0 +1,46 @@
+.. ProtoFlow API Reference
+
+ProtoFlow API Reference
+======================================
+
+Datasets
+--------------------------------------
+.. automodule:: prototorch.datasets
+   :members:
+   :undoc-members:
+
+Functions
+--------------------------------------
+
+**Dimensions:**
+
+- :math:`B` ... Batch size
+- :math:`P` ... Number of prototypes
+- :math:`n_x` ... Data dimension for vectorial data
+- :math:`n_w` ... Data dimension for vectorial prototypes
+
+Activations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: prototorch.functions.activations
+   :members:
+   :exclude-members: register_activation, get_activation
+   :undoc-members:
+
+Distances
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: prototorch.functions.distances
+   :members:
+   :exclude-members: sed
+   :undoc-members:
+
+Modules
+--------------------------------------
+.. automodule:: prototorch.modules
+   :members:
+   :undoc-members:
+
+Utilities
+--------------------------------------
+.. automodule:: prototorch.utils
+   :members:
+   :undoc-members:

docs/source/conf.py

@@ -0,0 +1,188 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("../../"))
+
+# -- Project information -----------------------------------------------------
+
+project = "ProtoTorch"
+copyright = "2021, Jensun Ravichandran"
+author = "Jensun Ravichandran"
+
+# The full version, including alpha/beta/rc tags
+#
+release = "0.4.2"
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+needs_sphinx = "1.6"
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named "sphinx.ext.*") or your custom
+# ones.
+extensions = [
+    "recommonmark",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.doctest",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
+    "sphinx_rtd_theme",
+    "sphinxcontrib.katex",
+]
+
+# katex_prerender = True
+katex_prerender = False
+
+napoleon_use_ivar = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+source_suffix = [".rst", ".md"]
+
+# The master toctree document.
+master_doc = "index"
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use. Choose from:
+# ["default", "emacs", "friendly", "colorful", "autumn", "murphy", "manni",
+#  "monokai", "perldoc", "pastie", "borland", "trac", "native", "fruity", "bw",
+#  "vim", "vs", "tango", "rrt", "xcode", "igor", "paraiso-light", "paraiso-dark",
+#  "lovelace", "algol", "algol_nu", "arduino", "rainbo w_dash", "abap",
+#  "solarized-dark", "solarized-light", "sas", "stata", "stata-light",
+#  "stata-dark", "inkpot"]
+pygments_style = "monokai"
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+# Disable docstring inheritance
+autodoc_inherit_docstrings = False
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+# https://sphinx-themes.org/
+html_theme = "sphinx_rtd_theme"
+
+html_logo = "_static/img/horizontal-lockup.png"
+
+html_theme_options = {
+    "logo_only": True,
+    "display_version": True,
+    "prev_next_buttons_location": "bottom",
+    "style_external_links": False,
+    "style_nav_header_background": "#ffffff",
+    # Toc options
+    "collapse_navigation": True,
+    "sticky_navigation": True,
+    "navigation_depth": 4,
+    "includehidden": True,
+    "titles_only": False,
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+html_css_files = [
+    "https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css",
+]
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "protoflowdoc"
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ("letterpaper" or "a4paper").
+    #
+    # "papersize": "letterpaper",
+    # The font size ("10pt", "11pt" or "12pt").
+    #
+    # "pointsize": "10pt",
+    # Additional stuff for the LaTeX preamble.
+    #
+    # "preamble": "",
+    # Latex figure (float) alignment
+    #
+    # "figure_align": "htbp",
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (
+        master_doc,
+        "prototorch.tex",
+        "ProtoTorch Documentation",
+        "Jensun Ravichandran",
+        "manual",
+    ),
+]
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(master_doc, "ProtoTorch", "ProtoTorch Documentation", [author],
+              1)]
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc,
+        "prototorch",
+        "ProtoTorch Documentation",
+        author,
+        "prototorch",
+        "Prototype-based machine learning in PyTorch.",
+        "Miscellaneous",
+    ),
+]
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/", None),
+    "numpy": ("https://docs.scipy.org/doc/numpy/", None),
+}
+
+# -- Options for Epub output ----------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-epub-output
+
+epub_cover = ()
+version = release

docs/source/index.rst

@@ -0,0 +1,22 @@
+.. ProtoTorch documentation master file
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+About ProtoTorch
+================
+
+.. toctree::
+   :hidden:
+   :maxdepth: 3
+   :caption: Contents:
+
+   self
+   api
+
+ProtoTorch is a PyTorch-based Python toolbox for bleeding-edge
+research in prototype-based machine learning algorithms.
+
+Indices
+=======
+* :ref:`genindex`
+* :ref:`modindex`

examples/glvq_iris.py

@@ -1,18 +1,20 @@
-"""ProtoTorch GLVQ example using 2D Iris data"""
+"""ProtoTorch GLVQ example using 2D Iris data."""
 
 import numpy as np
 import torch
 from matplotlib import pyplot as plt
 from sklearn.datasets import load_iris
 from sklearn.preprocessing import StandardScaler
+from torchinfo import summary
 
+from prototorch.functions.competitions import wtac
 from prototorch.functions.distances import euclidean_distance
 from prototorch.modules.losses import GLVQLoss
-from prototorch.modules.prototypes import AddPrototypes1D
+from prototorch.modules.prototypes import Prototypes1D
 
 # Prepare and preprocess the data
 scaler = StandardScaler()
-x_train, y_train = load_iris(True)
+x_train, y_train = load_iris(return_X_y=True)
 x_train = x_train[:, [0, 2]]
 scaler.fit(x_train)
 x_train = scaler.transform(x_train)
@@ -20,16 +22,20 @@ x_train = scaler.transform(x_train)
 
 # Define the GLVQ model
 class Model(torch.nn.Module):
-    def __init__(self, **kwargs):
+    def __init__(self):
+        """GLVQ model for training on 2D Iris data."""
         super().__init__()
-        self.p1 = AddPrototypes1D(input_dim=2,
+        self.proto_layer = Prototypes1D(
-                                  prototypes_per_class=1,
+            input_dim=2,
-                                  nclasses=3,
+            prototypes_per_class=3,
-                                  prototype_initializer='zeros')
+            nclasses=3,
+            prototype_initializer="stratified_random",
+            data=[x_train, y_train],
+        )
 
     def forward(self, x):
-        protos = self.p1.prototypes
+        protos = self.proto_layer.prototypes
-        plabels = self.p1.prototype_labels
+        plabels = self.proto_layer.prototype_labels
         dis = euclidean_distance(x, protos)
         return dis, plabels
 
@@ -37,17 +43,30 @@ class Model(torch.nn.Module):
 # Build the GLVQ model
 model = Model()
 
+# Print summary using torchinfo (might be buggy/incorrect)
+print(summary(model))
+
 # Optimize using SGD optimizer from `torch.optim`
 optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
-criterion = GLVQLoss(squashing='sigmoid_beta', beta=10)
+criterion = GLVQLoss(squashing="sigmoid_beta", beta=10)
+
+x_in = torch.Tensor(x_train)
+y_in = torch.Tensor(y_train)
 
 # Training loop
-fig = plt.figure('Prototype Visualization')
+title = "Prototype Visualization"
+fig = plt.figure(title)
 for epoch in range(70):
-    # Compute loss.
+    # Compute loss
-    distances, plabels = model(torch.tensor(x_train))
+    dis, plabels = model(x_in)
-    loss = criterion([distances, plabels], torch.tensor(y_train))
+    loss = criterion([dis, plabels], y_in)
-    print(f'Epoch: {epoch + 1:03d} Loss: {loss.item():02.02f}')
+    with torch.no_grad():
+        pred = wtac(dis, plabels)
+        correct = pred.eq(y_in.view_as(pred)).sum().item()
+    acc = 100.0 * correct / len(x_train)
+    print(
+        f"Epoch: {epoch + 1:03d} Loss: {loss.item():05.02f} Acc: {acc:05.02f}%"
+    )
 
     # Take a gradient descent step
     optimizer.zero_grad()
@@ -55,44 +74,41 @@ for epoch in range(70):
     optimizer.step()
 
     # Get the prototypes form the model
-    protos = model.p1.prototypes.data.numpy()
+    protos = model.proto_layer.prototypes.data.numpy()
+    if np.isnan(np.sum(protos)):
+        print("Stopping training because of `nan` in prototypes.")
+        break
 
     # Visualize the data and the prototypes
     ax = fig.gca()
     ax.cla()
-    cmap = 'viridis'
+    ax.set_title(title)
-    ax.scatter(x_train[:, 0], x_train[:, 1], c=y_train, edgecolor='k')
+    ax.set_xlabel("Data dimension 1")
-    ax.scatter(protos[:, 0],
+    ax.set_ylabel("Data dimension 2")
-               protos[:, 1],
+    cmap = "viridis"
-               c=plabels,
+    ax.scatter(x_train[:, 0], x_train[:, 1], c=y_train, edgecolor="k")
-               cmap=cmap,
+    ax.scatter(
-               edgecolor='k',
+        protos[:, 0],
-               marker='D',
+        protos[:, 1],
-               s=50)
+        c=plabels,
+        cmap=cmap,
+        edgecolor="k",
+        marker="D",
+        s=50,
+    )
 
     # Paint decision regions
-    border = 1
-    resolution = 50
     x = np.vstack((x_train, protos))
-    x_min, x_max = x[:, 0].min(), x[:, 0].max()
+    x_min, x_max = x[:, 0].min() - 1, x[:, 0].max() + 1
-    y_min, y_max = x[:, 1].min(), x[:, 1].max()
+    y_min, y_max = x[:, 1].min() - 1, x[:, 1].max() + 1
-    x_min, x_max = x_min - border, x_max + border
+    xx, yy = np.meshgrid(np.arange(x_min, x_max, 1 / 50),
-    y_min, y_max = y_min - border, y_max + border
+                         np.arange(y_min, y_max, 1 / 50))
-    try:
-        xx, yy = np.meshgrid(np.arange(x_min, x_max, 1.0 / resolution),
-                             np.arange(y_min, y_max, 1.0 / resolution))
-    except ValueError as ve:
-        print(ve)
-        raise ValueError(f'x_min: {x_min}, x_max: {x_max}. '
-                         f'x_min - x_max is {x_max - x_min}.')
-    except MemoryError as me:
-        print(me)
-        raise ValueError('Too many points. ' 'Try reducing the resolution.')
     mesh_input = np.c_[xx.ravel(), yy.ravel()]
 
-    torch_input = torch.from_numpy(mesh_input)
+    torch_input = torch.Tensor(mesh_input)
     d = model(torch_input)[0]
-    y_pred = np.argmin(d.detach().numpy(), axis=1)
+    w_indices = torch.argmin(d, dim=1)
+    y_pred = torch.index_select(plabels, 0, w_indices)
     y_pred = y_pred.reshape(xx.shape)
 
     # Plot voronoi regions
@@ -100,4 +116,5 @@ for epoch in range(70):
 
     ax.set_xlim(left=x_min + 0, right=x_max - 0)
     ax.set_ylim(bottom=y_min + 0, top=y_max - 0)
+
     plt.pause(0.1)

examples/gmlvq_tecator.py

@@ -0,0 +1,104 @@
+"""ProtoTorch "siamese" GMLVQ example using Tecator."""
+
+import matplotlib.pyplot as plt
+import torch
+from torch.utils.data import DataLoader
+
+from prototorch.datasets.tecator import Tecator
+from prototorch.functions.distances import sed
+from prototorch.modules import Prototypes1D
+from prototorch.modules.losses import GLVQLoss
+from prototorch.utils.colors import get_legend_handles
+
+# Prepare the dataset and dataloader
+train_data = Tecator(root="./artifacts", train=True)
+train_loader = DataLoader(train_data, batch_size=128, shuffle=True)
+
+
+class Model(torch.nn.Module):
+    def __init__(self, **kwargs):
+        """GMLVQ model as a siamese network."""
+        super().__init__()
+        x, y = train_data.data, train_data.targets
+        self.p1 = Prototypes1D(
+            input_dim=100,
+            prototypes_per_class=2,
+            nclasses=2,
+            prototype_initializer="stratified_random",
+            data=[x, y],
+        )
+        self.omega = torch.nn.Linear(in_features=100,
+                                     out_features=100,
+                                     bias=False)
+        torch.nn.init.eye_(self.omega.weight)
+
+    def forward(self, x):
+        protos = self.p1.prototypes
+        plabels = self.p1.prototype_labels
+
+        # Process `x` and `protos` through `omega`
+        x_map = self.omega(x)
+        protos_map = self.omega(protos)
+
+        # Compute distances and output
+        dis = sed(x_map, protos_map)
+        return dis, plabels
+
+
+# Build the GLVQ model
+model = Model()
+
+# Print a summary of the model
+print(model)
+
+# Optimize using Adam optimizer from `torch.optim`
+optimizer = torch.optim.Adam(model.parameters(), lr=0.001_0)
+scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=75, gamma=0.1)
+criterion = GLVQLoss(squashing="identity", beta=10)
+
+# Training loop
+for epoch in range(150):
+    epoch_loss = 0.0  # zero-out epoch loss
+    optimizer.zero_grad()  # zero-out gradients
+    for xb, yb in train_loader:
+        # Compute loss
+        distances, plabels = model(xb)
+        loss = criterion([distances, plabels], yb)
+        epoch_loss += loss.item()
+        # Backprop
+        loss.backward()
+    # Take a gradient descent step
+    optimizer.step()
+    scheduler.step()
+
+    lr = optimizer.param_groups[0]["lr"]
+    print(f"Epoch: {epoch + 1:03d} Loss: {epoch_loss:06.02f} lr: {lr:07.06f}")
+
+# Get the omega matrix form the model
+omega = model.omega.weight.data.numpy().T
+
+# Visualize the lambda matrix
+title = "Lambda Matrix Visualization"
+fig = plt.figure(title)
+ax = fig.gca()
+ax.set_title(title)
+im = ax.imshow(omega.dot(omega.T), cmap="viridis")
+plt.show()
+
+# Get the prototypes form the model
+protos = model.p1.prototypes.data.numpy()
+plabels = model.p1.prototype_labels
+
+# Visualize the prototypes
+title = "Tecator Prototypes"
+fig = plt.figure(title)
+ax = fig.gca()
+ax.set_title(title)
+ax.set_xlabel("Spectral frequencies")
+ax.set_ylabel("Absorption")
+clabels = ["Class 0 - Low fat", "Class 1 - High fat"]
+handles, colors = get_legend_handles(clabels, marker="line", zero_indexed=True)
+for x, y in zip(protos, plabels):
+    ax.plot(x, c=colors[int(y)])
+ax.legend(handles, clabels)
+plt.show()

examples/gtlvq_mnist.py

@@ -0,0 +1,184 @@
+"""
+ProtoTorch GTLVQ example using MNIST data.
+The GTLVQ is placed as an classification model on
+top of a CNN, considered as featurer extractor.
+Initialization of subpsace and prototypes in
+Siamnese fashion
+For more info about GTLVQ see:
+DOI:10.1109/IJCNN.2016.7727534
+"""
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torchvision
+from torchvision import transforms
+
+from prototorch.functions.helper import calculate_prototype_accuracy
+from prototorch.modules.losses import GLVQLoss
+from prototorch.modules.models import GTLVQ
+
+# Parameters and options
+n_epochs = 50
+batch_size_train = 64
+batch_size_test = 1000
+learning_rate = 0.1
+momentum = 0.5
+log_interval = 10
+cuda = "cuda:1"
+random_seed = 1
+device = torch.device(cuda if torch.cuda.is_available() else "cpu")
+
+# Configures reproducability
+torch.manual_seed(random_seed)
+np.random.seed(random_seed)
+
+# Prepare and preprocess the data
+train_loader = torch.utils.data.DataLoader(
+    torchvision.datasets.MNIST(
+        "./files/",
+        train=True,
+        download=True,
+        transform=torchvision.transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Normalize((0.1307, ), (0.3081, ))
+        ]),
+    ),
+    batch_size=batch_size_train,
+    shuffle=True,
+)
+
+test_loader = torch.utils.data.DataLoader(
+    torchvision.datasets.MNIST(
+        "./files/",
+        train=False,
+        download=True,
+        transform=torchvision.transforms.Compose([
+            transforms.ToTensor(),
+            transforms.Normalize((0.1307, ), (0.3081, ))
+        ]),
+    ),
+    batch_size=batch_size_test,
+    shuffle=True,
+)
+
+
+# Define the GLVQ model plus appropriate feature extractor
+class CNNGTLVQ(torch.nn.Module):
+    def __init__(
+        self,
+        num_classes,
+        subspace_data,
+        prototype_data,
+        tangent_projection_type="local",
+        prototypes_per_class=2,
+        bottleneck_dim=128,
+    ):
+        super(CNNGTLVQ, self).__init__()
+
+        # Feature Extractor - Simple CNN
+        self.fe = nn.Sequential(
+            nn.Conv2d(1, 32, 3, 1),
+            nn.ReLU(),
+            nn.Conv2d(32, 64, 3, 1),
+            nn.ReLU(),
+            nn.MaxPool2d(2),
+            nn.Dropout(0.25),
+            nn.Flatten(),
+            nn.Linear(9216, bottleneck_dim),
+            nn.Dropout(0.5),
+            nn.LeakyReLU(),
+            nn.LayerNorm(bottleneck_dim),
+        )
+
+        # Forward pass of subspace and prototype initialization data through feature extractor
+        subspace_data = self.fe(subspace_data)
+        prototype_data[0] = self.fe(prototype_data[0])
+
+        # Initialization of GTLVQ
+        self.gtlvq = GTLVQ(
+            num_classes,
+            subspace_data,
+            prototype_data,
+            tangent_projection_type=tangent_projection_type,
+            feature_dim=bottleneck_dim,
+            prototypes_per_class=prototypes_per_class,
+        )
+
+    def forward(self, x):
+        # Feature Extraction
+        x = self.fe(x)
+
+        # GTLVQ Forward pass
+        dis = self.gtlvq(x)
+        return dis
+
+
+# Get init data
+subspace_data = torch.cat(
+    [next(iter(train_loader))[0],
+     next(iter(test_loader))[0]])
+prototype_data = next(iter(train_loader))
+
+# Build the CNN GTLVQ  model
+model = CNNGTLVQ(
+    10,
+    subspace_data,
+    prototype_data,
+    tangent_projection_type="local",
+    bottleneck_dim=128,
+).to(device)
+
+# Optimize using SGD optimizer from `torch.optim`
+optimizer = torch.optim.Adam(
+    [{
+        "params": model.fe.parameters()
+    }, {
+        "params": model.gtlvq.parameters()
+    }],
+    lr=learning_rate,
+)
+criterion = GLVQLoss(squashing="sigmoid_beta", beta=10)
+
+# Training loop
+for epoch in range(n_epochs):
+    for batch_idx, (x_train, y_train) in enumerate(train_loader):
+        model.train()
+        x_train, y_train = x_train.to(device), y_train.to(device)
+        optimizer.zero_grad()
+
+        distances = model(x_train)
+        plabels = model.gtlvq.cls.prototype_labels.to(device)
+
+        # Compute loss.
+        loss = criterion([distances, plabels], y_train)
+        loss.backward()
+        optimizer.step()
+
+        # GTLVQ uses projected SGD, which means to orthogonalize the subspaces after every gradient update.
+        model.gtlvq.orthogonalize_subspace()
+
+        if batch_idx % log_interval == 0:
+            acc = calculate_prototype_accuracy(distances, y_train, plabels)
+            print(
+                f"Epoch: {epoch + 1:02d}/{n_epochs:02d} Epoch Progress: {100. * batch_idx / len(train_loader):02.02f} % Loss: {loss.item():02.02f} \
+              Train Acc: {acc.item():02.02f}")
+
+    # Test
+    with torch.no_grad():
+        model.eval()
+        correct = 0
+        total = 0
+        for x_test, y_test in test_loader:
+            x_test, y_test = x_test.to(device), y_test.to(device)
+            test_distances = model(torch.tensor(x_test))
+            test_plabels = model.gtlvq.cls.prototype_labels.to(device)
+            i = torch.argmin(test_distances, 1)
+            correct += torch.sum(y_test == test_plabels[i])
+            total += y_test.size(0)
+        print("Accuracy of the network on the test images: %d %%" %
+              (torch.true_divide(correct, total) * 100))
+
+# Save the model
+PATH = "./glvq_mnist_model.pth"
+torch.save(model.state_dict(), PATH)

examples/lgmlvq_iris.py

@@ -0,0 +1,110 @@
+"""ProtoTorch LGMLVQ example using 2D Iris data."""
+
+import numpy as np
+import torch
+from matplotlib import pyplot as plt
+from sklearn.datasets import load_iris
+from sklearn.metrics import accuracy_score
+
+from prototorch.functions.competitions import stratified_min
+from prototorch.functions.distances import lomega_distance
+from prototorch.functions.init import eye_
+from prototorch.modules.losses import GLVQLoss
+from prototorch.modules.prototypes import Prototypes1D
+
+# Prepare training data
+x_train, y_train = load_iris(True)
+x_train = x_train[:, [0, 2]]
+
+
+# Define the model
+class Model(torch.nn.Module):
+    def __init__(self):
+        """Local-GMLVQ model."""
+        super().__init__()
+        self.p1 = Prototypes1D(
+            input_dim=2,
+            prototype_distribution=[1, 2, 2],
+            prototype_initializer="stratified_random",
+            data=[x_train, y_train],
+        )
+        omegas = torch.zeros(5, 2, 2)
+        self.omegas = torch.nn.Parameter(omegas)
+        eye_(self.omegas)
+
+    def forward(self, x):
+        protos = self.p1.prototypes
+        plabels = self.p1.prototype_labels
+        omegas = self.omegas
+        dis = lomega_distance(x, protos, omegas)
+        return dis, plabels
+
+
+# Build the model
+model = Model()
+
+# Optimize using Adam optimizer from `torch.optim`
+optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
+criterion = GLVQLoss(squashing="sigmoid_beta", beta=10)
+
+x_in = torch.Tensor(x_train)
+y_in = torch.Tensor(y_train)
+
+# Training loop
+title = "Prototype Visualization"
+fig = plt.figure(title)
+for epoch in range(100):
+    # Compute loss
+    dis, plabels = model(x_in)
+    loss = criterion([dis, plabels], y_in)
+    y_pred = np.argmin(stratified_min(dis, plabels).detach().numpy(), axis=1)
+    acc = accuracy_score(y_train, y_pred)
+    log_string = f"Epoch: {epoch + 1:03d} Loss: {loss.item():05.02f} "
+    log_string += f"Acc: {acc * 100:05.02f}%"
+    print(log_string)
+
+    # Take a gradient descent step
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+
+    # Get the prototypes form the model
+    protos = model.p1.prototypes.data.numpy()
+
+    # Visualize the data and the prototypes
+    ax = fig.gca()
+    ax.cla()
+    ax.set_title(title)
+    ax.set_xlabel("Data dimension 1")
+    ax.set_ylabel("Data dimension 2")
+    cmap = "viridis"
+    ax.scatter(x_train[:, 0], x_train[:, 1], c=y_train, edgecolor="k")
+    ax.scatter(
+        protos[:, 0],
+        protos[:, 1],
+        c=plabels,
+        cmap=cmap,
+        edgecolor="k",
+        marker="D",
+        s=50,
+    )
+
+    # Paint decision regions
+    x = np.vstack((x_train, protos))
+    x_min, x_max = x[:, 0].min() - 1, x[:, 0].max() + 1
+    y_min, y_max = x[:, 1].min() - 1, x[:, 1].max() + 1
+    xx, yy = np.meshgrid(np.arange(x_min, x_max, 1 / 50),
+                         np.arange(y_min, y_max, 1 / 50))
+    mesh_input = np.c_[xx.ravel(), yy.ravel()]
+
+    d, plabels = model(torch.Tensor(mesh_input))
+    y_pred = np.argmin(stratified_min(d, plabels).detach().numpy(), axis=1)
+    y_pred = y_pred.reshape(xx.shape)
+
+    # Plot voronoi regions
+    ax.contourf(xx, yy, y_pred, cmap=cmap, alpha=0.35)
+
+    ax.set_xlim(left=x_min + 0, right=x_max - 0)
+    ax.set_ylim(bottom=y_min + 0, top=y_max - 0)
+
+    plt.pause(0.1)

examples/new_components.py

@@ -0,0 +1,65 @@
+"""This example script shows the usage of the new components architecture.
+
+Serialization/deserialization also works as expected.
+"""
+
+# DATASET
+import torch
+from sklearn.datasets import load_iris
+from sklearn.preprocessing import StandardScaler
+
+scaler = StandardScaler()
+x_train, y_train = load_iris(return_X_y=True)
+x_train = x_train[:, [0, 2]]
+scaler.fit(x_train)
+x_train = scaler.transform(x_train)
+
+x_train = torch.Tensor(x_train)
+y_train = torch.Tensor(y_train)
+num_classes = len(torch.unique(y_train))
+
+# CREATE NEW COMPONENTS
+from prototorch.components import *
+from prototorch.components.initializers import *
+
+unsupervised = Components(6, SelectionInitializer(x_train))
+print(unsupervised())
+
+prototypes = LabeledComponents(
+    (3, 2), StratifiedSelectionInitializer(x_train, y_train))
+print(prototypes())
+
+components = ReasoningComponents(
+    (3, 6), StratifiedSelectionInitializer(x_train, y_train))
+print(components())
+
+# TEST SERIALIZATION
+import io
+
+save = io.BytesIO()
+torch.save(unsupervised, save)
+save.seek(0)
+serialized_unsupervised = torch.load(save)
+
+assert torch.all(unsupervised.components == serialized_unsupervised.components
+                 ), "Serialization of Components failed."
+
+save = io.BytesIO()
+torch.save(prototypes, save)
+save.seek(0)
+serialized_prototypes = torch.load(save)
+
+assert torch.all(prototypes.components == serialized_prototypes.components
+                 ), "Serialization of Components failed."
+assert torch.all(prototypes.component_labels == serialized_prototypes.
+                 component_labels), "Serialization of Components failed."
+
+save = io.BytesIO()
+torch.save(components, save)
+save.seek(0)
+serialized_components = torch.load(save)
+
+assert torch.all(components.components == serialized_components.components
+                 ), "Serialization of Components failed."
+assert torch.all(components.reasonings == serialized_components.reasonings
+                 ), "Serialization of Components failed."

prototorch/__init__.py

@@ -1 +1,42 @@
-__version__ = '0.1.1-dev0'
+"""ProtoTorch package."""
+
+# Core Setup
+__version__ = "0.4.2"
+
+__all_core__ = [
+    "datasets",
+    "functions",
+    "modules",
+]
+
+from .datasets import *
+
+# Plugin Loader
+import pkgutil
+
+import pkg_resources
+
+__path__ = pkgutil.extend_path(__path__, __name__)
+
+
+def discover_plugins():
+    return {
+        entry_point.name: entry_point.load()
+        for entry_point in pkg_resources.iter_entry_points(
+            "prototorch.plugins")
+    }
+
+
+discovered_plugins = discover_plugins()
+locals().update(discovered_plugins)
+
+# Generate combines __version__ and __all__
+version_plugins = "\n".join([
+    "- " + name + ": v" + plugin.__version__
+    for name, plugin in discovered_plugins.items()
+])
+if version_plugins != "":
+    version_plugins = "\nPlugins: \n" + version_plugins
+
+version = "core: v" + __version__ + version_plugins
+__all__ = __all_core__ + list(discovered_plugins.keys())

prototorch/components/__init__.py

@@ -0,0 +1,2 @@
+from prototorch.components.components import *
+from prototorch.components.initializers import *

prototorch/components/components.py

@@ -0,0 +1,151 @@
+"""ProtoTorch components modules."""
+
+import warnings
+from typing import Tuple
+
+import torch
+from prototorch.components.initializers import (ClassAwareInitializer,
+                                                ComponentsInitializer,
+                                                EqualLabelsInitializer,
+                                                UnequalLabelsInitializer,
+                                                ZeroReasoningsInitializer)
+from prototorch.functions.initializers import get_initializer
+from torch.nn.parameter import Parameter
+
+
+class Components(torch.nn.Module):
+    """Components is a set of learnable Tensors."""
+    def __init__(self,
+                 number_of_components=None,
+                 initializer=None,
+                 *,
+                 initialized_components=None,
+                 dtype=torch.float32):
+        super().__init__()
+
+        # Ignore all initialization settings if initialized_components is given.
+        if initialized_components is not None:
+            self._components = Parameter(initialized_components)
+            if number_of_components is not None or initializer is not None:
+                wmsg = "Arguments ignored while initializing Components"
+                warnings.warn(wmsg)
+        else:
+            self._initialize_components(number_of_components, initializer)
+
+    def _precheck_initializer(self, initializer):
+        if not isinstance(initializer, ComponentsInitializer):
+            emsg = f"`initializer` has to be some subtype of " \
+                f"{ComponentsInitializer}. " \
+                f"You have provided: {initializer=} instead."
+            raise TypeError(emsg)
+
+    def _initialize_components(self, number_of_components, initializer):
+        self._precheck_initializer(initializer)
+        self._components = Parameter(
+            initializer.generate(number_of_components))
+
+    @property
+    def components(self):
+        """Tensor containing the component tensors."""
+        return self._components.detach().cpu()
+
+    def forward(self):
+        return self._components
+
+    def extra_repr(self):
+        return f"components.shape: {tuple(self._components.shape)}"
+
+
+class LabeledComponents(Components):
+    """LabeledComponents generate a set of components and a set of labels.
+
+    Every Component has a label assigned.
+    """
+    def __init__(self,
+                 distribution=None,
+                 initializer=None,
+                 *,
+                 initialized_components=None):
+        if initialized_components is not None:
+            super().__init__(initialized_components=initialized_components[0])
+            self._labels = initialized_components[1]
+        else:
+            self._initialize_labels(distribution)
+            super().__init__(number_of_components=len(self._labels),
+                             initializer=initializer)
+
+    def _initialize_components(self, number_of_components, initializer):
+        if isinstance(initializer, ClassAwareInitializer):
+            self._precheck_initializer(initializer)
+            self._components = Parameter(
+                initializer.generate(number_of_components, self.distribution))
+        else:
+            super()._initialize_components(self, number_of_components,
+                                           initializer)
+
+    def _initialize_labels(self, distribution):
+        if type(distribution) == tuple:
+            num_classes, prototypes_per_class = distribution
+            labels = EqualLabelsInitializer(num_classes, prototypes_per_class)
+        elif type(distribution) == list:
+            labels = UnequalLabelsInitializer(distribution)
+
+        self.distribution = labels.distribution
+        self._labels = labels.generate()
+
+    @property
+    def component_labels(self):
+        """Tensor containing the component tensors."""
+        return self._labels.detach().cpu()
+
+    def forward(self):
+        return super().forward(), self._labels
+
+
+class ReasoningComponents(Components):
+    """ReasoningComponents generate a set of components and a set of reasoning matrices.
+
+    Every Component has a reasoning matrix assigned.
+
+    A reasoning matrix is a Nx2 matrix, where N is the number of Classes. The
+    first element is called positive reasoning :math:`p`, the second negative
+    reasoning :math:`n`. A components can reason in favour (positive) of a
+    class, against (negative) a class or not at all (neutral).
+
+    It holds that :math:`0 \leq n \leq 1`, :math:`0 \leq p \leq 1` and :math:`0
+    \leq n+p \leq 1`. Therefore :math:`n` and :math:`p` are two elements of a
+    three element probability distribution.
+
+    """
+    def __init__(self,
+                 reasonings=None,
+                 initializer=None,
+                 *,
+                 initialized_components=None):
+        if initialized_components is not None:
+            super().__init__(initialized_components=initialized_components[0])
+            self._reasonings = initialized_components[1]
+        else:
+            self._initialize_reasonings(reasonings)
+            super().__init__(number_of_components=len(self._reasonings),
+                             initializer=initializer)
+
+    def _initialize_reasonings(self, reasonings):
+        if type(reasonings) == tuple:
+            num_classes, number_of_components = reasonings
+            reasonings = ZeroReasoningsInitializer(num_classes,
+                                                   number_of_components)
+
+        self._reasonings = reasonings.generate()
+
+    @property
+    def reasonings(self):
+        """Returns Reasoning Matrix.
+
+        Dimension NxCx2
+
+        """
+        return self._reasonings.detach().cpu()
+
+    def forward(self):
+        return super().forward(), self._reasonings

prototorch/components/initializers.py

@@ -0,0 +1,197 @@
+"""ProtoTroch Initializers."""
+import warnings
+from collections.abc import Iterable
+from itertools import chain
+
+import torch
+from torch.utils.data import DataLoader, Dataset
+
+
+def parse_init_arg(arg):
+    if isinstance(arg, Dataset):
+        data, labels = next(iter(DataLoader(arg, batch_size=len(arg))))
+        # data = data.view(len(arg), -1)  # flatten
+    else:
+        data, labels = arg
+        if not isinstance(data, torch.Tensor):
+            wmsg = f"Converting data to {torch.Tensor}."
+            warnings.warn(wmsg)
+            data = torch.Tensor(data)
+        if not isinstance(labels, torch.Tensor):
+            wmsg = f"Converting labels to {torch.Tensor}."
+            warnings.warn(wmsg)
+            labels = torch.Tensor(labels)
+    return data, labels
+
+
+# Components
+class ComponentsInitializer(object):
+    def generate(self, number_of_components):
+        raise NotImplementedError("Subclasses should implement this!")
+
+
+class DimensionAwareInitializer(ComponentsInitializer):
+    def __init__(self, c_dims):
+        super().__init__()
+        if isinstance(c_dims, Iterable):
+            self.components_dims = tuple(c_dims)
+        else:
+            self.components_dims = (c_dims, )
+
+
+class OnesInitializer(DimensionAwareInitializer):
+    def generate(self, length):
+        gen_dims = (length, ) + self.components_dims
+        return torch.ones(gen_dims)
+
+
+class ZerosInitializer(DimensionAwareInitializer):
+    def generate(self, length):
+        gen_dims = (length, ) + self.components_dims
+        return torch.zeros(gen_dims)
+
+
+class UniformInitializer(DimensionAwareInitializer):
+    def __init__(self, c_dims, min=0.0, max=1.0):
+        super().__init__(c_dims)
+
+        self.min = min
+        self.max = max
+
+    def generate(self, length):
+        gen_dims = (length, ) + self.components_dims
+        return torch.ones(gen_dims).uniform_(self.min, self.max)
+
+
+class PositionAwareInitializer(ComponentsInitializer):
+    def __init__(self, positions):
+        super().__init__()
+        self.data = positions
+
+
+class SelectionInitializer(PositionAwareInitializer):
+    def generate(self, length):
+        indices = torch.LongTensor(length).random_(0, len(self.data))
+        return self.data[indices]
+
+
+class MeanInitializer(PositionAwareInitializer):
+    def generate(self, length):
+        mean = torch.mean(self.data, dim=0)
+        repeat_dim = [length] + [1] * len(mean.shape)
+        return mean.repeat(repeat_dim)
+
+
+class ClassAwareInitializer(ComponentsInitializer):
+    def __init__(self, arg):
+        super().__init__()
+        data, labels = parse_init_arg(arg)
+        self.data = data
+        self.labels = labels
+
+        self.clabels = torch.unique(self.labels)
+        self.num_classes = len(self.clabels)
+
+    def _get_samples_from_initializer(self, length, dist):
+        if not dist:
+            per_class = length // self.num_classes
+            dist = self.num_classes * [per_class]
+        samples_list = [
+            init.generate(n) for init, n in zip(self.initializers, dist)
+        ]
+        return torch.vstack(samples_list)
+
+
+class StratifiedMeanInitializer(ClassAwareInitializer):
+    def __init__(self, arg):
+        super().__init__(arg)
+
+        self.initializers = []
+        for clabel in self.clabels:
+            class_data = self.data[self.labels == clabel]
+            class_initializer = MeanInitializer(class_data)
+            self.initializers.append(class_initializer)
+
+    def generate(self, length, dist=[]):
+        samples = self._get_samples_from_initializer(length, dist)
+        return samples
+
+
+class StratifiedSelectionInitializer(ClassAwareInitializer):
+    def __init__(self, arg, *, noise=None):
+        super().__init__(arg)
+        self.noise = noise
+
+        self.initializers = []
+        for clabel in self.clabels:
+            class_data = self.data[self.labels == clabel]
+            class_initializer = SelectionInitializer(class_data)
+            self.initializers.append(class_initializer)
+
+    def add_noise(self, x):
+        """Shifts some dimensions of the data randomly."""
+        n1 = torch.rand_like(x)
+        n2 = torch.rand_like(x)
+        mask = torch.bernoulli(n1) - torch.bernoulli(n2)
+        return x + (self.noise * mask)
+
+    def generate(self, length, dist=[]):
+        samples = self._get_samples_from_initializer(length, dist)
+        if self.noise is not None:
+            # samples = self.add_noise(samples)
+            samples = samples + self.noise
+        return samples
+
+
+# Labels
+class LabelsInitializer:
+    def generate(self):
+        raise NotImplementedError("Subclasses should implement this!")
+
+
+class UnequalLabelsInitializer(LabelsInitializer):
+    def __init__(self, dist):
+        self.dist = dist
+
+    @property
+    def distribution(self):
+        return self.dist
+
+    def generate(self):
+        clabels = range(len(self.dist))
+        labels = list(chain(*[[i] * n for i, n in zip(clabels, self.dist)]))
+        return torch.tensor(labels)
+
+
+class EqualLabelsInitializer(LabelsInitializer):
+    def __init__(self, classes, per_class):
+        self.classes = classes
+        self.per_class = per_class
+
+    @property
+    def distribution(self):
+        return self.classes * [self.per_class]
+
+    def generate(self):
+        return torch.arange(self.classes).repeat(self.per_class, 1).T.flatten()
+
+
+# Reasonings
+class ReasoningsInitializer:
+    def generate(self, length):
+        raise NotImplementedError("Subclasses should implement this!")
+
+
+class ZeroReasoningsInitializer(ReasoningsInitializer):
+    def __init__(self, classes, length):
+        self.classes = classes
+        self.length = length
+
+    def generate(self):
+        return torch.zeros((self.length, self.classes, 2))
+
+
+# Aliases
+SSI = StratifiedSampleInitializer = StratifiedSelectionInitializer
+SMI = StratifiedMeanInitializer
+Random = RandomInitializer = UniformInitializer

prototorch/datasets/__init__.py

@@ -0,0 +1,11 @@
+"""ProtoTorch datasets."""
+
+from .abstract import NumpyDataset
+from .spiral import Spiral
+from .tecator import Tecator
+
+__all__ = [
+    "NumpyDataset",
+    "Spiral",
+    "Tecator",
+]

prototorch/datasets/abstract.py

@@ -0,0 +1,96 @@
+"""ProtoTorch abstract dataset classes.
+
+Based on `torchvision.VisionDataset` and `torchvision.MNIST`
+
+For the original code, see:
+https://github.com/pytorch/vision/blob/master/torchvision/datasets/vision.py
+https://github.com/pytorch/vision/blob/master/torchvision/datasets/mnist.py
+"""
+
+import os
+
+import torch
+
+
+class NumpyDataset(torch.utils.data.TensorDataset):
+    """Create a PyTorch TensorDataset from NumPy arrays."""
+    def __init__(self, *arrays):
+        tensors = [torch.Tensor(arr) for arr in arrays]
+        super().__init__(*tensors)
+
+
+class Dataset(torch.utils.data.Dataset):
+    """Abstract dataset class to be inherited."""
+
+    _repr_indent = 2
+
+    def __init__(self, root):
+        if isinstance(root, torch._six.string_classes):
+            root = os.path.expanduser(root)
+        self.root = root
+
+    def __getitem__(self, index):
+        raise NotImplementedError
+
+    def __len__(self):
+        raise NotImplementedError
+
+
+class ProtoDataset(Dataset):
+    """Abstract dataset class to be inherited."""
+
+    training_file = "training.pt"
+    test_file = "test.pt"
+
+    def __init__(self, root, train=True, download=True, verbose=True):
+        super().__init__(root)
+        self.train = train  # training set or test set
+        self.verbose = verbose
+
+        if download:
+            self._download()
+
+        if not self._check_exists():
+            raise RuntimeError("Dataset not found. "
+                               "You can use download=True to download it")
+
+        data_file = self.training_file if self.train else self.test_file
+
+        self.data, self.targets = torch.load(
+            os.path.join(self.processed_folder, data_file))
+
+    @property
+    def raw_folder(self):
+        return os.path.join(self.root, self.__class__.__name__, "raw")
+
+    @property
+    def processed_folder(self):
+        return os.path.join(self.root, self.__class__.__name__, "processed")
+
+    @property
+    def class_to_idx(self):
+        return {_class: i for i, _class in enumerate(self.classes)}
+
+    def _check_exists(self):
+        return os.path.exists(
+            os.path.join(
+                self.processed_folder, self.training_file)) and os.path.exists(
+                    os.path.join(self.processed_folder, self.test_file))
+
+    def __repr__(self):
+        head = "Dataset " + self.__class__.__name__
+        body = ["Number of datapoints: {}".format(self.__len__())]
+        if self.root is not None:
+            body.append("Root location: {}".format(self.root))
+        body += self.extra_repr().splitlines()
+        lines = [head] + [" " * self._repr_indent + line for line in body]
+        return "\n".join(lines)
+
+    def extra_repr(self):
+        return f"Split: {'Train' if self.train is True else 'Test'}"
+
+    def __len__(self):
+        return len(self.data)
+
+    def _download(self):
+        raise NotImplementedError

prototorch/datasets/spiral.py

@@ -0,0 +1,33 @@
+"""Spiral dataset for binary classification."""
+
+import numpy as np
+import torch
+
+
+def make_spiral(n_samples=500, noise=0.3):
+    def get_samples(n, delta_t):
+        points = []
+        for i in range(n):
+            r = i / n_samples * 5
+            t = 1.75 * i / n * 2 * np.pi + delta_t
+            x = r * np.sin(t) + np.random.rand(1) * noise
+            y = r * np.cos(t) + np.random.rand(1) * noise
+            points.append([x, y])
+        return points
+
+    n = n_samples // 2
+    positive = get_samples(n=n, delta_t=0)
+    negative = get_samples(n=n, delta_t=np.pi)
+    x = np.concatenate(
+        [np.array(positive).reshape(n, -1),
+         np.array(negative).reshape(n, -1)],
+        axis=0)
+    y = np.concatenate([np.zeros(n), np.ones(n)])
+    return x, y
+
+
+class Spiral(torch.utils.data.TensorDataset):
+    """Spiral dataset for binary classification."""
+    def __init__(self, n_samples=500, noise=0.3):
+        x, y = make_spiral(n_samples, noise)
+        super().__init__(torch.Tensor(x), torch.LongTensor(y))

prototorch/datasets/tecator.py

@@ -0,0 +1,106 @@
+"""Tecator dataset for classification.
+
+URL:
+    http://lib.stat.cmu.edu/datasets/tecator
+
+LICENCE / TERMS / COPYRIGHT:
+    This is the Tecator data set: The task is to predict the fat content
+    of a meat sample on the basis of its near infrared absorbance spectrum.
+    -------------------------------------------------------------------------
+    1. Statement of permission from Tecator (the original data source)
+
+    These data are recorded on a Tecator Infratec Food and Feed Analyzer
+    working in the wavelength range 850 - 1050 nm by the Near Infrared
+    Transmission (NIT) principle. Each sample contains finely chopped pure
+    meat with different moisture, fat and protein contents.
+
+    If results from these data are used in a publication we want you to
+    mention the instrument and company name (Tecator) in the publication.
+    In addition, please send a preprint of your article to
+
+        Karin Thente, Tecator AB,
+        Box 70, S-263 21 Hoganas, Sweden
+
+    The data are available in the public domain with no responsability from
+    the original data source. The data can be redistributed as long as this
+    permission note is attached.
+
+    For more information about the instrument - call Perstorp Analytical's
+    representative in your area.
+
+Description:
+    For each meat sample the data consists of a 100 channel spectrum of
+    absorbances and the contents of moisture (water), fat and protein.
+    The absorbance is -log10 of the transmittance
+    measured by the spectrometer. The three contents, measured in percent,
+    are determined by analytic chemistry.
+"""
+
+import os
+
+import numpy as np
+import torch
+from torchvision.datasets.utils import download_file_from_google_drive
+
+from prototorch.datasets.abstract import ProtoDataset
+
+
+class Tecator(ProtoDataset):
+    """
+    `Tecator Dataset <http://lib.stat.cmu.edu/datasets/tecator>`__
+    for classification.
+    """
+
+    _resources = [
+        ("1P9WIYnyxFPh6f1vqAbnKfK8oYmUgyV83",
+         "ba5607c580d0f91bb27dc29d13c2f8df"),
+    ]  # (google_storage_id, md5hash)
+    classes = ["0 - low_fat", "1 - high_fat"]
+
+    def __getitem__(self, index):
+        img, target = self.data[index], int(self.targets[index])
+        return img, target
+
+    def _download(self):
+        """Download the data if it doesn't exist in already."""
+        if self._check_exists():
+            return
+
+        if self.verbose:
+            print("Making directories...")
+        os.makedirs(self.raw_folder, exist_ok=True)
+        os.makedirs(self.processed_folder, exist_ok=True)
+
+        if self.verbose:
+            print("Downloading...")
+        for fileid, md5 in self._resources:
+            filename = "tecator.npz"
+            download_file_from_google_drive(fileid,
+                                            root=self.raw_folder,
+                                            filename=filename,
+                                            md5=md5)
+
+        if self.verbose:
+            print("Processing...")
+        with np.load(os.path.join(self.raw_folder, "tecator.npz"),
+                     allow_pickle=False) as f:
+            x_train, y_train = f["x_train"], f["y_train"]
+            x_test, y_test = f["x_test"], f["y_test"]
+        training_set = [
+            torch.tensor(x_train, dtype=torch.float32),
+            torch.tensor(y_train),
+        ]
+        test_set = [
+            torch.tensor(x_test, dtype=torch.float32),
+            torch.tensor(y_test),
+        ]
+
+        with open(os.path.join(self.processed_folder, self.training_file),
+                  "wb") as f:
+            torch.save(training_set, f)
+        with open(os.path.join(self.processed_folder, self.test_file),
+                  "wb") as f:
+            torch.save(test_set, f)
+
+        if self.verbose:
+            print("Done!")

prototorch/functions/__init__.py

@@ -0,0 +1,12 @@
+"""ProtoTorch functions."""
+
+from .activations import identity, sigmoid_beta, swish_beta
+from .competitions import knnc, wtac
+
+__all__ = [
+    "identity",
+    "sigmoid_beta",
+    "swish_beta",
+    "knnc",
+    "wtac",
+]

prototorch/functions/activations.py

@@ -8,47 +8,60 @@ ACTIVATIONS = dict()
 # def register_activation(scriptf):
 #     ACTIVATIONS[scriptf.name] = scriptf
 #     return scriptf
-def register_activation(f):
+def register_activation(function):
-    ACTIVATIONS[f.__name__] = f
+    """Add the activation function to the registry."""
-    return f
+    ACTIVATIONS[function.__name__] = function
+    return function
 
 
 @register_activation
 # @torch.jit.script
-def identity(input, beta=torch.tensor([0])):
+def identity(x, beta=0.0):
-    """:math:`f(x) = x`"""
+    """Identity activation function.
-    return input
 
-
+    Definition:
-@register_activation
+    :math:`f(x) = x`
-# @torch.jit.script
-def sigmoid_beta(input, beta=torch.tensor([10])):
-    """:math:`f(x) = \\frac{1}{1 + e^{-\\beta x}}`
 
     Keyword Arguments:
-        beta (float): Parameter :math:`\\beta`
+        beta (`float`): Ignored.
     """
-    out = torch.reciprocal(1.0 + torch.exp(-int(beta.item()) * input))
+    return x
+
+
+@register_activation
+# @torch.jit.script
+def sigmoid_beta(x, beta=10.0):
+    r"""Sigmoid activation function with scaling.
+
+    Definition:
+    :math:`f(x) = \frac{1}{1 + e^{-\beta x}}`
+
+    Keyword Arguments:
+        beta (`float`): Scaling parameter :math:`\beta`
+    """
+    out = 1.0 / (1.0 + torch.exp(-1.0 * beta * x))
     return out
 
 
 @register_activation
 # @torch.jit.script
-def swish_beta(input, beta=torch.tensor([10])):
+def swish_beta(x, beta=10.0):
-    """:math:`f(x) = \\frac{x}{1 + e^{-\\beta x}}`
+    r"""Swish activation function with scaling.
+
+    Definition:
+    :math:`f(x) = \frac{x}{1 + e^{-\beta x}}`
 
     Keyword Arguments:
-        beta (float): Parameter :math:`\\beta`
+        beta (`float`): Scaling parameter :math:`\beta`
     """
-    out = input * sigmoid_beta(input, beta=beta)
+    out = x * sigmoid_beta(x, beta=beta)
     return out
 
 
 def get_activation(funcname):
+    """Deserialize the activation function."""
     if callable(funcname):
         return funcname
-    else:
+    if funcname in ACTIVATIONS:
-        if funcname in ACTIVATIONS:
+        return ACTIVATIONS.get(funcname)
-            return ACTIVATIONS.get(funcname)
+    raise NameError(f"Activation {funcname} was not found.")
-        else:
-            raise NameError(f'Activation {funcname} was not found.')

prototorch/functions/competitions.py

@@ -3,6 +3,34 @@
 import torch
 
 
+# @torch.jit.script
+def stratified_min(distances, labels):
+    clabels = torch.unique(labels, dim=0)
+    nclasses = clabels.size()[0]
+    if distances.size()[1] == nclasses:
+        # skip if only one prototype per class
+        return distances
+    batch_size = distances.size()[0]
+    winning_distances = torch.zeros(nclasses, batch_size)
+    inf = torch.full_like(distances.T, fill_value=float("inf"))
+    # distances_to_wpluses = torch.where(matcher, distances, inf)
+    for i, cl in enumerate(clabels):
+        # cdists = distances.T[labels == cl]
+        matcher = torch.eq(labels.unsqueeze(dim=1), cl)
+        if labels.ndim == 2:
+            # if the labels are one-hot vectors
+            matcher = torch.eq(torch.sum(matcher, dim=-1), nclasses)
+        cdists = torch.where(matcher, distances.T, inf).T
+        winning_distances[i] = torch.min(cdists, dim=1,
+                                         keepdim=True).values.squeeze()
+    if labels.ndim == 2:
+        # Transpose to return with `batch_size` first and
+        # reverse the columns to fix the ordering of the classes
+        return torch.flip(winning_distances.T, dims=(1, ))
+
+    return winning_distances.T  # return with `batch_size` first
+
+
 # @torch.jit.script
 def wtac(distances, labels):
     winning_indices = torch.min(distances, dim=1).indices

prototorch/functions/distances.py

@@ -1,13 +1,25 @@
 """ProtoTorch distance functions."""
 
+import numpy as np
 import torch
 
+from prototorch.functions.helper import (
+    _check_shapes,
+    _int_and_mixed_shape,
+    equal_int_shape,
+)
+
 
 def squared_euclidean_distance(x, y):
-    """Compute the squared Euclidean distance between :math:`x` and :math:`y`.
+    r"""Compute the squared Euclidean distance between :math:`\bm x` and :math:`\bm y`.
 
-    Expected dimension of x is 2.
+    Compute :math:`{\langle \bm x - \bm y \rangle}_2`
-    Expected dimension of y is 2.
+
+    :param `torch.tensor` x: Two dimensional vector
+    :param `torch.tensor` y: Two dimensional vector
+
+    **Alias:**
+    ``prototorch.functions.distances.sed``
     """
     expanded_x = x.unsqueeze(dim=1)
     batchwise_difference = y - expanded_x
@@ -17,34 +29,58 @@ def squared_euclidean_distance(x, y):
 
 
 def euclidean_distance(x, y):
-    """Compute the Euclidean distance between :math:`x` and :math:`y`.
+    r"""Compute the Euclidean distance between :math:`x` and :math:`y`.
 
-    Expected dimension of x is 2.
+    Compute :math:`\sqrt{{\langle \bm x - \bm y \rangle}_2}`
-    Expected dimension of y is 2.
+
+    :param `torch.tensor` x: Input Tensor of shape :math:`X \times N`
+    :param `torch.tensor` y: Input Tensor of shape :math:`Y \times N`
+
+    :returns: Distance Tensor of shape :math:`X \times Y`
+    :rtype: `torch.tensor`
     """
     distances_raised = squared_euclidean_distance(x, y)
     distances = torch.sqrt(distances_raised)
     return distances
 
 
-def lpnorm_distance(x, y, p):
+def euclidean_distance_v2(x, y):
-    """Compute :math:`{\\langle x, y \\rangle}_p`.
+    diff = y - x.unsqueeze(1)
+    pairwise_distances = (diff @ diff.permute((0, 2, 1))).sqrt()
+    # Passing `dim1=-2` and `dim2=-1` to `diagonal()` takes the
+    # batch diagonal. See:
+    # https://pytorch.org/docs/stable/generated/torch.diagonal.html
+    distances = torch.diagonal(pairwise_distances, dim1=-2, dim2=-1)
+    # print(f"{diff.shape=}")  # (nx, ny, ndim)
+    # print(f"{pairwise_distances.shape=}")  # (nx, ny, ny)
+    # print(f"{distances.shape=}")  # (nx, ny)
+    return distances
 
-    Expected dimension of x is 2.
+
-    Expected dimension of y is 2.
+def lpnorm_distance(x, y, p):
+    r"""Calculate the lp-norm between :math:`\bm x` and :math:`\bm y`.
+    Also known as Minkowski distance.
+
+    Compute :math:`{\| \bm x - \bm y \|}_p`.
+
+    Calls ``torch.cdist``
+
+    :param `torch.tensor` x: Two dimensional vector
+    :param `torch.tensor` y: Two dimensional vector
+    :param p: p parameter of the lp norm
     """
     distances = torch.cdist(x, y, p=p)
     return distances
 
 
 def omega_distance(x, y, omega):
-    """Omega distance.
+    r"""Omega distance.
 
-    Compute :math:`{\\langle \\Omega x, \\Omega y \\rangle}_p`
+    Compute :math:`{\| \Omega \bm x - \Omega \bm y \|}_p`
 
-    Expected dimension of x is 2.
+    :param `torch.tensor` x: Two dimensional vector
-    Expected dimension of y is 2.
+    :param `torch.tensor` y: Two dimensional vector
-    Expected dimension of omega is 2.
+    :param `torch.tensor` omega: Two dimensional matrix
     """
     projected_x = x @ omega
     projected_y = y @ omega
@@ -53,13 +89,13 @@ def omega_distance(x, y, omega):
 
 
 def lomega_distance(x, y, omegas):
-    """Localized Omega distance.
+    r"""Localized Omega distance.
 
-    Compute :math:`{\\langle \\Omega_k x, \\Omega_k y_k \\rangle}_p`
+    Compute :math:`{\| \Omega_k \bm x - \Omega_k \bm y_k \|}_p`
 
-    Expected dimension of x is 2.
+    :param `torch.tensor` x: Two dimensional vector
-    Expected dimension of y is 2.
+    :param `torch.tensor` y: Two dimensional vector
-    Expected dimension of omegas is 3.
+    :param `torch.tensor` omegas: Three dimensional matrix
     """
     projected_x = x @ omegas
     projected_y = torch.diagonal(y @ omegas).T
@@ -69,3 +105,245 @@ def lomega_distance(x, y, omegas):
     distances = torch.sum(differences_squared, dim=2)
     distances = distances.permute(1, 0)
     return distances
+
+
+def euclidean_distance_matrix(x, y, squared=False, epsilon=1e-10):
+    r"""Computes an euclidean distances matrix given two distinct vectors.
+    last dimension must be the vector dimension!
+    compute the distance via the identity of the dot product. This avoids the memory overhead due to the subtraction!
+
+    - ``x.shape = (number_of_x_vectors, vector_dim)``
+    - ``y.shape = (number_of_y_vectors, vector_dim)``
+
+    output: matrix of distances (number_of_x_vectors, number_of_y_vectors)
+    """
+    for tensor in [x, y]:
+        if tensor.ndim != 2:
+            raise ValueError(
+                "The tensor dimension must be two. You provide: tensor.ndim=" +
+                str(tensor.ndim) + ".")
+    if not equal_int_shape([tuple(x.shape)[1]], [tuple(y.shape)[1]]):
+        raise ValueError(
+            "The vector shape must be equivalent in both tensors. You provide: tuple(y.shape)[1]="
+            + str(tuple(x.shape)[1]) + " and  tuple(y.shape)(y)[1]=" +
+            str(tuple(y.shape)[1]) + ".")
+
+    y = torch.transpose(y)
+
+    diss = (torch.sum(x**2, axis=1, keepdims=True) - 2 * torch.dot(x, y) +
+            torch.sum(y**2, axis=0, keepdims=True))
+
+    if not squared:
+        if epsilon == 0:
+            diss = torch.sqrt(diss)
+        else:
+            diss = torch.sqrt(torch.max(diss, epsilon))
+
+    return diss
+
+
+def tangent_distance(signals, protos, subspaces, squared=False, epsilon=1e-10):
+    r"""Tangent distances based on the tensorflow implementation of Sascha Saralajews
+
+    For more info about Tangen distances see
+
+    DOI:10.1109/IJCNN.2016.7727534.
+
+    The subspaces is always assumed as transposed and must be orthogonal!
+    For local non sparse signals subspaces must be provided!
+
+    - shape(signals): batch x proto_number x channels x dim1 x dim2 x ... x dimN
+    - shape(protos): proto_number x dim1 x dim2 x ... x dimN
+    - shape(subspaces): (optional [proto_number]) x prod(dim1 * dim2 * ... * dimN)  x prod(projected_atom_shape)
+
+    subspace should be orthogonalized
+    Pytorch implementation of Sascha Saralajew's tensorflow code.
+    Translation by Christoph Raab
+    """
+    signal_shape, signal_int_shape = _int_and_mixed_shape(signals)
+    proto_shape, proto_int_shape = _int_and_mixed_shape(protos)
+    subspace_int_shape = tuple(subspaces.shape)
+
+    # check if the shapes are correct
+    _check_shapes(signal_int_shape, proto_int_shape)
+
+    atom_axes = list(range(3, len(signal_int_shape)))
+    # for sparse signals, we use the memory efficient implementation
+    if signal_int_shape[1] == 1:
+        signals = torch.reshape(signals, [-1, np.prod(signal_shape[3:])])
+
+        if len(atom_axes) > 1:
+            protos = torch.reshape(protos, [proto_shape[0], -1])
+
+        if subspaces.ndim == 2:
+            # clean solution without map if the matrix_scope is global
+            projectors = torch.eye(subspace_int_shape[-2]) - torch.dot(
+                subspaces, torch.transpose(subspaces))
+
+            projected_signals = torch.dot(signals, projectors)
+            projected_protos = torch.dot(protos, projectors)
+
+            diss = euclidean_distance_matrix(projected_signals,
+                                             projected_protos,
+                                             squared=squared,
+                                             epsilon=epsilon)
+
+            diss = torch.reshape(
+                diss, [signal_shape[0], signal_shape[2], proto_shape[0]])
+
+            return torch.permute(diss, [0, 2, 1])
+
+        else:
+
+            # no solution without map possible --> memory efficient but slow!
+            projectors = torch.eye(subspace_int_shape[-2]) - torch.bmm(
+                subspaces,
+                subspaces)  # K.batch_dot(subspaces, subspaces, [2, 2])
+
+            projected_protos = (protos @ subspaces
+                                ).T  # K.batch_dot(projectors, protos, [1, 1]))
+
+            def projected_norm(projector):
+                return torch.sum(torch.dot(signals, projector)**2, axis=1)
+
+            diss = (torch.transpose(map(projected_norm, projectors)) -
+                    2 * torch.dot(signals, projected_protos) +
+                    torch.sum(projected_protos**2, axis=0, keepdims=True))
+
+            if not squared:
+                if epsilon == 0:
+                    diss = torch.sqrt(diss)
+                else:
+                    diss = torch.sqrt(torch.max(diss, epsilon))
+
+            diss = torch.reshape(
+                diss, [signal_shape[0], signal_shape[2], proto_shape[0]])
+
+            return torch.permute(diss, [0, 2, 1])
+
+    else:
+        signals = signals.permute([0, 2, 1] + atom_axes)
+
+        diff = signals - protos
+
+        # global tangent space
+        if subspaces.ndim == 2:
+            # Scope Projectors
+            projectors = subspaces  #
+
+            # Scope: Tangentspace Projections
+            diff = torch.reshape(
+                diff, (signal_shape[0] * signal_shape[2], signal_shape[1], -1))
+            projected_diff = diff @ projectors
+            projected_diff = torch.reshape(
+                projected_diff,
+                (signal_shape[0], signal_shape[2], signal_shape[1]) +
+                signal_shape[3:],
+            )
+
+            diss = torch.norm(projected_diff, 2, dim=-1)
+            return diss.permute([0, 2, 1])
+
+        # local tangent spaces
+        else:
+            # Scope: Calculate Projectors
+            projectors = subspaces
+
+            # Scope: Tangentspace Projections
+            diff = torch.reshape(
+                diff, (signal_shape[0] * signal_shape[2], signal_shape[1], -1))
+            diff = diff.permute([1, 0, 2])
+            projected_diff = torch.bmm(diff, projectors)
+            projected_diff = torch.reshape(
+                projected_diff,
+                (signal_shape[1], signal_shape[0], signal_shape[2]) +
+                signal_shape[3:],
+            )
+
+            diss = torch.norm(projected_diff, 2, dim=-1)
+            return diss.permute([1, 0, 2]).squeeze(-1)
+
+
+class KernelDistance:
+    r"""Kernel Distance
+
+    Distance based on a kernel function.
+    """
+    def __init__(self, kernel_fn):
+        self.kernel_fn = kernel_fn
+
+    def __call__(self, x_batch: torch.Tensor, y_batch: torch.Tensor):
+        return self._single_call(x_batch, y_batch)
+
+    def _single_call(self, x, y):
+        remove_dims = []
+        if len(x.shape) == 1:
+            x = x.unsqueeze(0)
+            remove_dims.append(0)
+        if len(y.shape) == 1:
+            y = y.unsqueeze(0)
+            remove_dims.append(-1)
+
+        output = self.kernel_fn(x, x).diag().unsqueeze(1) - 2 * self.kernel_fn(
+            x, y) + self.kernel_fn(y, y).diag()
+
+        for dim in remove_dims:
+            output.squeeze_(dim)
+
+        return torch.sqrt(output)
+
+
+class BatchKernelDistance:
+    r"""Kernel Distance
+
+    Distance based on a kernel function.
+    """
+    def __init__(self, kernel_fn):
+        self.kernel_fn = kernel_fn
+
+    def __call__(self, x_batch: torch.Tensor, y_batch: torch.Tensor):
+        remove_dims = 0
+        # Extend Single inputs
+        if len(x_batch.shape) == 1:
+            x_batch = x_batch.unsqueeze(0)
+            remove_dims += 1
+        if len(y_batch.shape) == 1:
+            y_batch = y_batch.unsqueeze(0)
+            remove_dims += 1
+
+        # Loop over batches
+        output = torch.FloatTensor(len(x_batch), len(y_batch))
+        for i, x in enumerate(x_batch):
+            for j, y in enumerate(y_batch):
+                output[i][j] = self._single_call(x, y)
+
+        for _ in range(remove_dims):
+            output.squeeze_(0)
+
+        return output
+
+    def _single_call(self, x, y):
+        kappa_xx = self.kernel_fn(x, x)
+        kappa_xy = self.kernel_fn(x, y)
+        kappa_yy = self.kernel_fn(y, y)
+
+        squared_distance = kappa_xx - 2 * kappa_xy + kappa_yy
+
+        return torch.sqrt(squared_distance)
+
+
+class SquaredKernelDistance(KernelDistance):
+    r"""Squared Kernel Distance
+
+    Kernel distance without final squareroot.
+    """
+    def single_call(self, x, y):
+        kappa_xx = self.kernel_fn(x, x)
+        kappa_xy = self.kernel_fn(x, y)
+        kappa_yy = self.kernel_fn(y, y)
+
+        return kappa_xx - 2 * kappa_xy + kappa_yy
+
+
+# Aliases
+sed = squared_euclidean_distance

prototorch/functions/helper.py

@@ -0,0 +1,89 @@
+import torch
+
+
+def calculate_prototype_accuracy(y_pred, y_true, plabels):
+    """Computes the accuracy of a prototype based model.
+    via Winner-Takes-All rule.
+    Requirement:
+    y_pred.shape == y_true.shape
+    unique(y_pred) in plabels
+    """
+    with torch.no_grad():
+        idx = torch.argmin(y_pred, axis=1)
+        return torch.true_divide(torch.sum(y_true == plabels[idx]),
+                                 len(y_pred)) * 100
+
+
+def predict_label(y_pred, plabels):
+    r""" Predicts labels given a prediction of a prototype based model.
+    """
+    with torch.no_grad():
+        return plabels[torch.argmin(y_pred, 1)]
+
+
+def mixed_shape(inputs):
+    if not torch.is_tensor(inputs):
+        raise ValueError("Input must be a tensor.")
+    else:
+        int_shape = list(inputs.shape)
+        # sometimes int_shape returns mixed integer types
+        int_shape = [int(i) if i is not None else i for i in int_shape]
+        tensor_shape = inputs.shape
+
+        for i, s in enumerate(int_shape):
+            if s is None:
+                int_shape[i] = tensor_shape[i]
+        return tuple(int_shape)
+
+
+def equal_int_shape(shape_1, shape_2):
+    if not isinstance(shape_1,
+                      (tuple, list)) or not isinstance(shape_2, (tuple, list)):
+        raise ValueError("Input shapes must list or tuple.")
+    for shape in [shape_1, shape_2]:
+        if not all([isinstance(x, int) or x is None for x in shape]):
+            raise ValueError(
+                "Input shapes must be list or tuple of int and None values.")
+
+    if len(shape_1) != len(shape_2):
+        return False
+    else:
+        for axis, value in enumerate(shape_1):
+            if value is not None and shape_2[axis] not in {value, None}:
+                return False
+        return True
+
+
+def _check_shapes(signal_int_shape, proto_int_shape):
+    if len(signal_int_shape) < 4:
+        raise ValueError(
+            "The number of signal dimensions must be >=4. You provide: " +
+            str(len(signal_int_shape)))
+
+    if len(proto_int_shape) < 2:
+        raise ValueError(
+            "The number of proto dimensions must be >=2. You provide: " +
+            str(len(proto_int_shape)))
+
+    if not equal_int_shape(signal_int_shape[3:], proto_int_shape[1:]):
+        raise ValueError(
+            "The atom shape of signals must be equal protos. You provide: signals.shape[3:]="
+            + str(signal_int_shape[3:]) + " != protos.shape[1:]=" +
+            str(proto_int_shape[1:]))
+
+    # not a sparse signal
+    if signal_int_shape[1] != 1:
+        if not equal_int_shape(signal_int_shape[1:2], proto_int_shape[0:1]):
+            raise ValueError(
+                "If the signal is not sparse, the number of prototypes must be equal in signals and "
+                "protos. You provide: " + str(signal_int_shape[1]) + " != " +
+                str(proto_int_shape[0]))
+
+    return True
+
+
+def _int_and_mixed_shape(tensor):
+    shape = mixed_shape(tensor)
+    int_shape = tuple([i if isinstance(i, int) else None for i in shape])
+
+    return shape, int_shape

prototorch/functions/initializers.py

@@ -7,87 +7,101 @@ import torch
 INITIALIZERS = dict()
 
 
-def register_initializer(func):
+def register_initializer(function):
-    INITIALIZERS[func.__name__] = func
+    """Add the initializer to the registry."""
-    return func
+    INITIALIZERS[function.__name__] = function
+    return function
 
 
-def labels_from(distribution):
+def labels_from(distribution, one_hot=True):
     """Takes a distribution tensor and returns a labels tensor."""
     nclasses = distribution.shape[0]
     llist = [[i] * n for i, n in zip(range(nclasses), distribution)]
     # labels = [l for cl in llist for l in cl]  # flatten the list of lists
-    labels = list(chain(*llist))  # flatten using itertools.chain
+    flat_llist = list(chain(*llist))  # flatten label list with itertools.chain
-    return torch.tensor(labels, requires_grad=False)
+    plabels = torch.tensor(flat_llist, requires_grad=False)
+    if one_hot:
+        return torch.eye(nclasses)[plabels]
+    return plabels
 
 
 @register_initializer
-def ones(x_train, y_train, prototype_distribution):
+def ones(x_train, y_train, prototype_distribution, one_hot=True):
     nprotos = torch.sum(prototype_distribution)
     protos = torch.ones(nprotos, *x_train.shape[1:])
-    plabels = labels_from(prototype_distribution)
+    plabels = labels_from(prototype_distribution, one_hot)
     return protos, plabels
 
 
 @register_initializer
-def zeros(x_train, y_train, prototype_distribution):
+def zeros(x_train, y_train, prototype_distribution, one_hot=True):
     nprotos = torch.sum(prototype_distribution)
     protos = torch.zeros(nprotos, *x_train.shape[1:])
-    plabels = labels_from(prototype_distribution)
+    plabels = labels_from(prototype_distribution, one_hot)
     return protos, plabels
 
 
 @register_initializer
-def rand(x_train, y_train, prototype_distribution):
+def rand(x_train, y_train, prototype_distribution, one_hot=True):
     nprotos = torch.sum(prototype_distribution)
     protos = torch.rand(nprotos, *x_train.shape[1:])
-    plabels = labels_from(prototype_distribution)
+    plabels = labels_from(prototype_distribution, one_hot)
     return protos, plabels
 
 
 @register_initializer
-def randn(x_train, y_train, prototype_distribution):
+def randn(x_train, y_train, prototype_distribution, one_hot=True):
     nprotos = torch.sum(prototype_distribution)
     protos = torch.randn(nprotos, *x_train.shape[1:])
-    plabels = labels_from(prototype_distribution)
+    plabels = labels_from(prototype_distribution, one_hot)
     return protos, plabels
 
 
 @register_initializer
-def stratified_mean(x_train, y_train, prototype_distribution):
+def stratified_mean(x_train, y_train, prototype_distribution, one_hot=True):
     nprotos = torch.sum(prototype_distribution)
     pdim = x_train.shape[1]
     protos = torch.empty(nprotos, pdim)
-    plabels = labels_from(prototype_distribution)
+    plabels = labels_from(prototype_distribution, one_hot)
-    for i, l in enumerate(plabels):
+    for i, label in enumerate(plabels):
-        xl = x_train[y_train == l]
+        matcher = torch.eq(label.unsqueeze(dim=0), y_train)
+        if one_hot:
+            nclasses = y_train.size()[1]
+            matcher = torch.eq(torch.sum(matcher, dim=-1), nclasses)
+        xl = x_train[matcher]
         mean_xl = torch.mean(xl, dim=0)
         protos[i] = mean_xl
+    plabels = labels_from(prototype_distribution, one_hot=one_hot)
     return protos, plabels
 
 
 @register_initializer
-def stratified_random(x_train, y_train, prototype_distribution):
+def stratified_random(x_train,
-    gen = torch.manual_seed(torch.initial_seed())
+                      y_train,
+                      prototype_distribution,
+                      one_hot=True,
+                      epsilon=1e-7):
     nprotos = torch.sum(prototype_distribution)
     pdim = x_train.shape[1]
     protos = torch.empty(nprotos, pdim)
-    plabels = labels_from(prototype_distribution)
+    plabels = labels_from(prototype_distribution, one_hot)
-    for i, l in enumerate(plabels):
+    for i, label in enumerate(plabels):
-        xl = x_train[y_train == l]
+        matcher = torch.eq(label.unsqueeze(dim=0), y_train)
-        rand_index = torch.zeros(1).long().random_(0,
+        if one_hot:
-                                                   xl.shape[1] - 1,
+            nclasses = y_train.size()[1]
-                                                   generator=gen)
+            matcher = torch.eq(torch.sum(matcher, dim=-1), nclasses)
+        xl = x_train[matcher]
+        rand_index = torch.zeros(1).long().random_(0, xl.shape[0] - 1)
         random_xl = xl[rand_index]
-        protos[i] = random_xl
+        protos[i] = random_xl + epsilon
+    plabels = labels_from(prototype_distribution, one_hot=one_hot)
     return protos, plabels
 
 
 def get_initializer(funcname):
+    """Deserialize the initializer."""
     if callable(funcname):
         return funcname
-    else:
+    if funcname in INITIALIZERS:
-        if funcname in INITIALIZERS:
+        return INITIALIZERS.get(funcname)
-            return INITIALIZERS.get(funcname)
+    raise NameError(f"Initializer {funcname} was not found.")
-        else:
-            raise NameError(f'Initializer {funcname} was not found.')

prototorch/functions/kernels.py

@@ -0,0 +1,28 @@
+"""
+Experimental Kernels
+"""
+
+import torch
+
+
+class ExplicitKernel:
+    def __init__(self, projection=torch.nn.Identity()):
+        self.projection = projection
+
+    def __call__(self, x, y):
+        return self.projection(x) @ self.projection(y).T
+
+
+class RadialBasisFunctionKernel:
+    def __init__(self, sigma) -> None:
+        self.s2 = sigma * sigma
+
+    def __call__(self, x, y):
+        remove_dim = False
+        if len(x.shape) > 1:
+            x = x.unsqueeze(1)
+            remove_dim = True
+        output = torch.exp(-torch.sum((x - y)**2, dim=-1) / (2 * self.s2))
+        if remove_dim:
+            output = output.squeeze(1)
+        return output

prototorch/functions/losses.py

@@ -3,20 +3,54 @@
 import torch
 
 
-def glvq_loss(distances, target_labels, prototype_labels):
+def _get_matcher(targets, labels):
-    """GLVQ loss function with support for one-hot labels."""
+    """Returns a boolean tensor."""
-    matcher = torch.eq(target_labels.unsqueeze(dim=1), prototype_labels)
+    matcher = torch.eq(targets.unsqueeze(dim=1), labels)
-    if prototype_labels.ndim == 2:
+    if labels.ndim == 2:
         # if the labels are one-hot vectors
-        nclasses = target_labels.size()[1]
+        nclasses = targets.size()[1]
         matcher = torch.eq(torch.sum(matcher, dim=-1), nclasses)
+    return matcher
+
+
+def _get_dp_dm(distances, targets, plabels):
+    """Returns the d+ and d- values for a batch of distances."""
+    matcher = _get_matcher(targets, plabels)
     not_matcher = torch.bitwise_not(matcher)
 
-    inf = torch.full_like(distances, fill_value=float('inf'))
+    inf = torch.full_like(distances, fill_value=float("inf"))
-    distances_to_wpluses = torch.where(matcher, distances, inf)
+    d_matching = torch.where(matcher, distances, inf)
-    distances_to_wminuses = torch.where(not_matcher, distances, inf)
+    d_unmatching = torch.where(not_matcher, distances, inf)
-    dpluses = torch.min(distances_to_wpluses, dim=1, keepdim=True).values
+    dp = torch.min(d_matching, dim=1, keepdim=True).values
-    dminuses = torch.min(distances_to_wminuses, dim=1, keepdim=True).values
+    dm = torch.min(d_unmatching, dim=1, keepdim=True).values
+    return dp, dm
 
-    mu = (dpluses - dminuses) / (dpluses + dminuses)
+
+def glvq_loss(distances, target_labels, prototype_labels):
+    """GLVQ loss function with support for one-hot labels."""
+    dp, dm = _get_dp_dm(distances, target_labels, prototype_labels)
+    mu = (dp - dm) / (dp + dm)
     return mu
+
+
+def lvq1_loss(distances, target_labels, prototype_labels):
+    """LVQ1 loss function with support for one-hot labels.
+
+    See Section 4 [Sado&Yamada]
+    https://papers.nips.cc/paper/1995/file/9c3b1830513cc3b8fc4b76635d32e692-Paper.pdf
+    """
+    dp, dm = _get_dp_dm(distances, target_labels, prototype_labels)
+    mu = dp
+    mu[dp > dm] = -dm[dp > dm]
+    return mu
+
+
+def lvq21_loss(distances, target_labels, prototype_labels):
+    """LVQ2.1 loss function with support for one-hot labels.
+    
+    See Section 4 [Sado&Yamada]
+    https://papers.nips.cc/paper/1995/file/9c3b1830513cc3b8fc4b76635d32e692-Paper.pdf
+    """
+    dp, dm = _get_dp_dm(distances, target_labels, prototype_labels)
+    mu = dp - dm
+    return mu

prototorch/functions/normalization.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function
+
+import torch
+
+
+def orthogonalization(tensors):
+    r""" Orthogonalization of a given tensor via polar decomposition.
+    """
+    u, _, v = torch.svd(tensors, compute_uv=True)
+    u_shape = tuple(list(u.shape))
+    v_shape = tuple(list(v.shape))
+
+    # reshape to (num x N x M)
+    u = torch.reshape(u, (-1, u_shape[-2], u_shape[-1]))
+    v = torch.reshape(v, (-1, v_shape[-2], v_shape[-1]))
+
+    out = u @ v.permute([0, 2, 1])
+
+    out = torch.reshape(out, u_shape[:-1] + (v_shape[-2], ))
+
+    return out
+
+
+def trace_normalization(tensors):
+    r""" Trace normalization
+    """
+    epsilon = torch.tensor([1e-10], dtype=torch.float64)
+    # Scope trace_normalization
+    constant = torch.trace(tensors)
+
+    if epsilon != 0:
+        constant = torch.max(constant, epsilon)
+
+    return tensors / constant

prototorch/functions/similarities.py

@@ -0,0 +1,18 @@
+"""ProtoTorch similarity functions."""
+
+import torch
+
+
+def cosine_similarity(x, y):
+    """Compute the cosine similarity between :math:`x` and :math:`y`.
+
+    Expected dimension of x is 2.
+    Expected dimension of y is 2.
+    """
+    norm_x = x.pow(2).sum(1).sqrt()
+    norm_y = y.pow(2).sum(1).sqrt()
+    norm_mat = norm_x.unsqueeze(-1) @ norm_y.unsqueeze(-1).T
+    epsilon = torch.finfo(norm_mat.dtype).eps
+    norm_mat.clamp_(min=epsilon)
+    similarities = (x @ y.T) / norm_mat
+    return similarities

prototorch/modules/__init__.py

@@ -0,0 +1,7 @@
+"""ProtoTorch modules."""
+
+from .prototypes import Prototypes1D
+
+__all__ = [
+    "Prototypes1D",
+]

prototorch/modules/losses.py

@@ -7,8 +7,7 @@ from prototorch.functions.losses import glvq_loss
 
 
 class GLVQLoss(torch.nn.Module):
-    """GLVQ Loss."""
+    def __init__(self, margin=0.0, squashing="identity", beta=10, **kwargs):
-    def __init__(self, margin=0.0, squashing='identity', beta=10, **kwargs):
         super().__init__(**kwargs)
         self.margin = margin
         self.squashing = get_activation(squashing)
@@ -16,6 +15,26 @@ class GLVQLoss(torch.nn.Module):
 
     def forward(self, outputs, targets):
         distances, plabels = outputs
-        mu = glvq_loss(distances, targets, plabels)
+        mu = glvq_loss(distances, targets, prototype_labels=plabels)
         batch_loss = self.squashing(mu + self.margin, beta=self.beta)
         return torch.sum(batch_loss, dim=0)
+
+
+class NeuralGasEnergy(torch.nn.Module):
+    def __init__(self, lm):
+        super().__init__()
+        self.lm = lm
+
+    def forward(self, d):
+        order = torch.argsort(d, dim=1)
+        ranks = torch.argsort(order, dim=1)
+        cost = torch.sum(self._nghood_fn(ranks, self.lm) * d)
+
+        return cost, order
+
+    def extra_repr(self):
+        return f"lambda: {self.lm}"
+
+    @staticmethod
+    def _nghood_fn(rankings, lm):
+        return torch.exp(-rankings / lm)

prototorch/modules/models.py

@@ -0,0 +1,194 @@
+import torch
+from torch import nn
+
+from prototorch.functions.distances import euclidean_distance_matrix, tangent_distance
+from prototorch.functions.helper import _check_shapes, _int_and_mixed_shape
+from prototorch.functions.normalization import orthogonalization
+from prototorch.modules.prototypes import Prototypes1D
+
+
+class GTLVQ(nn.Module):
+    r""" Generalized Tangent Learning Vector Quantization
+
+    Parameters
+    ----------
+    num_classes: int
+        Number of classes of the given classification problem.
+
+    subspace_data: torch.tensor of shape (n_batch,feature_dim,feature_dim)
+        Subspace data for the point approximation, required
+
+    prototype_data: torch.tensor of shape (n_init_data,feature_dim) (optional)
+        prototype data for initalization of the prototypes used in GTLVQ.
+
+    subspace_size: int (default=256,optional)
+        Subspace dimension of the Projectors. Currently only supported
+        with tagnent_projection_type=global.
+
+    tangent_projection_type: string
+        Specifies the tangent projection type
+        options:    local
+                    local_proj
+                    global
+        local: computes the tangent distances without emphasizing projected
+        data. Only distances are available
+        local_proj: computs tangent distances and returns the projected data
+        for further use. Be careful: data is repeated by number of prototypes
+        global: Number of subspaces is set to one and every prototypes
+        uses the same.
+
+    prototypes_per_class: int (default=2,optional)
+    Number of prototypes per class
+
+    feature_dim: int (default=256)
+    Dimensionality of the feature space specified as integer.
+    Prototype dimension.
+
+    Notes
+    -----
+    The GTLVQ [1] is a prototype-based classification learning model. The
+    GTLVQ uses the Tangent-Distances for a local point approximation
+    of an assumed data manifold via prototypial representations.
+
+    The GTLVQ requires subspace projectors for transforming the data
+    and prototypes into the affine subspace. Every prototype is
+    equipped with a specific subpspace and represents a point
+    approximation of the assumed manifold.
+
+    In practice prototypes and data are projected on this manifold
+    and pairwise euclidean distance computes.
+
+    References
+    ----------
+    .. [1] Saralajew, Sascha; Villmann, Thomas: Transfer learning
+    in classification based on manifolc. models and its relation
+    to tangent metric learning. In: 2017 International Joint
+    Conference on Neural Networks (IJCNN).
+    Bd. 2017-May : IEEE, 2017, S. 1756–1765
+    """
+    def __init__(
+        self,
+        num_classes,
+        subspace_data=None,
+        prototype_data=None,
+        subspace_size=256,
+        tangent_projection_type="local",
+        prototypes_per_class=2,
+        feature_dim=256,
+    ):
+        super(GTLVQ, self).__init__()
+
+        self.num_protos = num_classes * prototypes_per_class
+        self.subspace_size = feature_dim if subspace_size is None else subspace_size
+        self.feature_dim = feature_dim
+
+        if subspace_data is None:
+            raise ValueError("Init Data must be specified!")
+
+        self.tpt = tangent_projection_type
+        with torch.no_grad():
+            if self.tpt == "local" or self.tpt == "local_proj":
+                self.init_local_subspace(subspace_data)
+            elif self.tpt == "global":
+                self.init_gobal_subspace(subspace_data, subspace_size)
+            else:
+                self.subspaces = None
+
+        # Hypothesis-Margin-Classifier
+        self.cls = Prototypes1D(
+            input_dim=feature_dim,
+            prototypes_per_class=prototypes_per_class,
+            nclasses=num_classes,
+            prototype_initializer="stratified_mean",
+            data=prototype_data,
+        )
+
+    def forward(self, x):
+        # Tangent Projection
+        if self.tpt == "local_proj":
+            x_conform = (x.unsqueeze(1).repeat_interleave(self.num_protos,
+                                                          1).unsqueeze(2))
+            dis, proj_x = self.local_tangent_projection(x_conform)
+
+            proj_x = proj_x.reshape(x.shape[0] * self.num_protos,
+                                    self.feature_dim)
+            return proj_x, dis
+        elif self.tpt == "local":
+            x_conform = (x.unsqueeze(1).repeat_interleave(self.num_protos,
+                                                          1).unsqueeze(2))
+            dis = tangent_distance(x_conform, self.cls.prototypes,
+                                   self.subspaces)
+        elif self.tpt == "gloabl":
+            dis = self.global_tangent_distances(x)
+        else:
+            dis = (x @ self.cls.prototypes.T) / (
+                torch.norm(x, dim=1, keepdim=True) @ torch.norm(
+                    self.cls.prototypes, dim=1, keepdim=True).T)
+        return dis
+
+    def init_gobal_subspace(self, data, num_subspaces):
+        _, _, v = torch.svd(data)
+        subspace = (torch.eye(v.shape[0]) - (v @ v.T)).T
+        subspaces = subspace[:, :num_subspaces]
+        self.subspaces = (torch.nn.Parameter(
+            subspaces).clone().detach().requires_grad_(True))
+
+    def init_local_subspace(self, data):
+        _, _, v = torch.svd(data)
+        inital_projector = (torch.eye(v.shape[0]) - (v @ v.T)).T
+        subspaces = inital_projector.unsqueeze(0).repeat_interleave(
+            self.num_protos, 0)
+        self.subspaces = (torch.nn.Parameter(
+            subspaces).clone().detach().requires_grad_(True))
+
+    def global_tangent_distances(self, x):
+        # Tangent Projection
+        x, projected_prototypes = (
+            x @ self.subspaces,
+            self.cls.prototypes @ self.subspaces,
+        )
+        # Euclidean Distance
+        return euclidean_distance_matrix(x, projected_prototypes)
+
+    def local_tangent_projection(self, signals):
+        # Note: subspaces is always assumed as transposed and must be orthogonal!
+        # shape(signals): batch x proto_number x channels x dim1 x dim2 x ... x dimN
+        # shape(protos): proto_number x dim1 x dim2 x ... x dimN
+        # shape(subspaces): (optional [proto_number]) x prod(dim1 * dim2 * ... * dimN)  x prod(projected_atom_shape)
+        # subspace should be orthogonalized
+        # Origin Source Code
+        # Origin Author:
+        protos = self.cls.prototypes
+        subspaces = self.subspaces
+        signal_shape, signal_int_shape = _int_and_mixed_shape(signals)
+        _, proto_int_shape = _int_and_mixed_shape(protos)
+
+        # check if the shapes are correct
+        _check_shapes(signal_int_shape, proto_int_shape)
+
+        # Tangent Data Projections
+        projected_protos = torch.bmm(protos.unsqueeze(1), subspaces).squeeze(1)
+        data = signals.squeeze(2).permute([1, 0, 2])
+        projected_data = torch.bmm(data, subspaces)
+        projected_data = projected_data.permute([1, 0, 2]).unsqueeze(1)
+        diff = projected_data - projected_protos
+        projected_diff = torch.reshape(
+            diff, (signal_shape[1], signal_shape[0], signal_shape[2]) +
+            signal_shape[3:])
+        diss = torch.norm(projected_diff, 2, dim=-1)
+        return diss.permute([1, 0, 2]).squeeze(-1), projected_data.squeeze(1)
+
+    def get_parameters(self):
+        return {
+            "params": self.cls.prototypes,
+        }, {
+            "params": self.subspaces
+        }
+
+    def orthogonalize_subspace(self):
+        if self.subspaces is not None:
+            with torch.no_grad():
+                ortho_subpsaces = (orthogonalization(self.subspaces)
+                                   if self.tpt == "global" else
+                                   torch.nn.init.orthogonal_(self.subspaces))
+                self.subspaces.copy_(ortho_subpsaces)

prototorch/modules/prototypes.py

@@ -1,57 +1,137 @@
 """ProtoTorch prototype modules."""
 
+import warnings
+
 import torch
 
 from prototorch.functions.initializers import get_initializer
 
 
-class AddPrototypes1D(torch.nn.Module):
+class _Prototypes(torch.nn.Module):
-    def __init__(self,
+    """Abstract prototypes class."""
-                 prototypes_per_class=1,
+    def __init__(self, *args, **kwargs):
-                 prototype_distribution=None,
+        super().__init__(*args, **kwargs)
-                 prototype_initializer='ones',
+
-                 data=None,
+    def _validate_prototype_distribution(self):
-                 **kwargs):
+        if 0 in self.prototype_distribution:
+            warnings.warn("Are you sure about the `0` in "
+                          "`prototype_distribution`?")
+
+    def extra_repr(self):
+        return f"prototypes.shape: {tuple(self.prototypes.shape)}"
+
+    def forward(self):
+        return self.prototypes, self.prototype_labels
+
+
+class Prototypes1D(_Prototypes):
+    """Create a learnable set of one-dimensional prototypes.
+
+    TODO Complete this doc-string.
+    """
+    def __init__(
+        self,
+        prototypes_per_class=1,
+        prototype_initializer="ones",
+        prototype_distribution=None,
+        data=None,
+        dtype=torch.float32,
+        one_hot_labels=False,
+        **kwargs,
+    ):
+        warnings.warn(
+            PendingDeprecationWarning(
+                "Prototypes1D will be replaced in future versions."))
+
+        # Convert tensors to python lists before processing
+        if prototype_distribution is not None:
+            if not isinstance(prototype_distribution, list):
+                prototype_distribution = prototype_distribution.tolist()
 
         if data is None:
-            if 'input_dim' not in kwargs:
+            if "input_dim" not in kwargs:
-                raise NameError('`input_dim` required if '
+                raise NameError("`input_dim` required if "
-                                'no `data` is provided.')
+                                "no `data` is provided.")
-            if prototype_distribution is not None:
+            if prototype_distribution:
-                nclasses = sum(prototype_distribution)
+                kwargs_nclasses = sum(prototype_distribution)
             else:
-                if 'nclasses' not in kwargs:
+                if "nclasses" not in kwargs:
-                    raise NameError('`prototype_distribution` required if '
+                    raise NameError("`prototype_distribution` required if "
-                                    'both `data` and `nclasses` are not '
+                                    "both `data` and `nclasses` are not "
-                                    'provided.')
+                                    "provided.")
-                nclasses = kwargs.pop('nclasses')
+                kwargs_nclasses = kwargs.pop("nclasses")
-            input_dim = kwargs.pop('input_dim')
+            input_dim = kwargs.pop("input_dim")
-            # input_shape = (input_dim, )
+            if prototype_initializer in [
-            x_train = torch.rand(nclasses, input_dim)
+                    "stratified_mean", "stratified_random"
-            y_train = torch.arange(nclasses)
+            ]:
+                warnings.warn(
+                    f"`prototype_initializer`: `{prototype_initializer}` "
+                    "requires `data`, but `data` is not provided. "
+                    "Using randomly generated data instead.")
+            x_train = torch.rand(kwargs_nclasses, input_dim)
+            y_train = torch.arange(kwargs_nclasses)
+            if one_hot_labels:
+                y_train = torch.eye(kwargs_nclasses)[y_train]
+            data = [x_train, y_train]
 
-        else:
+        x_train, y_train = data
-            x_train, y_train = data
+        x_train = torch.as_tensor(x_train).type(dtype)
-            x_train = torch.as_tensor(x_train)
+        y_train = torch.as_tensor(y_train).type(torch.int)
-            y_train = torch.as_tensor(y_train)
+        nclasses = torch.unique(y_train, dim=-1).shape[-1]
+
+        if nclasses == 1:
+            warnings.warn("Are you sure about having one class only?")
+
+        if x_train.ndim != 2:
+            raise ValueError("`data[0].ndim != 2`.")
+
+        if y_train.ndim == 2:
+            if y_train.shape[1] == 1 and one_hot_labels:
+                raise ValueError("`one_hot_labels` is set to `True` "
+                                 "but target labels are not one-hot-encoded.")
+            if y_train.shape[1] != 1 and not one_hot_labels:
+                raise ValueError("`one_hot_labels` is set to `False` "
+                                 "but target labels in `data` "
+                                 "are one-hot-encoded.")
+        if y_train.ndim == 1 and one_hot_labels:
+            raise ValueError("`one_hot_labels` is set to `True` "
+                             "but target labels are not one-hot-encoded.")
+
+        # Verify input dimension if `input_dim` is provided
+        if "input_dim" in kwargs:
+            input_dim = kwargs.pop("input_dim")
+            if input_dim != x_train.shape[1]:
+                raise ValueError(f"Provided `input_dim`={input_dim} does "
+                                 "not match data dimension "
+                                 f"`data[0].shape[1]`={x_train.shape[1]}")
+
+        # Verify the number of classes if `nclasses` is provided
+        if "nclasses" in kwargs:
+            kwargs_nclasses = kwargs.pop("nclasses")
+            if kwargs_nclasses != nclasses:
+                raise ValueError(f"Provided `nclasses={kwargs_nclasses}` does "
+                                 "not match data labels "
+                                 "`torch.unique(data[1]).shape[0]`"
+                                 f"={nclasses}")
 
         super().__init__(**kwargs)
-        self.prototypes_per_class = prototypes_per_class
+
+        if not prototype_distribution:
+            prototype_distribution = [prototypes_per_class] * nclasses
         with torch.no_grad():
-            if not prototype_distribution:
+            self.prototype_distribution = torch.tensor(prototype_distribution)
-                num_classes = torch.unique(y_train).shape[0]
+
-                self.prototype_distribution = torch.tensor(
+        self._validate_prototype_distribution()
-                    [self.prototypes_per_class] * num_classes)
+
-            else:
-                self.prototype_distribution = torch.tensor(
-                    prototype_distribution)
         self.prototype_initializer = get_initializer(prototype_initializer)
         prototypes, prototype_labels = self.prototype_initializer(
             x_train,
             y_train,
-            prototype_distribution=self.prototype_distribution)
+            prototype_distribution=self.prototype_distribution,
-        self.prototypes = torch.nn.Parameter(prototypes)
+            one_hot=one_hot_labels,
-        self.prototype_labels = prototype_labels
+        )
 
-    def forward(self):
+        # Register module parameters
-        return self.prototypes, self.prototype_labels
+        self.prototypes = torch.nn.Parameter(prototypes)
+        self.prototype_labels = torch.nn.Parameter(
+            prototype_labels.type(dtype)).requires_grad_(False)

prototorch/utils/celluloid.py

@@ -0,0 +1,46 @@
+"""Easy matplotlib animation. From https://github.com/jwkvam/celluloid."""
+
+from collections import defaultdict
+from typing import Dict, List
+
+from matplotlib.animation import ArtistAnimation
+from matplotlib.artist import Artist
+from matplotlib.figure import Figure
+
+__version__ = "0.2.0"
+
+
+class Camera:
+    """Make animations easier."""
+    def __init__(self, figure: Figure) -> None:
+        """Create camera from matplotlib figure."""
+        self._figure = figure
+        # need to keep track off artists for each axis
+        self._offsets: Dict[str, Dict[int, int]] = {
+            k: defaultdict(int)
+            for k in
+            ["collections", "patches", "lines", "texts", "artists", "images"]
+        }
+        self._photos: List[List[Artist]] = []
+
+    def snap(self) -> List[Artist]:
+        """Capture current state of the figure."""
+        frame_artists: List[Artist] = []
+        for i, axis in enumerate(self._figure.axes):
+            if axis.legend_ is not None:
+                axis.add_artist(axis.legend_)
+            for name in self._offsets:
+                new_artists = getattr(axis, name)[self._offsets[name][i]:]
+                frame_artists += new_artists
+                self._offsets[name][i] += len(new_artists)
+        self._photos.append(frame_artists)
+        return frame_artists
+
+    def animate(self, *args, **kwargs) -> ArtistAnimation:
+        """Animate the snapshots taken.
+        Uses matplotlib.animation.ArtistAnimation
+        Returns
+        -------
+        ArtistAnimation
+        """
+        return ArtistAnimation(self._figure, self._photos, *args, **kwargs)

prototorch/utils/colors.py

@@ -0,0 +1,78 @@
+"""ProtoFlow color utilities."""
+
+import matplotlib.lines as mlines
+from matplotlib import cm
+from matplotlib.colors import Normalize, to_hex, to_rgb
+
+
+def color_scheme(n,
+                 cmap="viridis",
+                 form="hex",
+                 tikz=False,
+                 zero_indexed=False):
+    """Return *n* colors from the color scheme.
+
+    Arguments:
+        n (int): number of colors to return
+
+    Keyword Arguments:
+        cmap (str): Name of a matplotlib `colormap\
+            <https://matplotlib.org/3.1.1/gallery/color/colormap_reference.html>`_.
+        form (str): Colorformat (supports "hex" and "rgb").
+        tikz (bool): Output as `TikZ <https://github.com/pgf-tikz/pgf>`_
+            command.
+        zero_indexed (bool): Use zero indexing for output array.
+
+    Returns:
+        (list): List of colors
+    """
+    cmap = cm.get_cmap(cmap)
+    colornorm = Normalize(vmin=1, vmax=n)
+    hex_map = dict()
+    rgb_map = dict()
+    for cl in range(1, n + 1):
+        if zero_indexed:
+            hex_map[cl - 1] = to_hex(cmap(colornorm(cl)))
+            rgb_map[cl - 1] = to_rgb(cmap(colornorm(cl)))
+        else:
+            hex_map[cl] = to_hex(cmap(colornorm(cl)))
+            rgb_map[cl] = to_rgb(cmap(colornorm(cl)))
+    if tikz:
+        for k, v in rgb_map.items():
+            print(f"\\definecolor{{color-{k}}}{{rgb}}{{{v[0]},{v[1]},{v[2]}}}")
+    if form == "hex":
+        return hex_map
+    elif form == "rgb":
+        return rgb_map
+    else:
+        return hex_map
+
+
+def get_legend_handles(labels, marker="dots", zero_indexed=False):
+    """Return matplotlib legend handles and colors."""
+    handles = list()
+    n = len(labels)
+    colors = color_scheme(n,
+                          cmap="viridis",
+                          form="hex",
+                          zero_indexed=zero_indexed)
+    for label, color in zip(labels, colors.values()):
+        if marker == "dots":
+            handle = mlines.Line2D(
+                [],
+                [],
+                color="white",
+                markerfacecolor=color,
+                marker="o",
+                markersize=10,
+                markeredgecolor="k",
+                label=label,
+            )
+        else:
+            handle = mlines.Line2D([], [],
+                                   color=color,
+                                   marker="",
+                                   markersize=15,
+                                   label=label)
+            handles.append(handle)
+    return handles, colors

prototorch/utils/utils.py

@@ -0,0 +1,243 @@
+"""Utilities that provide various small functionalities."""
+
+import os
+import pickle
+import sys
+from time import time
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def progressbar(title, value, end, bar_width=20):
+    percent = float(value) / end
+    arrow = "=" * int(round(percent * bar_width) - 1) + ">"
+    spaces = "." * (bar_width - len(arrow))
+    sys.stdout.write("\r{}: [{}] {}%".format(title, arrow + spaces,
+                                             int(round(percent * 100))))
+    sys.stdout.flush()
+    if percent == 1.0:
+        print()
+
+
+def prettify_string(inputs, start="", sep=" ", end="\n"):
+    outputs = start + " ".join(inputs.split()) + end
+    return outputs
+
+
+def pretty_print(inputs):
+    print(prettify_string(inputs))
+
+
+def writelog(self, *logs, logdir="./logs", logfile="run.txt"):
+    f = os.path.join(logdir, logfile)
+    with open(f, "a+") as fh:
+        for log in logs:
+            fh.write(log)
+            fh.write("\n")
+
+
+def start_tensorboard(self, logdir="./logs"):
+    cmd = f"tensorboard --logdir={logdir} --port=6006"
+    os.system(cmd)
+
+
+def make_directory(save_dir):
+    if not os.path.exists(save_dir):
+        print(f"Making directory {save_dir}.")
+        os.mkdir(save_dir)
+
+
+def make_gif(filenames, duration, output_file=None):
+    try:
+        import imageio
+    except ModuleNotFoundError as e:
+        print("Please install Protoflow with [other] extra requirements.")
+        raise (e)
+
+    images = list()
+    for filename in filenames:
+        images.append(imageio.imread(filename))
+    if not output_file:
+        output_file = f"makegif.gif"
+    if images:
+        imageio.mimwrite(output_file, images, duration=duration)
+
+
+def gif_from_dir(directory,
+                 duration,
+                 prefix="",
+                 output_file=None,
+                 verbose=True):
+    images = os.listdir(directory)
+    if verbose:
+        print(f"Making gif from {len(images)} images under {directory}.")
+    filenames = list()
+    # Sort images
+    images = sorted(
+        images,
+        key=lambda img: int(os.path.splitext(img)[0].replace(prefix, "")))
+    for image in images:
+        fname = os.path.join(directory, image)
+        filenames.append(fname)
+    if not output_file:
+        output_file = os.path.join(directory, "makegif.gif")
+    make_gif(filenames=filenames, duration=duration, output_file=output_file)
+
+
+def accuracy_score(y_true, y_pred):
+    accuracy = np.sum(y_true == y_pred)
+    normalized_acc = accuracy / float(len(y_true))
+    return normalized_acc
+
+
+def predict_and_score(clf,
+                      x_test,
+                      y_test,
+                      verbose=False,
+                      title="Test accuracy"):
+    y_pred = clf.predict(x_test)
+    accuracy = np.sum(y_test == y_pred)
+    normalized_acc = accuracy / float(len(y_test))
+    if verbose:
+        print(f"{title}: {normalized_acc * 100:06.04f}%")
+    return normalized_acc
+
+
+def remove_nan_rows(arr):
+    """Remove all rows with `nan` values in `arr`."""
+    mask = np.isnan(arr).any(axis=1)
+    return arr[~mask]
+
+
+def remove_nan_cols(arr):
+    """Remove all columns with `nan` values in `arr`."""
+    mask = np.isnan(arr).any(axis=0)
+    return arr[~mask]
+
+
+def replace_in(arr, replacement_dict, inplace=False):
+    """Replace the keys found in `arr` with the values from
+    the `replacement_dict`.
+    """
+    if inplace:
+        new_arr = arr
+    else:
+        import copy
+
+        new_arr = copy.deepcopy(arr)
+    for k, v in replacement_dict.items():
+        new_arr[arr == k] = v
+    return new_arr
+
+
+def train_test_split(data, train=0.7, val=0.15, shuffle=None, return_xy=False):
+    """Split a classification dataset in such a way so as to
+    preserve the class distribution in subsamples of the dataset.
+    """
+    if train + val > 1.0:
+        raise ValueError("Invalid split values for train and val.")
+    Y = data[:, -1]
+    labels = set(Y)
+    hist = dict()
+    for l in labels:
+        data_l = data[Y == l]
+        nl = len(data_l)
+        nl_train = int(nl * train)
+        nl_val = int(nl * val)
+        nl_test = nl - (nl_train + nl_val)
+        hist[l] = (nl_train, nl_val, nl_test)
+
+    train_data = list()
+    val_data = list()
+    test_data = list()
+    for l, (nl_train, nl_val, nl_test) in hist.items():
+        data_l = data[Y == l]
+        if shuffle:
+            np.random.shuffle(data_l)
+        train_l = data_l[:nl_train]
+        val_l = data_l[nl_train:nl_train + nl_val]
+        test_l = data_l[nl_train + nl_val:nl_train + nl_val + nl_test]
+        train_data.append(train_l)
+        val_data.append(val_l)
+        test_data.append(test_l)
+
+    def _squash(data_list):
+        data = np.array(data_list[0])
+        for item in data_list[1:]:
+            data = np.vstack((data, np.array(item)))
+        return data
+
+    train_data = _squash(train_data)
+    if val_data:
+        val_data = _squash(val_data)
+    if test_data:
+        test_data = _squash(test_data)
+    if return_xy:
+        x_train = train_data[:, :-1]
+        y_train = train_data[:, -1]
+        x_val = val_data[:, :-1]
+        y_val = val_data[:, -1]
+        x_test = test_data[:, :-1]
+        y_test = test_data[:, -1]
+        return (x_train, y_train), (x_val, y_val), (x_test, y_test)
+    return train_data, val_data, test_data
+
+
+def class_histogram(data, title="Untitled"):
+    plt.figure(title)
+    plt.clf()
+    plt.title(title)
+    dist, counts = np.unique(data[:, -1], return_counts=True)
+    plt.bar(dist, counts)
+    plt.xticks(dist)
+    print("Call matplotlib.pyplot.show() to see the plot.")
+
+
+def ntimer(n=10):
+    """Wraps a function which wraps another function to time it."""
+    if n < 1:
+        raise (Exception(f"Invalid n = {n} given."))
+
+    def timer(func):
+        """Wraps `func` with a timer and returns the wrapped `func`."""
+        def wrapper(*args, **kwargs):
+            rv = None
+            before = time()
+            for _ in range(n):
+                rv = func(*args, **kwargs)
+            after = time()
+            elapsed = after - before
+            print(f"Elapsed: {elapsed*1e3:02.02f} ms")
+            return rv
+
+        return wrapper
+
+    return timer
+
+
+def memoize(verbose=True):
+    """Wraps a function which wraps another function that memoizes."""
+    def memoizer(func):
+        """Memoize (cache) return values of `func`.
+        Wraps `func` and returns the wrapped `func` so that `func`
+        is executed when the results are not available in the cache.
+        """
+        cache = {}
+
+        def wrapper(*args, **kwargs):
+            t = (pickle.dumps(args), pickle.dumps(kwargs))
+            if t not in cache:
+                if verbose:
+                    print(f"Adding NEW rv {func.__name__}{args}{kwargs} "
+                          "to cache.")
+                cache[t] = func(*args, **kwargs)
+            else:
+                if verbose:
+                    print(f"Using OLD rv {func.__name__}{args}{kwargs} "
+                          "from cache.")
+            return cache[t]
+
+        return wrapper
+
+    return memoizer

setup.py

@@ -1,49 +1,83 @@
-"""Install ProtoTorch."""
+"""
+  _____           _     _______             _
+ |  __ \         | |   |__   __|           | |
+ | |__) | __ ___ | |_ ___ | | ___  _ __ ___| |__
+ |  ___/ '__/ _ \| __/ _ \| |/ _ \| '__/ __| '_ \
+ | |   | | | (_) | || (_) | | (_) | | | (__| | | |
+ |_|   |_|  \___/ \__\___/|_|\___/|_|  \___|_| |_|
 
-from setuptools import setup
+ProtoTorch Core Package
-from setuptools import find_packages
+"""
+from setuptools import find_packages, setup
 
-PROJECT_URL = 'https://github.com/si-cim/prototorch'
+PROJECT_URL = "https://github.com/si-cim/prototorch"
-DOWNLOAD_URL = 'https://github.com/si-cim/prototorch.git'
+DOWNLOAD_URL = "https://github.com/si-cim/prototorch.git"
 
-with open('README.md', 'r') as fh:
+with open("README.md", "r") as fh:
     long_description = fh.read()
 
-setup(name='prototorch',
+INSTALL_REQUIRES = [
-      version='0.1.1-dev0',
+    "torch>=1.3.1",
-      description='Highly extensible, GPU-supported '
+    "torchvision>=0.5.0",
-      'Learning Vector Quantization (LVQ) toolbox '
+    "numpy>=1.9.1",
-      'built using PyTorch and its nn API.',
+]
-      long_description=long_description,
+DATASETS = [
-      long_description_content_type='text/markdown',
+    "requests",
-      author='Jensun Ravichandran',
+    "tqdm",
-      author_email='jjensun@gmail.com',
+]
-      url=PROJECT_URL,
+DEV = ["bumpversion"]
-      download_url=DOWNLOAD_URL,
+DOCS = [
-      license='MIT',
+    "recommonmark",
-      install_requires=[
+    "sphinx",
-          'torch>=1.3.1',
+    "sphinx_rtd_theme",
-          'torchvision>=0.5.0',
+    "sphinxcontrib-katex",
-          'numpy>=1.9.1',
+]
-      ],
+EXAMPLES = [
-      extras_require={
+    "sklearn",
-          'examples': [
+    "matplotlib",
-              'sklearn',
+    "torchinfo",
-              'matplotlib',
+]
-          ],
+TESTS = ["codecov", "pytest"]
-          'tests': ['pytest'],
+ALL = DATASETS + DEV + DOCS + EXAMPLES + TESTS
-      },
+
-      classifiers=[
+setup(
-          'Development Status :: 2 - Pre-Alpha', 'Environment :: Console',
+    name="prototorch",
-          'Intended Audience :: Developers', 'Intended Audience :: Education',
+    version="0.4.2",
-          'Intended Audience :: Science/Research',
+    description="Highly extensible, GPU-supported "
-          'License :: OSI Approved :: MIT License',
+    "Learning Vector Quantization (LVQ) toolbox "
-          'Programming Language :: Python :: 3.6',
+    "built using PyTorch and its nn API.",
-          'Programming Language :: Python :: 3.7',
+    long_description=long_description,
-          'Programming Language :: Python :: 3.8',
+    long_description_content_type="text/markdown",
-          'Operating System :: OS Independent',
+    author="Jensun Ravichandran",
-          'Topic :: Scientific/Engineering :: Artificial Intelligence',
+    author_email="jjensun@gmail.com",
-          'Topic :: Software Development :: Libraries',
+    url=PROJECT_URL,
-          'Topic :: Software Development :: Libraries :: Python Modules'
+    download_url=DOWNLOAD_URL,
-      ],
+    license="MIT",
-      packages=find_packages())
+    install_requires=INSTALL_REQUIRES,
+    extras_require={
+        "docs": DOCS,
+        "datasets": DATASETS,
+        "examples": EXAMPLES,
+        "tests": TESTS,
+        "all": ALL,
+    },
+    classifiers=[
+        "Development Status :: 2 - Pre-Alpha",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Education",
+        "Intended Audience :: Science/Research",
+        "License :: OSI Approved :: MIT License",
+        "Natural Language :: English",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Operating System :: OS Independent",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+        "Topic :: Software Development :: Libraries",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+    ],
+    packages=find_packages(),
+    zip_safe=False,
+)

tests/test_datasets.py

@@ -0,0 +1,95 @@
+"""ProtoTorch datasets test suite."""
+
+import os
+import shutil
+import unittest
+
+import torch
+
+from prototorch.datasets import abstract, tecator
+
+
+class TestAbstract(unittest.TestCase):
+    def test_getitem(self):
+        with self.assertRaises(NotImplementedError):
+            abstract.Dataset("./artifacts")[0]
+
+    def test_len(self):
+        with self.assertRaises(NotImplementedError):
+            len(abstract.Dataset("./artifacts"))
+
+
+class TestProtoDataset(unittest.TestCase):
+    def test_getitem(self):
+        with self.assertRaises(NotImplementedError):
+            abstract.ProtoDataset("./artifacts")[0]
+
+    def test_download(self):
+        with self.assertRaises(NotImplementedError):
+            abstract.ProtoDataset("./artifacts").download()
+
+
+class TestTecator(unittest.TestCase):
+    def setUp(self):
+        self.artifacts_dir = "./artifacts/Tecator"
+        self._remove_artifacts()
+
+    def _remove_artifacts(self):
+        if os.path.exists(self.artifacts_dir):
+            shutil.rmtree(self.artifacts_dir)
+
+    def test_download_false(self):
+        rootdir = self.artifacts_dir.rpartition("/")[0]
+        self._remove_artifacts()
+        with self.assertRaises(RuntimeError):
+            _ = tecator.Tecator(rootdir, download=False)
+
+    def test_download_caching(self):
+        rootdir = self.artifacts_dir.rpartition("/")[0]
+        _ = tecator.Tecator(rootdir, download=True, verbose=False)
+        _ = tecator.Tecator(rootdir, download=False, verbose=False)
+
+    def test_repr(self):
+        rootdir = self.artifacts_dir.rpartition("/")[0]
+        train = tecator.Tecator(rootdir, download=True, verbose=True)
+        self.assertTrue("Split: Train" in train.__repr__())
+
+    def test_download_train(self):
+        rootdir = self.artifacts_dir.rpartition("/")[0]
+        train = tecator.Tecator(root=rootdir,
+                                train=True,
+                                download=True,
+                                verbose=False)
+        train = tecator.Tecator(root=rootdir, download=True, verbose=False)
+        x_train, y_train = train.data, train.targets
+        self.assertEqual(x_train.shape[0], 144)
+        self.assertEqual(y_train.shape[0], 144)
+        self.assertEqual(x_train.shape[1], 100)
+
+    def test_download_test(self):
+        rootdir = self.artifacts_dir.rpartition("/")[0]
+        test = tecator.Tecator(root=rootdir, train=False, verbose=False)
+        x_test, y_test = test.data, test.targets
+        self.assertEqual(x_test.shape[0], 71)
+        self.assertEqual(y_test.shape[0], 71)
+        self.assertEqual(x_test.shape[1], 100)
+
+    def test_class_to_idx(self):
+        rootdir = self.artifacts_dir.rpartition("/")[0]
+        test = tecator.Tecator(root=rootdir, train=False, verbose=False)
+        _ = test.class_to_idx
+
+    def test_getitem(self):
+        rootdir = self.artifacts_dir.rpartition("/")[0]
+        test = tecator.Tecator(root=rootdir, train=False, verbose=False)
+        x, y = test[0]
+        self.assertEqual(x.shape[0], 100)
+        self.assertIsInstance(y, int)
+
+    def test_loadable_with_dataloader(self):
+        rootdir = self.artifacts_dir.rpartition("/")[0]
+        test = tecator.Tecator(root=rootdir, train=False, verbose=False)
+        _ = torch.utils.data.DataLoader(test, batch_size=64, shuffle=True)
+
+    def tearDown(self):
+        pass

tests/test_functions.py

@@ -5,13 +5,18 @@ import unittest
 import numpy as np
 import torch
 
-from prototorch.functions import (activations, competitions, distances,
+from prototorch.functions import (
-                                  initializers, losses)
+    activations,
+    competitions,
+    distances,
+    initializers,
+    losses,
+)
 
 
 class TestActivations(unittest.TestCase):
     def setUp(self):
-        self.flist = ['identity', 'sigmoid_beta', 'swish_beta']
+        self.flist = ["identity", "sigmoid_beta", "swish_beta"]
         self.x = torch.randn(1024, 1)
 
     def test_registry(self):
@@ -39,7 +44,7 @@ class TestActivations(unittest.TestCase):
             self.assertEqual(1, f(1))
 
     def test_unknown_deserialization(self):
-        for funcname in ['blubb', 'foobar']:
+        for funcname in ["blubb", "foobar"]:
             with self.assertRaises(NameError):
                 _ = activations.get_activation(funcname)
 
@@ -52,7 +57,7 @@ class TestActivations(unittest.TestCase):
         self.assertIsNone(mismatch)
 
     def test_sigmoid_beta1(self):
-        actual = activations.sigmoid_beta(self.x, beta=torch.tensor(1))
+        actual = activations.sigmoid_beta(self.x, beta=1.0)
         desired = torch.sigmoid(self.x)
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
@@ -60,7 +65,7 @@ class TestActivations(unittest.TestCase):
         self.assertIsNone(mismatch)
 
     def test_swish_beta1(self):
-        actual = activations.swish_beta(self.x, beta=torch.tensor(1))
+        actual = activations.swish_beta(self.x, beta=1.0)
         desired = self.x * torch.sigmoid(self.x)
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
@@ -76,7 +81,7 @@ class TestCompetitions(unittest.TestCase):
         pass
 
     def test_wtac(self):
-        d = torch.tensor([[2., 3., 1.99, 3.01], [2., 3., 2.01, 3.]])
+        d = torch.tensor([[2.0, 3.0, 1.99, 3.01], [2.0, 3.0, 2.01, 3.0]])
         labels = torch.tensor([0, 1, 2, 3])
         actual = competitions.wtac(d, labels)
         desired = torch.tensor([2, 0])
@@ -85,8 +90,18 @@ class TestCompetitions(unittest.TestCase):
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
+    def test_wtac_unequal_dist(self):
+        d = torch.tensor([[2.0, 3.0, 4.0], [2.0, 3.0, 1.0]])
+        labels = torch.tensor([0, 1, 1])
+        actual = competitions.wtac(d, labels)
+        desired = torch.tensor([0, 1])
+        mismatch = np.testing.assert_array_almost_equal(actual,
+                                                        desired,
+                                                        decimal=5)
+        self.assertIsNone(mismatch)
+
     def test_wtac_one_hot(self):
-        d = torch.tensor([[1.99, 3.01], [3., 2.01]])
+        d = torch.tensor([[1.99, 3.01], [3.0, 2.01]])
         labels = torch.tensor([[0, 1], [1, 0]])
         actual = competitions.wtac(d, labels)
         desired = torch.tensor([[0, 1], [1, 0]])
@@ -95,8 +110,39 @@ class TestCompetitions(unittest.TestCase):
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
+    def test_stratified_min(self):
+        d = torch.tensor([[1.0, 0.0, 2.0, 3.0], [9.0, 8.0, 0, 1]])
+        labels = torch.tensor([0, 0, 1, 2])
+        actual = competitions.stratified_min(d, labels)
+        desired = torch.tensor([[0.0, 2.0, 3.0], [8.0, 0.0, 1.0]])
+        mismatch = np.testing.assert_array_almost_equal(actual,
+                                                        desired,
+                                                        decimal=5)
+        self.assertIsNone(mismatch)
+
+    def test_stratified_min_one_hot(self):
+        d = torch.tensor([[1.0, 0.0, 2.0, 3.0], [9.0, 8.0, 0, 1]])
+        labels = torch.tensor([0, 0, 1, 2])
+        labels = torch.eye(3)[labels]
+        actual = competitions.stratified_min(d, labels)
+        desired = torch.tensor([[0.0, 2.0, 3.0], [8.0, 0.0, 1.0]])
+        mismatch = np.testing.assert_array_almost_equal(actual,
+                                                        desired,
+                                                        decimal=5)
+        self.assertIsNone(mismatch)
+
+    def test_stratified_min_simple(self):
+        d = torch.tensor([[0.0, 2.0, 3.0], [8.0, 0, 1]])
+        labels = torch.tensor([0, 1, 2])
+        actual = competitions.stratified_min(d, labels)
+        desired = torch.tensor([[0.0, 2.0, 3.0], [8.0, 0.0, 1.0]])
+        mismatch = np.testing.assert_array_almost_equal(actual,
+                                                        desired,
+                                                        decimal=5)
+        self.assertIsNone(mismatch)
+
     def test_knnc_k1(self):
-        d = torch.tensor([[2., 3., 1.99, 3.01], [2., 3., 2.01, 3.]])
+        d = torch.tensor([[2.0, 3.0, 1.99, 3.01], [2.0, 3.0, 2.01, 3.0]])
         labels = torch.tensor([0, 1, 2, 3])
         actual = competitions.knnc(d, labels, k=torch.tensor([1]))
         desired = torch.tensor([2, 0])
@@ -153,12 +199,12 @@ class TestDistances(unittest.TestCase):
         desired = torch.empty(self.nx, self.ny)
         for i in range(self.nx):
             for j in range(self.ny):
-                desired[i][j] = torch.nn.functional.pairwise_distance(
+                desired[i][j] = (torch.nn.functional.pairwise_distance(
                     self.x[i].reshape(1, -1),
                     self.y[j].reshape(1, -1),
                     p=2,
                     keepdim=False,
-                )**2
+                )**2)
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
                                                         decimal=2)
@@ -213,14 +259,14 @@ class TestDistances(unittest.TestCase):
         self.assertIsNone(mismatch)
 
     def test_lpnorm_pinf(self):
-        actual = distances.lpnorm_distance(self.x, self.y, p=float('inf'))
+        actual = distances.lpnorm_distance(self.x, self.y, p=float("inf"))
         desired = torch.empty(self.nx, self.ny)
         for i in range(self.nx):
             for j in range(self.ny):
                 desired[i][j] = torch.nn.functional.pairwise_distance(
                     self.x[i].reshape(1, -1),
                     self.y[j].reshape(1, -1),
-                    p=float('inf'),
+                    p=float("inf"),
                     keepdim=False,
                 )
         mismatch = np.testing.assert_array_almost_equal(actual,
@@ -234,12 +280,12 @@ class TestDistances(unittest.TestCase):
         desired = torch.empty(self.nx, self.ny)
         for i in range(self.nx):
             for j in range(self.ny):
-                desired[i][j] = torch.nn.functional.pairwise_distance(
+                desired[i][j] = (torch.nn.functional.pairwise_distance(
                     self.x[i].reshape(1, -1),
                     self.y[j].reshape(1, -1),
                     p=2,
                     keepdim=False,
-                )**2
+                )**2)
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
                                                         decimal=2)
@@ -252,12 +298,12 @@ class TestDistances(unittest.TestCase):
         desired = torch.empty(self.nx, self.ny)
         for i in range(self.nx):
             for j in range(self.ny):
-                desired[i][j] = torch.nn.functional.pairwise_distance(
+                desired[i][j] = (torch.nn.functional.pairwise_distance(
                     self.x[i].reshape(1, -1),
                     self.y[j].reshape(1, -1),
                     p=2,
                     keepdim=False,
-                )**2
+                )**2)
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
                                                         decimal=2)
@@ -270,8 +316,12 @@ class TestDistances(unittest.TestCase):
 class TestInitializers(unittest.TestCase):
     def setUp(self):
         self.flist = [
-            'zeros', 'ones', 'rand', 'randn', 'stratified_mean',
+            "zeros",
-            'stratified_random'
+            "ones",
+            "rand",
+            "randn",
+            "stratified_mean",
+            "stratified_random",
         ]
         self.x = torch.tensor(
             [[0, -1, -2], [10, 11, 12], [0, 0, 0], [2, 2, 2]],
@@ -299,7 +349,7 @@ class TestInitializers(unittest.TestCase):
             self.assertEqual(1, f(1))
 
     def test_unknown_deserialization(self):
-        for funcname in ['blubb', 'foobar']:
+        for funcname in ["blubb", "foobar"]:
             with self.assertRaises(NameError):
                 _ = initializers.get_initializer(funcname)
 
@@ -341,8 +391,8 @@ class TestInitializers(unittest.TestCase):
 
     def test_stratified_mean_equal1(self):
         pdist = torch.tensor([1, 1])
-        actual, _ = initializers.stratified_mean(self.x, self.y, pdist)
+        actual, _ = initializers.stratified_mean(self.x, self.y, pdist, False)
-        desired = torch.tensor([[5., 5., 5.], [1., 1., 1.]])
+        desired = torch.tensor([[5.0, 5.0, 5.0], [1.0, 1.0, 1.0]])
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
                                                         decimal=5)
@@ -350,8 +400,9 @@ class TestInitializers(unittest.TestCase):
 
     def test_stratified_random_equal1(self):
         pdist = torch.tensor([1, 1])
-        actual, _ = initializers.stratified_random(self.x, self.y, pdist)
+        actual, _ = initializers.stratified_random(self.x, self.y, pdist,
-        desired = torch.tensor([[0., -1., -2.], [2., 2., 2.]])
+                                                   False)
+        desired = torch.tensor([[0.0, -1.0, -2.0], [0.0, 0.0, 0.0]])
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
                                                         decimal=5)
@@ -359,9 +410,20 @@ class TestInitializers(unittest.TestCase):
 
     def test_stratified_mean_equal2(self):
         pdist = torch.tensor([2, 2])
-        actual, _ = initializers.stratified_mean(self.x, self.y, pdist)
+        actual, _ = initializers.stratified_mean(self.x, self.y, pdist, False)
-        desired = torch.tensor([[5., 5., 5.], [5., 5., 5.], [1., 1., 1.],
+        desired = torch.tensor([[5.0, 5.0, 5.0], [5.0, 5.0, 5.0],
-                                [1., 1., 1.]])
+                                [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])
+        mismatch = np.testing.assert_array_almost_equal(actual,
+                                                        desired,
+                                                        decimal=5)
+        self.assertIsNone(mismatch)
+
+    def test_stratified_random_equal2(self):
+        pdist = torch.tensor([2, 2])
+        actual, _ = initializers.stratified_random(self.x, self.y, pdist,
+                                                   False)
+        desired = torch.tensor([[0.0, -1.0, -2.0], [0.0, -1.0, -2.0],
+                                [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
                                                         decimal=5)
@@ -369,9 +431,9 @@ class TestInitializers(unittest.TestCase):
 
     def test_stratified_mean_unequal(self):
         pdist = torch.tensor([1, 3])
-        actual, _ = initializers.stratified_mean(self.x, self.y, pdist)
+        actual, _ = initializers.stratified_mean(self.x, self.y, pdist, False)
-        desired = torch.tensor([[5., 5., 5.], [1., 1., 1.], [1., 1., 1.],
+        desired = torch.tensor([[5.0, 5.0, 5.0], [1.0, 1.0, 1.0],
-                                [1., 1., 1.]])
+                                [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
                                                         decimal=5)
@@ -379,14 +441,45 @@ class TestInitializers(unittest.TestCase):
 
     def test_stratified_random_unequal(self):
         pdist = torch.tensor([1, 3])
-        actual, _ = initializers.stratified_random(self.x, self.y, pdist)
+        actual, _ = initializers.stratified_random(self.x, self.y, pdist,
-        desired = torch.tensor([[0., -1., -2.], [2., 2., 2.], [0., 0., 0.],
+                                                   False)
-                                [0., 0., 0.]])
+        desired = torch.tensor([[0.0, -1.0, -2.0], [0.0, 0.0, 0.0],
+                                [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])
         mismatch = np.testing.assert_array_almost_equal(actual,
                                                         desired,
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
+    def test_stratified_mean_unequal_one_hot(self):
+        pdist = torch.tensor([1, 3])
+        y = torch.eye(2)[self.y]
+        desired1 = torch.tensor([[5.0, 5.0, 5.0], [1.0, 1.0, 1.0],
+                                 [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])
+        actual1, actual2 = initializers.stratified_mean(self.x, y, pdist)
+        desired2 = torch.tensor([[1, 0], [0, 1], [0, 1], [0, 1]])
+        mismatch = np.testing.assert_array_almost_equal(actual1,
+                                                        desired1,
+                                                        decimal=5)
+        mismatch = np.testing.assert_array_almost_equal(actual2,
+                                                        desired2,
+                                                        decimal=5)
+        self.assertIsNone(mismatch)
+
+    def test_stratified_random_unequal_one_hot(self):
+        pdist = torch.tensor([1, 3])
+        y = torch.eye(2)[self.y]
+        actual1, actual2 = initializers.stratified_random(self.x, y, pdist)
+        desired1 = torch.tensor([[0.0, -1.0, -2.0], [0.0, 0.0, 0.0],
+                                 [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])
+        desired2 = torch.tensor([[1, 0], [0, 1], [0, 1], [0, 1]])
+        mismatch = np.testing.assert_array_almost_equal(actual1,
+                                                        desired1,
+                                                        decimal=5)
+        mismatch = np.testing.assert_array_almost_equal(actual2,
+                                                        desired2,
+                                                        decimal=5)
+        self.assertIsNone(mismatch)
+
     def tearDown(self):
         del self.x, self.y, self.gen
         _ = torch.seed()
@@ -417,5 +510,17 @@ class TestLosses(unittest.TestCase):
         loss_value = torch.sum(batch_loss, dim=0)
         self.assertEqual(loss_value, -100)
 
+    def test_glvq_loss_one_hot_unequal(self):
+        dlist = [torch.ones(100), torch.zeros(100), torch.zeros(100)]
+        d = torch.stack(dlist, dim=1)
+        labels = torch.tensor([[0, 1], [1, 0], [1, 0]])
+        wl = torch.tensor([1, 0])
+        targets = torch.stack([wl for _ in range(100)], dim=0)
+        batch_loss = losses.glvq_loss(distances=d,
+                                      target_labels=targets,
+                                      prototype_labels=labels)
+        loss_value = torch.sum(batch_loss, dim=0)
+        self.assertEqual(loss_value, -100)
+
     def tearDown(self):
         pass

tests/test_kernels.py

@@ -0,0 +1,98 @@
+"""ProtoTorch kernels test suite."""
+
+import unittest
+
+import numpy as np
+import torch
+
+from prototorch.functions.distances import KernelDistance
+from prototorch.functions.kernels import ExplicitKernel, RadialBasisFunctionKernel
+
+
+class TestExplicitKernel(unittest.TestCase):
+    def setUp(self):
+        self.single_x = torch.randn(1024)
+        self.single_y = torch.randn(1024)
+
+        self.batch_x = torch.randn(32, 1024)
+        self.batch_y = torch.randn(32, 1024)
+
+    def test_single_values(self):
+        kernel = ExplicitKernel()
+        self.assertEqual(
+            kernel(self.single_x, self.single_y).shape, torch.Size([]))
+
+    def test_single_batch(self):
+        kernel = ExplicitKernel()
+        self.assertEqual(
+            kernel(self.single_x, self.batch_y).shape, torch.Size([32]))
+
+    def test_batch_single(self):
+        kernel = ExplicitKernel()
+        self.assertEqual(
+            kernel(self.batch_x, self.single_y).shape, torch.Size([32]))
+
+    def test_batch_values(self):
+        kernel = ExplicitKernel()
+        self.assertEqual(
+            kernel(self.batch_x, self.batch_y).shape, torch.Size([32, 32]))
+
+
+class TestRadialBasisFunctionKernel(unittest.TestCase):
+    def setUp(self):
+        self.single_x = torch.randn(1024)
+        self.single_y = torch.randn(1024)
+
+        self.batch_x = torch.randn(32, 1024)
+        self.batch_y = torch.randn(32, 1024)
+
+    def test_single_values(self):
+        kernel = RadialBasisFunctionKernel(1)
+        self.assertEqual(
+            kernel(self.single_x, self.single_y).shape, torch.Size([]))
+
+    def test_single_batch(self):
+        kernel = RadialBasisFunctionKernel(1)
+        self.assertEqual(
+            kernel(self.single_x, self.batch_y).shape, torch.Size([32]))
+
+    def test_batch_single(self):
+        kernel = RadialBasisFunctionKernel(1)
+        self.assertEqual(
+            kernel(self.batch_x, self.single_y).shape, torch.Size([32]))
+
+    def test_batch_values(self):
+        kernel = RadialBasisFunctionKernel(1)
+        self.assertEqual(
+            kernel(self.batch_x, self.batch_y).shape, torch.Size([32, 32]))
+
+
+class TestKernelDistance(unittest.TestCase):
+    def setUp(self):
+        self.single_x = torch.randn(1024)
+        self.single_y = torch.randn(1024)
+
+        self.batch_x = torch.randn(32, 1024)
+        self.batch_y = torch.randn(32, 1024)
+
+        self.kernel = ExplicitKernel()
+
+    def test_single_values(self):
+        distance = KernelDistance(self.kernel)
+        self.assertEqual(
+            distance(self.single_x, self.single_y).shape, torch.Size([]))
+
+    def test_single_batch(self):
+        distance = KernelDistance(self.kernel)
+        self.assertEqual(
+            distance(self.single_x, self.batch_y).shape, torch.Size([32]))
+
+    def test_batch_single(self):
+        distance = KernelDistance(self.kernel)
+        self.assertEqual(
+            distance(self.batch_x, self.single_y).shape, torch.Size([32]))
+
+    def test_batch_values(self):
+        distance = KernelDistance(self.kernel)
+        self.assertEqual(
+            distance(self.batch_x, self.batch_y).shape, torch.Size([32, 32]))

tests/test_modules.py

@@ -5,7 +5,7 @@ import unittest
 import numpy as np
 import torch
 
-from prototorch.modules import prototypes, losses
+from prototorch.modules import losses, prototypes
 
 
 class TestPrototypes(unittest.TestCase):
@@ -16,19 +16,25 @@ class TestPrototypes(unittest.TestCase):
         self.y = torch.tensor([0, 0, 1, 1])
         self.gen = torch.manual_seed(42)
 
-    def test_addprototypes1d_init_without_input_dim(self):
+    def test_prototypes1d_init_without_input_dim(self):
         with self.assertRaises(NameError):
-            _ = prototypes.AddPrototypes1D(nclasses=1)
+            _ = prototypes.Prototypes1D(nclasses=2)
 
-    def test_addprototypes1d_init_without_nclasses(self):
+    def test_prototypes1d_init_without_nclasses(self):
         with self.assertRaises(NameError):
-            _ = prototypes.AddPrototypes1D(input_dim=1)
+            _ = prototypes.Prototypes1D(input_dim=1)
 
-    def test_addprototypes1d_init_without_pdist(self):
+    def test_prototypes1d_init_with_nclasses_1(self):
-        p1 = prototypes.AddPrototypes1D(input_dim=6,
+        with self.assertWarns(UserWarning):
-                                        nclasses=2,
+            _ = prototypes.Prototypes1D(nclasses=1, input_dim=1)
-                                        prototypes_per_class=4,
+
-                                        prototype_initializer='ones')
+    def test_prototypes1d_init_without_pdist(self):
+        p1 = prototypes.Prototypes1D(
+            input_dim=6,
+            nclasses=2,
+            prototypes_per_class=4,
+            prototype_initializer="ones",
+        )
         protos = p1.prototypes
         actual = protos.detach().numpy()
         desired = torch.ones(8, 6)
@@ -37,11 +43,11 @@ class TestPrototypes(unittest.TestCase):
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
-    def test_addprototypes1d_init_without_data(self):
+    def test_prototypes1d_init_without_data(self):
         pdist = [2, 2]
-        p1 = prototypes.AddPrototypes1D(input_dim=3,
+        p1 = prototypes.Prototypes1D(input_dim=3,
-                                        prototype_distribution=pdist,
+                                     prototype_distribution=pdist,
-                                        prototype_initializer='zeros')
+                                     prototype_initializer="zeros")
         protos = p1.prototypes
         actual = protos.detach().numpy()
         desired = torch.zeros(4, 3)
@@ -50,23 +56,21 @@ class TestPrototypes(unittest.TestCase):
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
-    # def test_addprototypes1d_init_torch_pdist(self):
+    def test_prototypes1d_proto_init_without_data(self):
-    #     pdist = torch.tensor([2, 2])
+        with self.assertWarns(UserWarning):
-    #     p1 = prototypes.AddPrototypes1D(input_dim=3,
+            _ = prototypes.Prototypes1D(
-    #                                     prototype_distribution=pdist,
+                input_dim=3,
-    #                                     prototype_initializer='zeros')
+                nclasses=2,
-    #     protos = p1.prototypes
+                prototypes_per_class=1,
-    #     actual = protos.detach().numpy()
+                prototype_initializer="stratified_mean",
-    #     desired = torch.zeros(4, 3)
+                data=None,
-    #     mismatch = np.testing.assert_array_almost_equal(actual,
+            )
-    #                                                     desired,
-    #                                                     decimal=5)
-    #     self.assertIsNone(mismatch)
 
-    def test_addprototypes1d_init_with_ppc(self):
+    def test_prototypes1d_init_torch_pdist(self):
-        p1 = prototypes.AddPrototypes1D(data=[self.x, self.y],
+        pdist = torch.tensor([2, 2])
-                                        prototypes_per_class=2,
+        p1 = prototypes.Prototypes1D(input_dim=3,
-                                        prototype_initializer='zeros')
+                                     prototype_distribution=pdist,
+                                     prototype_initializer="zeros")
         protos = p1.prototypes
         actual = protos.detach().numpy()
         desired = torch.zeros(4, 3)
@@ -75,10 +79,133 @@ class TestPrototypes(unittest.TestCase):
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
-    def test_addprototypes1d_init_with_pdist(self):
+    def test_prototypes1d_init_without_inputdim_with_data(self):
-        p1 = prototypes.AddPrototypes1D(data=[self.x, self.y],
+        _ = prototypes.Prototypes1D(
-                                        prototype_distribution=[6, 9],
+            nclasses=2,
-                                        prototype_initializer='zeros')
+            prototypes_per_class=1,
+            prototype_initializer="stratified_mean",
+            data=[[[1.0], [0.0]], [1, 0]],
+        )
+
+    def test_prototypes1d_init_with_int_data(self):
+        _ = prototypes.Prototypes1D(
+            nclasses=2,
+            prototypes_per_class=1,
+            prototype_initializer="stratified_mean",
+            data=[[[1], [0]], [1, 0]],
+        )
+
+    def test_prototypes1d_init_one_hot_without_data(self):
+        _ = prototypes.Prototypes1D(
+            input_dim=1,
+            nclasses=2,
+            prototypes_per_class=1,
+            prototype_initializer="stratified_mean",
+            data=None,
+            one_hot_labels=True,
+        )
+
+    def test_prototypes1d_init_one_hot_labels_false(self):
+        """Test if ValueError is raised when `one_hot_labels` is set to `False`
+        but the provided `data` has one-hot encoded labels.
+        """
+        with self.assertRaises(ValueError):
+            _ = prototypes.Prototypes1D(
+                input_dim=1,
+                nclasses=2,
+                prototypes_per_class=1,
+                prototype_initializer="stratified_mean",
+                data=([[0.0], [1.0]], [[0, 1], [1, 0]]),
+                one_hot_labels=False,
+            )
+
+    def test_prototypes1d_init_1d_y_data_one_hot_labels_true(self):
+        """Test if ValueError is raised when `one_hot_labels` is set to `True`
+        but the provided `data` does not contain one-hot encoded labels.
+        """
+        with self.assertRaises(ValueError):
+            _ = prototypes.Prototypes1D(
+                input_dim=1,
+                nclasses=2,
+                prototypes_per_class=1,
+                prototype_initializer="stratified_mean",
+                data=([[0.0], [1.0]], [0, 1]),
+                one_hot_labels=True,
+            )
+
+    def test_prototypes1d_init_one_hot_labels_true(self):
+        """Test if ValueError is raised when `one_hot_labels` is set to `True`
+        but the provided `data` contains 2D targets but
+        does not contain one-hot encoded labels.
+        """
+        with self.assertRaises(ValueError):
+            _ = prototypes.Prototypes1D(
+                input_dim=1,
+                nclasses=2,
+                prototypes_per_class=1,
+                prototype_initializer="stratified_mean",
+                data=([[0.0], [1.0]], [[0], [1]]),
+                one_hot_labels=True,
+            )
+
+    def test_prototypes1d_init_with_int_dtype(self):
+        with self.assertRaises(RuntimeError):
+            _ = prototypes.Prototypes1D(
+                nclasses=2,
+                prototypes_per_class=1,
+                prototype_initializer="stratified_mean",
+                data=[[[1], [0]], [1, 0]],
+                dtype=torch.int32,
+            )
+
+    def test_prototypes1d_inputndim_with_data(self):
+        with self.assertRaises(ValueError):
+            _ = prototypes.Prototypes1D(input_dim=1,
+                                        nclasses=1,
+                                        prototypes_per_class=1,
+                                        data=[[1.0], [1]])
+
+    def test_prototypes1d_inputdim_with_data(self):
+        with self.assertRaises(ValueError):
+            _ = prototypes.Prototypes1D(
+                input_dim=2,
+                nclasses=2,
+                prototypes_per_class=1,
+                prototype_initializer="stratified_mean",
+                data=[[[1.0], [0.0]], [1, 0]],
+            )
+
+    def test_prototypes1d_nclasses_with_data(self):
+        """Test ValueError raise if provided `nclasses` is not the same
+        as the one computed from the provided `data`.
+        """
+        with self.assertRaises(ValueError):
+            _ = prototypes.Prototypes1D(
+                input_dim=1,
+                nclasses=1,
+                prototypes_per_class=1,
+                prototype_initializer="stratified_mean",
+                data=[[[1.0], [2.0]], [1, 2]],
+            )
+
+    def test_prototypes1d_init_with_ppc(self):
+        p1 = prototypes.Prototypes1D(data=[self.x, self.y],
+                                     prototypes_per_class=2,
+                                     prototype_initializer="zeros")
+        protos = p1.prototypes
+        actual = protos.detach().numpy()
+        desired = torch.zeros(4, 3)
+        mismatch = np.testing.assert_array_almost_equal(actual,
+                                                        desired,
+                                                        decimal=5)
+        self.assertIsNone(mismatch)
+
+    def test_prototypes1d_init_with_pdist(self):
+        p1 = prototypes.Prototypes1D(
+            data=[self.x, self.y],
+            prototype_distribution=[6, 9],
+            prototype_initializer="zeros",
+        )
         protos = p1.prototypes
         actual = protos.detach().numpy()
         desired = torch.zeros(15, 3)
@@ -87,14 +214,16 @@ class TestPrototypes(unittest.TestCase):
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
-    def test_addprototypes1d_func_initializer(self):
+    def test_prototypes1d_func_initializer(self):
         def my_initializer(*args, **kwargs):
-            return torch.full((2, 99), 99), torch.tensor([0, 1])
+            return torch.full((2, 99), 99.0), torch.tensor([0, 1])
 
-        p1 = prototypes.AddPrototypes1D(input_dim=99,
+        p1 = prototypes.Prototypes1D(
-                                        nclasses=2,
+            input_dim=99,
-                                        prototypes_per_class=1,
+            nclasses=2,
-                                        prototype_initializer=my_initializer)
+            prototypes_per_class=1,
+            prototype_initializer=my_initializer,
+        )
         protos = p1.prototypes
         actual = protos.detach().numpy()
         desired = 99 * torch.ones(2, 99)
@@ -103,8 +232,8 @@ class TestPrototypes(unittest.TestCase):
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
-    def test_addprototypes1d_forward(self):
+    def test_prototypes1d_forward(self):
-        p1 = prototypes.AddPrototypes1D(data=[self.x, self.y])
+        p1 = prototypes.Prototypes1D(data=[self.x, self.y])
         protos, _ = p1()
         actual = protos.detach().numpy()
         desired = torch.ones(2, 3)
@@ -113,6 +242,16 @@ class TestPrototypes(unittest.TestCase):
                                                         decimal=5)
         self.assertIsNone(mismatch)
 
+    def test_prototypes1d_dist_validate(self):
+        p1 = prototypes.Prototypes1D(input_dim=0, prototype_distribution=[0])
+        with self.assertWarns(UserWarning):
+            _ = p1._validate_prototype_distribution()
+
+    def test_prototypes1d_validate_extra_repr_not_empty(self):
+        p1 = prototypes.Prototypes1D(input_dim=0, prototype_distribution=[0])
+        rep = p1.extra_repr()
+        self.assertNotEqual(rep, "")
+
     def tearDown(self):
         del self.x, self.y, self.gen
         _ = torch.seed()
@@ -123,11 +262,11 @@ class TestLosses(unittest.TestCase):
         pass
 
     def test_glvqloss_init(self):
-        _ = losses.GLVQLoss(0, 'swish_beta', beta=20)
+        _ = losses.GLVQLoss(0, "swish_beta", beta=20)
 
-    def test_glvqloss_forward(self):
+    def test_glvqloss_forward_1ppc(self):
         criterion = losses.GLVQLoss(margin=0,
-                                    squashing='sigmoid_beta',
+                                    squashing="sigmoid_beta",
                                     beta=100)
         d = torch.stack([torch.ones(100), torch.zeros(100)], dim=1)
         labels = torch.tensor([0, 1])
@@ -137,5 +276,23 @@ class TestLosses(unittest.TestCase):
         loss_value = loss.item()
         self.assertAlmostEqual(loss_value, 0.0)
 
+    def test_glvqloss_forward_2ppc(self):
+        criterion = losses.GLVQLoss(margin=0,
+                                    squashing="sigmoid_beta",
+                                    beta=100)
+        d = torch.stack([
+            torch.ones(100),
+            torch.ones(100),
+            torch.zeros(100),
+            torch.ones(100)
+        ],
+                        dim=1)
+        labels = torch.tensor([0, 0, 1, 1])
+        targets = torch.ones(100)
+        outputs = [d, labels]
+        loss = criterion(outputs, targets)
+        loss_value = loss.item()
+        self.assertAlmostEqual(loss_value, 0.0)
+
     def tearDown(self):
         pass

tox.ini

@@ -1,15 +0,0 @@
-# tox (https://tox.readthedocs.io/) is a tool for running tests
-# in multiple virtualenvs. This configuration file will run the
-# test suite on all supported python versions. To use it, "pip install tox"
-# and then run "tox" from this directory.
-
-[tox]
-envlist = py36,py37,py38
-
-[testenv]
-deps =
-    pytest
-    coverage
-commands =
-    pip install -e .
-    coverage run -m pytest