Add TeLU activation function

Signed-off-by: Emanuele Ballarin <[email protected]>

ebtorch/__init__.py

@@ -100,6 +100,7 @@
 del SolvePoissonTensor
 del StatefulTupleSelect
 del SwiGLU
+del TeLU
 del TupleDecouple
 del TupleSelect
 del ViTStem

ebtorch/nn/__init__.py

@@ -89,6 +89,7 @@
 from .serlu import SERLU
 from .sinlu import SinLU
 from .smelu import SmeLU
+from .telu import TeLU
 from .utils import *
 
 # Deletions (from .)
@@ -109,10 +110,12 @@
 del serlu
 del sinlu
 del smelu
+del telu
 
 # Deletions (from .functional)
 # del mish  # (already done by chance!)
 # del serf  # (already done by chance!)
+# del telu # (already done by chance!)
 
 # Deletions (from .utils)
 del AdverApply

ebtorch/nn/architectures_resnets_dm.py

@@ -389,7 +389,8 @@ def __init__(
             nn.AdaptiveAvgPool2d((1, 1)) if autopool else nn.AvgPool2d(4)
         )
 
-    def _make_layer(  # Do not make static.
+    # noinspection PyMethodMayBeStatic
+    def _make_layer(
         self,
         in_planes: int,
         out_planes: int,

ebtorch/nn/functional/__init__.py

@@ -22,6 +22,7 @@
 from .inner_functional import serlu
 from .inner_functional import silhouette_score
 from .inner_functional import smelu
+from .inner_functional import telu
 from .inner_functional import tensor_replicate
 
 # Deletions (from .)

ebtorch/nn/functional/inner_functional.py

@@ -40,17 +40,18 @@
 from torch import Tensor
 
 __all__ = [
+    "bisided_thresholding",
+    "cummatmul",
     "field_transform",
+    "logit_to_prob",
     "mish",
-    "serlu",
-    "smelu",
-    "serf",
     "oldtranspose",
+    "serf",
+    "serlu",
     "silhouette_score",
-    "cummatmul",
+    "smelu",
+    "telu",
     "tensor_replicate",
-    "logit_to_prob",
-    "bisided_thresholding",
 ]
 
 
@@ -122,6 +123,12 @@ def serf(x: Tensor) -> Tensor:
     return torch.erf(x / math.sqrt(2.0))  # type: ignore
 
 
+@torch.jit.script
+def telu(x: Tensor) -> Tensor:
+    """Applies the TeLU function, element-wise."""
+    return x * torch.tanh(torch.exp(x))
+
+
 def oldtranspose(x: Tensor) -> Tensor:
     """
     Transpose a tensor along all dimensions, emulating x.T.

ebtorch/nn/reshapelayers.py

@@ -40,5 +40,6 @@ class FlatChannelize2DLayer(nn.Module):
     def __init__(self):
         super().__init__()
 
-    def forward(self, x):  # Do not make static!
+    # noinspection PyMethodMayBeStatic
+    def forward(self, x):
         return x.reshape(*x.shape, 1, 1)

ebtorch/nn/serf.py

@@ -34,5 +34,6 @@ class ScaledERF(torch.nn.Module):
     def __init__(self) -> None:
         super(ScaledERF, self).__init__()
 
-    def forward(self, x: Tensor) -> Tensor:  # Do not make static!
+    # noinspection PyMethodMayBeStatic
+    def forward(self, x: Tensor) -> Tensor:
         return fserf(x)

ebtorch/nn/telu.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+from typing import List
+
+from torch import nn
+from torch import Tensor
+
+from .functional import telu as ftelu
+
+__all__: List[str] = ["TeLU"]
+
+
+class TeLU(nn.Module):
+    """TeLU Function."""
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    # noinspection PyMethodMayBeStatic
+    def forward(self, x: Tensor) -> Tensor:
+        return ftelu(x)

setup.py

@@ -26,7 +26,7 @@ def read(fname):
 
 setup(
     name=PACKAGENAME,
-    version="0.28.9",
+    version="0.28.10",
     author="Emanuele Ballarin",
     author_email="[email protected]",
     url="https://github.com/emaballarin/ebtorch",