Merge etna_v1 into inference-v2.1-etna-v1

.github/workflows/docs-on-pr.yml

@@ -16,6 +16,7 @@ jobs:
        - name: Install Poetry
          uses: snok/install-poetry@v1
          with:
+           version: 1.4.0 # TODO: remove after poetry fix
            virtualenvs-create: true
            virtualenvs-in-project: true
        - name: Load cached venv

.github/workflows/docs-unstable.yml

@@ -17,6 +17,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: 1.4.0 # TODO: remove after poetry fix
           virtualenvs-create: true
           virtualenvs-in-project: true
       - name: Load cached venv

.github/workflows/notebooks.yml

@@ -26,6 +26,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: 1.4.0 # TODO: remove after poetry fix
           virtualenvs-create: true
           virtualenvs-in-project: true
       - name: Install dependencies

.github/workflows/test.yml

@@ -21,7 +21,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          pip install poetry
+          pip install poetry==1.4.0 # TODO: remove after poetry fix
           poetry --version
           poetry config virtualenvs.in-project true
           poetry install -E style --no-root
@@ -48,6 +48,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: 1.4.0 # TODO: remove after poetry fix
           virtualenvs-create: true
           virtualenvs-in-project: true
 
@@ -86,6 +87,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: 1.4.0 # TODO: remove after poetry fix
           virtualenvs-create: true
           virtualenvs-in-project: true
 
@@ -123,6 +125,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: 1.4.0 # TODO: remove after poetry fix
           virtualenvs-create: true
           virtualenvs-in-project: true
 
@@ -160,6 +163,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: 1.4.0 # TODO: remove after poetry fix
           virtualenvs-create: true
           virtualenvs-in-project: true
 
@@ -199,6 +203,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
+          version: 1.4.0 # TODO: remove after poetry fix
           virtualenvs-create: true
           virtualenvs-in-project: true
 

CHANGELOG.md

@@ -5,26 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-
 ## Unreleased
 ### Added
-- `RMSE` metric & `rmse` functional metric ([#1051](https://github.com/tinkoff-ai/etna/pull/1051))
-- `MaxDeviation` metric & `max_deviation` functional metric ([#1061](https://github.com/tinkoff-ai/etna/pull/1061))
-- Add saving/loading for transforms, models, pipelines, ensembles; tutorial for saving/loading ([#1068](https://github.com/tinkoff-ai/etna/pull/1068))
--
--
 - Add `refit` parameter into `backtest` ([#1159](https://github.com/tinkoff-ai/etna/pull/1159))
 - Add `stride` parameter into `backtest` ([#1165](https://github.com/tinkoff-ai/etna/pull/1165))
 - Add optional parameter `ts` into `forecast` method of pipelines ([#1071](https://github.com/tinkoff-ai/etna/pull/1071))
 - Add tests on `transform` method of transforms on subset of segments, on new segments, on future with gap ([#1094](https://github.com/tinkoff-ai/etna/pull/1094))
 - Add tests on `inverse_transform` method of transforms on subset of segments, on new segments, on future with gap ([#1127](https://github.com/tinkoff-ai/etna/pull/1127))
+- 
+- 
+- 
+- 
 ### Changed
--
--
 - Add more scenarios into tests for models ([#1082](https://github.com/tinkoff-ai/etna/pull/1082))
-- 
 - Decouple `SeasonalMovingAverageModel` from `PerSegmentModelMixin` ([#1132](https://github.com/tinkoff-ai/etna/pull/1132))
 - Decouple `DeadlineMovingAverageModel` from `PerSegmentModelMixin` ([#1140](https://github.com/tinkoff-ai/etna/pull/1140))
+- 
+- 
+- 
 ### Fixed
 - Fix inference tests on new segments for `DeepARModel` and `TFTModel` ([#1109](https://github.com/tinkoff-ai/etna/pull/1109))
 - Fix alignment during forecasting in new NNs, add validation of context size during forecasting in new NNs, add validation of batch in `MLPNet` ([#1108](https://github.com/tinkoff-ai/etna/pull/1108))
@@ -35,6 +33,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix `OutliersTransform` and its children to raise error on new segments ([#1139](https://github.com/tinkoff-ai/etna/pull/1139))
 - Fix `DifferencingTransform` to raise error on new segments during `transform` and `inverse_transform` in inplace mode ([#1141](https://github.com/tinkoff-ai/etna/pull/1141))
 - Teach `DifferencingTransform` to `inverse_transform` with NaNs ([#1155](https://github.com/tinkoff-ai/etna/pull/1155))
+- 
+- 
+- 
+
+## [1.15.1] - 2023-03-02
+### Changed
+- Impose specific order of columns on return value of `TSDataset.to_flatten` ([#1095](https://github.com/tinkoff-ai/etna/pull/1095))
+### Fixed
+- Fix bug in `GaleShapleyFeatureSelectionTransform` with wrong number of remaining features ([#1110](https://github.com/tinkoff-ai/etna/pull/1110))
+
+## [1.15.0] - 2023-01-31
+### Added
+- `RMSE` metric & `rmse` functional metric ([#1051](https://github.com/tinkoff-ai/etna/pull/1051))
+- `MaxDeviation` metric & `max_deviation` functional metric ([#1061](https://github.com/tinkoff-ai/etna/pull/1061))
+- Add saving/loading for transforms, models, pipelines, ensembles; tutorial for saving/loading ([#1068](https://github.com/tinkoff-ai/etna/pull/1068))
+- Add hierarchical time series support([#1083](https://github.com/tinkoff-ai/etna/pull/1083))
+- Add `WAPE` metric & `wape` functional metric ([#1085](https://github.com/tinkoff-ai/etna/pull/1085))
+### Fixed
+- Missed kwargs in TFT init([#1078](https://github.com/tinkoff-ai/etna/pull/1078))
+
 ## [1.14.0] - 2022-12-16
 ### Added
 - Add python 3.10 support ([#1005](https://github.com/tinkoff-ai/etna/pull/1005))

README.md

@@ -181,6 +181,7 @@ We have also prepared a set of tutorials for an easy introduction:
 | [Exogenous data](https://github.com/tinkoff-ai/etna/tree/master/examples/exogenous_data.ipynb) | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/tinkoff-ai/etna/master?filepath=examples/exogenous_data.ipynb) |
 | [Forecasting strategies](https://github.com/tinkoff-ai/etna/blob/master/examples/forecasting_strategies.ipynb) | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/tinkoff-ai/etna/master?filepath=examples/forecasting_strategies.ipynb) |
 | [Classification](https://github.com/tinkoff-ai/etna/blob/master/examples/classification.ipynb) | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/tinkoff-ai/etna/master?filepath=examples/classification.ipynb) |
+| [Hierarchical time series](https://github.com/tinkoff-ai/etna/blob/master/examples/hierarchical_pipeline.ipynb) | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/tinkoff-ai/etna/master?filepath=examples/hierarchical_pipeline.ipynb) |
 
 ## Documentation
 

docs/source/api.rst

@@ -14,6 +14,7 @@ API
    transforms
    ensembles
    pipeline
+   reconciliation
    analysis
    auto
    clustering

docs/source/index.rst

@@ -13,6 +13,7 @@ Welcome to ETNA's documentation
    transforms
    ensembles
    pipeline
+   reconciliation
    analysis
    auto
    clustering

docs/source/reconciliation.rst

@@ -0,0 +1,20 @@
+Reconciliation
+==============
+
+.. _reconciliation:
+
+.. currentmodule:: etna
+
+Details of ETNA reconciliators
+------------------------------
+
+See the API documentation for further details on reconciliators:
+
+.. currentmodule:: etna
+
+.. moduleautosummary::
+   :toctree: api/
+   :template: custom-module-template.rst
+   :recursive:
+
+   etna.reconciliation

docs/source/tutorials.rst

@@ -16,3 +16,4 @@ Tutorials
    tutorials/ensembles
    tutorials/NN_examples
    tutorials/classification
+   tutorials/hierarchical_pipeline

etna/datasets/__init__.py

@@ -1,7 +1,9 @@
 from etna.datasets.datasets_generation import generate_ar_df
 from etna.datasets.datasets_generation import generate_const_df
 from etna.datasets.datasets_generation import generate_from_patterns_df
+from etna.datasets.datasets_generation import generate_hierarchical_df
 from etna.datasets.datasets_generation import generate_periodic_df
+from etna.datasets.hierarchical_structure import HierarchicalStructure
 from etna.datasets.tsdataset import TSDataset
 from etna.datasets.utils import duplicate_data
 from etna.datasets.utils import set_columns_wide

etna/datasets/datasets_generation.py

@@ -186,3 +186,103 @@ def generate_from_patterns_df(
     df["timestamp"] = pd.date_range(start=start_time, freq=freq, periods=periods)
     df = df.melt(id_vars=["timestamp"], value_name="target", var_name="segment")
     return df
+
+
+def generate_hierarchical_df(
+    periods: int,
+    n_segments: List[int],
+    freq: str = "D",
+    start_time: str = "2000-01-01",
+    ar_coef: Optional[list] = None,
+    sigma: float = 1,
+    random_seed: int = 1,
+) -> pd.DataFrame:
+    """
+    Create DataFrame with hierarchical structure and AR process data.
+
+    The hierarchical structure is generated as follows:
+        1. Number of levels in the structure is the same as length of ``n_segments`` parameter
+        2. Each level contains the number of segments set in ``n_segments``
+        3. Connections from parent to child level are generated randomly.
+
+    Parameters
+    ----------
+    periods:
+        number of timestamps
+    n_segments:
+        number of segments on each level.
+    freq:
+        pandas frequency string for :py:func:`pandas.date_range` that is used to generate timestamp
+    start_time:
+        start timestamp
+    ar_coef:
+        AR coefficients
+    sigma:
+        scale of AR noise
+    random_seed:
+        random seed
+
+    Returns
+    -------
+    :
+        DataFrame at the bottom level of the hierarchy
+
+    Raises
+    ------
+    ValueError:
+        ``n_segments`` is empty
+    ValueError:
+        ``n_segments`` contains not positive integers
+    ValueError:
+        ``n_segments`` represents not non-decreasing sequence
+    """
+    if len(n_segments) == 0:
+        raise ValueError("`n_segments` should contain at least one positive integer!")
+
+    if (np.less_equal(n_segments, 0)).any():
+        raise ValueError("All `n_segments` elements should be positive!")
+
+    if (np.diff(n_segments) < 0).any():
+        raise ValueError("`n_segments` should represent non-decreasing sequence!")
+
+    rnd = RandomState(seed=random_seed)
+
+    bottom_df = generate_ar_df(
+        periods=periods,
+        start_time=start_time,
+        ar_coef=ar_coef,
+        sigma=sigma,
+        n_segments=n_segments[-1],
+        freq=freq,
+        random_seed=random_seed,
+    )
+
+    bottom_segments = np.unique(bottom_df["segment"])
+
+    n_levels = len(n_segments)
+    child_to_parent = dict()
+    for level_id in range(1, n_levels):
+        prev_level_n_segments = n_segments[level_id - 1]
+        cur_level_n_segments = n_segments[level_id]
+
+        # ensure all parents have at least one child
+        seen_ids = set()
+        child_ids = rnd.choice(cur_level_n_segments, prev_level_n_segments, replace=False)
+        for parent_id, child_id in enumerate(child_ids):
+            seen_ids.add(child_id)
+            child_to_parent[f"l{level_id}s{child_id}"] = f"l{level_id - 1}s{parent_id}"
+
+        for child_id in range(cur_level_n_segments):
+            if child_id not in seen_ids:
+                parent_id = rnd.choice(prev_level_n_segments, 1).item()
+                child_to_parent[f"l{level_id}s{child_id}"] = f"l{level_id - 1}s{parent_id}"
+
+    bottom_segments_map = {segment: f"l{n_levels - 1}s{idx}" for idx, segment in enumerate(bottom_segments)}
+    bottom_df[f"level_{n_levels - 1}"] = bottom_df["segment"].map(lambda x: bottom_segments_map[x])
+
+    for level_id in range(n_levels - 2, -1, -1):
+        bottom_df[f"level_{level_id}"] = bottom_df[f"level_{level_id + 1}"].map(lambda x: child_to_parent[x])
+
+    bottom_df.drop(columns=["segment"], inplace=True)
+
+    return bottom_df