From c0463d489237686919e5594772f73b02e3a444b6 Mon Sep 17 00:00:00 2001 From: Lorenzo Stella Date: Tue, 6 Sep 2022 10:07:43 +0200 Subject: [PATCH] Backports for v0.10.6 (#2273) * Use brand colors in docs. (#2257) * Docs: Reformatting table, badge colors. (#2258) * Improve len() for ParquetFile. (#2261) * Move max_idle_transform usage to GluonEstimator. (#2262) * Fix ignore hidden files when generating datasets (#2263) * Fix: set max idle transforms in PyTorch estimators (#2266) * Fix `QuantileForecast.plot()` to use `DateTimeIndex` (#2269) * Docs: update contribution guidelines and dev setup (#2270) * Update project_urls. (#2274) Co-authored-by: Jasper Co-authored-by: Kashif Rasul Co-authored-by: Abdul Fatir --- .github/ISSUE_TEMPLATE/config.yml | 2 +- .../workflows/test_release_unix_nightly.yml | 2 +- .../workflows/test_release_win32_nightly.yml | 4 +- CONTRIBUTING.md | 67 ++---- README.md | 14 +- docs/community/contribute.rst | 190 +----------------- docs/community/{devsetup.rst => devsetup.md} | 103 +++++----- docs/conf.py | 35 ++-- docs/getting_started/install.md | 2 +- docs/getting_started/models.md | 102 +++++----- examples/GluonTS_SageMaker_SDK_Tutorial.ipynb | 4 +- examples/dockerfiles/README.md | 2 +- setup.py | 6 +- src/gluonts/dataset/arrow/file.py | 7 +- .../dataset/repository/_gp_copula_2019.py | 6 +- src/gluonts/meta/__init__.py | 12 ++ src/gluonts/meta/colors.py | 21 ++ src/gluonts/model/canonical/_estimator.py | 8 +- src/gluonts/model/deep_factor/_estimator.py | 8 +- src/gluonts/model/deepar/_estimator.py | 8 +- src/gluonts/model/deepstate/_estimator.py | 8 +- src/gluonts/model/deepvar/_estimator.py | 8 +- src/gluonts/model/forecast.py | 5 +- src/gluonts/model/gp_forecaster/_estimator.py | 8 +- src/gluonts/model/gpvar/_estimator.py | 8 +- src/gluonts/model/lstnet/_estimator.py | 8 +- src/gluonts/model/n_beats/_estimator.py | 8 +- src/gluonts/model/renewal/_estimator.py | 24 +-- src/gluonts/model/san/_estimator.py | 8 +- .../model/seq2seq/_forking_estimator.py | 8 +- .../model/seq2seq/_seq2seq_estimator.py | 8 +- .../model/simple_feedforward/_estimator.py | 8 +- src/gluonts/model/tft/_estimator.py | 8 +- src/gluonts/model/tpp/deeptpp/_estimator.py | 8 +- src/gluonts/model/transformer/_estimator.py | 8 +- src/gluonts/model/wavenet/_estimator.py | 8 +- src/gluonts/mx/batchify.py | 2 +- src/gluonts/mx/model/estimator.py | 37 ++-- .../0. README.md | 4 +- .../nursery/sagemaker_sdk/estimator.py | 2 +- src/gluonts/nursery/tsbench/README.md | 4 +- src/gluonts/nursery/tsbench/pyproject.toml | 2 +- src/gluonts/torch/model/estimator.py | 53 ++--- src/gluonts/transform/_base.py | 9 +- test/model/test_forecast.py | 2 + test/mx/test_mx_rolling.py | 2 +- 46 files changed, 311 insertions(+), 550 deletions(-) rename docs/community/{devsetup.rst => devsetup.md} (54%) create mode 100644 src/gluonts/meta/__init__.py create mode 100644 src/gluonts/meta/colors.py diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4ebe8b7105..f2581d9ea3 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: Questions and Discussions - url: https://github.com/awslabs/gluon-ts/discussions + url: https://github.com/awslabs/gluonts/discussions about: Use GitHub Discussions to ask and answer questions, exchange ideas, and share learning. diff --git a/.github/workflows/test_release_unix_nightly.yml b/.github/workflows/test_release_unix_nightly.yml index 42eed21a0d..aead88c93d 100644 --- a/.github/workflows/test_release_unix_nightly.yml +++ b/.github/workflows/test_release_unix_nightly.yml @@ -21,7 +21,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Clone and install dependencies run: | - git clone https://github.com/awslabs/gluon-ts --branch $(curl https://api.github.com/repos/awslabs/gluon-ts/releases/latest | grep tag_name | cut -d : -f 2,3 | tr -d \"\ | tr -d \,\ ) + git clone https://github.com/awslabs/gluonts --branch $(curl https://api.github.com/repos/awslabs/gluonts/releases/latest | grep tag_name | cut -d : -f 2,3 | tr -d \"\ | tr -d \,\ ) cd gluon-ts python -m pip install pip==20.2 pip install mxnet~=1.8.0 diff --git a/.github/workflows/test_release_win32_nightly.yml b/.github/workflows/test_release_win32_nightly.yml index efcadc57f5..fa2048382d 100644 --- a/.github/workflows/test_release_win32_nightly.yml +++ b/.github/workflows/test_release_win32_nightly.yml @@ -21,9 +21,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Clone and install dependencies run: | - $tmp=(Invoke-WebRequest -Uri https://api.github.com/repos/awslabs/gluon-ts/releases/latest).Content | ConvertFrom-Json | Select-Object tag_name + $tmp=(Invoke-WebRequest -Uri https://api.github.com/repos/awslabs/gluonts/releases/latest).Content | ConvertFrom-Json | Select-Object tag_name $tmp=$tmp.psobject.properties.value.trim() - git clone https://github.com/awslabs/gluon-ts --branch $tmp + git clone https://github.com/awslabs/gluonts --branch $tmp cd gluon-ts python -m pip install -U pip pip install mxnet~=1.7.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37c1c0f4fd..ad8b2ccc22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,17 +6,15 @@ documentation, we greatly value feedback and contributions from our community. Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. - ## Questions and discussion topics -Questions and discussion topics can be proposed using [discussions](https://github.com/awslabs/gluon-ts/discussions). - +Questions and discussion topics can be proposed using [discussions](https://github.com/awslabs/gluonts/discussions). ## Reporting Bugs/Feature Requests -We welcome you to use the GitHub [issue tracker](https://github.com/awslabs/gluon-ts/issues/new/choose) to report bugs or suggest features. +We welcome you to use the GitHub [issue tracker](https://github.com/awslabs/gluonts/issues/new/choose) to report bugs or suggest features. -When filing an issue, please check [existing open](https://github.com/awslabs/gluon-ts/issues), or [recently closed](https://github.com/awslabs/gluon-ts/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already +When filing an issue, please check [existing open](https://github.com/awslabs/gluonts/issues), or [recently closed](https://github.com/awslabs/gluonts/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: * A reproducible test case or series of steps @@ -24,11 +22,11 @@ reported the issue. Please try to include as much information as you can. Detail * Any modifications you've made relevant to the bug * Anything unusual about your environment or deployment - ## Contributing via Pull Requests + Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: -1. You are working against the latest source on the *dev* branch. +1. You are working against the latest source on the `dev` branch. 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. @@ -44,64 +42,29 @@ To send us a pull request, please: GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). +## Development setup -## Development setup and coding style - -For development you need the requirements listed in all `requirements...txt` files. - -We use [black](https://github.com/python/black) and mypy for style and -type checking. Any code changes that you contribute should pass these checks. - -The easiest way to get set up for development is to run the following script: - -```bash -# first fork the repo on github - -# then clone your fork -git clone https://github.com//gluon-ts.git - -# enter the cloned repo -cd gluon-ts - -# run development setup -./dev_setup.sh - -# install GluonTS from current source with shell dependencies -pip install -e ".[shell]" -``` -Note that windows and other versions of linux (not mac versions) will require -something slightly different to initiate the `dev_setup.sh` script. - -This will install all the requirements and also install a git hook that runs -all code checks when you commit. This also separates your workflow so that -pull-requests go through your forked repo and then into upstream if approved. - -You may wish to familiarize yourself with forked repositories on github, for -more see: -https://help.github.com/en/articles/working-with-forks - -To avoid conflicts with existing python installations you may want to setup -a virtual environment. For more on virtual environments in python see: -https://docs.python.org/3/tutorial/venv.html -for python virtual environments and -https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html -for conda virtual environments. +Please refer to the [documentation](https://ts.gluon.ai/dev/community/devsetup.html) on how to set up your development environment. ## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/gluon-ts/labels/help%20wanted) issues is a great place to start. + +Looking at the existing issues is a great way to find something to contribute on: issues labeled with +['good first issue'](https://github.com/awslabs/gluonts/labels/good%20first%20issue) or +['help wanted'](https://github.com/awslabs/gluonts/labels/help%20wanted) +are a great place to start. ## Code of Conduct + This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact opensource-codeofconduct@amazon.com with any additional questions or comments. - ## Security issue notifications -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. ## Licensing -See the [LICENSE](https://github.com/awslabs/gluon-ts/blob/dev/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. +See the [LICENSE](https://github.com/awslabs/gluonts/blob/dev/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. diff --git a/README.md b/README.md index 1debb7fa77..1592a855a2 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ # GluonTS - Probabilistic Time Series Modeling in Python -[![PyPI](https://img.shields.io/pypi/v/gluonts.svg?style=flat-square)](https://pypi.org/project/gluonts/) -[![GitHub](https://img.shields.io/github/license/awslabs/gluon-ts.svg?style=flat-square)](./LICENSE) -[![Static](https://img.shields.io/static/v1?label=docs&message=stable&color=blue&style=flat-square)](https://ts.gluon.ai/) -[![Static](https://img.shields.io/static/v1?label=docs&message=dev&color=blue&style=flat-square)](https://ts.gluon.ai/dev/) -[![PyPI Downloads](https://pepy.tech/badge/gluonts)](https://pypi.org/project/gluonts/) +[![PyPI](https://img.shields.io/pypi/v/gluonts.svg?style=flat-square&color=b75347)](https://pypi.org/project/gluonts/) +[![GitHub](https://img.shields.io/github/license/awslabs/gluonts.svg?style=flat-square&color=df7e66)](./LICENSE) +[![Static](https://img.shields.io/static/v1?label=docs&message=stable&color=edc775&style=flat-square)](https://ts.gluon.ai/) +[![Static](https://img.shields.io/static/v1?label=docs&message=dev&color=edc775&style=flat-square)](https://ts.gluon.ai/dev/) +[![PyPI Downloads](https://img.shields.io/pypi/dm/gluonts?style=flat-square&color=94b594)](https://pypi.org/project/gluonts/) GluonTS is a Python package for probabilistic time series modeling, focusing on deep learning based models. @@ -69,7 +69,7 @@ centered around the median. ## Contributing If you wish to contribute to the project, please refer to our -[contribution guidelines](https://github.com/awslabs/gluon-ts/tree/dev/CONTRIBUTING.md). +[contribution guidelines](https://github.com/awslabs/gluonts/tree/dev/CONTRIBUTING.md). ## Citing @@ -115,7 +115,7 @@ in addition to any model-specific references that are relevant for your work: * [JMLR MLOSS Paper](http://www.jmlr.org/papers/v21/19-820.html) * [ArXiv Paper](https://arxiv.org/abs/1906.05264) -* [Collected Papers from the group behind GluonTS](https://github.com/awslabs/gluon-ts/tree/dev/REFERENCES.md): a bibliography. +* [Collected Papers from the group behind GluonTS](https://github.com/awslabs/gluonts/tree/dev/REFERENCES.md): a bibliography. ### Tutorials and Workshops diff --git a/docs/community/contribute.rst b/docs/community/contribute.rst index d7045d81a3..f2c987c718 100644 --- a/docs/community/contribute.rst +++ b/docs/community/contribute.rst @@ -1,189 +1 @@ -Contribute -========== - -GluonTS community welcomes contributions from anyone! Latest documentation can be found `here `__. - -There are lots of opportunities for you to become our `contributors `__: - -- Ask or answer questions on `GitHub issues `__. -- Propose ideas, or review proposed design ideas on `GitHub issues `__. -- Improve the `documentation `__. -- Contribute bug reports `GitHub issues `__. -- Write new `tutorials `__. -- Most importantly, if you have an idea of how to contribute, then do it! - -For a list of open starter tasks, check `good first issues `__. - -- `Make changes <#make-changes>`__ - -- `Contribute tutorials <#contribute-tutorials>`__ - -- `Contribute new Models <#contribute-new-model>`__ - -- `Git Workflow Howtos <#git-workflow-howtos>`__ - - - `How to submit pull request <#how-to-submit-pull-request>`__ - - `How to resolve conflict with - dev <#how-to-resolve-conflict-with-dev>`__ - - `How to combine multiple commits into - one <#how-to-combine-multiple-commits-into-one>`__ - - `What is the consequence of force - push <#what-is-the-consequence-of-force-push>`__ - - -Make changes ------------- - -Our package uses continuous integration and code coverage tools for verifying pull requests. Before -submitting, contributor should perform the following checks: - -- `Lint (code style) check `__. -- `Py3 `__ tests. - - -Contribute tutorials --------------------- - -Our :doc:`tutorials <../tutorials/index>` are intended for people who -are interested in time series and want to get better familiarized on different parts in time series. In order for -people to easily understand the content, the code needs to be clean and readable, accompanied by -explanation with good writing. - -See `existing tutorials `__. - -To make the review process easy, we adopt `notedown `_ as the -tutorial format. Notedown notebooks are regular markdown files with code blocks that can be -converted into `Jupyter notebooks `_. - -We suggest you start the example with `Jupyter notebook `_. When the content is ready, please: - -- Clear the output cells in the jupyter notebook, -- `Install notedown `_. -- Run `notedown input.ipynb --to markdown > output.md` -- Submit the `.md` file for review. - -Notebook Guidelines: - -- Less is better. Only show the code that needs people's attention. -- Have a block upfront about the key takeaway of the notebook. -- Explain the motivation of the notebook to guide readers. Add figures if they help. -- Try to have < 10 lines of code per block, < 100 lines of code per notebook. -- Hide uninteresting complex functions in .py and import them. -- Hide uninteresting model parameters. We can make some of them default parameters in model definition. Maybe out of 30 we just show 5 interesting ones and pass those to model constructor. -- Only import module instead of classes and functions (i.e. from gluonts import model and use model.get_model, instead of from gluonts.model import get_model) -- Make tutorials more engaging, interactive, prepare practice questions for people to try it out. For example, for embedding evaluation, we can ask questions to the audience like what's the most similar word to xxx. -- Make sure the notebook can be zoomed in and still render well. This helps accommodate different viewing devices. -- For low level APIs such as BeamSearch and Scorer, explain the API with examples so ppl know how to play with it / hack it. - - -Contribute Docs ---------------- - -Documentation is at least as important as code. Good documentation delivers the correct message clearly and concisely. -If you see any issue in the existing documentation, a patch to fix is most welcome! To locate the -code responsible for the doc, you may use "Edit on Github" in the top right corner, or the -"[source]" links after each API. Also, `git grep` works nicely for searching for a specific string. - -Git Workflow Howtos -------------------- - -How to submit pull request -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- Before submit, please rebase your code on the most recent version of - dev, you can do it by - -.. code:: bash - - git remote add upstream https://github.com/awslabs/gluon-ts - git fetch upstream - git rebase upstream/dev - -- If you have multiple small commits, it might be good to merge them - together(use git rebase then squash) into more meaningful groups. -- Send the pull request! - - - Fix the problems reported by automatic checks - - If you are contributing a new module or new function, add a test. - -How to resolve conflict with dev -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- First rebase to most recent dev - -.. code:: bash - - # The first two steps can be skipped after you do it once. - git remote add upstream https://github.com/awslabs/gluon-ts - git fetch upstream - git rebase upstream/dev - -- The git may show some conflicts it cannot merge, say - ``conflicted.py``. - - - Manually modify the file to resolve the conflict. - - After you resolved the conflict, mark it as resolved by - - .. code:: bash - - git add conflicted.py - -- Then you can continue rebase by - -.. code:: bash - - git rebase --continue - -- Finally push to your fork, you may need to force push here. - -.. code:: bash - - git push --force - -How to combine multiple commits into one -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Sometimes we want to combine multiple commits, especially when later -commits are only fixes to previous ones, to create a PR with set of -meaningful commits. You can do it by following steps. - Before doing so, -configure the default editor of git if you haven't done so before. - -.. code:: bash - - git config core.editor the-editor-you-like - -- Assume we want to merge last 3 commits, type the following commands - -.. code:: bash - - git rebase -i HEAD~3 - -- It will pop up an text editor. Set the first commit as ``pick``, and - change later ones to ``squash``. -- After you saved the file, it will pop up another text editor to ask - you modify the combined commit message. -- Push the changes to your fork, you need to force push. - -.. code:: bash - - git push --force - -Reset to the most recent dev -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You can always use git reset to reset your version to the most recent -dev. Note that all your ***local changes will get lost***. So only do -it when you do not have local changes or when your pull request just get -merged. - -.. code:: bash - - git reset --hard [hash tag of dev] - git push --force - -What is the consequence of force push -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The previous two tips requires force push, this is because we altered -the path of the commits. It is fine to force push to your own fork, as -long as the commits changed are only yours. +.. mdinclude:: ../../CONTRIBUTING.md diff --git a/docs/community/devsetup.rst b/docs/community/devsetup.md similarity index 54% rename from docs/community/devsetup.rst rename to docs/community/devsetup.md index 8ee8e7a364..e0b54f98be 100644 --- a/docs/community/devsetup.rst +++ b/docs/community/devsetup.md @@ -1,48 +1,51 @@ -Development Setup -================= +# Development Setup -.. highlight:: bash +This setup guide assumes that the ``python`` command references a Python 3.7 version or higher. +We recommend using [pyenv][pyenv] for managing Python versions. -GluonTS requires Python 3.7 or higher to run. This setup guide assumes that the -``python`` command references a Python 3.7 version or higher. We recommend -using pyenv_ for managing Python versions. +Upon checking out this package, please run the following: -Upon checking out this package, please run the following:: +```bash +./dev_setup.sh +pip install -e .[dev] - ./dev_setup.sh - pip install -e .[dev] - - # if you use zsh you might need to escape `[` and `]` - pip install -e ".[dev]" +# if you use zsh you might need to escape `[` and `]` +pip install -e ".[dev]" +``` This will install all required packages with pip and setup a Git hook that does automated type and style checks when you try to create a new Git commit. -When you create commits on a |WIP| branch, you can disable these checks for a +When you create commits on a branch, you can disable these checks for a with the ``--no-verify`` Git commit option. -.. _pyenv: https://github.com/pyenv/pyenv +[pyenv]: https://github.com/pyenv/pyenv -Build Instructions ------------------- +## Build Instructions -To run the project tests:: +To run the project tests: - pytest - # or - python setup.py tests +```bash +pytest +# or +python setup.py tests +``` -To build the project documentation:: +To build the project documentation: - python setup.py docs +```bash +python setup.py docs +``` This will put the documentation in ``docs/_build/html``, where you can inspect it by opening ``index.html``. -You can also run the code quality checks manually using ``setup.py``:: +You can also run the code quality checks manually using ``setup.py``: - python setup.py type_check # for Mypy type checks - python setup.py style_check # for Black code style checks +```bash +python setup.py type_check # for Mypy type checks +python setup.py style_check # for Black code style checks +``` Note that the above commands are executed automatically as part of the ``build`` command. Developers that want to merge their code changes in the @@ -52,14 +55,13 @@ using ``./dev_setup.sh`` and are not relying on the ``--no-verify`` option, this should already be asserted when you create your commits. -Writing Type-Safe Code ----------------------- +## Writing Type-Safe Code -The codebase makes extensive use of `type hints`_. The benefits of using types +The codebase makes extensive use of [type hints][type hints]. The benefits of using types are twofold. On the one hand, typing the arguments and the return type of methods and functions provides additional layer of meta-information and improves the readability of the code. On the other, with the help of an -external type-checker such as `mypy`_ we can statically catch a number of +external type-checker such as [mypy][mypy] we can statically catch a number of corner cases that can possibly cause bugs in production. As explained above, in order to implement the latter, the ``type_check`` @@ -79,42 +81,41 @@ to ensure that code added to the repository is type-safe and type-checked. marked with return type ``None``. If you adhere to the above guidelines, you should be able to run -``python setup.py style_check`` and catch type errors early directly on your -|WIP| branch. +``python setup.py style_check`` and catch type errors early directly on your branch. -.. _type hints: https://docs.python.org/3.7/library/typing.html -.. _mypy: https://mypy.readthedocs.io/en/latest/ +[type hints]: https://docs.python.org/3.7/library/typing.html +[mypy]: https://mypy.readthedocs.io/en/latest/ -Editing the documentation -------------------------- +## Editing the documentation -GluonTS documentation follows the `NumPy docstring format`_. +GluonTS documentation follows the [NumPy docstring format][numpy_doc]. If you are editing docstrings in source code, you can preview them with the -following commands:: +following commands: - make -C docs html # generate the docs - open docs/_build/html/index.html # open the generated docs in a browser +```bash +make -C docs html # generate the docs +open docs/_build/html/index.html # open the generated docs in a browser +``` Ensure that there are no syntax errors and warnings before committing a PR. If you are directly editing ``*.rst`` files within the ``docs`` folder, you can use a ``sphinx-autobuild`` autobuild session that starts a web server and a watchdog that automatically rebuilds the documentation when you change an -``*.rst`` file:: +``*.rst`` file: - cd docs # go to the docs folder - make livehtml # run the autobuild watchdog, ensure that - # there are no syntax errors and warnings - open http://127.0.0.1:8000 # open the autobuild preview +```bash +cd docs # go to the docs folder +make livehtml # run the autobuild watchdog, ensure that + # there are no syntax errors and warnings +open http://127.0.0.1:8000 # open the autobuild preview +``` Here are some useful links summarizing the Sphinx syntax: -- `numpydoc docstring guide `_ -- `rst cheat sheet `_ -- `rst basics `_ - -.. _NumPy docstring format: https://numpydoc.readthedocs.io/en/latest/format.html -.. |WIP| raw:: html +- [numpydoc docstring guide](https://numpydoc.readthedocs.io/en/latest/format.html) +- [rst cheat sheet](https://github.com/ralsina/rst-cheatsheet/blob/master/rst-cheatsheet.rst) +- [rst basics](http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) - WIP +[numpy_doc]: https://numpydoc.readthedocs.io/en/latest/format.html diff --git a/docs/conf.py b/docs/conf.py index 17ff72a84d..abc996fab2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,6 +15,7 @@ sys.path.insert(0, os.path.abspath(".")) +from gluonts.meta import colors # -- Project information ----------------------------------------------------- @@ -65,22 +66,26 @@ html_theme = "furo" +html_theme_options = { + "light_css_variables": { + "color-brand-primary": colors.RED, + "color-brand-content": colors.RED, + "color-announcement-text": colors.RED, + "color-announcement-background": "var(--color-background-secondary)", + }, + "dark_css_variables": { + "color-brand-primary": colors.GREEN, + "color-brand-content": colors.GREEN, + "color-announcement-text": colors.GREEN, + "color-announcement-background": "var(--color-background-secondary)", + }, +} + if os.environ.get("GITHUB_REF_NAME") == "dev": - html_theme_options = { - "announcement": "Warning: You are looking at the development docs.", - "light_css_variables": { - "color-announcement-background": "var(--color-background-secondary)", - "color-announcement-text": "#db6a00", - "color-brand-primary": "#ff6f00", - "color-brand-content": "#ff6f00", - }, - "dark_css_variables": { - "color-announcement-background": "var(--color-background-secondary)", - "color-announcement-text": "#db6a00", - "color-brand-primary": "#ff6f00", - "color-brand-content": "#ff6f00", - }, - } + html_theme_options[ + "announcement" + ] = "Note: You are looking at the development docs." + # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/getting_started/install.md b/docs/getting_started/install.md index a51c4532e0..4700e65ca9 100644 --- a/docs/getting_started/install.md +++ b/docs/getting_started/install.md @@ -13,7 +13,7 @@ GluonTS uses [Semantic Versioning](https://semver.org) for managing versions. Since the library is actively developed we use `v0` as the major version. We plan to release a new minor version at the end of each quarter. The current -planned releases can be found on [GitHub](https://github.com/awslabs/gluon-ts/milestones). +planned releases can be found on [GitHub](https://github.com/awslabs/gluonts/milestones). **Version Guarantees** diff --git a/docs/getting_started/models.md b/docs/getting_started/models.md index 0cff3b926e..4ee350f666 100644 --- a/docs/getting_started/models.md +++ b/docs/getting_started/models.md @@ -1,31 +1,31 @@ ## Available models -Model | Local/global | Data layout | Architecture/method | Implementation -------------------------------------------------------------|--------------|--------------------------|---------------------|---------------- -DeepAR [[Salinas et al. 2020][Salinas2020]] | Global | Univariate | RNN | [MXNet][DeepAR_mx], [PyTorch][DeepAR_torch] -DeepState [[Rangapuram et al. 2018][Rangapuram2018]] | Global | Univariate | RNN, state-space model | [MXNet][DeepState] -DeepFactor [[Wang et al. 2019][Wang2019]] | Global | Univariate | RNN, state-space model, Gaussian process | [MXNet][DeepFactor] -Deep Renewal Processes [[Türkmen et al. 2021][Turkmen2021]] | Global | Univariate | RNN | [MXNet][DeepRenewal] -GPForecaster | Global | Univariate | MLP, Gaussian process | [MXNet][GP] -MQ-CNN [[Wen et al. 2017][Wen2017]] | Global | Univariate | CNN encoder, MLP decoder | [MXNet][MQDNN] -MQ-RNN [[Wen et al. 2017][Wen2017]] | Global | Univariate | RNN encoder, MLP encoder | [MXNet][MQDNN] -N-BEATS [[Oreshkin et al. 2019][Oreshkin2019]] | Global | Univariate | MLP, residual links | [MXNet][NBeats] -Rotbaum [[Hasson et al. 2021][Hasson2021]] | Global | Univariate | XGBoost, Quantile Regression Forests, LightGBM, Level Set Forecaster | [Numpy][Rotbaum] -Causal Convolutional Transformer [[Li et al. 2019][Li2019]] | Global | Univariate | Causal convolution, self attention | [MXNet][SAN] -Temporal Fusion Transformer [[Lim et al. 2021][Lim2021]] | Global | Univariate | LSTM, self attention | [MXNet][TFT] -Transformer [[Vaswani et al. 2017][Vaswani2017]] | Global | Univariate | MLP, multi-head attention | [MXNet][Transformer] -WaveNet [[van den Oord et al. 2016][vanDenOord2016]] | Global | Univariate | Dilated convolution | [MXNet][WaveNet] -SimpleFeedForward | Global | Univariate | MLP | [MXNet][SFF_mx], [PyTorch][SFF_torch] -DeepVAR [[Salinas et al. 2019][Salinas2019]] | Global | Multivariate | RNN | [MXNet][DeepVAR] -GPVAR [[Salinas et al. 2019][Salinas2019]] | Global | Multivariate | RNN, Gaussian process | [MXNet][GPVAR] -LSTNet [[Lai et al. 2018][Lai2018]] | Global | Multivariate | LSTM | [MXNet][LSTNet] -DeepTPP [[Shchur et al. 2020][Shchur2020]] | Global | Multivariate events | RNN, temporal point process | [MXNet][DeepTPP] -DeepVARHierarchical [[Rangapuram et al. 2021][Rangapuram2021]] | Global | Hierarchical | RNN | [MXNet][DeepVARHierarchical] -RForecast [[Hyndman et al. 2008][Hyndman2008]] | Local | Univariate | ARIMA, ETS, Croston, TBATS | [Wrapped R package][RForecast] -Prophet [[Taylor et al. 2017][Taylor2017]] | Local | Univariate | - | [Wrapped Python package][Prophet] -NaiveSeasonal [[Hyndman et al. 2018][Hyndman2018]] | Local | Univariate | - | [Numpy][NaiveSeasonal] -Naive2 [[Makridakis et al. 1998][Makridakis1998]] | Local | Univariate | - | [Numpy][Naive2] -NPTS | Local | Univariate | - | [Numpy][NPTS] +Model + Paper | Local/global | Data layout | Architecture/method | Implementation +-------------------------------------------------------------|--------------|--------------------------|---------------------|---------------- +DeepAR
[Salinas et al. 2020][Salinas2020] | Global | Univariate | RNN | [MXNet][DeepAR_mx], [PyTorch][DeepAR_torch] +DeepState
[Rangapuram et al. 2018][Rangapuram2018] | Global | Univariate | RNN, state-space model | [MXNet][DeepState] +DeepFactor
[Wang et al. 2019][Wang2019] | Global | Univariate | RNN, state-space model, Gaussian process | [MXNet][DeepFactor] +Deep Renewal Processes
[Türkmen et al. 2021][Turkmen2021] | Global | Univariate | RNN | [MXNet][DeepRenewal] +GPForecaster | Global | Univariate | MLP, Gaussian process | [MXNet][GP] +MQ-CNN
[Wen et al. 2017][Wen2017] | Global | Univariate | CNN encoder, MLP decoder | [MXNet][MQDNN] +MQ-RNN
[Wen et al. 2017][Wen2017] | Global | Univariate | RNN encoder, MLP encoder | [MXNet][MQDNN] +N-BEATS
[Oreshkin et al. 2019][Oreshkin2019] | Global | Univariate | MLP, residual links | [MXNet][NBeats] +Rotbaum
[Hasson et al. 2021][Hasson2021] | Global | Univariate | XGBoost, Quantile Regression Forests, LightGBM, Level Set Forecaster | [Numpy][Rotbaum] +Causal Convolutional Transformer
[Li et al. 2019][Li2019] | Global | Univariate | Causal convolution, self attention | [MXNet][SAN] +Temporal Fusion Transformer
[Lim et al. 2021][Lim2021] | Global | Univariate | LSTM, self attention | [MXNet][TFT] +Transformer
[Vaswani et al. 2017][Vaswani2017] | Global | Univariate | MLP, multi-head attention | [MXNet][Transformer] +WaveNet
[van den Oord et al. 2016][vanDenOord2016] | Global | Univariate | Dilated convolution | [MXNet][WaveNet] +SimpleFeedForward | Global | Univariate | MLP | [MXNet][SFF_mx], [PyTorch][SFF_torch] +DeepVAR
[Salinas et al. 2019][Salinas2019] | Global | Multivariate | RNN | [MXNet][DeepVAR] +GPVAR
[Salinas et al. 2019][Salinas2019] | Global | Multivariate | RNN, Gaussian process | [MXNet][GPVAR] +LSTNet
[Lai et al. 2018][Lai2018] | Global | Multivariate | LSTM | [MXNet][LSTNet] +DeepTPP
[Shchur et al. 2020][Shchur2020] | Global | Multivariate events | RNN, temporal point process | [MXNet][DeepTPP] +DeepVARHierarchical
[Rangapuram et al. 2021][Rangapuram2021] | Global | Hierarchical | RNN | [MXNet][DeepVARHierarchical] +RForecast
[Hyndman et al. 2008][Hyndman2008] | Local | Univariate | ARIMA, ETS, Croston, TBATS | [Wrapped R package][RForecast] +Prophet
[Taylor et al. 2017][Taylor2017] | Local | Univariate | - | [Wrapped Python package][Prophet] +NaiveSeasonal
[Hyndman et al. 2018][Hyndman2018] | Local | Univariate | - | [Numpy][NaiveSeasonal] +Naive2
[Makridakis et al. 1998][Makridakis1998] | Local | Univariate | - | [Numpy][Naive2] +NPTS | Local | Univariate | - | [Numpy][NPTS] @@ -51,28 +51,28 @@ NPTS | Local | Uni -[DeepAR_mx]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/deepar/_estimator.py -[DeepAR_torch]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/torch/model/deepar/estimator.py -[DeepState]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/deepstate/_estimator.py -[DeepFactor]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/deep_factor/_estimator.py -[DeepRenewal]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/renewal/_estimator.py -[GP]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/gp_forecaster/_estimator.py -[MQDNN]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/seq2seq/_mq_dnn_estimator.py -[NBeats]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/n_beats/_estimator.py -[Rotbaum]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/model/rotbaum/_estimator.py -[SAN]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/san/_estimator.py -[TFT]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/tft/_estimator.py -[Transformer]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/transformer/_estimator.py -[WaveNet]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/wavenet/_estimator.py -[SFF_mx]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/simple_feedforward/_estimator.py -[SFF_torch]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/torch/model/simple_feedforward/estimator.py -[DeepVAR]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/deepvar/_estimator.py -[DeepVARHierarchical]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/deepvar_hierarchical/_estimator.py -[GPVAR]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/gpvar/_estimator.py -[LSTNet]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/lstnet/_estimator.py -[DeepTPP]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/mx/model/tpp/deeptpp/_estimator.py -[RForecast]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/model/r_forecast/_predictor.py -[Prophet]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/model/prophet/_predictor.py -[NaiveSeasonal]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/model/seasonal_naive/_predictor.py -[Naive2]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/model/naive_2/_predictor.py -[NPTS]: https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/model/npts/_predictor.py +[DeepAR_mx]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/deepar/_estimator.py +[DeepAR_torch]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/torch/model/deepar/estimator.py +[DeepState]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/deepstate/_estimator.py +[DeepFactor]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/deep_factor/_estimator.py +[DeepRenewal]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/renewal/_estimator.py +[GP]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/gp_forecaster/_estimator.py +[MQDNN]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/seq2seq/_mq_dnn_estimator.py +[NBeats]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/n_beats/_estimator.py +[Rotbaum]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/model/rotbaum/_estimator.py +[SAN]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/san/_estimator.py +[TFT]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/tft/_estimator.py +[Transformer]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/transformer/_estimator.py +[WaveNet]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/wavenet/_estimator.py +[SFF_mx]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/simple_feedforward/_estimator.py +[SFF_torch]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/torch/model/simple_feedforward/estimator.py +[DeepVAR]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/deepvar/_estimator.py +[DeepVARHierarchical]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/deepvar_hierarchical/_estimator.py +[GPVAR]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/gpvar/_estimator.py +[LSTNet]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/lstnet/_estimator.py +[DeepTPP]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/mx/model/tpp/deeptpp/_estimator.py +[RForecast]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/model/r_forecast/_predictor.py +[Prophet]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/model/prophet/_predictor.py +[NaiveSeasonal]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/model/seasonal_naive/_predictor.py +[Naive2]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/model/naive_2/_predictor.py +[NPTS]: https://github.com/awslabs/gluonts/blob/dev/src/gluonts/model/npts/_predictor.py diff --git a/examples/GluonTS_SageMaker_SDK_Tutorial.ipynb b/examples/GluonTS_SageMaker_SDK_Tutorial.ipynb index 65fad03ce1..06ff52e504 100644 --- a/examples/GluonTS_SageMaker_SDK_Tutorial.ipynb +++ b/examples/GluonTS_SageMaker_SDK_Tutorial.ipynb @@ -42,7 +42,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install --upgrade mxnet==1.6 git+https://github.com/awslabs/gluon-ts.git#egg=gluonts[dev]" + "!pip install --upgrade mxnet==1.6 git+https://github.com/awslabs/gluonts.git#egg=gluonts[dev]" ] }, { @@ -364,7 +364,7 @@ "outputs": [], "source": [ "requirements_dot_txt_file_name = \"requirements.txt\"\n", - "requirements_dot_txt_file_content = \"git+https://github.com/awslabs/gluon-ts.git\"" + "requirements_dot_txt_file_content = \"git+https://github.com/awslabs/gluonts.git\"" ] }, { diff --git a/examples/dockerfiles/README.md b/examples/dockerfiles/README.md index 6880ab1634..78570179a4 100644 --- a/examples/dockerfiles/README.md +++ b/examples/dockerfiles/README.md @@ -13,7 +13,7 @@ docker build ../.. -f ``` The built images are compatible with sagemaker. -For more information about the shell and the available params, see the [shell documentation](https://github.com/awslabs/gluon-ts/tree/dev/src/gluonts/shell). +For more information about the shell and the available params, see the [shell documentation](https://github.com/awslabs/gluonts/tree/dev/src/gluonts/shell). ## How to choose between the dockerfiles diff --git a/setup.py b/setup.py index d67600ecaf..b312b83e65 100644 --- a/setup.py +++ b/setup.py @@ -222,7 +222,11 @@ def run(self): ), long_description=read("README.md"), long_description_content_type="text/markdown", - url="https://github.com/awslabs/gluon-ts", + url="https://github.com/awslabs/gluonts/", + project_urls={ + "Documentation": "https://ts.gluon.ai/stable/", + "Source Code": "https://github.com/awslabs/gluonts/", + }, author="Amazon", author_email="gluon-ts-dev@amazon.com", maintainer_email="gluon-ts-dev@amazon.com", diff --git a/src/gluonts/dataset/arrow/file.py b/src/gluonts/dataset/arrow/file.py index ee47f00aa1..806b59b2f7 100644 --- a/src/gluonts/dataset/arrow/file.py +++ b/src/gluonts/dataset/arrow/file.py @@ -162,7 +162,6 @@ def __len__(self): class ParquetFile(File): path: Path reader: pq.ParquetFile = field(init=False) - _length: Optional[int] = field(default=None, init=False) def __post_init__(self): self.reader = pq.ParquetFile(self.path) @@ -182,7 +181,5 @@ def __iter__(self): yield from self.decoder.decode_batch(batch) def __len__(self): - if self._length is None: - self._length = self.reader.scan_contents() - - return self._length + # One would think that pq.ParquetFile had a nicer way to get its length + return self.reader.metadata.num_rows diff --git a/src/gluonts/dataset/repository/_gp_copula_2019.py b/src/gluonts/dataset/repository/_gp_copula_2019.py index aa799abcfa..08ae696381 100644 --- a/src/gluonts/dataset/repository/_gp_copula_2019.py +++ b/src/gluonts/dataset/repository/_gp_copula_2019.py @@ -89,7 +89,7 @@ class GPCopulaDataset(NamedTuple): name="wiki-rolling_nips", # That file lives on GitHub Large file storage (lfs). We need to use # the exact link, otherwise it will only open the lfs pointer file. - url="https://github.com/awslabs/gluon-ts/raw/1553651ca1fca63a16e012b8927bd9ce72b8e79e/datasets/wiki-rolling_nips.tar.gz", + url="https://github.com/awslabs/gluonts/raw/1553651ca1fca63a16e012b8927bd9ce72b8e79e/datasets/wiki-rolling_nips.tar.gz", num_series=9535, prediction_length=30, freq="D", @@ -154,11 +154,11 @@ def get_data(dataset_path: Path, ds_info: GPCopulaDataset): "item_id": cat, } for cat, data_entry in enumerate( - FileDataset(dataset_path, freq=ds_info.freq) + FileDataset(dataset_path, freq=ds_info.freq, pattern="[!._]*") ) ] def clean_up_dataset(dataset_path: Path, ds_info: GPCopulaDataset): os.remove(dataset_path.parent / f"{ds_info.name}.tar.gz") - shutil.rmtree(dataset_path / "metadata") + shutil.rmtree(dataset_path / "metadata", ignore_errors=True) diff --git a/src/gluonts/meta/__init__.py b/src/gluonts/meta/__init__.py new file mode 100644 index 0000000000..f342912f9b --- /dev/null +++ b/src/gluonts/meta/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. diff --git a/src/gluonts/meta/colors.py b/src/gluonts/meta/colors.py new file mode 100644 index 0000000000..37a7bf4f9f --- /dev/null +++ b/src/gluonts/meta/colors.py @@ -0,0 +1,21 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + + +COPPER = "#6d2f20" +RED = "#b75347" +SALMON = "#df7e66" +ORANGE = "#e09351" +YELLOW = "#edc775" +GREEN = "#94b594" +BLUE = "#224b5e" diff --git a/src/gluonts/model/canonical/_estimator.py b/src/gluonts/model/canonical/_estimator.py index ca9be1667f..9888917d96 100644 --- a/src/gluonts/model/canonical/_estimator.py +++ b/src/gluonts/model/canonical/_estimator.py @@ -24,7 +24,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.mx.batchify import batchify from gluonts.mx.block.feature import FeatureEmbedder @@ -34,7 +33,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import time_features_from_frequency_str from gluonts.transform import ( AddTimeFeatures, @@ -125,8 +123,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(CanonicalTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -141,8 +138,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(CanonicalTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/deep_factor/_estimator.py b/src/gluonts/model/deep_factor/_estimator.py index 0fd2edc93e..f4e5ec63e4 100644 --- a/src/gluonts/model/deep_factor/_estimator.py +++ b/src/gluonts/model/deep_factor/_estimator.py @@ -23,7 +23,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.mx.batchify import batchify from gluonts.mx.block.feature import FeatureEmbedder @@ -32,7 +31,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import time_features_from_frequency_str from gluonts.transform import ( AddTimeFeatures, @@ -205,8 +203,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepFactorTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -221,8 +218,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepFactorTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/deepar/_estimator.py b/src/gluonts/model/deepar/_estimator.py index e710dfa239..935890ad75 100644 --- a/src/gluonts/model/deepar/_estimator.py +++ b/src/gluonts/model/deepar/_estimator.py @@ -26,7 +26,6 @@ ValidationDataLoader, ) from gluonts.dataset.stat import calculate_dataset_statistics -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.mx.batchify import batchify from gluonts.mx.distribution import DistributionOutput, StudentTOutput @@ -34,7 +33,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import ( TimeFeature, get_lags_for_frequency, @@ -400,8 +398,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepARTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -416,8 +413,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepARTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/deepstate/_estimator.py b/src/gluonts/model/deepstate/_estimator.py index dcfd965736..12a72465ab 100644 --- a/src/gluonts/model/deepstate/_estimator.py +++ b/src/gluonts/model/deepstate/_estimator.py @@ -26,7 +26,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.deepstate.issm import ISSM, CompositeISSM from gluonts.model.predictor import Predictor from gluonts.mx.batchify import batchify @@ -35,7 +34,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import ( TimeFeature, norm_freq_str, @@ -345,8 +343,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepStateTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -361,8 +358,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepStateTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/deepvar/_estimator.py b/src/gluonts/model/deepvar/_estimator.py index ede1b1e758..4923d9a133 100644 --- a/src/gluonts/model/deepvar/_estimator.py +++ b/src/gluonts/model/deepvar/_estimator.py @@ -27,7 +27,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.mx.batchify import batchify from gluonts.mx.distribution import ( @@ -38,7 +37,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import TimeFeature, norm_freq_str from gluonts.transform import ( AddObservedValuesIndicator, @@ -410,8 +408,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepVARTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -426,8 +423,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepVARTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/forecast.py b/src/gluonts/model/forecast.py index 9f30fe388d..faddb670b6 100644 --- a/src/gluonts/model/forecast.py +++ b/src/gluonts/model/forecast.py @@ -779,14 +779,11 @@ def plot(self, label=None, output_file=None, keys=None, *args, **kwargs): keys = self.forecast_keys for k, v in zip(keys, self.forecast_array): - plt.plot( - self.index, - v, + pd.Series(data=v, index=self.index.to_timestamp()).plot( label=f"{label_prefix}q{k}", *args, **kwargs, ) - plt.legend() if output_file: plt.savefig(output_file) diff --git a/src/gluonts/model/gp_forecaster/_estimator.py b/src/gluonts/model/gp_forecaster/_estimator.py index db8e4f28f1..cac75d0c29 100644 --- a/src/gluonts/model/gp_forecaster/_estimator.py +++ b/src/gluonts/model/gp_forecaster/_estimator.py @@ -25,7 +25,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.mx.batchify import batchify from gluonts.mx.kernels import KernelOutput, RBFKernelOutput @@ -33,7 +32,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import TimeFeature, time_features_from_frequency_str from gluonts.transform import ( AddTimeFeatures, @@ -200,8 +198,7 @@ def create_training_data_loader( input_names = get_hybrid_forward_input_names( GaussianProcessTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -218,8 +215,7 @@ def create_validation_data_loader( input_names = get_hybrid_forward_input_names( GaussianProcessTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/gpvar/_estimator.py b/src/gluonts/model/gpvar/_estimator.py index 4178b3ba8b..fcfbd0fb03 100644 --- a/src/gluonts/model/gpvar/_estimator.py +++ b/src/gluonts/model/gpvar/_estimator.py @@ -24,7 +24,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.deepvar._estimator import ( get_lags_for_frequency, time_features_from_frequency_str, @@ -37,7 +36,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import TimeFeature from gluonts.transform import ( AddObservedValuesIndicator, @@ -342,8 +340,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(GPVARTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -358,8 +355,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(GPVARTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/lstnet/_estimator.py b/src/gluonts/model/lstnet/_estimator.py index 10b65801d0..64061e94b5 100644 --- a/src/gluonts/model/lstnet/_estimator.py +++ b/src/gluonts/model/lstnet/_estimator.py @@ -25,7 +25,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.lstnet._network import LSTNetPredict, LSTNetTrain from gluonts.model.predictor import Predictor from gluonts.mx.batchify import batchify @@ -33,7 +32,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.transform import ( AddObservedValuesIndicator, AsNumpyArray, @@ -211,8 +209,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(LSTNetTrain) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -227,8 +224,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(LSTNetTrain) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/n_beats/_estimator.py b/src/gluonts/model/n_beats/_estimator.py index d05d758229..826300b9d1 100644 --- a/src/gluonts/model/n_beats/_estimator.py +++ b/src/gluonts/model/n_beats/_estimator.py @@ -24,14 +24,12 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.mx.batchify import batchify from gluonts.mx.model.estimator import GluonEstimator from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.transform import ( AddObservedValuesIndicator, ExpectedNumInstanceSampler, @@ -303,8 +301,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(NBEATSTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -319,8 +316,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(NBEATSTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/renewal/_estimator.py b/src/gluonts/model/renewal/_estimator.py index 39f6567f1e..ab8574848a 100644 --- a/src/gluonts/model/renewal/_estimator.py +++ b/src/gluonts/model/renewal/_estimator.py @@ -23,7 +23,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.itertools import Cyclic from gluonts.model.predictor import Predictor from gluonts.model.renewal._network import ( @@ -37,7 +36,6 @@ from gluonts.mx.model.estimator import GluonEstimator from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters -from gluonts.itertools import maybe_len from gluonts.transform import ( AddObservedValuesIndicator, AsNumpyArray, @@ -216,12 +214,11 @@ def create_training_data_loader( data: Dataset, **kwargs, ) -> DataLoader: - with env._let(max_idle_transforms=maybe_len(data) or 0): - train_transform = ( - self._create_instance_splitter("training") - + self._create_post_split_transform() - + SelectFields(["past_target", "valid_length"]) - ) + train_transform = ( + self._create_instance_splitter("training") + + self._create_post_split_transform() + + SelectFields(["past_target", "valid_length"]) + ) return TrainDataLoader( train_transform.apply(Cyclic(data)), batch_size=self.batch_size, @@ -233,12 +230,11 @@ def create_validation_data_loader( data: Dataset, **kwargs, ) -> DataLoader: - with env._let(max_idle_transforms=maybe_len(data) or 0): - validation_transform = ( - self._create_instance_splitter("validation") - + self._create_post_split_transform() - + SelectFields(["past_target", "valid_length"]) - ) + validation_transform = ( + self._create_instance_splitter("validation") + + self._create_post_split_transform() + + SelectFields(["past_target", "valid_length"]) + ) return ValidationDataLoader( validation_transform.apply(data), batch_size=self.batch_size, diff --git a/src/gluonts/model/san/_estimator.py b/src/gluonts/model/san/_estimator.py index f52fddd0fa..3f50433d54 100644 --- a/src/gluonts/model/san/_estimator.py +++ b/src/gluonts/model/san/_estimator.py @@ -24,14 +24,12 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.forecast_generator import QuantileForecastGenerator from gluonts.mx.batchify import batchify from gluonts.mx.model.estimator import GluonEstimator from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import TimeFeature, time_features_from_frequency_str from gluonts.transform import ( AddAgeFeature, @@ -272,8 +270,7 @@ def create_training_data_loader( input_names = get_hybrid_forward_input_names( SelfAttentionTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -290,8 +287,7 @@ def create_validation_data_loader( input_names = get_hybrid_forward_input_names( SelfAttentionTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/seq2seq/_forking_estimator.py b/src/gluonts/model/seq2seq/_forking_estimator.py index 9bc2831ff9..8cbeb45501 100644 --- a/src/gluonts/model/seq2seq/_forking_estimator.py +++ b/src/gluonts/model/seq2seq/_forking_estimator.py @@ -24,7 +24,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.forecast import Quantile from gluonts.model.forecast_generator import ( DistributionForecastGenerator, @@ -41,7 +40,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import time_features_from_frequency_str from gluonts.transform import ( AddAgeFeature, @@ -470,8 +468,7 @@ def create_training_data_loader( input_names = get_hybrid_forward_input_names( ForkingSeq2SeqTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -488,8 +485,7 @@ def create_validation_data_loader( input_names = get_hybrid_forward_input_names( ForkingSeq2SeqTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/seq2seq/_seq2seq_estimator.py b/src/gluonts/model/seq2seq/_seq2seq_estimator.py index 64bb22ca5c..2cf14f117e 100644 --- a/src/gluonts/model/seq2seq/_seq2seq_estimator.py +++ b/src/gluonts/model/seq2seq/_seq2seq_estimator.py @@ -25,7 +25,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.forecast import Quantile from gluonts.model.forecast_generator import QuantileForecastGenerator from gluonts.model.predictor import Predictor @@ -45,7 +44,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import time_features_from_frequency_str from gluonts.transform import ( ExpectedNumInstanceSampler, @@ -176,8 +174,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(Seq2SeqTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -192,8 +189,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(Seq2SeqTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/simple_feedforward/_estimator.py b/src/gluonts/model/simple_feedforward/_estimator.py index 5290f38183..d4135a08b6 100644 --- a/src/gluonts/model/simple_feedforward/_estimator.py +++ b/src/gluonts/model/simple_feedforward/_estimator.py @@ -24,7 +24,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.forecast_generator import DistributionForecastGenerator from gluonts.mx.batchify import batchify from gluonts.mx.distribution import DistributionOutput, StudentTOutput @@ -32,7 +31,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.transform import ( AddObservedValuesIndicator, ExpectedNumInstanceSampler, @@ -236,8 +234,7 @@ def create_training_data_loader( input_names = get_hybrid_forward_input_names( SimpleFeedForwardTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -254,8 +251,7 @@ def create_validation_data_loader( input_names = get_hybrid_forward_input_names( SimpleFeedForwardTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/tft/_estimator.py b/src/gluonts/model/tft/_estimator.py index 8924357981..5ba116c110 100644 --- a/src/gluonts/model/tft/_estimator.py +++ b/src/gluonts/model/tft/_estimator.py @@ -25,14 +25,12 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.forecast_generator import QuantileForecastGenerator from gluonts.mx.batchify import batchify from gluonts.mx.model.estimator import GluonEstimator from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import ( Constant, TimeFeature, @@ -339,8 +337,7 @@ def create_training_data_loader( input_names = get_hybrid_forward_input_names( TemporalFusionTransformerTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -357,8 +354,7 @@ def create_validation_data_loader( input_names = get_hybrid_forward_input_names( TemporalFusionTransformerTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/tpp/deeptpp/_estimator.py b/src/gluonts/model/tpp/deeptpp/_estimator.py index d2fe6b8219..eaa8e22c59 100644 --- a/src/gluonts/model/tpp/deeptpp/_estimator.py +++ b/src/gluonts/model/tpp/deeptpp/_estimator.py @@ -21,7 +21,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.model.tpp import PointProcessGluonPredictor from gluonts.model.tpp.distribution import TPPDistributionOutput, WeibullOutput @@ -29,7 +28,6 @@ from gluonts.mx.model.estimator import GluonEstimator from gluonts.mx.trainer import Trainer from gluonts.mx.util import get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.transform import ( Chain, ContinuousTimeInstanceSplitter, @@ -198,8 +196,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepTPPTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -214,8 +211,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(DeepTPPTrainingNetwork) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/transformer/_estimator.py b/src/gluonts/model/transformer/_estimator.py index 708176fc62..5a2c2e92ba 100644 --- a/src/gluonts/model/transformer/_estimator.py +++ b/src/gluonts/model/transformer/_estimator.py @@ -24,7 +24,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.model.transformer._network import ( TransformerPredictionNetwork, @@ -38,7 +37,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import ( TimeFeature, get_lags_for_frequency, @@ -318,8 +316,7 @@ def create_training_data_loader( input_names = get_hybrid_forward_input_names( TransformerTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -336,8 +333,7 @@ def create_validation_data_loader( input_names = get_hybrid_forward_input_names( TransformerTrainingNetwork ) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/model/wavenet/_estimator.py b/src/gluonts/model/wavenet/_estimator.py index a46fcb644e..f90ca50ca7 100644 --- a/src/gluonts/model/wavenet/_estimator.py +++ b/src/gluonts/model/wavenet/_estimator.py @@ -27,7 +27,6 @@ TrainDataLoader, ValidationDataLoader, ) -from gluonts.env import env from gluonts.model.predictor import Predictor from gluonts.model.wavenet._network import ( WaveNet, @@ -39,7 +38,6 @@ from gluonts.mx.model.predictor import RepresentableBlockPredictor from gluonts.mx.trainer import Trainer from gluonts.mx.util import copy_parameters, get_hybrid_forward_input_names -from gluonts.itertools import maybe_len from gluonts.time_feature import ( get_seasonality, time_features_from_frequency_str, @@ -334,8 +332,7 @@ def create_training_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(WaveNetTraining) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("training") + instance_splitter = self._create_instance_splitter("training") return TrainDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), @@ -350,8 +347,7 @@ def create_validation_data_loader( **kwargs, ) -> DataLoader: input_names = get_hybrid_forward_input_names(WaveNetTraining) - with env._let(max_idle_transforms=maybe_len(data) or 0): - instance_splitter = self._create_instance_splitter("validation") + instance_splitter = self._create_instance_splitter("validation") return ValidationDataLoader( dataset=data, transform=instance_splitter + SelectFields(input_names), diff --git a/src/gluonts/mx/batchify.py b/src/gluonts/mx/batchify.py index d512770303..0d3807828e 100644 --- a/src/gluonts/mx/batchify.py +++ b/src/gluonts/mx/batchify.py @@ -75,7 +75,7 @@ def stack( data = _pad_arrays(data, axis=0, is_right_pad=is_right_pad) if isinstance(data[0], mx.nd.NDArray): # TODO: think about using shared context NDArrays - # https://github.com/awslabs/gluon-ts/blob/42bee73409f801e7bca73245ca21cd877891437c/src/gluonts/dataset/parallelized_loader.py#L157 + # https://github.com/awslabs/gluonts/blob/42bee73409f801e7bca73245ca21cd877891437c/src/gluonts/dataset/parallelized_loader.py#L157 return mx.nd.stack(*data) if isinstance(data[0], np.ndarray): data = mx.nd.array(data, dtype=dtype, ctx=ctx) diff --git a/src/gluonts/mx/model/estimator.py b/src/gluonts/mx/model/estimator.py index a2e8c8682a..be71658caa 100644 --- a/src/gluonts/mx/model/estimator.py +++ b/src/gluonts/mx/model/estimator.py @@ -14,6 +14,7 @@ from typing import NamedTuple, Optional, Type import numpy as np + from gluonts.core import fqname_for from gluonts.core.component import ( GluonTSHyperparametersError, @@ -22,6 +23,7 @@ ) from gluonts.dataset.common import Dataset from gluonts.dataset.loader import DataLoader +from gluonts.env import env from gluonts.itertools import Cached from gluonts.model.estimator import Estimator from gluonts.model.predictor import Predictor @@ -170,27 +172,34 @@ def train_model( shuffle_buffer_length: Optional[int] = None, cache_data: bool = False, ) -> TrainOutput: + transformation = self.create_transformation() - transformed_training_data = transformation.apply(training_data) + with env._let(max_idle_transforms=max(len(training_data), 100)): + transformed_training_data = transformation.apply(training_data) + if cache_data: + transformed_training_data = Cached(transformed_training_data) - training_data_loader = self.create_training_data_loader( - transformed_training_data - if not cache_data - else Cached(transformed_training_data), - shuffle_buffer_length=shuffle_buffer_length, - ) + training_data_loader = self.create_training_data_loader( + transformed_training_data, + shuffle_buffer_length=shuffle_buffer_length, + ) validation_data_loader = None if validation_data is not None: - transformed_validation_data = transformation.apply(validation_data) - - validation_data_loader = self.create_validation_data_loader( - transformed_validation_data - if not cache_data - else Cached(transformed_validation_data), - ) + with env._let(max_idle_transforms=max(len(validation_data), 100)): + transformed_validation_data = transformation.apply( + validation_data + ) + if cache_data: + transformed_validation_data = Cached( + transformed_validation_data + ) + + validation_data_loader = self.create_validation_data_loader( + transformed_validation_data + ) training_network = self.create_training_network() diff --git a/src/gluonts/nursery/QRX-Wrapped-M5-Accuracy-Solution/0. README.md b/src/gluonts/nursery/QRX-Wrapped-M5-Accuracy-Solution/0. README.md index 2dbfce9a0d..822aa6589b 100644 --- a/src/gluonts/nursery/QRX-Wrapped-M5-Accuracy-Solution/0. README.md +++ b/src/gluonts/nursery/QRX-Wrapped-M5-Accuracy-Solution/0. README.md @@ -1,9 +1,9 @@ # Introduction / Hilaf Hasson -This work wraps the [winning solution in the M5 Accuracy competition](https://github.com/Mcompetitions/M5-methods/tree/master/Code%20of%20Winning%20Methods/A1) (by YeonJun Im) with [QRX](https://github.com/awslabs/gluon-ts/blob/dev/src/gluonts/model/rotbaum/_model.py). +This work wraps the [winning solution in the M5 Accuracy competition](https://github.com/Mcompetitions/M5-methods/tree/master/Code%20of%20Winning%20Methods/A1) (by YeonJun Im) with [QRX](https://github.com/awslabs/gluonts/blob/dev/src/gluonts/model/rotbaum/_model.py). -QRX is an algorithm that takes a point forecaster as input and outputs a probabilistic forecaster. I designed it originally for the purpose of being the engine for [Rotbaum](https://github.com/awslabs/gluon-ts/tree/dev/src/gluonts/model/rotbaum), the flagship tree-based algorithm in [gluonts](https://github.com/awslabs/gluon-ts). +QRX is an algorithm that takes a point forecaster as input and outputs a probabilistic forecaster. I designed it originally for the purpose of being the engine for [Rotbaum](https://github.com/awslabs/gluonts/tree/dev/src/gluonts/model/rotbaum), the flagship tree-based algorithm in [gluonts](https://github.com/awslabs/gluonts). The rough idea behind the logic of QRX is that it uses the point forecaster's predictions to create bins of true values that are being sampled at inference. We will outline the exact algorithm in a future paper. diff --git a/src/gluonts/nursery/sagemaker_sdk/estimator.py b/src/gluonts/nursery/sagemaker_sdk/estimator.py index 32096ab709..2084134669 100644 --- a/src/gluonts/nursery/sagemaker_sdk/estimator.py +++ b/src/gluonts/nursery/sagemaker_sdk/estimator.py @@ -113,7 +113,7 @@ class GluonTSFramework(Framework): defined by the "entry_point" argument of the :meth:`GluonTSFramework.run` method. Technical documentation on preparing GluonTSFramework scripts for SageMaker training and using the GluonTsFramework Estimator is available on - the project home-page: https://github.com/awslabs/gluon-ts. See + the project home-page: https://github.com/awslabs/gluonts. See how_to_notebooks for examples of how to use this SDK. Parameters diff --git a/src/gluonts/nursery/tsbench/README.md b/src/gluonts/nursery/tsbench/README.md index 14835c739a..524271f767 100644 --- a/src/gluonts/nursery/tsbench/README.md +++ b/src/gluonts/nursery/tsbench/README.md @@ -33,8 +33,8 @@ In order to use the code in this repository, you should first clone the GluonTS go into the directory of this project: ```bash -git clone git@github.com:awslabs/gluon-ts.git -cd gluon-ts/src/gluonts/nursery/tsbench +git clone git@github.com:awslabs/gluonts.git +cd gluonts/src/gluonts/nursery/tsbench ``` Then, in the root of the repository, you can install all dependencies via diff --git a/src/gluonts/nursery/tsbench/pyproject.toml b/src/gluonts/nursery/tsbench/pyproject.toml index 03853a0dca..6dcbc32ff0 100644 --- a/src/gluonts/nursery/tsbench/pyproject.toml +++ b/src/gluonts/nursery/tsbench/pyproject.toml @@ -15,7 +15,7 @@ catch22 = "^0.2.0" click = "^7.1.2" fastparquet = "^0.6.1" fbprophet = "^0.7.1" -gluonts = {git = "https://github.com/awslabs/gluon-ts.git", rev = "7c94c1149875f6ad2e0d7b0a6bcee952f14d3fb1"} +gluonts = {git = "https://github.com/awslabs/gluonts.git", rev = "7c94c1149875f6ad2e0d7b0a6bcee952f14d3fb1"} holidays = "^0.11.1" lightkit = "^0.3.6" mxnet = "1.8.0.post0" diff --git a/src/gluonts/torch/model/estimator.py b/src/gluonts/torch/model/estimator.py index cf932d2498..bfc1335b41 100644 --- a/src/gluonts/torch/model/estimator.py +++ b/src/gluonts/torch/model/estimator.py @@ -20,6 +20,7 @@ from gluonts.core.component import validated from gluonts.dataset.common import Dataset +from gluonts.env import env from gluonts.itertools import Cached from gluonts.model.estimator import Estimator from gluonts.torch.model.predictor import PyTorchPredictor @@ -154,36 +155,40 @@ def train_model( ) -> TrainOutput: transformation = self.create_transformation() - transformed_training_data = transformation.apply( - training_data, is_train=True - ) - - training_network = self.create_lightning_module() - - training_data_loader = self.create_training_data_loader( - transformed_training_data - if not cache_data - else Cached(transformed_training_data), - training_network, - num_workers=num_workers, - shuffle_buffer_length=shuffle_buffer_length, - ) - - validation_data_loader = None - - if validation_data is not None: - transformed_validation_data = transformation.apply( - validation_data, is_train=True + with env._let(max_idle_transforms=max(len(training_data), 100)): + transformed_training_data = transformation.apply( + training_data, is_train=True ) + if cache_data: + transformed_training_data = Cached(transformed_training_data) + + training_network = self.create_lightning_module() - validation_data_loader = self.create_validation_data_loader( - transformed_validation_data - if not cache_data - else Cached(transformed_validation_data), + training_data_loader = self.create_training_data_loader( + transformed_training_data, training_network, num_workers=num_workers, + shuffle_buffer_length=shuffle_buffer_length, ) + validation_data_loader = None + + with env._let(max_idle_transforms=max(len(training_data), 100)): + if validation_data is not None: + transformed_validation_data = transformation.apply( + validation_data, is_train=True + ) + if cache_data: + transformed_validation_data = Cached( + transformed_validation_data + ) + + validation_data_loader = self.create_validation_data_loader( + transformed_validation_data, + training_network, + num_workers=num_workers, + ) + monitor = "train_loss" if validation_data is None else "val_loss" checkpoint = pl.callbacks.ModelCheckpoint( monitor=monitor, mode="min", verbose=True diff --git a/src/gluonts/transform/_base.py b/src/gluonts/transform/_base.py index 99f9835e08..0d4d252fb2 100644 --- a/src/gluonts/transform/_base.py +++ b/src/gluonts/transform/_base.py @@ -169,7 +169,7 @@ class FlatMapTransformation(Transformation): @validated() def __init__(self): - self.max_idle_transforms = max(env.max_idle_transforms, 100) + self.max_idle_transforms = env.max_idle_transforms def __call__( self, data_it: Iterable[DataEntry], is_train: bool @@ -180,7 +180,12 @@ def __call__( for result in self.flatmap_transform(data_entry.copy(), is_train): num_idle_transforms = 0 yield result - if num_idle_transforms > self.max_idle_transforms: + + if ( + # negative values disable the check + self.max_idle_transforms > 0 + and num_idle_transforms > self.max_idle_transforms + ): raise Exception( "Reached maximum number of idle transformation" " calls.\nThis means the transformation looped over" diff --git a/test/model/test_forecast.py b/test/model/test_forecast.py index ab8ca97258..e2fc894361 100644 --- a/test/model/test_forecast.py +++ b/test/model/test_forecast.py @@ -60,6 +60,8 @@ def percentile(value): assert len(forecast.index) == pred_length assert forecast.index[0] == START_DATE + forecast.plot() + @pytest.mark.parametrize( "forecast, exp_index", diff --git a/test/mx/test_mx_rolling.py b/test/mx/test_mx_rolling.py index d8b87ff1d0..ce29eb9168 100644 --- a/test/mx/test_mx_rolling.py +++ b/test/mx/test_mx_rolling.py @@ -68,7 +68,7 @@ def test_dynamic_integration( ): """ Trains an estimator on a rolled dataset with dynamic features. - Tests https://github.com/awslabs/gluon-ts/issues/1390 + Tests https://github.com/awslabs/gluonts/issues/1390 """ train_ds = create_dynamic_dataset( target_start, train_length, num_dynamic_feat