From 13c1eef9ef17395c32c731625de1382a0beb62e9 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 10:53:29 +0200 Subject: [PATCH 01/31] ci: rebuild all notebooks in key req. & assistant changed --- .actions/assistant.py | 4 +++- .azure/ipynb-publish.yml | 2 +- .azure/ipynb-validate.yml | 2 +- _requirements/data.txt | 2 -- _requirements/devel.txt | 3 +++ 5 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 _requirements/data.txt diff --git a/.actions/assistant.py b/.actions/assistant.py index 685791a6b..093b2ac52 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -20,6 +20,8 @@ _PATH_ROOT = os.path.dirname(_PATH_HERE) PATH_SCRIPT_RENDER = os.path.join(_PATH_HERE, "_ipynb-render.sh") PATH_SCRIPT_TEST = os.path.join(_PATH_HERE, "_ipynb-validate.sh") +PATH_DEV_SCRIPT = os.path.join(".actions", "assistant.py") +PATH_DEV_REQUIREMENTS = os.path.join("_requirements", "devel.txt") # https://askubuntu.com/questions/909918/how-to-show-unzip-progress UNZIP_PROGRESS_BAR = ' | awk \'BEGIN {ORS=" "} {if(NR%10==0)print "."}\'' REPO_NAME = "lightning-tutorials" @@ -536,7 +538,7 @@ def group_folders( fpath_drop_folders: str = "dropped-folders.txt", fpath_actual_dirs: Sequence[str] = tuple(), strict: bool = True, - root_path: str = "", + root_path: str = ".", ) -> None: """Parsing the raw git diff and group changes by folders. diff --git a/.azure/ipynb-publish.yml b/.azure/ipynb-publish.yml index df1efb99e..ef9964ec4 100644 --- a/.azure/ipynb-publish.yml +++ b/.azure/ipynb-publish.yml @@ -145,7 +145,7 @@ jobs: pip --version # todo: export requirements for notebooks to file and execute # todo: adjust torch ecosystem versions - pip install -r requirements.txt -r _requirements/data.txt + pip install -r requirements.txt displayName: "Install dependencies" timeoutInMinutes: "15" diff --git a/.azure/ipynb-validate.yml b/.azure/ipynb-validate.yml index d165d1ffa..7e3d5992c 100644 --- a/.azure/ipynb-validate.yml +++ b/.azure/ipynb-validate.yml @@ -82,7 +82,7 @@ jobs: - bash: | set -e pip --version - pip install -r requirements.txt -r _requirements/data.txt + pip install -r requirements.txt pip list displayName: "Install dependencies" diff --git a/_requirements/data.txt b/_requirements/data.txt deleted file mode 100644 index 08fe31381..000000000 --- a/_requirements/data.txt +++ /dev/null @@ -1,2 +0,0 @@ -# fixed version to be able to call it as `python -m kaggle` -https://github.com/Borda/kaggle-api/archive/refs/heads/setup/python-m.zip diff --git a/_requirements/devel.txt b/_requirements/devel.txt index 58decc8ee..179a6e0bc 100644 --- a/_requirements/devel.txt +++ b/_requirements/devel.txt @@ -11,3 +11,6 @@ papermill>=2.3.4, <2.5.0 # rendering matplotlib # all is based on PyTorch torch + +# fixed version to be able to call it as `python -m kaggle` +https://github.com/Borda/kaggle-api/archive/refs/heads/setup/python-m.zip \ No newline at end of file From 1c422a4d8f63b13308a35bee293a5c5fa7ce38c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 08:55:27 +0000 Subject: [PATCH 02/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- _requirements/devel.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_requirements/devel.txt b/_requirements/devel.txt index 179a6e0bc..0dfa5eab3 100644 --- a/_requirements/devel.txt +++ b/_requirements/devel.txt @@ -13,4 +13,4 @@ matplotlib torch # fixed version to be able to call it as `python -m kaggle` -https://github.com/Borda/kaggle-api/archive/refs/heads/setup/python-m.zip \ No newline at end of file +https://github.com/Borda/kaggle-api/archive/refs/heads/setup/python-m.zip From a723636c4e9f815c3f8bde34c0f4d900067f398b Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 11:06:19 +0200 Subject: [PATCH 03/31] update req. --- README.md | 2 +- course_UvA-DL/01-introduction-to-pytorch/requirements.txt | 0 course_UvA-DL/02-activation-functions/requirements.txt | 2 -- .../03-initialization-and-optimization/requirements.txt | 2 -- course_UvA-DL/04-inception-resnet-densenet/requirements.txt | 2 -- course_UvA-DL/05-transformers-and-MH-attention/requirements.txt | 2 -- .../07-deep-energy-based-generative-models/requirements.txt | 2 -- course_UvA-DL/08-deep-autoencoders/requirements.txt | 2 -- course_UvA-DL/09-normalizing-flows/requirements.txt | 2 -- course_UvA-DL/10-autoregressive-image-modeling/requirements.txt | 2 -- course_UvA-DL/11-vision-transformer/requirements.txt | 2 -- course_UvA-DL/12-meta-learning/requirements.txt | 2 -- course_UvA-DL/13-contrastive-learning/requirements.txt | 2 -- course_UvA-DL/requirements.txt | 2 ++ 14 files changed, 3 insertions(+), 23 deletions(-) delete mode 100644 course_UvA-DL/01-introduction-to-pytorch/requirements.txt delete mode 100644 course_UvA-DL/02-activation-functions/requirements.txt delete mode 100644 course_UvA-DL/03-initialization-and-optimization/requirements.txt delete mode 100644 course_UvA-DL/05-transformers-and-MH-attention/requirements.txt delete mode 100644 course_UvA-DL/10-autoregressive-image-modeling/requirements.txt diff --git a/README.md b/README.md index c06aa87c0..ffc8fa266 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ The addition has to formed as new folder: accelerator: - CPU ``` -- requirements listed in `requirements.txt` in the particular folder (in case you need some other packaged then listed the parent folder) +- _[optional]_ requirements listed in `requirements.txt` in the particular folder (in case you need some other packaged then listed the parent folder) ## Using datasets diff --git a/course_UvA-DL/01-introduction-to-pytorch/requirements.txt b/course_UvA-DL/01-introduction-to-pytorch/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/course_UvA-DL/02-activation-functions/requirements.txt b/course_UvA-DL/02-activation-functions/requirements.txt deleted file mode 100644 index c505f4cb6..000000000 --- a/course_UvA-DL/02-activation-functions/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torchvision -seaborn diff --git a/course_UvA-DL/03-initialization-and-optimization/requirements.txt b/course_UvA-DL/03-initialization-and-optimization/requirements.txt deleted file mode 100644 index c505f4cb6..000000000 --- a/course_UvA-DL/03-initialization-and-optimization/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torchvision -seaborn diff --git a/course_UvA-DL/04-inception-resnet-densenet/requirements.txt b/course_UvA-DL/04-inception-resnet-densenet/requirements.txt index c0a786caa..80b42774c 100644 --- a/course_UvA-DL/04-inception-resnet-densenet/requirements.txt +++ b/course_UvA-DL/04-inception-resnet-densenet/requirements.txt @@ -1,4 +1,2 @@ -torchvision -seaborn tabulate tensorboard diff --git a/course_UvA-DL/05-transformers-and-MH-attention/requirements.txt b/course_UvA-DL/05-transformers-and-MH-attention/requirements.txt deleted file mode 100644 index c505f4cb6..000000000 --- a/course_UvA-DL/05-transformers-and-MH-attention/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torchvision -seaborn diff --git a/course_UvA-DL/07-deep-energy-based-generative-models/requirements.txt b/course_UvA-DL/07-deep-energy-based-generative-models/requirements.txt index 16b455672..8ba46efc1 100644 --- a/course_UvA-DL/07-deep-energy-based-generative-models/requirements.txt +++ b/course_UvA-DL/07-deep-energy-based-generative-models/requirements.txt @@ -1,3 +1 @@ -torchvision -matplotlib tensorboard diff --git a/course_UvA-DL/08-deep-autoencoders/requirements.txt b/course_UvA-DL/08-deep-autoencoders/requirements.txt index 66448f2cf..8ba46efc1 100644 --- a/course_UvA-DL/08-deep-autoencoders/requirements.txt +++ b/course_UvA-DL/08-deep-autoencoders/requirements.txt @@ -1,3 +1 @@ -torchvision -seaborn tensorboard diff --git a/course_UvA-DL/09-normalizing-flows/requirements.txt b/course_UvA-DL/09-normalizing-flows/requirements.txt index 27b6e8dd1..a5d51591e 100644 --- a/course_UvA-DL/09-normalizing-flows/requirements.txt +++ b/course_UvA-DL/09-normalizing-flows/requirements.txt @@ -1,3 +1 @@ -torchvision -seaborn tabulate diff --git a/course_UvA-DL/10-autoregressive-image-modeling/requirements.txt b/course_UvA-DL/10-autoregressive-image-modeling/requirements.txt deleted file mode 100644 index c505f4cb6..000000000 --- a/course_UvA-DL/10-autoregressive-image-modeling/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torchvision -seaborn diff --git a/course_UvA-DL/11-vision-transformer/requirements.txt b/course_UvA-DL/11-vision-transformer/requirements.txt index 66448f2cf..8ba46efc1 100644 --- a/course_UvA-DL/11-vision-transformer/requirements.txt +++ b/course_UvA-DL/11-vision-transformer/requirements.txt @@ -1,3 +1 @@ -torchvision -seaborn tensorboard diff --git a/course_UvA-DL/12-meta-learning/requirements.txt b/course_UvA-DL/12-meta-learning/requirements.txt index 9410eba20..de07b966e 100644 --- a/course_UvA-DL/12-meta-learning/requirements.txt +++ b/course_UvA-DL/12-meta-learning/requirements.txt @@ -1,4 +1,2 @@ -torchvision -seaborn tensorboard scipy diff --git a/course_UvA-DL/13-contrastive-learning/requirements.txt b/course_UvA-DL/13-contrastive-learning/requirements.txt index 66448f2cf..8ba46efc1 100644 --- a/course_UvA-DL/13-contrastive-learning/requirements.txt +++ b/course_UvA-DL/13-contrastive-learning/requirements.txt @@ -1,3 +1 @@ -torchvision -seaborn tensorboard diff --git a/course_UvA-DL/requirements.txt b/course_UvA-DL/requirements.txt index 2e0dba9cd..54a071e28 100644 --- a/course_UvA-DL/requirements.txt +++ b/course_UvA-DL/requirements.txt @@ -2,4 +2,6 @@ numpy <2.0 # needed for older Torch torch >=1.8.1,<2.1.0 pytorch-lightning >=2.0,<2.1.0 torchmetrics >=1.0,<1.3 +torchvision matplotlib +seaborn From 59c3daa5e5894862b84a062038fbababf61260d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 09:06:44 +0000 Subject: [PATCH 04/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ffc8fa266..78b2b8657 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ The addition has to formed as new folder: accelerator: - CPU ``` -- _[optional]_ requirements listed in `requirements.txt` in the particular folder (in case you need some other packaged then listed the parent folder) +- _\[optional\]_ requirements listed in `requirements.txt` in the particular folder (in case you need some other packaged then listed the parent folder) ## Using datasets From 6c17549bc8ea99ee54f5e7931ee4789f1bee34de Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 11:15:22 +0200 Subject: [PATCH 05/31] os.getcwd() --- .actions/assistant.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.actions/assistant.py b/.actions/assistant.py index 093b2ac52..05ef1b156 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -560,6 +560,8 @@ def group_folders( --fpath_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']" """ + if root_path == ".": + root_path = os.getcwd() with open(fpath_gitdiff) as fopen: changed = [ln.strip() for ln in fopen.readlines()] dirs_changed = [os.path.dirname(ln) for ln in changed] From ac2feb6d8c4385c71b121b08153d773a71ce1493 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 11:31:56 +0200 Subject: [PATCH 06/31] all notebooks --- .actions/assistant.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 05ef1b156..c3a1b9448 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -538,7 +538,7 @@ def group_folders( fpath_drop_folders: str = "dropped-folders.txt", fpath_actual_dirs: Sequence[str] = tuple(), strict: bool = True, - root_path: str = ".", + root_path: str = "", ) -> None: """Parsing the raw git diff and group changes by folders. @@ -560,16 +560,21 @@ def group_folders( --fpath_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']" """ + dirs = [] + # replace here notation with path, so later it does not fall to ignore if root_path == ".": root_path = os.getcwd() + # loading the generated changes with git diff with open(fpath_gitdiff) as fopen: changed = [ln.strip() for ln in fopen.readlines()] dirs_changed = [os.path.dirname(ln) for ln in changed] + # in case of ay key component changed, rebuild all notebooks + if any(p_key in p_changed for p_key in (PATH_DEV_SCRIPT, PATH_DEV_REQUIREMENTS) for p_changed in dirs_changed): + dirs_changed += [p for p in os.listdir(_PATH_ROOT) if os.path.isdir(p) and p[0] not in (".", "_")] # append a path to root in case you call this from other path then root if root_path: dirs_changed = [os.path.join(root_path, d) for d in dirs_changed] # append all subfolders in case of parent requirements has been changed all related notebooks shall be updated - dirs = [] for dir in dirs_changed: # in case that the diff item comes from removed folder if not os.path.isdir(dir): From 552b92cf845ba29d251771687a36984532338944 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 11:44:38 +0200 Subject: [PATCH 07/31] all notebooks --- .actions/assistant.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index c3a1b9448..eb4415148 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -567,9 +567,12 @@ def group_folders( # loading the generated changes with git diff with open(fpath_gitdiff) as fopen: changed = [ln.strip() for ln in fopen.readlines()] + # in case of ay key component changed, later rebuild all notebooks + rebuild_all = any( + p_key in p_changed for p_key in (PATH_DEV_SCRIPT, PATH_DEV_REQUIREMENTS) for p_changed in changed + ) dirs_changed = [os.path.dirname(ln) for ln in changed] - # in case of ay key component changed, rebuild all notebooks - if any(p_key in p_changed for p_key in (PATH_DEV_SCRIPT, PATH_DEV_REQUIREMENTS) for p_changed in dirs_changed): + if rebuild_all: dirs_changed += [p for p in os.listdir(_PATH_ROOT) if os.path.isdir(p) and p[0] not in (".", "_")] # append a path to root in case you call this from other path then root if root_path: From 37a55f26c35a9ed874ab55de03f47e3ca7c7bbd4 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 12:08:32 +0200 Subject: [PATCH 08/31] realpath --- .actions/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index eb4415148..f4933ce24 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -16,7 +16,7 @@ from pip._internal.operations import freeze from wcmatch import glob -_PATH_HERE = os.path.dirname(__file__) +_PATH_HERE = os.path.realpath(os.path.dirname(__file__)) _PATH_ROOT = os.path.dirname(_PATH_HERE) PATH_SCRIPT_RENDER = os.path.join(_PATH_HERE, "_ipynb-render.sh") PATH_SCRIPT_TEST = os.path.join(_PATH_HERE, "_ipynb-validate.sh") From 19ca1a573e15b4f5859c18ac6f650d1afac9c4d4 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:38:16 +0200 Subject: [PATCH 09/31] Apply suggestions from code review --- .actions/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index f4933ce24..0880d1384 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -16,7 +16,7 @@ from pip._internal.operations import freeze from wcmatch import glob -_PATH_HERE = os.path.realpath(os.path.dirname(__file__)) +_PATH_HERE = os.path.dirname(os.path.realpath(__file__)) _PATH_ROOT = os.path.dirname(_PATH_HERE) PATH_SCRIPT_RENDER = os.path.join(_PATH_HERE, "_ipynb-render.sh") PATH_SCRIPT_TEST = os.path.join(_PATH_HERE, "_ipynb-validate.sh") From 30dd7a227fa861907cb88710071f57db5f369d26 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 12:43:04 +0200 Subject: [PATCH 10/31] freeze --- course_UvA-DL/06-graph-neural-networks/requirements.txt | 9 +++++---- lightning_examples/finetuning-scheduler/requirements.txt | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/course_UvA-DL/06-graph-neural-networks/requirements.txt b/course_UvA-DL/06-graph-neural-networks/requirements.txt index 072e876e8..1b27f14d2 100644 --- a/course_UvA-DL/06-graph-neural-networks/requirements.txt +++ b/course_UvA-DL/06-graph-neural-networks/requirements.txt @@ -1,5 +1,6 @@ -torch-scatter -torch-sparse -torch-cluster -torch-spline-conv +torch <2.1 +torch-scatter ==2.1.* +torch-sparse ==0.6.* +torch-cluster ==1.6.* +torch-spline-conv ==1.2.* torch-geometric>=2.0.0,<2.5.0 diff --git a/lightning_examples/finetuning-scheduler/requirements.txt b/lightning_examples/finetuning-scheduler/requirements.txt index 46890c5ae..aa74d49cf 100644 --- a/lightning_examples/finetuning-scheduler/requirements.txt +++ b/lightning_examples/finetuning-scheduler/requirements.txt @@ -1,3 +1,3 @@ -finetuning-scheduler[examples]>=2.0.0 -datasets -torch>=1.12.1 # to avoid https://github.com/pytorch/pytorch/issues/80809 with torch 1.12.0 +finetuning-scheduler[examples] ==2.3.* +datasets ==2.17.* +torch >=2.0.0 From 00b3cdc6c8752f958ee810f75f6c4d1be47e327d Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 13:17:58 +0200 Subject: [PATCH 11/31] lit --- .actions/assistant.py | 7 +++ .../finetuning-scheduler.py | 44 +++++++++---------- .../finetuning-scheduler/requirements.txt | 1 + .../text-transformers/requirements.txt | 10 ++--- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 0880d1384..4786f08b7 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -1,6 +1,7 @@ import base64 import json import os +import platform import re import warnings from datetime import datetime @@ -809,6 +810,12 @@ def update_env_details(folder: str, base_path: str = DIR_NOTEBOOKS) -> str: """ meta = AssistantCLI._load_meta(folder) + meta.update( + { + "OS": platform.system(), + "python": platform.python_version(), + } + ) # load local and parent requirements requires = set(load_requirements(folder) + load_requirements(os.path.dirname(folder))) diff --git a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py index ce4a45c7f..52d79298b 100644 --- a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py +++ b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py @@ -47,9 +47,9 @@ # using default [FTSEarlyStopping](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts_supporters.html#finetuning_scheduler.fts_supporters.FTSEarlyStopping) and [FTSCheckpoint](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts_supporters.html#finetuning_scheduler.fts_supporters.FTSCheckpoint) callbacks with ``monitor=val_loss``. # # ```python -# import pytorch_lightning as pl +# import lightning as L # from finetuning_scheduler import FinetuningScheduler -# trainer = pl.Trainer(callbacks=[FinetuningScheduler()]) +# trainer = L.Trainer(callbacks=[FinetuningScheduler()]) # ``` # %% [markdown] @@ -67,11 +67,11 @@ # 1. First, generate the default schedule to ``Trainer.log_dir``. It will be named after your # ``LightningModule`` subclass with the suffix ``_ft_schedule.yaml``. # -# ```python -# import pytorch_lightning as pl +# ```python +# import lightning as L # from finetuning_scheduler import FinetuningScheduler -# trainer = pl.Trainer(callbacks=[FinetuningScheduler(gen_ft_sched_only=True)]) -# ``` +# trainer = L.Trainer(callbacks=[FinetuningScheduler(gen_ft_sched_only=True)]) +# ``` # # 2. Alter the schedule as desired. # @@ -80,12 +80,12 @@ # 3. Once the fine-tuning schedule has been altered as desired, pass it to # [FinetuningScheduler](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts.html#finetuning_scheduler.fts.FinetuningScheduler) to commence scheduled training: # -# ```python -# import pytorch_lightning as pl -# from finetuning_scheduler import FinetuningScheduler +# ```python +# import lightning as L +# from finetuning_scheduler import FinetuningScheduler # -# trainer = pl.Trainer(callbacks=[FinetuningScheduler(ft_schedule="/path/to/my/schedule/my_schedule.yaml")]) -# ``` +# trainer = L.Trainer(callbacks=[FinetuningScheduler(ft_schedule="/path/to/my/schedule/my_schedule.yaml")]) +# ``` # %% [markdown] # ## Early-Stopping and Epoch-Driven Phase Transition Criteria @@ -119,9 +119,9 @@ # # # ```python -# import pytorch_lightning as pl +# import lightning as L # from finetuning_scheduler import FinetuningScheduler -# trainer = pl.Trainer(callbacks=[FinetuningScheduler()]) +# trainer = L.Trainer(callbacks=[FinetuningScheduler()]) # trainer.ckpt_path="some/path/to/my_checkpoint.ckpt" # trainer.fit(...) # ``` @@ -131,7 +131,7 @@ # By default ([FinetuningScheduler.restore_best](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts.html?highlight=restore_best#finetuning_scheduler.fts.FinetuningScheduler.params.restore_best) is ``True``), [FinetuningScheduler](https://finetuning-scheduler.readthedocs.io/en/stable/api/finetuning_scheduler.fts.html#finetuning_scheduler.fts.FinetuningScheduler) will attempt to restore the best available checkpoint before fine-tuning depth transitions. # # ```python -# trainer = pl.Trainer(callbacks=[FinetuningScheduler()]) +# trainer = L.Trainer(callbacks=[FinetuningScheduler()]) # trainer.ckpt_path="some/path/to/my_kth_best_checkpoint.ckpt" # trainer.fit(...) # ``` @@ -172,13 +172,13 @@ # Import the `FinetuningScheduler` PyTorch Lightning extension module we want to use. This will import all necessary callbacks. import finetuning_scheduler as fts # isort: split -import pytorch_lightning as pl +import lightning as L import sentencepiece as sp # noqa: F401 # isort: split import torch from datasets import logging as datasets_logging -from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint -from pytorch_lightning.loggers.tensorboard import TensorBoardLogger -from pytorch_lightning.utilities import rank_zero_warn +from lightning.pytorch.callbacks import EarlyStopping, ModelCheckpoint +from lightning.pytorch.loggers.tensorboard import TensorBoardLogger +from lightning.pytorch.utilities import rank_zero_warn from torch.optim.adamw import AdamW from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts from torch.utils.data import DataLoader @@ -205,7 +205,7 @@ # %% -class RteBoolqDataModule(pl.LightningDataModule): +class RteBoolqDataModule(L.LightningDataModule): """A ``LightningDataModule`` designed for both the RTE or BoolQ SuperGLUE Hugging Face datasets.""" TASK_TEXT_FIELD_MAP = {"rte": ("premise", "hypothesis"), "boolq": ("question", "passage")} @@ -308,7 +308,7 @@ def _convert_to_features(self, example_batch: datasets.arrow_dataset.LazyDict) - # %% -class RteBoolqModule(pl.LightningModule): +class RteBoolqModule(L.LightningModule): """A ``LightningModule`` that can be used to fine-tune a foundation model on either the RTE or BoolQ SuperGLUE tasks using Hugging Face implementations of a given model and the `SuperGLUE Hugging Face dataset.""" @@ -444,7 +444,7 @@ def configure_optimizers(self): # %% datasets.logging.disable_progress_bar() -pl.seed_everything(42) +L.seed_everything(42) dm = RteBoolqDataModule(model_name_or_path="microsoft/deberta-v3-base", tokenizers_parallelism=True) # %% [markdown] @@ -522,7 +522,7 @@ def configure_optimizers(self): def train() -> None: - trainer = pl.Trainer( + trainer = L.Trainer( enable_progress_bar=enable_progress_bar, max_epochs=100, precision="16-mixed", diff --git a/lightning_examples/finetuning-scheduler/requirements.txt b/lightning_examples/finetuning-scheduler/requirements.txt index aa74d49cf..e1416188b 100644 --- a/lightning_examples/finetuning-scheduler/requirements.txt +++ b/lightning_examples/finetuning-scheduler/requirements.txt @@ -1,3 +1,4 @@ finetuning-scheduler[examples] ==2.3.* datasets ==2.17.* torch >=2.0.0 +lightning>=2.0, <2.1.0 # todo: the tuner depends on L so later using with PL crash diff --git a/lightning_examples/text-transformers/requirements.txt b/lightning_examples/text-transformers/requirements.txt index 5b0d2ded8..0bd6a1fb8 100644 --- a/lightning_examples/text-transformers/requirements.txt +++ b/lightning_examples/text-transformers/requirements.txt @@ -1,7 +1,7 @@ -transformers -datasets scipy scikit-learn -torchtext>=0.9 -lightning>=2.0.0,<2.1.0 -torchmetrics<1.2 +torch ==2.0.* +torchmetrics >=1.0, <1.2 +torchtext ==0.15.* +transformers ==4.43.* +datasets ==2.20.* From 46aec00d90da3d5a9fa4bba7b2d6f26a046efcd7 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 13:41:26 +0200 Subject: [PATCH 12/31] import --- _dockers/ubuntu-cuda/Dockerfile | 5 +++-- lightning_examples/finetuning-scheduler/requirements.txt | 2 +- .../text-transformers/{transformers.py => notebook.py} | 0 lightning_examples/text-transformers/requirements.txt | 7 +++---- 4 files changed, 7 insertions(+), 7 deletions(-) rename lightning_examples/text-transformers/{transformers.py => notebook.py} (100%) diff --git a/_dockers/ubuntu-cuda/Dockerfile b/_dockers/ubuntu-cuda/Dockerfile index 56f4b5bae..7c4c59557 100644 --- a/_dockers/ubuntu-cuda/Dockerfile +++ b/_dockers/ubuntu-cuda/Dockerfile @@ -16,7 +16,7 @@ ARG UBUNTU_VERSION=22.04 ARG CUDA_VERSION=11.8.0 -FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} ARG PYTHON_VERSION=3.10 ARG PYTORCH_VERSION=2.4 @@ -27,10 +27,11 @@ ENV \ DEBIAN_FRONTEND="noninteractive" \ TZ="Etc/UTC" \ PATH="$PATH:/root/.local/bin" \ + CUDA_HOME="/usr/local/cuda" \ CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda" \ MKL_THREADING_LAYER="GNU" \ # MAKEFLAGS="-j$(nproc)" - MAKEFLAGS="-j2" + MAKEFLAGS="-j4" RUN \ apt-get -y update --fix-missing && \ diff --git a/lightning_examples/finetuning-scheduler/requirements.txt b/lightning_examples/finetuning-scheduler/requirements.txt index e1416188b..07054309b 100644 --- a/lightning_examples/finetuning-scheduler/requirements.txt +++ b/lightning_examples/finetuning-scheduler/requirements.txt @@ -1,4 +1,4 @@ +lightning # todo: the tuner depends on L so later using with PL crash finetuning-scheduler[examples] ==2.3.* datasets ==2.17.* torch >=2.0.0 -lightning>=2.0, <2.1.0 # todo: the tuner depends on L so later using with PL crash diff --git a/lightning_examples/text-transformers/transformers.py b/lightning_examples/text-transformers/notebook.py similarity index 100% rename from lightning_examples/text-transformers/transformers.py rename to lightning_examples/text-transformers/notebook.py diff --git a/lightning_examples/text-transformers/requirements.txt b/lightning_examples/text-transformers/requirements.txt index 0bd6a1fb8..689422379 100644 --- a/lightning_examples/text-transformers/requirements.txt +++ b/lightning_examples/text-transformers/requirements.txt @@ -1,7 +1,6 @@ scipy scikit-learn -torch ==2.0.* torchmetrics >=1.0, <1.2 -torchtext ==0.15.* -transformers ==4.43.* -datasets ==2.20.* +torchtext +transformers +datasets From 9df6b2ee3d0cfb0055361768583ec649f32c568e Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 14:41:30 +0200 Subject: [PATCH 13/31] push nvcc --- .github/workflows/docker-build.yml | 2 +- _dockers/ubuntu-cuda/Dockerfile | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 649574c50..0e3783cb7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: - PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} + PUSH_DOCKERHUB: true # ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} jobs: build-cuda: diff --git a/_dockers/ubuntu-cuda/Dockerfile b/_dockers/ubuntu-cuda/Dockerfile index 7c4c59557..e00bfb209 100644 --- a/_dockers/ubuntu-cuda/Dockerfile +++ b/_dockers/ubuntu-cuda/Dockerfile @@ -39,6 +39,7 @@ RUN \ build-essential \ ca-certificates \ software-properties-common \ + nvidia-cuda-toolkit \ pkg-config \ libopenmpi-dev \ openmpi-bin \ @@ -88,5 +89,6 @@ RUN \ # Show what we have pip --version && \ pip list && \ + nvcc --version && \ python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \ python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__" From be4baf82a41dcb5a791ee55226eff2c6efaa61b0 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 14:50:06 +0200 Subject: [PATCH 14/31] use --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0e3783cb7..649574c50 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: - PUSH_DOCKERHUB: true # ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} + PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} jobs: build-cuda: From 2da490f0611e17c993bf9debaadf5f538b8f6fa9 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 15:24:54 +0200 Subject: [PATCH 15/31] cuda 12.1 --- .github/workflows/docker-build.yml | 2 +- _dockers/ubuntu-cuda/Dockerfile | 4 ++-- _requirements/devel.txt | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 649574c50..0e3783cb7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: - PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} + PUSH_DOCKERHUB: true # ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} jobs: build-cuda: diff --git a/_dockers/ubuntu-cuda/Dockerfile b/_dockers/ubuntu-cuda/Dockerfile index e00bfb209..95da2df55 100644 --- a/_dockers/ubuntu-cuda/Dockerfile +++ b/_dockers/ubuntu-cuda/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. ARG UBUNTU_VERSION=22.04 -ARG CUDA_VERSION=11.8.0 +ARG CUDA_VERSION=12.1.1 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} @@ -82,7 +82,7 @@ RUN \ CUDA_VERSION_MM=${CUDA_VERSION%.*} && \ CU_VERSION_MM=${CUDA_VERSION_MM//'.'/''} && \ pip install --no-cache-dir -r requirements.txt \ - --find-links "https://download.pytorch.org/whl/cu${CU_VERSION_MM}/torch_stable.html" && \ + --extra-index-url="https://download.pytorch.org/whl/cu${CU_VERSION_MM}" && \ rm -rf requirements.txt .actions/ _requirements/ RUN \ diff --git a/_requirements/devel.txt b/_requirements/devel.txt index 0dfa5eab3..1ee1b5839 100644 --- a/_requirements/devel.txt +++ b/_requirements/devel.txt @@ -11,6 +11,9 @@ papermill>=2.3.4, <2.5.0 # rendering matplotlib # all is based on PyTorch torch +torchaudio +torchtext +torchvision # fixed version to be able to call it as `python -m kaggle` https://github.com/Borda/kaggle-api/archive/refs/heads/setup/python-m.zip From ef560359d3c48987a6022b7fdea6d2d53e3d99a7 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 16:12:05 +0200 Subject: [PATCH 16/31] use --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0e3783cb7..649574c50 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: - PUSH_DOCKERHUB: true # ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} + PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} jobs: build-cuda: From b99ea338a6016c814476ba11fe8e54f5ebc38390 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 16:23:48 +0200 Subject: [PATCH 17/31] maxParallel: "10" --- .azure/ipynb-validate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure/ipynb-validate.yml b/.azure/ipynb-validate.yml index 7e3d5992c..8f9f3c081 100644 --- a/.azure/ipynb-validate.yml +++ b/.azure/ipynb-validate.yml @@ -47,6 +47,8 @@ jobs: condition: and(succeeded(), ne(dependencies.check_diff.outputs['mtrx.dirs'], '')) strategy: matrix: $[ dependencies.check_diff.outputs['mtrx.dirs'] ] + # Maximum number of jobs running in parallel + maxParallel: "10" # how long to run the job before automatically cancelling timeoutInMinutes: "95" # how much time to give 'run always even if cancelled tasks' before stopping them From 21e6b1de42305dc48f0e15840a5af6ce895c529f Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 16:49:05 +0200 Subject: [PATCH 18/31] audio --- .github/workflows/docker-build.yml | 2 +- _requirements/devel.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 649574c50..0e3783cb7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: - PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} + PUSH_DOCKERHUB: true # ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} jobs: build-cuda: diff --git a/_requirements/devel.txt b/_requirements/devel.txt index 1ee1b5839..0feba5a68 100644 --- a/_requirements/devel.txt +++ b/_requirements/devel.txt @@ -11,7 +11,6 @@ papermill>=2.3.4, <2.5.0 # rendering matplotlib # all is based on PyTorch torch -torchaudio torchtext torchvision From e0072b4ba56dd1f08a0682e4559790058b9c6b60 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 16:58:28 +0200 Subject: [PATCH 19/31] use --- .github/workflows/docker-build.yml | 2 +- _requirements/devel.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 0e3783cb7..649574c50 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: - PUSH_DOCKERHUB: true # ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} + PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name != 'pull_request' }} jobs: build-cuda: diff --git a/_requirements/devel.txt b/_requirements/devel.txt index 0feba5a68..7155466d1 100644 --- a/_requirements/devel.txt +++ b/_requirements/devel.txt @@ -11,7 +11,6 @@ papermill>=2.3.4, <2.5.0 # rendering matplotlib # all is based on PyTorch torch -torchtext torchvision # fixed version to be able to call it as `python -m kaggle` From 7b1a95f2972ef1dbf90b45a76db61c69c71b7425 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 17:23:40 +0200 Subject: [PATCH 20/31] torch --- course_UvA-DL/requirements.txt | 2 +- lightning_examples/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/course_UvA-DL/requirements.txt b/course_UvA-DL/requirements.txt index 54a071e28..7b4749c59 100644 --- a/course_UvA-DL/requirements.txt +++ b/course_UvA-DL/requirements.txt @@ -1,6 +1,6 @@ numpy <2.0 # needed for older Torch torch >=1.8.1,<2.1.0 -pytorch-lightning >=2.0,<2.1.0 +pytorch-lightning >=2.0,<2.2 torchmetrics >=1.0,<1.3 torchvision matplotlib diff --git a/lightning_examples/requirements.txt b/lightning_examples/requirements.txt index e1804ec0c..4f06358dc 100644 --- a/lightning_examples/requirements.txt +++ b/lightning_examples/requirements.txt @@ -1,5 +1,5 @@ numpy <2.0 # needed for older Torch -torch>=1.8.1, <2.1.0 -pytorch-lightning>=2.0, <2.1.0 +torch>=1.8.1, <2.2 +pytorch-lightning>=2.0, <2.2 torchmetrics>=1.0, <1.3 matplotlib From 48e8e82e56b7ffec52e23badc1afbf909d6a3005 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 18:14:36 +0200 Subject: [PATCH 21/31] pin --- lightning_examples/finetuning-scheduler/requirements.txt | 4 +++- lightning_examples/reinforce-learning-DQN/requirements.txt | 3 +++ lightning_examples/text-transformers/requirements.txt | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lightning_examples/finetuning-scheduler/requirements.txt b/lightning_examples/finetuning-scheduler/requirements.txt index 07054309b..31dd30277 100644 --- a/lightning_examples/finetuning-scheduler/requirements.txt +++ b/lightning_examples/finetuning-scheduler/requirements.txt @@ -1,4 +1,6 @@ lightning # todo: the tuner depends on L so later using with PL crash finetuning-scheduler[examples] ==2.3.* datasets ==2.17.* -torch >=2.0.0 +# todo: pin version intill reinstall with PT-eco alignement +torch ==2.1.* +torchvision ==0.16.* diff --git a/lightning_examples/reinforce-learning-DQN/requirements.txt b/lightning_examples/reinforce-learning-DQN/requirements.txt index 0a593add8..f03a2e73c 100644 --- a/lightning_examples/reinforce-learning-DQN/requirements.txt +++ b/lightning_examples/reinforce-learning-DQN/requirements.txt @@ -2,3 +2,6 @@ gym <0.24 pygame pandas seaborn +# todo: pin version intill reinstall with PT-eco alignement +torch ==2.1.* +torchvision ==0.16.* diff --git a/lightning_examples/text-transformers/requirements.txt b/lightning_examples/text-transformers/requirements.txt index 689422379..e71fe5deb 100644 --- a/lightning_examples/text-transformers/requirements.txt +++ b/lightning_examples/text-transformers/requirements.txt @@ -4,3 +4,6 @@ torchmetrics >=1.0, <1.2 torchtext transformers datasets +# todo: pin version intill reinstall with PT-eco alignement +torch ==2.1.* +torchvision ==0.16.* \ No newline at end of file From eb7fe8527832d2091bd328457c2404a1f5585b6c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 16:14:58 +0000 Subject: [PATCH 22/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- lightning_examples/text-transformers/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightning_examples/text-transformers/requirements.txt b/lightning_examples/text-transformers/requirements.txt index e71fe5deb..c644efd04 100644 --- a/lightning_examples/text-transformers/requirements.txt +++ b/lightning_examples/text-transformers/requirements.txt @@ -6,4 +6,4 @@ transformers datasets # todo: pin version intill reinstall with PT-eco alignement torch ==2.1.* -torchvision ==0.16.* \ No newline at end of file +torchvision ==0.16.* From edae5f040c6b18b0056fc2fdcc0c1e31d71cba82 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 19:51:11 +0200 Subject: [PATCH 23/31] base --- _requirements/base.txt | 7 +++++++ _requirements/devel.txt | 5 ----- course_UvA-DL/06-graph-neural-networks/requirements.txt | 2 +- lightning_examples/finetuning-scheduler/requirements.txt | 2 +- lightning_examples/reinforce-learning-DQN/requirements.txt | 2 +- lightning_examples/text-transformers/requirements.txt | 4 ++-- requirements.txt | 1 + 7 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 _requirements/base.txt diff --git a/_requirements/base.txt b/_requirements/base.txt new file mode 100644 index 000000000..266bd275e --- /dev/null +++ b/_requirements/base.txt @@ -0,0 +1,7 @@ +matplotlib +# all is based on PyTorch +torch +torchvision + +pycuda +numba diff --git a/_requirements/devel.txt b/_requirements/devel.txt index 7155466d1..30ddd76ae 100644 --- a/_requirements/devel.txt +++ b/_requirements/devel.txt @@ -8,10 +8,5 @@ urllib3 # for ipython jupytext>=1.10, <1.15 # converting papermill>=2.3.4, <2.5.0 # rendering -matplotlib -# all is based on PyTorch -torch -torchvision - # fixed version to be able to call it as `python -m kaggle` https://github.com/Borda/kaggle-api/archive/refs/heads/setup/python-m.zip diff --git a/course_UvA-DL/06-graph-neural-networks/requirements.txt b/course_UvA-DL/06-graph-neural-networks/requirements.txt index 1b27f14d2..45c4f031c 100644 --- a/course_UvA-DL/06-graph-neural-networks/requirements.txt +++ b/course_UvA-DL/06-graph-neural-networks/requirements.txt @@ -1,4 +1,4 @@ -torch <2.1 +torch ==2.1.* torch-scatter ==2.1.* torch-sparse ==0.6.* torch-cluster ==1.6.* diff --git a/lightning_examples/finetuning-scheduler/requirements.txt b/lightning_examples/finetuning-scheduler/requirements.txt index 31dd30277..109e98ef6 100644 --- a/lightning_examples/finetuning-scheduler/requirements.txt +++ b/lightning_examples/finetuning-scheduler/requirements.txt @@ -1,6 +1,6 @@ lightning # todo: the tuner depends on L so later using with PL crash finetuning-scheduler[examples] ==2.3.* datasets ==2.17.* -# todo: pin version intill reinstall with PT-eco alignement +# todo: pin version intill reinstall with PT-eco alignment torch ==2.1.* torchvision ==0.16.* diff --git a/lightning_examples/reinforce-learning-DQN/requirements.txt b/lightning_examples/reinforce-learning-DQN/requirements.txt index f03a2e73c..9dc9c9be6 100644 --- a/lightning_examples/reinforce-learning-DQN/requirements.txt +++ b/lightning_examples/reinforce-learning-DQN/requirements.txt @@ -2,6 +2,6 @@ gym <0.24 pygame pandas seaborn -# todo: pin version intill reinstall with PT-eco alignement +# todo: pin version intill reinstall with PT-eco alignment torch ==2.1.* torchvision ==0.16.* diff --git a/lightning_examples/text-transformers/requirements.txt b/lightning_examples/text-transformers/requirements.txt index e71fe5deb..1a5997e27 100644 --- a/lightning_examples/text-transformers/requirements.txt +++ b/lightning_examples/text-transformers/requirements.txt @@ -4,6 +4,6 @@ torchmetrics >=1.0, <1.2 torchtext transformers datasets -# todo: pin version intill reinstall with PT-eco alignement +# todo: pin version intill reinstall with PT-eco alignment torch ==2.1.* -torchvision ==0.16.* \ No newline at end of file +torchvision ==0.16.* diff --git a/requirements.txt b/requirements.txt index 4db5e3fed..d0879eaf4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +-r _requirements/base.txt -r _requirements/devel.txt -r .actions/requires.txt From 5704880394b870db855657e8fe0cde3148184570 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 22:11:02 +0200 Subject: [PATCH 24/31] aggreg base --- .actions/assistant.py | 34 +++++++++++++++++++++++++++------ _dockers/ubuntu-cuda/Dockerfile | 12 +++++++----- _requirements/base.txt | 7 ------- _requirements/devel.txt | 2 ++ requirements.txt | 1 - 5 files changed, 37 insertions(+), 19 deletions(-) delete mode 100644 _requirements/base.txt diff --git a/.actions/assistant.py b/.actions/assistant.py index 4786f08b7..66fc78483 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -1,4 +1,5 @@ import base64 +import itertools import json import os import platform @@ -95,19 +96,21 @@ """ -def load_requirements(folder: str, fname: str = "requirements.txt") -> List[str]: +def load_requirements(folder: str = ".", fname: str = "requirements.txt", full_path: str = "") -> List[str]: """Load the requirements from a file. Args: folder: path to the folder with requirements - fname: filename + fname: filename in the given folder + full_path: alternative and overwrite the composition of folder + fname """ - path_req = os.path.join(folder, fname) - if not os.path.isfile(path_req): - warnings.warn(f"Missing expected requirement file '{path_req}'") + if not full_path: + full_path = os.path.join(folder, fname) + if not os.path.isfile(full_path): + warnings.warn(f"Missing expected requirement file '{full_path}'") return [] - with open(path_req) as fopen: + with open(full_path) as fopen: req = fopen.readlines() req = [r[: r.index("#")] if "#" in r else r for r in req] req = [r.strip() for r in req] @@ -849,6 +852,25 @@ def list_dirs(folder: str = "", include_file_ext: str = "") -> str: dirs = [p for p in dirs if os.path.isdir(p)] return os.linesep.join(sorted(dirs)) + @staticmethod + def aggregate_requirements(req_pattern: str, result_file: str) -> None: + """Load all requirements from given path pattern and dump them to a single file. + + Args: + req_pattern: search pattern such as '*/requirements.txt' + result_file: path to save aggregated requirements + + """ + ls_requirements = glob.glob(req_pattern) + assert len(ls_requirements) > 0, "no requirements found" + # load and append all particular requirements + reqs = [load_requirements(full_path=fp) for fp in ls_requirements] + # try to filer only unique + reqs = set([r.replace(" ", "") for r in itertools.chain(*reqs)]) + # dump result to a file + with open(result_file, "w") as fopen: + fopen.writelines(os.linesep.join(reqs)) + if __name__ == "__main__": fire.Fire(AssistantCLI) diff --git a/_dockers/ubuntu-cuda/Dockerfile b/_dockers/ubuntu-cuda/Dockerfile index 95da2df55..cb891d92b 100644 --- a/_dockers/ubuntu-cuda/Dockerfile +++ b/_dockers/ubuntu-cuda/Dockerfile @@ -74,16 +74,18 @@ RUN \ ENV PYTHONPATH="/usr/lib/python${PYTHON_VERSION}/site-packages" -COPY .actions/ .actions/ -COPY requirements.txt . -COPY _requirements/ _requirements/ +COPY ./ repo/ RUN \ CUDA_VERSION_MM=${CUDA_VERSION%.*} && \ CU_VERSION_MM=${CUDA_VERSION_MM//'.'/''} && \ - pip install --no-cache-dir -r requirements.txt \ + cd repo && \ + python .actions/assistant.py aggregate-requirements "./*/requirements.txt" "./_requirements/base.txt" && \ + cat _requirements/base.txt && \ + pip install --no-cache-dir pycuda numba -r requirements.txt -r _requirements/base.txt \ --extra-index-url="https://download.pytorch.org/whl/cu${CU_VERSION_MM}" && \ - rm -rf requirements.txt .actions/ _requirements/ + cd .. && \ + rm -rf repo RUN \ # Show what we have diff --git a/_requirements/base.txt b/_requirements/base.txt deleted file mode 100644 index 266bd275e..000000000 --- a/_requirements/base.txt +++ /dev/null @@ -1,7 +0,0 @@ -matplotlib -# all is based on PyTorch -torch -torchvision - -pycuda -numba diff --git a/_requirements/devel.txt b/_requirements/devel.txt index 30ddd76ae..51cddf7c0 100644 --- a/_requirements/devel.txt +++ b/_requirements/devel.txt @@ -8,5 +8,7 @@ urllib3 # for ipython jupytext>=1.10, <1.15 # converting papermill>=2.3.4, <2.5.0 # rendering +matplotlib + # fixed version to be able to call it as `python -m kaggle` https://github.com/Borda/kaggle-api/archive/refs/heads/setup/python-m.zip diff --git a/requirements.txt b/requirements.txt index d0879eaf4..4db5e3fed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ --r _requirements/base.txt -r _requirements/devel.txt -r .actions/requires.txt From ec9c3fcffecfcc03f0d65589f06759a8f123e4f6 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 22:16:26 +0200 Subject: [PATCH 25/31] REQ --- course_UvA-DL/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/course_UvA-DL/requirements.txt b/course_UvA-DL/requirements.txt index 7b4749c59..cc195f481 100644 --- a/course_UvA-DL/requirements.txt +++ b/course_UvA-DL/requirements.txt @@ -1,6 +1,6 @@ numpy <2.0 # needed for older Torch -torch >=1.8.1,<2.1.0 -pytorch-lightning >=2.0,<2.2 +torch >=1.8.1,<2.2 +pytorch-lightning >=2.0,<2.1 torchmetrics >=1.0,<1.3 torchvision matplotlib From a5d685375cf8046f95885f9d8775bf92334aa4dc Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 22:18:29 +0200 Subject: [PATCH 26/31] dosker --- _dockers/ubuntu-cuda/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/_dockers/ubuntu-cuda/Dockerfile b/_dockers/ubuntu-cuda/Dockerfile index cb891d92b..21bb63f02 100644 --- a/_dockers/ubuntu-cuda/Dockerfile +++ b/_dockers/ubuntu-cuda/Dockerfile @@ -80,9 +80,10 @@ RUN \ CUDA_VERSION_MM=${CUDA_VERSION%.*} && \ CU_VERSION_MM=${CUDA_VERSION_MM//'.'/''} && \ cd repo && \ + pip install --no-cache-dir -r requirements.txt && \ python .actions/assistant.py aggregate-requirements "./*/requirements.txt" "./_requirements/base.txt" && \ cat _requirements/base.txt && \ - pip install --no-cache-dir pycuda numba -r requirements.txt -r _requirements/base.txt \ + pip install --no-cache-dir pycuda numba-r _requirements/base.txt \ --extra-index-url="https://download.pytorch.org/whl/cu${CU_VERSION_MM}" && \ cd .. && \ rm -rf repo From 789356e6c1f623cdadbebf09b71ba7a420ef613c Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 22:23:58 +0200 Subject: [PATCH 27/31] typo --- _dockers/ubuntu-cuda/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_dockers/ubuntu-cuda/Dockerfile b/_dockers/ubuntu-cuda/Dockerfile index 21bb63f02..550950438 100644 --- a/_dockers/ubuntu-cuda/Dockerfile +++ b/_dockers/ubuntu-cuda/Dockerfile @@ -83,7 +83,7 @@ RUN \ pip install --no-cache-dir -r requirements.txt && \ python .actions/assistant.py aggregate-requirements "./*/requirements.txt" "./_requirements/base.txt" && \ cat _requirements/base.txt && \ - pip install --no-cache-dir pycuda numba-r _requirements/base.txt \ + pip install pycuda numba --no-cache-dir -r _requirements/base.txt \ --extra-index-url="https://download.pytorch.org/whl/cu${CU_VERSION_MM}" && \ cd .. && \ rm -rf repo From 48fd073541b27784f6b6768eb4fd1716af007737 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 22:30:54 +0200 Subject: [PATCH 28/31] assert --- _dockers/ubuntu-cuda/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/_dockers/ubuntu-cuda/Dockerfile b/_dockers/ubuntu-cuda/Dockerfile index 550950438..9ed6edcd3 100644 --- a/_dockers/ubuntu-cuda/Dockerfile +++ b/_dockers/ubuntu-cuda/Dockerfile @@ -19,7 +19,6 @@ ARG CUDA_VERSION=12.1.1 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} ARG PYTHON_VERSION=3.10 -ARG PYTORCH_VERSION=2.4 SHELL ["/bin/bash", "-c"] # https://techoverflow.net/2019/05/18/how-to-fix-configuring-tzdata-interactive-input-when-building-docker-images/ @@ -94,4 +93,4 @@ RUN \ pip list && \ nvcc --version && \ python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \ - python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__" + python -c "import torch; assert torch.__version__.startswith('2.'), torch.__version__" From 4a52ef30e831f42e984c36b58a4df90c488147c5 Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 25 Jul 2024 22:49:54 +0200 Subject: [PATCH 29/31] value_range --- .../07-deep-energy-based-generative-models/notebook.py | 8 ++++---- course_UvA-DL/08-deep-autoencoders/notebook.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py b/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py index 989f80b6c..4ee5c260d 100644 --- a/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py +++ b/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py @@ -568,7 +568,7 @@ def on_epoch_end(self, trainer, pl_module): step_size = self.num_steps // self.vis_steps imgs_to_plot = imgs_per_step[step_size - 1 :: step_size, i] grid = torchvision.utils.make_grid( - imgs_to_plot, nrow=imgs_to_plot.shape[0], normalize=True, range=(-1, 1) + imgs_to_plot, nrow=imgs_to_plot.shape[0], normalize=True, value_range=(-1, 1) ) trainer.logger.experiment.add_image("generation_%i" % i, grid, global_step=trainer.current_epoch) @@ -600,7 +600,7 @@ def __init__(self, num_imgs=32, every_n_epochs=5): def on_epoch_end(self, trainer, pl_module): if trainer.current_epoch % self.every_n_epochs == 0: exmp_imgs = torch.cat(random.choices(pl_module.sampler.examples, k=self.num_imgs), dim=0) - grid = torchvision.utils.make_grid(exmp_imgs, nrow=4, normalize=True, range=(-1, 1)) + grid = torchvision.utils.make_grid(exmp_imgs, nrow=4, normalize=True, value_range=(-1, 1)) trainer.logger.experiment.add_image("sampler", grid, global_step=trainer.current_epoch) @@ -724,7 +724,7 @@ def train_model(**kwargs): imgs_to_plot = imgs_per_step[step_size - 1 :: step_size, i] imgs_to_plot = torch.cat([imgs_per_step[0:1, i], imgs_to_plot], dim=0) grid = torchvision.utils.make_grid( - imgs_to_plot, nrow=imgs_to_plot.shape[0], normalize=True, range=(-1, 1), pad_value=0.5, padding=2 + imgs_to_plot, nrow=imgs_to_plot.shape[0], normalize=True, value_range=(-1, 1), pad_value=0.5, padding=2 ) grid = grid.permute(1, 2, 0) plt.figure(figsize=(8, 8)) @@ -795,7 +795,7 @@ def compare_images(img1, img2): imgs = torch.stack([img1, img2], dim=0).to(model.device) score1, score2 = model.cnn(imgs).cpu().chunk(2, dim=0) grid = torchvision.utils.make_grid( - [img1.cpu(), img2.cpu()], nrow=2, normalize=True, range=(-1, 1), pad_value=0.5, padding=2 + [img1.cpu(), img2.cpu()], nrow=2, normalize=True, value_range=(-1, 1), pad_value=0.5, padding=2 ) grid = grid.permute(1, 2, 0) plt.figure(figsize=(4, 4)) diff --git a/course_UvA-DL/08-deep-autoencoders/notebook.py b/course_UvA-DL/08-deep-autoencoders/notebook.py index 44c468be4..499a9c559 100644 --- a/course_UvA-DL/08-deep-autoencoders/notebook.py +++ b/course_UvA-DL/08-deep-autoencoders/notebook.py @@ -321,7 +321,7 @@ def compare_imgs(img1, img2, title_prefix=""): # Calculate MSE loss between both images loss = F.mse_loss(img1, img2, reduction="sum") # Plot images for visual comparison - grid = torchvision.utils.make_grid(torch.stack([img1, img2], dim=0), nrow=2, normalize=True, range=(-1, 1)) + grid = torchvision.utils.make_grid(torch.stack([img1, img2], dim=0), nrow=2, normalize=True, value_range=(-1, 1)) grid = grid.permute(1, 2, 0) plt.figure(figsize=(4, 2)) plt.title(f"{title_prefix} Loss: {loss.item():4.2f}") @@ -373,7 +373,7 @@ def on_train_epoch_end(self, trainer, pl_module): pl_module.train() # Plot and add to tensorboard imgs = torch.stack([input_imgs, reconst_imgs], dim=1).flatten(0, 1) - grid = torchvision.utils.make_grid(imgs, nrow=2, normalize=True, range=(-1, 1)) + grid = torchvision.utils.make_grid(imgs, nrow=2, normalize=True, value_range=(-1, 1)) trainer.logger.experiment.add_image("Reconstructions", grid, global_step=trainer.global_step) @@ -472,7 +472,7 @@ def visualize_reconstructions(model, input_imgs): # Plotting imgs = torch.stack([input_imgs, reconst_imgs], dim=1).flatten(0, 1) - grid = torchvision.utils.make_grid(imgs, nrow=4, normalize=True, range=(-1, 1)) + grid = torchvision.utils.make_grid(imgs, nrow=4, normalize=True, value_range=(-1, 1)) grid = grid.permute(1, 2, 0) plt.figure(figsize=(7, 4.5)) plt.title("Reconstructed from %i latents" % (model.hparams.latent_dim)) @@ -557,7 +557,7 @@ def visualize_reconstructions(model, input_imgs): imgs = model.decoder(latent_vectors) imgs = imgs.cpu() -grid = torchvision.utils.make_grid(imgs, nrow=4, normalize=True, range=(-1, 1), pad_value=0.5) +grid = torchvision.utils.make_grid(imgs, nrow=4, normalize=True, value_range=(-1, 1), pad_value=0.5) grid = grid.permute(1, 2, 0) plt.figure(figsize=(8, 5)) plt.imshow(grid) @@ -617,7 +617,7 @@ def find_similar_images(query_img, query_z, key_embeds, K=8): dist, indices = torch.sort(dist) # Plot K closest images imgs_to_display = torch.cat([query_img[None], key_embeds[0][indices[:K]]], dim=0) - grid = torchvision.utils.make_grid(imgs_to_display, nrow=K + 1, normalize=True, range=(-1, 1)) + grid = torchvision.utils.make_grid(imgs_to_display, nrow=K + 1, normalize=True, value_range=(-1, 1)) grid = grid.permute(1, 2, 0) plt.figure(figsize=(12, 3)) plt.imshow(grid) From 5ddbd7cab9771a39f3f0c4ab4dfdaeed250e4efe Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 26 Jul 2024 01:06:16 +0200 Subject: [PATCH 30/31] torch-geometric --- course_UvA-DL/06-graph-neural-networks/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/course_UvA-DL/06-graph-neural-networks/requirements.txt b/course_UvA-DL/06-graph-neural-networks/requirements.txt index 45c4f031c..542bf36a6 100644 --- a/course_UvA-DL/06-graph-neural-networks/requirements.txt +++ b/course_UvA-DL/06-graph-neural-networks/requirements.txt @@ -3,4 +3,4 @@ torch-scatter ==2.1.* torch-sparse ==0.6.* torch-cluster ==1.6.* torch-spline-conv ==1.2.* -torch-geometric>=2.0.0,<2.5.0 +torch-geometric ==2.1.* From 4cdb54aeb85b3a1c1566552eb109fd30fb274029 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 26 Jul 2024 07:49:53 +0200 Subject: [PATCH 31/31] binaries --- course_UvA-DL/06-graph-neural-networks/.meta.yml | 3 ++- course_UvA-DL/06-graph-neural-networks/requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/course_UvA-DL/06-graph-neural-networks/.meta.yml b/course_UvA-DL/06-graph-neural-networks/.meta.yml index 4bb21b23e..c3075597b 100644 --- a/course_UvA-DL/06-graph-neural-networks/.meta.yml +++ b/course_UvA-DL/06-graph-neural-networks/.meta.yml @@ -17,6 +17,7 @@ description: | The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io. pip__find-link: # - https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html - - https://pytorch-geometric.com/whl/torch-%(TORCH_MAJOR_DOT_MINOR)s.0+%(DEVICE)s.html + # - https://pytorch-geometric.com/whl/torch-%(TORCH_MAJOR_DOT_MINOR)s.0+%(DEVICE)s.html + - https://pytorch-geometric.com/whl/torch-2.1.2+%(DEVICE)s.html accelerator: - GPU diff --git a/course_UvA-DL/06-graph-neural-networks/requirements.txt b/course_UvA-DL/06-graph-neural-networks/requirements.txt index 542bf36a6..1c8316b8d 100644 --- a/course_UvA-DL/06-graph-neural-networks/requirements.txt +++ b/course_UvA-DL/06-graph-neural-networks/requirements.txt @@ -1,4 +1,5 @@ -torch ==2.1.* +# when this is updated you also shall update `pip__find-link` in metadata +torch ==2.1.2 # pined to be able to load binaries, see: https://github.com/rusty1s/pytorch_sparse#binaries torch-scatter ==2.1.* torch-sparse ==0.6.* torch-cluster ==1.6.*