ci: rebuild all notebooks in key req. | assistant changed ++ make default res.

.actions/assistant.py

@@ -16,10 +16,12 @@
 from pip._internal.operations import freeze
 from wcmatch import glob
 
-_PATH_HERE = os.path.dirname(__file__)
+_PATH_HERE = os.path.realpath(os.path.dirname(__file__))
 _PATH_ROOT = os.path.dirname(_PATH_HERE)
 PATH_SCRIPT_RENDER = os.path.join(_PATH_HERE, "_ipynb-render.sh")
 PATH_SCRIPT_TEST = os.path.join(_PATH_HERE, "_ipynb-validate.sh")
+PATH_DEV_SCRIPT = os.path.join(".actions", "assistant.py")
+PATH_DEV_REQUIREMENTS = os.path.join("_requirements", "devel.txt")
 # https://askubuntu.com/questions/909918/how-to-show-unzip-progress
 UNZIP_PROGRESS_BAR = ' | awk \'BEGIN {ORS=" "} {if(NR%10==0)print "."}\''
 REPO_NAME = "lightning-tutorials"
@@ -558,14 +560,24 @@ def group_folders(
                 --fpath_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']"
 
         """
+        dirs = []
+        # replace here notation with path, so later it does not fall to ignore
+        if root_path == ".":
+            root_path = os.getcwd()
+        # loading the generated changes with git diff
         with open(fpath_gitdiff) as fopen:
             changed = [ln.strip() for ln in fopen.readlines()]
+        # in case of ay key component changed, later rebuild all notebooks
+        rebuild_all = any(
+            p_key in p_changed for p_key in (PATH_DEV_SCRIPT, PATH_DEV_REQUIREMENTS) for p_changed in changed
+        )
         dirs_changed = [os.path.dirname(ln) for ln in changed]
+        if rebuild_all:
+            dirs_changed += [p for p in os.listdir(_PATH_ROOT) if os.path.isdir(p) and p[0] not in (".", "_")]
         # append a path to root in case you call this from other path then root
         if root_path:
             dirs_changed = [os.path.join(root_path, d) for d in dirs_changed]
         # append all subfolders in case of parent requirements has been changed all related notebooks shall be updated
-        dirs = []
         for dir in dirs_changed:
             # in case that the diff item comes from removed folder
             if not os.path.isdir(dir):

.azure/ipynb-publish.yml

@@ -145,7 +145,7 @@ jobs:
           pip --version
           # todo: export requirements for notebooks to file and execute
           # todo: adjust torch ecosystem versions
-          pip install -r requirements.txt -r _requirements/data.txt
+          pip install -r requirements.txt
         displayName: "Install dependencies"
         timeoutInMinutes: "15"
 

.azure/ipynb-validate.yml

@@ -82,7 +82,7 @@ jobs:
       - bash: |
           set -e
           pip --version
-          pip install -r requirements.txt -r _requirements/data.txt
+          pip install -r requirements.txt
           pip list
         displayName: "Install dependencies"
 

README.md

@@ -44,7 +44,7 @@ The addition has to formed as new folder:
   accelerator:
     - CPU
   ```
-- requirements listed in `requirements.txt` in the particular folder (in case you need some other packaged then listed the parent folder)
+- _\[optional\]_ requirements listed in `requirements.txt` in the particular folder (in case you need some other packaged then listed the parent folder)
 
 ## Using datasets
 

_requirements/devel.txt

@@ -11,3 +11,6 @@ papermill>=2.3.4, <2.5.0  # rendering
 matplotlib
 # all is based on PyTorch
 torch
+
+# fixed version to be able to call it as `python -m kaggle`
+https://github.com/Borda/kaggle-api/archive/refs/heads/setup/python-m.zip

course_UvA-DL/04-inception-resnet-densenet/requirements.txt

@@ -1,4 +1,2 @@
-torchvision
-seaborn
 tabulate
 tensorboard

course_UvA-DL/07-deep-energy-based-generative-models/requirements.txt

@@ -1,3 +1 @@
-torchvision
-matplotlib
 tensorboard

course_UvA-DL/08-deep-autoencoders/requirements.txt

@@ -1,3 +1 @@
-torchvision
-seaborn
 tensorboard

course_UvA-DL/09-normalizing-flows/requirements.txt

@@ -1,3 +1 @@
-torchvision
-seaborn
 tabulate

course_UvA-DL/11-vision-transformer/requirements.txt

@@ -1,3 +1 @@
-torchvision
-seaborn
 tensorboard

course_UvA-DL/12-meta-learning/requirements.txt

@@ -1,4 +1,2 @@
-torchvision
-seaborn
 tensorboard
 scipy

course_UvA-DL/13-contrastive-learning/requirements.txt

@@ -1,3 +1 @@
-torchvision
-seaborn
 tensorboard

course_UvA-DL/requirements.txt

@@ -2,4 +2,6 @@ numpy <2.0 # needed for older Torch
 torch >=1.8.1,<2.1.0
 pytorch-lightning >=2.0,<2.1.0
 torchmetrics >=1.0,<1.3
+torchvision
 matplotlib
+seaborn