From 5a2884b2d466c7948ba0043cea773d0584dcde11 Mon Sep 17 00:00:00 2001 From: KanaiYuma-aist <105629713+KanaiYuma-aist@users.noreply.github.com> Date: Mon, 27 Jan 2025 10:34:21 +0900 Subject: [PATCH] Resolved warnings during document generation. (#408) * Resolve warning. * Resolve warning. * Add docstrfmt. * Format by docstrfmt. * Remove -s inc . * Remove search-field . * Fix for docstrfmt. --------- Co-authored-by: Yoshiaki Bando --- .github/workflows/lint.yaml | 3 + docs/source/api_reference/hpo.rst | 45 ++-- docs/source/api_reference/index.rst | 13 +- docs/source/api_reference/torch.rst | 28 +-- docs/source/api_reference/utils.rst | 8 +- docs/source/conf.py | 3 +- docs/source/contribution_guide/index.rst | 9 +- docs/source/contribution_guide/issues.md | 4 +- .../contribution_guide/pull_requests.md | 17 +- docs/source/index.rst | 69 +++---- docs/source/user_guide/hpo.rst | 192 ++++++++++-------- docs/source/user_guide/index.rst | 10 +- docs/source/user_guide/torch.rst | 2 - setup.py | 1 + 14 files changed, 211 insertions(+), 193 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 348c520ef..beb58aeb7 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -29,3 +29,6 @@ jobs: - name: Perform mypy run: | mypy --config-file mypy.ini . + - name: Perform docstrfmt + run: | + docstrfmt --check docs/source/ diff --git a/docs/source/api_reference/hpo.rst b/docs/source/api_reference/hpo.rst index 963c92a09..ba09f6265 100644 --- a/docs/source/api_reference/hpo.rst +++ b/docs/source/api_reference/hpo.rst @@ -1,41 +1,48 @@ Hyperparameter Optimization =========================== - Algorithms ---------- .. currentmodule:: aiaccel.hpo.algorithms + .. autosummary:: - :toctree: generated/ + :toctree: generated/ - NelderMeadAlgorism + NelderMeadAlgorism Optuna Utilities ---------------- Samplers -^^^^^^^^ +~~~~~~~~ .. currentmodule:: aiaccel.hpo.optuna.samplers + .. autosummary:: - :toctree: generated/ + :toctree: generated/ - NelderMeadSampler + NelderMeadSampler Suggest Wrappers -^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~ + .. currentmodule:: aiaccel.hpo.optuna.suggest_wrapper + .. autosummary:: - :toctree: generated/ - - Suggest - Const - SuggestFloat - SuggestInt - SuggestCategorical - SuggestDiscreteUniform - SuggestLogUniform - SuggestUniform -.. lightning.OptimizerLightningModule -.. lightning.OptimizerConfig \ No newline at end of file + :toctree: generated/ + + Suggest + Const + SuggestFloat + SuggestInt + SuggestCategorical + SuggestDiscreteUniform + SuggestLogUniform + SuggestUniform + +.. + lightning.OptimizerLightningModule + +.. + lightning.OptimizerConfig diff --git a/docs/source/api_reference/index.rst b/docs/source/api_reference/index.rst index f1d204dc6..77a4c4aeb 100644 --- a/docs/source/api_reference/index.rst +++ b/docs/source/api_reference/index.rst @@ -1,12 +1,9 @@ - -************* API Reference -************* - +============= .. toctree:: - :maxdepth: 2 + :maxdepth: 2 - torch - hpo - utils \ No newline at end of file + torch + hpo + utils diff --git a/docs/source/api_reference/torch.rst b/docs/source/api_reference/torch.rst index e90c0962b..3bab48fbe 100644 --- a/docs/source/api_reference/torch.rst +++ b/docs/source/api_reference/torch.rst @@ -1,32 +1,36 @@ PyTorch/Lightning Toolkit ========================= - Datasets -------- + .. currentmodule:: aiaccel.torch.datasets + .. autosummary:: - :toctree: generated/ + :toctree: generated/ - CachedDataset - HDF5Dataset - RawHDF5Dataset + CachedDataset + HDF5Dataset + RawHDF5Dataset Dataset Utilities ----------------- + .. currentmodule:: aiaccel.torch.datasets -.. autosummary:: - :toctree: generated/ - scatter_dataset +.. autosummary:: + :toctree: generated/ + scatter_dataset Lightning Utilities ------------------- + .. currentmodule:: aiaccel.torch.lightning + .. autosummary:: - :toctree: generated/ + :toctree: generated/ - ABCIEnvironment - OptimizerLightningModule - OptimizerConfig + ABCIEnvironment + OptimizerLightningModule + OptimizerConfig diff --git a/docs/source/api_reference/utils.rst b/docs/source/api_reference/utils.rst index a67f286a5..cae699147 100644 --- a/docs/source/api_reference/utils.rst +++ b/docs/source/api_reference/utils.rst @@ -1,13 +1,13 @@ Miscellaneous Utilities ======================= - Config Utilities ---------------- .. currentmodule:: aiaccel.utils + .. autosummary:: - :toctree: generated/ + :toctree: generated/ - print_config - pathlib2str_config + print_config + pathlib2str_config diff --git a/docs/source/conf.py b/docs/source/conf.py index 250353fa1..7479e781c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -39,7 +39,6 @@ "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.viewcode", - "sphinx_fontawesome", "myst_parser", "pydata_sphinx_theme", "sphinx.ext.doctest", @@ -76,7 +75,7 @@ html_show_sourcelink = False html_show_sphinx = False # (commentout) html_static_path = ["_static"] -html_sidebars = {"**": ["search-field", "sidebar-nav-bs", "sourcelink.html"]} +html_sidebars = {"**": ["sidebar-nav-bs", "sourcelink.html"]} source_suffix = {".rst": "restructuredtext", ".txt": "markdown", ".md": "markdown"} # Add any paths that contain custom static files (such as style sheets) here, diff --git a/docs/source/contribution_guide/index.rst b/docs/source/contribution_guide/index.rst index 36a7cf247..9caffa590 100644 --- a/docs/source/contribution_guide/index.rst +++ b/docs/source/contribution_guide/index.rst @@ -1,11 +1,10 @@ Contribution Guide ================== -Thank you for contributing to aiaccel! -This document introduces how to contribute. +Thank you for contributing to aiaccel! This document introduces how to contribute. .. toctree:: - :maxdepth: 2 + :maxdepth: 2 - issues - pull_requests \ No newline at end of file + issues + pull_requests diff --git a/docs/source/contribution_guide/issues.md b/docs/source/contribution_guide/issues.md index 79b1e284d..d45579255 100644 --- a/docs/source/contribution_guide/issues.md +++ b/docs/source/contribution_guide/issues.md @@ -2,14 +2,14 @@ When you find any problems or have requests for new features, please first check to ensure that there is no duplicate issues already posted. We usually use Japanese for internal development, but we are more than happy to communicate with you in English. -### Bug report +## Bug report A bug report should briefly summarize the following details: * What the bug is * Steps to reproduce the bug * What you expected to happen * The execution environment -### Feature request +## Feature request A feature request should briefly summarize the following details: diff --git a/docs/source/contribution_guide/pull_requests.md b/docs/source/contribution_guide/pull_requests.md index 17eb09790..32f6eaa6a 100644 --- a/docs/source/contribution_guide/pull_requests.md +++ b/docs/source/contribution_guide/pull_requests.md @@ -2,7 +2,7 @@ When you want to the modified code to be reflected in the repository, please execute pull request. -### Procedures +## Procedures - Please fork aiaccel repository on GitHub. - After forking, run `git clone` command for aiaccel repository. @@ -11,7 +11,7 @@ When you want to the modified code to be reflected in the repository, please exe git clone https://github.com/[YOUR USERNAME]/aiaccel.git ~~~ -### Development +## Development - Update a local repository to the latest version. ~~~bash @@ -31,16 +31,16 @@ git checkout -b feature/add-new-feature - The message should be written in such a way that their contents can be understood without refering code. -### Submitting +## Submitting Before submit *Pull request*, confirm the following: - Did you discuss it with other developer on issues in advance? - Can it be distributed under the MIT licence? -- Is there appropriate [unit tests](#test)? -- Can the [unit tests](#test) be run on local? +- Is there appropriate [unit tests](test) ? +- Can the [unit tests](test) be run on local? - Does the public function have a docstring? -- Can the [documentation](#documentation-wip) be rendered correctly? -- Is the [coding style](#coding-style) appropriate? +- Can the [documentation](documentation-wip) be rendered correctly? +- Is the [coding style](coding-style) appropriate? - Is the commit message appropriate? - For larger commit, please provide the example (docs/source/examples) and the description of module level. - If you are adding complied codes, have you modified setup.py? @@ -71,6 +71,7 @@ git push origin feature/add-new-feature - When updating codes in the pull request, please commit the changes in the local repository and push the changes to the fork only if they have been successfully tested in the local environment. - If the pull request has been reviewed and approved by at least one member of the aiaccel development team, it will be merged into the main branch. +(documentation-wip)= # Documentation (WIP) ## Docstrings @@ -110,6 +111,7 @@ make gettext sphinx-intl update -p build/gettext -l en -l ja ~~~ +(test)= # Test ## Adding tests @@ -166,6 +168,7 @@ pytest --cov=aiaccel pytest --cov=aiaccel --cov-branch ~~~ +(coding-style)= # Coding style ## Basic rules diff --git a/docs/source/index.rst b/docs/source/index.rst index 8f9409f3a..4e61f7d51 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,76 +1,71 @@ -.. aiaccel-dev documentation master file, created by - sphinx-quickstart on Mon Jan 18 15:46:13 2021. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - +.. + aiaccel-dev documentation master file, created by + sphinx-quickstart on Mon Jan 18 15:46:13 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. Aiaccel ======= AIST toolkit for accelerating machine learning research. - :octicon:`cpu;1em;sd-text-primary` High-Performance Computing (HPC) - Intended to use in the HPC clusters including AI Bridging Cloud Infrastructure (ABCI). + Intended to use in the HPC clusters including AI Bridging Cloud Infrastructure + (ABCI). :octicon:`server;1em;sd-text-primary` Highly Modular Design - Designed to let you pick up any part of aiaccel with minimal dependencies as you prefer. + Designed to let you pick up any part of aiaccel with minimal dependencies as you + prefer. :octicon:`zap;1em;sd-text-primary` High Compatibility - Compatible with the modern standard frameworks such as PyTorch Lightning and Optuna. - + Compatible with the modern standard frameworks such as PyTorch Lightning and Optuna. .. grid:: - .. grid-item-card:: PyTorch/Lightning Toolkit - :link: api_reference/torch.html - - Training toolkit for HPC clusters. + .. grid-item-card:: PyTorch/Lightning Toolkit + :link: api_reference/torch.html - .. grid-item-card:: Hyperparameter Optimization (HPO) - :link: api_reference/hpo.html - - Ready-to-use HPO algorithms/tools. + Training toolkit for HPC clusters. + .. grid-item-card:: Hyperparameter Optimization (HPO) + :link: api_reference/hpo.html + Ready-to-use HPO algorithms/tools. Getting Started --------- +--------------- .. toctree:: - :maxdepth: 2 - - getting_started.md + :maxdepth: 2 + getting_started.md User Guide --------- +---------- .. toctree:: - :maxdepth: 2 - - user_guide/index.rst + :maxdepth: 2 + user_guide/index.rst API Reference --------- - +------------- .. toctree:: - :maxdepth: 2 - - api_reference/index.rst + :maxdepth: 2 + api_reference/index.rst Contribution Guide --------- +------------------ .. toctree:: - :maxdepth: 2 - - contribution_guide/index.rst + :maxdepth: 2 + contribution_guide/index.rst Acknowledgments --------- -* Part of this work was developed under a commissioned project of the New Energy and Industrial Technology Development Organization (NEDO). +--------------- + +- Part of this work was developed under a commissioned project of the New Energy and + Industrial Technology Development Organization (NEDO). diff --git a/docs/source/user_guide/hpo.rst b/docs/source/user_guide/hpo.rst index cc83ac5a9..a8b6c55ce 100644 --- a/docs/source/user_guide/hpo.rst +++ b/docs/source/user_guide/hpo.rst @@ -1,97 +1,100 @@ Optimizing Your Hyperparameters =============================== -Hyperparameter optimization (HPO) is an indispensable step to make it work in real world. +Hyperparameter optimization (HPO) is an indispensable step to make it work in real +world. Usage --------- +----- Creating an Objective File -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~ Create a file that defines the objective function to be optimized: .. code-block:: python - # objective.py - import json - from argparse import ArgumentParser - from pathlib import Path + # objective.py + import json + from argparse import ArgumentParser + from pathlib import Path - def main() -> None: - parser = ArgumentParser() - parser.add_argument("dst_filename", type=Path) - parser.add_argument("--x1", type=float) - parser.add_argument("--x2", type=float) - args = parser.parse_args() - x1, x2 = args.x1, args.x2 + def main() -> None: + parser = ArgumentParser() + parser.add_argument("dst_filename", type=Path) + parser.add_argument("--x1", type=float) + parser.add_argument("--x2", type=float) + args = parser.parse_args() - # Objective function: (x1^2 - 4x1 + x2^2 - x2 - x1x2) - y = (x1**2) - (4.0 * x1) + (x2**2) - x2 - (x1 * x2) + x1, x2 = args.x1, args.x2 - with open(args.dst_filename, 'w') as f: - json.dump(y, f) + # Objective function: (x1^2 - 4x1 + x2^2 - x2 - x1x2) + y = (x1**2) - (4.0 * x1) + (x2**2) - x2 - (x1 * x2) - if __name__ == "__main__": - main() + with open(args.dst_filename, "w") as f: + json.dump(y, f) + + + if __name__ == "__main__": + main() Creating an Execution Script -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Create a script for environment setup and objective function execution: .. code-block:: bash - # objective.sh - #!/bin/bash + # objective.sh + #!/bin/bash - #$-l rt_C.small=1 - #$-cwd + #$-l rt_C.small=1 + #$-cwd - source /etc/profile.d/modules.sh - module load gcc/13.2.0 - module load python/3.10/3.10.14 + source /etc/profile.d/modules.sh + module load gcc/13.2.0 + module load python/3.10/3.10.14 - python objective.py $@ + python objective.py $@ Configuration -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~ Basic configuration example: .. code-block:: yaml - study: - _target_: optuna.create_study - direction: minimize + study: + _target_: optuna.create_study + direction: minimize - executor: - _target_: aiaccel.hpo.job_executors.LocalJobExecutor - n_max_jobs: 4 + executor: + _target_: aiaccel.hpo.job_executors.LocalJobExecutor + n_max_jobs: 4 - result: - _target_: aiaccel.results.JsonResult - filename_template: "{job.cwd}/{job.job_name}_result.json" + result: + _target_: aiaccel.results.JsonResult + filename_template: "{job.cwd}/{job.job_name}_result.json" - params: - x1: [0, 1] - x2: [0, 1] + params: + x1: [0, 1] + x2: [0, 1] - n_trials: 30 + n_trials: 30 Study Configuration -~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~ (WIP) Sampler Configuration -~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~ (WIP) Executor Configuration -~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~ Two types of executors are available: @@ -99,21 +102,21 @@ Local Execution: .. code-block:: yaml - executor: - _target_: aiaccel.hpo.job_executors.LocalJobExecutor - n_max_jobs: 4 + executor: + _target_: aiaccel.hpo.job_executors.LocalJobExecutor + n_max_jobs: 4 ABCI Execution: .. code-block:: yaml - executor: - _target_: aiaccel.hpo.job_executors.AbciJobExecutor - n_max_jobs: 4 - group: gaa50000 + executor: + _target_: aiaccel.hpo.job_executors.AbciJobExecutor + n_max_jobs: 4 + group: gaa50000 Result Configuration -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~ Three formats are supported for saving results: @@ -121,83 +124,92 @@ JSON Format (Default): .. code-block:: yaml - result: - _target_: aiaccel.results.JsonResult - filename_template: "{job.cwd}/{job.job_name}_result.json" + result: + _target_: aiaccel.results.JsonResult + filename_template: "{job.cwd}/{job.job_name}_result.json" Example objective function for JSON: .. code-block:: python - import json - ... - def main() -> None: - # ... argument parsing ... - y = (x1**2) - (4.0 * x1) + (x2**2) - x2 - (x1 * x2) - with open(args.dst_filename, 'w') as f: - json.dump(y, f) + import json + + ... + + + def main() -> None: + # ... argument parsing ... + y = (x1**2) - (4.0 * x1) + (x2**2) - x2 - (x1 * x2) + with open(args.dst_filename, "w") as f: + json.dump(y, f) Pickle Format: .. code-block:: yaml - result: - _target_: aiaccel.results.PickleResult - filename_template: "{job.cwd}/{job.job_name}_result.pkl" + result: + _target_: aiaccel.results.PickleResult + filename_template: "{job.cwd}/{job.job_name}_result.pkl" Example objective function for Pickle: .. code-block:: python - import pickle - ... - def main() -> None: - # ... argument parsing ... - y = (x1**2) - (4.0 * x1) + (x2**2) - x2 - (x1 * x2) - with open(args.dst_filename, 'wb') as f: - pickle.dump(y, f) + import pickle + + ... + + + def main() -> None: + # ... argument parsing ... + y = (x1**2) - (4.0 * x1) + (x2**2) - x2 - (x1 * x2) + with open(args.dst_filename, "wb") as f: + pickle.dump(y, f) Stdout Format: .. code-block:: yaml - result: - _target_: aiaccel.results.StdoutResult - filename_template: "{job.cwd}/{job.job_name}_result.txt" + result: + _target_: aiaccel.results.StdoutResult + filename_template: "{job.cwd}/{job.job_name}_result.txt" Example objective function for Stdout: .. code-block:: python - ... - def main() -> None: - # ... argument parsing ... - y = (x1**2) - (4.0 * x1) + (x2**2) - x2 - (x1 * x2) - print(y) + ... + + + def main() -> None: + # ... argument parsing ... + y = (x1**2) - (4.0 * x1) + (x2**2) - x2 - (x1 * x2) + print(y) Parameters Configuration -~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~ (WIP) Execution Methods -~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~ Basic Usage -+++++++++++++++++++++++++++ ++++++++++++ For more complex configurations, you can use a YAML configuration file: .. code-block:: bash - python -m aiaccel.hpo.apps.optimize objective.sh --config config.yaml + python -m aiaccel.hpo.apps.optimize objective.sh --config config.yaml Combining Configuration File and Command Line Parameters -+++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -You can override configuration file settings using command line parameters. Command line parameters take precedence over configuration file values: +You can override configuration file settings using command line parameters. Command line +parameters take precedence over configuration file values: .. code-block:: bash - # Override parameters from config file - python -m aiaccel.hpo.apps.optimize objective.sh --config config.yaml --params x1="[0,2]" x2="[0,2]" + # Override parameters from config file + python -m aiaccel.hpo.apps.optimize objective.sh --config config.yaml --params x1="[0,2]" x2="[0,2]" diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst index f18378e35..9d5405490 100644 --- a/docs/source/user_guide/index.rst +++ b/docs/source/user_guide/index.rst @@ -1,8 +1,8 @@ User Guide -================== +========== .. toctree:: - :maxdepth: 2 - - torch - hpo \ No newline at end of file + :maxdepth: 2 + + torch + hpo diff --git a/docs/source/user_guide/torch.rst b/docs/source/user_guide/torch.rst index e4ee80c0f..0c488e4a7 100644 --- a/docs/source/user_guide/torch.rst +++ b/docs/source/user_guide/torch.rst @@ -7,10 +7,8 @@ Writing a simple training script Running inference ----------------- - Writing a DDP training script ----------------------------- - Accelerating your training -------------------------- diff --git a/setup.py b/setup.py index c022a994b..2e178b553 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ "pandas", "pandas-stubs", "matplotlib", + "docstrfmt", ] github_actions_dependencies = [ "pytest-github-actions-annotate-failures",