diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index 045c36c13..5b7fcbe87 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -61,6 +61,10 @@ jobs: if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version != '3.10' run: | make test-no-spark + - name: RTD build for kedro-datasets + if: inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10' + run: | + make rtd lint: defaults: diff --git a/.github/workflows/run-docs-build.yml b/.github/workflows/run-docs-build.yml new file mode 100644 index 000000000..79ee833f4 --- /dev/null +++ b/.github/workflows/run-docs-build.yml @@ -0,0 +1,25 @@ +name: Run Doc builds on Kedro-Datasets + +on: + push: + branches: + - main + - try-rtd-build + paths-ignore: + - "kedro-airflow/**" + - "kedro-docker/**" + - "kedro-telemetry/**" + pull_request: + branches: + - main + - try-rtd-build + paths-ignore: + - "kedro-airflow/**" + - "kedro-docker/**" + - "kedro-telemetry/**" + +jobs: + datasets-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-datasets diff --git a/Makefile b/Makefile index 074095ba7..1c6c7e478 100644 --- a/Makefile +++ b/Makefile @@ -60,3 +60,6 @@ test-no-spark-sequential: # kedro-datasets/snowflake tests skipped from default scope test-snowflake-only: cd kedro-datasets && pytest tests --no-cov --numprocesses 1 --dist loadfile -m snowflake + +rtd: + cd kedro-datasets && python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck diff --git a/kedro-datasets/docs/build-docs.sh b/kedro-datasets/docs/build-docs.sh new file mode 100755 index 000000000..d55076e11 --- /dev/null +++ b/kedro-datasets/docs/build-docs.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +# Exit script if you try to use an uninitialized variable. +set -o nounset + +action=$1 + +if [ "$action" == "linkcheck" ]; then + sphinx-build -WETan -j auto -D language=en -b linkcheck -d docs/build/doctrees docs/source docs/build/linkcheck +elif [ "$action" == "docs" ]; then + sphinx-build -WETa -j auto -D language=en -b html -d docs/build/doctrees docs/source docs/build/html +fi diff --git a/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css b/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css new file mode 100644 index 000000000..3f11d0cee --- /dev/null +++ b/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css @@ -0,0 +1,493 @@ +@import url("https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600&display=swap"); + +html, body.wy-body-for-nav { + margin: 0; + padding: 0; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + font-family: 'Inter', sans-serif !important; + font-weight: 400; + line-height: 2rem; +} + +html { + font-size: 62.5%; +} + +:focus { + outline-color: #006dff; +} + +body.wy-body-for-nav { + font-size: 1.4289rem; + background: rgb(250, 250, 250) !important; + color: #1e1e21; + position: relative; +} + +.wy-side-nav-search { + text-align: left; +} + +.wy-side-nav-search input[type=text] { + display: block; + box-sizing: border-box; + width: 100%; + padding: 6px 12px; + color: #666; + background-color: #fff; + font-family: 'Inter', sans-serif !important; + font-size: 1.4289rem; + border: 1px #ccc solid; + border-radius: 2px; + transition: all ease 0.15s; + box-shadow: none; +} + +.wy-side-nav-search input[type=text]:focus { + border-color: #888; + color: #333; +} + +.wy-body-for-nav .wy-nav-side { + position: fixed; + top: 0; + bottom: 0; + left: 0; + padding-bottom: 0; + width: 300px; + overflow-x: hidden; + min-height: 100%; + background: white; + z-index: 200; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, .05); +} + +.wy-body-for-nav .wy-side-scroll { + display: flex; + flex-direction: column; + width: 100%; + position: relative; + overflow-x: initial; + overflow-y: initial; +} + +.wy-body-for-nav .wy-side-nav-search { + width: 100%; + background: none; + margin: 0; + padding: 0 20px; +} +.wy-body-for-nav .wy-side-nav-search a { + display: inline-flex; + flex-direction: row-reverse; + align-items: center; + font-size: 4rem; + margin: 0; + padding: 0; + height: 7rem; + line-height: 7rem; + color: black; +} +.wy-body-for-nav .wy-side-nav-search a:before { + display: none; +} +.wy-body-for-nav .wy-side-nav-search a img.logo { + width: 4.5rem; + height: auto; + margin: 0 0.3rem 0 -0.6rem; + padding: 0; +} +.wy-body-for-nav .wy-side-nav-search>div.version { + color: #555; + display: inline-block; + margin-left: 0.5em; + font-size: 1.4rem; +} + +.wy-body-for-nav .wy-menu-vertical { + width: 300px; + margin: 0; + padding: 0px 20px 64px 20px; + overflow-y: auto; + height: 100%; +} + +.wy-body-for-nav .wy-menu-vertical p.caption { + margin: 1.5em 0 0.2em; + padding: 0; + font-size: 1.4289rem; + color: #161616; + font-weight: normal; + text-transform: none; +} + +.wy-body-for-nav .wy-menu-vertical p.caption:first-child { + margin-top: 1em; +} + +.wy-body-for-nav .wy-menu-vertical li.on a, .wy-body-for-nav .wy-menu-vertical li.current>a { + border-top: none; + border-bottom: none; +} + +.wy-body-for-nav .wy-menu-vertical li.current { + background: none; +} + +.wy-body-for-nav .wy-menu-vertical li { + margin: 0; + padding: 0 0 0 20px; +} + +.wy-body-for-nav .wy-menu-vertical li a { + display: block; + margin: 0; + padding: 0.9rem 0 !important; + font-size: 1.4289rem; + line-height: 1.2; + color: #222; + background: none !important; +} + +.wy-menu-vertical li.toctree-l1>a { + font-size: 1.4289rem; +} + +.rst-content.style-external-links a.reference.external:after { + color: inherit; + opacity: 0.8; +} + +.rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal { + color: #e72000; +} + +.highlight .nv { + color: #bd269e; +} + +.highlight .c1, +.highlight .sd { + color: #147087; +} + +.highlight .nn { + color: #0e84b5; +} + +.highlight .nn { + color: #0047bf; +} + +.wy-body-for-nav .wy-menu-vertical a:hover { + color: #000; +} + +.wy-body-for-nav li span.toctree-expand { + margin-left: -20px; +} + +.wy-body-for-nav .toctree-expand:before { + font-size: 15px; + margin-right: 5px; +} + +.wy-body-for-nav .wy-nav-content-wrap { + margin-left: 300px; + background: #fafafa; +} + +.wy-body-for-nav .wy-nav-content { + padding: 0; + max-width: initial; +} + +.wy-body-for-nav .wy-breadcrumbs { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + width: 100%; + height: 7rem; + padding-left: 3.2rem; + background: white; + background-image: initial; + background-position-x: initial; + background-position-y: initial; + background-size: initial; + background-repeat-x: initial; + background-repeat-y: initial; + background-attachment: initial; + background-origin: initial; + background-clip: initial; + background-color: white; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, .05); +} + +.wy-body-for-nav .wy-breadcrumbs + hr { + display: none; +} +.wy-body-for-nav .wy-breadcrumbs li { + font-size: 1.8rem; +} + +.wy-body-for-nav .wy-breadcrumbs li:not(:first-child) { + margin-left: 6px; +} + +.wy-body-for-nav .wy-breadcrumbs li.wy-breadcrumbs-aside { + margin-left: 24px; +} + +.wy-body-for-nav .wy-nav-content .document { + padding: 10.2rem 3.2rem 2rem; + max-width: 100rem; +} + +.kedro-logo { + margin-bottom: 3rem; +} + +.wy-body-for-nav footer { + padding: 32px; +} + +.wy-body-for-nav .wy-body-for-nav { + background: #fafafa !important; + background-image: none; + background-size: initial; +} + +.wy-body-for-nav .wy-menu-vertical li.current a:hover { + background: none; + font-weight: 600; +} + +.wy-menu-vertical a span.toctree-expand { + color: #555 !important; +} + +.wy-menu-vertical a:hover span.toctree-expand { + color: #000 !important; +} + +.wy-body-for-nav .wy-menu-vertical li.current a { + border-right: none; + font-weight: normal; +} + +.wy-body-for-nav .wy-menu-vertical li.current a.current { + font-weight: bold; +} + +.wy-body-for-nav .toctree-l2, +.wy-body-for-nav .toctree-l3, +.wy-body-for-nav .toctree-l4, +.wy-body-for-nav .toctree-l5 { + padding-left: 8px !important; +} + +.wy-body-for-nav .toctree-l4 a { + word-break: break-word; +} + +.wy-plain-list-disc li, +.rst-content .section ul li, +.rst-content .toctree-wrapper ul li, +article ul li { + list-style: square; + margin-top: 0.35em; + margin-bottom: 0.35em; +} + +.rst-content .toctree-wrapper ul li { + margin-top: 0.5em; + margin-bottom: 0.5em; +} + +h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend { + font-family: 'Inter', sans-serif !important; + margin: 1.2em 0 1em 0; + line-height: 1.2em; + color: #272c2e; + font-weight: normal; +} + +.wy-body-for-nav .document > div > .section > *:not(:empty):first-of-type { + margin-top: 0; +} + +.wy-body-for-nav h1 { + font-size: 2.6rem; + letter-spacing: -0.3px; +} + +.wy-body-for-nav h2 { + font-size: 2.3rem; +} + +.wy-body-for-nav h3 { + font-size: 2.1rem; +} + +.wy-body-for-nav h4 { + font-size: 1.95rem; +} + +.wy-body-for-nav h5 { + font-size: 1.625rem; +} + +.wy-body-for-nav h6 { + font-size: 1.4625rem; +} + +.wy-body-for-nav p { + font-size: 100%; + margin: 0 0 1em 0; + line-height: 1.7; +} + +.wy-body-for-nav p + details { + margin: -4px 0 15px; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #6a6a6a; +} + +footer span.commit code, +footer span.commit .rst-content tt, +.rst-content footer span.commit tt { + color: #6f7070; +} + +.rst-content tt, .rst-content tt, .rst-content code { + padding: 0.16em 0.48em; +} + +.wy-body-for-nav .rst-content a, .wy-body-for-nav footer a { + font-family: 'Inter', sans-serif !important; + font-size: inherit; + color: #006ea7; + text-decoration: none; +} + +.wy-body-for-nav .rst-content a:visited, .wy-body-for-nav footer a:visited { + color: #447087; + text-decoration-color: #739eb4; +} + +.wy-body-for-nav .rst-content .btn, .wy-body-for-nav footer .btn, +.wy-body-for-nav .rst-content .btn:hover, .wy-body-for-nav footer .btn:hover, +.wy-body-for-nav .rst-content .headerlink, .wy-body-for-nav .rst-content .headerlink:hover, +.wy-body-for-nav .wy-breadcrumbs li a, +.wy-nav-content .wy-breadcrumbs .fa-github, .wy-nav-content .wy-breadcrumbs .fa-github:hover { + text-decoration: none; +} + +.wy-body-for-nav .rst-content a, .wy-body-for-nav footer a, +.wy-body-for-nav .rst-content a:hover, .wy-body-for-nav footer a:hover { + text-decoration: underline; + text-underline-position: under; + text-decoration-color: #b7d9f2; +} + +.rst-content a code span { + text-decoration: underline; + text-decoration-color: #fff; +} + +.rst-content a code { + border-bottom: 1px solid #ff8c8c; +} + +.wy-body-for-nav .wy-nav-top { + padding: 5px 20px; + background: white; + color: black; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, 0.05); +} + +.wy-body-for-nav .wy-nav-top i { + transform: translateY(6px); +} + +.wy-body-for-nav .wy-nav-top a { + font-size: 2.8rem; + color: black !important; +} + +.wy-side-scroll p.caption[role="heading"] span.caption-text { + font-weight: 700; +} + +.wy-nav-content h1, +.wy-nav-content p.caption[role="heading"] .caption-text { + font-size: 1.7857rem; + font-weight: 700; +} + + +@media screen and (max-width: 768px) { + .wy-body-for-nav .wy-nav-side { + transform: translate(-300px, 0); + transition: all ease 0.3s; + } + .wy-body-for-nav .wy-nav-side.shift { + width: 85%; + transform: translate(0, 0); + } + .wy-body-for-nav .wy-nav-content-wrap { + margin-left: 0; + transform: translate(0, 0); + transition: all ease 0.3s; + } + .wy-body-for-nav .wy-nav-content-wrap.shift { + position: relative; + left: 0; + transform: translate(85%, 0); + } + .wy-body-for-nav .wy-breadcrumbs { + display: none; + } + .wy-body-for-nav .wy-nav-content .document { + padding: 20px; + } +} + +@media screen and (min-width: 1600px) { + .wy-body-for-nav .wy-nav-side { + width: 350px; + } + .wy-body-for-nav .wy-nav-content-wrap { + margin-left: 350px; + } + html { + font-size: 70%; + } +} + +/* Fix Read The Docs side-effects */ +.wy-body-for-nav .rst-versions { + font-size: 16px; + line-height: 1; +} + +.wy-breadcrumbs { + position: fixed; + z-index: 2; +} + +/* Offset target anchor to below the fixed header */ +:target:before { + display: block; + content: ' '; + width: 1px; +} diff --git a/kedro-datasets/docs/source/_static/css/theme-overrides.css b/kedro-datasets/docs/source/_static/css/theme-overrides.css new file mode 100644 index 000000000..bdf28d927 --- /dev/null +++ b/kedro-datasets/docs/source/_static/css/theme-overrides.css @@ -0,0 +1,35 @@ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + .wy-table-responsive table td { + white-space: normal; + } + + .wy-table-responsive { + overflow: visible; + } +} + +/* override the table font-size and line-height */ +html.writer-html5 .rst-content .wy-table-responsive table.docutils th > p, +html.writer-html5 .rst-content .wy-table-responsive table.docutils td > p { + font-size: 1em; + line-height: 1em; +} + +img[alt^="mermaid-"] { + max-width: 600px; +} + +.rst-content .important { + background: #ffedcc; +} +.rst-content .important .admonition-title { + background-color: #f0b37e; +} + +/* Ensure the section title is visible when linked via a hash in the URL */ +:target:before { + content: ""; + display: block; + height: 80px; +} diff --git a/kedro-datasets/docs/source/_templates/autosummary/base.rst b/kedro-datasets/docs/source/_templates/autosummary/base.rst new file mode 100644 index 000000000..b7556ebf7 --- /dev/null +++ b/kedro-datasets/docs/source/_templates/autosummary/base.rst @@ -0,0 +1,5 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. auto{{ objtype }}:: {{ objname }} diff --git a/kedro-datasets/docs/source/_templates/autosummary/class.rst b/kedro-datasets/docs/source/_templates/autosummary/class.rst new file mode 100644 index 000000000..10c8ff8be --- /dev/null +++ b/kedro-datasets/docs/source/_templates/autosummary/class.rst @@ -0,0 +1,32 @@ +{{ fullname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :undoc-members: + :inherited-members: + + {% block attributes %} + {% if attributes %} + .. rubric:: Attributes + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block methods %} + {% if methods %} + .. rubric:: Methods + + .. autosummary:: + {% for item in all_methods %} + {%- if not item.startswith('_') %} + ~{{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/kedro-datasets/docs/source/_templates/autosummary/module.rst b/kedro-datasets/docs/source/_templates/autosummary/module.rst new file mode 100644 index 000000000..a496ca3f5 --- /dev/null +++ b/kedro-datasets/docs/source/_templates/autosummary/module.rst @@ -0,0 +1,56 @@ +{{ fullname | escape | underline }} + +.. rubric:: Description + +.. automodule:: {{ fullname }} + + {% block functions %} + {% if functions %} + .. rubric:: Functions + + .. autosummary:: + :toctree: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: Classes + + .. autosummary:: + :toctree: + :template: autosummary/class.rst + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: Exceptions + + .. autosummary:: + :toctree: + :template: autosummary/class.rst + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/kedro-datasets/docs/source/_templates/breadcrumbs.html b/kedro-datasets/docs/source/_templates/breadcrumbs.html new file mode 100644 index 000000000..49fa4779f --- /dev/null +++ b/kedro-datasets/docs/source/_templates/breadcrumbs.html @@ -0,0 +1,94 @@ +{# Support for Sphinx 1.3+ page_source_suffix, but don't break old builds. #} + +{% if page_source_suffix %} +{% set suffix = page_source_suffix %} +{% else %} +{% set suffix = source_suffix %} +{% endif %} + +{# modification to enable custom github_url #} + +{% if meta is not defined or meta is none %} + {% set meta = {} %} +{% endif %} + +{% if github_url is defined %} + {% set _dummy = meta.update({'github_url': github_url}) %} +{% endif %} + +{# // modification to enable custom github_url #} + +{% if meta is defined and meta is not none %} +{% set check_meta = True %} +{% else %} +{% set check_meta = False %} +{% endif %} + +{% if check_meta and 'github_url' in meta %} +{% set display_github = True %} +{% endif %} + +{% if check_meta and 'bitbucket_url' in meta %} +{% set display_bitbucket = True %} +{% endif %} + +{% if check_meta and 'gitlab_url' in meta %} +{% set display_gitlab = True %} +{% endif %} + +
+ + + + {% if (theme_prev_next_buttons_location == 'top' or theme_prev_next_buttons_location == 'both') and (next or prev) %} + + {% endif %} +
+
diff --git a/kedro-datasets/docs/source/_templates/layout.html b/kedro-datasets/docs/source/_templates/layout.html new file mode 100644 index 000000000..ecdde06f1 --- /dev/null +++ b/kedro-datasets/docs/source/_templates/layout.html @@ -0,0 +1,8 @@ +{% extends "!layout.html" %} + +{%- block extrahead %} + +{% endblock %} diff --git a/kedro-datasets/docs/source/conf.py b/kedro-datasets/docs/source/conf.py new file mode 100644 index 000000000..ca28f16c5 --- /dev/null +++ b/kedro-datasets/docs/source/conf.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 +# +# Kedro documentation build configuration file, created by +# sphinx-quickstart on Mon Dec 18 11:31:24 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. +from __future__ import annotations + +import importlib +import os +import re +import sys +from inspect import getmembers, isclass, isfunction +from pathlib import Path + +from click import secho, style + +from kedro import __version__ as release + +# -- Project information ----------------------------------------------------- + +project = "kedro" +author = "kedro" + +# The short X.Y version. +version = re.match(r"^([0-9]+\.[0-9]+).*", release).group(1) + + +# -- General configuration --------------------------------------------------- +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx_autodoc_typehints", + "sphinx.ext.doctest", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx_copybutton", + "sphinxcontrib.mermaid", + "myst_parser", + "notfound.extension", +] + +# enable autosummary plugin (table of contents for modules/classes/class +# methods) +autosummary_generate = True +autosummary_generate_overwrite = False +napoleon_include_init_with_doc = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] +html_static_path = ["_static"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = {".rst": "restructuredtext", ".md": "markdown"} + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [ + "**.ipynb_checkpoints", + "_templates", + "modules.rst", + "source", + "kedro_docs_style_guide.md", +] + + +type_targets = { + "py:class": ( + "object", + "bool", + "int", + "float", + "str", + "tuple", + "Any", + "Dict", + "dict", + "list", + "set", + "typing.Dict", + "typing.Iterable", + "typing.List", + "typing.Tuple", + "typing.Type", + "typing.Set", + "kedro.config.config.ConfigLoader", + "kedro.io.core.AbstractDataSet", + "kedro.io.AbstractDataSet", + "kedro.io.core.AbstractVersionedDataSet", + "kedro.io.core.DataSetError", + "kedro.io.core.Version", + "kedro.io.data_catalog.DataCatalog", + "kedro.io.memory_dataset.MemoryDataSet", + "kedro.io.partitioned_dataset.PartitionedDataSet", + "kedro.pipeline.pipeline.Pipeline", + "kedro.runner.runner.AbstractRunner", + "kedro.runner.parallel_runner._SharedMemoryDataSet", + "kedro.runner.parallel_runner._SharedMemoryDataset", + "kedro.framework.context.context.KedroContext", + "kedro.framework.startup.ProjectMetadata", + "abc.ABC", + "Path", + "pathlib.Path", + "PurePosixPath", + "pathlib.PurePosixPath", + "requests.auth.AuthBase", + "google.oauth2.credentials.Credentials", + "Exception", + "CONF_SOURCE", + "integer -- return number of occurrences of value", + "integer -- return first index of value.", + "kedro.extras.datasets.pandas.json_dataset.JSONDataSet", + "kedro_datasets.pandas.json_dataset.JSONDataSet", + "pluggy._manager.PluginManager", + "PluginManager", + "_DI", + "_DO", + "deltalake.table.Metadata", + # The statements below were added after subclassing UserDict in AbstractConfigLoader. + "None. Remove all items from D.", + "a shallow copy of D", + "a set-like object providing a view on D's items", + "a set-like object providing a view on D's keys", + "v, remove specified key and return the corresponding value.", + "None. Update D from dict/iterable E and F.", + "an object providing a view on D's values", + "(k, v), remove and return some (key, value) pair", + "D.get(k,d), also set D[k]=d if k not in D", + "None. Update D from mapping/iterable E and F.", + "DataCatalog" + ), + "py:data": ( + "typing.Any", + "typing.Callable", + "typing.Union", + "typing.Optional", + "typing.Tuple", + ), + "py:exc": ( + "ValueError", + "BadConfigException", + "MissingConfigException", + "DataSetError", + "ImportError", + "KedroCliError", + "Exception", + "TypeError", + "SyntaxError", + "CircularDependencyError", + "OutputNotUniqueError", + "ConfirmNotUniqueError", + "ParserError", + "DatasetError", + ), +} +# https://stackoverflow.com/questions/61770698/sphinx-nit-picky-mode-but-only-for-links-i-explicitly-wrote +nitpick_ignore = [(key, value) for key in type_targets for value in type_targets[key]] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" +here = Path(__file__).parent.absolute() + +# Theme options are theme-specific and customise the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = {"collapse_navigation": False, "style_external_links": True} + +# html_extra_path used to define a path to robots.txt which is used by webcrawlers +# to ignore or allow certain links. +html_extra_path = [str(here / "robots.txt")] + +# Removes, from all docs, the copyright footer. +html_show_copyright = False + +# some of these complain that the sections don't exist (which is not true), +# too many requests, or forbidden URL +linkcheck_ignore = [ + "http://127.0.0.1:8787/status", # Dask's diagnostics dashboard + "https://datacamp.com/community/tutorials/docstrings-python", # "forbidden" url + "https://github.com/argoproj/argo/blob/master/README.md#quickstart", + "https://console.aws.amazon.com/batch/home#/jobs", + "https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python", + "https://github.com/jazzband/pip-tools#example-usage-for-pip-compile", + "https://www.astronomer.io/docs/cloud/stable/get-started/quickstart#", + "https://eternallybored.org/misc/wget/", + "https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.from_pandas", + "https://www.oracle.com/java/technologies/javase-downloads.html", # "forbidden" url + "https://www.java.com/en/download/help/download_options.html", # "403 Client Error: Forbidden for url" + # "anchor not found" but it's a valid selector for code examples + "https://docs.delta.io/latest/delta-update.html#language-python", + "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml", + "https://github.com/kedro-org/kedro/blob/main/README.md#the-humans-behind-kedro", # "anchor not found" but is valid + "https://opensource.org/license/apache2-0-php/", + "https://docs.github.com/en/rest/overview/other-authentication-methods#via-username-and-password", + "https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/api/snowflake.snowpark.DataFrameWriter.saveAsTable.html", + "https://www.educative.io/blog/advanced-yaml-syntax-cheatsheet#anchors" +] + +# retry before render a link broken (fix for "too many requests") +linkcheck_retries = 5 +linkcheck_rate_limit_timeout = 2.0 + +html_context = { + "display_github": True, + "github_url": "https://github.com/kedro-org/kedro/tree/main/docs/source", +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + +html_show_sourcelink = False + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "Kedrodoc" + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [(master_doc, "Kedro.tex", "Kedro Documentation", "Kedro", "manual")] + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "kedro", "Kedro Documentation", [author], 1)] + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "Kedro", + "Kedro Documentation", + author, + "Kedro", + "Kedro is a Python framework for creating reproducible, maintainable and modular data science code.", + "Data-Science", + ) +] + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Kedro specific configuration ----------------------------------------- +KEDRO_MODULES = [ + "kedro.io", + "kedro.pipeline", + "kedro.runner", + "kedro.config", + "kedro.extras.datasets", + "kedro.extras.logging", + "kedro_datasets", +] + + +def get_classes(module): + importlib.import_module(module) + return [obj[0] for obj in getmembers(sys.modules[module], lambda obj: isclass(obj))] + + +def get_functions(module): + importlib.import_module(module) + return [ + obj[0] for obj in getmembers(sys.modules[module], lambda obj: isfunction(obj)) + ] + + +def remove_arrows_in_examples(lines): + for i, line in enumerate(lines): + lines[i] = line.replace(">>>", "") + + +def autolink_replacements(what: str) -> list[tuple[str, str, str]]: + """ + Create a list containing replacement tuples of the form: + (``regex``, ``replacement``, ``obj``) for all classes and methods which are + imported in ``KEDRO_MODULES`` ``__init__.py`` files. The ``replacement`` + is a reStructuredText link to their documentation. + + For example, if the docstring reads: + This LambdaDataSet loads and saves ... + + Then the word ``LambdaDataSet``, will be replaced by + :class:`~kedro.io.LambdaDataSet` + + Works for plural as well, e.g: + These ``LambdaDataSet``s load and save + + Will convert to: + These :class:`kedro.io.LambdaDataSet` load and save + + Args: + what: The objects to create replacement tuples for. Possible values + ["class", "func"]. + + Returns: + A list of tuples: (regex, replacement, obj), for all "what" objects + imported in __init__.py files of ``KEDRO_MODULES``. + + """ + replacements = [] + suggestions = [] + for module in KEDRO_MODULES: + if what == "class": + objects = get_classes(module) + elif what == "func": + objects = get_functions(module) + + # Look for recognised class names/function names which are + # surrounded by double back-ticks + if what == "class": + # first do plural only for classes + replacements += [ + ( + rf"``{obj}``s", + f":{what}:`~{module}.{obj}`\\\\s", + obj, + ) + for obj in objects + ] + + # singular + replacements += [ + (rf"``{obj}``", f":{what}:`~{module}.{obj}`", obj) for obj in objects + ] + + # Look for recognised class names/function names which are NOT + # surrounded by double back-ticks, so that we can log these in the + # terminal + if what == "class": + # first do plural only for classes + suggestions += [ + (rf"(?>>" in lines[i]: + continue + + for existing, replacement, obj in suggestions: + new = re.sub(existing, rf"{replacement}", lines[i]) + if new == lines[i]: + continue + if ":rtype:" in lines[i] or ":type " in lines[i]: + continue + + if not title_printed: + secho("-" * 50 + "\n" + name + ":\n" + "-" * 50, fg="blue") + title_printed = True + + print( + "[" + + str(i) + + "] " + + re.sub(existing, r"{}".format(style(obj, fg="magenta")), lines[i]) + ) + print( + "[" + + str(i) + + "] " + + re.sub(existing, r"``{}``".format(style(obj, fg="green")), lines[i]) + ) + + if title_printed: + print("\n") + + +def autolink_classes_and_methods(lines): + for i in range(len(lines)): + if ">>>" in lines[i]: + continue + + for existing, replacement, obj in replacements: + lines[i] = re.sub(existing, rf"{replacement}", lines[i]) + + +def autodoc_process_docstring(app, what, name, obj, options, lines): + try: + # guarded method to make sure build never fails + log_suggestions(lines, name) + autolink_classes_and_methods(lines) + except Exception as e: + print( + style( + "Failed to check for class name mentions that can be " + "converted to reStructuredText links in docstring of {}. " + "Error is: \n{}".format(name, str(e)), + fg="red", + ) + ) + + remove_arrows_in_examples(lines) + + +def env_override(default_appid): + build_version = os.getenv("READTHEDOCS_VERSION") + + if build_version == "latest": + return os.environ["HEAP_APPID_QA"] + if build_version == "stable": + return os.environ["HEAP_APPID_PROD"] + + return default_appid # default to Development for local builds + + +def _add_jinja_filters(app): + # https://github.com/crate/crate/issues/10833 + from sphinx.builders.latex import LaTeXBuilder + from sphinx.builders.linkcheck import CheckExternalLinksBuilder + + # LaTeXBuilder is used in the PDF docs build, + # and it doesn't have attribute 'templates' + if not ( + isinstance(app.builder, (LaTeXBuilder,CheckExternalLinksBuilder)) + ): + app.builder.templates.environment.filters["env_override"] = env_override + + +def _override_permalinks_icon(app): + # https://github.com/readthedocs/sphinx_rtd_theme/issues/98#issuecomment-1503211439 + app.config.html_permalinks_icon = "ΒΆ" + + +def setup(app): + app.connect("builder-inited", _add_jinja_filters) + app.connect("builder-inited", _override_permalinks_icon) + app.connect("autodoc-process-docstring", autodoc_process_docstring) + app.add_css_file("css/qb1-sphinx-rtd.css") + # fix a bug with table wraps in Read the Docs Sphinx theme: + # https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html + app.add_css_file("css/theme-overrides.css") + + +# (regex, restructuredText link replacement, object) list +replacements = [] + +# (regex, class/function name surrounded with back-ticks, object) list +suggestions = [] + +try: + # guarded code to make sure build never fails + replacements_f, suggestions_f = autolink_replacements("func") + replacements_c, suggestions_c = autolink_replacements("class") + replacements = replacements_f + replacements_c + suggestions = suggestions_f + suggestions_c +except Exception as e: + print( + style( + "Failed to create list of (regex, reStructuredText link " + "replacement) for class names and method names in docstrings. " + "Error is: \n{}".format(str(e)), + fg="red", + ) + ) + +user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0" + +myst_heading_anchors = 5 + +# https://github.com/kedro-org/kedro/issues/1772 +mermaid_output_format = "png" +# https://github.com/mermaidjs/mermaid.cli#linux-sandbox-issue +# https://github.com/mermaid-js/mermaid-cli/issues/544 +mermaid_params = ["-p", here / "puppeteer-config.json", "-s", "2"] +# https://github.com/kedro-org/kedro/issues/2451 +mermaid_version = mermaid_init_js = "" diff --git a/kedro-datasets/docs/source/index.rst b/kedro-datasets/docs/source/index.rst new file mode 100644 index 000000000..84decee2a --- /dev/null +++ b/kedro-datasets/docs/source/index.rst @@ -0,0 +1,22 @@ +.. Kedro documentation master file, created by + sphinx-quickstart on Mon Dec 18 11:31:24 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + + +API documentation +================= + +.. autosummary:: + :toctree: + :caption: API documentation + :template: autosummary/module.rst + :recursive: + + kedro_datasets + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` diff --git a/kedro-datasets/docs/source/kedro_datasets.rst b/kedro-datasets/docs/source/kedro_datasets.rst new file mode 100644 index 000000000..08147ba5e --- /dev/null +++ b/kedro-datasets/docs/source/kedro_datasets.rst @@ -0,0 +1,56 @@ +kedro_datasets +============== + +.. rubric:: Description + +.. automodule:: kedro_datasets + +.. rubric:: Classes + +.. autosummary:: + :toctree: + :template: autosummary/class.rst + + kedro_datasets.api.APIDataSet + kedro_datasets.biosequence.BioSequenceDataSet + kedro_datasets.dask.ParquetDataSet + kedro_datasets.databricks.ManagedTableDataSet + kedro_datasets.email.EmailMessageDataSet + kedro_datasets.geopandas.GeoJSONDataSet + kedro_datasets.holoviews.HoloviewsWriter + kedro_datasets.json.JSONDataSet + kedro_datasets.matplotlib.MatplotlibWriter + kedro_datasets.networkx.GMLDataSet + kedro_datasets.networkx.GraphMLDataSet + kedro_datasets.networkx.JSONDataSet + kedro_datasets.pandas.CSVDataSet + kedro_datasets.pandas.DeltaTableDataSet + kedro_datasets.pandas.ExcelDataSet + kedro_datasets.pandas.FeatherDataSet + kedro_datasets.pandas.GBQQueryDataSet + kedro_datasets.pandas.GBQTableDataSet + kedro_datasets.pandas.GenericDataSet + kedro_datasets.pandas.HDFDataSet + kedro_datasets.pandas.JSONDataSet + kedro_datasets.pandas.ParquetDataSet + kedro_datasets.pandas.SQLQueryDataSet + kedro_datasets.pandas.SQLTableDataSet + kedro_datasets.pandas.XMLDataSet + kedro_datasets.pickle.PickleDataSet + kedro_datasets.pillow.ImageDataSet + kedro_datasets.plotly.JSONDataSet + kedro_datasets.plotly.PlotlyDataSet + kedro_datasets.polars.CSVDataSet + kedro_datasets.redis.PickleDataSet + kedro_datasets.spark.DeltaTableDataSet + kedro_datasets.spark.SparkDataSet + kedro_datasets.spark.SparkHiveDataSet + kedro_datasets.spark.SparkJDBCDataSet + kedro_datasets.spark.SparkStreamingDataSet + kedro_datasets.svmlight.SVMLightDataSet + kedro_datasets.tensorflow.TensorFlowModelDataSet + kedro_datasets.text.TextDataSet + kedro_datasets.tracking.JSONDataSet + kedro_datasets.tracking.MetricsDataSet + kedro_datasets.video.VideoDataSet + kedro_datasets.yaml.YAMLDataSet diff --git a/kedro-datasets/docs/source/puppeteer-config.json b/kedro-datasets/docs/source/puppeteer-config.json new file mode 100644 index 000000000..5adc866e9 --- /dev/null +++ b/kedro-datasets/docs/source/puppeteer-config.json @@ -0,0 +1,4 @@ +{ + "args": ["--no-sandbox"], + "headless": "old" +} diff --git a/kedro-datasets/docs/source/robots.txt b/kedro-datasets/docs/source/robots.txt new file mode 100644 index 000000000..9bd9ee90d --- /dev/null +++ b/kedro-datasets/docs/source/robots.txt @@ -0,0 +1,5 @@ +User-agent: * +Disallow: * +Allow: /en/stable +Allow: /en/latest +Allow: /en/0.18.*