diff --git a/.github/workflows/docs.sh b/.github/workflows/docs.sh index 1d08bbbafef26d..2b9c9126e15c22 100755 --- a/.github/workflows/docs.sh +++ b/.github/workflows/docs.sh @@ -24,27 +24,8 @@ javadoc -J-version # workaround for a git security patch git config --global --add safe.directory /root/flink -git submodule update --init --recursive -cd docs - -# setup hugo -source setup_hugo.sh - -# Setup the external documentation modules -source setup_docs.sh - -cd .. - -# Build the docs -hugo --source docs - -# generate docs into docs/target -hugo -v --source docs --destination target -if [ $? -ne 0 ]; then - echo "Error building the docs" - exit 1 -fi +./docs/build_docs.sh build # build Flink; required for Javadoc step mvn clean install -B -DskipTests -Dfast -Pskip-webui-build diff --git a/docs/README.md b/docs/README.md index bc59ea579eb9eb..2f6dd8aa22081a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,15 +12,20 @@ The Flink documentation uses [Hugo](https://gohugo.io/getting-started/installing To build the documentation, you can install Hugo locally or use a Docker image. -Both methods require you to execute commands in the directory of this module (`docs/`). The built site is served at http://localhost:1313/. - -#### Using Hugo Docker image: +Both methods require you to execute commands in the directory of this module (`docs/`). The built site is served at http://localhost:1313/.: ```sh -$ git submodule update --init --recursive -$ ./setup_docs.sh -$ docker pull jakejarvis/hugo-extended:latest -$ docker run -v $(pwd):/src -p 1313:1313 jakejarvis/hugo-extended:latest server --buildDrafts --buildFuture --bind 0.0.0.0 +# installs the Hugo binaries locally +$ ./build_docs.sh install-hugo + +# serves the docs locally using a Docker image +$ ./build_docs.sh serve --docker + +# serves the docs using a local Hugo installation +$ ./build_docs.sh serve --local + +# build docs locally (requires a local Hugo installation) +$ ./build_docs.sh build ``` #### Local Hugo installation: @@ -28,13 +33,13 @@ $ docker run -v $(pwd):/src -p 1313:1313 jakejarvis/hugo-extended:latest server Make sure you have installed [Hugo](https://gohugo.io/getting-started/installing/) on your system. ```sh -$ ./setup_hugo.sh +$ ./build_docs.sh docker-build ``` Then build the docs from source: ```sh -$ ./build_docs.sh +$ ./build_docs.sh local-build ``` The site can be viewed at http://localhost:1313/ @@ -49,8 +54,8 @@ Adding new externally hosted documentation requires the following steps to be ta 1. (If necessary) Move the existing documentation to the new repository -2. In the Flink repository, edit the `docs/setup_docs.sh` file and add a reference to your now -externally hosted documentation. The reference will look like `integrate_connector_docs `. +2. In the Flink repository, edit the `docs/connector_docs.conf` file and add a reference to your now +externally hosted documentation. The reference will look like ` `. Replace with the name of your connector, e.g., `elasticsearch` for `flink-connector-elasticsearch`. diff --git a/docs/build_docs.sh b/docs/build_docs.sh index 80f7d0e251580a..761b4cb0cb7d99 100755 --- a/docs/build_docs.sh +++ b/docs/build_docs.sh @@ -17,13 +17,64 @@ # limitations under the License. ################################################################################ -if ! command -v hugo &> /dev/null -then - echo "Hugo must be installed to run the docs locally" - echo "Please see docs/README.md for more details" - exit 1 -fi -git submodule update --init --recursive -./setup_docs.sh -hugo mod get -u -hugo -b "" serve +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +source "${SCRIPT_DIR}/docs_utils.sh" + +function fetch_submodules() { + # the git command has to happen from within the git checkout + pushd ${SCRIPT_DIR} || exit + git submodule update --init --recursive + popd || exit + + load_connector_docs_from_file "${SCRIPT_DIR}/connector_docs.conf" "${SCRIPT_DIR}/tmp" "${SCRIPT_DIR}/themes/connectors" +} + +SCRIPT_COMMAND="$1" +shift 1 + +HUGO_VERSION="0.110.0" +HUGO_COMMAND="hugo" +case "${SCRIPT_COMMAND}" in + "install-hugo") + install_hugo "${HUGO_VERSION}" + ;; + "serve") + if [[ "${1}" == "--local" ]]; then + assert_hugo_installation "${HUGO_VERSION}" + elif [[ "${1}" == "--docker" ]]; then + # overwrites HUGO_COMMAND to utilize docker container + HUGO_COMMAND="docker run -v ${SCRIPT_DIR}:/src -p 1313:1313 jakejarvis/hugo-extended:${HUGO_VERSION}" + else + echo "[ERROR] Unexpected parameter for serve: $*" + exit 1 + fi + + fetch_submodules + eval "${HUGO_COMMAND} mod get -u" + eval "${HUGO_COMMAND} server --buildDrafts --buildFuture --bind 0.0.0.0 --baseURL ''" + + popd || exit + ;; + "build") + assert_hugo_installation "${HUGO_VERSION}" + fetch_submodules + + source_dir="${SCRIPT_DIR}" + dest_dir="${SCRIPT_DIR}/../target" + ${HUGO_COMMAND} --source "${source_dir}" + ${HUGO_COMMAND} --verbose --source "${source_dir}" --destination "${dest_dir}" + echo "[INFO] Flink docs build from ${source_dir} into ${dest_dir}." + + if [ $? -ne 0 ]; then + echo "[ERROR] An error occurred while building the Flink docs." + exit 1 + fi + ;; + *) + echo "Usage: (install-hugo|serve|local-build) []" + echo " install-hugo Installs Hugo ${HUGO_VERSION} locally." + echo " serve (--local|--docker) Builds the docs and starts a local webserver (either using the local Hugo binaries or a Hugo Docker image)." + echo " build Builds the docs (this step is only supported using a local Hugo installation)." + ;; +esac diff --git a/docs/setup_hugo.sh b/docs/connector_docs.conf similarity index 70% rename from docs/setup_hugo.sh rename to docs/connector_docs.conf index 5f3fa29028472b..d0ce9bbc1b5616 100644 --- a/docs/setup_hugo.sh +++ b/docs/connector_docs.conf @@ -17,11 +17,20 @@ # limitations under the License. ################################################################################ -# setup hugo -HUGO_REPO=https://github.com/gohugoio/hugo/releases/download/v0.110.0/hugo_extended_0.110.0_Linux-64bit.tar.gz -HUGO_ARTIFACT=hugo_extended_0.110.0_Linux-64bit.tar.gz -if ! curl --fail -OL $HUGO_REPO ; then - echo "Failed to download Hugo binary" - exit 1 -fi -tar -zxvf $HUGO_ARTIFACT -C /usr/local/bin +# space-separated list for the connector docs +# should match the connector's GitHub repository apache/flink- +# refers to the branch the docs should be fetched from +# +# row format of this file: + +elasticsearch v3.0 +aws v4.2 +cassandra v3.1 +pulsar v4.0 +jdbc v3.1 +rabbitmq v3.0 +gcp-pubsub v3.0 +mongodb v1.0 +opensearch v1.1 +kafka v3.0 +hbase v3.0 diff --git a/docs/docs_utils.sh b/docs/docs_utils.sh new file mode 100755 index 00000000000000..78a2c7ed288613 --- /dev/null +++ b/docs/docs_utils.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +function install_hugo { + local hugo_version hugo_binary_arch hugo_artifact hugo_download_url install_dir + + hugo_version="$1" + hugo_binary_arch="Linux-64bit" + + hugo_artifact="hugo_extended_${hugo_version}_${hugo_binary_arch}.tar.gz" + hugo_download_url="https://github.com/gohugoio/hugo/releases/download/v${hugo_version}/${hugo_artifact}" + if ! curl --silent --fail -OL "${hugo_download_url}" ; then + echo "Failed to download Hugo ${hugo_version} binary" + exit 1 + fi + + install_dir="/usr/local/bin" + tar -zxf "${hugo_artifact}" -C "${install_dir}" + echo "[INFO] Hugo (v${hugo_version}) installed to '${install_dir}'." + rm "${hugo_artifact}" +} + +function load_connector_docs { + local connector_name ref temp_dir theme_dir + connector_name=$1 + ref=$2 + temp_dir=$3 + theme_dir=$4 + + pushd "${temp_dir}" || exit + git clone --single-branch --branch "${ref}" "https://github.com/apache/flink-connector-${connector_name}" + rsync -a flink-connector-${connector_name}/docs/* "${theme_dir}/" + popd || exit +} + +function load_connector_docs_from_file { + local config_file temp_dir theme_dir + + config_file="$1" + temp_dir="$2" + theme_dir="$3" + + rm -rf "${theme_dir}" + mkdir -p "${theme_dir}" + + rm -rf "${temp_dir}" + mkdir -p "${temp_dir}" + + while read -r line; do + if [[ "${line}" =~ ^# ]] || [[ "${line}" =~ ^[^a-zA-Z]*$ ]]; then + # skip comments or empty lines + continue + fi + + connector_name="$(echo $line | cut -d' ' -f1)" + ref="$(echo $line | cut -d' ' -f2)" + load_connector_docs "${connector_name}" "${ref}" "${temp_dir}" "${theme_dir}" + done < "${config_file}" +} + +function assert_hugo_installation { + local expected_hugo_version + expected_hugo_version="${1}" + if ! which hugo &> /dev/null ; then + echo "[ERROR] No hugo binaries installed." + exit 1 + elif [[ "$(hugo version | grep -c "hugo v${expected_hugo_version}")" == 0 ]]; then + echo "[ERROR] Invalid Hugo version installed: (expected ${expected_hugo_version}):" + hugo version + exit 1 + fi +} diff --git a/docs/setup_docs.sh b/docs/setup_docs.sh deleted file mode 100755 index 439249c72663b5..00000000000000 --- a/docs/setup_docs.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -HERE=` basename "$PWD"` -if [[ "$HERE" != "docs" ]]; then - echo "Please only execute in the docs/ directory"; - exit 1; -fi - -function integrate_connector_docs { - local connector ref additional_folders - connector=$1 - ref=$2 - - git clone --single-branch --branch ${ref} https://github.com/apache/flink-connector-${connector} - theme_dir="../themes/connectors" - mkdir -p "${theme_dir}" - - rsync -a flink-connector-${connector}/docs/* "${theme_dir}/" -} - - -# Integrate the connector documentation - -rm -rf themes/connectors/* -rm -rf tmp -mkdir tmp -cd tmp - -integrate_connector_docs elasticsearch v3.0 -integrate_connector_docs aws v4.2 -integrate_connector_docs cassandra v3.1 -integrate_connector_docs pulsar v4.0 -integrate_connector_docs jdbc v3.1 -integrate_connector_docs rabbitmq v3.0 -integrate_connector_docs gcp-pubsub v3.0 -integrate_connector_docs mongodb v1.0 -integrate_connector_docs opensearch v1.1 -integrate_connector_docs kafka v3.0 -integrate_connector_docs hbase v3.0 - -cd .. -rm -rf tmp diff --git a/tools/ci/docs.sh b/tools/ci/docs.sh index 30e1ea89b8b302..a4bb0f2688dd0e 100755 --- a/tools/ci/docs.sh +++ b/tools/ci/docs.sh @@ -17,16 +17,5 @@ # limitations under the License. ################################################################################ -git submodule update --init --recursive -# Setup the external documentation modules -cd docs -source setup_hugo.sh -source setup_docs.sh -cd .. -# Build the docs -hugo --source docs - -if [ $? -ne 0 ]; then - echo "Error building the docs" - exit 1 -fi +./docs/build_docs.sh install-hugo +./docs/build_docs.sh build