Skip to content

Commit

Permalink
[FLINK-34045][ci] Refactors docs creation scripts
Browse files Browse the repository at this point in the history
Merges all the logic for building docs into a single file ./docs/build_docs.sh.
  • Loading branch information
XComp committed Feb 16, 2024
1 parent 9caa3bb commit 26e9f4c
Show file tree
Hide file tree
Showing 7 changed files with 185 additions and 121 deletions.
21 changes: 1 addition & 20 deletions .github/workflows/docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,27 +24,8 @@ javadoc -J-version

# workaround for a git security patch
git config --global --add safe.directory /root/flink
git submodule update --init --recursive

cd docs

# setup hugo
source setup_hugo.sh

# Setup the external documentation modules
source setup_docs.sh

cd ..

# Build the docs
hugo --source docs

# generate docs into docs/target
hugo -v --source docs --destination target
if [ $? -ne 0 ]; then
echo "Error building the docs"
exit 1
fi
./docs/build_docs.sh build

# build Flink; required for Javadoc step
mvn clean install -B -DskipTests -Dfast -Pskip-webui-build
Expand Down
27 changes: 16 additions & 11 deletions docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,29 +12,34 @@ The Flink documentation uses [Hugo](https://gohugo.io/getting-started/installing

To build the documentation, you can install Hugo locally or use a Docker image.

Both methods require you to execute commands in the directory of this module (`docs/`). The built site is served at http://localhost:1313/.

#### Using Hugo Docker image:
Both methods require you to execute commands in the directory of this module (`docs/`). The built site is served at http://localhost:1313/.:

```sh
$ git submodule update --init --recursive
$ ./setup_docs.sh
$ docker pull jakejarvis/hugo-extended:latest
$ docker run -v $(pwd):/src -p 1313:1313 jakejarvis/hugo-extended:latest server --buildDrafts --buildFuture --bind 0.0.0.0
# installs the Hugo binaries locally
$ ./build_docs.sh install-hugo

# serves the docs locally using a Docker image
$ ./build_docs.sh serve --docker

# serves the docs using a local Hugo installation
$ ./build_docs.sh serve --local

# build docs locally (requires a local Hugo installation)
$ ./build_docs.sh build
```

#### Local Hugo installation:

Make sure you have installed [Hugo](https://gohugo.io/getting-started/installing/) on your system.

```sh
$ ./setup_hugo.sh
$ ./build_docs.sh docker-build
```

Then build the docs from source:

```sh
$ ./build_docs.sh
$ ./build_docs.sh local-build
```

The site can be viewed at http://localhost:1313/
Expand All @@ -49,8 +54,8 @@ Adding new externally hosted documentation requires the following steps to be ta

1. (If necessary) Move the existing documentation to the new repository

2. In the Flink repository, edit the `docs/setup_docs.sh` file and add a reference to your now
externally hosted documentation. The reference will look like `integrate_connector_docs <connector_name> <branch_or_tag>`.
2. In the Flink repository, edit the `docs/connector_docs.conf` file and add a reference to your now
externally hosted documentation. The reference will look like `<connector_name> <branch_or_tag>`.

Replace <connector_name> with the name of your connector, e.g., `elasticsearch` for `flink-connector-elasticsearch`.

Expand Down
71 changes: 61 additions & 10 deletions docs/build_docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,64 @@
# limitations under the License.
################################################################################

if ! command -v hugo &> /dev/null
then
echo "Hugo must be installed to run the docs locally"
echo "Please see docs/README.md for more details"
exit 1
fi
git submodule update --init --recursive
./setup_docs.sh
hugo mod get -u
hugo -b "" serve
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"

source "${SCRIPT_DIR}/docs_utils.sh"

function fetch_submodules() {
# the git command has to happen from within the git checkout
pushd ${SCRIPT_DIR} || exit
git submodule update --init --recursive
popd || exit

load_connector_docs_from_file "${SCRIPT_DIR}/connector_docs.conf" "${SCRIPT_DIR}/tmp" "${SCRIPT_DIR}/themes/connectors"
}

SCRIPT_COMMAND="$1"
shift 1

HUGO_VERSION="0.110.0"
HUGO_COMMAND="hugo"
case "${SCRIPT_COMMAND}" in
"install-hugo")
install_hugo "${HUGO_VERSION}"
;;
"serve")
if [[ "${1}" == "--local" ]]; then
assert_hugo_installation "${HUGO_VERSION}"
elif [[ "${1}" == "--docker" ]]; then
# overwrites HUGO_COMMAND to utilize docker container
HUGO_COMMAND="docker run -v ${SCRIPT_DIR}:/src -p 1313:1313 jakejarvis/hugo-extended:${HUGO_VERSION}"
else
echo "[ERROR] Unexpected parameter for serve: $*"
exit 1
fi

fetch_submodules
eval "${HUGO_COMMAND} mod get -u"
eval "${HUGO_COMMAND} server --buildDrafts --buildFuture --bind 0.0.0.0 --baseURL ''"

popd || exit
;;
"build")
assert_hugo_installation "${HUGO_VERSION}"
fetch_submodules

source_dir="${SCRIPT_DIR}"
dest_dir="${SCRIPT_DIR}/../target"
${HUGO_COMMAND} --source "${source_dir}"
${HUGO_COMMAND} --verbose --source "${source_dir}" --destination "${dest_dir}"
echo "[INFO] Flink docs build from ${source_dir} into ${dest_dir}."

if [ $? -ne 0 ]; then
echo "[ERROR] An error occurred while building the Flink docs."
exit 1
fi
;;
*)
echo "Usage: (install-hugo|serve|local-build) [<Hugo parameters>]"
echo " install-hugo Installs Hugo ${HUGO_VERSION} locally."
echo " serve (--local|--docker) Builds the docs and starts a local webserver (either using the local Hugo binaries or a Hugo Docker image)."
echo " build Builds the docs (this step is only supported using a local Hugo installation)."
;;
esac
25 changes: 17 additions & 8 deletions docs/setup_hugo.sh → docs/connector_docs.conf
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,20 @@
# limitations under the License.
################################################################################

# setup hugo
HUGO_REPO=https://github.com/gohugoio/hugo/releases/download/v0.110.0/hugo_extended_0.110.0_Linux-64bit.tar.gz
HUGO_ARTIFACT=hugo_extended_0.110.0_Linux-64bit.tar.gz
if ! curl --fail -OL $HUGO_REPO ; then
echo "Failed to download Hugo binary"
exit 1
fi
tar -zxvf $HUGO_ARTIFACT -C /usr/local/bin
# space-separated list for the connector docs
# <connector-name> should match the connector's GitHub repository apache/flink-<connector-name>
# <branch-ref> refers to the branch the docs should be fetched from
#
# row format of this file: <connector-name> <branch-ref>

elasticsearch v3.0
aws v4.2
cassandra v3.1
pulsar v4.0
jdbc v3.1
rabbitmq v3.0
gcp-pubsub v3.0
mongodb v1.0
opensearch v1.1
kafka v3.0
hbase v3.0
88 changes: 88 additions & 0 deletions docs/docs_utils.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#!/usr/bin/env bash
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

function install_hugo {
local hugo_version hugo_binary_arch hugo_artifact hugo_download_url install_dir

hugo_version="$1"
hugo_binary_arch="Linux-64bit"

hugo_artifact="hugo_extended_${hugo_version}_${hugo_binary_arch}.tar.gz"
hugo_download_url="https://github.com/gohugoio/hugo/releases/download/v${hugo_version}/${hugo_artifact}"
if ! curl --silent --fail -OL "${hugo_download_url}" ; then
echo "Failed to download Hugo ${hugo_version} binary"
exit 1
fi

install_dir="/usr/local/bin"
tar -zxf "${hugo_artifact}" -C "${install_dir}"
echo "[INFO] Hugo (v${hugo_version}) installed to '${install_dir}'."
rm "${hugo_artifact}"
}

function load_connector_docs {
local connector_name ref temp_dir theme_dir
connector_name=$1
ref=$2
temp_dir=$3
theme_dir=$4

pushd "${temp_dir}" || exit
git clone --single-branch --branch "${ref}" "https://github.com/apache/flink-connector-${connector_name}"
rsync -a flink-connector-${connector_name}/docs/* "${theme_dir}/"
popd || exit
}

function load_connector_docs_from_file {
local config_file temp_dir theme_dir

config_file="$1"
temp_dir="$2"
theme_dir="$3"

rm -rf "${theme_dir}"
mkdir -p "${theme_dir}"

rm -rf "${temp_dir}"
mkdir -p "${temp_dir}"

while read -r line; do
if [[ "${line}" =~ ^# ]] || [[ "${line}" =~ ^[^a-zA-Z]*$ ]]; then
# skip comments or empty lines
continue
fi

connector_name="$(echo $line | cut -d' ' -f1)"
ref="$(echo $line | cut -d' ' -f2)"
load_connector_docs "${connector_name}" "${ref}" "${temp_dir}" "${theme_dir}"
done < "${config_file}"
}

function assert_hugo_installation {
local expected_hugo_version
expected_hugo_version="${1}"
if ! which hugo &> /dev/null ; then
echo "[ERROR] No hugo binaries installed."
exit 1
elif [[ "$(hugo version | grep -c "hugo v${expected_hugo_version}")" == 0 ]]; then
echo "[ERROR] Invalid Hugo version installed: (expected ${expected_hugo_version}):"
hugo version
exit 1
fi
}
59 changes: 0 additions & 59 deletions docs/setup_docs.sh

This file was deleted.

15 changes: 2 additions & 13 deletions tools/ci/docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,5 @@
# limitations under the License.
################################################################################

git submodule update --init --recursive
# Setup the external documentation modules
cd docs
source setup_hugo.sh
source setup_docs.sh
cd ..
# Build the docs
hugo --source docs

if [ $? -ne 0 ]; then
echo "Error building the docs"
exit 1
fi
./docs/build_docs.sh install-hugo
./docs/build_docs.sh build

0 comments on commit 26e9f4c

Please sign in to comment.