Introduce initial version of Dart-based site tooling (#5472)

This tooling is accessible from the root directory of the repository by
calling `dart run dart_site <command> <args>`.

It replaces our various scripts and will be the home of future staging
and deploy commands after the migration to 11ty.

To learn more about the commands available, run `dart run dart_site
--help`. You can add a shortcut for `dart run dart_site` if you'd like,
or I'll likely come up with a standard shortcut we can use as part of
followup.

I'll improve the formatting of the output and add a proper guide after
the 11ty migration lands, as I expect to change some in that process.

Contributes to #4163 and
#5177

.github/workflows/test.yml

@@ -43,24 +43,39 @@ jobs:
       - uses: dart-lang/setup-dart@ca7e6fee45ffbd82b555a7ebfc236d2c86439f5b
         with:
           sdk: ${{ matrix.sdk }}
-      - run: dart pub get
-      - run: tool/test.sh
-        env:
-          DART_CHANNEL: ${{ matrix.sdk }}
+      - name: Fetch packages
+        run: dart pub get
+      - name: Check Dart code formatting
+        run: dart run dart_site format-dart --check
+      - name: Analyze Dart code
+        run: dart run dart_site analyze-dart
+      - name: Run Dart tests
+        run: dart run dart_site test-dart
+      - name: Check if excerpts are up to date
+        run: dart run dart_site refresh-excerpts --fail-on-update
 
   linkcheck:
     name: Build site and check links
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
         with:
           submodules: recursive
       - run: make build
+      - uses: dart-lang/setup-dart@ca7e6fee45ffbd82b555a7ebfc236d2c86439f5b
+        with:
+          sdk: stable
+      - name: Fetch packages
+        run: dart pub get
+      - name: Check for broken Markdown links
+        run: dart run dart_site check-link-references
+      - name: Validate the firebase.json file
+        run: dart run dart_site verify-firebase-json
       - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8
         with:
           node-version: ${{ env.NODE_VERSION }}
       - run: npm install -g [email protected]
-      - uses: dart-lang/setup-dart@ca7e6fee45ffbd82b555a7ebfc236d2c86439f5b
-        with:
-          sdk: stable
-      - run: tool/check-links.sh
+      - name: Check internal site links are functional
+        run: dart run dart_site check-links

Dockerfile

@@ -57,27 +57,12 @@ RUN set -eu; \
     BASEURL="https://storage.googleapis.com/dart-archive/channels"; \
     URL="$BASEURL/$DART_CHANNEL/release/$DART_VERSION/sdk/$SDK"; \
     curl -fsSLO "$URL"; \
-    echo "$DART_SHA256 *$SDK" | sha256sum --check --status --strict - || (\
-        echo -e "\n\nDART CHECKSUM FAILED! Run 'make fetch-sums' for updated values.\n\n" && \
-        rm "$SDK" && \
-        exit 1 \
-    ); \
     unzip "$SDK" > /dev/null && mv dart-sdk "$DART_SDK" && rm "$SDK";
 ENV PUB_CACHE="${HOME}/.pub-cache"
 RUN dart --disable-analytics
 RUN echo -e "Successfully installed Dart SDK:" && dart --version
 
 
-# ============== DART-TESTS ==============
-from dart as dart-tests
-WORKDIR /app
-COPY ./ ./
-RUN dart pub get
-ENV BASE_DIR=/app
-ENV TOOL_DIR=$BASE_DIR/tool
-CMD ["./tool/test.sh"]
-
-
 # ============== NODEJS INSTALL ==============
 FROM dart as node
 

Makefile

@@ -2,9 +2,8 @@
 -include .env
 
 
-all: clean up down debug shell serve test-build test-run setup \
-	serve emulate stage test build-image build deploy deploy-ci \
-	fetch-sums test-builds test-run
+all: clean up down shell serve test-build test-run run setup \
+	serve emulate stage test build-image build deploy deploy-ci
 
 .PHONY: all
 .DEFAULT_GOAL := up
@@ -67,15 +66,6 @@ serve:
 		--incremental \
 		--trace
 
-# Run all tests inside a built container
-test:
-	DOCKER_BUILDKIT=1 docker build \
-		-t dart-tests:${DART_CHANNEL} \
-		--target dart-tests \
-		--build-arg DART_VERSION=${DART_VERSION} \
-		--build-arg DART_CHANNEL=${DART_CHANNEL} .
-	docker run --rm -v ${PWD}:/app dart-tests:${DART_CHANNEL}
-
 # Build docker image with optional target
 # Usage: `make build-image [BUILD_CONFIGS=<config1,[config2,]>]`
 build-image:
@@ -133,38 +123,3 @@ emulate:
 	npx firebase emulators:start \
 		--only hosting \
 		--project ${FIREBASE_PROJECT}
-
-
-
-################## UTILS ##################
-
-# Fetch SDK sums for current Dart SDKs by arch
-# This outputs a bash case format to be copied to Dockerfile
-fetch-sums:
-	tool/fetch-dart-sdk-sums.sh \
-		--version ${DART_VERSION} \
-		--channel ${DART_CHANNEL}
-
-# Check Dart sums pulls the set of Dart SDK SHA256 hashes
-# and writes them to a temp file.
-check-sums:
-	tool/check-dart-sdk.sh
-
-# Update Dart sums replaces the Dart SDK SHA256 hashes
-# in the Dockerfile and deletes the temp file.
-update-sums:
-	tool/update-dart-sdk.sh
-
-# Test the dev container with pure docker
-test-builds:
-	docker build -t ${BUILD_TAG}:stable \
-		--no-cache --target=dart-tests .
-	docker build -t ${BUILD_TAG}:beta \
-		--no-cache --target=dart-tests --build-arg DART_CHANNEL=beta .
-	docker build -t ${BUILD_TAG}:dev \
-		--no-cache --target=dart-tests --build-arg DART_CHANNEL=dev .
-
-# Test stable run with volume
-TEST_CHANNEL =? stable
-test-run:
-	docker run --rm -it -v ${PWD}:/app ${BUILD_TAG}:${TEST_CHANNEL} bash

README.md

@@ -164,17 +164,14 @@ _choose one_ of the following submodule-cloning techniques:
 ### Checking documentation and example code
 
 If you've made changes to this site's documentation and/or example code,
-and committed locally, then run the following command before pushing your work:
+and committed locally, then run the following commands before pushing your work:
 
 ```terminal
 # Enter a running Docker container shell
 $ make run
 
 # Check/validate example code
-$ tool/test.sh
-
-# Check links for 404 errors
-$ tool/check-links.sh
+$ dart run dart_site check-all
 ```
 
 If these scripts report errors or warnings,
@@ -226,44 +223,6 @@ personal Firebase hosting staging site as follows:
   the staged version, the names of your reviewers, and so on.
 
 
-## Creating and/or editing DartPad example code
-
-Most of the code used to create [DartPad][] examples is hosted on GitHub.
-However, this repo also contains some `*.dart` files
-responsible for DartPad example code.
-
-### DartPad picker
-
-The DartPad example picker must be manually compiled if changes are made.
-This will regenerate the associated JavaScript file in `src/assets/dash/js`:
-
-```terminal
-$ tool/compile.sh
-```
-
-## Dockerfile Maintenance
-
-### Dart SDK and Node PPA Checksum values
-
-Since the Dart SDK setup fetches remote files,
-it's important to verify checksum values.
-Both installs use `latest` and `lts` respectively,
-so these files may be periodically updated.
-When this happens,
-local checksums may fail and **This will break the Docker/Compose setup/build**.
-You will see the relevant output in your shell e.g. `DART CHECKSUM FAILED!...`.
-When this happens, run the following command:
-
-```terminal
-$ make fetch-sums
-```
-
-This command will output the updated checksum values for Dart,
-and that output will be formatted similar
-or the same as what is currently in the Dockerfile.
-Copy this output and replace the relevant install code in the Dockerfile,
-then rerun your setup/build again.
-
 [Build Status SVG]: https://github.com/dart-lang/site-www/workflows/build/badge.svg
 [OpenSSF Scorecard SVG]: https://api.securityscorecards.dev/projects/github.com/dart-lang/site-www/badge
 [Scorecard Results]: https://deps.dev/project/github/dart-lang%2Fsite-www

pubspec.yaml

@@ -1,6 +1,5 @@
 name: site_www
 publish_to: none
-
 homepage: https://dart.dev
 
 environment:
@@ -12,5 +11,6 @@ dev_dependencies:
     path: site-shared/packages/code_excerpt_updater
   code_excerpter:
     path: site-shared/packages/code_excerpter
-  linkcheck: ^3.0.0
+  dart_site:
+    path: tool/dart_site
   lints: ^3.0.0