diff --git a/src/docker_clojure/config.clj b/src/docker_clojure/config.clj index 18b894a..5a606c5 100644 --- a/src/docker_clojure/config.clj +++ b/src/docker_clojure/config.clj @@ -55,14 +55,9 @@ "debian" #{:debian-slim/bookworm-slim :debian/bookworm :debian-slim/bullseye-slim :debian/bullseye}}) -(def default-architectures +(def architectures #{"amd64" "arm64v8"}) -(def distro-architectures - "Map of distro types to architectures it supports if different from - default-architectures." - {:alpine #{"amd64"}}) - (def default-distros "The default distro to use for tags that don't specify one, keyed by jdk-version. :default is a fallback for jdk versions not o/w specified." @@ -84,9 +79,10 @@ "1.12.0.1479" "94f29b9b66183bd58307c46fb561fd9e9148666bac13a4518a9931b6f989d830"}}) (def exclusions ; don't build these for whatever reason(s) - #{;; commented out example - #_{:jdk-version 8 - :distro :alpine/alpine}}) + #{;; No upstream ARM alpine images available before JDK 21 + {:jdk-version #(< % 21) + :architecture "arm64v8" + :distro :alpine/alpine}}) (def maintainers ["Paul Lam (@Quantisan)" diff --git a/src/docker_clojure/core.clj b/src/docker_clojure/core.clj index a1e2a38..c8ac88d 100644 --- a/src/docker_clojure/core.clj +++ b/src/docker_clojure/core.clj @@ -1,158 +1,51 @@ (ns docker-clojure.core (:require + [clojure.core.async :refer [ distro namespace keyword)) - base {:jdk-version jdk-version - :base-image base-image - :base-image-tag (base-image-tag base-image - jdk-version distro) - :distro distro - :build-tool build-tool - :build-tool-version build-tool-version - :maintainer (str/join " & " cfg/maintainers)}] - (-> base - (assoc :docker-tag (default-docker-tag base)) - (assoc-if #(nil? (:build-tool-version base)) :build-tool-versions - cfg/build-tools) - (assoc-if #(seq variant-arch) :architectures variant-arch)))) - -(defn pull-image [image] - (sh "docker" "pull" image)) + :opt-un [::cfg/build-tool-versions ::cfg/architecture])) -(defn generate-dockerfile! [installer-hashes variant] - (let [build-dir (df/build-dir variant) - filename "Dockerfile"] - (log "Generating" (str build-dir "/" filename)) - (df/write-file build-dir filename installer-hashes variant) - (assoc variant - :build-dir build-dir - :dockerfile filename))) - -(defn build-image - [installer-hashes {:keys [docker-tag base-image architectures] :as variant}] - (let [image-tag (str "clojure:" docker-tag) - _ (log "Pulling base image" base-image) - _ (pull-image base-image) - - {:keys [dockerfile build-dir]} - (generate-dockerfile! installer-hashes variant) - - host-arch (let [jvm-arch (System/getProperty "os.arch")] - (if (= "aarch64" jvm-arch) - "arm64v8" - jvm-arch)) - platform-flag (if (contains? (or architectures - cfg/default-architectures) - host-arch) - nil - (str "--platform=linux/" (first architectures))) - - build-cmd (remove nil? ["docker" "buildx" "build" "--no-cache" - "-t" image-tag platform-flag "--load" - "-f" dockerfile "."])] - (apply log "Running" build-cmd) - (let [{:keys [out err exit]} - (with-sh-dir build-dir (apply sh build-cmd))] - (if (zero? exit) - (log "Succeeded building" (str "clojure:" docker-tag)) - (log "ERROR building" (str "clojure:" docker-tag ":") err out)))) - (log) - [::done variant]) - -(def latest-variant +(def latest-variants "The latest variant is special because we include all 3 build tools via the [::all] value on the end." - (list (-> cfg/base-images :default first) - cfg/default-jdk-version - (get-or-default cfg/default-distros cfg/default-jdk-version) - [::all])) - -(defn image-variant-combinations - [base-images jdk-versions distros build-tools] - (reduce - (fn [variants jdk-version] - (concat - variants - (let [jdk-base-images (get-or-default base-images jdk-version)] - (loop [[bi & r] jdk-base-images - acc #{}] - (let [vs (combo/cartesian-product #{bi} - #{jdk-version} - (get-or-default distros bi) - build-tools) - acc' (concat acc vs)] - (if (seq r) - (recur r acc') - acc')))))) - #{} jdk-versions)) + (for [arch cfg/architectures] + (list (-> cfg/base-images :default first) + cfg/default-jdk-version + (get-or-default cfg/default-distros cfg/default-jdk-version) + [::all] + arch))) (defn image-variants - [base-images jdk-versions distros build-tools] + [base-images jdk-versions distros build-tools architectures] (into #{} (comp - (map variant-map) + (map variant/->map) (remove #(= ::s/invalid (s/conform ::variant %)))) - (conj - (image-variant-combinations base-images jdk-versions distros - build-tools) - latest-variant))) + (concat + (variant/combinations base-images jdk-versions distros build-tools + architectures) + latest-variants))) (defn rand-delay "Runs argument f w/ any supplied args after a random delay of 100-1000 ms" @@ -164,12 +57,12 @@ (defn build-images [parallelization installer-hashes variants] (log "Building images" parallelization "at a time") - (let [variants-ch (to-chan! variants) - builds-ch (chan parallelization)] + (let [variants-ch (to-chan! variants) + builds-ch (chan parallelization)] ;; Kick off builds with a random delay so we don't have Docker race ;; conditions (e.g. build container name collisions) (async/thread (pipeline-blocking parallelization builds-ch - (map (partial rand-delay build-image + (map (partial rand-delay docker/build-image installer-hashes)) variants-ch)) (while (> ["git" "rev-parse" "HEAD"] (apply sh) :out) target-file (or (first args) :stdout) manifest (manifest/generate {:maintainers cfg/maintainers - :architectures cfg/default-architectures + :architectures cfg/architectures :git-repo cfg/git-repo} git-head variants)] (log "Writing manifest of" (count variants) "variants to" target-file "...") @@ -199,24 +92,6 @@ (when (not= :stdout target-file) (.close output-writer))))) -(defn sort-variants - [variants] - (sort - (fn [v1 v2] - (cond - (= "latest" (:docker-tag v1)) -1 - (= "latest" (:docker-tag v2)) 1 - :else (let [c (compare (:jdk-version v1) (:jdk-version v2))] - (if (not= c 0) - c - (let [c (compare (full-docker-tag v1) (full-docker-tag v2))] - (if (not= c 0) - c - (throw - (ex-info "No two variants should have the same full Docker tag" - {:v1 v1, :v2 v2})))))))) - variants)) - (defn generate-variants [args] (let [key-vals (->> args @@ -239,7 +114,7 @@ (case cmd :clean (df/clean-all) :dockerfiles (generate-dockerfiles! cfg/installer-hashes variants) - :manifest (-> variants sort-variants (generate-manifest! args)) + :manifest (generate-manifest! variants args) :build-images (build-images parallelization cfg/installer-hashes variants))) (logger/stop)) diff --git a/src/docker_clojure/docker.clj b/src/docker_clojure/docker.clj new file mode 100644 index 0000000..955cf27 --- /dev/null +++ b/src/docker_clojure/docker.clj @@ -0,0 +1,115 @@ +(ns docker-clojure.docker + (:require [clojure.java.shell :refer [sh with-sh-dir]] + [clojure.string :as str] + [docker-clojure.config :as cfg] + [docker-clojure.core :as-alias core] + [docker-clojure.dockerfile :as df] + [docker-clojure.util :refer [get-or-default]] + [docker-clojure.log :refer [log]])) + +(defn pull-image [image] + (sh "docker" "pull" image)) + +(defn build-image + [installer-hashes {:keys [docker-tag base-image architecture] :as variant}] + (let [image-tag (str "clojure:" docker-tag) + _ (log "Pulling base image" base-image) + _ (pull-image base-image) + + {:keys [dockerfile build-dir]} + (df/generate! installer-hashes variant) + + host-arch (let [jvm-arch (System/getProperty "os.arch")] + (if (= "aarch64" jvm-arch) + "arm64v8" + jvm-arch)) + platform-flag (if (= architecture host-arch) + nil + (str "--platform=linux/" architecture)) + + build-cmd (remove nil? ["docker" "buildx" "build" "--no-cache" + "-t" image-tag platform-flag "--load" + "-f" dockerfile "."])] + (apply log "Running" build-cmd) + (let [{:keys [out err exit]} + (with-sh-dir build-dir (apply sh build-cmd))] + (if (zero? exit) + (log "Succeeded building" (str "clojure:" docker-tag)) + (log "ERROR building" (str "clojure:" docker-tag ":") err out)))) + (log) + [::done variant]) + +(defn base-image-tag + [base-image jdk-version distro] + (str base-image ":" + (case base-image + "eclipse-temurin" (str jdk-version "-jdk-") + "debian" "" + "-") + (name distro))) + +(defn jdk-label + [omit-default? jdk-version base-image] + (if (and omit-default? (= cfg/default-jdk-version jdk-version) + (= (first (get-or-default cfg/base-images jdk-version)) + base-image)) + nil + (str + (case base-image + ("eclipse-temurin" "debian") "temurin" + base-image) + "-" jdk-version))) + +(defn tag + "Returns the Docker tag for the given variant with truthy keys from first arg + left out when possible." + [{:keys [omit-all? omit-jdk? omit-build-tool? omit-build-tool-version? + omit-distro?]} + {:keys [base-image jdk-version distro build-tool + build-tool-version] :as _variant}] + (if (= ::core/all build-tool) + "latest" + (let [jdk (jdk-label (or omit-all? omit-jdk?) + jdk-version base-image) + dd (get-or-default cfg/default-distros jdk-version) + distro-label (if (and (or omit-all? omit-distro?) (= dd distro)) + nil + (when distro (name distro))) + tag-elements (remove nil? [jdk distro-label]) + build-tool-label (if (and (seq tag-elements) ; ensure tag is non-empty + (or omit-all? omit-build-tool?) + (= build-tool cfg/default-build-tool)) + nil + build-tool) + build-tool-version-label (if (or omit-all? omit-build-tool? + omit-build-tool-version?) + nil + build-tool-version)] + (str/join "-" (remove nil? [jdk build-tool-label + build-tool-version-label + distro-label]))))) + +(def full-tag + (partial tag {})) + +(def default-tag + (partial tag {:omit-jdk? true, :omit-distro? true})) + +(defn all-tags + "Returns all Docker tags for the give variant" + [variant] + (let [short-tag (:docker-tag variant) + full-tag (full-tag variant) + base (into #{} [short-tag full-tag])] + (-> base + (conj + (tag {:omit-jdk? true} variant) + (tag {:omit-build-tool? true} variant) + (tag {:omit-build-tool-version? true} variant) + (tag {:omit-distro? true} variant) + (tag {:omit-distro? true, :omit-build-tool-version? true} variant) + (tag {:omit-jdk? true, :omit-build-tool-version? true} variant) + (tag {:omit-jdk? true, :omit-distro? true + :omit-build-tool-version? true} variant)) + vec + sort))) diff --git a/src/docker_clojure/dockerfile.clj b/src/docker_clojure/dockerfile.clj index ab3321f..e2aa056 100644 --- a/src/docker_clojure/dockerfile.clj +++ b/src/docker_clojure/dockerfile.clj @@ -4,7 +4,8 @@ [clojure.string :as str] [docker-clojure.dockerfile.lein :as lein] [docker-clojure.dockerfile.tools-deps :as tools-deps] - [docker-clojure.dockerfile.shared :refer [copy-resource-file! entrypoint]])) + [docker-clojure.dockerfile.shared :refer [copy-resource-file! entrypoint]] + [docker-clojure.log :refer [log]])) (defn build-dir [{:keys [base-image-tag jdk-version build-tool]}] (str/join "/" ["target" @@ -79,5 +80,14 @@ (throw (ex-info (str "Error creating directory " dir) {:error err}))))) +(defn generate! [installer-hashes variant] + (let [build-dir (build-dir variant) + filename "Dockerfile"] + (log "Generating" (str build-dir "/" filename)) + (write-file build-dir filename installer-hashes variant) + (assoc variant + :build-dir build-dir + :dockerfile filename))) + (defn clean-all [] (sh "sh" "-c" "rm -rf target/*")) diff --git a/src/docker_clojure/manifest.clj b/src/docker_clojure/manifest.clj index 7adb77f..f74a132 100644 --- a/src/docker_clojure/manifest.clj +++ b/src/docker_clojure/manifest.clj @@ -1,26 +1,8 @@ (ns docker-clojure.manifest (:require [clojure.string :as str] + [docker-clojure.docker :as docker] [docker-clojure.dockerfile :as df] - [docker-clojure.util :refer [docker-tag full-docker-tag]])) - -(defn variant-tags - "Generates all the Docker Hub tag variations for the given variant" - [variant] - (let [short-tag (:docker-tag variant) - full-tag (full-docker-tag variant) - base (into #{} [short-tag full-tag])] - (-> base - (conj - (docker-tag {:omit-jdk? true} variant) - (docker-tag {:omit-build-tool? true} variant) - (docker-tag {:omit-build-tool-version? true} variant) - (docker-tag {:omit-distro? true} variant) - (docker-tag {:omit-distro? true, :omit-build-tool-version? true} variant) - (docker-tag {:omit-jdk? true, :omit-build-tool-version? true} variant) - (docker-tag {:omit-jdk? true, :omit-distro? true - :omit-build-tool-version? true} variant)) - vec - sort))) + [docker-clojure.variant :as variant])) (defn variant->manifest [variant] @@ -28,19 +10,24 @@ (conj (remove nil? [(str/join " " (conj ["Tags:"] (->> variant - variant-tags + docker/all-tags (str/join ", ")))) (when-let [arch (:architectures variant)] - (str/join " " ["Architectures:" (str/join ", " arch)])) - (str/join " " ["Directory:" (df/build-dir variant)])]) + (str/join " " ["Architectures:" + (str/join ", " arch)])) + (str/join " " ["Directory:" + (df/build-dir variant)])]) nil))) (defn generate "Generates Docker manifest file for a given git commit and returns it as a string." [{:keys [maintainers architectures git-repo]} git-commit variants] - (let [maintainers-label "Maintainers:" - maintainers-sep (apply str ",\n" (repeat (inc (count maintainers-label)) " "))] + (let [merged-arch-variants (variant/merge-architectures variants + architectures) + maintainers-label "Maintainers:" + maintainers-sep (apply str ",\n" (repeat (inc (count maintainers-label)) + " "))] (str/join "\n" (concat [(str/join " " [maintainers-label @@ -49,7 +36,6 @@ (str/join " " ["GitRepo:" git-repo]) (str/join " " ["GitCommit:" git-commit])] - (map variant->manifest variants) + (map variant->manifest merged-arch-variants) [nil])))) - diff --git a/src/docker_clojure/util.clj b/src/docker_clojure/util.clj index cbe128b..ec28ec8 100644 --- a/src/docker_clojure/util.clj +++ b/src/docker_clojure/util.clj @@ -7,50 +7,3 @@ "Returns the value in map m for key k or else the value for key :default." [m k] (get m k (get m :default))) - -(defn jdk-label - [omit-default? jdk-version base-image] - (if (and omit-default? (= cfg/default-jdk-version jdk-version) - (= (first (get-or-default cfg/base-images jdk-version)) - base-image)) - nil - (str - (case base-image - ("eclipse-temurin" "debian") "temurin" - base-image) - "-" jdk-version))) - -(defn docker-tag - "Returns the Docker tag for the given variant with truthy keys from first arg - left out when possible." - [{:keys [omit-all? omit-jdk? omit-build-tool? omit-build-tool-version? - omit-distro?]} - {:keys [base-image jdk-version distro build-tool - build-tool-version] :as _variant}] - (if (= ::core/all build-tool) - "latest" - (let [jdk (jdk-label (or omit-all? omit-jdk?) - jdk-version base-image) - dd (get-or-default cfg/default-distros jdk-version) - distro-label (if (and (or omit-all? omit-distro?) (= dd distro)) - nil - (when distro (name distro))) - tag-elements (remove nil? [jdk distro-label]) - build-tool-label (if (and (seq tag-elements) ; ensure tag is non-empty - (or omit-all? omit-build-tool?) - (= build-tool cfg/default-build-tool)) - nil - build-tool) - build-tool-version-label (if (or omit-all? omit-build-tool? - omit-build-tool-version?) - nil - build-tool-version)] - (str/join "-" (remove nil? [jdk build-tool-label - build-tool-version-label - distro-label]))))) - -(def full-docker-tag - (partial docker-tag {})) - -(def default-docker-tag - (partial docker-tag {:omit-jdk? true, :omit-distro? true})) diff --git a/src/docker_clojure/variant.clj b/src/docker_clojure/variant.clj new file mode 100644 index 0000000..82ff1c3 --- /dev/null +++ b/src/docker_clojure/variant.clj @@ -0,0 +1,113 @@ +(ns docker-clojure.variant + (:refer-clojure :exclude [compare sort]) + (:require [clojure.math.combinatorics :as combo] + [clojure.string :as str] + [docker-clojure.config :as cfg] + [docker-clojure.docker :as docker] + [docker-clojure.util :refer [get-or-default]])) + +(defn assoc-if + [m pred k v] + (if (pred) + (assoc m k v) + m)) + +(defn ->map [[base-image jdk-version distro + [build-tool build-tool-version] architecture]] + (let [base {:jdk-version jdk-version + :architecture architecture + :base-image base-image + :base-image-tag (docker/base-image-tag base-image jdk-version + distro) + :distro distro + :build-tool build-tool + :build-tool-version build-tool-version + :maintainer (str/join " & " cfg/maintainers)}] + (-> base + (assoc :docker-tag (docker/default-tag base)) + (assoc-if #(nil? (:build-tool-version base)) :build-tool-versions + cfg/build-tools)))) + +(defn exclude? + "Returns true if the map `variant` contains every key-value pair in the map + `exclusion`. `variant` may contain additional keys that are not in + `exclusion`. Some values of `exclusion` can also be a predicate of one + argument which is then tested against the respective value from `variant`. + Returns false if any of the keys in `exclusions` are missing from `variant` or + have different values, or the predicate value returned false." + [variant exclusion] + (every? (fn [[k v]] + (if (fn? v) + (v (get variant k)) + (= v (get variant k)))) + exclusion)) + +(defn compare + [v1 v2] + (let [c (clojure.core/compare (:jdk-version v1) (:jdk-version v2))] + (if (not= c 0) + c + (let [c (clojure.core/compare (docker/full-tag v1) + (docker/full-tag v2))] + (if (not= c 0) + c + (clojure.core/compare (:architecture v1) (:architecture v2))))))) + +(defn sort + [variants] + (clojure.core/sort + (fn [v1 v2] + (cond + (= "latest" (:docker-tag v1)) -1 + (= "latest" (:docker-tag v2)) 1 + :else (compare v1 v2))) + variants)) + +(defn equal? + [v1 v2] + (= 0 (compare v1 v2))) + +(defn equal-except-architecture? + [v1 v2] + (= 0 (compare (dissoc v1 :architecture) (dissoc v2 :architecture)))) + +(defn combinations + [base-images jdk-versions distros build-tools architectures] + (reduce + (fn [variants jdk-version] + (concat + variants + (let [jdk-base-images (get-or-default base-images jdk-version)] + (loop [[bi & r] jdk-base-images + acc #{}] + (let [vs (combo/cartesian-product #{bi} #{jdk-version} + (get-or-default distros bi) + build-tools + architectures) + acc' (concat acc vs)] + (if (seq r) + (recur r acc') + acc')))))) + #{} jdk-versions)) + +(defn merge-architectures + [variants default-architectures] + (->> variants + (map #(assoc % :architectures #{(:architecture %)})) + (reduce + (fn [mav v] + (if-let [matching + (some #(when + (equal-except-architecture? v %) + %) + mav)] + (-> mav + (->> (remove #(= % matching))) + (conj (update matching :architectures conj + (:architecture v)))) + (conj mav v))) + []) + (map #(if (= (:architectures %) default-architectures) + (dissoc % :architectures :architecture) + (dissoc % :architecture))) + sort))