Merge pull request #1 from AkihiroSuda/dev

dockerfile: download repro-get binary by default

.github/workflows/release.yml

@@ -37,9 +37,9 @@ jobs:
           (Changes to be documented)
 
           ## About the files
-          - repro-get-<VERSION>-linux-amd64:          Regular binary for x86_64
-          - repro-get-<VERSION>-linux-amd64.upx:      Compressed binary for x86_64
-          - repro-get-<VERSION>.go-mod-vendor.tar.gz: Backup of \`go mod vendor\` (Not needed for most people)
+          - repro-get-\<VERSION\>-linux-amd64:          Regular binary for x86_64
+          - repro-get-\<VERSION\>-linux-amd64.upx:      Compressed binary for x86_64
+          - repro-get-\<VERSION\>.go-mod-vendor.tar.gz: Backup of \`go mod vendor\` (Not needed for most people)
           - - -
           The binaries were built automatically on GitHub Actions.
           The build log is available for 90 days: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

Makefile

@@ -3,14 +3,16 @@ PREFIX ?= /usr/local
 DEST := $(shell echo "$(DESTDIR)/$(PREFIX)" | sed 's:///*:/:g; s://*$$::')
 
 VERSION ?=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
+# DOWNLOADABLE_VERSION <= VERSION
+DOWNLOADABLE_VERSION ?= $(shell git describe --abbrev=0)
 PACKAGE := github.com/reproducible-containers/repro-get
 
 export CGO_ENABLED ?= 0
 export DOCKER_BUILDKIT := 1
 export SOURCE_DATE_EPOCH ?= $(shell git log -1 --pretty=%ct)
 
 GO ?= go
-GO_BUILD ?= $(GO) build -trimpath -ldflags="-s -w -X $(PACKAGE)/pkg/version.Version=$(VERSION)"
+GO_BUILD ?= $(GO) build -trimpath -ldflags="-s -w -X $(PACKAGE)/pkg/version.Version=$(VERSION) -X $(PACKAGE)/pkg/version.DownloadableVersion=$(DOWNLOADABLE_VERSION)"
 DOCKER ?= docker
 DOCKER_BUILD ?= $(DOCKER) build
 UPX ?= upx --best --lzma

README.md

@@ -157,20 +157,14 @@ The following example produces an image with `gcc`, using the packages from 2021
 # Generate "Dockerfile.generate-hash" and "Dockerfile" in the current directory
 repro-get --distro=debian dockerfile generate . debian:bullseye-20211220 gcc build-essential
 
-# Copy the repro-get binary into the current directory
-cp $(command -v repro-get) ./repro-get.linux-amd64
-
-# Enable BuildKit
+ Enable BuildKit
 export DOCKER_BUILDKIT=1
 
 # Generate "SHA256SUMS-amd64" file in the current directory (needed by the next step)
 docker build --output . -f Dockerfile.generate-hash .
 
 # Build the image
 docker build .
-
-# Clean up
-rm -f repro-get.linux-amd64
 ```
 
 See [`./examples/gcc`](./examples/gcc) for an example output.

cmd/repro-get/dockerfile_generate.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"os"
 	"regexp"
@@ -11,25 +12,20 @@ import (
 	"github.com/reproducible-containers/repro-get/pkg/archutil"
 	"github.com/reproducible-containers/repro-get/pkg/distro"
 	"github.com/reproducible-containers/repro-get/pkg/ocidistutil"
+	"github.com/reproducible-containers/repro-get/pkg/version"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
 func helpForBuildingDockerfiles(needsToGenerateHash bool) string {
-	const tmpl = `# Copy the repro-get binary into the current directory
-cp $(command -v repro-get) ./repro-get.linux-{{.OCIArchDashVariant}}
-
-# Enable BuildKit
+	const tmpl = `# Enable BuildKit
 export DOCKER_BUILDKIT=1
 {{if .NeedsToGenerateHash}}
 # Generate "SHA256SUMS-{{.OCIArchDashVariant}}" in the current directory
 docker build --output . -f Dockerfile.generate-hash .
 {{else}}{{end}}
 # Build the image
 docker build .
-
-# Clean up
-rm -f ./repro-get.linux-{{.OCIArchDashVariant}}
 `
 	parsed, err := template.New("").Parse(tmpl)
 	if err != nil {
@@ -103,12 +99,22 @@ func dockerfileGenerateAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	if version.DownloadableVersion == "" {
+		return errors.New("variable pkg/version.DownloadableVersion is unset (Hint: rebuild the binary with the upstream Makefile)")
+	}
+	shasha, err := version.SHASHA(ctx, version.DownloadableVersion)
+	if err != nil {
+		return fmt.Errorf("failed to get the URL of the SHA256SUMS file for version %q: %w", version.DownloadableVersion, err)
+	}
+
 	templateArgs := distro.DockerfileTemplateArgs{
 		BaseImage:          resolvedWithDigest,
 		BaseImageOrig:      baseImageOrig,
 		Packages:           pkgs,
 		OCIArchDashVariant: archutil.OCIArchDashVariant(),
 		Providers:          providers,
+		ReproGetVersion:    version.DownloadableVersion,
+		ReproGetSHASHA:     shasha,
 	}
 	opts := distro.DockerfileOpts{
 		GenerateHash: len(pkgs) > 0,

examples/Makefile

@@ -14,9 +14,7 @@ endef
 
 define gen_full
 	$(REPRO_GET) $(REPRO_GET_FLAGS) dockerfile generate ./$(1) $(BASE_IMAGE) $(shell cat ./$(1)/PACKAGES)
-	cp -f $(REPRO_GET) ./$(1)/repro-get.linux-$(BUILDARCH)
 	$(DOCKER_BUILD) --output type=local,dest=./$(1) --platform=$(PLATFORMS) -f ./$(1)/Dockerfile.generate-hash ./$(1)
-	rm -f ./$(1)/repro-get.linux-$(BUILDARCH)
 	$(REPRO_GET) $(REPRO_GET_FLAGS) dockerfile generate ./$(1) $(BASE_IMAGE)
 endef
 

examples/gcc/Dockerfile

@@ -7,19 +7,44 @@
 # Usage:
 # Make sure that the hash file "SHA256SUMS-amd64" is present in the current directory.
 # ----------------------------------------------------------
-# cp $(command -v repro-get) ./repro-get.linux-amd64
 # export DOCKER_BUILDKIT=1
 # docker build .
 # ----------------------------------------------------------
 
 ARG BASE_IMAGE=docker.io/library/debian:bullseye-20211220@sha256:2906804d2a64e8a13a434a1a127fe3f6a28bf7cf3696be4223b06276f32f1f2d # debian:bullseye-20211220
 ARG REPRO_GET_PROVIDER=http://deb.debian.org/debian/{{.Name}},http://deb.debian.org/debian-security/{{.Name}},http://debian.notset.fr/snapshot/by-hash/SHA256/{{.SHA256}}
 
-FROM scratch AS repro-get
+ARG REPRO_GET_VERSION=v0.1.0
+ARG REPRO_GET_SHA256SUMS_SHA256SUM=a23ee0e0a2a2e940809b968befc84aa928323c86d3f4eef1f1653c96c2861632
+# "download" or "local"
+ARG REPRO_GET_FETCH_MODE=download
+ARG REPRO_GET_BINARY_URL=https://github.com/reproducible-containers/repro-get/releases/download/${REPRO_GET_VERSION}/repro-get-${REPRO_GET_VERSION}.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}
+ARG REPRO_GET_SHA256SUMS_URL=https://github.com/reproducible-containers/repro-get/releases/download/${REPRO_GET_VERSION}/SHA256SUMS
+
+FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE} AS repro-get-download
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG REPRO_GET_VERSION
+ARG REPRO_GET_BINARY_URL
+ARG REPRO_GET_SHA256SUMS_URL
+ARG REPRO_GET_SHA256SUMS_SHA256SUM
+ADD ${REPRO_GET_BINARY_URL} .
+ADD ${REPRO_GET_SHA256SUMS_URL} .
+RUN \
+  echo "${REPRO_GET_SHA256SUMS_SHA256SUM}  SHA256SUMS" >SHASHA && \
+  sha256sum -c SHASHA && \
+  sha256sum --ignore-missing -c SHA256SUMS && \
+  mv repro-get-${REPRO_GET_VERSION}.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}} repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}  && \
+  chmod +x repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}
+
+FROM scratch AS repro-get-local
 ARG TARGETARCH
 ARG TARGETVARIANT
 COPY repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}} /
 
+FROM repro-get-${REPRO_GET_FETCH_MODE} AS repro-get
+
+
 FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE}
 ARG TARGETARCH
 ARG TARGETVARIANT

examples/gcc/Dockerfile.generate-hash

@@ -6,7 +6,6 @@
 
 # Usage:
 # ----------------------------------------------------------
-# cp $(command -v repro-get) ./repro-get.linux-amd64
 # export DOCKER_BUILDKIT=1
 # docker build --output . -f Dockerfile.generate-hash .
 # ----------------------------------------------------------
@@ -17,17 +16,46 @@
 ARG BASE_IMAGE=docker.io/library/debian:bullseye-20211220@sha256:2906804d2a64e8a13a434a1a127fe3f6a28bf7cf3696be4223b06276f32f1f2d # debian:bullseye-20211220
 ARG PACKAGES="gcc build-essential"
 ARG SNAPSHOT_ARCHIVE_BASE=http://snapshot.debian.org/archive/
+# Backports do not seem cached in https://debian.notset.fr/
+ARG BACKPORTS=0
 
-FROM scratch AS repro-get
+ARG REPRO_GET_VERSION=v0.1.0
+ARG REPRO_GET_SHA256SUMS_SHA256SUM=a23ee0e0a2a2e940809b968befc84aa928323c86d3f4eef1f1653c96c2861632
+# "download" or "local"
+ARG REPRO_GET_FETCH_MODE=download
+ARG REPRO_GET_BINARY_URL=https://github.com/reproducible-containers/repro-get/releases/download/${REPRO_GET_VERSION}/repro-get-${REPRO_GET_VERSION}.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}
+ARG REPRO_GET_SHA256SUMS_URL=https://github.com/reproducible-containers/repro-get/releases/download/${REPRO_GET_VERSION}/SHA256SUMS
+
+FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE} AS repro-get-download
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG REPRO_GET_VERSION
+ARG REPRO_GET_BINARY_URL
+ARG REPRO_GET_SHA256SUMS_URL
+ARG REPRO_GET_SHA256SUMS_SHA256SUM
+ADD ${REPRO_GET_BINARY_URL} .
+ADD ${REPRO_GET_SHA256SUMS_URL} .
+RUN \
+  echo "${REPRO_GET_SHA256SUMS_SHA256SUM}  SHA256SUMS" >SHASHA && \
+  sha256sum -c SHASHA && \
+  sha256sum --ignore-missing -c SHA256SUMS && \
+  mv repro-get-${REPRO_GET_VERSION}.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}} repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}  && \
+  chmod +x repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}
+
+FROM scratch AS repro-get-local
 ARG TARGETARCH
 ARG TARGETVARIANT
 COPY repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}} /
 
+FROM repro-get-${REPRO_GET_FETCH_MODE} AS repro-get
+
+
 FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE} AS generate-hash
 ARG PACKAGES
 ARG SNAPSHOT_ARCHIVE_BASE
 ARG TARGETARCH
 ARG TARGETVARIANT
+ARG BACKPORTS
 SHELL ["/bin/bash", "-c"]
 RUN \
   --mount=type=cache,target=/var/cache/apt \
@@ -42,7 +70,7 @@ RUN \
   echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME} main" >/etc/apt/sources.list && \
   echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian-security/${snapshot} ${VERSION_CODENAME}-security main" >>/etc/apt/sources.list && \
   echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-updates main" >>/etc/apt/sources.list && \
-  echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-backports main" >>/etc/apt/sources.list && \
+  if [ "${BACKPORTS}" = 1 ] ; then echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-backports main" >>/etc/apt/sources.list; fi&& \
   rm -f /etc/apt/apt.conf.d/docker-clean && \
   echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
   apt-get update && \

examples/gcc/README.md

@@ -5,9 +5,6 @@ The files in this directory were generated and verified with the following comma
 # Generate "Dockerfile.generate-hash" and "Dockerfile" in the current directory
 repro-get --distro=debian dockerfile generate . debian:bullseye-20211220 $(cat PACKAGES)
 
-# Copy the repro-get binary into the current directory
-cp $(command -v repro-get) ./repro-get.linux-amd64
-
 # Enable BuildKit
 export DOCKER_BUILDKIT=1
 
@@ -16,7 +13,4 @@ docker build --output . -f Dockerfile.generate-hash .
 
 # Build the image
 docker build .
-
-# Clean up
-rm -f ./repro-get.linux-amd64
 ```

examples/gcc/SHA256SUMS-amd64

@@ -36,7 +36,7 @@ a9a5e1f53b7e27a3f2b8388929bb622d3c6c35a4e42ac166697444e5ed662fd5  pool/main/g/gc
 5e7a75814131f83334ef795e93f48c2b035df592a7cca49c0c339b8520a967e7  pool/main/libt/libtirpc/libtirpc-dev_1.3.1-1_amd64.deb
 4ea9ce5df578309538e684fcbca0c4403c6d7ca01b3f44868829e98a55548e2a  pool/main/g/gcc-10/libtsan0_10.2.1-6_amd64.deb
 9435da9b51aba0f9b128ccf3e80cc1e8233aa6b816d7bddd5688ca6c4afb6e4d  pool/main/g/gcc-10/libubsan1_10.2.1-6_amd64.deb
-e12ce36f6a15a3be1595261ff5a0d26bd615483b6f2ccfe17a9aa5999ff689e2  pool/main/l/linux/linux-libc-dev_5.14.9-2~bpo11+1_amd64.deb
+1bb053863873916cb8d5fa877cc4972a6279931783c1fd9e4339d0369a617af4  pool/main/l/linux/linux-libc-dev_5.10.84-1_amd64.deb
 a1a83af8cbd854af887b72ad196b1f4af58387815e21ced1000253a116a46e2a  pool/main/m/make-dfsg/make_4.3-4.1_amd64.deb
 8c6d49b771530dbe26d7bd060582dc7d2b4eeb603a20789debc1ef4bbbc4ef67  pool/main/p/patch/patch_2.7.6-7_amd64.deb
 6fa15be322c3c89ec4a07d704ad58d4a2d1aabf866135a859f6d8d58c59e9df4  pool/main/p/perl/perl-modules-5.32_5.32.1-4+deb11u2_all.deb

examples/hello/Dockerfile

@@ -7,19 +7,44 @@
 # Usage:
 # Make sure that the hash file "SHA256SUMS-amd64" is present in the current directory.
 # ----------------------------------------------------------
-# cp $(command -v repro-get) ./repro-get.linux-amd64
 # export DOCKER_BUILDKIT=1
 # docker build .
 # ----------------------------------------------------------
 
 ARG BASE_IMAGE=docker.io/library/debian:bullseye-20211220@sha256:2906804d2a64e8a13a434a1a127fe3f6a28bf7cf3696be4223b06276f32f1f2d # debian:bullseye-20211220
 ARG REPRO_GET_PROVIDER=http://deb.debian.org/debian/{{.Name}},http://deb.debian.org/debian-security/{{.Name}},http://debian.notset.fr/snapshot/by-hash/SHA256/{{.SHA256}}
 
-FROM scratch AS repro-get
+ARG REPRO_GET_VERSION=v0.1.0
+ARG REPRO_GET_SHA256SUMS_SHA256SUM=a23ee0e0a2a2e940809b968befc84aa928323c86d3f4eef1f1653c96c2861632
+# "download" or "local"
+ARG REPRO_GET_FETCH_MODE=download
+ARG REPRO_GET_BINARY_URL=https://github.com/reproducible-containers/repro-get/releases/download/${REPRO_GET_VERSION}/repro-get-${REPRO_GET_VERSION}.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}
+ARG REPRO_GET_SHA256SUMS_URL=https://github.com/reproducible-containers/repro-get/releases/download/${REPRO_GET_VERSION}/SHA256SUMS
+
+FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE} AS repro-get-download
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG REPRO_GET_VERSION
+ARG REPRO_GET_BINARY_URL
+ARG REPRO_GET_SHA256SUMS_URL
+ARG REPRO_GET_SHA256SUMS_SHA256SUM
+ADD ${REPRO_GET_BINARY_URL} .
+ADD ${REPRO_GET_SHA256SUMS_URL} .
+RUN \
+  echo "${REPRO_GET_SHA256SUMS_SHA256SUM}  SHA256SUMS" >SHASHA && \
+  sha256sum -c SHASHA && \
+  sha256sum --ignore-missing -c SHA256SUMS && \
+  mv repro-get-${REPRO_GET_VERSION}.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}} repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}  && \
+  chmod +x repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}}
+
+FROM scratch AS repro-get-local
 ARG TARGETARCH
 ARG TARGETVARIANT
 COPY repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}} /
 
+FROM repro-get-${REPRO_GET_FETCH_MODE} AS repro-get
+
+
 FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE}
 ARG TARGETARCH
 ARG TARGETVARIANT

examples/hello/README.md

@@ -7,15 +7,9 @@ The `Dockerfile` was generated and verified with the following commands:
 # Generate "Dockerfile" in the current directory
 repro-get --distro=debian dockerfile generate . debian:bullseye-20211220
 
-# Copy the repro-get binary into the current directory
-cp $(command -v repro-get) ./repro-get.linux-amd64
-
 # Enable BuildKit
 export DOCKER_BUILDKIT=1
 
 # Build the image
 docker build .
-
-# Clean up
-rm -f ./repro-get.linux-amd64
 ```

pkg/distro/debian/Dockerfile.generate-hash.tmpl

@@ -6,7 +6,6 @@
 
 # Usage:
 # ----------------------------------------------------------
-# cp $(command -v repro-get) ./repro-get.linux-{{.OCIArchDashVariant}}
 # export DOCKER_BUILDKIT=1
 # docker build --output . -f Dockerfile.generate-hash .
 # ----------------------------------------------------------
@@ -17,17 +16,19 @@
 ARG BASE_IMAGE={{.BaseImage}} # {{.BaseImageOrig}}
 ARG PACKAGES="{{join .Packages " "}}"
 ARG SNAPSHOT_ARCHIVE_BASE=http://snapshot.debian.org/archive/
+# Backports do not seem cached in https://debian.notset.fr/
+ARG BACKPORTS=0
 
-FROM scratch AS repro-get
-ARG TARGETARCH
-ARG TARGETVARIANT
-COPY repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}} /
+ARG REPRO_GET_VERSION={{.ReproGetVersion}}
+ARG REPRO_GET_SHA256SUMS_SHA256SUM={{.ReproGetSHASHA}}
+{{snippet "fetch-repro-get"}}
 
 FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE} AS generate-hash
 ARG PACKAGES
 ARG SNAPSHOT_ARCHIVE_BASE
 ARG TARGETARCH
 ARG TARGETVARIANT
+ARG BACKPORTS
 SHELL ["/bin/bash", "-c"]
 RUN \
   --mount=type=cache,target=/var/cache/apt \
@@ -42,7 +43,7 @@ RUN \
   echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME} main" >/etc/apt/sources.list && \
   echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian-security/${snapshot} ${VERSION_CODENAME}-security main" >>/etc/apt/sources.list && \
   echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-updates main" >>/etc/apt/sources.list && \
-  echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-backports main" >>/etc/apt/sources.list && \
+  if [ "${BACKPORTS}" = 1 ] ; then echo "deb [check-valid-until=no] ${SNAPSHOT_ARCHIVE_BASE}debian/${snapshot} ${VERSION_CODENAME}-backports main" >>/etc/apt/sources.list; fi&& \
   rm -f /etc/apt/apt.conf.d/docker-clean && \
   echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
   apt-get update && \

pkg/distro/debian/Dockerfile.tmpl

@@ -7,18 +7,16 @@
 # Usage:
 # Make sure that the hash file "SHA256SUMS-{{.OCIArchDashVariant}}" is present in the current directory.
 # ----------------------------------------------------------
-# cp $(command -v repro-get) ./repro-get.linux-{{.OCIArchDashVariant}}
 # export DOCKER_BUILDKIT=1
 # docker build .
 # ----------------------------------------------------------
 
 ARG BASE_IMAGE={{.BaseImage}} # {{.BaseImageOrig}}
 ARG REPRO_GET_PROVIDER={{join .Providers ","}}
 
-FROM scratch AS repro-get
-ARG TARGETARCH
-ARG TARGETVARIANT
-COPY repro-get.linux-${TARGETARCH}${TARGETVARIANT:+-${TARGETVARIANT}} /
+ARG REPRO_GET_VERSION={{.ReproGetVersion}}
+ARG REPRO_GET_SHA256SUMS_SHA256SUM={{.ReproGetSHASHA}}
+{{snippet "fetch-repro-get"}}
 
 FROM --platform=${TARGETPLATFORM} ${BASE_IMAGE}
 ARG TARGETARCH