From 17b1a28f663d9b243911c60092ef9ccb3bed1ea5 Mon Sep 17 00:00:00 2001 From: Pat Maddox Date: Fri, 13 Dec 2024 11:57:30 -0800 Subject: [PATCH 1/4] build: Sort distributekernel METALOG when using -DNO_ROOT The metalog is produced by install -M, which is not sorted. This results in non-deterministic file ordering in kernel.txz. Order the files in kernel.txz to support reproducible builds. PR: 283214 Signed-off-by: Pat Maddox --- Makefile.inc1 | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 5c3d190e4c3ede..dc4d9e72b6b483 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1903,9 +1903,7 @@ distributekernel distributekernel.debug: .PHONY false .endif mkdir -p ${DESTDIR}/${DISTDIR} -.if defined(NO_ROOT) - @echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.premeta -.endif + rm -f ${DESTDIR}/${DISTDIR}/kernel.premeta ${_+_}cd ${KRNLOBJDIR}/${INSTALLKERNEL}; \ ${IMAKEENV} ${IMAKE_INSTALL:S/METALOG/kernel.premeta/} \ ${IMAKE_MTREE} PATH=${TMPPATH:Q} ${MAKE} KERNEL=${INSTKERNNAME} \ @@ -1913,15 +1911,14 @@ distributekernel distributekernel.debug: .PHONY METALOG=${METALOG:S/METALOG/kernel.premeta/} \ ${.TARGET:S/distributekernel/install/} .if defined(NO_ROOT) - @sed -e 's|^./kernel|.|' ${DESTDIR}/${DISTDIR}/kernel.premeta > \ - ${DESTDIR}/${DISTDIR}/kernel.meta + echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.meta + sed -e 's|^./kernel|.|' ${DESTDIR}/${DISTDIR}/kernel.premeta | \ + ${METALOG_SORT_CMD} >> ${DESTDIR}/${DISTDIR}/kernel.meta .endif .endif .if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} -.if defined(NO_ROOT) - @echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta -.endif + rm -f ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta ${_+_}cd ${KRNLOBJDIR}/${_kernel}; \ ${IMAKEENV} ${IMAKE_INSTALL:S/METALOG/kernel.${_kernel}.premeta/} \ ${IMAKE_MTREE} PATH=${TMPPATH:Q} ${MAKE} \ @@ -1930,9 +1927,10 @@ distributekernel distributekernel.debug: .PHONY METALOG=${METALOG:S/METALOG/kernel.${_kernel}.premeta/} \ ${.TARGET:S/distributekernel/install/} .if defined(NO_ROOT) - @sed -e "s|^./kernel.${_kernel}|.|" \ - ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta > \ - ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta + echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta + sed -e "s|^./kernel.${_kernel}|.|" \ + ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta | \ + ${METALOG_SORT_CMD} >> ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta .endif .endfor .endif From a6fbdd754929749ff654a08469e80a17f2f25bac Mon Sep 17 00:00:00 2001 From: Pat Maddox Date: Fri, 13 Dec 2024 12:12:52 -0800 Subject: [PATCH 2/4] build: Set METALOG time when using -DWITH_REPRODUCIBLE_BUILD When using -DWITHOUT_REPRODUCIBLE_BUILD, files in *.txz will be added with their built timestamps. This results in non-deterministic tarballs. -DWITH_REPRODUCIBLE_BUILD sets the time in the METALOG, so that all files have the time defined by PKG_TIMESTAMP. kernel is added to its metalog via install -M, which does not support configuring the mtree keys. Rewrite the time for its metalog entry. PR: 283214 Signed-off-by: Pat Maddox --- Makefile.inc1 | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index dc4d9e72b6b483..9e3a6c805d4fea 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1441,7 +1441,11 @@ DEBUG_DISTRIBUTIONS= DEBUG_DISTRIBUTIONS+= base ${EXTRA_DISTRIBUTIONS:S,tests,,} .endif -MTREE_MAGIC?= mtree 2.0 +MTREE_MAGIC?= mtree 2.0 +.if ${MK_REPRODUCIBLE_BUILD} == "yes" +MTREE_TIME= time=${SOURCE_DATE_EPOCH}.000000000 +SED_REPLACE_TIME= -e "s/ time=[[:digit:].]*/ ${MTREE_TIME}/" +.endif distributeworld installworld stageworld: _installcheck_world .PHONY mkdir -p ${INSTALLTMP} @@ -1470,6 +1474,7 @@ distributeworld installworld stageworld: _installcheck_world .PHONY .if defined(NO_ROOT) -mkdir -p ${METALOG:H} echo "#${MTREE_MAGIC}" > ${METALOG} + echo "/set ${MTREE_TIME}" >> ${METALOG} .endif .if make(distributeworld) .for dist in ${EXTRA_DISTRIBUTIONS} @@ -1548,7 +1553,7 @@ distributeworld installworld stageworld: _installcheck_world .PHONY @# the relevant mtree line. cd ${DESTDIR}/${DISTDIR}; \ find ./${dist}${path} | ${METALOG_SORT_CMD} -u ${METALOG} - | \ - awk 'BEGIN { print "#${MTREE_MAGIC}" } !/ type=/ { file = $$1 } / type=/ { if ($$1 == file) { sub(/^\.\/${dist}/, "."); print } }' > \ + awk 'BEGIN { print "#${MTREE_MAGIC}"; print "/set ${MTREE_TIME}" } !/ type=/ { file = $$1 } / type=/ { if ($$1 == file) { sub(/^\.\/${dist}/, "."); print } }' > \ ${DESTDIR}/${DISTDIR}/${dist}${suffix} .endfor .endfor @@ -1912,7 +1917,8 @@ distributekernel distributekernel.debug: .PHONY ${.TARGET:S/distributekernel/install/} .if defined(NO_ROOT) echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.meta - sed -e 's|^./kernel|.|' ${DESTDIR}/${DISTDIR}/kernel.premeta | \ + echo "/set ${MTREE_TIME}" >> ${DESTDIR}/${DISTDIR}/kernel.meta + sed -e 's|^./kernel|.|' ${SED_REPLACE_TIME} ${DESTDIR}/${DISTDIR}/kernel.premeta | \ ${METALOG_SORT_CMD} >> ${DESTDIR}/${DISTDIR}/kernel.meta .endif .endif @@ -1928,7 +1934,8 @@ distributekernel distributekernel.debug: .PHONY ${.TARGET:S/distributekernel/install/} .if defined(NO_ROOT) echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta - sed -e "s|^./kernel.${_kernel}|.|" \ + echo "/set ${MTREE_TIME}" >> ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta + sed -e "s|^./kernel.${_kernel}|.|" ${SED_REPLACE_TIME} \ ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta | \ ${METALOG_SORT_CMD} >> ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta .endif From a5eb00d3814d07207b58095cabd4d00335e6b16a Mon Sep 17 00:00:00 2001 From: Pat Maddox Date: Fri, 13 Dec 2024 15:19:25 -0800 Subject: [PATCH 3/4] build: Wait to build mandoc.db when making distributeworld distributeworld builds man pages using distribute, rather than install. Check for distribute, and wait to build etc. This ensures that makedb is called after the man pages have been distributed. PR: 283214 Signed-off-by: Pat Maddox --- Makefile.inc1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 9e3a6c805d4fea..43a3b7bf26d07f 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -462,7 +462,7 @@ SUBDIR+= ${_DIR} # by calling 'makedb' in share/man. This is only relevant for # install/distribute so they build the whatis file after every manpage is # installed. -.if make(installworld) || make(install) +.if make(installworld) || make(install) || make(distribute) SUBDIR+=.WAIT .endif SUBDIR+=etc From 6bc903d5c9cfd89fa8d05c1f895fa2d3a59d4f1b Mon Sep 17 00:00:00 2001 From: Pat Maddox Date: Fri, 13 Dec 2024 18:43:58 -0800 Subject: [PATCH 4/4] release: Document reproducible release tarballs PR: 283214 Signed-off-by: Pat Maddox --- share/man/man7/release.7 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/share/man/man7/release.7 b/share/man/man7/release.7 index 5c5c11efd0fbb9..0f4cd6897b112e 100644 --- a/share/man/man7/release.7 +++ b/share/man/man7/release.7 @@ -283,6 +283,16 @@ This is intended for use only when .Fa /usr/ports is expected to exist by alternative means. .El +.Sh REPRODUCIBLE BUILDS +Different builders can produce bit-identical release tarballs using +.Va PKG_TIMESTAMP +and +.Va REVISION : +.Bd -literal -offset indent +git checkout +make -DWITH_REPRODUCIBLE_BUILD buildworld buildkernel +make -C release -DWITH_REPRODUCIBLE_BUILD PKG_TIMESTAMP= packagesystem +.Ed .Sh EMBEDDED BUILDS The following .Fa release.conf