diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5fce8a666e..092c6ddce1 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -72,7 +72,7 @@ global_job_config: - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_DEV_REGISTRY - export LATEST_TAG=$BRANCH_TAG-latest - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" - - export DOCKER_REPOS="confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm" + - export DOCKER_REPOS="confluentinc/cp-base-java confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm" - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 @@ -102,7 +102,7 @@ blocks: - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 - $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true + $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true -Dmaven.test.skip - . cache-maven store - >- for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; @@ -135,7 +135,7 @@ blocks: - ci-tools ci-update-version - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean install dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 - $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true + $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true -Dmaven.test.skip - . cache-maven store - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: diff --git a/base-java/Dockerfile.ubi8 b/base-java/Dockerfile.ubi8 new file mode 100644 index 0000000000..344ebdb3f3 --- /dev/null +++ b/base-java/Dockerfile.ubi8 @@ -0,0 +1,68 @@ +ARG UBI_MICRO_VERSION +ARG TEMURIN_JDK_VERSION +ARG DOCKER_UPSTREAM_REGISTRY +ARG DOCKER_UPSTREAM_TAG +ARG GOLANG_VERSION +ARG UBI_MINIMAL_VERSION + +FROM docker.io/golang:${GOLANG_VERSION} AS build-ub-package-dedupe +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +WORKDIR /build/package_dedupe +COPY --chown=appuser:appuser package_dedupe/ ./ +RUN go build -ldflags="-w -s" ./package_dedupe.go +WORKDIR /build/ub +COPY --chown=appuser:appuser ub/ ./ +RUN go build -ldflags="-w -s" ./ub.go +USER appuser +RUN go test ./... + +FROM registry.access.redhat.com/ubi9-minimal:${UBI_MINIMAL_VERSION} AS REFRESH +ARG PROJECT_VERSION +ARG ARTIFACT_ID + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for new Confluent lightweight Docker images." +LABEL description="Common base image for Confluent lightweight Docker images." +LABEL io.confluent.docker=true +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" + +RUN printf "[temurin-jre] \n\ +name=temurin-jre \n\ +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo + +RUN echo "installing temurin-21-jre:${TEMURIN_JDK_VERSION}" \ + && microdnf install -y temurin-21-jre${TEMURIN_JDK_VERSION} \ + && microdnf clean all \ + && useradd --no-log-init --create-home --shell /bin/bash appuser + +COPY --from=build-ub-package-dedupe /build/package_dedupe /usr/bin/package_dedupe +COPY --from=build-ub-package-dedupe /build/ub /usr/bin/ub + + +COPY target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +COPY --chown=appuser:appuser include/etc/cp-base-java /etc/cp-base-java + + +USER appuser +WORKDIR /home/appuser diff --git a/base-lite/README.md b/base-java/README.md similarity index 100% rename from base-lite/README.md rename to base-java/README.md diff --git a/base-lite/include/etc/confluent/docker/bash-config b/base-java/include/etc/confluent/docker/bash-config similarity index 100% rename from base-lite/include/etc/confluent/docker/bash-config rename to base-java/include/etc/confluent/docker/bash-config diff --git a/base-lite/include/etc/confluent/docker/mesos-setup.sh b/base-java/include/etc/confluent/docker/mesos-setup.sh similarity index 100% rename from base-lite/include/etc/confluent/docker/mesos-setup.sh rename to base-java/include/etc/confluent/docker/mesos-setup.sh diff --git a/base-java/include/etc/cp-base-java/log4j.properties b/base-java/include/etc/cp-base-java/log4j.properties new file mode 120000 index 0000000000..c6aeab5903 --- /dev/null +++ b/base-java/include/etc/cp-base-java/log4j.properties @@ -0,0 +1 @@ +../../../../base/include/etc/cp-base-new/log4j.properties \ No newline at end of file diff --git a/base-java/license.txt b/base-java/license.txt new file mode 100644 index 0000000000..bb7cc57a2e --- /dev/null +++ b/base-java/license.txt @@ -0,0 +1 @@ +Copyright 2023 Confluent, Inc. \ No newline at end of file diff --git a/base-java/package_dedupe/package_dedupe.go b/base-java/package_dedupe/package_dedupe.go new file mode 100644 index 0000000000..7af27b59d4 --- /dev/null +++ b/base-java/package_dedupe/package_dedupe.go @@ -0,0 +1,68 @@ +package main + +import ( + "crypto/sha1" + + "fmt" + + "io" + "log" + "os" + "path/filepath" +) + +func dedupe_packages(rootPath string) { + sha2path := make(map[string]string) + err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil + } + sha, err := shaSum(path) + if err != nil { + return err + } + if orig, exists := sha2path[sha]; exists { + relPath, err := filepath.Rel(filepath.Dir(path), orig) + if err != nil { + return err + } + os.Remove(path) + err = os.Symlink(relPath, path) + if err != nil { + return err + } + log.Printf("DEDUP: ln -sf %s %s\n", orig, path) + } else { + sha2path[sha] = path + } + return nil + }) + if err != nil { + log.Fatal(err) + } +} + +func shaSum(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha1.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func main() { + if len(os.Args) != 2 { + fmt.Println("Usage: dedupe_packages ") + os.Exit(1) + } + basePath := os.Args[1] + dedupe_packages(basePath) +} \ No newline at end of file diff --git a/base-java/pom.xml b/base-java/pom.xml new file mode 100644 index 0000000000..d59938eba4 --- /dev/null +++ b/base-java/pom.xml @@ -0,0 +1,118 @@ + + + + + 4.0.0 + + + io.confluent + common-docker + 8.0.0-0 + + + pom + + cp-base-java + + Base for new Confluent lightweight Docker images + ${project.artifactId} + + + false + false + true + + + + + io.confluent + utility-belt + ${io.confluent.common-docker.version} + + + junit + junit + ${junit.version} + test + + + + + + + maven-resources-plugin + ${maven-resources-plugin.version} + + + copy-resources + compile + + copy-resources + + + true + target + + + ${basedir}/../docker-utils/target + + docker-utils-${CONFLUENT_VERSION}-jar-with-dependencies.jar + + + + + + + + + com.spotify + dockerfile-maven-plugin + + + ${ubi9.image.version} + -${ubi.temurin.jdk.version} + ${docker.skip-security-update-check} + ${golang.version} + + + + + io.fabric8 + docker-maven-plugin + 0.43.4 + + + + + + ${ubi9.image.version} + -${ubi.temurin.jdk.version} + + ${docker.skip-security-update-check} + + ${golang.version} + + + + + + + + + diff --git a/base-lite/requirements.txt b/base-java/requirements.txt similarity index 100% rename from base-lite/requirements.txt rename to base-java/requirements.txt diff --git a/base-lite/setup.py b/base-java/setup.py similarity index 100% rename from base-lite/setup.py rename to base-java/setup.py diff --git a/base-java/test/test_base_java_image.py b/base-java/test/test_base_java_image.py new file mode 100644 index 0000000000..561155734e --- /dev/null +++ b/base-java/test/test_base_java_image.py @@ -0,0 +1,28 @@ +import os +import unittest + +import confluent.docker_utils as utils + + +class BaseJavaImageTest(unittest.TestCase): + + def setUp(self): + self.image = "{0}confluentinc/cp-base-java:{1}".format(os.environ["DOCKER_REGISTRY"], os.environ["DOCKER_TAG"]) + + def test_image_build(self): + self.assertTrue(utils.image_exists(self.image)) + + def test_ub_exists(self): + self.assertTrue(utils.path_exists_in_image(self.image, "/usr/bin/ub")) + + def test_ub_runnable(self): + ub_cmd = "bash -c '/usr/bin/ub/ub -h'" + self.assertTrue(b"utility commands" in utils.run_docker_command(image=self.image, command=ub_cmd)) + + def test_kafka_ready_jar(self): + java_cmd = "bash -c 'java -cp \"/usr/share/java/cp-base-java/*\" io.confluent.admin.utils.cli.KafkaReadyCommand -h'" + self.assertTrue(b"Check if Kafka is ready" in utils.run_docker_command(image=self.image, command=java_cmd)) + + +if __name__ == '__main__': + unittest.main() diff --git a/base-lite/tox.ini b/base-java/tox.ini similarity index 100% rename from base-lite/tox.ini rename to base-java/tox.ini diff --git a/base-lite/ub/go.mod b/base-java/ub/go.mod similarity index 100% rename from base-lite/ub/go.mod rename to base-java/ub/go.mod diff --git a/base-lite/ub/go.sum b/base-java/ub/go.sum similarity index 100% rename from base-lite/ub/go.sum rename to base-java/ub/go.sum diff --git a/base-lite/ub/testResources/sampleFile b/base-java/ub/testResources/sampleFile similarity index 100% rename from base-lite/ub/testResources/sampleFile rename to base-java/ub/testResources/sampleFile diff --git a/base-lite/ub/testResources/sampleFile2 b/base-java/ub/testResources/sampleFile2 similarity index 100% rename from base-lite/ub/testResources/sampleFile2 rename to base-java/ub/testResources/sampleFile2 diff --git a/base-lite/ub/testResources/sampleLog4j.template b/base-java/ub/testResources/sampleLog4j.template similarity index 100% rename from base-lite/ub/testResources/sampleLog4j.template rename to base-java/ub/testResources/sampleLog4j.template diff --git a/base-lite/ub/ub.go b/base-java/ub/ub.go similarity index 100% rename from base-lite/ub/ub.go rename to base-java/ub/ub.go diff --git a/base-lite/ub/ub_test.go b/base-java/ub/ub_test.go similarity index 100% rename from base-lite/ub/ub_test.go rename to base-java/ub/ub_test.go diff --git a/base-lite/Dockerfile.ubi8 b/base-lite/Dockerfile.ubi8 index beb233c355..9f36d40076 100644 --- a/base-lite/Dockerfile.ubi8 +++ b/base-lite/Dockerfile.ubi8 @@ -13,18 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG UBI_MINIMAL_VERSION="latest" -ARG GOLANG_VERSION - -FROM golang:${GOLANG_VERSION} AS build-ub -WORKDIR /build -RUN useradd --no-log-init --create-home --shell /bin/bash appuser -COPY --chown=appuser:appuser ub/ ./ -RUN go build -ldflags="-w -s" ./ub.go -USER appuser -RUN go test ./... - -FROM registry.access.redhat.com/ubi8/ubi-minimal:${UBI_MINIMAL_VERSION} +ARG DOCKER_UPSTREAM_REGISTRY +ARG DOCKER_TAG +FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-java:${DOCKER_TAG} ARG PROJECT_VERSION ARG ARTIFACT_ID @@ -56,24 +47,15 @@ ARG TEMURIN_JDK_VERSION="" ENV UB_CLASSPATH=/usr/share/java/cp-base-lite/* -RUN printf "[temurin-jdk] \n\ -name=temurin-jdk \n\ -baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ -enabled=1 \n\ -gpgcheck=1 \n\ -gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ -" > /etc/yum.repos.d/adoptium.repo - -RUN microdnf --nodocs install yum \ - && yum --nodocs update -y \ - && yum --nodocs install -y --setopt=install_weak_deps=False \ +USER root + +RUN yum --nodocs update -y \ + && yum --nodocs install -y --setopt=install_weak_deps=False --allowerasing \ "curl${CURL_VERSION}" \ - "temurin-21-jre${TEMURIN_JDK_VERSION}" \ && microdnf clean all \ && yum clean all \ && rm -rf /tmp/* \ && mkdir -p /etc/confluent/docker /usr/logs \ - && useradd --no-log-init --create-home --shell /bin/bash appuser \ && chown appuser:appuser -R /etc/confluent/ /usr/logs # This is a step that will cause the build to fail of the package manager detects a package update is availible and isn't installed. @@ -88,12 +70,10 @@ RUN yum --disablerepo="temurin-jdk" check-update || "${SKIP_SECURITY_UPDATE_CHEC COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker COPY --chown=appuser:appuser include/etc/cp-base-lite /etc/cp-base-lite -COPY --from=build-ub /build/ub /usr/bin RUN mkdir /licenses COPY license.txt /licenses USER appuser -WORKDIR /home/appuser +WORKDIR /home/appuser \ No newline at end of file diff --git a/base-lite/test/test_base_lite_image.py b/base-lite/test/test_base_java_image.py similarity index 90% rename from base-lite/test/test_base_lite_image.py rename to base-lite/test/test_base_java_image.py index 404a6db9d3..87e55c2294 100644 --- a/base-lite/test/test_base_lite_image.py +++ b/base-lite/test/test_base_java_image.py @@ -4,7 +4,7 @@ import confluent.docker_utils as utils -class BaseLiteImageTest(unittest.TestCase): +class BaseJavaImageTest(unittest.TestCase): def setUp(self): self.image = "{0}confluentinc/cp-base-lite:{1}".format(os.environ["DOCKER_REGISTRY"], os.environ["DOCKER_TAG"]) @@ -16,7 +16,7 @@ def test_ub_exists(self): self.assertTrue(utils.path_exists_in_image(self.image, "/usr/bin/ub")) def test_ub_runnable(self): - ub_cmd = "bash -c '/usr/bin/ub -h'" + ub_cmd = "bash -c '/usr/bin/ub/ub -h'" self.assertTrue(b"utility commands" in utils.run_docker_command(image=self.image, command=ub_cmd)) def test_kafka_ready_jar(self): diff --git a/pom.xml b/pom.xml index 8b91a611b7..8f1ffc8e34 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ utility-belt docker-utils + base-java base base-lite jmxterm @@ -34,6 +35,7 @@ 8.0.0-0 8.10-1154 + 9.5-1736404155 3.0.9