Skip to content

Commit

Permalink
fix(digests): do not mandate sha256 as the only algorithm used for ha…
Browse files Browse the repository at this point in the history 
…shing blobs

Signed-off-by: Andrei Aaron <[email protected]>
  • Loading branch information
@andaaron
andaaron committed Nov 28, 2023
1 parent 0de2210 commit 9eed9c5
Show file tree
Hide file tree
Showing 14 changed files with 482 additions and 133 deletions.
239 changes: 239 additions & 0 deletions pkg/api/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10533,6 +10533,245 @@ func RunAuthorizationTests(t *testing.T, client *resty.Client, baseURL, user str
})
}

func TestSupportedDigestAlgorithms(t *testing.T) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)

conf := config.New()
conf.HTTP.Port = port

dir := t.TempDir()

ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = dir
ctlr.Config.Storage.Dedupe = false
ctlr.Config.Storage.GC = false

cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()

Check failure on line 10552 in pkg/api/controller_test.go

View workflow job for this annotation

GitHub Actions / lint 

only one cuddle assignment allowed before defer statement (wsl)

Convey("Test SHA512 single-arch image", t, func() {
image := CreateImageWithDigestAlgorithm(godigest.SHA512).
RandomLayers(1, 10).DefaultConfig().Build()

name := "algo-sha256"
tag := "singlearch"

err := UploadImage(image, baseURL, name, tag)
So(err, ShouldBeNil)

client := resty.New()

// The server picks canonical digests when tags are pushed
// See https://github.com/opencontainers/distribution-spec/issues/494
// It would be nice to be able to push tags with other digest algorithms and verify those are returned
// but there is no way to specify a client preference
// so all we can do is verify the correct algorithm is returned

expectedDigestStr := image.DigestForAlgorithm(godigest.Canonical).String()

headResponse, err := client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag))
So(err, ShouldBeNil)
So(headResponse, ShouldNotBeNil)
So(headResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr := headResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err := client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err = client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err = client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)
})

Convey("Test SHA384 single-arch image", t, func() {
image := CreateImageWithDigestAlgorithm(godigest.SHA384).
RandomLayers(1, 10).DefaultConfig().Build()

name := "algo-sha384"
tag := "singlearch"

err := UploadImage(image, baseURL, name, tag)
So(err, ShouldBeNil)

client := resty.New()

// The server picks canonical digests when tags are pushed
// See https://github.com/opencontainers/distribution-spec/issues/494
// It would be nice to be able to push tags with other digest algorithms and verify those are returned
// but there is no way to specify a client preference
// so all we can do is verify the correct algorithm is returned

expectedDigestStr := image.DigestForAlgorithm(godigest.Canonical).String()

headResponse, err := client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag))
So(err, ShouldBeNil)
So(headResponse, ShouldNotBeNil)
So(headResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr := headResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err := client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err = client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err = client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)
})

Convey("Test SHA512 multi-arch image", t, func() {
subImage1 := CreateImageWithDigestAlgorithm(godigest.SHA512).RandomLayers(1, 10).
DefaultConfig().Build()
subImage2 := CreateImageWithDigestAlgorithm(godigest.SHA512).RandomLayers(1, 10).
DefaultConfig().Build()
multiarch := CreateMultiarchWithDigestAlgorithm(godigest.SHA512).
Images([]Image{subImage1, subImage2}).Build()

name := "algo-sha256"
tag := "multiarch"

err := UploadMultiarchImage(multiarch, baseURL, name, tag)
So(err, ShouldBeNil)

client := resty.New()

// The server picks canonical digests when tags are pushed
// See https://github.com/opencontainers/distribution-spec/issues/494
// It would be nice to be able to push tags with other digest algorithms and verify those are returned
// but there is no way to specify a client preference
// so all we can do is verify the correct algorithm is returned

expectedDigestStr := multiarch.DigestForAlgorithm(godigest.Canonical).String()

headResponse, err := client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag))
So(err, ShouldBeNil)
So(headResponse, ShouldNotBeNil)
So(headResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr := headResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err := client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err = client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err = client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)
})

Convey("Test SHA384 multi-arch image", t, func() {
subImage1 := CreateImageWithDigestAlgorithm(godigest.SHA384).RandomLayers(1, 10).
DefaultConfig().Build()
subImage2 := CreateImageWithDigestAlgorithm(godigest.SHA384).RandomLayers(1, 10).
DefaultConfig().Build()
multiarch := CreateMultiarchWithDigestAlgorithm(godigest.SHA384).
Images([]Image{subImage1, subImage2}).Build()

name := "algo-sha384"
tag := "multiarch"

err := UploadMultiarchImage(multiarch, baseURL, name, tag)
So(err, ShouldBeNil)

client := resty.New()

// The server picks canonical digests when tags are pushed
// See https://github.com/opencontainers/distribution-spec/issues/494
// It would be nice to be able to push tags with other digest algorithms and verify those are returned
// but there is no way to specify a client preference
// so all we can do is verify the correct algorithm is returned

expectedDigestStr := multiarch.DigestForAlgorithm(godigest.Canonical).String()

headResponse, err := client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag))
So(err, ShouldBeNil)
So(headResponse, ShouldNotBeNil)
So(headResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr := headResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err := client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err = client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)

getResponse, err = client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr))
So(err, ShouldBeNil)
So(getResponse, ShouldNotBeNil)
So(getResponse.StatusCode(), ShouldEqual, http.StatusOK)

canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest")
So(canonicalDigestStr, ShouldEqual, expectedDigestStr)
})
}

func getEmptyImageConfig() ([]byte, godigest.Digest) {
config := ispec.Image{}

Expand Down
53 changes: 31 additions & 22 deletions pkg/storage/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,19 +64,19 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc

func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte,
log zlog.Logger,
) (godigest.Digest, error) {
) error {
// validate the manifest
if !IsSupportedMediaType(mediaType) {
log.Debug().Interface("actual", mediaType).
Msg("bad manifest media type")

return "", zerr.ErrBadManifest
return zerr.ErrBadManifest

Check warning on line 73 in pkg/storage/common/common.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/common/common.go#L73 

Added line #L73 was not covered by tests
}

if len(body) == 0 {
log.Debug().Int("len", len(body)).Msg("invalid body length")

return "", zerr.ErrBadManifest
return zerr.ErrBadManifest

Check warning on line 79 in pkg/storage/common/common.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/common/common.go#L79 

Added line #L79 was not covered by tests
}

switch mediaType {
Expand All @@ -87,13 +87,13 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy
if err := ValidateManifestSchema(body); err != nil {
log.Error().Err(err).Msg("OCIv1 image manifest schema validation failed")

return "", zerr.NewError(zerr.ErrBadManifest).AddDetail("jsonSchemaValidation", err.Error())
return zerr.NewError(zerr.ErrBadManifest).AddDetail("jsonSchemaValidation", err.Error())
}

if err := json.Unmarshal(body, &manifest); err != nil {
log.Error().Err(err).Msg("unable to unmarshal JSON")

return "", zerr.ErrBadManifest
return zerr.ErrBadManifest

Check warning on line 96 in pkg/storage/common/common.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/common/common.go#L96 

Added line #L96 was not covered by tests
}

// validate blobs only for known media types
Expand All @@ -104,7 +104,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy
if !ok || err != nil {
log.Error().Err(err).Str("digest", manifest.Config.Digest.String()).Msg("missing config blob")

return "", zerr.ErrBadManifest
return zerr.ErrBadManifest
}

// validate layers - a lightweight check if the blob is present
Expand All @@ -120,7 +120,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy
if !ok || err != nil {
log.Error().Err(err).Str("digest", layer.Digest.String()).Msg("missing layer blob")

return "", zerr.ErrBadManifest
return zerr.ErrBadManifest
}
}
}
Expand All @@ -129,49 +129,58 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy
if err := json.Unmarshal(body, &m); err != nil {
log.Error().Err(err).Msg("unable to unmarshal JSON")

return "", zerr.ErrBadManifest
return zerr.ErrBadManifest
}
case ispec.MediaTypeImageIndex:
// validate manifest
if err := ValidateImageIndexSchema(body); err != nil {
log.Error().Err(err).Msg("OCIv1 image index manifest schema validation failed")

return "", zerr.NewError(zerr.ErrBadManifest).AddDetail("jsonSchemaValidation", err.Error())
return zerr.NewError(zerr.ErrBadManifest).AddDetail("jsonSchemaValidation", err.Error())
}

var indexManifest ispec.Index
if err := json.Unmarshal(body, &indexManifest); err != nil {
log.Error().Err(err).Msg("unable to unmarshal JSON")

return "", zerr.ErrBadManifest
return zerr.ErrBadManifest

Check warning on line 146 in pkg/storage/common/common.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/common/common.go#L146 

Added line #L146 was not covered by tests
}

for _, manifest := range indexManifest.Manifests {
if ok, _, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil {
log.Error().Err(err).Str("digest", manifest.Digest.String()).Msg("missing manifest blob")

return "", zerr.ErrBadManifest
return zerr.ErrBadManifest
}
}
}

return "", nil
return nil
}

func GetAndValidateRequestDigest(body []byte, digestStr string, log zlog.Logger) (godigest.Digest, error) {
bodyDigest := godigest.FromBytes(body)
// Returns the canonical digest or the digest provided by the reference if any
// Per spec, the canonical digest would always be returned to the client in
// request headers, but that does not make sense if the client requested a different digest algorithm
// See https://github.com/opencontainers/distribution-spec/issues/494
func GetAndValidateRequestDigest(body []byte, reference string, log zlog.Logger) (
godigest.Digest, error,
) {
expectedDigest, err := godigest.Parse(reference)
if err != nil {
// This is a non-digest reference
return godigest.Canonical.FromBytes(body), err
}

actualDigest := expectedDigest.Algorithm().FromBytes(body)

d, err := godigest.Parse(digestStr)
if err == nil {
if d.String() != bodyDigest.String() {
log.Error().Str("actual", bodyDigest.String()).Str("expected", d.String()).
Msg("manifest digest is not valid")
if expectedDigest.String() != actualDigest.String() {
log.Error().Str("actual", actualDigest.String()).Str("expected", expectedDigest.String()).
Msg("manifest digest is not valid")

return "", zerr.ErrBadManifest
}
return actualDigest, zerr.ErrBadManifest
}

return bodyDigest, err
return actualDigest, nil
}

/*
Expand Down
Loading

0 comments on commit 9eed9c5

Please sign in to comment.