From b4282a3a11cf2d9adaa091a764c6d53692867331 Mon Sep 17 00:00:00 2001 From: Stefan Wiedemann Date: Fri, 31 May 2024 15:27:32 +0200 Subject: [PATCH 1/5] add debugging --- tir/tirClient.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tir/tirClient.go b/tir/tirClient.go index c35e3e65..0040f266 100644 --- a/tir/tirClient.go +++ b/tir/tirClient.go @@ -3,10 +3,12 @@ package tir import ( "encoding/json" "errors" - "github.com/procyon-projects/chrono" + "io" "net/http" "time" + "github.com/procyon-projects/chrono" + "github.com/bxcodec/httpcache" "github.com/fiware/VCVerifier/common" "github.com/fiware/VCVerifier/config" @@ -129,6 +131,8 @@ func (tc TirHttpClient) GetTrustedIssuer(tirEndpoints []string, did string) (exi trustedIssuer, err := parseTirResponse(*resp) if err != nil { logging.Log().Warnf("Was not able to parse the response from tir %s for %s. Err: %v", tirEndpoint, did, err) + body, _ := io.ReadAll(resp.Body) + logging.Log().Debugf("Response was %s ", string(body)) continue } logging.Log().Debugf("Got issuer %s.", logging.PrettyPrintObject(trustedIssuer)) From 0fea9233c2d20debb0682726e0a594432ab1a6b7 Mon Sep 17 00:00:00 2001 From: Stefan Wiedemann Date: Mon, 3 Jun 2024 09:03:04 +0200 Subject: [PATCH 2/5] logging --- tir/tirClient.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tir/tirClient.go b/tir/tirClient.go index 0040f266..9762cb59 100644 --- a/tir/tirClient.go +++ b/tir/tirClient.go @@ -3,7 +3,6 @@ package tir import ( "encoding/json" "errors" - "io" "net/http" "time" @@ -130,9 +129,8 @@ func (tc TirHttpClient) GetTrustedIssuer(tirEndpoints []string, did string) (exi } trustedIssuer, err := parseTirResponse(*resp) if err != nil { - logging.Log().Warnf("Was not able to parse the response from tir %s for %s. Err: %v", tirEndpoint, did, err) - body, _ := io.ReadAll(resp.Body) - logging.Log().Debugf("Response was %s ", string(body)) + logging.Log().Warnf("Was not able to parse the response from til %s for %s. Err: %v", tirEndpoint, did, err) + logging.Log().Debugf("Response was %v ", resp) continue } logging.Log().Debugf("Got issuer %s.", logging.PrettyPrintObject(trustedIssuer)) From c0b677da4e4e0ad27a24eaff4953954cc1853ca8 Mon Sep 17 00:00:00 2001 From: Stefan Wiedemann Date: Mon, 3 Jun 2024 09:48:34 +0200 Subject: [PATCH 3/5] change caching --- config/config.go | 4 +++ config/provider_test.go | 10 ++++-- tir/tirClient.go | 77 ++++++++++++++++++++++++----------------- tir/tirClient_test.go | 16 ++++++--- verifier/verifier.go | 2 +- 5 files changed, 69 insertions(+), 40 deletions(-) diff --git a/config/config.go b/config/config.go index 98006d6e..259b3a21 100644 --- a/config/config.go +++ b/config/config.go @@ -59,6 +59,10 @@ type Verifier struct { Did string `mapstructure:"did"` // address of the (ebsi-compatible) trusted-issuers-registry for verifying the issuer TirAddress string `mapstructure:"tirAddress"` + // expiry of the tir-cache entries + TirCacheExpiry int `mapstructure:"tirCacheExpiry" default:"30"` + // expiry of the til-cache entries + TilCacheExpiry int `mapstructure:"tilCacheExpiry" default:"30"` // expiry of auth sessions SessionExpiry int `mapstructure:"sessionExpiry" default:"30"` // policies that shall be checked diff --git a/config/provider_test.go b/config/provider_test.go index b77e60f7..0b10f6f4 100644 --- a/config/provider_test.go +++ b/config/provider_test.go @@ -27,9 +27,11 @@ func Test_ReadConfig(t *testing.T) { StaticDir: "views/static", }, Verifier: Verifier{ - Did: "did:key:somekey", - TirAddress: "https://test.dev/trusted_issuer/v3/issuers/", - SessionExpiry: 30, + Did: "did:key:somekey", + TirAddress: "https://test.dev/trusted_issuer/v3/issuers/", + TirCacheExpiry: 30, + TilCacheExpiry: 30, + SessionExpiry: 30, PolicyConfig: Policies{ DefaultPolicies: PolicyMap{ "SignaturePolicy": {}, @@ -84,6 +86,8 @@ func Test_ReadConfig(t *testing.T) { }, Verifier: Verifier{Did: "", TirAddress: "", + TirCacheExpiry: 30, + TilCacheExpiry: 30, SessionExpiry: 30, ValidationMode: "none", KeyAlgorithm: "RS256", diff --git a/tir/tirClient.go b/tir/tirClient.go index 9762cb59..86c88361 100644 --- a/tir/tirClient.go +++ b/tir/tirClient.go @@ -8,7 +8,6 @@ import ( "github.com/procyon-projects/chrono" - "github.com/bxcodec/httpcache" "github.com/fiware/VCVerifier/common" "github.com/fiware/VCVerifier/config" "github.com/fiware/VCVerifier/logging" @@ -37,7 +36,9 @@ type TirClient interface { * A client to retrieve infromation from EBSI-compatible TrustedIssuerRegistry APIs. */ type TirHttpClient struct { - client HttpGetClient + client HttpGetClient + tirCache common.Cache + tilCache common.Cache } /** @@ -78,18 +79,17 @@ type Claim struct { AllowedValues []interface{} `json:"allowedValues"` } -func NewTirHttpClient(tokenProvider TokenProvider, config config.M2M) (client TirClient, err error) { +func NewTirHttpClient(tokenProvider TokenProvider, m2mConfig config.M2M, verifierConfig config.Verifier) (client TirClient, err error) { httpClient := &http.Client{} - _, err = httpcache.NewWithInmemoryCache(httpClient, true, time.Second*60) - if err != nil { - logging.Log().Errorf("Was not able to inject the cache to the client. Err: %v", err) - return - } + + tirCache := cache.New(time.Duration(verifierConfig.TirCacheExpiry)*time.Second, time.Duration(2*verifierConfig.TirCacheExpiry)*time.Second) + tilCache := cache.New(time.Duration(verifierConfig.TilCacheExpiry)*time.Second, time.Duration(2*verifierConfig.TilCacheExpiry)*time.Second) + var httpGetClient HttpGetClient - if config.AuthEnabled { + if m2mConfig.AuthEnabled { logging.Log().Debug("Authorization for the trusted-issuers-registry is enabled.") - authorizingHttpClient := AuthorizingHttpClient{httpClient: httpClient, tokenProvider: tokenProvider, clientId: config.ClientId} + authorizingHttpClient := AuthorizingHttpClient{httpClient: httpClient, tokenProvider: tokenProvider, clientId: m2mConfig.ClientId} _, err := chrono.NewDefaultTaskScheduler().ScheduleAtFixedRate(authorizingHttpClient.FillMetadataCache, time.Duration(30)*time.Second) if err != nil { @@ -102,7 +102,7 @@ func NewTirHttpClient(tokenProvider TokenProvider, config config.M2M) (client Ti httpGetClient = NoAuthHttpClient{httpClient: httpClient} } - return TirHttpClient{client: httpGetClient}, err + return TirHttpClient{client: httpGetClient, tirCache: tirCache, tilCache: tilCache}, err } func (tc TirHttpClient) IsTrustedParticipant(tirEndpoints []string, did string) (trusted bool) { @@ -118,23 +118,28 @@ func (tc TirHttpClient) IsTrustedParticipant(tirEndpoints []string, did string) func (tc TirHttpClient) GetTrustedIssuer(tirEndpoints []string, did string) (exists bool, trustedIssuer TrustedIssuer, err error) { for _, tirEndpoint := range tirEndpoints { - resp, err := tc.requestIssuer(tirEndpoint, did) - if err != nil { - logging.Log().Warnf("Was not able to get the issuer %s from %s because of err: %v.", did, tirEndpoint, err) - continue - } - if resp.StatusCode != 200 { - logging.Log().Debugf("Issuer %s is not known at %s.", did, tirEndpoint) - continue - } - trustedIssuer, err := parseTirResponse(*resp) - if err != nil { - logging.Log().Warnf("Was not able to parse the response from til %s for %s. Err: %v", tirEndpoint, did, err) - logging.Log().Debugf("Response was %v ", resp) - continue + trustedIssuer, hit := tc.tilCache.Get(tirEndpoint + did) + if !hit { + resp, err := tc.requestIssuer(tirEndpoint, did) + if err != nil { + logging.Log().Warnf("Was not able to get the issuer %s from %s because of err: %v.", did, tirEndpoint, err) + continue + } + if resp.StatusCode != 200 { + logging.Log().Debugf("Issuer %s is not known at %s.", did, tirEndpoint) + continue + } + trustedIssuer, err = parseTirResponse(*resp) + if err != nil { + logging.Log().Warnf("Was not able to parse the response from til %s for %s. Err: %v", tirEndpoint, did, err) + logging.Log().Debugf("Response was %v ", resp) + continue + } + logging.Log().Debugf("Got issuer %s.", logging.PrettyPrintObject(trustedIssuer)) + tc.tilCache.Add(tirEndpoint+did, trustedIssuer, cache.DefaultExpiration) } - logging.Log().Debugf("Got issuer %s.", logging.PrettyPrintObject(trustedIssuer)) - return true, trustedIssuer, err + return true, trustedIssuer.(TrustedIssuer), err + } return false, trustedIssuer, err } @@ -156,13 +161,20 @@ func parseTirResponse(resp http.Response) (trustedIssuer TrustedIssuer, err erro func (tc TirHttpClient) issuerExists(tirEndpoint string, did string) (trusted bool) { - resp, err := tc.requestIssuer(tirEndpoint, did) - if err != nil { - return false + exists, hit := tc.tirCache.Get(tirEndpoint + did) + + if !hit { + resp, err := tc.requestIssuer(tirEndpoint, did) + if err != nil { + return false + } + logging.Log().Debugf("Issuer %s response from %s is %v", did, tirEndpoint, resp.StatusCode) + exists = resp.StatusCode == 200 + tc.tirCache.Add(tirEndpoint, exists, cache.DefaultExpiration) } - logging.Log().Debugf("Issuer %s response from %s is %v", did, tirEndpoint, resp.StatusCode) + // if a 200 is returned, the issuer exists. We dont have to parse the whole response - return resp.StatusCode == 200 + return exists.(bool) } func (tc TirHttpClient) requestIssuer(tirEndpoint string, did string) (response *http.Response, err error) { @@ -171,6 +183,7 @@ func (tc TirHttpClient) requestIssuer(tirEndpoint string, did string) (response logging.Log().Debugf("Got error %v", err) return tc.requestIssuerWithVersion(tirEndpoint, getIssuerV3Url(did)) } + if response.StatusCode != 200 { logging.Log().Debugf("Got status %v", response.StatusCode) return tc.requestIssuerWithVersion(tirEndpoint, getIssuerV3Url(did)) diff --git a/tir/tirClient_test.go b/tir/tirClient_test.go index 14b671a0..70964511 100644 --- a/tir/tirClient_test.go +++ b/tir/tirClient_test.go @@ -3,12 +3,14 @@ package tir import ( "errors" "fmt" - "github.com/fiware/VCVerifier/common" "io" "net/http" "net/http/httptest" "strings" "testing" + "time" + + "github.com/fiware/VCVerifier/common" "github.com/stretchr/testify/assert" ) @@ -32,6 +34,12 @@ func (mc mockClient) Get(tirAddress string, tirPath string) (resp *http.Response return mc.responses[tirAddress+"/"+tirPath], mc.errors[tirAddress+"/"+tirPath] } +type mockCache struct{} + +func (mc mockCache) Add(k string, x interface{}, d time.Duration) error { return nil } +func (mc mockCache) Get(k string) (interface{}, bool) { return nil, false } +func (mc mockCache) Delete(k string) {} + func TestIsTrustedParticipant(t *testing.T) { type test struct { testName string @@ -61,7 +69,7 @@ func TestIsTrustedParticipant(t *testing.T) { for _, tc := range tests { common.ResetGlobalCache() t.Run(tc.testName, func(t *testing.T) { - tirClient := TirHttpClient{mockClient{responses: tc.mockResponses, errors: tc.mockErrors}} + tirClient := TirHttpClient{client: mockClient{responses: tc.mockResponses, errors: tc.mockErrors}, tilCache: mockCache{}, tirCache: mockCache{}} isTrusted := tirClient.IsTrustedParticipant(tc.testEndpoints, tc.testIssuer) if tc.expectedResult != isTrusted { @@ -103,7 +111,7 @@ func TestGetTrustedIssuer(t *testing.T) { for _, tc := range tests { common.ResetGlobalCache() t.Run(tc.testName, func(t *testing.T) { - tirClient := TirHttpClient{mockClient{responses: tc.mockResponses, errors: tc.mockErrors}} + tirClient := TirHttpClient{client: mockClient{responses: tc.mockResponses, errors: tc.mockErrors}, tilCache: mockCache{}, tirCache: mockCache{}} exists, issuer, err := tirClient.GetTrustedIssuer(tc.testEndpoints, tc.testIssuer) if tc.expectedError != err { t.Errorf("%s - Expected error %v but was %v.", tc.testName, tc.expectedError, err) @@ -146,7 +154,7 @@ func TestDidRegistry(t *testing.T) { // Close the server when test finishes defer server.Close() - tirClient := TirHttpClient{getClient{server.Client()}} + tirClient := TirHttpClient{client: getClient{server.Client()}, tilCache: mockCache{}, tirCache: mockCache{}} trusted := tirClient.issuerExists(server.URL, "did:elsi:eu.eori.denhaag19902304") assert.Equal(t, true, trusted, "Should return that issuer is trusted") diff --git a/verifier/verifier.go b/verifier/verifier.go index b2be3c7f..fec81b19 100644 --- a/verifier/verifier.go +++ b/verifier/verifier.go @@ -228,7 +228,7 @@ func InitVerifier(config *configModel.Configuration) (err error) { logging.Log().Infof("Auth disabled.") } - tirClient, err := tir.NewTirHttpClient(tokenProvider, config.M2M) + tirClient, err := tir.NewTirHttpClient(tokenProvider, config.M2M, config.Verifier) if err != nil { logging.Log().Errorf("Was not able to instantiate the trusted-issuers-registry client. Err: %v", err) return err From 971b40db745c95676e32fe809521b920d5a00d55 Mon Sep 17 00:00:00 2001 From: Stefan Wiedemann Date: Mon, 3 Jun 2024 09:49:21 +0200 Subject: [PATCH 4/5] clean deps --- go.mod | 3 --- go.sum | 10 ---------- 2 files changed, 13 deletions(-) diff --git a/go.mod b/go.mod index fbc83141..59e72b8a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/fiware/VCVerifier go 1.21 require ( - github.com/bxcodec/httpcache v1.0.0-beta.3 github.com/deepmap/oapi-codegen v1.12.3 github.com/foolin/goview v0.3.0 github.com/gin-contrib/cors v1.4.0 @@ -31,7 +30,6 @@ require ( github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.1.3 // indirect github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect - github.com/bxcodec/gotcha v1.0.0-beta.8 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.1 // indirect @@ -119,5 +117,4 @@ require ( golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - software.sslmate.com/src/go-pkcs12 v0.4.0 ) diff --git a/go.sum b/go.sum index fb7724df..56b28f62 100644 --- a/go.sum +++ b/go.sum @@ -75,11 +75,6 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bxcodec/gotcha v1.0.0-beta.2/go.mod h1:MEL9PRYL9Squu1zxreMIzJU6xtMouPmQybWEtXrL1nk= -github.com/bxcodec/gotcha v1.0.0-beta.8 h1:2yA1RzMHh4KPc2Fou6mQFd5rgrPHI8nj52un2fQyXkU= -github.com/bxcodec/gotcha v1.0.0-beta.8/go.mod h1:UnXdYOHIGqan8v5AACIHh8nLoCBLb5YifKBeGJOTNBg= -github.com/bxcodec/httpcache v1.0.0-beta.3 h1:4h+Yoda40PthVkiFa5hPtCoH46qbDRq/iCT7M5ACoR4= -github.com/bxcodec/httpcache v1.0.0-beta.3/go.mod h1:NdIkH0u1smImRyp35nIGeqG5IFeKbnyyBIv8cEGSXMY= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -173,9 +168,6 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/goccy/go-yaml v1.10.0 h1:rBi+5HGuznOxx0JZ+60LDY85gc0dyIJCIMvsMJTKSKQ= github.com/goccy/go-yaml v1.10.0/go.mod h1:h/18Lr6oSQ3mvmqFoWmQ47KChOgpfHpTyIHl3yVmpiY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -829,5 +821,3 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= -software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= -software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= From 29cfe12ef600a00bccf073106455ee484883ef1e Mon Sep 17 00:00:00 2001 From: Stefan Wiedemann Date: Mon, 3 Jun 2024 09:58:35 +0200 Subject: [PATCH 5/5] fix test --- verifier/verifier_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/verifier/verifier_test.go b/verifier/verifier_test.go index 8a81f55e..0c7c549f 100644 --- a/verifier/verifier_test.go +++ b/verifier/verifier_test.go @@ -9,6 +9,7 @@ import ( "errors" "net/http" "net/url" + "slices" "testing" "time" @@ -787,7 +788,10 @@ func TestGetOpenIDConfiguration(t *testing.T) { actualOpenID, _ := verifier.GetOpenIDConfiguration(tc.serviceIdentifier) assert.Equal(t, tc.expectedOpenID.Issuer, actualOpenID.Issuer) - assert.Equal(t, tc.expectedOpenID.ScopesSupported, actualOpenID.ScopesSupported) + assert.Equal(t, len(tc.expectedOpenID.ScopesSupported), len(actualOpenID.ScopesSupported)) + for _, scope := range tc.expectedOpenID.ScopesSupported { + assert.True(t, slices.Contains(actualOpenID.ScopesSupported, scope)) + } }) } }