diff --git a/internal/infoblox/infoblox.go b/internal/infoblox/infoblox.go index b6746bf..71e0e61 100644 --- a/internal/infoblox/infoblox.go +++ b/internal/infoblox/infoblox.go @@ -195,12 +195,12 @@ func (p *Provider) Records(_ context.Context) (endpoints []*endpoint.Endpoint, e for _, zone := range zones { log.Debugf("fetch records from zone '%s'", zone.Fqdn) - searchParams := recordQueryParams(zone.Fqdn, p.config.View) + searchParams := map[string]string{"zone": zone.Fqdn, "view": p.config.View} var resA []ibclient.RecordA objA := ibclient.NewEmptyRecordA() objA.View = p.config.View objA.Zone = zone.Fqdn - err = p.client.GetObject(objA, "", searchParams, &resA) + err = PagingGetObject(p.client, objA, "", searchParams, &resA) if err != nil && !isNotFoundError(err) { return nil, fmt.Errorf("could not fetch A records from zone '%s': %w", zone.Fqdn, err) } @@ -212,7 +212,7 @@ func (p *Provider) Records(_ context.Context) (endpoints []*endpoint.Endpoint, e objH := ibclient.NewEmptyHostRecord() objH.View = &p.config.View objH.Zone = zone.Fqdn - err = p.client.GetObject(objH, "", searchParams, &resH) + err = PagingGetObject(p.client, objH, "", searchParams, &resH) if err != nil && !isNotFoundError(err) { return nil, fmt.Errorf("could not fetch host records from zone '%s': %w", zone.Fqdn, err) } @@ -223,7 +223,7 @@ func (p *Provider) Records(_ context.Context) (endpoints []*endpoint.Endpoint, e objC := ibclient.NewEmptyRecordCNAME() objC.View = &p.config.View objC.Zone = zone.Fqdn - err = p.client.GetObject(objC, "", searchParams, &resC) + err = PagingGetObject(p.client, objC, "", searchParams, &resC) if err != nil && !isNotFoundError(err) { return nil, fmt.Errorf("could not fetch CNAME records from zone '%s': %w", zone.Fqdn, err) } @@ -234,7 +234,7 @@ func (p *Provider) Records(_ context.Context) (endpoints []*endpoint.Endpoint, e objT := ibclient.NewEmptyRecordTXT() objT.View = &p.config.View objT.Zone = zone.Fqdn - err = p.client.GetObject(objT, "", searchParams, &resT) + err = PagingGetObject(p.client, objT, "", searchParams, &resT) if err != nil && !isNotFoundError(err) { return nil, fmt.Errorf("could not fetch TXT records from zone '%s': %w", zone.Fqdn, err) } diff --git a/internal/infoblox/infoblox_test.go b/internal/infoblox/infoblox_test.go index 1afb14a..d9076bc 100644 --- a/internal/infoblox/infoblox_test.go +++ b/internal/infoblox/infoblox_test.go @@ -190,6 +190,19 @@ func (client *mockIBConnector) CreateObject(obj ibclient.IBObject) (ref string, // nolint: gocyclo func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, queryParams *ibclient.QueryParams, res interface{}) (err error) { + isPagingType := false + switch res.(type) { + case *pagingResponseStruct[ibclient.RecordA]: + isPagingType = true + case *pagingResponseStruct[ibclient.HostRecord]: + isPagingType = true + case *pagingResponseStruct[ibclient.RecordTXT]: + isPagingType = true + case *pagingResponseStruct[ibclient.RecordPTR]: + isPagingType = true + case *pagingResponseStruct[ibclient.RecordCNAME]: + isPagingType = true + } req := getObjectRequest{ obj: obj.ObjectType(), ref: ref, @@ -226,7 +239,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer result = append(result, *object.(*ibclient.RecordA)) } } - *res.(*[]ibclient.RecordA) = result + if isPagingType { + res.(*pagingResponseStruct[ibclient.RecordA]).Result = result + } else { + *res.(*[]ibclient.RecordA) = result + } case recordCname: var result []ibclient.RecordCNAME for _, object := range *client.mockInfobloxObjects { @@ -250,7 +267,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer result = append(result, *object.(*ibclient.RecordCNAME)) } } - *res.(*[]ibclient.RecordCNAME) = result + if isPagingType { + res.(*pagingResponseStruct[ibclient.RecordCNAME]).Result = result + } else { + *res.(*[]ibclient.RecordCNAME) = result + } case recordHost: var result []ibclient.HostRecord for _, object := range *client.mockInfobloxObjects { @@ -274,7 +295,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer result = append(result, *object.(*ibclient.HostRecord)) } } - *res.(*[]ibclient.HostRecord) = result + if isPagingType { + res.(*pagingResponseStruct[ibclient.HostRecord]).Result = result + } else { + *res.(*[]ibclient.HostRecord) = result + } case recordTxt: var result []ibclient.RecordTXT for _, object := range *client.mockInfobloxObjects { @@ -298,7 +323,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer result = append(result, *object.(*ibclient.RecordTXT)) } } - *res.(*[]ibclient.RecordTXT) = result + if isPagingType { + res.(*pagingResponseStruct[ibclient.RecordTXT]).Result = result + } else { + *res.(*[]ibclient.RecordTXT) = result + } case recordPtr: var result []ibclient.RecordPTR for _, object := range *client.mockInfobloxObjects { @@ -323,7 +352,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer result = append(result, *object.(*ibclient.RecordPTR)) } } - *res.(*[]ibclient.RecordPTR) = result + if isPagingType { + res.(*pagingResponseStruct[ibclient.RecordPTR]).Result = result + } else { + *res.(*[]ibclient.RecordPTR) = result + } case "zone_auth": *res.(*[]ibclient.ZoneAuth) = *client.mockInfobloxZones } @@ -613,13 +646,33 @@ func TestInfobloxRecords(t *testing.T) { client.verifyGetObjectRequest(t, "zone_auth", "", &map[string]string{}). ExpectNotRequestURLQueryParam(t, "view"). ExpectNotRequestURLQueryParam(t, "zone") - client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{"zone": "example.com"}). + client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "view": "", + "zone": "example.com"}). ExpectRequestURLQueryParam(t, "zone", "example.com") - client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{"zone": "example.com"}). + client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "view": "", + "zone": "example.com"}). ExpectRequestURLQueryParam(t, "zone", "example.com") - client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{"zone": "example.com"}). + client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "view": "", + "zone": "example.com"}). ExpectRequestURLQueryParam(t, "zone", "example.com") - client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{"zone": "example.com"}). + client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "view": "", + "zone": "example.com"}). ExpectRequestURLQueryParam(t, "zone", "example.com") client.verifyNoMoreGetObjectRequests(t) } @@ -649,28 +702,58 @@ func TestInfobloxRecordsWithView(t *testing.T) { client.verifyGetObjectRequest(t, "zone_auth", "", &map[string]string{"view": "Inside"}). ExpectRequestURLQueryParam(t, "view", "Inside"). ExpectNotRequestURLQueryParam(t, "zone") - client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{"zone": "foo.example.com", "view": "Inside"}). + client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "zone": "foo.example.com", + "view": "Inside"}). ExpectRequestURLQueryParam(t, "zone", "foo.example.com"). ExpectRequestURLQueryParam(t, "view", "Inside") - client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{"zone": "foo.example.com", "view": "Inside"}). + client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "zone": "foo.example.com", "view": "Inside"}). ExpectRequestURLQueryParam(t, "zone", "foo.example.com"). ExpectRequestURLQueryParam(t, "view", "Inside") - client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{"zone": "foo.example.com", "view": "Inside"}). + client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "zone": "foo.example.com", "view": "Inside"}). ExpectRequestURLQueryParam(t, "zone", "foo.example.com"). ExpectRequestURLQueryParam(t, "view", "Inside") - client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{"zone": "foo.example.com", "view": "Inside"}). + client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "zone": "foo.example.com", "view": "Inside"}). ExpectRequestURLQueryParam(t, "zone", "foo.example.com"). ExpectRequestURLQueryParam(t, "view", "Inside") - client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{"zone": "bar.example.com", "view": "Inside"}). + client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", "zone": "bar.example.com", "view": "Inside"}). ExpectRequestURLQueryParam(t, "zone", "bar.example.com"). ExpectRequestURLQueryParam(t, "view", "Inside") - client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{"zone": "bar.example.com", "view": "Inside"}). + client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", "zone": "bar.example.com", "view": "Inside"}). ExpectRequestURLQueryParam(t, "zone", "bar.example.com"). ExpectRequestURLQueryParam(t, "view", "Inside") - client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{"zone": "bar.example.com", "view": "Inside"}). + client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", "zone": "bar.example.com", "view": "Inside"}). ExpectRequestURLQueryParam(t, "zone", "bar.example.com"). ExpectRequestURLQueryParam(t, "view", "Inside") - client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{"zone": "bar.example.com", "view": "Inside"}). + client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{ + "_max_results": "1000", + "_paging": "1", + "_return_as_object": "1", + "zone": "bar.example.com", "view": "Inside"}). ExpectRequestURLQueryParam(t, "zone", "bar.example.com"). ExpectRequestURLQueryParam(t, "view", "Inside") client.verifyNoMoreGetObjectRequests(t) diff --git a/internal/infoblox/paging_ibclient.go b/internal/infoblox/paging_ibclient.go new file mode 100644 index 0000000..d8e0eef --- /dev/null +++ b/internal/infoblox/paging_ibclient.go @@ -0,0 +1,78 @@ +package infoblox + +/* +Copyright 2024 The external-dns-infoblox-webhook Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "fmt" + "reflect" + + ibclient "github.com/infobloxopen/infoblox-go-client/v2" +) + +func PagingGetObject[T any]( + c ibclient.IBConnector, + obj ibclient.IBObject, + ref string, + queryParams map[string]string, + res *[]T, +) (err error) { + + pagingResponse := pagingResponseStruct[T]{ + NextPageId: "", + Result: make([]T, 0), + } + + //copy query params and update them + queryParamsCopy := map[string]string{} + for k, v := range queryParams { + queryParamsCopy[k] = v + } + + queryParamsCopy["_return_as_object"] = "1" + queryParamsCopy["_paging"] = "1" + queryParamsCopy["_max_results"] = "1000" + + err = c.GetObject(obj, "", ibclient.NewQueryParams(false, queryParamsCopy), &pagingResponse) + if err != nil { + return fmt.Errorf("could not fetch object: %s", err) + } else { + *res = append(*res, pagingResponse.Result...) + } + + for { + if pagingResponse.NextPageId == "" { + return + } + queryParamsCopy["_page_id"] = pagingResponse.NextPageId + pagingResponse.NextPageId = "" + pagingResponse.Result = make([]T, 0) + err = c.GetObject(obj, "", ibclient.NewQueryParams(false, queryParamsCopy), &pagingResponse) + if err != nil { + return fmt.Errorf("could not fetch object: %s", err) + } + + *res = append(*res, pagingResponse.Result...) + fmt.Print(fmt.Sprintln("Paging to retrieve", reflect.TypeOf(obj), len(*res))) + } +} + +type pagingResponseStruct[T any] struct { + NextPageId string `json:"next_page_id,omitempty"` + Result []T `json:"result,omitempty"` +}