Skip to content

Commit

Permalink
feat: Infoblox paging support
Browse files Browse the repository at this point in the history
  • Loading branch information
skalpin committed Jun 5, 2024
1 parent 5946007 commit ece6e33
Show file tree
Hide file tree
Showing 3 changed files with 183 additions and 22 deletions.
10 changes: 5 additions & 5 deletions internal/infoblox/infoblox.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,12 +195,12 @@ func (p *Provider) Records(_ context.Context) (endpoints []*endpoint.Endpoint, e

for _, zone := range zones {
log.Debugf("fetch records from zone '%s'", zone.Fqdn)
searchParams := recordQueryParams(zone.Fqdn, p.config.View)
searchParams := map[string]string{"zone": zone.Fqdn, "view": p.config.View}
var resA []ibclient.RecordA
objA := ibclient.NewEmptyRecordA()
objA.View = p.config.View
objA.Zone = zone.Fqdn
err = p.client.GetObject(objA, "", searchParams, &resA)
err = PagingGetObject(p.client, objA, "", searchParams, &resA)
if err != nil && !isNotFoundError(err) {
return nil, fmt.Errorf("could not fetch A records from zone '%s': %w", zone.Fqdn, err)
}
Expand All @@ -212,7 +212,7 @@ func (p *Provider) Records(_ context.Context) (endpoints []*endpoint.Endpoint, e
objH := ibclient.NewEmptyHostRecord()
objH.View = &p.config.View
objH.Zone = zone.Fqdn
err = p.client.GetObject(objH, "", searchParams, &resH)
err = PagingGetObject(p.client, objH, "", searchParams, &resH)
if err != nil && !isNotFoundError(err) {
return nil, fmt.Errorf("could not fetch host records from zone '%s': %w", zone.Fqdn, err)
}
Expand All @@ -223,7 +223,7 @@ func (p *Provider) Records(_ context.Context) (endpoints []*endpoint.Endpoint, e
objC := ibclient.NewEmptyRecordCNAME()
objC.View = &p.config.View
objC.Zone = zone.Fqdn
err = p.client.GetObject(objC, "", searchParams, &resC)
err = PagingGetObject(p.client, objC, "", searchParams, &resC)
if err != nil && !isNotFoundError(err) {
return nil, fmt.Errorf("could not fetch CNAME records from zone '%s': %w", zone.Fqdn, err)
}
Expand All @@ -234,7 +234,7 @@ func (p *Provider) Records(_ context.Context) (endpoints []*endpoint.Endpoint, e
objT := ibclient.NewEmptyRecordTXT()
objT.View = &p.config.View
objT.Zone = zone.Fqdn
err = p.client.GetObject(objT, "", searchParams, &resT)
err = PagingGetObject(p.client, objT, "", searchParams, &resT)
if err != nil && !isNotFoundError(err) {
return nil, fmt.Errorf("could not fetch TXT records from zone '%s': %w", zone.Fqdn, err)
}
Expand Down
117 changes: 100 additions & 17 deletions internal/infoblox/infoblox_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,19 @@ func (client *mockIBConnector) CreateObject(obj ibclient.IBObject) (ref string,

// nolint: gocyclo
func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, queryParams *ibclient.QueryParams, res interface{}) (err error) {
isPagingType := false
switch res.(type) {
case *pagingResponseStruct[ibclient.RecordA]:
isPagingType = true
case *pagingResponseStruct[ibclient.HostRecord]:
isPagingType = true
case *pagingResponseStruct[ibclient.RecordTXT]:
isPagingType = true
case *pagingResponseStruct[ibclient.RecordPTR]:
isPagingType = true
case *pagingResponseStruct[ibclient.RecordCNAME]:
isPagingType = true
}
req := getObjectRequest{
obj: obj.ObjectType(),
ref: ref,
Expand Down Expand Up @@ -226,7 +239,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer
result = append(result, *object.(*ibclient.RecordA))
}
}
*res.(*[]ibclient.RecordA) = result
if isPagingType {
res.(*pagingResponseStruct[ibclient.RecordA]).Result = result
} else {
*res.(*[]ibclient.RecordA) = result
}
case recordCname:
var result []ibclient.RecordCNAME
for _, object := range *client.mockInfobloxObjects {
Expand All @@ -250,7 +267,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer
result = append(result, *object.(*ibclient.RecordCNAME))
}
}
*res.(*[]ibclient.RecordCNAME) = result
if isPagingType {
res.(*pagingResponseStruct[ibclient.RecordCNAME]).Result = result
} else {
*res.(*[]ibclient.RecordCNAME) = result
}
case recordHost:
var result []ibclient.HostRecord
for _, object := range *client.mockInfobloxObjects {
Expand All @@ -274,7 +295,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer
result = append(result, *object.(*ibclient.HostRecord))
}
}
*res.(*[]ibclient.HostRecord) = result
if isPagingType {
res.(*pagingResponseStruct[ibclient.HostRecord]).Result = result
} else {
*res.(*[]ibclient.HostRecord) = result
}
case recordTxt:
var result []ibclient.RecordTXT
for _, object := range *client.mockInfobloxObjects {
Expand All @@ -298,7 +323,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer
result = append(result, *object.(*ibclient.RecordTXT))
}
}
*res.(*[]ibclient.RecordTXT) = result
if isPagingType {
res.(*pagingResponseStruct[ibclient.RecordTXT]).Result = result
} else {
*res.(*[]ibclient.RecordTXT) = result
}
case recordPtr:
var result []ibclient.RecordPTR
for _, object := range *client.mockInfobloxObjects {
Expand All @@ -323,7 +352,11 @@ func (client *mockIBConnector) GetObject(obj ibclient.IBObject, ref string, quer
result = append(result, *object.(*ibclient.RecordPTR))
}
}
*res.(*[]ibclient.RecordPTR) = result
if isPagingType {
res.(*pagingResponseStruct[ibclient.RecordPTR]).Result = result
} else {
*res.(*[]ibclient.RecordPTR) = result
}
case "zone_auth":
*res.(*[]ibclient.ZoneAuth) = *client.mockInfobloxZones
}
Expand Down Expand Up @@ -613,13 +646,33 @@ func TestInfobloxRecords(t *testing.T) {
client.verifyGetObjectRequest(t, "zone_auth", "", &map[string]string{}).
ExpectNotRequestURLQueryParam(t, "view").
ExpectNotRequestURLQueryParam(t, "zone")
client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{"zone": "example.com"}).
client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"view": "",
"zone": "example.com"}).
ExpectRequestURLQueryParam(t, "zone", "example.com")
client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{"zone": "example.com"}).
client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"view": "",
"zone": "example.com"}).
ExpectRequestURLQueryParam(t, "zone", "example.com")
client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{"zone": "example.com"}).
client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"view": "",
"zone": "example.com"}).
ExpectRequestURLQueryParam(t, "zone", "example.com")
client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{"zone": "example.com"}).
client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"view": "",
"zone": "example.com"}).
ExpectRequestURLQueryParam(t, "zone", "example.com")
client.verifyNoMoreGetObjectRequests(t)
}
Expand Down Expand Up @@ -649,28 +702,58 @@ func TestInfobloxRecordsWithView(t *testing.T) {
client.verifyGetObjectRequest(t, "zone_auth", "", &map[string]string{"view": "Inside"}).
ExpectRequestURLQueryParam(t, "view", "Inside").
ExpectNotRequestURLQueryParam(t, "zone")
client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{"zone": "foo.example.com", "view": "Inside"}).
client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"zone": "foo.example.com",
"view": "Inside"}).
ExpectRequestURLQueryParam(t, "zone", "foo.example.com").
ExpectRequestURLQueryParam(t, "view", "Inside")
client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{"zone": "foo.example.com", "view": "Inside"}).
client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"zone": "foo.example.com", "view": "Inside"}).
ExpectRequestURLQueryParam(t, "zone", "foo.example.com").
ExpectRequestURLQueryParam(t, "view", "Inside")
client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{"zone": "foo.example.com", "view": "Inside"}).
client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"zone": "foo.example.com", "view": "Inside"}).
ExpectRequestURLQueryParam(t, "zone", "foo.example.com").
ExpectRequestURLQueryParam(t, "view", "Inside")
client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{"zone": "foo.example.com", "view": "Inside"}).
client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"zone": "foo.example.com", "view": "Inside"}).
ExpectRequestURLQueryParam(t, "zone", "foo.example.com").
ExpectRequestURLQueryParam(t, "view", "Inside")
client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{"zone": "bar.example.com", "view": "Inside"}).
client.verifyGetObjectRequest(t, "record:a", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1", "zone": "bar.example.com", "view": "Inside"}).
ExpectRequestURLQueryParam(t, "zone", "bar.example.com").
ExpectRequestURLQueryParam(t, "view", "Inside")
client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{"zone": "bar.example.com", "view": "Inside"}).
client.verifyGetObjectRequest(t, "record:host", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1", "zone": "bar.example.com", "view": "Inside"}).
ExpectRequestURLQueryParam(t, "zone", "bar.example.com").
ExpectRequestURLQueryParam(t, "view", "Inside")
client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{"zone": "bar.example.com", "view": "Inside"}).
client.verifyGetObjectRequest(t, "record:cname", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1", "zone": "bar.example.com", "view": "Inside"}).
ExpectRequestURLQueryParam(t, "zone", "bar.example.com").
ExpectRequestURLQueryParam(t, "view", "Inside")
client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{"zone": "bar.example.com", "view": "Inside"}).
client.verifyGetObjectRequest(t, "record:txt", "", &map[string]string{
"_max_results": "1000",
"_paging": "1",
"_return_as_object": "1",
"zone": "bar.example.com", "view": "Inside"}).
ExpectRequestURLQueryParam(t, "zone", "bar.example.com").
ExpectRequestURLQueryParam(t, "view", "Inside")
client.verifyNoMoreGetObjectRequests(t)
Expand Down
78 changes: 78 additions & 0 deletions internal/infoblox/paging_ibclient.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package infoblox

/*
Copyright 2024 The external-dns-infoblox-webhook Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic
*/

import (
"fmt"
"reflect"

ibclient "github.com/infobloxopen/infoblox-go-client/v2"
)

func PagingGetObject[T any](
c ibclient.IBConnector,
obj ibclient.IBObject,
ref string,
queryParams map[string]string,
res *[]T,
) (err error) {

pagingResponse := pagingResponseStruct[T]{
NextPageId: "",
Result: make([]T, 0),
}

//copy query params and update them
queryParamsCopy := map[string]string{}
for k, v := range queryParams {
queryParamsCopy[k] = v
}

queryParamsCopy["_return_as_object"] = "1"
queryParamsCopy["_paging"] = "1"
queryParamsCopy["_max_results"] = "1000"

err = c.GetObject(obj, "", ibclient.NewQueryParams(false, queryParamsCopy), &pagingResponse)
if err != nil {
return fmt.Errorf("could not fetch object: %s", err)
} else {
*res = append(*res, pagingResponse.Result...)
}

for {
if pagingResponse.NextPageId == "" {
return
}
queryParamsCopy["_page_id"] = pagingResponse.NextPageId
pagingResponse.NextPageId = ""
pagingResponse.Result = make([]T, 0)
err = c.GetObject(obj, "", ibclient.NewQueryParams(false, queryParamsCopy), &pagingResponse)
if err != nil {
return fmt.Errorf("could not fetch object: %s", err)
}

*res = append(*res, pagingResponse.Result...)
fmt.Print(fmt.Sprintln("Paging to retrieve", reflect.TypeOf(obj), len(*res)))
}
}

type pagingResponseStruct[T any] struct {
NextPageId string `json:"next_page_id,omitempty"`
Result []T `json:"result,omitempty"`
}

0 comments on commit ece6e33

Please sign in to comment.