Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

An attempt to add paging support in a generic way. #147

Merged
merged 1 commit into from
May 28, 2024
Merged

An attempt to add paging support in a generic way. #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 99 additions & 51 deletions src/CostApi/AzureCostApiRetriever.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ private Uri DeterminePath(Scope scope, string path)

}

private async Task<HttpResponseMessage> ExecuteCallToCostApi(bool includeDebugOutput, object? payload, Uri uri)
private async Task<QueryResponse> ExecutePagedCallToCostApi(bool includeDebugOutput, object? payload, Uri uri)
{
await RetrieveToken(includeDebugOutput);

Expand All @@ -138,22 +138,91 @@ private async Task<HttpResponseMessage> ExecuteCallToCostApi(bool includeDebugOu
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};

var response = payload == null
? await _client.GetAsync(uri)
: await _client.PostAsJsonAsync(uri, payload, options);
QueryResponse? combinedResponse = new QueryResponse { properties = new Properties { rows = new List<JsonElement>() } };

if (includeDebugOutput)
Uri? nextUri = uri;

while (nextUri != null)
{
AnsiConsole.WriteLine(
$"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}");
if (!response.IsSuccessStatusCode)
var response = payload == null
? await _client.GetAsync(nextUri)
: await _client.PostAsJsonAsync(nextUri, payload, options);

if (includeDebugOutput)
{
AnsiConsole.WriteLine(
$"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}");
if (!response.IsSuccessStatusCode)
{
AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}");
}
}

response.EnsureSuccessStatusCode();

QueryResponse? content = await response.Content.ReadFromJsonAsync<QueryResponse>();

if (content == null)
{
AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}");
throw new Exception("Failed to deserialize the response from the API.");
}

combinedResponse.Combine(content);

nextUri = string.IsNullOrEmpty(content.properties.nextLink) ? null : new Uri(content.properties.nextLink);
}

response.EnsureSuccessStatusCode();
return response;
return combinedResponse;
}

private async Task<T?> ExecuteTypedCallToCostApi<T>(bool includeDebugOutput, object? payload, Uri uri)
where T : class
{
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

var content = await response.Content.ReadFromJsonAsync<T>();
return content;
}

private async Task<HttpResponseMessage> ExecuteCallToCostApi(bool includeDebugOutput, object? payload, Uri uri)
{
await RetrieveToken(includeDebugOutput);

if (includeDebugOutput)
{
AnsiConsole.WriteLine($"Retrieving data from {uri} using the following payload:");
AnsiConsole.Write(new JsonText(JsonSerializer.Serialize(payload)));
AnsiConsole.WriteLine();
}

if (!string.Equals(_client.BaseAddress?.ToString(), CostApiAddress))
{
_client.BaseAddress = new Uri(CostApiAddress);
}

var options = new JsonSerializerOptions
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};


var response = payload == null
? await _client.GetAsync(uri)
: await _client.PostAsJsonAsync(uri, payload, options);

if (includeDebugOutput)
{
AnsiConsole.WriteLine(
$"Response status code is {response.StatusCode} and got payload size of {response.Content.Headers.ContentLength}");
if (!response.IsSuccessStatusCode)
{
AnsiConsole.WriteLine($"Response content: {await response.Content.ReadAsStringAsync()}");
}
}

response.EnsureSuccessStatusCode();
return response;
}

public async Task<IEnumerable<CostItem>> RetrieveCosts(bool includeDebugOutput, Scope scope,
Expand Down Expand Up @@ -202,10 +271,8 @@ public async Task<IEnumerable<CostItem>> RetrieveCosts(bool includeDebugOutput,
}
};

var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -275,10 +342,8 @@ public async Task<IEnumerable<CostNamedItem>> RetrieveCostByServiceName(bool inc
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostNamedItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -347,10 +412,8 @@ public async Task<IEnumerable<CostNamedItem>> RetrieveCostByLocation(bool includ
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostNamedItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -424,10 +487,8 @@ public async Task<IEnumerable<CostNamedItem>> RetrieveCostByResourceGroup(bool i
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostNamedItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -501,10 +562,8 @@ public async Task<IEnumerable<CostNamedItem>> RetrieveCostBySubscription(bool in
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostNamedItem>();
foreach (var row in content.properties.rows)
{
Expand Down Expand Up @@ -579,9 +638,7 @@ public async Task<IEnumerable<CostDailyItem>> RetrieveDailyCost(bool includeDebu
filter = GenerateFilters(filter)
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();
var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostDailyItem>();
foreach (var row in content.properties.rows)
Expand Down Expand Up @@ -632,10 +689,8 @@ public async Task<Subscription> RetrieveSubscription(bool includeDebugOutput, Gu
$"/subscriptions/{subscriptionId}/?api-version=2019-11-01",
UriKind.Relative);

var response = await ExecuteCallToCostApi(includeDebugOutput, null, uri);

var content = await response.Content.ReadFromJsonAsync<Subscription>();

var content = await ExecuteTypedCallToCostApi<Subscription>(includeDebugOutput, null, uri);

if (includeDebugOutput)
{
var json = JsonSerializer.Serialize(content, new JsonSerializerOptions { WriteIndented = true });
Expand Down Expand Up @@ -692,11 +747,9 @@ public async Task<IEnumerable<CostItem>> RetrieveForecastedCosts(bool includeDeb
try
{
// Allow this one to fail, as it is not supported for all subscriptions
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);


foreach (var row in content.properties.rows)
{
var date = DateOnly.ParseExact(row[1].ToString(), "yyyyMMdd", CultureInfo.InvariantCulture);
Expand Down Expand Up @@ -847,10 +900,8 @@ public async Task<IEnumerable<CostResourceItem>> RetrieveCostForResources(bool i
grouping = grouping,
}
};
var response = await ExecuteCallToCostApi(includeDebugOutput, payload, uri);

CostQueryResponse? content = await response.Content.ReadFromJsonAsync<CostQueryResponse>();

var content = await ExecutePagedCallToCostApi(includeDebugOutput, payload, uri);

var items = new List<CostResourceItem>();
foreach (JsonElement row in content.properties.rows)
{
Expand Down Expand Up @@ -933,11 +984,8 @@ public async Task<IEnumerable<UsageDetails>> RetrieveUsageDetails(bool includeDe

while (uri != null)
{
var response = await ExecuteCallToCostApi(includeDebugOutput, null, uri);

UsageDetailsResponse payload = await response.Content.ReadFromJsonAsync<UsageDetailsResponse>() ??
new UsageDetailsResponse();

var payload = await ExecuteTypedCallToCostApi<UsageDetailsResponse>(includeDebugOutput, null, uri);

items.AddRange(payload.value);
uri = payload.nextLink != null ? new Uri(payload.nextLink, UriKind.Relative) : null;
}
Expand Down
15 changes: 12 additions & 3 deletions src/CostApi/CostQueryResponse.cs → src/CostApi/QueryResponse.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,34 @@

namespace AzureCostCli.CostApi;

public class CostQueryResponse
public class QueryResponse
{
public object eTag { get; set; }

Check warning on line 7 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'eTag' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public string id { get; set; }

Check warning on line 8 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'id' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public object location { get; set; }
public string name { get; set; }
public Properties properties { get; set; }
public string type { get; set; }

// Combine method to merge results
public void Combine(QueryResponse other)
{
if (other?.properties?.rows != null)
{
this.properties.rows.AddRange(other.properties.rows);
}
}
}

public class Columns
{
public string name { get; set; }

Check warning on line 26 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'name' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public string type { get; set; }

Check warning on line 27 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'type' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
}

public class Properties
{
public Columns[] columns { get; set; }

Check warning on line 32 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'columns' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public object nextLink { get; set; }
public JsonElement[] rows { get; set; }
public string nextLink { get; set; }

Check warning on line 33 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'nextLink' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
public List<JsonElement> rows { get; set; }

Check warning on line 34 in src/CostApi/QueryResponse.cs

View workflow job for this annotation

GitHub Actions / build 

Non-nullable property 'rows' must contain a non-null value when exiting constructor. Consider declaring the property as nullable.
}
Loading