Add support for getting paginated results (#213)

* Add support for getting paginated results * Support multi page results in GithubApi * Return async stream of JToken instead of string * Update release notes
github · Jan 24, 2022 · 70408d4 · 70408d4
1 parent 581d669
commit 70408d4
Show file tree

Hide file tree

Showing 6 changed files with 373 additions and 31 deletions.
diff --git a/RELEASENOTES.md b/RELEASENOTES.md
@@ -4,3 +4,5 @@
 - Show a generic error message instead of the actual one for unhandled exceptions in non-verbose mode.
 - Exit code is now 1 instead of 0 in case of an error.
 - Errors are written to std error instead of std out. 
+- Adding Support to get multi page results from Github API.
+- The Github to Github migrations are no longer limited to 30 repos. 
diff --git a/src/Octoshift/GithubApi.cs b/src/Octoshift/GithubApi.cs
@@ -38,22 +38,16 @@ public virtual async Task<string> CreateTeam(string org, string teamName)
 
         public virtual async Task<IEnumerable<string>> GetTeamMembers(string org, string teamName)
         {
-            var url = $"https://api.github.com/orgs/{org}/teams/{teamName}/members";
+            var url = $"https://api.github.com/orgs/{org}/teams/{teamName}/members?per_page=100";
 
-            var response = await _client.GetAsync(url);
-            var data = JArray.Parse(response);
-
-            return data.Children().Select(x => (string)x["login"]).ToList();
+            return await _client.GetAllAsync(url).Select(x => (string)x["login"]).ToListAsync();
         }
 
         public virtual async Task<IEnumerable<string>> GetRepos(string org)
         {
-            var url = $"https://api.github.com/orgs/{org}/repos";
-
-            var response = await _client.GetAsync(url);
-            var data = JArray.Parse(response);
+            var url = $"https://api.github.com/orgs/{org}/repos?per_page=100";
 
-            return data.Children().Select(x => (string)x["name"]).ToList();
+            return await _client.GetAllAsync(url).Select(x => (string)x["name"]).ToListAsync();
         }
 
         public virtual async Task RemoveTeamMember(string org, string teamName, string member)

diff --git a/src/Octoshift/GithubClient.cs b/src/Octoshift/GithubClient.cs
@@ -1,7 +1,11 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
 using OctoshiftCLI.Extensions;
 
 namespace OctoshiftCLI
@@ -25,20 +29,36 @@ public GithubClient(OctoLogger log, HttpClient httpClient, string personalAccess
             }
         }
 
-        public virtual async Task<string> GetAsync(string url) => await SendAsync(HttpMethod.Get, url);
+        public virtual async Task<string> GetAsync(string url) => (await SendAsync(HttpMethod.Get, url)).Content;
+
+        public virtual async IAsyncEnumerable<JToken> GetAllAsync(string url)
+        {
+            var nextUrl = url;
+            do
+            {
+                var (content, headers) = await SendAsync(HttpMethod.Get, nextUrl);
+                foreach (var jToken in JArray.Parse(content))
+                {
+                    yield return jToken;
+                }
+
+                nextUrl = GetNextUrl(headers);
+            } while (nextUrl != null);
+        }
 
         public virtual async Task<string> PostAsync(string url, object body) =>
-            await SendAsync(HttpMethod.Post, url, body);
+            (await SendAsync(HttpMethod.Post, url, body)).Content;
 
         public virtual async Task<string> PutAsync(string url, object body) =>
-            await SendAsync(HttpMethod.Put, url, body);
+            (await SendAsync(HttpMethod.Put, url, body)).Content;
 
         public virtual async Task<string> PatchAsync(string url, object body) =>
-            await SendAsync(HttpMethod.Patch, url, body);
+            (await SendAsync(HttpMethod.Patch, url, body)).Content;
 
-        public virtual async Task<string> DeleteAsync(string url) => await SendAsync(HttpMethod.Delete, url);
+        public virtual async Task<string> DeleteAsync(string url) => (await SendAsync(HttpMethod.Delete, url)).Content;
 
-        private async Task<string> SendAsync(HttpMethod httpMethod, string url, object body = null)
+        private async Task<(string Content, KeyValuePair<string, IEnumerable<string>>[] ResponseHeaders)> SendAsync(
+            HttpMethod httpMethod, string url, object body = null)
         {
             url = url?.Replace(" ", "%20");
 
@@ -50,7 +70,7 @@ private async Task<string> SendAsync(HttpMethod httpMethod, string url, object b
             }
 
             using var payload = body?.ToJson().ToStringContent();
-            var response = httpMethod.ToString() switch
+            using var response = httpMethod.ToString() switch
             {
                 "GET" => await _httpClient.GetAsync(url),
                 "DELETE" => await _httpClient.DeleteAsync(url),
@@ -64,7 +84,29 @@ private async Task<string> SendAsync(HttpMethod httpMethod, string url, object b
 
             response.EnsureSuccessStatusCode();
 
-            return content;
+            return (content, response.Headers.ToArray());
+        }
+
+        private string GetNextUrl(KeyValuePair<string, IEnumerable<string>>[] headers)
+        {
+            var linkHeaderValue = ExtractLinkHeader(headers);
+
+            var nextUrl = linkHeaderValue?
+                .Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)
+                .Select(link =>
+                {
+                    var rx = new Regex(@"<(?<url>.+)>;\s*rel=""(?<rel>.+)""");
+                    var url = rx.Match(link).Groups["url"].Value;
+                    var rel = rx.Match(link).Groups["rel"].Value; // first, next, last, prev
+
+                    return (Url: url, Rel: rel);
+                })
+                .FirstOrDefault(x => x.Rel == "next").Url;
+
+            return nextUrl;
         }
+
+        private string ExtractLinkHeader(KeyValuePair<string, IEnumerable<string>>[] headers) =>
+            headers.SingleOrDefault(kvp => kvp.Key == "Link").Value?.FirstOrDefault();
     }
 }
diff --git a/src/Octoshift/Octoshift.csproj b/src/Octoshift/Octoshift.csproj
@@ -7,6 +7,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="3.11.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="System.Linq.Async" Version="5.1.0" />
   </ItemGroup>
 
 </Project>
diff --git a/src/OctoshiftCLI.Tests/GithubApiTests.cs b/src/OctoshiftCLI.Tests/GithubApiTests.cs
@@ -1,7 +1,10 @@
+using System.Collections.Generic;
+using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
 using FluentAssertions;
 using Moq;
+using Newtonsoft.Json.Linq;
 using OctoshiftCLI.Extensions;
 using Xunit;
 
@@ -69,10 +72,11 @@ public async Task GetTeamMembers_Returns_Team_Members()
             const string org = "ORG";
             const string teamName = "TEAM_NAME";
 
-            var url = $"https://api.github.com/orgs/{org}/teams/{teamName}/members";
+            var url = $"https://api.github.com/orgs/{org}/teams/{teamName}/members?per_page=100";
+
             const string teamMember1 = "TEAM_MEMBER_1";
             const string teamMember2 = "TEAM_MEMBER_2";
-            var response = $@"
+            var responsePage1 = $@"
             [
                 {{
                     ""login"": ""{teamMember1}"",
@@ -84,29 +88,57 @@ public async Task GetTeamMembers_Returns_Team_Members()
                 }}
             ]";
 
+            const string teamMember3 = "TEAM_MEMBER_3";
+            const string teamMember4 = "TEAM_MEMBER_4";
+            var responsePage2 = $@"
+            [
+                {{
+                    ""login"": ""{teamMember3}"",
+                    ""id"": 3
+                }},
+                {{
+                    ""login"": ""{teamMember4}"", 
+                    ""id"": 4
+                }}
+            ]";
+
+            async IAsyncEnumerable<JToken> GetAllPages()
+            {
+                var jArrayPage1 = JArray.Parse(responsePage1);
+                yield return jArrayPage1[0];
+                yield return jArrayPage1[1];
+
+                var jArrayPage2 = JArray.Parse(responsePage2);
+                yield return jArrayPage2[0];
+                yield return jArrayPage2[1];
+
+                await Task.CompletedTask;
+            }
+
             var githubClientMock = new Mock<GithubClient>(null, null, null);
             githubClientMock
-                .Setup(m => m.GetAsync(url))
-                .ReturnsAsync(response);
+                .Setup(m => m.GetAllAsync(url))
+                .Returns(GetAllPages);
 
             // Act
             var githubApi = new GithubApi(githubClientMock.Object);
-            var result = await githubApi.GetTeamMembers(org, teamName);
+            var result = (await githubApi.GetTeamMembers(org, teamName)).ToArray();
 
             // Assert
-            result.Should().Equal(teamMember1, teamMember2);
+            result.Should().HaveCount(4);
+            result.Should().Equal(teamMember1, teamMember2, teamMember3, teamMember4);
         }
 
         [Fact]
         public async Task GetRepos_Returns_Names_Of_All_Repositories()
         {
             // Arrange
             const string org = "ORG";
-            var url = $"https://api.github.com/orgs/{org}/repos";
+            var url = $"https://api.github.com/orgs/{org}/repos?per_page=100";
 
             const string repoName1 = "FOO";
             const string repoName2 = "BAR";
-            var response = $@"
+            var responsePage1 = $@"
             [
                 {{
                     ""id"": 1,
@@ -118,18 +150,45 @@ public async Task GetRepos_Returns_Names_Of_All_Repositories()
                 }}
             ]";
 
+            const string repoName3 = "BAZ";
+            const string repoName4 = "QUX";
+            var responsePage2 = $@"
+            [
+                {{
+                    ""id"": 3,
+                    ""name"": ""{repoName3}""
+                }},
+                {{
+                    ""id"": 4,
+                    ""name"": ""{repoName4}""
+                }}
+            ]";
+
+            async IAsyncEnumerable<JToken> GetAllPages()
+            {
+                var jArrayPage1 = JArray.Parse(responsePage1);
+                yield return jArrayPage1[0];
+                yield return jArrayPage1[1];
+
+                var jArrayPage2 = JArray.Parse(responsePage2);
+                yield return jArrayPage2[0];
+                yield return jArrayPage2[1];
+
+                await Task.CompletedTask;
+            }
+
             var githubClientMock = new Mock<GithubClient>(null, null, null);
             githubClientMock
-                .Setup(m => m.GetAsync(url))
-                .ReturnsAsync(response);
+                .Setup(m => m.GetAllAsync(url))
+                .Returns(GetAllPages);
 
             // Act
             var githubApi = new GithubApi(githubClientMock.Object);
-            var result = await githubApi.GetRepos(org);
+            var result = (await githubApi.GetRepos(org)).ToArray();
 
             // Assert
-            result.Should().HaveCount(2);
-            result.Should().Equal(repoName1, repoName2);
+            result.Should().HaveCount(4);
+            result.Should().Equal(repoName1, repoName2, repoName3, repoName4);
         }
 
         [Fact]