Fixed data exporters not being able to access data when having more t…

…han 1.000 cards
danniehansen · Apr 19, 2024 · e0d6c2c · e0d6c2c
1 parent 53b9ce9
commit e0d6c2c
Show file tree

Hide file tree

Showing 4 changed files with 134 additions and 34 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,13 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Released]
 
-## [2.8.1] - 2024-04-16
+## [2.8.1] - 2024-04-19
 
 ### Security
 
 - Updated packages to latest version matching version selectors.
 - Changed IAM based authentication for CDK to use OIDC
 
+### Fixed
+
+- Fixed an issue with time tracking / estiamtes data exporter not being able to pull cards data if the /all API responded with more than 1.000 cards. We will now fall back to fetch closed, open and visible cards one-by-one to try and display as much data as possible. Disclaimer added to both pages when this is happening.
+
 ## [2.8.0] - 2024-03-30
 
 ### Security

diff --git a/src/main.ts b/src/main.ts
@@ -20,6 +20,7 @@ import Dropdown from 'primevue/dropdown';
 import MultiSelect from 'primevue/multiselect';
 import Checkbox from 'primevue/checkbox';
 import InputText from 'primevue/inputtext';
+import Message from 'primevue/message';
 import InputNumber from 'primevue/inputnumber';
 import ColumnGroup from 'primevue/columngroup';
 import Slider from 'primevue/slider';
@@ -82,6 +83,7 @@ app.component('InputText', InputText);
 app.component('InputNumber', InputNumber);
 app.component('ColumnGroup', ColumnGroup);
 app.component('Slider', Slider);
+app.component('Message', Message);
 app.component('Row', Row);
 
 // eslint-disable-next-line vue/no-reserved-component-names

diff --git a/src/pages/DataExporter/Estimates/index.vue b/src/pages/DataExporter/Estimates/index.vue
@@ -38,6 +38,11 @@
     v-else-if="ready && isAuthorized"
     class="authorized flex flex-column gap-3"
   >
+    <Message v-if="apiDisclaimer" severity="info"
+      >Note: Some Trello cards may be unavailable; we prioritize fetching open,
+      closed, and visible ones for optimal service.</Message
+    >
+
     <div class="flex flex-column gap-3">
       <div class="flex flex-wrap gap-3">
         <div class="p-float-label">
@@ -184,6 +189,7 @@ const columns = ref<string[]>([]);
 const listOptions = ref<Option[]>([]);
 const lists = ref<string[]>([]);
 const isIncognito = ref(false);
+const apiDisclaimer = ref(false);
 const unrecognizedError = ref(false);
 const rejectedAuth = ref(false);
 const groupByCard = ref(false);
@@ -471,47 +477,82 @@ async function getData() {
   const board = await getTrelloInstance().board('id');
 
   try {
-    const data: Trello.PowerUp.Card[] = [];
-    let moreData = true;
-    let page = 0;
-
-    while (moreData) {
-      const rawData = await fetch(
-        `https://api.trello.com/1/boards/${
-          board.id
-        }/cards/all?pluginData=true&limit=1&page=${page}&fields=id,idList,name,desc,labels,pluginData,closed&key=${getAppKey()}&token=${token}&r=${new Date().getTime()}`
-      ).then<Trello.PowerUp.Card[]>((res) => res.json());
-
-      moreData = rawData.length > 0;
-      page++;
-
-      data.push(...rawData);
-    }
+    const data = await fetch(
+      `https://api.trello.com/1/boards/${
+        board.id
+      }/cards/all?pluginData=true&fields=id,idList,name,desc,labels,idBoard,pluginData,closed&key=${getAppKey()}&token=${token}&r=${new Date().getTime()}`
+    ).then<Trello.PowerUp.Card[]>((res) => res.json());
 
     const boardData = await fetch(
       `https://api.trello.com/1/boards/${
         board.id
       }?fields=name&key=${getAppKey()}&token=${token}&r=${new Date().getTime()}`
     ).then<Trello.PowerUp.Board>((res) => res.json());
 
-    cards = data
-      // Remove cards which has been archived
-      .filter((card) => !card.closed)
-      .map<ApiCard>((card) => {
-        return new ApiCard(boardData, card, listById, memberById, members);
-      });
+    cards = data.map<ApiCard>((card) => {
+      return new ApiCard(boardData, card, listById, memberById, members);
+    });
 
     lastDataFetch.value = Date.now();
 
     getUniqueLabels();
   } catch (e) {
-    try {
-      await clearToken();
-    } catch (e) {
-      // Ignore exceptions in case no token exists
+    // Fallback to query the card types one at a time if there is simply too many in /all
+
+    const boardData = await fetch(
+      `https://api.trello.com/1/boards/${
+        board.id
+      }?fields=name&key=${getAppKey()}&token=${token}&r=${new Date().getTime()}`
+    ).then<Trello.PowerUp.Board>((res) => res.json());
+
+    let failedFetches = 0;
+
+    // Reset cards before we push new items into it
+    cards = [];
+
+    // To avoid duplicates in case of overlapping filters, we keep track of already added cards here
+    const takenCards: string[] = [];
+
+    for (const type of ['closed', 'open', 'visible']) {
+      try {
+        const data = (
+          await fetch(
+            `https://api.trello.com/1/boards/${
+              board.id
+            }/cards/${type}?pluginData=true&fields=id,idList,name,desc,labels,idBoard,pluginData,closed&key=${getAppKey()}&token=${token}&r=${new Date().getTime()}`
+          ).then<Trello.PowerUp.Card[]>((res) => res.json())
+        ).filter((card) => !takenCards.includes(card.id));
+
+        cards.push(
+          ...data.map<ApiCard>((card) => {
+            return new ApiCard(boardData, card, listById, memberById, members);
+          })
+        );
+
+        takenCards.push(...data.map((card) => card.id));
+
+        apiDisclaimer.value = true;
+      } catch (e) {
+        failedFetches++;
+      }
     }
 
-    await trelloTick();
+    // If all of the fetches fail, then we assume the token is invalid
+    if (failedFetches === 4) {
+      try {
+        await clearToken();
+      } catch (e) {
+        // Ignore exceptions in case no token exists
+      }
+
+      await getTrelloCard().getRestApi().clearToken();
+
+      await trelloTick();
+    } else {
+      lastDataFetch.value = Date.now();
+
+      getUniqueLabels();
+    }
   }
 
   await new Promise((resolve) =>

diff --git a/src/pages/DataExporter/TimeTracking/index.vue b/src/pages/DataExporter/TimeTracking/index.vue
@@ -38,6 +38,11 @@
     v-else-if="ready && isAuthorized"
     class="authorized flex flex-column gap-3"
   >
+    <Message v-if="apiDisclaimer" severity="info"
+      >Note: Some Trello cards may be unavailable; we prioritize fetching open,
+      closed, and visible ones for optimal service.</Message
+    >
+
     <div class="flex flex-wrap column-gap-3 row-gap-4">
       <div class="p-float-label">
         <MultiSelect
@@ -209,6 +214,7 @@ const members = ref<string[]>([]);
 const labels = ref<string[]>([]);
 const columns = ref<string[]>([]);
 const dateFrom = ref('');
+const apiDisclaimer = ref(false);
 const dateTo = ref('');
 const listOptions = ref<Option[]>([]);
 const lists = ref<string[]>([]);
@@ -667,15 +673,62 @@ async function getData() {
 
     getUniqueLabels();
   } catch (e) {
-    try {
-      await clearToken();
-    } catch (e) {
-      // Ignore exceptions in case no token exists
+    // Fallback to query the card types one at a time if there is simply too many in /all
+
+    const boardData = await fetch(
+      `https://api.trello.com/1/boards/${
+        board.id
+      }?fields=name&key=${getAppKey()}&token=${token}&r=${new Date().getTime()}`
+    ).then<Trello.PowerUp.Board>((res) => res.json());
+
+    let failedFetches = 0;
+
+    // Reset cards before we push new items into it
+    cards = [];
+
+    // To avoid duplicates in case of overlapping filters, we keep track of already added cards here
+    const takenCards: string[] = [];
+
+    for (const type of ['closed', 'open', 'visible']) {
+      try {
+        const data = (
+          await fetch(
+            `https://api.trello.com/1/boards/${
+              board.id
+            }/cards/${type}?pluginData=true&fields=id,idList,name,desc,labels,idBoard,pluginData,closed&key=${getAppKey()}&token=${token}&r=${new Date().getTime()}`
+          ).then<Trello.PowerUp.Card[]>((res) => res.json())
+        ).filter((card) => !takenCards.includes(card.id));
+
+        cards.push(
+          ...data.map<ApiCard>((card) => {
+            return new ApiCard(boardData, card, listById, memberById, members);
+          })
+        );
+
+        takenCards.push(...data.map((card) => card.id));
+
+        apiDisclaimer.value = true;
+      } catch (e) {
+        failedFetches++;
+      }
     }
 
-    await getTrelloCard().getRestApi().clearToken();
+    // If all of the fetches fail, then we assume the token is invalid
+    if (failedFetches === 4) {
+      try {
+        await clearToken();
+      } catch (e) {
+        // Ignore exceptions in case no token exists
+      }
 
-    await trelloTick();
+      await getTrelloCard().getRestApi().clearToken();
+
+      await trelloTick();
+    } else {
+      lastDataFetch.value = Date.now();
+
+      getUniqueLabels();
+    }
   }
 
   await new Promise((resolve) =>