Skip to content

Commit

Permalink
feat: only do full update per day
Browse files Browse the repository at this point in the history
  • Loading branch information
JasonKhew96 committed Dec 15, 2024
1 parent 879c3bf commit 085d37d
Show file tree
Hide file tree
Showing 4 changed files with 150 additions and 11 deletions.
9 changes: 9 additions & 0 deletions .github/actions/build/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@ runs:
key: ${{ runner.os }}-gatsby-public-cache-${{ github.run_id }}
restore-keys: ${{ runner.os }}-gatsby-public-cache-
save-always: true
- name: cache graphql response
uses: actions/cache@v4
with:
path: cached_graphql.json
key: ${{ runner.os }}-github-graphql-response-${{ github.run_id }}
restore-keys: ${{ runner.os }}-github-graphql-response-
save-always: true
- name: Restore cache
shell: bash
run: node gh-pages-cache-restore.js
Expand All @@ -55,6 +62,7 @@ runs:
echo "modules.lsposed.org" > ./public/CNAME
env:
GRAPHQL_TOKEN: ${{ inputs.token }}
REPO: ${{ inputs.repo }}
GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES: true
- name: clean up caches on failure
if: ${{ failure() || cancelled() }}
Expand All @@ -63,6 +71,7 @@ runs:
rm -rf public/*
rm -rf public-cache/*
rm -rf .cache/*
rm -f cached_graphql.json
- name: Refresh cache
shell: bash
run: node gh-pages-cache.js
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ on:
repo:
description: 'repo name'
required: false
schedule:
- cron: "0 0 * * *"

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,5 @@ yarn-error.log
!.yarn/releases
!.yarn/sdks
!.yarn/versions

cached_graphql.json
148 changes: 137 additions & 11 deletions gatsby-node.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,112 @@ const { execFileSync } = require('child_process')

const { fetchFromGithub, replacePrivateImage } = require('./github-source')

function makeRepositoryQuery (name) {
return gql`
{
repository(owner: "Xposed-Modules-Repo", name: "${name}") {
name
description
url
homepageUrl
collaborators(affiliation: DIRECT, first: 100) {
edges {
node {
login
name
}
}
}
readme: object(expression: "HEAD:README.md") {
... on Blob {
text
}
}
summary: object(expression: "HEAD:SUMMARY") {
... on Blob {
text
}
}
scope: object(expression: "HEAD:SCOPE") {
... on Blob {
text
}
}
sourceUrl: object(expression: "HEAD:SOURCE_URL") {
... on Blob {
text
}
}
hide: object(expression: "HEAD:HIDE") {
... on Blob {
text
}
}
additionalAuthors: object(expression: "HEAD:ADDITIONAL_AUTHORS") {
... on Blob {
text
}
}
latestRelease {
name
url
isDraft
description
descriptionHTML
createdAt
publishedAt
updatedAt
tagName
isPrerelease
releaseAssets(first: 50) {
edges {
node {
name
contentType
downloadUrl
downloadCount
size
}
}
}
}
releases(first: 20) {
edges {
node {
name
url
isDraft
description
descriptionHTML
createdAt
publishedAt
updatedAt
tagName
isPrerelease
isLatest
releaseAssets(first: 50) {
edges {
node {
name
contentType
downloadUrl
downloadCount
size
}
}
}
}
}
}
updatedAt
createdAt
stargazerCount
}
}
`
}


const PAGINATION = 10
function makeRepositoriesQuery (cursor) {
const arg = cursor ? `, after: "${cursor}"` : ''
Expand Down Expand Up @@ -292,7 +398,7 @@ exports.sourceNodes = async (
let cursor = null
let page = 1
let total
const mergedResult = {
let mergedResult = {
data: {
organization: {
repositories: {
Expand All @@ -301,23 +407,43 @@ exports.sourceNodes = async (
}
}
}
while (true) {
console.log(`Querying GitHub API, page ${page}, total ${Math.ceil(total / PAGINATION) || 'unknown'}, cursor: ${cursor}`)
const result = await fetchFromGithub(makeRepositoriesQuery(cursor))
const repo_name = process.env.REPO ? process.env.REPO.split('/')[1] : null
if (repo_name && fs.existsSync('./cached_graphql.json')) {
const data = fs.readFileSync('./cached_graphql.json')
mergedResult = JSON.parse(data)
mergedResult.data.organization.repositories.edges.forEach((value, index, array) => {
if (value.node.name === repo_name) {
array.splice(index, 1)
}
})
console.log(`Fetching ${repo_name} from GitHub API`)
const result = await fetchFromGithub(makeRepositoryQuery(repo_name))
if (result.errors || !result.data) {
const errMsg = result.errors || 'result.data is null'
console.error(errMsg)
throw errMsg
}
mergedResult.data.organization.repositories.edges =
mergedResult.data.organization.repositories.edges.concat(result.data.organization.repositories.edges)
if (!result.data.organization.repositories.pageInfo.hasNextPage) {
break
mergedResult.data.organization.repositories.edges.unshift({'node': result.data.repository})
} else {
while (true) {
console.log(`Querying GitHub API, page ${page}, total ${Math.ceil(total / PAGINATION) || 'unknown'}, cursor: ${cursor}`)
const result = await fetchFromGithub(makeRepositoriesQuery(cursor))
if (result.errors || !result.data) {
const errMsg = result.errors || 'result.data is null'
console.error(errMsg)
throw errMsg
}
mergedResult.data.organization.repositories.edges =
mergedResult.data.organization.repositories.edges.concat(result.data.organization.repositories.edges)
if (!result.data.organization.repositories.pageInfo.hasNextPage) {
break
}
cursor = result.data.organization.repositories.pageInfo.endCursor
total = result.data.organization.repositories.totalCount
page++
}
cursor = result.data.organization.repositories.pageInfo.endCursor
total = result.data.organization.repositories.totalCount
page++
}
fs.writeFileSync('./cached_graphql.json', JSON.stringify(mergedResult))
generateGatsbyNode(mergedResult, createNode)
}

Expand Down

0 comments on commit 085d37d

Please sign in to comment.