From c6ef8cd69ba150791646ea05e0de55f4f861f6cd Mon Sep 17 00:00:00 2001 From: Arnau Casau <47946624+arnaucasau@users.noreply.github.com> Date: Fri, 2 Feb 2024 14:19:20 +0100 Subject: [PATCH] Automate the regeneration of the API docs (#739) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Thanks to #737, the API generation script doesn't need an argument to specify the CI artifact URL used in the generation. Because of that, we can automatize the script to regenerate all API docs in the repository. ### New command The PR adds a new command `regen-apis` where we can regenerate multiple versions of our three APIs with one command: ```bash npm run regen-apis ``` By default, all minor releases of each API will be regenerated, but we can add the `--current-apis-only` argument to regenerate only the latest stable versions. ```bash npm run regen-apis -- --current-apis-only ``` Alternatively, you can regenerate only one of the three available APIs. We can use the `-p` argument, which could be combined with the other arguments, similarly to the API generation script: ```bash npm run regen-apis -- -p ``` ### Git instructions To run the script, we need to create a dedicated branch and have a clean git status. The script will create a commit per version regenerated and the developers can git cherry-pick them how they want to create different PRs. If the regeneration fails for a specific version, the script will call `git restore` to avoid mixing changes from two different versions when regenerating the next one. ### Output At the end of each call to the new command, we will see a summarization of all versions regenerated with three different possible outcomes per version:
Possible Output Meaning Creates a new commit Restores the files
✅ regenerated correctly The regeneration was successful Yes No
☑️ is up-to-date The regeneration was successful, but no files were modified No No
❌ failed to regenerate The regeneration failed No Yes
Closes #720 --------- Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> --- README.md | 14 ++ package.json | 1 + scripts/api-html-artifacts.json | 51 ++++---- scripts/commands/regenerateApiDocs.ts | 181 ++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 25 deletions(-) create mode 100644 scripts/commands/regenerateApiDocs.ts diff --git a/README.md b/README.md index fea0368599f..653ca4594ff 100644 --- a/README.md +++ b/README.md @@ -242,12 +242,26 @@ To check that formatting is valid without actually making changes, run `npm run This is useful when we make improvements to the API generation script. +You can regenerate all API docs versions following these steps: + +1. Create a dedicated branch for the regeneration other than `main` using `git checkout -b `. +2. Ensure there are no pending changes by running `git status` and creating a new commit for them if necessary. +3. Run `npm run regen-apis` to regenerate all API docs versions for `qiskit`, `qiskit-ibm-provider`, and `qiskit-ibm-runtime`. + +Each regenerated version will be saved as a distinct commit. If the changes are too large for one single PR, consider splitting it up into multiple PRs by using `git cherry-pick` or `git rebase -i` so each PR only has the commits it wants to target. + +If you only want to regenerate the latest stable minor release of each package, then add `--current-apis-only` as an argument, and in case you only want to regenerate versions of one package, then you can use the `-p ` argument. + +Alternatively, you can also regenerate one specific version: + 1. Choose which documentation you want to generate (`qiskit`, `qiskit-ibm-provider`, or `qiskit-ibm-runtime`) and its version. 2. Run `npm run gen-api -- -p -v `, e.g. `npm run gen-api -- -p qiskit -v 0.45.0` If the version is not for the latest stable minor release series, then add `--historical` to the arguments. For example, use `--historical` if the latest stable release is 0.45.\* but you're generating docs for the patch release 0.44.3. +In this case, no commit will be automatically created. + ## Generate new API docs This is useful when new docs content is published, usually corresponding to new releases or hotfixes for content issues. If you're generating a patch release, also see the below subsection for additional steps. diff --git a/package.json b/package.json index d44b5b06529..8c91ca30b9e 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "fmt": "prettier --write .", "test": "jest", "typecheck": "tsc", + "regen-apis": "node -r esbuild-register scripts/commands/regenerateApiDocs.ts", "gen-api": "node -r esbuild-register scripts/commands/updateApiDocs.ts", "make-historical": "node -r esbuild-register scripts/commands/convertApiDocsToHistorical.ts" }, diff --git a/scripts/api-html-artifacts.json b/scripts/api-html-artifacts.json index b8e05488ef3..c4195dfce24 100644 --- a/scripts/api-html-artifacts.json +++ b/scripts/api-html-artifacts.json @@ -2,35 +2,36 @@ "qiskit": { "0.46": "https://ibm.box.com/shared/static/74ppfh7b3rx73fntmk9p8pw4asf6pp4a.zip", "0.45": "https://ibm.box.com/shared/static/e2rwbrztml4dg6lkk1w3e0kmevkw9w7s.zip", - "0.44": "https://ibm.box.com/shared/static/yudjd08saugb3z9stciz4zxhpkroqrv4.zip", - "0.43": "https://ibm.box.com/shared/static/odcdrizljwpcz96upbu8umb5xpfvf73h.zip", - "0.42": "https://ibm.box.com/shared/static/muj1v0m4x36fuqyqo22wknexj9cq8yfv.zip", - "0.41": "https://ibm.box.com/shared/static/2j2qw33dgy5swf9vno0tfpq8ahey14uu.zip", - "0.40": "https://ibm.box.com/shared/static/6j9mbwbeqx0kfi20zl22hzp6524kvqz8.zip", - "0.39": "https://ibm.box.com/shared/static/za2u7phhpyd6lrsalv48xsxhz1ud17rd.zip", - "0.38": "https://ibm.box.com/shared/static/rcyb5ugvgjc6gpgb5hkdedz5r5buashu.zip", - "0.37": "https://ibm.box.com/shared/static/4nfrhcnu5ixod1i5ct1fps2qpoyfmgnb.zip", - "0.36": "https://ibm.box.com/shared/static/eekpk9r8kawwxr1bfaswrcc073mien1j.zip", - "0.35": "https://ibm.box.com/shared/static/dkqlxxllnhrq4yyae9fqlifab36fq2av.zip", - "0.33": "https://ibm.box.com/shared/static/kmgdgi0imgam3o7jzumyjhwdz00i9y5w.zip", - "0.32": "https://ibm.box.com/shared/static/tzkb58t2vgzz21i6ydg7r9g4m4995waq.zip", - "0.31": "https://ibm.box.com/shared/static/d4m9d1f4uaq95ovpqo9yot1mnm1jnofw.zip", - "0.30": "https://ibm.box.com/shared/static/loueznt45qyoo925gwt6cb5n1a0lp4o0.zip", - "0.29": "https://ibm.box.com/shared/static/uc8s5lcrmxuy23vkb2bkn8y930m533rk.zip", - "0.28": "https://ibm.box.com/shared/static/656e5bsov004vpnlmc9z40q718eq0ze2.zip", - "0.27": "https://ibm.box.com/shared/static/y2lb96gl2v32si02xgbswmjc06vfvfje.zip", - "0.26": "https://ibm.box.com/shared/static/5wlx2c8wr5xka5buzmlf1mqmz0zl600p.zip", - "0.25": "https://ibm.box.com/shared/static/h9dw4iq19f6nz6vp0egpoyjhxqnx7ot5.zip", - "0.24": "https://ibm.box.com/shared/static/pmcnjymbenwhwa3psxqsptvexs7hwrpd.zip", - "0.19": "https://ibm.box.com/shared/static/ov1hqihl0hlsai31dwyjyjj06k86k2nt.zip" + "0.44": "https://ibm.box.com/shared/static/myfk3g20xsp5xj8f3f1fd81p7ywgl7an.zip", + "0.43": "https://ibm.box.com/shared/static/cgqrwmz0u3ma8bld3xr39ysl0ghp8mfh.zip", + "0.42": "https://ibm.box.com/shared/static/icdn5tbrgy87k3aqxksptjezp98egqh6.zip", + "0.41": "https://ibm.box.com/shared/static/1xsrn1akx7qbr31l0yqc258gficf30cf.zip", + "0.40": "https://ibm.box.com/shared/static/j8rarno6kvv7zag58cv8uc63an3v7z69.zip", + "0.39": "https://ibm.box.com/shared/static/2mhunbme514sg5ewnngsmmtds5a3mpyx.zip", + "0.38": "https://ibm.box.com/shared/static/jl36955v51vezwiyyojs2wsexzu2hhdm.zip", + "0.37": "https://ibm.box.com/shared/static/fvn2x3tvqizvklf7ehnrxd5y149lma25.zip", + "0.36": "https://ibm.box.com/shared/static/p639y3r0xwbfuf5jps3mwdh6xiacdbs3.zip", + "0.35": "https://ibm.box.com/shared/static/2lw6x3tbnbgczigfh1yvr9z165fsauod.zip", + "0.33": "https://ibm.box.com/shared/static/xbmunamrm0yt46xv14q0hv4o104rv6q4.zip", + "0.32": "https://ibm.box.com/shared/static/xjan1nd4x4sviyz9pf43fygppczw1e1q.zip", + "0.31": "https://ibm.box.com/shared/static/wpvvg789qkx4dz0ctnj7su3tj2qfy68s.zip", + "0.30": "https://ibm.box.com/shared/static/gxpg8de5aof9ywar84266onvqfhbiy7t.zip", + "0.29": "https://ibm.box.com/shared/static/eqxzns3bv2yjyprne4s45vom5gzppg0o.zip", + "0.28": "https://ibm.box.com/shared/static/uv4l91bnjrhcqyqdunx1mqcec6fp31xg.zip", + "0.27": "https://ibm.box.com/shared/static/cqud425zvqpog6xd8w2dmfloqy894alb.zip", + "0.26": "https://ibm.box.com/shared/static/el5nseict90pjrs3a4x7gkxhh5203giq.zip", + "0.25": "https://ibm.box.com/shared/static/8qg4lb4outd8opd4a841b7xd18k3b5ke.zip", + "0.24": "https://ibm.box.com/shared/static/ygdswx8s7fj970kuulkzlh8r2mjuzn3k.zip", + "0.19": "https://ibm.box.com/shared/static/wjoea4x5tnxd0l4lgo2v3kxnx6btxvvl.zip" }, "qiskit-ibm-provider": { - "0.7": "https://ibm.box.com/shared/static/o5tjrdefdz72yi35nw09l9zbjjjecips.zip" + "0.7": "https://ibm.box.com/shared/static/t2vuik7bqboata3i34r4a83baabo95xn.zip" }, "qiskit-ibm-runtime": { - "0.17": "https://ibm.box.com/shared/static/5r7r2x65bst3hdtcyuowwfvno5o23xw0.zip", + "0.18": "https://ibm.box.com/shared/static/rbjdfogq8t1a4tquw1bh6gy4qdcv521v.zip", + "0.17": "https://ibm.box.com/shared/static/dnjto1rpvk0qknqb9ir8nr3yv6n0yzhk.zip", "0.16": "https://ibm.box.com/shared/static/xbtjc270jc2uu3s8tp7tqn8o9pckl37i.zip", - "0.15": "https://ibm.box.com/shared/static/j9wiuo9mga3lwihhqy9sdeqtsr4taanm.zip", - "0.14": "https://ibm.box.com/shared/static/bhv1xl2pid74qsanmphx3zm49cup6owv.zip" + "0.15": "https://ibm.box.com/shared/static/u9yqn2ya75cigotxsgl8zqaaqllzuudb.zip", + "0.14": "https://ibm.box.com/shared/static/t37e7jjsi0hii4j3xoorwpwso5m03jqn.zip" } } diff --git a/scripts/commands/regenerateApiDocs.ts b/scripts/commands/regenerateApiDocs.ts new file mode 100644 index 00000000000..09ef3255e61 --- /dev/null +++ b/scripts/commands/regenerateApiDocs.ts @@ -0,0 +1,181 @@ +// This code is a Qiskit project. +// +// (C) Copyright IBM 2024. +// +// This code is licensed under the Apache License, Version 2.0. You may +// obtain a copy of this license in the LICENSE file in the root directory +// of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +// +// Any modifications or derivative works of this code must retain this +// copyright notice, and modified files need to carry a notice indicating +// that they have been altered from the originals. + +import { readFile, readdir } from "fs/promises"; + +import yargs from "yargs/yargs"; +import { hideBin } from "yargs/helpers"; +import { $ } from "zx"; +import fs from "fs"; + +import { Pkg } from "../lib/api/Pkg"; +import { zxMain } from "../lib/zx"; + +interface Arguments { + [x: string]: unknown; + package?: string; + currentApisOnly: boolean; +} + +const readArgs = (): Arguments => { + return yargs(hideBin(process.argv)) + .version(false) + .option("package", { + alias: "p", + type: "string", + choices: Pkg.VALID_NAMES, + demandOption: false, + description: "Which package to update", + }) + .option("current-apis-only", { + type: "boolean", + default: false, + description: "Regenerate only the current API docs?", + }) + .parseSync(); +}; + +zxMain(async () => { + const args = readArgs(); + await validateGitStatus(); + + const results = new Map(); + for (const pkgName of Pkg.VALID_NAMES) { + if (args.package && pkgName != args.package) { + continue; + } + + const [historicalVersions, currentVersion] = await getPackageVersions( + pkgName, + args.currentApisOnly, + ); + const result = await processVersions( + pkgName, + historicalVersions, + currentVersion, + ); + results.set(pkgName, result); + } + + console.log(""); + results.forEach((result: string[], pkgName: string) => { + console.log(`Regeneration of ${pkgName}:`); + result.forEach((msg) => console.error(msg)); + console.log(""); + }); + + console.log(`Each regenerated version has been saved as a distinct commit. If the changes are +too large for one single PR, consider splitting it up into multiple PRs by using +git cherry-pick or git rebase -i so each PR only has the commits it wants to target.`); +}); + +async function processVersions( + pkgName: string, + historicalVersions: string[], + currentVersion: string, +): Promise { + const results: string[] = []; + + for (const historicalVersion of historicalVersions) { + results.push(await regenerateVersion(pkgName, historicalVersion)); + } + results.push(await regenerateVersion(pkgName, currentVersion, false)); + return results; +} + +async function regenerateVersion( + pkgName: string, + version: string, + historical: boolean = true, +): Promise { + try { + if (historical) { + await $`npm run gen-api -- -p ${pkgName} -v ${version} --historical`; + } else { + await $`npm run gen-api -- -p ${pkgName} -v ${version}`; + } + + if ((await gitStatus()) !== "") { + await gitCommit(`Regenerate ${pkgName} ${version}`); + return `✅ ${pkgName} ${version} regenerated correctly`; + } else { + return `☑️ ${pkgName} ${version} is up-to-date`; + } + } catch (_) { + await gitRestore("."); + return `❌ ${pkgName} ${version} failed to regenerate`; + } +} + +async function getPackageVersions( + pkgName: string, + currentApisOnly: boolean, +): Promise<[string[], string]> { + const pkgDocsPath = `docs/api/${pkgName}`; + const historicalVersions: string[] = []; + + if (!currentApisOnly) { + const historicalFolders = ( + await readdir(`${pkgDocsPath}`, { withFileTypes: true }) + ).filter((file) => file.isDirectory() && file.name.match(/[0-9].*/)); + + for (const folder of historicalFolders) { + const historicalVersion = JSON.parse( + await readFile(`${pkgDocsPath}/${folder.name}/_package.json`, "utf-8"), + ); + historicalVersions.push(historicalVersion.version); + } + } + + const currentVersion = JSON.parse( + await readFile(`${pkgDocsPath}/_package.json`, "utf-8"), + ); + + return [historicalVersions, currentVersion.version]; +} + +async function validateGitStatus(): Promise { + const initialStatus = await gitStatus(); + if (initialStatus !== "") { + console.error(` + Repository must have clean Git state when calling + \`npm run regenerate-apis\`. 'git status' returned:\n\n${initialStatus}`); + process.exit(1); + } + + const currentBranch = await gitBranch(); + if (currentBranch == "main\n") { + console.error(` + Please create a dedicated branch to regenerate the API docs correctly. + Use 'git checkout -b ' to continue`); + process.exit(1); + } +} + +async function gitStatus(): Promise { + const status = await $`git status --porcelain`.quiet(); + return status.stdout; +} + +async function gitBranch(): Promise { + const status = await $`git branch --show-current`.quiet(); + return status.stdout; +} + +async function gitCommit(message: string): Promise { + await $`git add .`; + await $`git commit -m ${message}`; +} + +async function gitRestore(path: string): Promise { + await $`git restore ${path}`; +}