From bbcc81405b1262c2c1d2dfc234e2722158a2a85b Mon Sep 17 00:00:00 2001 From: Kat Yang <69819079+yangkb09@users.noreply.github.com> Date: Tue, 24 Oct 2023 13:01:04 -0400 Subject: [PATCH] Chore: Add CI Pipeline to generate Grafana's OpenAPI specification (#75393) * chore: move over initial changes from sofia's pr #58029 * chore: remove go_image * chore: begin removing edition, remove unused imports * chore: remove edition from swagger_gen.star and generate .drone.yml * chore: regen drone.yml * fix: fix order of load statements * fix: try #2 fix order of load statements * linter fixes * chore: add doc comment explaining purpose of new clone_pr_branch step * fix: add placeholder documentation for ver_mode arg * attempt #1 to import and use clone_enterprise_step_pr * Update scripts/drone/pipelines/swagger_gen.star Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com> * attempt #2 to import and use clone_enterprise_step_pr * fix: fix drone lint err invalid or unknown step dependency * fix: regen hmac to make drone run * fix: fix drone lint err * fix: update swagger-clean cmd in step * attempt to return non zero exit code * test to see if pipeline fails * fix: add git to clone pr branch step * fix: add git and make to swagger-gen step * try to rerun drone * debug: figure out why cannot find make swagger-clean * debug: see if cd grafana/grafana fixes things * debug: undo cd grafana/grafana * debug: add more logging statements * debug: try and remove cd grafana in swagger-gen * debug: removed grafana after clone statement; add debug before cloning * fix: remove disable clone * regen specs to see if swagger-gen step passes now * add descriptive error message to swagger-gen step * remove api-spec.json from .gitignore * revert backend change, regen spec * add back backend change, regen specs * Update scripts/drone/pipelines/swagger_gen.star Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com> * Update scripts/drone/pipelines/swagger_gen.star Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com> * revert gitignore change, set enterprise source to drone source branch * chore: remove unused variable, change committish var name to source * testing functionality: make a new BE change without gen spec, pipeline should fail * test functionality: does removing err msg cause step to fail properly * test functionality: add back err msg and && after * chore: remove debug statements from swagger gen step * chore: remove debug lines from clone_pr_branch step * test functionality: regen specs, swagger_gen step should pass * test funcionality: regen specs again ???? * chore: update swagger-gen step err msg * test functionality: make BE change dont regen spec, swagger gen should fail * test functionality: regen the specs, swagger-gen should pass * chore: revert test BE change, regen spec * chore: remove unused clone step * chore: regen drone.yml --------- Co-authored-by: Timur Olzhabayev Co-authored-by: Sofia Papagiannaki <1632407+papagian@users.noreply.github.com> --- .drone.yml | 62 ++++++++++++++++++++- public/api-merged.json | 71 ++++++++++++++++-------- public/openapi3.json | 71 ++++++++++++++++-------- scripts/drone/events/pr.star | 8 +++ scripts/drone/pipelines/swagger_gen.star | 56 +++++++++++++++++++ 5 files changed, 219 insertions(+), 49 deletions(-) create mode 100644 scripts/drone/pipelines/swagger_gen.star diff --git a/.drone.yml b/.drone.yml index ead82b9229605..385229cdfb6cb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1132,6 +1132,66 @@ volumes: clone: retries: 3 depends_on: [] +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: pr-swagger-gen +node: + type: no-parallel +platform: + arch: amd64 + os: linux +services: [] +steps: +- commands: + - apk add --update curl jq bash + - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" + | jq .head.repo.fork) + - if [ "$is_fork" != false ]; then return 1; fi + - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" + grafana-enterprise + - cd grafana-enterprise + - if git checkout ${DRONE_SOURCE_BRANCH}; then echo "checked out ${DRONE_SOURCE_BRANCH}"; + elif git checkout main; then echo "git checkout main"; else git checkout main; + fi + environment: + GITHUB_TOKEN: + from_secret: github_token + image: alpine/git:2.40.1 + name: clone-enterprise +- commands: + - apk add --update git make + - make swagger-clean && make openapi3-gen + - for f in public/api-merged.json public/openapi3.json; do git add $f; done + - if [ -z "$(git diff --name-only --cached)" ]; then echo "Everything seems up to + date!"; else echo "Please ensure the branch is up-to-date, then regenerate the + specification by running make swagger-clean && make openapi3-gen" && return 1; + fi + depends_on: + - clone-enterprise + environment: + GITHUB_TOKEN: + from_secret: github_token + image: golang:1.20.10-alpine + name: swagger-gen +trigger: + event: + - pull_request + paths: + exclude: + - docs/** + - '*.md' + include: + - pkg/** +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +clone: + retries: 3 +depends_on: [] environment: EDITION: oss image_pull_secrets: @@ -4607,6 +4667,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 93de8a710e23d3f1d31860f9eed34cd841b0a0eb48637971de4e8ce60a7c3df1 +hmac: 29a933affceb9cc39b285d936de9e6327deedbb80f1285fa645d596f89ede442 ... diff --git a/public/api-merged.json b/public/api-merged.json index 69a907542a622..326dfb0aa1889 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -639,30 +639,6 @@ } } }, - "/admin/ldap-sync-status": { - "get": { - "description": "You need to have a permission with action `ldap.status:read`.", - "tags": [ - "ldap_debug" - ], - "summary": "Returns the current state of the LDAP background sync integration.", - "operationId": "getSyncStatus", - "responses": { - "200": { - "$ref": "#/responses/getSyncStatusResponse" - }, - "401": { - "$ref": "#/responses/unauthorisedError" - }, - "403": { - "$ref": "#/responses/forbiddenError" - }, - "500": { - "$ref": "#/responses/internalServerError" - } - } - } - }, "/admin/ldap/reload": { "post": { "security": [ @@ -15166,6 +15142,53 @@ } } }, + "JSONWebKey": { + "type": "object", + "title": "JSONWebKey represents a public or private key in JWK format.", + "properties": { + "Algorithm": { + "description": "Key algorithm, parsed from `alg` header.", + "type": "string" + }, + "CertificateThumbprintSHA1": { + "description": "X.509 certificate thumbprint (SHA-1), parsed from `x5t` header.", + "type": "array", + "items": { + "type": "integer", + "format": "uint8" + } + }, + "CertificateThumbprintSHA256": { + "description": "X.509 certificate thumbprint (SHA-256), parsed from `x5t#S256` header.", + "type": "array", + "items": { + "type": "integer", + "format": "uint8" + } + }, + "Certificates": { + "description": "X.509 certificate chain, parsed from `x5c` header.", + "type": "array", + "items": { + "$ref": "#/definitions/Certificate" + } + }, + "CertificatesURL": { + "$ref": "#/definitions/URL" + }, + "Key": { + "description": "Cryptographic key, can be a symmetric or asymmetric key." + }, + "KeyID": { + "description": "Key identifier, parsed from `kid` header.", + "type": "string" + }, + "Use": { + "description": "Key use, parsed from `use` header.", + "type": "string" + } + } + }, "Json": { "type": "object" }, diff --git a/public/openapi3.json b/public/openapi3.json index 5a66d55d215c1..d577849398e30 100644 --- a/public/openapi3.json +++ b/public/openapi3.json @@ -6234,6 +6234,53 @@ "title": "JSONWebKey represents a public or private key in JWK format.", "type": "object" }, + "JSONWebKey": { + "properties": { + "Algorithm": { + "description": "Key algorithm, parsed from `alg` header.", + "type": "string" + }, + "CertificateThumbprintSHA1": { + "description": "X.509 certificate thumbprint (SHA-1), parsed from `x5t` header.", + "items": { + "format": "uint8", + "type": "integer" + }, + "type": "array" + }, + "CertificateThumbprintSHA256": { + "description": "X.509 certificate thumbprint (SHA-256), parsed from `x5t#S256` header.", + "items": { + "format": "uint8", + "type": "integer" + }, + "type": "array" + }, + "Certificates": { + "description": "X.509 certificate chain, parsed from `x5c` header.", + "items": { + "$ref": "#/components/schemas/Certificate" + }, + "type": "array" + }, + "CertificatesURL": { + "$ref": "#/components/schemas/URL" + }, + "Key": { + "description": "Cryptographic key, can be a symmetric or asymmetric key." + }, + "KeyID": { + "description": "Key identifier, parsed from `kid` header.", + "type": "string" + }, + "Use": { + "description": "Key use, parsed from `use` header.", + "type": "string" + } + }, + "title": "JSONWebKey represents a public or private key in JWK format.", + "type": "object" + }, "Json": { "type": "object" }, @@ -12481,30 +12528,6 @@ ] } }, - "/admin/ldap-sync-status": { - "get": { - "description": "You need to have a permission with action `ldap.status:read`.", - "operationId": "getSyncStatus", - "responses": { - "200": { - "$ref": "#/components/responses/getSyncStatusResponse" - }, - "401": { - "$ref": "#/components/responses/unauthorisedError" - }, - "403": { - "$ref": "#/components/responses/forbiddenError" - }, - "500": { - "$ref": "#/components/responses/internalServerError" - } - }, - "summary": "Returns the current state of the LDAP background sync integration.", - "tags": [ - "ldap_debug" - ] - } - }, "/admin/ldap/reload": { "post": { "description": "If you are running Grafana Enterprise and have Fine-grained access control enabled, you need to have a permission with action `ldap.config:reload`.", diff --git a/scripts/drone/events/pr.star b/scripts/drone/events/pr.star index cbd425efcf6da..d7b751511975f 100644 --- a/scripts/drone/events/pr.star +++ b/scripts/drone/events/pr.star @@ -32,6 +32,10 @@ load( "scripts/drone/pipelines/shellcheck.star", "shellcheck_pipeline", ) +load( + "scripts/drone/pipelines/swagger_gen.star", + "swagger_gen", +) load( "scripts/drone/pipelines/test_backend.star", "test_backend", @@ -137,6 +141,10 @@ def pr_pipelines(): ), docs_pipelines(ver_mode, trigger_docs_pr()), shellcheck_pipeline(), + swagger_gen( + get_pr_trigger(include_paths = ["pkg/**"]), + ver_mode, + ), integration_benchmarks( prefix = ver_mode, ), diff --git a/scripts/drone/pipelines/swagger_gen.star b/scripts/drone/pipelines/swagger_gen.star new file mode 100644 index 0000000000000..755b84ee4cb23 --- /dev/null +++ b/scripts/drone/pipelines/swagger_gen.star @@ -0,0 +1,56 @@ +""" +This module returns all pipelines used in OpenAPI specification generation of Grafana HTTP APIs +""" + +load( + "scripts/drone/steps/lib.star", + "clone_enterprise_step_pr", +) +load( + "scripts/drone/utils/images.star", + "images", +) +load( + "scripts/drone/utils/utils.star", + "pipeline", +) +load( + "scripts/drone/vault.star", + "from_secret", +) + +def swagger_gen_step(ver_mode): + if ver_mode != "pr": + return None + + return { + "name": "swagger-gen", + "image": images["go"], + "environment": { + "GITHUB_TOKEN": from_secret("github_token"), + }, + "commands": [ + "apk add --update git make", + "make swagger-clean && make openapi3-gen", + "for f in public/api-merged.json public/openapi3.json; do git add $f; done", + 'if [ -z "$(git diff --name-only --cached)" ]; then echo "Everything seems up to date!"; else echo "Please ensure the branch is up-to-date, then regenerate the specification by running make swagger-clean && make openapi3-gen" && return 1; fi', + ], + "depends_on": [ + "clone-enterprise", + ], + } + +def swagger_gen(trigger, ver_mode, source = "${DRONE_SOURCE_BRANCH}"): + test_steps = [ + clone_enterprise_step_pr(source = source), + swagger_gen_step(ver_mode = ver_mode), + ] + + p = pipeline( + name = "{}-swagger-gen".format(ver_mode), + trigger = trigger, + services = [], + steps = test_steps, + ) + + return p