Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable linter analysis cache build phase and run the build tagger before linting #649

Merged
merged 1 commit into from
Nov 5, 2024
Merged

Enable linter analysis cache build phase and run the build tagger before linting #649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,9 @@ jobs:
- name: Lint
env:
GOLANGCI_LINT_CACHE: ${{ steps.go_cache.outputs.analysis_cache }}
SKIP_LINTER_ANALYSIS: true
RUN_BUILDTAGGER: false
SKIP_LINTER_ANALYSIS: false
RUN_BUILDTAGGER: true
GOGC: "50"
run: make lint

check-diff:
Expand Down
4 changes: 2 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ go.lint.analysiskey-interval:
go.lint.analysiskey:
@echo $$(make go.lint.analysiskey-interval)$$(sha1sum go.sum | cut -d' ' -f1)

.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA) load-pkg
.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.lint.analysiskey-interval go.lint.analysiskey go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA) load-pkg

build.init: kustomize-crds

Expand All @@ -417,7 +417,7 @@ build-lint-cache: $(GOLANGCILINT)
@# minimum.
@(BUILDTAGGER_DOWNLOAD_URL=$(BUILDTAGGER_DOWNLOAD_URL) ./scripts/tag.sh && \
(([[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] && $(OK) "Skipping analysis cache build phase because it's already been populated") && \
[[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags account,configregistry,configprovider,linter_run -v --disable-all --exclude '.*')) || $(FAIL)
[[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags activedirectory,configregistry,configprovider,linter_run -v --disable-all --exclude '.*')) || $(FAIL)
@$(OK) Running golangci-lint with the analysis cache building phase.

delete-build-tags:
Expand Down
17 changes: 17 additions & 0 deletions apis/linter_run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//go:build linter_run

// SPDX-FileCopyrightText: 2024 The Crossplane Authors <https://crossplane.io>
//
// SPDX-License-Identifier: Apache-2.0

package apis

import "k8s.io/apimachinery/pkg/runtime"

// AddToSchemes may be used to add all resources defined in the project to a Scheme
var AddToSchemes runtime.SchemeBuilder

// AddToScheme adds all Resources to the Scheme
func AddToScheme(s *runtime.Scheme) error {
panic(`Must not be called in provider runtime. The provider should not have been built with the "linter_run" build constraint.`)
}
0 config/common/config.go → config/common/common.go
View file
Open in desktop
File renamed without changes.
332 changes: 47 additions & 285 deletions config/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,6 @@
package config

import (
"context"
"strings"

// Note(ezgidemirel): we are importing this to embed provider schema document
_ "embed"

"github.com/crossplane/upjet/pkg/config"
ujconfig "github.com/crossplane/upjet/pkg/config"
"github.com/crossplane/upjet/pkg/config/conversion"
"github.com/crossplane/upjet/pkg/registry/reference"
"github.com/crossplane/upjet/pkg/schema/traverser"
conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson"
"github.com/crossplane/upjet/pkg/types/name"
tfjson "github.com/hashicorp/terraform-json"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-provider-google/google/provider"
"github.com/pkg/errors"

"github.com/upbound/provider-gcp/config/accessapproval"
"github.com/upbound/provider-gcp/config/accesscontextmanager"
"github.com/upbound/provider-gcp/config/apigee"
Expand Down Expand Up @@ -69,274 +51,54 @@ import (
"github.com/upbound/provider-gcp/config/tpu"
"github.com/upbound/provider-gcp/config/vertexai"
"github.com/upbound/provider-gcp/config/vpcaccess"
"github.com/upbound/provider-gcp/hack"
)

const (
resourcePrefix = "gcp"
modulePath = "github.com/upbound/provider-gcp"
)

var (
//go:embed schema.json
providerSchema string

//go:embed provider-metadata.yaml
providerMetadata []byte

// oldSingletonListAPIs is a newline-delimited list of Terraform resource
// names with converted singleton list APIs with at least CRD API version
// containing the old singleton list API. This is to prevent the API
// conversion for the newly added resources whose CRD APIs will already
// use embedded objects instead of the singleton lists and thus, will
// not possess a CRD API version with the singleton list. Thus, for
// the newly added resources (resources added after the singleton lists
// have been converted), we do not need the CRD API conversion
// functions that convert between singleton lists and embedded objects,
// but we need only the Terraform conversion functions.
// This list is immutable and represents the set of resources with the
// already generated CRD API versions with now converted singleton lists.
// Because new resources should never have singleton lists in their
// generated APIs, there should be no need to add them to this list.
// However, bugs might result in exceptions in the future.
// Please see:
// https://github.com/crossplane-contrib/provider-upjet-gcp/pull/508
// for more context on singleton list to embedded object conversions.
//go:embed old-singleton-list-apis.txt
oldSingletonListAPIs string
)

var skipList = []string{
// Note(turkenh): Following two resources conflicts their singular versions
// "google_access_context_manager_access_level" and
// "google_access_context_manager_service_perimeter". Skipping for now.
"google_access_context_manager_access_levels$",
"google_access_context_manager_service_perimeters$",
// Note(piotr): Following resources are potentially dangerous to implement
// details in: https://github.com/upbound/official-providers/issues/587
"google_kms_crypto_key_iam_policy",
"google_kms_crypto_key_iam_binding",
"google_kms_key_ring_iam_policy",
"google_kms_key_ring_iam_binding",
"google_cloudfunctions_function_iam_policy",
"google_cloudfunctions_function_iam_binding",
"google_compute_region_disk_iam_policy",
"google_compute_region_disk_iam_binding",
// Note(donovamuller): Following resources are potentially dangerous to implement
// details in: https://github.com/upbound/official-providers/issues/521
"google_project_iam_policy",
"google_project_iam_binding",
"google_organization_iam_binding",
"google_service_account_iam_policy",
"google_service_account_iam_binding",
"google_cloud_run_service_iam_policy",
"google_cloud_run_service_iam_binding",
"google_pubsub_topic_iam_policy",
"google_pubsub_topic_iam_binding",
"google_pubsub_subscription_iam_policy",
"google_pubsub_subscription_iam_binding",
"google_compute_disk_iam_policy",
"google_compute_disk_iam_binding",
"google_compute_instance_iam_policy",
"google_compute_instance_iam_binding",
"google_compute_image_iam_policy",
"google_compute_image_iam_binding",
"google_notebooks_instance_iam_policy",
"google_notebooks_instance_iam_binding",
"google_notebooks_runtime_iam_policy",
"google_notebooks_runtime_iam_binding",
"google_secret_manager_secret_iam_policy",
"google_secret_manager_secret_iam_binding",
"google_sourcerepo_repository_iam_policy",
"google_sourcerepo_repository_iam_binding",
"google_spanner_instance_iam_policy",
"google_spanner_instance_iam_binding",
"google_spanner_database_iam_policy",
"google_spanner_database_iam_binding",
"google_compute_subnetwork_iam_policy",
"google_compute_subnetwork_iam_binding",
"google_endpoints_service_iam_policy",
"google_endpoints_service_iam_binding",
"google_endpoints_service_consumers_iam_policy",
"google_endpoints_service_consumers_iam_binding",
}

// workaround for the TF Google v4.77.0-based no-fork release: We would like to
// keep the types in the generated CRDs intact
// (prevent number->int type replacements).
func getProviderSchema(s string) (*schema.Provider, error) {
ps := tfjson.ProviderSchemas{}
if err := ps.UnmarshalJSON([]byte(s)); err != nil {
panic(err)
}
if len(ps.Schemas) != 1 {
return nil, errors.Errorf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas))
}
var rs map[string]*tfjson.Schema
for _, v := range ps.Schemas {
rs = v.ResourceSchemas
break
}
return &schema.Provider{
ResourcesMap: conversiontfjson.GetV2ResourceMap(rs),
}, nil
}

// GetProvider returns provider configuration
func GetProvider(_ context.Context, generationProvider bool) (*ujconfig.Provider, error) {
sdkProvider := provider.Provider()

if generationProvider {
p, err := getProviderSchema(providerSchema)
if err != nil {
return nil, errors.Wrap(err, "cannot read the Terraform SDK provider from the JSON schema for code generation")
}
if err := traverser.TFResourceSchema(sdkProvider.ResourcesMap).Traverse(traverser.NewMaxItemsSync(p.ResourcesMap)); err != nil {
return nil, errors.Wrap(err, "cannot sync the MaxItems constraints between the Go schema and the JSON schema")
}
// use the JSON schema to temporarily prevent float64->int64
// conversions in the CRD APIs.
// We would like to convert to int64s with the next major release of
// the provider.
sdkProvider = p
}

pc := ujconfig.NewProvider([]byte(providerSchema), resourcePrefix, modulePath, providerMetadata,
ujconfig.WithDefaultResourceOptions(
groupOverrides(),
externalNameConfig(),
defaultVersion(),
resourceConfigurator(),
descriptionOverrides(),
),
ujconfig.WithRootGroup("gcp.upbound.io"),
ujconfig.WithShortName("gcp"),
// Comment out the following line to generate all resources.
ujconfig.WithIncludeList(resourceList(cliReconciledExternalNameConfigs)),
ujconfig.WithTerraformPluginSDKIncludeList(resourceList(terraformPluginSDKExternalNameConfigs)),
ujconfig.WithReferenceInjectors([]ujconfig.ReferenceInjector{reference.NewInjector(modulePath)}),
ujconfig.WithSkipList(skipList),
ujconfig.WithFeaturesPackage("internal/features"),
ujconfig.WithMainTemplate(hack.MainTemplate),
ujconfig.WithTerraformProvider(sdkProvider),
ujconfig.WithSchemaTraversers(&ujconfig.SingletonListEmbedder{}),
)

bumpVersionsWithEmbeddedLists(pc)
for _, configure := range []func(provider *ujconfig.Provider){
accessapproval.Configure,
accesscontextmanager.Configure,
apigee.Configure,
bigtable.Configure,
composer.Configure,
cloudfunctions.Configure,
cloudplatform.Configure,
cloudrun.Configure,
cloudscheduler.Configure,
cloudtasks.Configure,
containerattached.Configure,
containeraws.Configure,
containerazure.Configure,
compute.Configure,
container.Configure,
dataflow.Configure,
dataproc.Configure,
dns.Configure,
endpoints.Configure,
firebaserules.Configure,
gameservices.Configure,
iap.Configure,
identityplatform.Configure,
logging.Configure,
kms.Configure,
notebooks.Configure,
privateca.Configure,
oslogin.Configure,
project.Configure,
pubsub.Configure,
redis.Configure,
secretmanager.Configure,
servicenetworking.Configure,
sourcerepo.Configure,
spanner.Configure,
storage.Configure,
sql.Configure,
redis.Configure,
bigquery.Configure,
beyondcorp.Configure,
vertexai.Configure,
tags.Configure,
tpu.Configure,
vpcaccess.Configure,
healthcare.Configure,
gkehub.Configure,
monitoring.Configure,
} {
configure(pc)
}

pc.ConfigureResources()
return pc, nil
}

// resourceList returns the list of resources that have external
// name configured in the specified table.
func resourceList(t map[string]ujconfig.ExternalName) []string {
l := make([]string, len(t))
i := 0
for n := range t {
// Expected format is regex and we'd like to have exact matches.
l[i] = n + "$"
i++
}
return l
}

func bumpVersionsWithEmbeddedLists(pc *ujconfig.Provider) {
l := strings.Split(strings.TrimSpace(oldSingletonListAPIs), "\n")
oldSLAPIs := make(map[string]struct{}, len(l))
for _, n := range l {
oldSLAPIs[n] = struct{}{}
}

for n, r := range pc.Resources {
r := r
// nothing to do if no singleton list has been converted to
// an embedded object
if len(r.CRDListConversionPaths()) == 0 {
continue
}

if _, ok := oldSLAPIs[n]; ok {
r.Version = "v1beta2"
r.PreviousVersions = []string{"v1beta1"}
// we would like to set the storage version to v1beta1 to facilitate
// downgrades.
r.SetCRDStorageVersion("v1beta1")
// because the controller reconciles on the API version with the singleton list API,
// no need for a Terraform conversion.
r.ControllerReconcileVersion = "v1beta1"
r.Conversions = []conversion.Conversion{
conversion.NewIdentityConversionExpandPaths(conversion.AllVersions, conversion.AllVersions, conversion.DefaultPathPrefixes(), r.CRDListConversionPaths()...),
conversion.NewSingletonListConversion("v1beta1", "v1beta2", conversion.DefaultPathPrefixes(), r.CRDListConversionPaths(), conversion.ToEmbeddedObject),
conversion.NewSingletonListConversion("v1beta2", "v1beta1", conversion.DefaultPathPrefixes(), r.CRDListConversionPaths(), conversion.ToSingletonList)}
} else {
// the controller will be reconciling on the CRD API version
// with the converted API (with embedded objects in place of
// singleton lists), so we need the appropriate Terraform
// converter in this case.
r.TerraformConversions = []config.TerraformConversion{
config.NewTFSingletonConversion(),
}
}
pc.Resources[n] = r
}
}

func init() {
// GCP specific acronyms

// Todo(turkenh): move to Terrajet?
name.AddAcronym("idp", "IdP")
name.AddAcronym("oauth", "OAuth")
ProviderConfiguration.AddConfig(accessapproval.Configure)
ProviderConfiguration.AddConfig(accesscontextmanager.Configure)
ProviderConfiguration.AddConfig(apigee.Configure)
ProviderConfiguration.AddConfig(bigtable.Configure)
ProviderConfiguration.AddConfig(composer.Configure)
ProviderConfiguration.AddConfig(cloudfunctions.Configure)
ProviderConfiguration.AddConfig(cloudplatform.Configure)
ProviderConfiguration.AddConfig(cloudrun.Configure)
ProviderConfiguration.AddConfig(cloudscheduler.Configure)
ProviderConfiguration.AddConfig(cloudtasks.Configure)
ProviderConfiguration.AddConfig(containerattached.Configure)
ProviderConfiguration.AddConfig(containeraws.Configure)
ProviderConfiguration.AddConfig(containerazure.Configure)
ProviderConfiguration.AddConfig(compute.Configure)
ProviderConfiguration.AddConfig(container.Configure)
ProviderConfiguration.AddConfig(dataflow.Configure)
ProviderConfiguration.AddConfig(dataproc.Configure)
ProviderConfiguration.AddConfig(dns.Configure)
ProviderConfiguration.AddConfig(endpoints.Configure)
ProviderConfiguration.AddConfig(firebaserules.Configure)
ProviderConfiguration.AddConfig(gameservices.Configure)
ProviderConfiguration.AddConfig(iap.Configure)
ProviderConfiguration.AddConfig(identityplatform.Configure)
ProviderConfiguration.AddConfig(logging.Configure)
ProviderConfiguration.AddConfig(kms.Configure)
ProviderConfiguration.AddConfig(notebooks.Configure)
ProviderConfiguration.AddConfig(privateca.Configure)
ProviderConfiguration.AddConfig(oslogin.Configure)
ProviderConfiguration.AddConfig(project.Configure)
ProviderConfiguration.AddConfig(pubsub.Configure)
ProviderConfiguration.AddConfig(redis.Configure)
ProviderConfiguration.AddConfig(secretmanager.Configure)
ProviderConfiguration.AddConfig(servicenetworking.Configure)
ProviderConfiguration.AddConfig(sourcerepo.Configure)
ProviderConfiguration.AddConfig(spanner.Configure)
ProviderConfiguration.AddConfig(storage.Configure)
ProviderConfiguration.AddConfig(sql.Configure)
ProviderConfiguration.AddConfig(redis.Configure)
ProviderConfiguration.AddConfig(bigquery.Configure)
ProviderConfiguration.AddConfig(beyondcorp.Configure)
ProviderConfiguration.AddConfig(vertexai.Configure)
ProviderConfiguration.AddConfig(tags.Configure)
ProviderConfiguration.AddConfig(tpu.Configure)
ProviderConfiguration.AddConfig(vpcaccess.Configure)
ProviderConfiguration.AddConfig(healthcare.Configure)
ProviderConfiguration.AddConfig(gkehub.Configure)
ProviderConfiguration.AddConfig(monitoring.Configure)
}
Loading
Loading