Skip to content

Commit

Permalink
Enable linter analysis cache build phase and run the build tagger bef…
Browse files Browse the repository at this point in the history 
…ore linting

- Linting is now done in two phases: an initial analysis cache build phase,
  and then the actual linting phase, to decrease the memory footprint of
  linting.

Signed-off-by: Alper Rifat Ulucinar <[email protected]>
  • Loading branch information
@ulucinar
ulucinar committed Nov 5, 2024
1 parent 8d9c9fd commit 8c85121
Show file tree
Hide file tree
Showing 6 changed files with 332 additions and 289 deletions.
5 changes: 3 additions & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,9 @@ jobs:
- name: Lint
env:
GOLANGCI_LINT_CACHE: ${{ steps.go_cache.outputs.analysis_cache }}
SKIP_LINTER_ANALYSIS: true
RUN_BUILDTAGGER: false
SKIP_LINTER_ANALYSIS: false
RUN_BUILDTAGGER: true
GOGC: "50"
run: make lint

check-diff:
Expand Down
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ go.lint.analysiskey-interval:
go.lint.analysiskey:
@echo $$(make go.lint.analysiskey-interval)$$(sha1sum go.sum | cut -d' ' -f1)

.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA) load-pkg
.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.lint.analysiskey-interval go.lint.analysiskey go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA) load-pkg

build.init: kustomize-crds

Expand All @@ -417,7 +417,7 @@ build-lint-cache: $(GOLANGCILINT)
@# minimum.
@(BUILDTAGGER_DOWNLOAD_URL=$(BUILDTAGGER_DOWNLOAD_URL) ./scripts/tag.sh && \
(([[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] && $(OK) "Skipping analysis cache build phase because it's already been populated") && \
[[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags account,configregistry,configprovider,linter_run -v --disable-all --exclude '.*')) || $(FAIL)
[[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags activedirectory,configregistry,configprovider,linter_run -v --disable-all --exclude '.*')) || $(FAIL)
@$(OK) Running golangci-lint with the analysis cache building phase.

delete-build-tags:
Expand Down
17 changes: 17 additions & 0 deletions apis/linter_run.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//go:build linter_run

// SPDX-FileCopyrightText: 2024 The Crossplane Authors <https://crossplane.io>
//
// SPDX-License-Identifier: Apache-2.0

package apis

import "k8s.io/apimachinery/pkg/runtime"

// AddToSchemes may be used to add all resources defined in the project to a Scheme
var AddToSchemes runtime.SchemeBuilder

// AddToScheme adds all Resources to the Scheme
func AddToScheme(s *runtime.Scheme) error {
panic(`Must not be called in provider runtime. The provider should not have been built with the "linter_run" build constraint.`)
}
0 config/common/config.go → config/common/common.go
View file
File renamed without changes.
332 changes: 47 additions & 285 deletions config/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,6 @@
package config

import (
"context"
"strings"

// Note(ezgidemirel): we are importing this to embed provider schema document
_ "embed"

"github.com/crossplane/upjet/pkg/config"
ujconfig "github.com/crossplane/upjet/pkg/config"
"github.com/crossplane/upjet/pkg/config/conversion"
"github.com/crossplane/upjet/pkg/registry/reference"
"github.com/crossplane/upjet/pkg/schema/traverser"
conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson"
"github.com/crossplane/upjet/pkg/types/name"
tfjson "github.com/hashicorp/terraform-json"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-provider-google/google/provider"
"github.com/pkg/errors"

"github.com/upbound/provider-gcp/config/accessapproval"
"github.com/upbound/provider-gcp/config/accesscontextmanager"
"github.com/upbound/provider-gcp/config/apigee"
Expand Down Expand Up @@ -69,274 +51,54 @@ import (
"github.com/upbound/provider-gcp/config/tpu"
"github.com/upbound/provider-gcp/config/vertexai"
"github.com/upbound/provider-gcp/config/vpcaccess"
"github.com/upbound/provider-gcp/hack"
)

const (
resourcePrefix = "gcp"
modulePath = "github.com/upbound/provider-gcp"
)

var (
//go:embed schema.json
providerSchema string

//go:embed provider-metadata.yaml
providerMetadata []byte

// oldSingletonListAPIs is a newline-delimited list of Terraform resource
// names with converted singleton list APIs with at least CRD API version
// containing the old singleton list API. This is to prevent the API
// conversion for the newly added resources whose CRD APIs will already
// use embedded objects instead of the singleton lists and thus, will
// not possess a CRD API version with the singleton list. Thus, for
// the newly added resources (resources added after the singleton lists
// have been converted), we do not need the CRD API conversion
// functions that convert between singleton lists and embedded objects,
// but we need only the Terraform conversion functions.
// This list is immutable and represents the set of resources with the
// already generated CRD API versions with now converted singleton lists.
// Because new resources should never have singleton lists in their
// generated APIs, there should be no need to add them to this list.
// However, bugs might result in exceptions in the future.
// Please see:
// https://github.com/crossplane-contrib/provider-upjet-gcp/pull/508
// for more context on singleton list to embedded object conversions.
//go:embed old-singleton-list-apis.txt
oldSingletonListAPIs string
)

var skipList = []string{
// Note(turkenh): Following two resources conflicts their singular versions
// "google_access_context_manager_access_level" and
// "google_access_context_manager_service_perimeter". Skipping for now.
"google_access_context_manager_access_levels$",
"google_access_context_manager_service_perimeters$",
// Note(piotr): Following resources are potentially dangerous to implement
// details in: https://github.com/upbound/official-providers/issues/587
"google_kms_crypto_key_iam_policy",
"google_kms_crypto_key_iam_binding",
"google_kms_key_ring_iam_policy",
"google_kms_key_ring_iam_binding",
"google_cloudfunctions_function_iam_policy",
"google_cloudfunctions_function_iam_binding",
"google_compute_region_disk_iam_policy",
"google_compute_region_disk_iam_binding",
// Note(donovamuller): Following resources are potentially dangerous to implement
// details in: https://github.com/upbound/official-providers/issues/521
"google_project_iam_policy",
"google_project_iam_binding",
"google_organization_iam_binding",
"google_service_account_iam_policy",
"google_service_account_iam_binding",
"google_cloud_run_service_iam_policy",
"google_cloud_run_service_iam_binding",
"google_pubsub_topic_iam_policy",
"google_pubsub_topic_iam_binding",
"google_pubsub_subscription_iam_policy",
"google_pubsub_subscription_iam_binding",
"google_compute_disk_iam_policy",
"google_compute_disk_iam_binding",
"google_compute_instance_iam_policy",
"google_compute_instance_iam_binding",
"google_compute_image_iam_policy",
"google_compute_image_iam_binding",
"google_notebooks_instance_iam_policy",
"google_notebooks_instance_iam_binding",
"google_notebooks_runtime_iam_policy",
"google_notebooks_runtime_iam_binding",
"google_secret_manager_secret_iam_policy",
"google_secret_manager_secret_iam_binding",
"google_sourcerepo_repository_iam_policy",
"google_sourcerepo_repository_iam_binding",
"google_spanner_instance_iam_policy",
"google_spanner_instance_iam_binding",
"google_spanner_database_iam_policy",
"google_spanner_database_iam_binding",
"google_compute_subnetwork_iam_policy",
"google_compute_subnetwork_iam_binding",
"google_endpoints_service_iam_policy",
"google_endpoints_service_iam_binding",
"google_endpoints_service_consumers_iam_policy",
"google_endpoints_service_consumers_iam_binding",
}

// workaround for the TF Google v4.77.0-based no-fork release: We would like to
// keep the types in the generated CRDs intact
// (prevent number->int type replacements).
func getProviderSchema(s string) (*schema.Provider, error) {
ps := tfjson.ProviderSchemas{}
if err := ps.UnmarshalJSON([]byte(s)); err != nil {
panic(err)
}
if len(ps.Schemas) != 1 {
return nil, errors.Errorf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas))
}
var rs map[string]*tfjson.Schema
for _, v := range ps.Schemas {
rs = v.ResourceSchemas
break
}
return &schema.Provider{
ResourcesMap: conversiontfjson.GetV2ResourceMap(rs),
}, nil
}

// GetProvider returns provider configuration
func GetProvider(_ context.Context, generationProvider bool) (*ujconfig.Provider, error) {
sdkProvider := provider.Provider()

if generationProvider {
p, err := getProviderSchema(providerSchema)
if err != nil {
return nil, errors.Wrap(err, "cannot read the Terraform SDK provider from the JSON schema for code generation")
}
if err := traverser.TFResourceSchema(sdkProvider.ResourcesMap).Traverse(traverser.NewMaxItemsSync(p.ResourcesMap)); err != nil {
return nil, errors.Wrap(err, "cannot sync the MaxItems constraints between the Go schema and the JSON schema")
}
// use the JSON schema to temporarily prevent float64->int64
// conversions in the CRD APIs.
// We would like to convert to int64s with the next major release of
// the provider.
sdkProvider = p
}

pc := ujconfig.NewProvider([]byte(providerSchema), resourcePrefix, modulePath, providerMetadata,
ujconfig.WithDefaultResourceOptions(
groupOverrides(),
externalNameConfig(),
defaultVersion(),
resourceConfigurator(),
descriptionOverrides(),
),
ujconfig.WithRootGroup("gcp.upbound.io"),
ujconfig.WithShortName("gcp"),
// Comment out the following line to generate all resources.
ujconfig.WithIncludeList(resourceList(cliReconciledExternalNameConfigs)),
ujconfig.WithTerraformPluginSDKIncludeList(resourceList(terraformPluginSDKExternalNameConfigs)),
ujconfig.WithReferenceInjectors([]ujconfig.ReferenceInjector{reference.NewInjector(modulePath)}),
ujconfig.WithSkipList(skipList),
ujconfig.WithFeaturesPackage("internal/features"),
ujconfig.WithMainTemplate(hack.MainTemplate),
ujconfig.WithTerraformProvider(sdkProvider),
ujconfig.WithSchemaTraversers(&ujconfig.SingletonListEmbedder{}),
)

bumpVersionsWithEmbeddedLists(pc)
for _, configure := range []func(provider *ujconfig.Provider){
accessapproval.Configure,
accesscontextmanager.Configure,
apigee.Configure,
bigtable.Configure,
composer.Configure,
cloudfunctions.Configure,
cloudplatform.Configure,
cloudrun.Configure,
cloudscheduler.Configure,
cloudtasks.Configure,
containerattached.Configure,
containeraws.Configure,
containerazure.Configure,
compute.Configure,
container.Configure,
dataflow.Configure,
dataproc.Configure,
dns.Configure,
endpoints.Configure,
firebaserules.Configure,
gameservices.Configure,
iap.Configure,
identityplatform.Configure,
logging.Configure,
kms.Configure,
notebooks.Configure,
privateca.Configure,
oslogin.Configure,
project.Configure,
pubsub.Configure,
redis.Configure,
secretmanager.Configure,
servicenetworking.Configure,
sourcerepo.Configure,
spanner.Configure,
storage.Configure,
sql.Configure,
redis.Configure,
bigquery.Configure,
beyondcorp.Configure,
vertexai.Configure,
tags.Configure,
tpu.Configure,
vpcaccess.Configure,
healthcare.Configure,
gkehub.Configure,
monitoring.Configure,
} {
configure(pc)
}

pc.ConfigureResources()
return pc, nil
}

// resourceList returns the list of resources that have external
// name configured in the specified table.
func resourceList(t map[string]ujconfig.ExternalName) []string {
l := make([]string, len(t))
i := 0
for n := range t {
// Expected format is regex and we'd like to have exact matches.
l[i] = n + "$"
i++
}
return l
}

func bumpVersionsWithEmbeddedLists(pc *ujconfig.Provider) {
l := strings.Split(strings.TrimSpace(oldSingletonListAPIs), "\n")
oldSLAPIs := make(map[string]struct{}, len(l))
for _, n := range l {
oldSLAPIs[n] = struct{}{}
}

for n, r := range pc.Resources {
r := r
// nothing to do if no singleton list has been converted to
// an embedded object
if len(r.CRDListConversionPaths()) == 0 {
continue
}

if _, ok := oldSLAPIs[n]; ok {
r.Version = "v1beta2"
r.PreviousVersions = []string{"v1beta1"}
// we would like to set the storage version to v1beta1 to facilitate
// downgrades.
r.SetCRDStorageVersion("v1beta1")
// because the controller reconciles on the API version with the singleton list API,
// no need for a Terraform conversion.
r.ControllerReconcileVersion = "v1beta1"
r.Conversions = []conversion.Conversion{
conversion.NewIdentityConversionExpandPaths(conversion.AllVersions, conversion.AllVersions, conversion.DefaultPathPrefixes(), r.CRDListConversionPaths()...),
conversion.NewSingletonListConversion("v1beta1", "v1beta2", conversion.DefaultPathPrefixes(), r.CRDListConversionPaths(), conversion.ToEmbeddedObject),
conversion.NewSingletonListConversion("v1beta2", "v1beta1", conversion.DefaultPathPrefixes(), r.CRDListConversionPaths(), conversion.ToSingletonList)}
} else {
// the controller will be reconciling on the CRD API version
// with the converted API (with embedded objects in place of
// singleton lists), so we need the appropriate Terraform
// converter in this case.
r.TerraformConversions = []config.TerraformConversion{
config.NewTFSingletonConversion(),
}
}
pc.Resources[n] = r
}
}

func init() {
// GCP specific acronyms

// Todo(turkenh): move to Terrajet?
name.AddAcronym("idp", "IdP")
name.AddAcronym("oauth", "OAuth")
ProviderConfiguration.AddConfig(accessapproval.Configure)
ProviderConfiguration.AddConfig(accesscontextmanager.Configure)
ProviderConfiguration.AddConfig(apigee.Configure)
ProviderConfiguration.AddConfig(bigtable.Configure)
ProviderConfiguration.AddConfig(composer.Configure)
ProviderConfiguration.AddConfig(cloudfunctions.Configure)
ProviderConfiguration.AddConfig(cloudplatform.Configure)
ProviderConfiguration.AddConfig(cloudrun.Configure)
ProviderConfiguration.AddConfig(cloudscheduler.Configure)
ProviderConfiguration.AddConfig(cloudtasks.Configure)
ProviderConfiguration.AddConfig(containerattached.Configure)
ProviderConfiguration.AddConfig(containeraws.Configure)
ProviderConfiguration.AddConfig(containerazure.Configure)
ProviderConfiguration.AddConfig(compute.Configure)
ProviderConfiguration.AddConfig(container.Configure)
ProviderConfiguration.AddConfig(dataflow.Configure)
ProviderConfiguration.AddConfig(dataproc.Configure)
ProviderConfiguration.AddConfig(dns.Configure)
ProviderConfiguration.AddConfig(endpoints.Configure)
ProviderConfiguration.AddConfig(firebaserules.Configure)
ProviderConfiguration.AddConfig(gameservices.Configure)
ProviderConfiguration.AddConfig(iap.Configure)
ProviderConfiguration.AddConfig(identityplatform.Configure)
ProviderConfiguration.AddConfig(logging.Configure)
ProviderConfiguration.AddConfig(kms.Configure)
ProviderConfiguration.AddConfig(notebooks.Configure)
ProviderConfiguration.AddConfig(privateca.Configure)
ProviderConfiguration.AddConfig(oslogin.Configure)
ProviderConfiguration.AddConfig(project.Configure)
ProviderConfiguration.AddConfig(pubsub.Configure)
ProviderConfiguration.AddConfig(redis.Configure)
ProviderConfiguration.AddConfig(secretmanager.Configure)
ProviderConfiguration.AddConfig(servicenetworking.Configure)
ProviderConfiguration.AddConfig(sourcerepo.Configure)
ProviderConfiguration.AddConfig(spanner.Configure)
ProviderConfiguration.AddConfig(storage.Configure)
ProviderConfiguration.AddConfig(sql.Configure)
ProviderConfiguration.AddConfig(redis.Configure)
ProviderConfiguration.AddConfig(bigquery.Configure)
ProviderConfiguration.AddConfig(beyondcorp.Configure)
ProviderConfiguration.AddConfig(vertexai.Configure)
ProviderConfiguration.AddConfig(tags.Configure)
ProviderConfiguration.AddConfig(tpu.Configure)
ProviderConfiguration.AddConfig(vpcaccess.Configure)
ProviderConfiguration.AddConfig(healthcare.Configure)
ProviderConfiguration.AddConfig(gkehub.Configure)
ProviderConfiguration.AddConfig(monitoring.Configure)
}
Loading

0 comments on commit 8c85121

Please sign in to comment.