diff --git a/pf/internal/muxer/muxer.go b/pf/internal/muxer/muxer.go index 42fdb0c79..336c5c00a 100644 --- a/pf/internal/muxer/muxer.go +++ b/pf/internal/muxer/muxer.go @@ -133,6 +133,18 @@ func (m *ProviderShim) extend(provider shim.Provider) ([]string, []string) { return conflictingResources, conflictingDataSources } +func (m *ProviderShim) DetailedSchemaDump() []byte { + for _, p := range m.MuxedProviders { + defer func() { + if r := recover(); r != nil { + contract.IgnoreError(fmt.Errorf("DetailedSchemaDump failed: %v", r)) + } + }() + return p.DetailedSchemaDump() + } + return nil +} + func newProviderShim(provider shim.Provider) ProviderShim { return ProviderShim{ simpleSchemaProvider: simpleSchemaProvider{ @@ -253,4 +265,8 @@ func (p *simpleSchemaProvider) DataSourcesMap() shim.ResourceMap { return p.dataSources } +func (p *simpleSchemaProvider) DetailedSchemaDump() []byte { + panic("Unsupported") +} + var _ shim.Provider = (*simpleSchemaProvider)(nil) diff --git a/pf/internal/schemashim/provider.go b/pf/internal/schemashim/provider.go index 27d8e3261..ecdee3e9b 100644 --- a/pf/internal/schemashim/provider.go +++ b/pf/internal/schemashim/provider.go @@ -70,6 +70,10 @@ func (p *SchemaOnlyProvider) Config(ctx context.Context) (tftypes.Object, error) return schema.Type().TerraformType(ctx).(tftypes.Object), nil } +func (p *SchemaOnlyProvider) DetailedSchemaDump() []byte { + panic("Unsupported") +} + var _ shim.Provider = (*SchemaOnlyProvider)(nil) func (p *SchemaOnlyProvider) Schema() shim.SchemaMap { diff --git a/pf/proto/protov6.go b/pf/proto/protov6.go index 49edf4cff..04c2e29c4 100644 --- a/pf/proto/protov6.go +++ b/pf/proto/protov6.go @@ -122,3 +122,7 @@ func (p Provider) DataSourcesMap() shim.ResourceMap { } return resourceMap(v.DataSourceSchemas) } + +func (p Provider) DetailedSchemaDump() []byte { + panic("Unsupported") +} diff --git a/pf/tfbridge/main.go b/pf/tfbridge/main.go index eda13d179..f9495918b 100644 --- a/pf/tfbridge/main.go +++ b/pf/tfbridge/main.go @@ -38,6 +38,7 @@ import ( // // info.P must be constructed with ShimProvider or ShimProviderWithContext. func Main(ctx context.Context, pkg string, prov tfbridge.ProviderInfo, meta ProviderMetadata) { + handleGetSchemaFlag(prov) handleFlags(ctx, prov.Version, func() (*tfbridge.MarshallableProviderInfo, error) { pp, err := newProviderWithContext(ctx, prov, meta) @@ -54,6 +55,22 @@ func Main(ctx context.Context, pkg string, prov tfbridge.ProviderInfo, meta Prov } } +func handleGetSchemaFlag(prov tfbridge.ProviderInfo) { + flags := flag.NewFlagSet("get-schema-flags", flag.ContinueOnError) + + dumpSchema := flags.Bool("get-schema", false, "dump provider schema as JSON to stdout") + + flags.SetOutput(io.Discard) + + err := flags.Parse(os.Args[1:]) + contract.IgnoreError(err) + + if *dumpSchema { + fmt.Print(string(prov.P.DetailedSchemaDump())) + os.Exit(0) + } +} + func handleFlags( ctx context.Context, version string, getProviderInfo func() (*tfbridge.MarshallableProviderInfo, error), @@ -107,6 +124,7 @@ func MainWithMuxer(ctx context.Context, pkg string, info tfbridge.ProviderInfo, if len(info.MuxWith) > 0 { panic("mixin providers via tfbridge.ProviderInfo.MuxWith is currently not supported") } + handleGetSchemaFlag(info) handleFlags(ctx, info.Version, func() (*tfbridge.MarshallableProviderInfo, error) { info := info return tfbridge.MarshalProviderInfo(&info), nil diff --git a/pkg/tfbridge/main.go b/pkg/tfbridge/main.go index 78138c56a..1eca1d930 100644 --- a/pkg/tfbridge/main.go +++ b/pkg/tfbridge/main.go @@ -41,6 +41,7 @@ func Main(pkg string, version string, prov ProviderInfo, pulumiSchema []byte) { flags.SetOutput(io.Discard) dumpInfo := flags.Bool("get-provider-info", false, "dump provider info as JSON to stdout") + dumpSchema := flags.Bool("get-schema", false, "dump provider schema as JSON to stdout") providerVersion := flags.Bool("version", false, "get built provider version") err := flags.Parse(os.Args[1:]) @@ -62,6 +63,11 @@ func Main(pkg string, version string, prov ProviderInfo, pulumiSchema []byte) { os.Exit(0) } + if *dumpSchema { + fmt.Print(string(prov.P.DetailedSchemaDump())) + os.Exit(0) + } + if *providerVersion { fmt.Println(version) os.Exit(0) diff --git a/pkg/tfshim/schema/provider.go b/pkg/tfshim/schema/provider.go index 8a303c420..53b4b8127 100644 --- a/pkg/tfshim/schema/provider.go +++ b/pkg/tfshim/schema/provider.go @@ -123,3 +123,7 @@ func (ProviderShim) NewResourceConfig( func (ProviderShim) IsSet(ctx context.Context, v interface{}) ([]interface{}, bool) { panic("this provider is schema-only and does not support runtime operations") } + +func (ProviderShim) DetailedSchemaDump() []byte { + panic("this provider is schema-only and does not support runtime operations") +} diff --git a/pkg/tfshim/sdk-v1/provider.go b/pkg/tfshim/sdk-v1/provider.go index 3027f4870..dd983ea07 100644 --- a/pkg/tfshim/sdk-v1/provider.go +++ b/pkg/tfshim/sdk-v1/provider.go @@ -176,3 +176,7 @@ func (p v1Provider) IsSet(_ context.Context, v interface{}) ([]interface{}, bool } return nil, false } + +func (p v1Provider) DetailedSchemaDump() []byte { + panic("Unsupported") +} diff --git a/pkg/tfshim/sdk-v2/provider.go b/pkg/tfshim/sdk-v2/provider.go index 2eef34b72..553741db3 100644 --- a/pkg/tfshim/sdk-v2/provider.go +++ b/pkg/tfshim/sdk-v2/provider.go @@ -2,6 +2,7 @@ package sdkv2 import ( "context" + "encoding/json" "fmt" "os" @@ -11,6 +12,7 @@ import ( testing "github.com/mitchellh/go-testing-interface" shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" ) var _ = shim.Provider(v2Provider{}) @@ -222,3 +224,149 @@ func (p v2Provider) IsSet(_ context.Context, v interface{}) ([]interface{}, bool } return nil, false } + +type tfSchemaMarshaller struct { + Type schema.ValueType + ConfigMode schema.SchemaConfigMode + Required bool + Optional bool + Computed bool + ForceNew bool + DiffSuppressFuncDefined bool + DiffSuppressOnRefresh bool + Default interface{} + DefaultFuncDefined bool + Description string + InputDefault string + StateFuncDefined bool + Elem interface{} + MaxItems int + MinItems int + SetDefined bool + ComputedWhen []string + ConflictsWith []string + ExactlyOneOf []string + AtLeastOneOf []string + RequiredWith []string + Deprecated string + ValidateFuncDefined bool + ValidateDiagFuncDefined bool + Sensitive bool +} + +type tfResourceMarshaller struct { + Schema map[string]*tfSchemaMarshaller + SchemaVersion int + MigrateStateDefined bool + StateUpgradersLen int + CustomizeDiffDefined bool + DeprecationMessage string + Description string + UseJSONNumber bool + EnableLegacyTypeSystemApplyErrors bool + EnableLegacyTypeSystemPlanErrors bool +} + +type tfProvMarshaller struct { + Schema map[string]*tfSchemaMarshaller + ResourcesMap map[string]*tfResourceMarshaller + DataSourcesMap map[string]*tfResourceMarshaller + ConfigureFuncDefined bool + TerraformVersion string +} + +func convertTFSchemaOrResource(s interface{}) interface{} { + if s == nil { + return nil + } + switch s := s.(type) { + case *schema.Schema: + return convertTFSchema(s) + case *schema.Resource: + return convertTfResource(s) + } + panic(fmt.Sprintf("unexpected type %T", s)) +} + +func convertTFSchema(s *schema.Schema) *tfSchemaMarshaller { + return &tfSchemaMarshaller{ + Type: s.Type, + ConfigMode: s.ConfigMode, + Required: s.Required, + Optional: s.Optional, + Computed: s.Computed, + ForceNew: s.ForceNew, + DiffSuppressFuncDefined: s.DiffSuppressFunc != nil, + DiffSuppressOnRefresh: s.DiffSuppressOnRefresh, + Default: s.Default, + DefaultFuncDefined: s.DefaultFunc != nil, + Description: s.Description, + InputDefault: s.InputDefault, + StateFuncDefined: s.StateFunc != nil, + Elem: convertTFSchemaOrResource(s.Elem), + MaxItems: s.MaxItems, + MinItems: s.MinItems, + SetDefined: s.Set != nil, + ComputedWhen: s.ComputedWhen, //nolint:staticcheck + ConflictsWith: s.ConflictsWith, + ExactlyOneOf: s.ExactlyOneOf, + AtLeastOneOf: s.AtLeastOneOf, + RequiredWith: s.RequiredWith, + Deprecated: s.Deprecated, + ValidateFuncDefined: s.ValidateFunc != nil, + ValidateDiagFuncDefined: s.ValidateDiagFunc != nil, + Sensitive: s.Sensitive, + } +} + +func convertTfResource(r *schema.Resource) *tfResourceMarshaller { + tfResource := &tfResourceMarshaller{ + Schema: make(map[string]*tfSchemaMarshaller), + SchemaVersion: r.SchemaVersion, + MigrateStateDefined: r.MigrateState != nil, //nolint:staticcheck + StateUpgradersLen: len(r.StateUpgraders), + CustomizeDiffDefined: r.CustomizeDiff != nil, + DeprecationMessage: r.DeprecationMessage, + Description: r.Description, + UseJSONNumber: r.UseJSONNumber, + EnableLegacyTypeSystemApplyErrors: r.EnableLegacyTypeSystemApplyErrors, + EnableLegacyTypeSystemPlanErrors: r.EnableLegacyTypeSystemPlanErrors, + } + + for k, v := range r.Schema { + tfResource.Schema[k] = convertTFSchema(v) + } + + return tfResource +} + +func convertTfProv(p *schema.Provider) *tfProvMarshaller { + tfProv := &tfProvMarshaller{ + Schema: make(map[string]*tfSchemaMarshaller), + ResourcesMap: make(map[string]*tfResourceMarshaller), + DataSourcesMap: make(map[string]*tfResourceMarshaller), + ConfigureFuncDefined: p.ConfigureFunc != nil, //nolint:staticcheck + TerraformVersion: p.TerraformVersion, + } + + for k, v := range p.Schema { + tfProv.Schema[k] = convertTFSchema(v) + } + + for k, v := range p.ResourcesMap { + tfProv.ResourcesMap[k] = convertTfResource(v) + } + + for k, v := range p.DataSourcesMap { + tfProv.DataSourcesMap[k] = convertTfResource(v) + } + + return tfProv +} + +func (p v2Provider) DetailedSchemaDump() []byte { + prov := convertTfProv(p.tf) + sch, err := json.Marshal(prov) + contract.AssertNoErrorf(err, "failed to marshal schema") + return sch +} diff --git a/pkg/tfshim/sdk-v2/provider_test.go b/pkg/tfshim/sdk-v2/provider_test.go index 5f8216663..07d3f7642 100644 --- a/pkg/tfshim/sdk-v2/provider_test.go +++ b/pkg/tfshim/sdk-v2/provider_test.go @@ -1,12 +1,15 @@ package sdkv2 import ( + "bytes" "context" + "encoding/json" "testing" "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + "github.com/hexops/autogold/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -104,3 +107,209 @@ func TestProvider1UpgradeResourceState(t *testing.T) { }) } } + +func TestProviderDetailedSchemaDump(t *testing.T) { + prov := NewProvider(&schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "test_resource": { + Schema: map[string]*schema.Schema{ + "foo": {Type: schema.TypeString}, + "bar": {Type: schema.TypeInt}, + }, + }, + }, + DataSourcesMap: map[string]*schema.Resource{ + "test_data_source": { + Schema: map[string]*schema.Schema{ + "foo": {Type: schema.TypeString}, + "bar": {Type: schema.TypeInt}, + }, + }, + }, + Schema: map[string]*schema.Schema{ + "test_schema": {Type: schema.TypeString}, + }, + }) + + jsonArr := prov.DetailedSchemaDump() + var out bytes.Buffer + err := json.Indent(&out, jsonArr, "", " ") + require.NoError(t, err) + + autogold.Expect(`{ + "Schema": { + "test_schema": { + "Type": 4, + "ConfigMode": 0, + "Required": false, + "Optional": false, + "Computed": false, + "ForceNew": false, + "DiffSuppressFuncDefined": false, + "DiffSuppressOnRefresh": false, + "Default": null, + "DefaultFuncDefined": false, + "Description": "", + "InputDefault": "", + "StateFuncDefined": false, + "Elem": null, + "MaxItems": 0, + "MinItems": 0, + "SetDefined": false, + "ComputedWhen": null, + "ConflictsWith": null, + "ExactlyOneOf": null, + "AtLeastOneOf": null, + "RequiredWith": null, + "Deprecated": "", + "ValidateFuncDefined": false, + "ValidateDiagFuncDefined": false, + "Sensitive": false + } + }, + "ResourcesMap": { + "test_resource": { + "Schema": { + "bar": { + "Type": 2, + "ConfigMode": 0, + "Required": false, + "Optional": false, + "Computed": false, + "ForceNew": false, + "DiffSuppressFuncDefined": false, + "DiffSuppressOnRefresh": false, + "Default": null, + "DefaultFuncDefined": false, + "Description": "", + "InputDefault": "", + "StateFuncDefined": false, + "Elem": null, + "MaxItems": 0, + "MinItems": 0, + "SetDefined": false, + "ComputedWhen": null, + "ConflictsWith": null, + "ExactlyOneOf": null, + "AtLeastOneOf": null, + "RequiredWith": null, + "Deprecated": "", + "ValidateFuncDefined": false, + "ValidateDiagFuncDefined": false, + "Sensitive": false + }, + "foo": { + "Type": 4, + "ConfigMode": 0, + "Required": false, + "Optional": false, + "Computed": false, + "ForceNew": false, + "DiffSuppressFuncDefined": false, + "DiffSuppressOnRefresh": false, + "Default": null, + "DefaultFuncDefined": false, + "Description": "", + "InputDefault": "", + "StateFuncDefined": false, + "Elem": null, + "MaxItems": 0, + "MinItems": 0, + "SetDefined": false, + "ComputedWhen": null, + "ConflictsWith": null, + "ExactlyOneOf": null, + "AtLeastOneOf": null, + "RequiredWith": null, + "Deprecated": "", + "ValidateFuncDefined": false, + "ValidateDiagFuncDefined": false, + "Sensitive": false + } + }, + "SchemaVersion": 0, + "MigrateStateDefined": false, + "StateUpgradersLen": 0, + "CustomizeDiffDefined": false, + "DeprecationMessage": "", + "Description": "", + "UseJSONNumber": false, + "EnableLegacyTypeSystemApplyErrors": false, + "EnableLegacyTypeSystemPlanErrors": false + } + }, + "DataSourcesMap": { + "test_data_source": { + "Schema": { + "bar": { + "Type": 2, + "ConfigMode": 0, + "Required": false, + "Optional": false, + "Computed": false, + "ForceNew": false, + "DiffSuppressFuncDefined": false, + "DiffSuppressOnRefresh": false, + "Default": null, + "DefaultFuncDefined": false, + "Description": "", + "InputDefault": "", + "StateFuncDefined": false, + "Elem": null, + "MaxItems": 0, + "MinItems": 0, + "SetDefined": false, + "ComputedWhen": null, + "ConflictsWith": null, + "ExactlyOneOf": null, + "AtLeastOneOf": null, + "RequiredWith": null, + "Deprecated": "", + "ValidateFuncDefined": false, + "ValidateDiagFuncDefined": false, + "Sensitive": false + }, + "foo": { + "Type": 4, + "ConfigMode": 0, + "Required": false, + "Optional": false, + "Computed": false, + "ForceNew": false, + "DiffSuppressFuncDefined": false, + "DiffSuppressOnRefresh": false, + "Default": null, + "DefaultFuncDefined": false, + "Description": "", + "InputDefault": "", + "StateFuncDefined": false, + "Elem": null, + "MaxItems": 0, + "MinItems": 0, + "SetDefined": false, + "ComputedWhen": null, + "ConflictsWith": null, + "ExactlyOneOf": null, + "AtLeastOneOf": null, + "RequiredWith": null, + "Deprecated": "", + "ValidateFuncDefined": false, + "ValidateDiagFuncDefined": false, + "Sensitive": false + } + }, + "SchemaVersion": 0, + "MigrateStateDefined": false, + "StateUpgradersLen": 0, + "CustomizeDiffDefined": false, + "DeprecationMessage": "", + "Description": "", + "UseJSONNumber": false, + "EnableLegacyTypeSystemApplyErrors": false, + "EnableLegacyTypeSystemPlanErrors": false + } + }, + "ConfigureFuncDefined": false, + "TerraformVersion": "" +}`).Equal(t, out.String()) +} diff --git a/pkg/tfshim/shim.go b/pkg/tfshim/shim.go index fc8fed9c4..35e03a351 100644 --- a/pkg/tfshim/shim.go +++ b/pkg/tfshim/shim.go @@ -244,6 +244,7 @@ type Provider interface { // Checks if a value is representing a Set, and unpacks its elements on success. IsSet(ctx context.Context, v interface{}) ([]interface{}, bool) + DetailedSchemaDump() []byte } type TimeoutOptions struct { diff --git a/pkg/tfshim/tfplugin5/provider.go b/pkg/tfshim/tfplugin5/provider.go index a462f64a3..bf1613802 100644 --- a/pkg/tfshim/tfplugin5/provider.go +++ b/pkg/tfshim/tfplugin5/provider.go @@ -594,3 +594,7 @@ func (p *provider) IsSet(ctx context.Context, v interface{}) ([]interface{}, boo } return result, true } + +func (p *provider) DetailedSchemaDump() []byte { + panic("Unsupported") +} diff --git a/pkg/tfshim/util/filter.go b/pkg/tfshim/util/filter.go index ea583ab07..ca16e6e5c 100644 --- a/pkg/tfshim/util/filter.go +++ b/pkg/tfshim/util/filter.go @@ -141,6 +141,10 @@ func (p *FilteringProvider) IsSet(ctx context.Context, v interface{}) ([]interfa return p.Provider.IsSet(ctx, v) } +func (p *FilteringProvider) DetailedSchemaDump() []byte { + panic("Unsupported") +} + type filteringMap struct { inner shim.ResourceMap tokenFilter func(string) bool diff --git a/pkg/tfshim/util/util.go b/pkg/tfshim/util/util.go index d55dea758..52eeb361c 100644 --- a/pkg/tfshim/util/util.go +++ b/pkg/tfshim/util/util.go @@ -103,3 +103,7 @@ func (UnimplementedProvider) NewResourceConfig( func (UnimplementedProvider) IsSet(ctx context.Context, v interface{}) ([]interface{}, bool) { panic("unimplemented") } + +func (UnimplementedProvider) DetailedSchemaDump() []byte { + panic("unimplemented") +} diff --git a/scripts/tf_schema_splitter.py b/scripts/tf_schema_splitter.py new file mode 100644 index 000000000..f8c17c1b2 --- /dev/null +++ b/scripts/tf_schema_splitter.py @@ -0,0 +1,126 @@ +import argparse +import csv +from dataclasses import dataclass, field +import enum +import json +from typing import Any + + +class Element(enum.Enum): + RESOURCE_ELEMENT = "resource_element" + ATTRIBUTE_ELEMENT = "attribute_element" + + +@dataclass +class Results: + provider_name: str + resources: list[dict[str, Any]] = field(default_factory=list) + attributes: list[dict[str, Any]] = field(default_factory=list) + + def add_resource(self, name: str, resource: dict[str, Any]): + resource["Provider"] = self.provider_name + resource["Name"] = name + self.resources.append(resource) + + def add_attribute(self, name: str, attribute: dict[str, Any]): + attribute["Provider"] = self.provider_name + attribute["Name"] = name + self.attributes.append(attribute) + + def parse_res_or_attr(self, elem_schema: dict[str, Any], name: str) -> Element: + if "Schema" in elem_schema: + # this is a resource + self.add_resource(name, self.parse_resource(elem_schema, name)) + return Element.RESOURCE_ELEMENT + self.add_attribute(name, self.parse_attribute(elem_schema, name)) + return Element.ATTRIBUTE_ELEMENT + + def parse_attribute( + self, attribute_schema: dict[str, Any], name: str + ) -> dict[str, Any]: + res: dict[str, Any] = {} + for key, value in attribute_schema.items(): + if key == "Elem": + continue + res[key] = value + + elem = attribute_schema.get("Elem") + if elem: + elem_name = f"{name}.elem" + key = self.parse_res_or_attr(elem, elem_name) + res[key.value] = elem_name + + return res + + def parse_resource( + self, resource_schema: dict[str, Any], name: str + ) -> dict[str, Any]: + res: dict[str, Any] = {} + for key, value in resource_schema.items(): + if key == "Schema": + continue + res[key] = value + + schema_list: list[str] = [] + attributes: dict[str, Any] = resource_schema.get("Schema", {}) + for attribute_name, attribute_schema in attributes.items(): + full_name = f"{name}.{attribute_name}" + self.add_attribute( + full_name, self.parse_attribute(attribute_schema, full_name) + ) + schema_list.append(full_name) + res["Schema"] = schema_list + return res + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--tf-schema-file", type=str, required=True) + ap.add_argument("--provider-name", type=str, required=True) + args = ap.parse_args() + schema_file = args.tf_schema_file + provider_name = args.provider_name + + with open(schema_file, encoding="utf-8") as f: + provider_schema = json.load(f) + + res = Results(provider_name=provider_name) + + for resource_name, resource_schema in provider_schema["ResourcesMap"].items(): + res.add_resource( + resource_name, res.parse_resource(resource_schema, resource_name) + ) + + for datasource_name, datasource_schema in provider_schema["DataSourcesMap"].items(): + res.add_resource( + datasource_name, res.parse_resource(datasource_schema, datasource_name) + ) + + for attribute_name, attribute_schema in provider_schema["Schema"].items(): + res.add_attribute( + attribute_name, res.parse_attribute(attribute_schema, attribute_name) + ) + + res_keys = set(res.resources[0].keys()) + res_keys.update([Element.ATTRIBUTE_ELEMENT.value, Element.RESOURCE_ELEMENT.value]) + res_keys.remove("Name") + res_keys = ["Name"] + list(res_keys) + + attr_keys = set(res.attributes[0].keys()) + attr_keys.update([Element.ATTRIBUTE_ELEMENT.value, Element.RESOURCE_ELEMENT.value]) + attr_keys.remove("Name") + attr_keys = ["Name"] + list(attr_keys) + + with open(f"{provider_name}_resources.csv", "w", encoding="utf-8") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=res_keys) + writer.writeheader() + writer.writerows(res.resources) + + with open(f"{provider_name}_attributes.csv", "w", encoding="utf-8") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=attr_keys) + writer.writeheader() + writer.writerows(res.attributes) + + +if __name__ == "__main__": + main()