Skip to content

Commit

Permalink
Merge pull request #4167 from hashicorp/msgraph-importer-refactor
Browse files Browse the repository at this point in the history
`tools/importer-msgraph-metadata`: refactor to generate JSON definitions using `data-api-sdk`
  • Loading branch information
manicminer authored Jul 8, 2024
2 parents c9d8ed5 + f52ce24 commit dbffd4d
Show file tree
Hide file tree
Showing 44 changed files with 1,134 additions and 1,737 deletions.
5 changes: 5 additions & 0 deletions api-definitions/microsoft-graph/metadata.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"dataSource": "MicrosoftGraph",
"sourceInformation": "microsoftgraph/msgraph-metadata",
"gitRevision": "7b1feb7cf42dd1f08941b349209c1cf8602f47c8"
}
28 changes: 14 additions & 14 deletions config/microsoft-graph.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -13,70 +13,70 @@ service "appRoleAssignments" {

service "applications" {
name = "Applications"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "applicationTemplates" {
name = "ApplicationTemplates"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "directory" {
name = "Directory"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "directoryObjects" {
name = "DirectoryObjects"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "domains" {
name = "Domains"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "groups" {
name = "Groups"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "identity" {
name = "Identity"
available = ["v1.0"]
available = ["stable"]
}

service "invitations" {
name = "Invitations"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "me" {
name = "Me"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "oauth2PermissionGrants" {
name = "OAuth2PermissionGrants"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "policies" {
name = "Policies"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "roleManagement" {
name = "RoleManagement"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "servicePrincipals" {
name = "ServicePrincipals"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}

service "users" {
name = "Users"
available = ["v1.0", "beta"]
available = ["stable", "beta"]
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,7 @@ type OperationsStage struct {
// APIVersion specifies the APIVersion within the Service where the Operations exist.
APIVersion string

// CommonTypesForThisAPIVersion specifies a map of API Version (key) to CommonTypes (value)
// which defines the known Common Types for this APIVersion.
// CommonTypesForThisAPIVersion specifies the known CommonTypes for this APIVersion.
CommonTypesForThisAPIVersion sdkModels.CommonTypes

// Constants specifies the map of Constant Name (key) to SDKConstant (value) which should be
Expand All @@ -48,8 +47,8 @@ func (g OperationsStage) Generate(input *helpers.FileSystem, logger hclog.Logger
Constants: g.Constants,
Models: g.Models,
ResourceIds: map[string]sdkModels.ResourceID{},
CommonTypeConstants: make(map[string]sdkModels.SDKConstant),
CommonTypeModels: make(map[string]sdkModels.SDKModel),
CommonTypeConstants: g.CommonTypesForThisAPIVersion.Constants,
CommonTypeModels: g.CommonTypesForThisAPIVersion.Models,
}

operationDetails := g.Operations[operationName]
Expand Down
7 changes: 6 additions & 1 deletion tools/data-api-repository/repository/save_common_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,13 @@ func (r *repositoryImpl) SaveCommonTypes(opts SaveCommonTypesOptions) error {
}
}

dataDirectory, err := r.defaultDirectoryForSourceDataOrigin(opts.SourceDataOrigin)
if err != nil {
return fmt.Errorf("determining the default data directory for the Source Data Origin %q: %+v", opts.SourceDataOrigin, err)
}

r.logger.Debug("Persisting files to disk..")
if err := helpers.PersistFileSystem(r.workingDirectory, fs, r.logger); err != nil {
if err := helpers.PersistFileSystem(*dataDirectory, fs, r.logger); err != nil {
return fmt.Errorf("persisting files: %+v", err)
}

Expand Down
21 changes: 15 additions & 6 deletions tools/data-api-repository/repository/save_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ type SaveServiceOptions struct {
// This is only present when SourceDataType is ResourceManagerSourceDataType.
ResourceProvider *string

// CommonTypes specifies a map of API Version (key) to CommonTypes (value), which defines the known Common Types.
CommonTypes map[string]sdkModels.CommonTypes

// Service specifies details about this Service, including the available APIVersions.
Service sdkModels.Service

Expand Down Expand Up @@ -52,6 +55,11 @@ func (r *repositoryImpl) SaveService(opts SaveServiceOptions) error {
APIVersion: apiVersionDetails,
})

commonTypesForThisApiVersion := sdkModels.CommonTypes{}
if opts.CommonTypes != nil {
commonTypesForThisApiVersion, _ = opts.CommonTypes[apiVersion]
}

for apiResourceName, apiResourceDetails := range apiVersionDetails.Resources {
// Output the API Definitions for this APIResource

Expand All @@ -70,11 +78,12 @@ func (r *repositoryImpl) SaveService(opts SaveServiceOptions) error {
})

items = append(items, stages.OperationsStage{
APIVersion: apiVersion,
APIResource: apiResourceName,
Constants: apiResourceDetails.Constants,
Models: apiResourceDetails.Models,
Operations: apiResourceDetails.Operations,
APIVersion: apiVersion,
APIResource: apiResourceName,
CommonTypesForThisAPIVersion: commonTypesForThisApiVersion,
Constants: apiResourceDetails.Constants,
Models: apiResourceDetails.Models,
Operations: apiResourceDetails.Operations,
})

items = append(items, stages.ResourceIDsStage{
Expand Down Expand Up @@ -129,7 +138,7 @@ func (r *repositoryImpl) SaveService(opts SaveServiceOptions) error {

// then populate the source data information into the metadata file
if err := r.writeSourceDataInformation(opts.SourceDataOrigin, opts.SourceCommitSHA); err != nil {
return fmt.Errorf("populating the Source Data Information into %q: %+v", r.workingDirectory, err)
return fmt.Errorf("populating the Source Data Information into %q: %+v", *serviceDirectory, err)
}

return nil
Expand Down
3 changes: 3 additions & 0 deletions tools/data-api-sdk/v1/models/sdk_object_definition.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ type SDKObjectDefinition struct {
// is set to `ReferenceSDKObjectDefinitionType`.
ReferenceName *string `json:"referenceName,omitempty"`

// ReferenceNameIsCommonType specifies whether the referenced Constant or Model is a common type
ReferenceNameIsCommonType *bool `json:"referenceNameIsCommonType,omitempty"`

// Type specifies the Type that represents this SDK Object Definition. This can be either a
// Simple type (e.g. a String/Integer), a Reference to a Constant/Model or a more complex object
// (e.g. a CommonSchema type [such as a SystemAssignedIdentity] - or a List/Dictionary).
Expand Down
8 changes: 8 additions & 0 deletions tools/generator-go-sdk/internal/generator/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ func copyrightLinesForSource(input models.SourceDataOrigin) (*string, error) {
return &out, nil
}

if input == models.MicrosoftGraphMetaDataSourceDataOrigin {
out := `
// Copyright (c) HashiCorp Inc. All rights reserved.
// Licensed under the MIT License. See NOTICE.txt in the project root for license information.
`
return &out, nil
}

if input == models.AzureRestAPISpecsSourceDataOrigin {
out := `
// Copyright (c) Microsoft Corporation. All rights reserved.
Expand Down
5 changes: 2 additions & 3 deletions tools/importer-msgraph-metadata/GNUmakefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ import: build
./importer-msgraph-metadata import -services=$(SERVICES); \
fi

import-all-clean: build
rm -r ../../data/Pandora.Definitions.MicrosoftGraph.*/[A-Z]*/
./importer-msgraph-metadata import
lint:
golangci-lint run ./...

test: build
go test -v ./...
Expand Down
42 changes: 42 additions & 0 deletions tools/importer-msgraph-metadata/components/normalize/duplicates.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package normalize

import (
"regexp"
"strings"
)

func DeDuplicate(name string) string {
nameSpaced := regexp.MustCompile("([A-Z])").ReplaceAllString(name, " $1")
nameParts := strings.Split(strings.TrimSpace(nameSpaced), " ")
newParts := make([]string, 0)
for i := 0; i < len(nameParts); i++ {
if i > 0 && strings.EqualFold(nameParts[i], nameParts[i-1]) {
continue
}
newParts = append(newParts, nameParts[i])
}
return strings.Join(newParts, "")
}

func DeDuplicateName(name string) string {
words := make([]string, 0)
i := 0
for j, c := range name {
char := string(c)
if j > 0 && strings.ToUpper(char) == char {
i++
}
if len(words) <= i {
words = append(words, "")
}
words[i] = words[i] + char
}
out := ""
for k, word := range words {
if k > 0 && strings.EqualFold(words[k-1], word) {
continue
}
out = out + word
}
return out
}
124 changes: 124 additions & 0 deletions tools/importer-msgraph-metadata/components/normalize/normalize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package normalize

import (
"fmt"
"regexp"
"strings"

"golang.org/x/text/cases"
"golang.org/x/text/language"
)

func Singularize(name string) string {
if len(name) >= 3 && name[len(name)-3:] == "ies" {
return fmt.Sprintf("%sy", name[:len(name)-3])
}
if len(name) >= 3 && name[len(name)-3:] == "ses" {
return name[:len(name)-2]
}
if len(name) >= 1 && name[len(name)-1:] == "s" {
return name[:len(name)-1]
}
return name
}

func Pluralize(name string) string {
if name == "" {
return ""
}
ret := fmt.Sprintf("%ss", name)
if strings.EqualFold(name, "me") {
return name
}
if len(name) == 0 {
return ret
}
if strings.EqualFold(name[len(name)-2:], "ay") || strings.EqualFold(name[len(name)-2:], "ey") {
return fmt.Sprintf("%ss", name)
}
if strings.EqualFold(name[len(name)-1:], "y") {
return fmt.Sprintf("%sies", name[:len(name)-1])
}
if strings.EqualFold(name[len(name)-1:], "s") {
return name
}
if len(name) < 2 {
return ret
}
if strings.EqualFold(name[len(name)-2:], "Of") {
return name
}
return ret
}

func CleanName(name string) string {
// Trim a "microsoft.graph" prefix from type names
name = strings.TrimPrefix(name, "microsoft.graph")

// Replace all periods with spaces to allow for title casing
name = strings.ReplaceAll(name, ".", " ")

// Convert name to title case
name = cases.Title(language.AmericanEnglish, cases.NoLower).String(name)

// Remove all non-alphanumeric characters, except for any leading underscores
leading := regexp.MustCompile(`^_+`).FindString(name)
trailing := regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(name[len(leading):], "")
name = leading + trailing

// Odata should be OData
name = regexp.MustCompile("^Odata").ReplaceAllString(name, "OData")

// Innererror should be InnerError
name = regexp.MustCompile("^Innererror").ReplaceAllString(name, "InnerError")

// known issue where CloudPC appears with inconsistent casing
name = regexp.MustCompile("(?i)CloudPc").ReplaceAllString(name, "CloudPC")

return name
}

func CleanNameCamel(name string) string {
name = CleanName(name)
return strings.ToLower(name[0:1]) + name[1:]
}

type operationVerbs []string

func (ov operationVerbs) Match(operation string) (*string, bool) {
for _, v := range ov {
if regexp.MustCompile(fmt.Sprintf("^%s$", v)).MatchString(operation) {
return &v, true
}
if regexp.MustCompile(fmt.Sprintf("^%s[A-Z]", v)).MatchString(operation) {
return &v, true
}
}
return nil, false
}

var Verbs = operationVerbs{
"Acquire",
"Add",
"Assign",
"Check",
"Discover",
"Get",
"Instantiate",
"Parse",
"Pause",
"Provision",
"Remove",
"Renew",
"Restart",
"Restore",
"Set",
"Start",
"Stop",
"Unset",
"Update",
"Validate",
}
Loading

0 comments on commit dbffd4d

Please sign in to comment.