Merge pull request #2907 from hashicorp/bugfix/msgraph-importer-fixups

Graph Importer: various fixes and pausing the workflow
hashicorp · Aug 7, 2023 · cd3d7c6 · cd3d7c6
2 parents 5d96be0 + d75fbd8
commit cd3d7c6
Show file tree

Hide file tree

Showing 9 changed files with 261 additions and 160 deletions.
diff --git a/.github/workflows/automation-msgraph-metadata-importer.yaml b/.github/workflows/automation-msgraph-metadata-importer.yaml
@@ -1,13 +1,14 @@
 name: Regenerate API Data with New Microsoft Graph Changes after PR Merge
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - '.github/workflows/**'
-      - 'config/**'
-      - 'submodules/msgraph-metadata'
-      - 'tools/importer-msgraph/**'
+
+# manicminer: disabling until model consistency issue is resolved
+on: workflow_dispatch
+#  push:
+#    branches:
+#      - main
+#    paths:
+#      - '.github/workflows/**'
+#      - 'submodules/msgraph-metadata'
+#      - 'tools/importer-msgraph/**'
 
 
 concurrency:

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -11,3 +11,12 @@ This codebase contains a modified portion of code from Microsoft which can be ob
     * https://www.github.com/Azure/azure-rest-api-specs
 * LICENSE:
     * https://github.com/Azure/azure-rest-api-specs/blob/main/LICENSE
+
+--------------------------------------------------------------------------------------
+
+This codebase contains a modified portion of code from Microsoft which can be obtained at:
+
+* SOURCE:
+    * https://github.com/microsoftgraph/msgraph-metadata
+* LICENSE:
+    * https://github.com/microsoftgraph/msgraph-metadata/blob/master/LICENSE
diff --git a/tools/importer-msgraph-metadata/pipeline/normalize.go b/tools/importer-msgraph-metadata/pipeline/normalize.go
@@ -71,7 +71,7 @@ func deDuplicate(name string) string {
 }
 
 func cleanName(name string) string {
-	name = strings.Title(strings.TrimPrefix(name, "microsoft.graph."))
+	name = strings.Title(strings.TrimPrefix(name, "microsoft.graph"))
 	name = regexp.MustCompile("[.]").ReplaceAllString(name, "_")
 	name = regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(name, "")
 	name = regexp.MustCompile("^Odata").ReplaceAllString(name, "OData")
@@ -156,6 +156,7 @@ var verbs = operationVerbs{
 	"Check",
 	"Discover",
 	"Get",
+	"Instantiate",
 	"Parse",
 	"Pause",
 	"Provision",

diff --git a/tools/importer-msgraph-metadata/pipeline/parser_models.go b/tools/importer-msgraph-metadata/pipeline/parser_models.go
@@ -21,6 +21,8 @@ import (
    Schema leads to SchemaRefs and Schemas
 */
 
+const refPrefix = "#/components/schemas/"
+
 type Models map[string]*Model
 
 // Found returns true when the provided modelName was found in the Models map
@@ -80,17 +82,19 @@ func (m Models) Merge(m2 Models) {
 type Model struct {
 	Fields map[string]*ModelField
 	Common bool
+	Prefix string
 }
 
 type ModelField struct {
-	Title       string
-	Type        *DataType
-	Description string
-	Default     interface{}
-	Enum        []string
-	ItemType    *DataType
-	ModelName   *string
-	JsonField   string
+	Title        string
+	Type         *DataType
+	Description  string
+	Default      interface{}
+	Enum         []string
+	ItemType     *DataType
+	ConstantName *string
+	ModelName    *string
+	JsonField    string
 }
 
 // CSType returns a string containing the C# type name for the ModelField, either describing it as a literal type, a
@@ -113,8 +117,8 @@ func (f ModelField) CSType() *string {
 			return pointerTo(fmt.Sprintf("List<%sModel>", *f.ModelName))
 		}
 
-		if f.Title != "" && len(f.Enum) > 0 {
-			return pointerTo(fmt.Sprintf("List<%sConstant>", f.Title))
+		if f.ConstantName != nil {
+			return pointerTo(fmt.Sprintf("List<%sConstant>", *f.ConstantName))
 		}
 
 		if f.ItemType != nil {
@@ -126,8 +130,8 @@ func (f ModelField) CSType() *string {
 		return nil
 
 	case DataTypeString:
-		if f.Title != "" && len(f.Enum) > 0 {
-			return pointerTo(fmt.Sprintf("%sConstant", f.Title))
+		if f.ConstantName != nil {
+			return pointerTo(fmt.Sprintf("%sConstant", *f.ConstantName))
 		}
 	}
 
@@ -282,10 +286,11 @@ func parseCommonModels(schemas openapi3.Schemas) (models Models, err error) {
 	models = make(Models)
 	for modelName, schemaRef := range schemas {
 		name := cleanName(modelName)
-		if schema := parseSchemaRef(schemaRef); schema != nil {
-			var f flattenedSchema
-			f, _ = flattenSchema(schema, nil)
-			models = parseSchemas(f, name, models, true)
+		if schemaRef.Value != nil {
+			var f *flattenedSchema
+			if f, _ = flattenSchemaRef(schemaRef, nil); f != nil {
+				models = parseSchemas(*f, name, models, true)
+			}
 		}
 	}
 
@@ -294,22 +299,38 @@ func parseCommonModels(schemas openapi3.Schemas) (models Models, err error) {
 
 type flattenedSchema struct {
 	Schemas openapi3.Schemas
+	Prefix  string
 	Title   string
 	Type    string
 	Format  string
 	Enum    []interface{}
 }
 
-// flattenSchema attempts to recursively parse and flatten the provided *openapi3.Schema and returns a flattenedSchema
+// flattenSchemaRef attempts to recursively parse and flatten the provided *openapi3.Schema and returns a flattenedSchema
 // which is much more convenient to inspect for types. The returned map[string]bool is used when recursing to track
 // Refs which have been observed in order to avoid infinite recursion, and is usually not interesting to the caller.
-func flattenSchema(schema *openapi3.Schema, seenRefs map[string]bool) (flattenedSchema, map[string]bool) {
+func flattenSchemaRef(schemaRef *openapi3.SchemaRef, seenRefs map[string]bool) (*flattenedSchema, map[string]bool) {
 	if seenRefs == nil {
 		seenRefs = make(map[string]bool)
 	}
 
-	schemas := make(openapi3.Schemas, 0)
+	if schemaRef.Value == nil {
+		return nil, seenRefs
+	}
+
+	prefix := ""
 	title := ""
+	titleFromRef := false
+	if strings.HasPrefix(schemaRef.Ref, refPrefix) {
+		ref := schemaRef.Ref[len(refPrefix):]
+		if i := strings.LastIndex(ref, "."); i > 0 {
+			prefix = strings.Title(cleanName(ref[0:i]))
+		}
+		title = strings.Title(cleanName(ref))
+		titleFromRef = true
+	}
+	schema := schemaRef.Value
+	schemas := make(openapi3.Schemas, 0)
 	typ := ""
 	format := ""
 	enum := make([]interface{}, 0)
@@ -322,13 +343,20 @@ func flattenSchema(schema *openapi3.Schema, seenRefs map[string]bool) (flattened
 				}
 			}
 			seenRefs[r.Ref] = true
+			if title == "" && strings.HasPrefix(r.Ref, refPrefix) {
+				ref := r.Ref[len(refPrefix):]
+				if i := strings.LastIndex(ref, "."); i > 0 {
+					prefix = strings.Title(cleanName(ref[0:i]))
+				}
+				title = strings.Title(cleanName(ref))
+			}
 		}
 
-		if s := parseSchemaRef(r); s != nil {
-			var result flattenedSchema
-			result, seenRefs = flattenSchema(s, seenRefs)
-			if result.Title != "" {
-				title = result.Title
+		if r.Value != nil {
+			var result *flattenedSchema
+			result, seenRefs = flattenSchemaRef(r, seenRefs)
+			if title == "" && result.Title != "" {
+				title = strings.Title(cleanName(result.Title))
 			}
 			if result.Type != "" {
 				typ = result.Type
@@ -353,12 +381,21 @@ func flattenSchema(schema *openapi3.Schema, seenRefs map[string]bool) (flattened
 						}
 					}
 					seenRefs[r.Ref] = true
+					if !titleFromRef && strings.HasPrefix(r.Ref, refPrefix) {
+						ref := r.Ref[len(refPrefix):]
+						if i := strings.LastIndex(ref, "."); i > 0 {
+							prefix = strings.Title(cleanName(ref[0:i]))
+						}
+						title = strings.Title(cleanName(ref))
+						titleFromRef = true
+					}
 				}
-				if s := parseSchemaRef(r); s != nil {
-					var result flattenedSchema
-					result, seenRefs = flattenSchema(s, seenRefs)
-					if result.Title != "" {
-						title = result.Title
+
+				if r.Value != nil {
+					var result *flattenedSchema
+					result, seenRefs = flattenSchemaRef(r, seenRefs)
+					if !titleFromRef && result.Title != "" {
+						title = strings.Title(cleanName(result.Title))
 					}
 					if typ == "" && result.Type != "" {
 						typ = result.Type
@@ -385,13 +422,21 @@ func flattenSchema(schema *openapi3.Schema, seenRefs map[string]bool) (flattened
 						}
 					}
 					seenRefs[r.Ref] = true
+					if !titleFromRef && strings.HasPrefix(r.Ref, refPrefix) {
+						ref := r.Ref[len(refPrefix):]
+						if i := strings.LastIndex(ref, "."); i > 0 {
+							prefix = strings.Title(cleanName(ref[0:i]))
+						}
+						title = strings.Title(cleanName(ref))
+						titleFromRef = true
+					}
 				}
 
-				if s := parseSchemaRef(r); s != nil {
-					var result flattenedSchema
-					result, seenRefs = flattenSchema(s, seenRefs)
-					if result.Title != "" {
-						title = result.Title
+				if r.Value != nil {
+					var result *flattenedSchema
+					result, seenRefs = flattenSchemaRef(r, seenRefs)
+					if !titleFromRef && result.Title != "" {
+						title = strings.Title(cleanName(result.Title))
 					}
 					if typ == "" && result.Type != "" {
 						typ = result.Type
@@ -410,8 +455,9 @@ func flattenSchema(schema *openapi3.Schema, seenRefs map[string]bool) (flattened
 		}
 	}
 
+	// TODO: may need to prefer innermost title
 	if schema.Title != "" {
-		title = schema.Title
+		title = strings.Title(cleanName(schema.Title))
 	}
 
 	// prefer the innermost type
@@ -438,79 +484,81 @@ func flattenSchema(schema *openapi3.Schema, seenRefs map[string]bool) (flattened
 		schemas = nil
 	}
 
-	return flattenedSchema{
+	return &flattenedSchema{
 		Schemas: schemas,
+		Prefix:  prefix,
 		Title:   title,
 		Type:    typ,
 		Format:  format,
 		Enum:    enum,
 	}, seenRefs
 }
 
-func parseSchemaRef(schemaRef *openapi3.SchemaRef) *openapi3.Schema {
-	if schemaRef.Value != nil {
-		return schemaRef.Value
-	}
-	return nil
-}
-
 // parseSchemas inspects the provided flattenedSchema to parse out the fields for the provided modelName, optionally
 // marking it as a common model. The provided Models (map[string]Model) is mutated to append the new model and its fields.
+// Fields having the type of another model are parsed recursively to extract all known models that may not be directly
+// referenced in the root schema.
 func parseSchemas(input flattenedSchema, modelName string, models Models, common bool) Models {
 	if _, ok := models[modelName]; ok {
 		return models
 	}
+
 	model := Model{
 		Fields: make(map[string]*ModelField),
 		Common: common,
+		Prefix: input.Prefix,
 	}
+
+	// Add to models map before descending, to prevent recursion
 	models[modelName] = &model
 
 	for jsonField, schemaRef := range input.Schemas {
-		schema := parseSchemaRef(schemaRef)
-		result, _ := flattenSchema(schema, nil)
-		title := ""
-
-		if result.Title != "" {
-			title = strings.Title(result.Title)
-		} else {
-			title = strings.Title(jsonField)
-		}
+		if schema := schemaRef.Value; schema != nil {
+			field := ModelField{
+				Title:       strings.Title(jsonField),
+				Description: schema.Description,
+				Default:     schema.Default,
+				Enum:        parseEnum(schema.Enum),
+				JsonField:   jsonField,
+			}
 
-		field := ModelField{
-			Title:       title,
-			Description: schema.Description,
-			Default:     schema.Default,
-			Enum:        parseEnum(schema.Enum),
-			JsonField:   jsonField,
-		}
+			if field.Title == "" {
+				continue
+			}
 
-		if len(field.Enum) == 0 && len(result.Enum) > 0 {
-			field.Enum = parseEnum(result.Enum)
-		}
+			result, _ := flattenSchemaRef(schemaRef, nil)
 
-		if result.Schemas != nil {
-			if _, ok := models[title]; !ok {
-				models = parseSchemas(result, title, models, common)
+			if result != nil && len(result.Enum) > 0 && len(field.Enum) == 0 {
+				field.Enum = parseEnum(result.Enum)
 			}
-			field.ModelName = &title
-		}
 
-		if schema.Items != nil && schema.Items.Value != nil && schema.Items.Value.Type != "" {
-			field.ItemType = fieldType(schema.Items.Value.Type, schema.Items.Value.Format, field.ModelName != nil)
-		}
+			if result != nil && result.Title != "" && result.Schemas != nil {
+				if _, ok := models[result.Title]; !ok {
+					models = parseSchemas(*result, result.Title, models, common)
+				}
+				field.ModelName = &result.Title
+			}
 
-		if schema.Type == "" && schema.Format == "" && (result.Type != "" || result.Format != "") {
-			field.Type = fieldType(result.Type, result.Format, field.ModelName != nil)
-		} else {
-			field.Type = fieldType(schema.Type, schema.Format, field.ModelName != nil)
-		}
+			if schema.Items != nil && schema.Items.Value != nil && schema.Items.Value.Type != "" {
+				field.ItemType = fieldType(schema.Items.Value.Type, schema.Items.Value.Format, field.ModelName != nil)
+			}
 
-		if field.Type != nil && *field.Type == DataTypeArray && len(field.Enum) > 0 && (result.Type != "" || result.Format != "") {
-			field.ItemType = fieldType(result.Type, result.Format, field.ModelName != nil)
-		}
+			if result != nil && schema.Type == "" && schema.Format == "" && (result.Type != "" || result.Format != "") {
+				field.Type = fieldType(result.Type, result.Format, field.ModelName != nil)
+			} else {
+				field.Type = fieldType(schema.Type, schema.Format, field.ModelName != nil)
+			}
 
-		model.Fields[cleanName(jsonField)] = &field
+			if result != nil && field.Type != nil && *field.Type == DataTypeArray && len(field.Enum) > 0 && (result.Type != "" || result.Format != "") {
+				field.ItemType = fieldType(result.Type, result.Format, field.ModelName != nil)
+			}
+
+			if ((field.Type != nil && *field.Type == DataTypeString) || (field.ItemType != nil && *field.ItemType == DataTypeString)) && len(field.Enum) > 0 {
+				field.ConstantName = pointerTo(modelName + field.Title)
+			}
+
+			model.Fields[cleanName(jsonField)] = &field
+		}
 	}
 
 	return models