#539 Spark schema generation works with record relations

Update Spark Schema MDA generation to account for relations between
records. Updated the relation mda to include column and required fields.

Added record relation unit tests.

Spark Schema casting and to/from pojos work with records with relations.

Updated documentation to account for the new relation fields.

Implement validation for relations except one to M.

DRAFT_RELEASE_NOTES.md

@@ -13,7 +13,10 @@ Data access through [GraphQL](https://graphql.org/) has been deprecated and repl
 Spark and PySpark have been upgraded from version 3.5.2 to 3.5.4.
 
 ## Record Relation
-To enable nested data records, we have added a new relation feature to the record metamodel. This allows records to reference other records. For more details, refer to the [Record Relation Options].(https://boozallen.github.io/aissemble/aissemble/current-dev/record-metamodel.html#_record_relation_options)
+To enable nested data records, we have added a new relation feature to the record metamodel. This allows records to reference other records. For more details, refer to the [Record Relation Options](https://boozallen.github.io/aissemble/aissemble/current-dev/record-metamodel.html#_record_relation_options).
+Several features are still a work in progress:
+- Spark-based validation for records with a One to Many multiplicity. (POJO validation is available.)
+- PySpark schema generation for records with any multiplicity
 
 # Breaking Changes
 _Note: instructions for adapting to these changes are outlined in the upgrade instructions below._

build-parent/pom.xml

@@ -48,7 +48,7 @@
         <version.smallrye.graphql>2.9.1</version.smallrye.graphql>
         <version.smallrye.reactive.messaging>4.24.0</version.smallrye.reactive.messaging>
         <version.orphedomos.plugin>0.10.1</version.orphedomos.plugin>
-        <version.janino>3.0.8</version.janino>
+        <version.janino>3.1.12</version.janino>
         <version.jackson>2.15.0</version.jackson>
         <version.jclouds>2.6.0</version.jclouds>
         <version.maven.core>3.8.6</version.maven.core>

docs/modules/ROOT/pages/record-metamodel.adoc

@@ -247,13 +247,20 @@ namespacing (e.g., package in Java, namespace in XSD).
 | `relations/relation/documentation`
 | No
 | None
-| A description of the field.
+| A description of the relation.
 
 | `relations/relation/multiplicity`
 | No
 | One to Many (1-M)
 | Defines the multiplicity of the relation.  Options are ONE_TO_MANY (1-M), ONE_TO_ONE (1-1), and MANY_TO_ONE (M-1).
 
-|===
+| `relations/relation/required`
+| No
+| false
+| Setting `required` to `true` will mandate that the relation must be populated for a record to pass validation.
 
-\
+| `relations/relation/column`
+| No
+| None
+| The name of the storage field for data persistence.
+|===

extensions/extensions-data-delivery/extensions-data-delivery-spark/src/main/java/com/boozallen/aiops/data/delivery/spark/SparkSchema.java

@@ -127,6 +127,22 @@ protected void add(String name, DataType dataType, boolean nullable, String comm
         schema = schema.add(name, dataType, nullable, comment);
     }
 
+    /**
+     * Adds a field to the schema.
+     *
+     * @param name
+     *            the name of the field to add
+     * @param structType
+     *            the struct type of the field to add
+     * @param nullable
+     *            whether the field is nullable
+     * @param comment
+     *            a description of the field
+     */
+    protected void add(String name, StructType structType, boolean nullable, String comment) {
+        schema = schema.add(name, structType, nullable, comment);
+    }
+
     /**
      * Updates the data type of a field in the schema.
      * 

foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/BaseRecordRelationDecorator.java

@@ -2,19 +2,22 @@
 
 /*-
  * #%L
- * AIOps Foundation::AIOps MDA
+ * aiSSEMBLE::Foundation::MDA
  * %%
  * Copyright (C) 2021 Booz Allen
  * %%
  * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
  * #L%
  */
 
+import org.apache.commons.lang3.StringUtils;
 import org.technologybrewery.fermenter.mda.metamodel.element.MetamodelUtils;
 
+import com.boozallen.aiops.mda.generator.util.PipelineUtils;
+
 /**
  * Provides baseline decorator functionality for {@link Relation}.
- * 
+ *
  * The goal is to make it easier to apply the decorator pattern in various implementations of generators (e.g., Java,
  * python, Docker) so that each concrete decorator only has to decorate those aspects of the class that are needed, not
  * all the pass-through methods that each decorator would otherwise need to implement (that add no real value).
@@ -59,6 +62,16 @@ public String getName() {
         return wrapped.getName();
     }
 
+    @Override
+    public Boolean isRequired() {
+        return wrapped.isRequired();
+    }
+
+    @Override
+    public String getColumn() {
+        return wrapped.getColumn();
+    }
+
     @Override
     public void validate() {
         wrapped.validate();
@@ -71,4 +84,56 @@ public void validate() {
     public boolean isOneToManyRelation() {
         return wrapped.getMultiplicity().equals(Multiplicity.ONE_TO_MANY);
     }
+
+    /**
+     * Whether the Spark relation is nullable.
+     *
+     * @return true if the Spark field is nullable
+     */
+    public boolean isNullable() {
+        return wrapped.isRequired() == null || !wrapped.isRequired();
+    }
+
+    /**
+     * Returns the column name for the Spark relation.
+     *
+     * @return column name
+     */
+    public String getColumnName() {
+        String columnName;
+
+        if (StringUtils.isNotBlank(wrapped.getColumn())) {
+            columnName = wrapped.getColumn();
+        } else {
+            columnName = wrapped.getName();
+        }
+        return columnName;
+    }
+
+    /**
+     * Returns the relation name formatted to uppercase with underscores.
+     *
+     * @return name formatted to uppercase with underscores
+     */
+    public String getUpperSnakecaseName() {
+        return PipelineUtils.deriveUpperUnderscoreNameFromUpperCamelName(getName());
+    }
+
+    /**
+     * Returns the relation name, capitalized.
+     *
+     * @return capitalized name
+     */
+    public String getCapitalizedName() {
+        return StringUtils.capitalize(getName());
+    }
+
+    /**
+     * Returns the relation name, uncapitalized.
+     *
+     * @return uncapitalized name
+     */
+    public String getUncapitalizeName() {
+        return StringUtils.uncapitalize(getName());
+    }
 }

foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/Relation.java

@@ -33,6 +33,20 @@ public interface Relation extends NamespacedMetamodel {
 	 */
 	Multiplicity getMultiplicity();
 
+	/**
+	 * Returns whether the relation is required.
+	 *
+	 * @return required
+	 */
+	Boolean isRequired();
+
+	/**
+	 * Returns the column of the relation.
+	 *
+	 * @return column
+	 */
+	String getColumn();
+
 	/**
 	 * Enumerated values representing multiplicity options.
 	 */

foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/RelationElement.java

@@ -34,6 +34,11 @@ public class RelationElement extends NamespacedMetamodelElement implements Relat
     @JsonInclude(Include.NON_NULL)
     protected Multiplicity multiplicity;
 
+    @JsonInclude(Include.NON_NULL)
+    protected Boolean required;
+
+    @JsonInclude(Include.NON_NULL)
+    protected String column;
 
     /**
      * {@inheritDoc}
@@ -51,6 +56,16 @@ public Multiplicity getMultiplicity() {
         return multiplicity;
     }
 
+    @Override
+    public Boolean isRequired() {
+        return required;
+    }
+
+    @Override
+    public String getColumn() {
+        return column;
+    }
+
 
     /**
      * {@inheritDoc}

foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/spark/SparkRecord.java

@@ -15,6 +15,7 @@
 import java.util.Set;
 import java.util.TreeSet;
 
+import com.boozallen.aiops.mda.metamodel.element.Relation;
 import com.boozallen.aiops.mda.metamodel.element.java.JavaRecordFieldType;
 import com.boozallen.aiops.mda.metamodel.element.util.JavaElementUtils;
 import org.technologybrewery.fermenter.mda.TypeManager;
@@ -63,6 +64,18 @@ public List<RecordField> getFields() {
         return fields;
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public List<Relation> getRelations() {
+        List<Relation> relations = new ArrayList<>();
+        for (Relation relation: super.getRelations()){
+            relations.add(new SparkRecordRelation(relation));
+        }
+        return relations;
+    }
+
     /**
      * {@inheritDoc}
      */

foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/spark/SparkRecordRelation.java

@@ -0,0 +1,38 @@
+package com.boozallen.aiops.mda.metamodel.element.spark;
+
+/*-
+ * #%L
+ * aiSSEMBLE::Foundation::MDA
+ * %%
+ * Copyright (C) 2021 Booz Allen
+ * %%
+ * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
+ * #L%
+ */
+
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.boozallen.aiops.mda.metamodel.element.BaseRecordRelationDecorator;
+import com.boozallen.aiops.mda.metamodel.element.Relation;
+
+/**
+ * Decorates RecordField with Spark-specific functionality.
+ */
+public class SparkRecordRelation extends BaseRecordRelationDecorator {
+
+    /**
+     * {@inheritDoc}
+     */
+    public SparkRecordRelation(Relation recordRelationToDecorate) {
+        super(recordRelationToDecorate);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public String getDocumentation() {
+        return StringUtils.isNotBlank(super.getDocumentation()) ? super.getDocumentation() : "";
+    }
+}

foundation/foundation-mda/src/main/resources/aiops-record-schema.json

@@ -95,6 +95,12 @@
 					},
 					"multiplicity": {
 						"type": "string"
+					},
+					"required": {
+						"type": "string"
+					},
+					"column": {
+						"type": "string"
 					}
 				},
 				"required": [
@@ -106,4 +112,4 @@
 	"required": [
 		"name", "package", "fields"
 	]
-}
+}