[release-4.15] OCPBUGS-39521: UPSTREAM: <carry>:Add AWSMachine fields to control vpc placement for the instance, Drop the VPC field from the AWSMachine spec, look up the VPC from the subnet instead, Add support to specify PlacementGroupPartition of placement group, Ensure Volumes and Network Interfaces are also tagged on creation

api/v1beta1/awscluster_conversion.go

@@ -47,6 +47,7 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error {
 	if restored.Status.Bastion != nil {
 		dst.Status.Bastion.InstanceMetadataOptions = restored.Status.Bastion.InstanceMetadataOptions
 		dst.Status.Bastion.PlacementGroupName = restored.Status.Bastion.PlacementGroupName
+		dst.Status.Bastion.PlacementGroupPartition = restored.Status.Bastion.PlacementGroupPartition
 	}
 	dst.Spec.Partition = restored.Spec.Partition
 

api/v1beta1/awsmachine_conversion.go

@@ -38,6 +38,8 @@ func (src *AWSMachine) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.Ignition = restored.Spec.Ignition
 	dst.Spec.InstanceMetadataOptions = restored.Spec.InstanceMetadataOptions
 	dst.Spec.PlacementGroupName = restored.Spec.PlacementGroupName
+	dst.Spec.PlacementGroupPartition = restored.Spec.PlacementGroupPartition
+	dst.Spec.SecurityGroupOverrides = restored.Spec.SecurityGroupOverrides
 
 	return nil
 }
@@ -85,6 +87,8 @@ func (r *AWSMachineTemplate) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.Template.Spec.Ignition = restored.Spec.Template.Spec.Ignition
 	dst.Spec.Template.Spec.InstanceMetadataOptions = restored.Spec.Template.Spec.InstanceMetadataOptions
 	dst.Spec.Template.Spec.PlacementGroupName = restored.Spec.Template.Spec.PlacementGroupName
+	dst.Spec.Template.Spec.PlacementGroupPartition = restored.Spec.Template.Spec.PlacementGroupPartition
+	dst.Spec.Template.Spec.SecurityGroupOverrides = restored.Spec.Template.Spec.SecurityGroupOverrides
 
 	return nil
 }

api/v1beta2/awsmachine_types.go

@@ -114,6 +114,11 @@ type AWSMachineSpec struct {
 	// +optional
 	Subnet *AWSResourceReference `json:"subnet,omitempty"`
 
+	// SecurityGroupOverrides is an optional set of security groups to use for the node.
+	// This is optional - if not provided security groups from the cluster will be used.
+	// +optional
+	SecurityGroupOverrides map[SecurityGroupRole]string `json:"securityGroupOverrides,omitempty"`
+
 	// SSHKeyName is the name of the ssh key to attach to the instance. Valid values are empty string (do not use SSH keys), a valid SSH key name, or omitted (use the default SSH key name)
 	// +optional
 	SSHKeyName *string `json:"sshKeyName,omitempty"`
@@ -156,6 +161,14 @@ type AWSMachineSpec struct {
 	// +optional
 	PlacementGroupName string `json:"placementGroupName,omitempty"`
 
+	// PlacementGroupPartition is the partition number within the placement group in which to launch the instance.
+	// This value is only valid if the placement group, referred in `PlacementGroupName`, was created with
+	// strategy set to partition.
+	// +kubebuilder:validation:Minimum:=1
+	// +kubebuilder:validation:Maximum:=7
+	// +optional
+	PlacementGroupPartition int64 `json:"placementGroupPartition,omitempty"`
+
 	// Tenancy indicates if instance should run on shared or single-tenant hardware.
 	// +optional
 	// +kubebuilder:validation:Enum:=default;dedicated;host

api/v1beta2/types.go

@@ -198,6 +198,14 @@ type Instance struct {
 	// +optional
 	PlacementGroupName string `json:"placementGroupName,omitempty"`
 
+	// PlacementGroupPartition is the partition number within the placement group in which to launch the instance.
+	// This value is only valid if the placement group, referred in `PlacementGroupName`, was created with
+	// strategy set to partition.
+	// +kubebuilder:validation:Minimum:=1
+	// +kubebuilder:validation:Maximum:=7
+	// +optional
+	PlacementGroupPartition int64 `json:"placementGroupPartition,omitempty"`
+
 	// Tenancy indicates if instance should run on shared or single-tenant hardware.
 	// +optional
 	Tenancy string `json:"tenancy,omitempty"`

config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml

@@ -1031,6 +1031,15 @@ spec:
                     description: PlacementGroupName specifies the name of the placement
                       group in which to launch the instance.
                     type: string
+                  placementGroupPartition:
+                    description: PlacementGroupPartition is the partition number within
+                      the placement group in which to launch the instance. This value
+                      is only valid if the placement group, referred in `PlacementGroupName`,
+                      was created with strategy set to partition.
+                    format: int64
+                    maximum: 7
+                    minimum: 1
+                    type: integer
                   privateIp:
                     description: The private IPv4 address assigned to the instance.
                     type: string
@@ -2574,6 +2583,15 @@ spec:
                     description: PlacementGroupName specifies the name of the placement
                       group in which to launch the instance.
                     type: string
+                  placementGroupPartition:
+                    description: PlacementGroupPartition is the partition number within
+                      the placement group in which to launch the instance. This value
+                      is only valid if the placement group, referred in `PlacementGroupName`,
+                      was created with strategy set to partition.
+                    format: int64
+                    maximum: 7
+                    minimum: 1
+                    type: integer
                   privateIp:
                     description: The private IPv4 address assigned to the instance.
                     type: string

config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml

@@ -1588,6 +1588,15 @@ spec:
                     description: PlacementGroupName specifies the name of the placement
                       group in which to launch the instance.
                     type: string
+                  placementGroupPartition:
+                    description: PlacementGroupPartition is the partition number within
+                      the placement group in which to launch the instance. This value
+                      is only valid if the placement group, referred in `PlacementGroupName`,
+                      was created with strategy set to partition.
+                    format: int64
+                    maximum: 7
+                    minimum: 1
+                    type: integer
                   privateIp:
                     description: The private IPv4 address assigned to the instance.
                     type: string

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml

@@ -775,6 +775,15 @@ spec:
                 description: PlacementGroupName specifies the name of the placement
                   group in which to launch the instance.
                 type: string
+              placementGroupPartition:
+                description: PlacementGroupPartition is the partition number within
+                  the placement group in which to launch the instance. This value
+                  is only valid if the placement group, referred in `PlacementGroupName`,
+                  was created with strategy set to partition.
+                format: int64
+                maximum: 7
+                minimum: 1
+                type: integer
               providerID:
                 description: ProviderID is the unique identifier as specified by the
                   cloud provider.
@@ -824,6 +833,13 @@ spec:
                 required:
                 - size
                 type: object
+              securityGroupOverrides:
+                additionalProperties:
+                  type: string
+                description: SecurityGroupOverrides is an optional set of security
+                  groups to use for the node. This is optional - if not provided security
+                  groups from the cluster will be used.
+                type: object
               spotMarketOptions:
                 description: SpotMarketOptions allows users to configure instances
                   to be run using AWS Spot instances.

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml

@@ -729,6 +729,16 @@ spec:
                         description: PlacementGroupName specifies the name of the
                           placement group in which to launch the instance.
                         type: string
+                      placementGroupPartition:
+                        description: PlacementGroupPartition is the partition number
+                          within the placement group in which to launch the instance.
+                          This value is only valid if the placement group, referred
+                          in `PlacementGroupName`, was created with strategy set to
+                          partition.
+                        format: int64
+                        maximum: 7
+                        minimum: 1
+                        type: integer
                       providerID:
                         description: ProviderID is the unique identifier as specified
                           by the cloud provider.
@@ -781,6 +791,14 @@ spec:
                         required:
                         - size
                         type: object
+                      securityGroupOverrides:
+                        additionalProperties:
+                          type: string
+                        description: SecurityGroupOverrides is an optional set of
+                          security groups to use for the node. This is optional -
+                          if not provided security groups from the cluster will be
+                          used.
+                        type: object
                       spotMarketOptions:
                         description: SpotMarketOptions allows users to configure instances
                           to be run using AWS Spot instances.

controllers/awsmachine_controller_test.go

@@ -537,10 +537,6 @@ func mockedCreateSecretCall(s *mock_services.MockSecretInterfaceMockRecorder) {
 func mockedCreateInstanceCalls(m *mocks.MockEC2APIMockRecorder) {
 	m.DescribeInstances(gomock.Eq(&ec2.DescribeInstancesInput{
 		Filters: []*ec2.Filter{
-			{
-				Name:   aws.String("vpc-id"),
-				Values: aws.StringSlice([]string{"vpc-exists"}),
-			},
 			{
 				Name:   aws.String("tag:sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"),
 				Values: aws.StringSlice([]string{"owned"}),
@@ -649,10 +645,6 @@ func mockedCreateInstanceCalls(m *mocks.MockEC2APIMockRecorder) {
 			Name:   aws.String("state"),
 			Values: aws.StringSlice([]string{"pending", "available"}),
 		},
-		{
-			Name:   aws.String("vpc-id"),
-			Values: aws.StringSlice([]string{"vpc-exists"}),
-		},
 		{
 			Name:   aws.String("subnet-id"),
 			Values: aws.StringSlice([]string{"subnet-1"}),

docs/book/src/topics/bring-your-own-aws-infrastructure.md

@@ -120,6 +120,30 @@ spec:
 
 Users may either specify `failureDomain` on the Machine or MachineDeployment objects, _or_ users may explicitly specify subnet IDs on the AWSMachine or AWSMachineTemplate objects. If both are specified, the subnet ID is used and the `failureDomain` is ignored.
 
+### Placing EC2 Instances in Specific External VPCs
+
+CAPA clusters are deployed within a single VPC, but it's possible to place machines that live in external VPCs. For this kind of configuration, we assume that all the VPCs have the ability to communicate, either through external peering, a transit gateway, or some other mechanism already established outside of CAPA. CAPA will not create a tunnel or manage the network configuration for any secondary VPCs.
+
+The AWSMachineTemplate `subnet` field allows specifying filters or specific subnet ids for worker machine placement. If the filters or subnet id is specified in a secondary VPC, CAPA will place the machine in that VPC and subnet.
+
+```yaml
+spec:
+  template:
+    spec:
+      subnet:
+        filters:
+          name: "vpc-id"
+          values:
+            - "secondary-vpc-id"
+      securityGroupOverrides:
+        node: sg-04e870a3507a5ad2c5c8c2
+        node-eks-additional: sg-04e870a3507a5ad2c5c8c1
+```
+
+#### Caveats/Notes
+
+CAPA helpfully creates security groups for various roles in the cluster and automatically attaches them to workers. However, security groups are tied to a specific VPC, so workers placed in a VPC outside of the cluster will need to have these security groups created by some external process first and set in the `securityGroupOverrides` field, otherwise the ec2 creation will fail.
+
 ### Security Groups
 
 To use existing security groups for instances for a cluster, add this to the AWSCluster specification:
@@ -147,6 +171,15 @@ spec:
     - ...
 ```
 
+It's also possible to override the cluster security groups for an individual AWSMachine or AWSMachineTemplate:
+
+```yaml
+spec:
+  SecurityGroupOverrides:
+    node: sg-04e870a3507a5ad2c5c8c2
+    node-eks-additional: sg-04e870a3507a5ad2c5c8c1
+```
+
 ### Control Plane Load Balancer
 
 The cluster control plane is accessed through a Classic ELB. By default, Cluster API creates the Classic ELB. To use an existing Classic ELB, add its name to the AWSCluster specification: