diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a8b42eb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.retry diff --git a/README.md b/README.md new file mode 100644 index 0000000..70024f7 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# openshift4-ansible + +This playbook creates the OpenShift 4 UPI (User provided +Infrastructure) on AWS into an existing VPC with existing private and +public subnets and DNS Zones. + +It is also possible to deploy the API server without exposing it to +the Internet, this will require that the host that runs this Ansible +playbook can access the VPC subnets. + +The Cloudformation templates are based on these: +https://github.com/openshift/installer/tree/master/upi/aws/cloudformation + +Some information has to be provided. Mainly information about your AWS +VPC, your subnets etc. See `inventory/group_vars/all` + + +## Setup + +Create an administrative IAM user to perform the install. +See https://github.com/openshift/installer/blob/master/docs/user/aws/iam.md + +This user can be removed after the installation + +To set up a bastion host follow these steps: + +Start with a RHEL7 Instance. + +Become root and install the needed tools: + +```bash +sudo -i + +subscription-manager repos --enable rhel-7-server-ansible-2.8-rpms + +yum install -y ansible + +yum install -y \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + +yum -y install \ + python2-boto python2-boto3 python2-simplejson + +yum erase -y epel-release + +exit +``` + +With your own account, create ~/.aws/credentials with the following +content, replacing the AWSKEY and AWSSECRETKEY with the right values +from AWS. + +``` +[default] +aws_access_key_id = AWSKEY +aws_secret_access_key = AWSSECRETKEY +``` + +## Usage + +Modify `inventory/group_vars/all`. + +```bash +ansible-playbook install-upi.yaml +``` + +To delete all AWS resources that were created for an OpenShift cluster, use the same `inventory/group_vars/all` that was used for the +installation. In particular, the clustername has to match. You also need the `/tmp/CLUSTERNAME` directory that was created +by the installation playbook. + +```bash +ansible-playbook uninstall-upi.yaml +``` + +### Disk Encryption + +To enable encryption of the EBS volumes attached to the master and worker nodes, the RHCOS AMI needs to be copied before +the installation is started. This can be done by running + +```bash +ansible-playbook create-encrypted-ami.yaml +``` + +The playbook uses the AMI ID `rhcos_ami` from `vars.yaml` as the +source and creates a private AMI that is identical to the source AMI, +except that disk encryption is enabled. + +install-upi.yaml looks for a private AMI created by +`create-encrypted-ami.yaml`. If none is found, it uses AMI ID +`rhcos_ami` from `inventory/group_vars/all`. diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..d126ccc --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,7 @@ +[defaults] +inventory = inventory +host_key_checking = False +retry_files_enabled = False +callback_whitelist = profile_tasks +forks = 20 +interpreter_python = auto_silent diff --git a/create-encrypted-ami.yaml b/create-encrypted-ami.yaml new file mode 100644 index 0000000..b284de0 --- /dev/null +++ b/create-encrypted-ami.yaml @@ -0,0 +1,49 @@ +--- +- name: Create encrypted AMI image + gather_facts: false + hosts: localhost + tasks: + - name: Get unencrypted RCOS AMI + ec2_ami_facts: + filters: + image-id: "{{ rhcos_ami }}" + region: "{{ region }}" + register: ami_unencrypted + + - name: Get previous encrypted RHCOS AMI + ec2_ami_facts: + filters: + "tag:rhcos_version": "{{ rhcos_version }}" + "tag:latest_ami": 'true' + region: "{{ region }}" + register: ami_encrypted_old + + - name: Update latest_ami tag for previous encrypted AMIs + ec2_ami: + image_id: "{{ item.image_id }}" + region: "{{ region }}" + tags: + latest_ami: 'false' + loop: "{{ ami_encrypted_old.images }}" + + - name: Copy unencrypted RHCOS AMI and enable encryption + ec2_ami_copy: + source_image_id: "{{ rhcos_ami }}" + source_region: "{{ region }}" + name: "{{ ami_unencrypted.images[0]['name'] ~ '-encrypted' }}" + region: "{{ region }}" + encrypted: true + tags: + rhcos_version: "{{ rhcos_version }}" + latest_ami: 'true' + register: ami_encrypted + + - name: Wait for encrypted RHCOS AMI to become available + ec2_ami_facts: + image_ids: "{{ ami_encrypted.image_id }}" + region: "{{ region }}" + register: ami_check + until: ami_check.images[0].state == 'available' + retries: 60 + delay: 10 +... diff --git a/files/LDAPsecret.yaml.j2 b/files/LDAPsecret.yaml.j2 new file mode 100644 index 0000000..86d09b2 --- /dev/null +++ b/files/LDAPsecret.yaml.j2 @@ -0,0 +1,7 @@ +apiVersion: v1 +data: + bindPassword: "{{ ldap_bindPassword | b64encode }}" +kind: Secret +metadata: + namespace: openshift-config + name: "{{ ldap_secret_name }}" diff --git a/files/Oauth.yaml.j2 b/files/Oauth.yaml.j2 new file mode 100644 index 0000000..478e473 --- /dev/null +++ b/files/Oauth.yaml.j2 @@ -0,0 +1,30 @@ +apiVersion: config.openshift.io/v1 +kind: OAuth +metadata: + name: cluster +spec: + identityProviders: + # This provider name is prefixed to the returned user ID to form an identity name: + - name: "{{ ldap_idp_name }}" + # Controls how mappings are established between this provider’s identities and user objects: + mappingMethod: claim + type: LDAP + ldap: + attributes: + id: + - dn + email: + - mail + name: + - cn + preferredUsername: + - uid + bindDN: "{{ bindDN }}" + bindPassword: + name: "{{ ldap_secret_name }}" + # ca: + # name: ca-config-map1 + # LDAP or LDAPS: + insecure: false + # An RFC 2255 URL which specifies the LDAP host and search parameters to use: + url: "{{ ldap_url }}" diff --git a/files/cloudformation/01_vpc.yaml b/files/cloudformation/01_vpc.yaml new file mode 100644 index 0000000..5742846 --- /dev/null +++ b/files/cloudformation/01_vpc.yaml @@ -0,0 +1,369 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for Best Practice VPC with 1-3 AZs + +Parameters: + VpcCidr: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-4]))$ + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-24. + Default: 10.0.0.0/16 + Description: CIDR block for VPC. + Type: String + AvailabilityZoneCount: + ConstraintDescription: "The number of availability zones. (Min: 1, Max: 3)" + MinValue: 1 + MaxValue: 3 + Default: 1 + Description: "How many AZs to create VPC subnets for. (Min: 1, Max: 3)" + Type: Number + SubnetBits: + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/19-27. + MinValue: 5 + MaxValue: 13 + Default: 12 + Description: "Size of each subnet to create within the availability zones. (Min: 5 = /27, Max: 13 = /19)" + Type: Number + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Network Configuration" + Parameters: + - VpcCidr + - SubnetBits + - Label: + default: "Availability Zones" + Parameters: + - AvailabilityZoneCount + ParameterLabels: + AvailabilityZoneCount: + default: "Availability Zone Count" + VpcCidr: + default: "VPC CIDR" + SubnetBits: + default: "Bits Per Subnet" + +Conditions: + DoAz3: !Equals [3, !Ref AvailabilityZoneCount] + DoAz2: !Or [!Equals [2, !Ref AvailabilityZoneCount], Condition: DoAz3] + +Resources: + VPC: + Type: "AWS::EC2::VPC" + Properties: + EnableDnsSupport: "true" + EnableDnsHostnames: "true" + CidrBlock: !Ref VpcCidr + PublicSubnet: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [0, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 0 + - Fn::GetAZs: !Ref "AWS::Region" + PublicSubnet2: + Type: "AWS::EC2::Subnet" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [1, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 1 + - Fn::GetAZs: !Ref "AWS::Region" + PublicSubnet3: + Type: "AWS::EC2::Subnet" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [2, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 2 + - Fn::GetAZs: !Ref "AWS::Region" + InternetGateway: + Type: "AWS::EC2::InternetGateway" + GatewayToInternet: + Type: "AWS::EC2::VPCGatewayAttachment" + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + PublicRouteTable: + Type: "AWS::EC2::RouteTable" + Properties: + VpcId: !Ref VPC + PublicRoute: + Type: "AWS::EC2::Route" + DependsOn: GatewayToInternet + Properties: + RouteTableId: !Ref PublicRouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + PublicSubnetRouteTableAssociation: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PublicSubnet + RouteTableId: !Ref PublicRouteTable + PublicSubnetRouteTableAssociation2: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz2 + Properties: + SubnetId: !Ref PublicSubnet2 + RouteTableId: !Ref PublicRouteTable + PublicSubnetRouteTableAssociation3: + Condition: DoAz3 + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PublicSubnet3 + RouteTableId: !Ref PublicRouteTable + PublicNetworkAcl: + Type: "AWS::EC2::NetworkAcl" + Properties: + VpcId: !Ref VPC + InboundHTTPPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "100" + Protocol: "6" + RuleAction: allow + Egress: "false" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "80" + To: "80" + InboundHTTPSPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "101" + Protocol: "6" + RuleAction: allow + Egress: "false" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "443" + To: "443" + InboundSSHPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "102" + Protocol: "6" + RuleAction: allow + Egress: "false" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "22" + To: "22" + InboundEphemeralPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "103" + Protocol: "6" + RuleAction: allow + Egress: "false" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "1024" + To: "65535" + OutboundPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "100" + Protocol: "6" + RuleAction: allow + Egress: "true" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "0" + To: "65535" + PublicSubnetNetworkAclAssociation: + Type: "AWS::EC2::SubnetNetworkAclAssociation" + Properties: + SubnetId: !Ref PublicSubnet + NetworkAclId: !Ref PublicNetworkAcl + PublicSubnetNetworkAclAssociation2: + Type: "AWS::EC2::SubnetNetworkAclAssociation" + Condition: DoAz2 + Properties: + SubnetId: !Ref PublicSubnet2 + NetworkAclId: !Ref PublicNetworkAcl + PublicSubnetNetworkAclAssociation3: + Type: "AWS::EC2::SubnetNetworkAclAssociation" + Condition: DoAz3 + Properties: + SubnetId: !Ref PublicSubnet3 + NetworkAclId: !Ref PublicNetworkAcl + PrivateSubnet: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [3, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 0 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable: + Type: "AWS::EC2::RouteTable" + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PrivateSubnet + RouteTableId: !Ref PrivateRouteTable + NAT: + DependsOn: + - GatewayToInternet + Type: "AWS::EC2::NatGateway" + Properties: + AllocationId: + "Fn::GetAtt": + - EIP + - AllocationId + SubnetId: !Ref PublicSubnet + EIP: + Type: "AWS::EC2::EIP" + Properties: + Domain: vpc + Route: + Type: "AWS::EC2::Route" + Properties: + RouteTableId: + Ref: PrivateRouteTable + DestinationCidrBlock: 0.0.0.0/0 + NatGatewayId: + Ref: NAT + PrivateSubnet2: + Type: "AWS::EC2::Subnet" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [4, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 1 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable2: + Type: "AWS::EC2::RouteTable" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation2: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz2 + Properties: + SubnetId: !Ref PrivateSubnet2 + RouteTableId: !Ref PrivateRouteTable2 + NAT2: + DependsOn: + - GatewayToInternet + Type: "AWS::EC2::NatGateway" + Condition: DoAz2 + Properties: + AllocationId: + "Fn::GetAtt": + - EIP2 + - AllocationId + SubnetId: !Ref PublicSubnet2 + EIP2: + Type: "AWS::EC2::EIP" + Condition: DoAz2 + Properties: + Domain: vpc + Route2: + Type: "AWS::EC2::Route" + Condition: DoAz2 + Properties: + RouteTableId: + Ref: PrivateRouteTable2 + DestinationCidrBlock: 0.0.0.0/0 + NatGatewayId: + Ref: NAT2 + PrivateSubnet3: + Type: "AWS::EC2::Subnet" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [5, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 2 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable3: + Type: "AWS::EC2::RouteTable" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation3: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz3 + Properties: + SubnetId: !Ref PrivateSubnet3 + RouteTableId: !Ref PrivateRouteTable3 + NAT3: + DependsOn: + - GatewayToInternet + Type: "AWS::EC2::NatGateway" + Condition: DoAz3 + Properties: + AllocationId: + "Fn::GetAtt": + - EIP3 + - AllocationId + SubnetId: !Ref PublicSubnet3 + EIP3: + Type: "AWS::EC2::EIP" + Condition: DoAz3 + Properties: + Domain: vpc + Route3: + Type: "AWS::EC2::Route" + Condition: DoAz3 + Properties: + RouteTableId: + Ref: PrivateRouteTable3 + DestinationCidrBlock: 0.0.0.0/0 + NatGatewayId: + Ref: NAT3 + S3Endpoint: + Type: AWS::EC2::VPCEndpoint + Properties: + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: '*' + Action: + - '*' + Resource: + - '*' + RouteTableIds: + - !Ref PublicRouteTable + - !Ref PrivateRouteTable + - !If [DoAz2, !Ref PrivateRouteTable2, !Ref "AWS::NoValue"] + - !If [DoAz3, !Ref PrivateRouteTable3, !Ref "AWS::NoValue"] + ServiceName: !Join + - '' + - - com.amazonaws. + - !Ref 'AWS::Region' + - .s3 + VpcId: !Ref VPC + +Outputs: + VpcId: + Description: ID of the new VPC. + Value: !Ref VPC + PublicSubnetIds: + Description: Subnet IDs of the public subnets. + Value: + !Join [ + ",", + [!Ref PublicSubnet, !If [DoAz2, !Ref PublicSubnet2, !Ref "AWS::NoValue"], !If [DoAz3, !Ref PublicSubnet3, !Ref "AWS::NoValue"]] + ] + PrivateSubnetIds: + Description: Subnet IDs of the private subnets. + Value: + !Join [ + ",", + [!Ref PrivateSubnet, !If [DoAz2, !Ref PrivateSubnet2, !Ref "AWS::NoValue"], !If [DoAz3, !Ref PrivateSubnet3, !Ref "AWS::NoValue"]] + ] diff --git a/files/cloudformation/02_cluster_infra.yaml b/files/cloudformation/02_cluster_infra.yaml new file mode 100644 index 0000000..ea10e59 --- /dev/null +++ b/files/cloudformation/02_cluster_infra.yaml @@ -0,0 +1,380 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Network Elements (Route53 & LBs) + +Parameters: + ClusterName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Cluster name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, representative cluster name to use for host names and other identifying names. + Type: String + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag cloud resources and identify items owned or used by the cluster. + Type: String + HostedZoneId: + Description: The Route53 public zone ID to register the targets with, such as Z21IXYZABCZ2A4. + Type: String + HostedZoneName: + Description: The Route53 zone to register the targets with, such as example.com. Omit the trailing period. + Type: String + Default: "example.com" + PublicSubnets: + Description: The internet-facing subnets. + Type: List + PrivateSubnets: + Description: The internal subnets. + Type: List + VpcId: + Description: The VPC-scoped resources will belong to this VPC. + Type: AWS::EC2::VPC::Id + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - ClusterName + - InfrastructureName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - PublicSubnets + - PrivateSubnets + - Label: + default: "DNS" + Parameters: + - HostedZoneName + - HostedZoneId + ParameterLabels: + ClusterName: + default: "Cluster Name" + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + PublicSubnets: + default: "Public Subnets" + PrivateSubnets: + default: "Private Subnets" + HostedZoneName: + default: "Public Hosted Zone Name" + HostedZoneId: + default: "Public Hosted Zone ID" + +Resources: + ExtApiElb: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Join ["-", [!Ref InfrastructureName, "ext"]] + IpAddressType: ipv4 + Subnets: !Ref PublicSubnets + Type: network + + IntApiElb: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Join ["-", [!Ref InfrastructureName, "int"]] + Scheme: internal + IpAddressType: ipv4 + Subnets: !Ref PrivateSubnets + Type: network + + IntDns: + Type: "AWS::Route53::HostedZone" + Properties: + HostedZoneConfig: + Comment: "Managed by CloudFormation" + Name: !Join [".", [!Ref ClusterName, !Ref HostedZoneName]] + HostedZoneTags: + - Key: Name + Value: !Join ["-", [!Ref InfrastructureName, "int"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "owned" + VPCs: + - VPCId: !Ref VpcId + VPCRegion: !Ref "AWS::Region" + + ExternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref HostedZoneId + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt ExtApiElb.CanonicalHostedZoneID + DNSName: !GetAtt ExtApiElb.DNSName + + InternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref IntDns + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + - Name: + !Join [ + ".", + ["api-int", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + + ExternalApiListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: ExternalApiTargetGroup + LoadBalancerArn: + Ref: ExtApiElb + Port: 6443 + Protocol: TCP + + ExternalApiTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 6443 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + InternalApiListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalApiTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 6443 + Protocol: TCP + + InternalApiTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 6443 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + InternalServiceInternalListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalServiceTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 22623 + Protocol: TCP + + InternalServiceTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 22623 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + RegisterTargetLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "nlb", "lambda", "role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "master", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalApiTargetGroup + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalServiceTargetGroup + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref ExternalApiTargetGroup + + RegisterNlbIpTargets: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterTargetLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + elb = boto3.client('elbv2') + if event['RequestType'] == 'Delete': + elb.deregister_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + elif event['RequestType'] == 'Create': + elb.register_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['TargetArn']+event['ResourceProperties']['TargetIp']) + Runtime: "python3.7" + Timeout: 120 + + RegisterSubnetTagsLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "subnet-tags-lambda-role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "subnet-tagging-policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "ec2:DeleteTags", + "ec2:CreateTags" + ] + Resource: "arn:aws:ec2:*:*:subnet/*" + - Effect: "Allow" + Action: + [ + "ec2:DescribeSubnets", + "ec2:DescribeTags" + ] + Resource: "*" + + RegisterSubnetTags: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterSubnetTagsLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + ec2_client = boto3.client('ec2') + if event['RequestType'] == 'Delete': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.delete_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName']}]); + elif event['RequestType'] == 'Create': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.create_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName'], 'Value': 'shared'}]); + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['InfrastructureName']+event['ResourceProperties']['Subnets'][0]) + Runtime: "python3.7" + Timeout: 120 + + RegisterPublicSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PublicSubnets + + RegisterPrivateSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PrivateSubnets + +Outputs: + PrivateHostedZoneId: + Description: Hosted zone ID for the private DNS, which is required for private records. + Value: !Ref IntDns + ExternalApiLoadBalancerName: + Description: Full name of the External API load balancer created. + Value: !GetAtt ExtApiElb.LoadBalancerFullName + InternalApiLoadBalancerName: + Description: Full name of the Internal API load balancer created. + Value: !GetAtt IntApiElb.LoadBalancerFullName + ApiServerDnsName: + Description: Full hostname of the API server, which is required for the Ignition config files. + Value: !Join [".", ["api-int", !Ref ClusterName, !Ref HostedZoneName]] + RegisterNlbIpTargetsLambda: + Description: Lambda ARN useful to help register or deregister IP targets for these load balancers. + Value: !GetAtt RegisterNlbIpTargets.Arn + ExternalApiTargetGroupArn: + Description: ARN of External API target group. + Value: !Ref ExternalApiTargetGroup + InternalApiTargetGroupArn: + Description: ARN of Internal API target group. + Value: !Ref InternalApiTargetGroup + InternalServiceTargetGroupArn: + Description: ARN of internal service target group. + Value: !Ref InternalServiceTargetGroup diff --git a/files/cloudformation/02_cluster_infra.yaml.existingprivzone b/files/cloudformation/02_cluster_infra.yaml.existingprivzone new file mode 100644 index 0000000..f7fc831 --- /dev/null +++ b/files/cloudformation/02_cluster_infra.yaml.existingprivzone @@ -0,0 +1,383 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Network Elements (Route53 & LBs) + +Parameters: + ClusterName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Cluster name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, representative cluster name to use for host names and other identifying names. + Type: String + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag cloud resources and identify items owned or used by the cluster. + Type: String + HostedZoneId: + Description: The Route53 public zone ID to register the targets with, such as Z21IXYZABCZ2A4. + Type: String + HostedZoneName: + Description: The Route53 zone to register the targets with, such as example.com. Omit the trailing period. + Type: String + Default: "example.com" + PublicSubnets: + Description: The internet-facing subnets. + Type: List + PrivateSubnets: + Description: The internal subnets. + Type: List + VpcId: + Description: The VPC-scoped resources will belong to this VPC. + Type: AWS::EC2::VPC::Id + PrivateZoneId: + Description: The ID of the private zone + Type: String + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - ClusterName + - InfrastructureName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - PublicSubnets + - PrivateSubnets + - Label: + default: "DNS" + Parameters: + - HostedZoneName + - HostedZoneId + ParameterLabels: + ClusterName: + default: "Cluster Name" + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + PublicSubnets: + default: "Public Subnets" + PrivateSubnets: + default: "Private Subnets" + HostedZoneName: + default: "Public Hosted Zone Name" + HostedZoneId: + default: "Public Hosted Zone ID" + +Resources: + ExtApiElb: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Join ["-", [!Ref InfrastructureName, "ext"]] + IpAddressType: ipv4 + Subnets: !Ref PublicSubnets + Type: network + + IntApiElb: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Join ["-", [!Ref InfrastructureName, "int"]] + Scheme: internal + IpAddressType: ipv4 + Subnets: !Ref PrivateSubnets + Type: network + +# IntDns: +# Type: "AWS::Route53::HostedZone" +# Properties: +# HostedZoneConfig: +# Comment: "Managed by CloudFormation" +# Name: !Join [".", [!Ref ClusterName, !Ref HostedZoneName]] +# HostedZoneTags: +# - Key: Name +# Value: !Join ["-", [!Ref InfrastructureName, "int"]] +# - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] +# Value: "owned" +# VPCs: +# - VPCId: !Ref VpcId +# VPCRegion: !Ref "AWS::Region" + + ExternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref HostedZoneId + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt ExtApiElb.CanonicalHostedZoneID + DNSName: !GetAtt ExtApiElb.DNSName + + InternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref PrivateZoneId + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + - Name: + !Join [ + ".", + ["api-int", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + + ExternalApiListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: ExternalApiTargetGroup + LoadBalancerArn: + Ref: ExtApiElb + Port: 6443 + Protocol: TCP + + ExternalApiTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 6443 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + InternalApiListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalApiTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 6443 + Protocol: TCP + + InternalApiTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 6443 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + InternalServiceInternalListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalServiceTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 22623 + Protocol: TCP + + InternalServiceTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 22623 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + RegisterTargetLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "nlb", "lambda", "role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "master", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalApiTargetGroup + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalServiceTargetGroup + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref ExternalApiTargetGroup + + RegisterNlbIpTargets: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterTargetLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + elb = boto3.client('elbv2') + if event['RequestType'] == 'Delete': + elb.deregister_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + elif event['RequestType'] == 'Create': + elb.register_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['TargetArn']+event['ResourceProperties']['TargetIp']) + Runtime: "python3.7" + Timeout: 120 + + RegisterSubnetTagsLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "subnet-tags-lambda-role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "subnet-tagging-policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "ec2:DeleteTags", + "ec2:CreateTags" + ] + Resource: "arn:aws:ec2:*:*:subnet/*" + - Effect: "Allow" + Action: + [ + "ec2:DescribeSubnets", + "ec2:DescribeTags" + ] + Resource: "*" + + RegisterSubnetTags: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterSubnetTagsLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + ec2_client = boto3.client('ec2') + if event['RequestType'] == 'Delete': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.delete_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName']}]); + elif event['RequestType'] == 'Create': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.create_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName'], 'Value': 'shared'}]); + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['InfrastructureName']+event['ResourceProperties']['Subnets'][0]) + Runtime: "python3.7" + Timeout: 120 + + RegisterPublicSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PublicSubnets + + RegisterPrivateSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PrivateSubnets + +Outputs: +# PrivateHostedZoneId: +# Description: Hosted zone ID for the private DNS, which is required for private records. +# Value: !Ref IntDns + ExternalApiLoadBalancerName: + Description: Full name of the External API load balancer created. + Value: !GetAtt ExtApiElb.LoadBalancerFullName + InternalApiLoadBalancerName: + Description: Full name of the Internal API load balancer created. + Value: !GetAtt IntApiElb.LoadBalancerFullName + ApiServerDnsName: + Description: Full hostname of the API server, which is required for the Ignition config files. + Value: !Join [".", ["api-int", !Ref ClusterName, !Ref HostedZoneName]] + RegisterNlbIpTargetsLambda: + Description: Lambda ARN useful to help register or deregister IP targets for these load balancers. + Value: !GetAtt RegisterNlbIpTargets.Arn + ExternalApiTargetGroupArn: + Description: ARN of External API target group. + Value: !Ref ExternalApiTargetGroup + InternalApiTargetGroupArn: + Description: ARN of Internal API target group. + Value: !Ref InternalApiTargetGroup + InternalServiceTargetGroupArn: + Description: ARN of internal service target group. + Value: !Ref InternalServiceTargetGroup diff --git a/files/cloudformation/02_cluster_infra.yaml.internalonly b/files/cloudformation/02_cluster_infra.yaml.internalonly new file mode 100644 index 0000000..0169673 --- /dev/null +++ b/files/cloudformation/02_cluster_infra.yaml.internalonly @@ -0,0 +1,380 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Network Elements (Route53 & LBs) + +Parameters: + ClusterName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Cluster name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, representative cluster name to use for host names and other identifying names. + Type: String + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag cloud resources and identify items owned or used by the cluster. + Type: String + HostedZoneId: + Description: The Route53 public zone ID to register the targets with, such as Z21IXYZABCZ2A4. + Type: String + HostedZoneName: + Description: The Route53 zone to register the targets with, such as example.com. Omit the trailing period. + Type: String + Default: "example.com" + PublicSubnets: + Description: The internet-facing subnets. + Type: List + PrivateSubnets: + Description: The internal subnets. + Type: List + VpcId: + Description: The VPC-scoped resources will belong to this VPC. + Type: AWS::EC2::VPC::Id + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - ClusterName + - InfrastructureName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - PublicSubnets + - PrivateSubnets + - Label: + default: "DNS" + Parameters: + - HostedZoneName + - HostedZoneId + ParameterLabels: + ClusterName: + default: "Cluster Name" + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + PublicSubnets: + default: "Public Subnets" + PrivateSubnets: + default: "Private Subnets" + HostedZoneName: + default: "Public Hosted Zone Name" + HostedZoneId: + default: "Public Hosted Zone ID" + +Resources: +# ExtApiElb: +# Type: AWS::ElasticLoadBalancingV2::LoadBalancer +# Properties: +# Name: !Join ["-", [!Ref InfrastructureName, "ext"]] +# IpAddressType: ipv4 +# Subnets: !Ref PublicSubnets +# Type: network + + IntApiElb: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Join ["-", [!Ref InfrastructureName, "int"]] + Scheme: internal + IpAddressType: ipv4 + Subnets: !Ref PrivateSubnets + Type: network + + IntDns: + Type: "AWS::Route53::HostedZone" + Properties: + HostedZoneConfig: + Comment: "Managed by CloudFormation" + Name: !Join [".", [!Ref ClusterName, !Ref HostedZoneName]] + HostedZoneTags: + - Key: Name + Value: !Join ["-", [!Ref InfrastructureName, "int"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "owned" + VPCs: + - VPCId: !Ref VpcId + VPCRegion: !Ref "AWS::Region" + + ExternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref HostedZoneId + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + + InternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref IntDns + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + - Name: + !Join [ + ".", + ["api-int", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + + # ExternalApiListener: + # Type: AWS::ElasticLoadBalancingV2::Listener + # Properties: + # DefaultActions: + # - Type: forward + # TargetGroupArn: + # Ref: ExternalApiTargetGroup + # LoadBalancerArn: + # Ref: ExtApiElb + # Port: 6443 + # Protocol: TCP + +# ExternalApiTargetGroup: +# Type: AWS::ElasticLoadBalancingV2::TargetGroup +# Properties: +# Port: 6443 +# Protocol: TCP +# TargetType: ip +# VpcId: +# Ref: VpcId +# TargetGroupAttributes: +# - Key: deregistration_delay.timeout_seconds +# Value: 60 +# + InternalApiListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalApiTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 6443 + Protocol: TCP + + InternalApiTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 6443 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + InternalServiceInternalListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalServiceTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 22623 + Protocol: TCP + + InternalServiceTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 22623 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + RegisterTargetLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "nlb", "lambda", "role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "master", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalApiTargetGroup + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalServiceTargetGroup +# - Effect: "Allow" +# Action: +# [ +# "elasticloadbalancing:RegisterTargets", +# "elasticloadbalancing:DeregisterTargets", +# ] +# Resource: !Ref ExternalApiTargetGroup + + RegisterNlbIpTargets: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterTargetLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + elb = boto3.client('elbv2') + if event['RequestType'] == 'Delete': + elb.deregister_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + elif event['RequestType'] == 'Create': + elb.register_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['TargetArn']+event['ResourceProperties']['TargetIp']) + Runtime: "python3.7" + Timeout: 120 + + RegisterSubnetTagsLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "subnet-tags-lambda-role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "subnet-tagging-policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "ec2:DeleteTags", + "ec2:CreateTags" + ] + Resource: "arn:aws:ec2:*:*:subnet/*" + - Effect: "Allow" + Action: + [ + "ec2:DescribeSubnets", + "ec2:DescribeTags" + ] + Resource: "*" + + RegisterSubnetTags: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterSubnetTagsLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + ec2_client = boto3.client('ec2') + if event['RequestType'] == 'Delete': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.delete_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName']}]); + elif event['RequestType'] == 'Create': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.create_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName'], 'Value': 'shared'}]); + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['InfrastructureName']+event['ResourceProperties']['Subnets'][0]) + Runtime: "python3.7" + Timeout: 120 + + RegisterPublicSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PublicSubnets + + RegisterPrivateSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PrivateSubnets + +Outputs: + PrivateHostedZoneId: + Description: Hosted zone ID for the private DNS, which is required for private records. + Value: !Ref IntDns +# ExternalApiLoadBalancerName: +# Description: Full name of the External API load balancer created. +# Value: !GetAtt ExtApiElb.LoadBalancerFullName + InternalApiLoadBalancerName: + Description: Full name of the Internal API load balancer created. + Value: !GetAtt IntApiElb.LoadBalancerFullName + ApiServerDnsName: + Description: Full hostname of the API server, which is required for the Ignition config files. + Value: !Join [".", ["api-int", !Ref ClusterName, !Ref HostedZoneName]] + RegisterNlbIpTargetsLambda: + Description: Lambda ARN useful to help register or deregister IP targets for these load balancers. + Value: !GetAtt RegisterNlbIpTargets.Arn +# ExternalApiTargetGroupArn: +# Description: ARN of External API target group. +# Value: !Ref ExternalApiTargetGroup + InternalApiTargetGroupArn: + Description: ARN of Internal API target group. + Value: !Ref InternalApiTargetGroup + InternalServiceTargetGroupArn: + Description: ARN of internal service target group. + Value: !Ref InternalServiceTargetGroup diff --git a/files/cloudformation/02_cluster_infra.yaml.internalonly-existingprivzone b/files/cloudformation/02_cluster_infra.yaml.internalonly-existingprivzone new file mode 100644 index 0000000..4926a26 --- /dev/null +++ b/files/cloudformation/02_cluster_infra.yaml.internalonly-existingprivzone @@ -0,0 +1,383 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Network Elements (Route53 & LBs) + +Parameters: + ClusterName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Cluster name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, representative cluster name to use for host names and other identifying names. + Type: String + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag cloud resources and identify items owned or used by the cluster. + Type: String + HostedZoneId: + Description: The Route53 public zone ID to register the targets with, such as Z21IXYZABCZ2A4. + Type: String + HostedZoneName: + Description: The Route53 zone to register the targets with, such as example.com. Omit the trailing period. + Type: String + Default: "example.com" + PublicSubnets: + Description: The internet-facing subnets. + Type: List + PrivateSubnets: + Description: The internal subnets. + Type: List + VpcId: + Description: The VPC-scoped resources will belong to this VPC. + Type: AWS::EC2::VPC::Id + PrivateZoneId: + Description: The ID of the private zone + Type: String + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - ClusterName + - InfrastructureName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - PublicSubnets + - PrivateSubnets + - Label: + default: "DNS" + Parameters: + - HostedZoneName + - HostedZoneId + ParameterLabels: + ClusterName: + default: "Cluster Name" + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + PublicSubnets: + default: "Public Subnets" + PrivateSubnets: + default: "Private Subnets" + HostedZoneName: + default: "Public Hosted Zone Name" + HostedZoneId: + default: "Public Hosted Zone ID" + +Resources: +# ExtApiElb: +# Type: AWS::ElasticLoadBalancingV2::LoadBalancer +# Properties: +# Name: !Join ["-", [!Ref InfrastructureName, "ext"]] +# IpAddressType: ipv4 +# Subnets: !Ref PublicSubnets +# Type: network + + IntApiElb: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Join ["-", [!Ref InfrastructureName, "int"]] + Scheme: internal + IpAddressType: ipv4 + Subnets: !Ref PrivateSubnets + Type: network + +# IntDns: +# Type: "AWS::Route53::HostedZone" +# Properties: +# HostedZoneConfig: +# Comment: "Managed by CloudFormation" +# Name: !Join [".", [!Ref ClusterName, !Ref HostedZoneName]] +# HostedZoneTags: +# - Key: Name +# Value: !Join ["-", [!Ref InfrastructureName, "int"]] +# - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] +# Value: "owned" +# VPCs: +# - VPCId: !Ref VpcId +# VPCRegion: !Ref "AWS::Region" + + ExternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref HostedZoneId + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + + InternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref PrivateZoneId + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + - Name: + !Join [ + ".", + ["api-int", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + + # ExternalApiListener: + # Type: AWS::ElasticLoadBalancingV2::Listener + # Properties: + # DefaultActions: + # - Type: forward + # TargetGroupArn: + # Ref: ExternalApiTargetGroup + # LoadBalancerArn: + # Ref: ExtApiElb + # Port: 6443 + # Protocol: TCP + +# ExternalApiTargetGroup: +# Type: AWS::ElasticLoadBalancingV2::TargetGroup +# Properties: +# Port: 6443 +# Protocol: TCP +# TargetType: ip +# VpcId: +# Ref: VpcId +# TargetGroupAttributes: +# - Key: deregistration_delay.timeout_seconds +# Value: 60 +# + InternalApiListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalApiTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 6443 + Protocol: TCP + + InternalApiTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 6443 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + InternalServiceInternalListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalServiceTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 22623 + Protocol: TCP + + InternalServiceTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 22623 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + RegisterTargetLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "nlb", "lambda", "role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "master", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalApiTargetGroup + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalServiceTargetGroup +# - Effect: "Allow" +# Action: +# [ +# "elasticloadbalancing:RegisterTargets", +# "elasticloadbalancing:DeregisterTargets", +# ] +# Resource: !Ref ExternalApiTargetGroup + + RegisterNlbIpTargets: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterTargetLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + elb = boto3.client('elbv2') + if event['RequestType'] == 'Delete': + elb.deregister_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + elif event['RequestType'] == 'Create': + elb.register_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['TargetArn']+event['ResourceProperties']['TargetIp']) + Runtime: "python3.7" + Timeout: 120 + + RegisterSubnetTagsLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "subnet-tags-lambda-role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "subnet-tagging-policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "ec2:DeleteTags", + "ec2:CreateTags" + ] + Resource: "arn:aws:ec2:*:*:subnet/*" + - Effect: "Allow" + Action: + [ + "ec2:DescribeSubnets", + "ec2:DescribeTags" + ] + Resource: "*" + + RegisterSubnetTags: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterSubnetTagsLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + ec2_client = boto3.client('ec2') + if event['RequestType'] == 'Delete': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.delete_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName']}]); + elif event['RequestType'] == 'Create': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.create_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName'], 'Value': 'shared'}]); + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['InfrastructureName']+event['ResourceProperties']['Subnets'][0]) + Runtime: "python3.7" + Timeout: 120 + + RegisterPublicSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PublicSubnets + + RegisterPrivateSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PrivateSubnets + +Outputs: +# PrivateHostedZoneId: +# Description: Hosted zone ID for the private DNS, which is required for private records. +# Value: !Ref IntDns +# ExternalApiLoadBalancerName: +# Description: Full name of the External API load balancer created. +# Value: !GetAtt ExtApiElb.LoadBalancerFullName + InternalApiLoadBalancerName: + Description: Full name of the Internal API load balancer created. + Value: !GetAtt IntApiElb.LoadBalancerFullName + ApiServerDnsName: + Description: Full hostname of the API server, which is required for the Ignition config files. + Value: !Join [".", ["api-int", !Ref ClusterName, !Ref HostedZoneName]] + RegisterNlbIpTargetsLambda: + Description: Lambda ARN useful to help register or deregister IP targets for these load balancers. + Value: !GetAtt RegisterNlbIpTargets.Arn +# ExternalApiTargetGroupArn: +# Description: ARN of External API target group. +# Value: !Ref ExternalApiTargetGroup + InternalApiTargetGroupArn: + Description: ARN of Internal API target group. + Value: !Ref InternalApiTargetGroup + InternalServiceTargetGroupArn: + Description: ARN of internal service target group. + Value: !Ref InternalServiceTargetGroup diff --git a/files/cloudformation/03_cluster_security.yaml b/files/cloudformation/03_cluster_security.yaml new file mode 100644 index 0000000..f229715 --- /dev/null +++ b/files/cloudformation/03_cluster_security.yaml @@ -0,0 +1,339 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Security Elements (Security Groups & IAM) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag cloud resources and identify items owned or used by the cluster. + Type: String + VpcCidr: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-4]))$ + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-24. + Default: 10.0.0.0/16 + Description: CIDR block for VPC. + Type: String + VpcId: + Description: The VPC-scoped resources will belong to this VPC. + Type: AWS::EC2::VPC::Id + PrivateSubnets: + Description: The internal subnets. + Type: List + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - InfrastructureName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - VpcCidr + - PrivateSubnets + ParameterLabels: + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + VpcCidr: + default: "VPC CIDR" + PrivateSubnets: + default: "Private Subnets" + +Resources: + MasterSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupName: !Join ["-", ["master-sg", !Ref InfrastructureName]] + GroupDescription: Cluster Master Security Group + SecurityGroupIngress: + - IpProtocol: icmp + FromPort: 0 + ToPort: 0 + CidrIp: !Ref VpcCidr + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCidr + - IpProtocol: tcp + ToPort: 6443 + FromPort: 6443 + CidrIp: !Ref VpcCidr + - IpProtocol: tcp + FromPort: 22623 + ToPort: 22623 + CidrIp: !Ref VpcCidr + VpcId: !Ref VpcId + + WorkerSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupName: !Join ["-", ["worker-sg", !Ref InfrastructureName]] + GroupDescription: Cluster Worker Security Group + SecurityGroupIngress: + - IpProtocol: icmp + FromPort: 0 + ToPort: 0 + CidrIp: !Ref VpcCidr + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCidr + VpcId: !Ref VpcId + + MasterIngressEtcd: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: etcd + FromPort: 2379 + ToPort: 2380 + IpProtocol: tcp + + MasterIngressVxlan: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Vxlan packets + FromPort: 4789 + ToPort: 4789 + IpProtocol: udp + + MasterIngressWorkerVxlan: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Vxlan packets + FromPort: 4789 + ToPort: 4789 + IpProtocol: udp + + MasterIngressInternal: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Internal cluster communication + FromPort: 9000 + ToPort: 9999 + IpProtocol: tcp + + MasterIngressWorkerInternal: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Internal cluster communication + FromPort: 9000 + ToPort: 9999 + IpProtocol: tcp + + MasterIngressKube: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Kubernetes kubelet, scheduler and controller manager + FromPort: 10250 + ToPort: 10259 + IpProtocol: tcp + + MasterIngressWorkerKube: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Kubernetes kubelet, scheduler and controller manager + FromPort: 10250 + ToPort: 10259 + IpProtocol: tcp + + MasterIngressIngressServices: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Kubernetes ingress services + FromPort: 30000 + ToPort: 32767 + IpProtocol: tcp + + MasterIngressWorkerIngressServices: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Kubernetes ingress services + FromPort: 30000 + ToPort: 32767 + IpProtocol: tcp + + WorkerIngressVxlan: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Vxlan packets + FromPort: 4789 + ToPort: 4789 + IpProtocol: udp + + WorkerIngressWorkerVxlan: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Vxlan packets + FromPort: 4789 + ToPort: 4789 + IpProtocol: udp + + WorkerIngressInternal: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Internal cluster communication + FromPort: 9000 + ToPort: 9999 + IpProtocol: tcp + + WorkerIngressWorkerInternal: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Internal cluster communication + FromPort: 9000 + ToPort: 9999 + IpProtocol: tcp + + WorkerIngressKube: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Kubernetes secure kubelet port + FromPort: 10250 + ToPort: 10250 + IpProtocol: tcp + + WorkerIngressWorkerKube: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Internal Kubernetes communication + FromPort: 10250 + ToPort: 10250 + IpProtocol: tcp + + WorkerIngressIngressServices: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Kubernetes ingress services + FromPort: 30000 + ToPort: 32767 + IpProtocol: tcp + + WorkerIngressWorkerIngressServices: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Kubernetes ingress services + FromPort: 30000 + ToPort: 32767 + IpProtocol: tcp + + MasterIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "master", "role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "ec2.amazonaws.com" + Action: + - "sts:AssumeRole" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "master", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "ec2:*" + Resource: "*" + - Effect: "Allow" + Action: "elasticloadbalancing:*" + Resource: "*" + - Effect: "Allow" + Action: "iam:PassRole" + Resource: "*" + - Effect: "Allow" + Action: "s3:GetObject" + Resource: "*" + + MasterInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Roles: + - Ref: "MasterIamRole" + + WorkerIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "worker", "role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "ec2.amazonaws.com" + Action: + - "sts:AssumeRole" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "worker", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "ec2:Describe*" + Resource: "*" + + WorkerInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Roles: + - Ref: "WorkerIamRole" + +Outputs: + MasterSecurityGroupId: + Description: Master Security Group ID + Value: !GetAtt MasterSecurityGroup.GroupId + + WorkerSecurityGroupId: + Description: Worker Security Group ID + Value: !GetAtt WorkerSecurityGroup.GroupId + + MasterInstanceProfile: + Description: Master IAM Instance Profile + Value: !Ref MasterInstanceProfile + + WorkerInstanceProfile: + Description: Worker IAM Instance Profile + Value: !Ref WorkerInstanceProfile diff --git a/files/cloudformation/04_cluster_bootstrap.yaml b/files/cloudformation/04_cluster_bootstrap.yaml new file mode 100644 index 0000000..9dad64d --- /dev/null +++ b/files/cloudformation/04_cluster_bootstrap.yaml @@ -0,0 +1,216 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Bootstrap (EC2 Instance, Security Groups and IAM) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag cloud resources and identify items owned or used by the cluster. + Type: String + RhcosAmi: + Description: Current Red Hat Enterprise Linux CoreOS AMI to use for boostrap. + Type: AWS::EC2::Image::Id + AllowedBootstrapSshCidr: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|1[0-9]|2[0-9]|3[0-2]))$ + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/0-32. + Default: 0.0.0.0/0 + Description: CIDR block to allow SSH access to the bootstrap node. + Type: String + PublicSubnet: + Description: The public subnet to launch the bootstrap node into. + Type: AWS::EC2::Subnet::Id + MasterSecurityGroupId: + Description: The master security group ID for registering temporary rules. + Type: AWS::EC2::SecurityGroup::Id + VpcId: + Description: The VPC-scoped resources will belong to this VPC. + Type: AWS::EC2::VPC::Id + BootstrapIgnitionLocation: + Default: s3://my-s3-bucket/bootstrap.ign + Description: Ignition config file location. + Type: String + AutoRegisterELB: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke NLB registration, which requires a Lambda ARN parameter? + Type: String + RegisterNlbIpTargetsLambdaArn: + Description: ARN for NLB IP target registration lambda. + Type: String + ExternalApiTargetGroupArn: + Description: ARN for external API load balancer target group. + Type: String + InternalApiTargetGroupArn: + Description: ARN for internal API load balancer target group. + Type: String + InternalServiceTargetGroupArn: + Description: ARN for internal service load balancer target group. + Type: String + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - InfrastructureName + - Label: + default: "Host Information" + Parameters: + - RhcosAmi + - BootstrapIgnitionLocation + - MasterSecurityGroupId + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - AllowedBootstrapSshCidr + - PublicSubnet + - Label: + default: "Load Balancer Automation" + Parameters: + - AutoRegisterELB + - RegisterNlbIpTargetsLambdaArn + - ExternalApiTargetGroupArn + - InternalApiTargetGroupArn + - InternalServiceTargetGroupArn + ParameterLabels: + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + AllowedBootstrapSshCidr: + default: "Allowed SSH Source" + PublicSubnet: + default: "Public Subnet" + RhcosAmi: + default: "Red Hat Enterprise Linux CoreOS AMI ID" + BootstrapIgnitionLocation: + default: "Bootstrap Ignition Source" + MasterSecurityGroupId: + default: "Master Security Group ID" + AutoRegisterELB: + default: "Use Provided ELB Automation" + +Conditions: + DoRegistration: !Equals ["yes", !Ref AutoRegisterELB] + +Resources: + BootstrapIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "bootstrap", "role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "ec2.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "bootstrap", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "ec2:Describe*" + Resource: "*" + - Effect: "Allow" + Action: "ec2:AttachVolume" + Resource: "*" + - Effect: "Allow" + Action: "ec2:DetachVolume" + Resource: "*" + - Effect: "Allow" + Action: "s3:GetObject" + Resource: "*" + + BootstrapInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Path: "/" + Roles: + - Ref: "BootstrapIamRole" + + BootstrapSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupName: !Join ["-", ["bootstrap-sg", !Ref InfrastructureName]] + GroupDescription: Cluster Bootstrap Security Group + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref AllowedBootstrapSshCidr + - IpProtocol: tcp + ToPort: 19531 + FromPort: 19531 + CidrIp: 0.0.0.0/0 + VpcId: !Ref VpcId + + BootstrapInstance: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + IamInstanceProfile: !Ref BootstrapInstanceProfile + InstanceType: "i3.large" + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - !Ref "BootstrapSecurityGroup" + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "PublicSubnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"replace":{"source":"${S3Loc}","verification":{}}},"timeouts":{},"version":"2.1.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + S3Loc: !Ref BootstrapIgnitionLocation + } + Tags: + - Key: "Name" + Value: !Join ["-", [!Ref InfrastructureName, "bootstrap"]] + + RegisterBootstrapApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref ExternalApiTargetGroupArn + TargetIp: !GetAtt BootstrapInstance.PrivateIp + + RegisterBootstrapInternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt BootstrapInstance.PrivateIp + + RegisterBootstrapInternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt BootstrapInstance.PrivateIp + +Outputs: + BootstrapInstanceId: + Description: Bootstrap Instance ID. + Value: !Ref BootstrapInstance + + BootstrapPublicIp: + Description: The bootstrap node public IP address. + Value: !GetAtt BootstrapInstance.PublicIp + + BootstrapPrivateIp: + Description: The bootstrap node private IP address. + Value: !GetAtt BootstrapInstance.PrivateIp diff --git a/files/cloudformation/04_cluster_bootstrap.yaml.internalonly b/files/cloudformation/04_cluster_bootstrap.yaml.internalonly new file mode 100644 index 0000000..c76598a --- /dev/null +++ b/files/cloudformation/04_cluster_bootstrap.yaml.internalonly @@ -0,0 +1,214 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Bootstrap (EC2 Instance, Security Groups and IAM) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag cloud resources and identify items owned or used by the cluster. + Type: String + RhcosAmi: + Description: Current Red Hat Enterprise Linux CoreOS AMI to use for boostrap. + Type: AWS::EC2::Image::Id + AllowedBootstrapSshCidr: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|1[0-9]|2[0-9]|3[0-2]))$ + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/0-32. + Default: 0.0.0.0/0 + Description: CIDR block to allow SSH access to the bootstrap node. + Type: String + PublicSubnet: + Description: The public subnet to launch the bootstrap node into. + Type: AWS::EC2::Subnet::Id + MasterSecurityGroupId: + Description: The master security group ID for registering temporary rules. + Type: AWS::EC2::SecurityGroup::Id + VpcId: + Description: The VPC-scoped resources will belong to this VPC. + Type: AWS::EC2::VPC::Id + BootstrapIgnitionLocation: + Default: s3://my-s3-bucket/bootstrap.ign + Description: Ignition config file location. + Type: String + AutoRegisterELB: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke NLB registration, which requires a Lambda ARN parameter? + Type: String + RegisterNlbIpTargetsLambdaArn: + Description: ARN for NLB IP target registration lambda. + Type: String +# ExternalApiTargetGroupArn: +# Description: ARN for external API load balancer target group. +# Type: String + InternalApiTargetGroupArn: + Description: ARN for internal API load balancer target group. + Type: String + InternalServiceTargetGroupArn: + Description: ARN for internal service load balancer target group. + Type: String + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - InfrastructureName + - Label: + default: "Host Information" + Parameters: + - RhcosAmi + - BootstrapIgnitionLocation + - MasterSecurityGroupId + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - AllowedBootstrapSshCidr + - PublicSubnet + - Label: + default: "Load Balancer Automation" + Parameters: + - AutoRegisterELB + - RegisterNlbIpTargetsLambdaArn +# - ExternalApiTargetGroupArn + - InternalApiTargetGroupArn + - InternalServiceTargetGroupArn + ParameterLabels: + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + AllowedBootstrapSshCidr: + default: "Allowed SSH Source" + PublicSubnet: + default: "Public Subnet" + RhcosAmi: + default: "Red Hat Enterprise Linux CoreOS AMI ID" + BootstrapIgnitionLocation: + default: "Bootstrap Ignition Source" + MasterSecurityGroupId: + default: "Master Security Group ID" + AutoRegisterELB: + default: "Use Provided ELB Automation" + +Conditions: + DoRegistration: !Equals ["yes", !Ref AutoRegisterELB] + +Resources: + BootstrapIamRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "ec2.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "bootstrap", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "ec2:Describe*" + Resource: "*" + - Effect: "Allow" + Action: "ec2:AttachVolume" + Resource: "*" + - Effect: "Allow" + Action: "ec2:DetachVolume" + Resource: "*" + - Effect: "Allow" + Action: "s3:GetObject" + Resource: "*" + + BootstrapInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Path: "/" + Roles: + - Ref: "BootstrapIamRole" + + BootstrapSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Cluster Bootstrap Security Group + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref AllowedBootstrapSshCidr + - IpProtocol: tcp + ToPort: 19531 + FromPort: 19531 + CidrIp: 0.0.0.0/0 + VpcId: !Ref VpcId + + BootstrapInstance: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + IamInstanceProfile: !Ref BootstrapInstanceProfile + InstanceType: "i3.large" + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - !Ref "BootstrapSecurityGroup" + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "PublicSubnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"replace":{"source":"${S3Loc}","verification":{}}},"timeouts":{},"version":"2.1.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + S3Loc: !Ref BootstrapIgnitionLocation + } + Tags: + - Key: "Name" + Value: !Join ["-", [!Ref InfrastructureName, "bootstrap"]] + +# RegisterBootstrapApiTarget: +# Condition: DoRegistration +# Type: Custom::NLBRegister +# Properties: +# ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn +# TargetArn: !Ref ExternalApiTargetGroupArn +# TargetIp: !GetAtt BootstrapInstance.PrivateIp + + RegisterBootstrapInternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt BootstrapInstance.PrivateIp + + RegisterBootstrapInternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt BootstrapInstance.PrivateIp + +Outputs: + BootstrapInstanceId: + Description: Bootstrap Instance ID. + Value: !Ref BootstrapInstance + + BootstrapPublicIp: + Description: The bootstrap node public IP address. + Value: !GetAtt BootstrapInstance.PublicIp + + BootstrapPrivateIp: + Description: The bootstrap node private IP address. + Value: !GetAtt BootstrapInstance.PrivateIp diff --git a/files/cloudformation/05_cluster_master_nodes.yaml b/files/cloudformation/05_cluster_master_nodes.yaml new file mode 100644 index 0000000..8d387b4 --- /dev/null +++ b/files/cloudformation/05_cluster_master_nodes.yaml @@ -0,0 +1,391 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Node Launch (EC2 master instances) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag nodes for the kubelet cloud provider. + Type: String + RhcosAmi: + Description: Current Red Hat Enterprise Linux CoreOS AMI to use for boostrap. + Type: AWS::EC2::Image::Id + AutoRegisterDNS: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke DNS etcd registration, which requires Hosted Zone information? + Type: String + PrivateHostedZoneId: + Description: The Route53 private zone ID to register the etcd targets with, such as Z21IXYZABCZ2A4. + Type: String + PrivateHostedZoneName: + Description: The Route53 zone to register the targets with, such as cluster.example.com. Omit the trailing period. + Type: String + Master0Subnet: + Description: The subnets, recommend private, to launch the master nodes into. + Type: AWS::EC2::Subnet::Id + Master1Subnet: + Description: The subnets, recommend private, to launch the master nodes into. + Type: AWS::EC2::Subnet::Id + Master2Subnet: + Description: The subnets, recommend private, to launch the master nodes into. + Type: AWS::EC2::Subnet::Id + MasterSecurityGroupId: + Description: The master security group ID to associate with master nodes. + Type: AWS::EC2::SecurityGroup::Id + IgnitionLocation: + Default: https://api-int.$CLUSTER_NAME.$DOMAIN:22623/config/master + Description: Ignition config file location. + Type: String + CertificateAuthorities: + Default: data:text/plain;charset=utf-8;base64,ABC...xYz== + Description: Base64 encoded certificate authority string to use. + Type: String + MasterInstanceProfileName: + Description: IAM profile to associate with master nodes. + Type: String + MasterInstanceType: + Default: m4.xlarge + Type: String + AllowedValues: + - "m4.xlarge" + - "m4.2xlarge" + - "m4.4xlarge" + - "m4.8xlarge" + - "m4.10xlarge" + - "m4.16xlarge" + - "c4.2xlarge" + - "c4.4xlarge" + - "c4.8xlarge" + - "r4.xlarge" + - "r4.2xlarge" + - "r4.4xlarge" + - "r4.8xlarge" + - "r4.16xlarge" + AutoRegisterELB: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke NLB registration, which requires a Lambda ARN parameter? + Type: String + RegisterNlbIpTargetsLambdaArn: + Description: ARN for NLB IP target registration lambda. Supply the value from the cluster infrastructure or select "no" for AutoRegisterELB. + Type: String + ExternalApiTargetGroupArn: + Description: ARN for external API load balancer target group. Supply the value from the cluster infrastructure or select "no" for AutoRegisterELB. + Type: String + InternalApiTargetGroupArn: + Description: ARN for internal API load balancer target group. Supply the value from the cluster infrastructure or select "no" for AutoRegisterELB. + Type: String + InternalServiceTargetGroupArn: + Description: ARN for internal service load balancer target group. Supply the value from the cluster infrastructure or select "no" for AutoRegisterELB. + Type: String + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - InfrastructureName + - Label: + default: "Host Information" + Parameters: + - MasterInstanceType + - RhcosAmi + - IgnitionLocation + - CertificateAuthorities + - MasterSecurityGroupId + - MasterInstanceProfileName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - AllowedBootstrapSshCidr + - Master0Subnet + - Master1Subnet + - Master2Subnet + - Label: + default: "DNS" + Parameters: + - AutoRegisterDNS + - PrivateHostedZoneName + - PrivateHostedZoneId + - Label: + default: "Load Balancer Automation" + Parameters: + - AutoRegisterELB + - RegisterNlbIpTargetsLambdaArn + - ExternalApiTargetGroupArn + - InternalApiTargetGroupArn + - InternalServiceTargetGroupArn + ParameterLabels: + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + Master0Subnet: + default: "Master-0 Subnet" + Master1Subnet: + default: "Master-1 Subnet" + Master2Subnet: + default: "Master-2 Subnet" + MasterInstanceType: + default: "Master Instance Type" + MasterInstanceProfileName: + default: "Master Instance Profile Name" + RhcosAmi: + default: "Red Hat Enterprise Linux CoreOS AMI ID" + BootstrapIgnitionLocation: + default: "Master Ignition Source" + CertificateAuthorities: + default: "Ignition CA String" + MasterSecurityGroupId: + default: "Master Security Group ID" + AutoRegisterDNS: + default: "Use Provided DNS Automation" + AutoRegisterELB: + default: "Use Provided ELB Automation" + PrivateHostedZoneName: + default: "Private Hosted Zone Name" + PrivateHostedZoneId: + default: "Private Hosted Zone ID" + +Conditions: + DoRegistration: !Equals ["yes", !Ref AutoRegisterELB] + DoDns: !Equals ["yes", !Ref AutoRegisterDNS] + +Resources: + Master0: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "200" + VolumeType: "io1" + Iops: "4000" + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master0Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: "Name" + Value: !Join ["-", [!Ref InfrastructureName, "master", "0"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + + RegisterMaster0: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref ExternalApiTargetGroupArn + TargetIp: !GetAtt Master0.PrivateIp + + RegisterMaster0InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master0.PrivateIp + + RegisterMaster0InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master0.PrivateIp + + Master1: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "200" + VolumeType: "io1" + Iops: "4000" + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master1Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: "Name" + Value: !Join ["-", [!Ref InfrastructureName, "master", "1"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + + RegisterMaster1: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref ExternalApiTargetGroupArn + TargetIp: !GetAtt Master1.PrivateIp + + RegisterMaster1InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master1.PrivateIp + + RegisterMaster1InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master1.PrivateIp + + Master2: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "200" + VolumeType: "io1" + Iops: "4000" + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master2Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: "Name" + Value: !Join ["-", [!Ref InfrastructureName, "master", "2"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + + RegisterMaster2: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref ExternalApiTargetGroupArn + TargetIp: !GetAtt Master2.PrivateIp + + RegisterMaster2InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master2.PrivateIp + + RegisterMaster2InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master2.PrivateIp + + EtcdSrvRecords: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["_etcd-server-ssl._tcp", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-0", !Ref PrivateHostedZoneName]]], + ] + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-1", !Ref PrivateHostedZoneName]]], + ] + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-2", !Ref PrivateHostedZoneName]]], + ] + TTL: 60 + Type: SRV + + Etcd0Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-0", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master0.PrivateIp + TTL: 60 + Type: A + + Etcd1Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-1", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master1.PrivateIp + TTL: 60 + Type: A + + Etcd2Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-2", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master2.PrivateIp + TTL: 60 + Type: A + +Outputs: + PrivateIPs: + Description: The control-plane node private IP addresses. + Value: + !Join [ + ",", + [!GetAtt Master0.PrivateIp, !GetAtt Master1.PrivateIp, !GetAtt Master2.PrivateIp] + ] diff --git a/files/cloudformation/05_cluster_master_nodes.yaml.internalonly b/files/cloudformation/05_cluster_master_nodes.yaml.internalonly new file mode 100644 index 0000000..aadacb5 --- /dev/null +++ b/files/cloudformation/05_cluster_master_nodes.yaml.internalonly @@ -0,0 +1,391 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Node Launch (EC2 master instances) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag nodes for the kubelet cloud provider. + Type: String + RhcosAmi: + Description: Current Red Hat Enterprise Linux CoreOS AMI to use for boostrap. + Type: AWS::EC2::Image::Id + AutoRegisterDNS: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke DNS etcd registration, which requires Hosted Zone information? + Type: String + PrivateHostedZoneId: + Description: The Route53 private zone ID to register the etcd targets with, such as Z21IXYZABCZ2A4. + Type: String + PrivateHostedZoneName: + Description: The Route53 zone to register the targets with, such as cluster.example.com. Omit the trailing period. + Type: String + Master0Subnet: + Description: The subnets, recommend private, to launch the master nodes into. + Type: AWS::EC2::Subnet::Id + Master1Subnet: + Description: The subnets, recommend private, to launch the master nodes into. + Type: AWS::EC2::Subnet::Id + Master2Subnet: + Description: The subnets, recommend private, to launch the master nodes into. + Type: AWS::EC2::Subnet::Id + MasterSecurityGroupId: + Description: The master security group ID to associate with master nodes. + Type: AWS::EC2::SecurityGroup::Id + IgnitionLocation: + Default: https://api-int.$CLUSTER_NAME.$DOMAIN:22623/config/master + Description: Ignition config file location. + Type: String + CertificateAuthorities: + Default: data:text/plain;charset=utf-8;base64,ABC...xYz== + Description: Base64 encoded certificate authority string to use. + Type: String + MasterInstanceProfileName: + Description: IAM profile to associate with master nodes. + Type: String + MasterInstanceType: + Default: m4.xlarge + Type: String + AllowedValues: + - "m4.xlarge" + - "m4.2xlarge" + - "m4.4xlarge" + - "m4.8xlarge" + - "m4.10xlarge" + - "m4.16xlarge" + - "c4.2xlarge" + - "c4.4xlarge" + - "c4.8xlarge" + - "r4.xlarge" + - "r4.2xlarge" + - "r4.4xlarge" + - "r4.8xlarge" + - "r4.16xlarge" + AutoRegisterELB: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke NLB registration, which requires a Lambda ARN parameter? + Type: String + RegisterNlbIpTargetsLambdaArn: + Description: ARN for NLB IP target registration lambda. Supply the value from the cluster infrastructure or select "no" for AutoRegisterELB. + Type: String +# ExternalApiTargetGroupArn: +# Description: ARN for external API load balancer target group. Supply the value from the cluster infrastructure or select "no" for AutoRegisterELB. +# Type: String + InternalApiTargetGroupArn: + Description: ARN for internal API load balancer target group. Supply the value from the cluster infrastructure or select "no" for AutoRegisterELB. + Type: String + InternalServiceTargetGroupArn: + Description: ARN for internal service load balancer target group. Supply the value from the cluster infrastructure or select "no" for AutoRegisterELB. + Type: String + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - InfrastructureName + - Label: + default: "Host Information" + Parameters: + - MasterInstanceType + - RhcosAmi + - IgnitionLocation + - CertificateAuthorities + - MasterSecurityGroupId + - MasterInstanceProfileName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - AllowedBootstrapSshCidr + - Master0Subnet + - Master1Subnet + - Master2Subnet + - Label: + default: "DNS" + Parameters: + - AutoRegisterDNS + - PrivateHostedZoneName + - PrivateHostedZoneId + - Label: + default: "Load Balancer Automation" + Parameters: + - AutoRegisterELB + - RegisterNlbIpTargetsLambdaArn +# - ExternalApiTargetGroupArn + - InternalApiTargetGroupArn + - InternalServiceTargetGroupArn + ParameterLabels: + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + Master0Subnet: + default: "Master-0 Subnet" + Master1Subnet: + default: "Master-1 Subnet" + Master2Subnet: + default: "Master-2 Subnet" + MasterInstanceType: + default: "Master Instance Type" + MasterInstanceProfileName: + default: "Master Instance Profile Name" + RhcosAmi: + default: "Red Hat Enterprise Linux CoreOS AMI ID" + BootstrapIgnitionLocation: + default: "Master Ignition Source" + CertificateAuthorities: + default: "Ignition CA String" + MasterSecurityGroupId: + default: "Master Security Group ID" + AutoRegisterDNS: + default: "Use Provided DNS Automation" + AutoRegisterELB: + default: "Use Provided ELB Automation" + PrivateHostedZoneName: + default: "Private Hosted Zone Name" + PrivateHostedZoneId: + default: "Private Hosted Zone ID" + +Conditions: + DoRegistration: !Equals ["yes", !Ref AutoRegisterELB] + DoDns: !Equals ["yes", !Ref AutoRegisterDNS] + +Resources: + Master0: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "200" + VolumeType: "io1" + Iops: "4000" + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master0Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: "Name" + Value: !Join ["-", [!Ref InfrastructureName, "master", "0"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + +# RegisterMaster0: +# Condition: DoRegistration +# Type: Custom::NLBRegister +# Properties: +# ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn +# TargetArn: !Ref ExternalApiTargetGroupArn +# TargetIp: !GetAtt Master0.PrivateIp + + RegisterMaster0InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master0.PrivateIp + + RegisterMaster0InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master0.PrivateIp + + Master1: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "200" + VolumeType: "io1" + Iops: "4000" + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master1Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: "Name" + Value: !Join ["-", [!Ref InfrastructureName, "master", "1"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + +# RegisterMaster1: +# Condition: DoRegistration +# Type: Custom::NLBRegister +# Properties: +# ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn +# TargetArn: !Ref ExternalApiTargetGroupArn +# TargetIp: !GetAtt Master1.PrivateIp + + RegisterMaster1InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master1.PrivateIp + + RegisterMaster1InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master1.PrivateIp + + Master2: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "200" + VolumeType: "io1" + Iops: "4000" + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master2Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: "Name" + Value: !Join ["-", [!Ref InfrastructureName, "master", "2"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + +# RegisterMaster2: +# Condition: DoRegistration +# Type: Custom::NLBRegister +# Properties: +# ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn +# TargetArn: !Ref ExternalApiTargetGroupArn +# TargetIp: !GetAtt Master2.PrivateIp + + RegisterMaster2InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master2.PrivateIp + + RegisterMaster2InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master2.PrivateIp + + EtcdSrvRecords: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["_etcd-server-ssl._tcp", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-0", !Ref PrivateHostedZoneName]]], + ] + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-1", !Ref PrivateHostedZoneName]]], + ] + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-2", !Ref PrivateHostedZoneName]]], + ] + TTL: 60 + Type: SRV + + Etcd0Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-0", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master0.PrivateIp + TTL: 60 + Type: A + + Etcd1Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-1", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master1.PrivateIp + TTL: 60 + Type: A + + Etcd2Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-2", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master2.PrivateIp + TTL: 60 + Type: A + +Outputs: + PrivateIPs: + Description: The control-plane node private IP addresses. + Value: + !Join [ + ",", + [!GetAtt Master0.PrivateIp, !GetAtt Master1.PrivateIp, !GetAtt Master2.PrivateIp] + ] diff --git a/files/cloudformation/06_cluster_worker_node.yaml b/files/cloudformation/06_cluster_worker_node.yaml new file mode 100644 index 0000000..d4e6f21 --- /dev/null +++ b/files/cloudformation/06_cluster_worker_node.yaml @@ -0,0 +1,125 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for OpenShift Cluster Node Launch (EC2 worker instance) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,26})$ + MaxLength: 27 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter, and have a maximum of 27 characters. + Description: A short, unique cluster ID used to tag nodes for the kubelet cloud provider. + Type: String + RhcosAmi: + Description: Current Red Hat Enterprise Linux CoreOS AMI to use for boostrap. + Type: AWS::EC2::Image::Id + Subnet: + Description: The subnets, recommend private, to launch the master nodes into. + Type: AWS::EC2::Subnet::Id + WorkerSecurityGroupId: + Description: The master security group ID to associate with master nodes. + Type: AWS::EC2::SecurityGroup::Id + IgnitionLocation: + Default: https://api-int.$CLUSTER_NAME.$DOMAIN:22623/config/worker + Description: Ignition config file location. + Type: String + CertificateAuthorities: + Default: data:text/plain;charset=utf-8;base64,ABC...xYz== + Description: Base64 encoded certificate authority string to use. + Type: String + WorkerInstanceProfileName: + Description: IAM profile to associate with master nodes. + Type: String + WorkerInstanceType: + Default: m4.large + Type: String + AllowedValues: + - "m4.large" + - "m4.xlarge" + - "m4.2xlarge" + - "m4.4xlarge" + - "m4.8xlarge" + - "m4.10xlarge" + - "m4.16xlarge" + - "c4.large" + - "c4.xlarge" + - "c4.2xlarge" + - "c4.4xlarge" + - "c4.8xlarge" + - "r4.large" + - "r4.xlarge" + - "r4.2xlarge" + - "r4.4xlarge" + - "r4.8xlarge" + - "r4.16xlarge" + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - InfrastructureName + - Label: + default: "Host Information" + Parameters: + - WorkerInstanceType + - RhcosAmi + - IgnitionLocation + - CertificateAuthorities + - WorkerSecurityGroupId + - WorkerInstanceProfileName + - Label: + default: "Network Configuration" + Parameters: + - Subnet + ParameterLabels: + Subnet: + default: "Subnet" + InfrastructureName: + default: "Infrastructure Name" + WorkerInstanceType: + default: "Worker Instance Type" + WorkerInstanceProfileName: + default: "Worker Instance Profile Name" + RhcosAmi: + default: "Red Hat Enterprise Linux CoreOS AMI ID" + IgnitionLocation: + default: "Worker Ignition Source" + CertificateAuthorities: + default: "Ignition CA String" + WorkerSecurityGroupId: + default: "Worker Security Group ID" + +Resources: + Worker0: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: "120" + VolumeType: "gp2" + IamInstanceProfile: !Ref WorkerInstanceProfileName + InstanceType: !Ref WorkerInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "WorkerSecurityGroupId" + SubnetId: !Ref "Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + +Outputs: + PrivateIP: + Description: The compute node private IP address. + Value: !GetAtt Worker0.PrivateIp diff --git a/files/cluster-logging/cleanup-logging.sh b/files/cluster-logging/cleanup-logging.sh new file mode 100755 index 0000000..0440b63 --- /dev/null +++ b/files/cluster-logging/cleanup-logging.sh @@ -0,0 +1,7 @@ +#!/bin/bash +oc delete OperatorGroup/openshift-operators-redhat -n openshift-operators-redhat +oc delete project/openshift-operators-redhat +oc delete project/openshift-logging +oc delete CatalogSourceConfig/installed-redhat-openshift-logging -n openshift-marketplace +oc delete CatalogSourceConfig/elasticsearch -n openshift-marketplace +oc delete ClusterServiceVersion/$(oc get ClusterServiceVersion -n openshift-operator-lifecycle-manager | awk '{print $1}' | grep elasticsearch-operator) -n openshift-operator-lifecycle-manager diff --git a/files/cluster-logging/clo-namespace.yaml b/files/cluster-logging/clo-namespace.yaml new file mode 100644 index 0000000..08b3aa9 --- /dev/null +++ b/files/cluster-logging/clo-namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-logging + annotations: + openshift.io/node-selector: "" + labels: + openshift.io/cluster-logging: "true" + openshift.io/cluster-monitoring: "true" diff --git a/files/cluster-logging/cluster-logging-csc.cr.yaml b/files/cluster-logging/cluster-logging-csc.cr.yaml new file mode 100644 index 0000000..12044af --- /dev/null +++ b/files/cluster-logging/cluster-logging-csc.cr.yaml @@ -0,0 +1,12 @@ +apiVersion: operators.coreos.com/v1 +kind: CatalogSourceConfig +metadata: + finalizers: + - finalizer.catalogsourceconfigs.operators.coreos.com + name: installed-redhat-openshift-logging + namespace: openshift-marketplace +spec: + csDisplayName: Red Hat Operators + csPublisher: Red Hat + packages: cluster-logging + targetNamespace: openshift-logging diff --git a/files/cluster-logging/cluster-logging-og.yaml b/files/cluster-logging/cluster-logging-og.yaml new file mode 100644 index 0000000..052f503 --- /dev/null +++ b/files/cluster-logging/cluster-logging-og.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + generateName: openshift-logging- + name: openshift-logging + namespace: openshift-logging +spec: + serviceAccount: + metadata: + targetNamespaces: + - openshift-logging diff --git a/files/cluster-logging/cluster-logging-resourse.yaml b/files/cluster-logging/cluster-logging-resourse.yaml new file mode 100644 index 0000000..f5efaf1 --- /dev/null +++ b/files/cluster-logging/cluster-logging-resourse.yaml @@ -0,0 +1,27 @@ +apiVersion: "logging.openshift.io/v1" +kind: "ClusterLogging" +metadata: + name: "instance" + namespace: "openshift-logging" +spec: + managementState: "Managed" + logStore: + type: "elasticsearch" + elasticsearch: + nodeCount: 3 + storage: + storageClassName: gp2 + size: 10G + redundancyPolicy: "SingleRedundancy" + visualization: + type: "kibana" + kibana: + replicas: 1 + curation: + type: "curator" + curator: + schedule: "30 3 * * *" + collection: + logs: + type: "fluentd" + fluentd: {} diff --git a/files/cluster-logging/cluster-logging-sub.yaml b/files/cluster-logging/cluster-logging-sub.yaml new file mode 100644 index 0000000..6a5b6b6 --- /dev/null +++ b/files/cluster-logging/cluster-logging-sub.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: cluster-logging + namespace: openshift-logging +spec: + channel: preview + installPlanApproval: Automatic + name: cluster-logging + source: installed-redhat-openshift-logging + sourceNamespace: openshift-logging diff --git a/files/cluster-logging/cluster-monitoring-config.yml b/files/cluster-logging/cluster-monitoring-config.yml new file mode 100644 index 0000000..f382823 --- /dev/null +++ b/files/cluster-logging/cluster-monitoring-config.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + prometheusK8s: + volumeClaimTemplate: + spec: + storageClassName: gp2 + resources: + requests: + storage: 20Gi + retention: 7d diff --git a/files/cluster-logging/eo-csc.yaml b/files/cluster-logging/eo-csc.yaml new file mode 100644 index 0000000..0dfc95b --- /dev/null +++ b/files/cluster-logging/eo-csc.yaml @@ -0,0 +1,8 @@ +apiVersion: "operators.coreos.com/v1" +kind: "CatalogSourceConfig" +metadata: + name: "elasticsearch" + namespace: "openshift-marketplace" +spec: + targetNamespace: "openshift-operators-redhat" + packages: "elasticsearch-operator" diff --git a/files/cluster-logging/eo-namespace.yaml b/files/cluster-logging/eo-namespace.yaml new file mode 100644 index 0000000..f018c77 --- /dev/null +++ b/files/cluster-logging/eo-namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators-redhat + annotations: + openshift.io/node-selector: "" + labels: + openshift.io/cluster-logging: "true" + openshift.io/cluster-monitoring: "true" diff --git a/files/cluster-logging/eo-og.yaml b/files/cluster-logging/eo-og.yaml new file mode 100644 index 0000000..a8372d2 --- /dev/null +++ b/files/cluster-logging/eo-og.yaml @@ -0,0 +1,6 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-operators-redhat + namespace: openshift-operators-redhat +spec: {} diff --git a/files/cluster-logging/eo-rbac.yaml b/files/cluster-logging/eo-rbac.yaml new file mode 100644 index 0000000..28a047b --- /dev/null +++ b/files/cluster-logging/eo-rbac.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus-k8s + namespace: openshift-operators-redhat +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus-k8s + namespace: openshift-operators-redhat +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus-k8s +subjects: +- kind: ServiceAccount + name: prometheus-k8s +namespace: openshift-operators-redhat diff --git a/files/cluster-logging/eo-sub.yaml b/files/cluster-logging/eo-sub.yaml new file mode 100644 index 0000000..4f235e3 --- /dev/null +++ b/files/cluster-logging/eo-sub.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + generateName: "elasticsearch-" + namespace: "openshift-operators-redhat" +spec: + channel: "preview" + installPlanApproval: "Automatic" + source: "elasticsearch" + sourceNamespace: "openshift-operators-redhat" + name: "elasticsearch-operator" diff --git a/files/cluster-logging/logging.sh b/files/cluster-logging/logging.sh new file mode 100755 index 0000000..e28fed1 --- /dev/null +++ b/files/cluster-logging/logging.sh @@ -0,0 +1,21 @@ +#!/bin/bash +oc create -f eo-namespace.yaml +oc create -f clo-namespace.yaml +oc create -f eo-og.yaml +oc create -f eo-csc.yaml +oc create -f eo-sub.yaml +oc create -f eo-rbac.yaml +oc create -f cluster-logging-og.yaml +oc create -f cluster-logging-csc.cr.yaml +oc create -f cluster-logging-sub.yaml +oc create -f cluster-logging-resourse.yaml + +sleep 60 + +oc get csv -n openshift-logging +oc get ip -n openshift-logging +oc get catsrc -n openshift-logging +oc get sub -n openshift-logging +oc get og -n openshift-logging + +oc get deployments -n openshift-logging diff --git a/files/cluster-monitoring-config.yaml.j2 b/files/cluster-monitoring-config.yaml.j2 new file mode 100644 index 0000000..4893eb4 --- /dev/null +++ b/files/cluster-monitoring-config.yaml.j2 @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + prometheusK8s: + volumeClaimTemplate: + spec: + storageClassName: "{{ prometheus_storage_class }}" + resources: + requests: + storage: "{{ prometheus_storage_size }}" + retention: "{{ prometheus_retention_time }}" diff --git a/files/dnses.yaml.j2 b/files/dnses.yaml.j2 new file mode 100644 index 0000000..d7ae154 --- /dev/null +++ b/files/dnses.yaml.j2 @@ -0,0 +1,11 @@ +apiVersion: config.openshift.io/v1 +kind: DNS +metadata: + name: cluster +spec: + baseDomain: {{ clustername }}.{{ publiczonename }} + privateZone: + id: {{ privatezoneid }} + publicZone: + id: {{ publiczoneid }} + diff --git a/files/install-config.yaml.j2 b/files/install-config.yaml.j2 new file mode 100644 index 0000000..651f223 --- /dev/null +++ b/files/install-config.yaml.j2 @@ -0,0 +1,47 @@ +apiVersion: v1 +baseDomain: {{ publiczonename }} +compute: +- name: worker + platform: + aws: + type: {{ worker_instance }} + zones: + - {{ region }}a + - {{ region }}b + - {{ region }}c + replicas: 3 +controlPlane: + name: master + platform: + aws: + type: {{ master_instance }} + zones: + - {{ region }}a + - {{ region }}b + - {{ region }}c + replicas: 3 +metadata: + creationTimestamp: null + name: {{ clustername }} +networking: + clusterNetwork: + - cidr: {{ clustercidr }} + hostPrefix: 23 + machineCIDR: {{ vpccidr }} + networkType: OpenShiftSDN + serviceNetwork: + - {{ servicecidr }} +platform: + aws: + region: {{ region }} +{% if extra_instance_tags %} + userTags: {{ extra_instance_tags | to_yaml }} +{% endif %} +{% if pull_secret is match("^'.*") %} +pullSecret: {{ pull_secret }} +{% else %} +pullSecret: '{{ pull_secret }}' +{% endif %} +{% if ssh_key is defined %} +sshKey: {{ ssh_key }} +{% endif %} diff --git a/files/machinesets.yaml.j2 b/files/machinesets.yaml.j2 new file mode 100644 index 0000000..2ed3837 --- /dev/null +++ b/files/machinesets.yaml.j2 @@ -0,0 +1,60 @@ +apiVersion: machine.openshift.io/v1beta1 +kind: MachineSet +metadata: + labels: + machine.openshift.io/cluster-api-cluster: {{ infraid }} + machine.openshift.io/cluster-api-machine-role: worker + machine.openshift.io/cluster-api-machine-type: worker + name: {{ infraid }}-worker-{{ region }}{{item.key}} + namespace: openshift-machine-api +spec: + replicas: 1 + selector: + matchLabels: + machine.openshift.io/cluster-api-cluster: {{ infraid }} + machine.openshift.io/cluster-api-machineset: {{ infraid }}-worker-{{ region }}{{item.key}} + template: + metadata: + labels: + machine.openshift.io/cluster-api-cluster: {{ infraid }} + machine.openshift.io/cluster-api-machine-role: worker + machine.openshift.io/cluster-api-machine-type: worker + machine.openshift.io/cluster-api-machineset: {{ infraid }}-worker-{{ region }}{{item.key}} + spec: + providerSpec: + value: + ami: + id: "{{ rhcos_ami }}" + apiVersion: awsproviderconfig.openshift.io/v1beta1 + credentialsSecret: + name: aws-cloud-credentials + deviceIndex: 0 + iamInstanceProfile: + id: "{{ cluster_security_stack.stack_outputs.WorkerInstanceProfile }}" + instanceType: m5.2xlarge + kind: AWSMachineProviderConfig + placement: + availabilityZone: {{ region }}{{item.key}} + region: "{{ region }}" + publicIp: null + securityGroups: + - filters: + - name: group-id + values: + - "{{ cluster_security_stack.stack_outputs.WorkerSecurityGroupId }}" + subnet: + filters: + - name: subnet-id + values: + - "{{ privatesubnets.split(',')[item.value] }}" + tags: + - name: kubernetes.io/cluster/{{ infraid }} + value: owned +{% if automatic_shutdown is defined %} + - name: auto_shut_bool + value: "{{ automatic_shutdown }}" +{% endif %} + userDataSecret: + name: worker-user-data + versions: + kubelet: "" diff --git a/install-upi.yaml b/install-upi.yaml new file mode 100644 index 0000000..3ad6702 --- /dev/null +++ b/install-upi.yaml @@ -0,0 +1,318 @@ +--- +- name: Install OpenShift 4 + gather_facts: false + hosts: localhost + environment: + KUBECONFIG: /tmp/{{ clustername }}/auth/kubeconfig + + tasks: + - name: Check if virtualenv is installed + command: virtualenv --version + ignore_errors: yes + changed_when: false + register: virtualenv_cmd + + - name: Install python-virtualenv package + yum: + name: python-virtualenv + state: present + become: yes + when: not virtualenv_cmd is success + + - name: Create virtual Python environment + pip: + name: awscli + virtualenv: "{{ awscli_venv_path }}" + + - import_tasks: tasks/download-ocp-binaries.yaml + + - name: Get encrypted AMI + ec2_ami_facts: + filters: + "tag:rhcos_version": "{{ rhcos_version }}" + "tag:latest_ami": 'true' + region: "{{ region }}" + register: ami_encrypted + + - name: Override AMI ID + set_fact: + rhcos_ami: "{{ ami_encrypted.images[0].image_id }}" + when: ami_encrypted.images + + - import_tasks: tasks/installer-config-directory.yaml + + - name: "Check if /tmp/{{ clustername }}/metadata.json exists" + stat: + path: /tmp/{{ clustername }}/metadata.json + register: stat_result + + - name: render template + template: + src: files/install-config.yaml.j2 + dest: /tmp/{{ clustername }}/install-config.yaml + when: stat_result.stat.exists == False + + - name: Create the cluster ignition configs + command: "{{ openshift_install }} create ignition-configs --dir=/tmp/{{ clustername }}" + args: + creates: /tmp/{{ clustername }}/metadata.json + when: stat_result.stat.exists == False + + - name: read master CA info + command: cat /tmp/{{ clustername }}/master.ign + register: result + + - set_fact: + master_ignition: "{{ result.stdout | from_json }}" + + - import_tasks: tasks/get_infraid.yaml + +# - debug: +# msg: "{{ master_ignition.ignition.security.tls.certificateAuthorities[0].source }}" +# +# - debug: +# msg: "{{ metadata_json.infraID }}" + + - import_tasks: tasks/bootstrap-s3-bucket.yaml + + - name: upload bootstrap.ign to s3 bucket + aws_s3: + bucket: "{{ infraid }}-bootstrap" + object: /bootstrap.ign + src: /tmp/{{ clustername }}/bootstrap.ign + mode: "{{ state | default('put') }}" + retries: 3 + delay: 3 + + - name: Create UPI Network Elements (Route53 & LBs) + cloudformation: + stack_name: "{{ infraid }}-cluster-infra" + state: present + region: "{{ region }}" + template: "files/cloudformation/02_cluster_infra.yaml{% if api_internal_only %}{% if privatezoneid is defined %}.internalonly-existingprivzone{% else %}.internalonly{% endif %}{% elif privatezoneid is defined %}.existingprivzone{% endif %}" + template_parameters: + ClusterName: "{{ clustername }}" + InfrastructureName: "{{ infraid }}" + PrivateSubnets: "{{ privatesubnets}}" + PrivateZoneId: "{{ privatezoneid | default(omit) }}" + PublicSubnets: "{{ publicsubnets }}" + HostedZoneId: "{{ publiczoneid }}" + HostedZoneName: "{{ publiczonename }}" + VpcId: "{{ vpcid }}" + tags: + clustername: "{{ clustername }}" + infraid: "{{ infraid }}" + register: cluster_infra_stack + + - set_fact: + privatezoneid: "{{ cluster_infra_stack.stack_outputs.PrivateHostedZoneId }}" + when: privatezoneid is not defined + + - name: Create UPI Security Elements (Security Groups & IAM) + cloudformation: + stack_name: "{{ infraid }}-cluster-security" + state: present + region: "{{ region }}" + template: "files/cloudformation/03_cluster_security.yaml" + template_parameters: + InfrastructureName: "{{ infraid }}" + PrivateSubnets: "{{ privatesubnets}}" + VpcId: "{{ vpcid }}" + VpcCidr: "{{ vpccidr }}" + tags: + clustername: "{{ clustername }}" + infraid: "{{ infraid }}" + register: cluster_security_stack + +# - debug: +# msg: "{{ cluster_security_stack.stack_outputs.MasterSecurityGroupId }}" + + - name: Set default tags + set_fact: + default_tags: + clustername: "{{ clustername }}" + infraid: "{{ infraid }}" + + - name: Create UPI Bootstrap (EC2 Instance, Security Groups and IAM) + cloudformation: + stack_name: "{{ infraid }}-cluster-bootstrap" + state: present + region: "{{ region }}" + template: "files/cloudformation/04_cluster_bootstrap.yaml{% if api_internal_only is sameas true %}.internalonly{% endif %}" + template_parameters: + InfrastructureName: "{{ infraid }}" + BootstrapIgnitionLocation: s3://{{ infraid }}-bootstrap//bootstrap.ign + VpcId: "{{ vpcid }}" + AutoRegisterELB: "yes" + AllowedBootstrapSshCidr: "0.0.0.0/0" + MasterSecurityGroupId: "{{ cluster_security_stack.stack_outputs.MasterSecurityGroupId }}" + PublicSubnet: "{{ publicsubnets.split(',')[0] }}" + RhcosAmi: "{{ rhcos_ami }}" + ExternalApiTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.ExternalApiTargetGroupArn | default(omit) }}" + InternalApiTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.InternalApiTargetGroupArn }}" + InternalServiceTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.InternalServiceTargetGroupArn }}" + RegisterNlbIpTargetsLambdaArn: "{{ cluster_infra_stack.stack_outputs.RegisterNlbIpTargetsLambda }}" + tags: "{{ default_tags | combine(extra_instance_tags) }}" + + - name: Create UPI Node Launch (EC2 master instances) + cloudformation: + stack_name: "{{ infraid }}-cluster-master-nodes" + state: present + region: "{{ region }}" + template: "files/cloudformation/05_cluster_master_nodes.yaml{% if api_internal_only is sameas true %}.internalonly{% endif %}" + template_parameters: + InfrastructureName: "{{ infraid }}" + AutoRegisterELB: "yes" + MasterSecurityGroupId: "{{ cluster_security_stack.stack_outputs.MasterSecurityGroupId }}" + Master0Subnet: "{{ privatesubnets.split(',')[0] }}" + Master1Subnet: "{{ privatesubnets.split(',')[1] }}" + Master2Subnet: "{{ privatesubnets.split(',')[2] }}" + MasterInstanceProfileName: "{{ cluster_security_stack.stack_outputs.MasterInstanceProfile }}" + MasterInstanceType: "{{ master_instance }}" + RhcosAmi: "{{ rhcos_ami }}" + ExternalApiTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.ExternalApiTargetGroupArn | default(omit) }}" + InternalApiTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.InternalApiTargetGroupArn }}" + InternalServiceTargetGroupArn: "{{ cluster_infra_stack.stack_outputs.InternalServiceTargetGroupArn }}" + RegisterNlbIpTargetsLambdaArn: "{{ cluster_infra_stack.stack_outputs.RegisterNlbIpTargetsLambda }}" + AutoRegisterDNS: "yes" + IgnitionLocation: "https://api-int.{{ clustername }}.{{ publiczonename }}:22623/config/master" + PrivateHostedZoneId: "{{ privatezoneid }}" +#this is confusing but right for in some cases. in others use the privatezonename var + PrivateHostedZoneName: "{{ clustername }}.{{ publiczonename }}" + CertificateAuthorities: "{{ master_ignition.ignition.security.tls.certificateAuthorities[0].source }}" + tags: "{{ default_tags | combine(extra_instance_tags) }}" + +#TODO: change master ec2 instance termination protection to true and then remove old machine api objects after + + - name: tag public subnets + command: "{{ aws }} --region {{ region }} ec2 create-tags --resources {{ item }} --tags Key=kubernetes.io/cluster/{{ infraid }},Value=shared" + with_items: "{{ publicsubnets.split(',') }}" + + - name: tag private subnets + command: "{{ aws }} --region {{ region }} ec2 create-tags --resources {{ item }} --tags Key=kubernetes.io/role/internal-elb,Value=''" + with_items: "{{ privatesubnets.split(',') }}" + + - name: "Check if /tmp/{{ clustername }}/bootstrap-complete exists" + stat: + path: /tmp/{{ clustername }}/bootstrap-complete + register: bootstrap_result + + - name: Wait until bootstrap is complete + shell: > + {{ openshift_install }} wait-for bootstrap-complete --dir=/tmp/{{ clustername }} && + touch /tmp/{{ clustername }}/bootstrap-complete + when: bootstrap_result.stat.exists == False + + - name: "Check if /tmp/{{ clustername }}/bootstrap-complete exists" + stat: + path: /tmp/{{ clustername }}/finish + register: finish + + - name: approve master certificates + shell: "{{ oc }} adm certificate approve $(oc get csr -o 'jsonpath={..metadata.name}')" + register: oc_approve + retries: 12 + delay: 10 + until: oc_approve.rc == 0 + when: not finish.stat.exists + ignore_errors: yes + +# - name: render machineset template +# template: +# src: files/machinesets.yaml.j2 +# dest: /tmp/{{ clustername }}/machineset-{{ item.value }}.yaml +# with_dict: {a: 0, b: 1, c: 2} +# when: finish.stat.exists == False + +# - name: import machinesets +# command: "{{ oc }} -n openshift-machine-api replace -f /tmp/{{ clustername }}/machineset-{{ item }}.yaml" +# with_sequence: start=0 count=3 +# ignore_errors: yes +# when: finish.stat.exists == False + + - name: Fix worker machineset, securitygroup, subnet, ami and instanceprofile setting + shell: | + {{ oc }} -n openshift-machine-api patch machineset {{ infraid }}-worker-{{ region }}{{item.key}} -p '{"spec":{"template":{"spec":{"providerSpec":{"value":{"securityGroups":[{"filters":[{"name": "group-id", "values":["{{ cluster_security_stack.stack_outputs.WorkerSecurityGroupId }}"]}]}]}}}}}}' --type merge + {{ oc }} -n openshift-machine-api patch machineset {{ infraid }}-worker-{{ region }}{{item.key}} -p '{"spec":{"template":{"spec":{"providerSpec":{"value":{"subnet":{"filters":[{"name": "subnet-id", "values":["{{ privatesubnets.split(',')[item.value | int] }}"]}]}}}}}}}' --type merge + {{ oc }} -n openshift-machine-api patch machineset {{ infraid }}-worker-{{ region }}{{item.key}} -p '{"spec":{"template":{"spec":{"providerSpec":{"value":{"iamInstanceProfile":{"id": "{{ cluster_security_stack.stack_outputs.WorkerInstanceProfile }}"}}}}}}}' --type merge + {{ oc }} -n openshift-machine-api patch machineset {{ infraid }}-worker-{{ region }}{{item.key}} -p '{"spec":{"template":{"spec":{"providerSpec":{"value":{"ami":{"id": "{{ rhcos_ami }}"}}}}}}}' --type merge + with_dict: {a: 0, b: 1, c: 2} + when: finish.stat.exists == False + + - name: delete broken machines + command: "{{ oc }} -n openshift-machine-api delete machine -l machine.openshift.io/cluster-api-machineset={{ infraid }}-worker-{{ region }}{{ item.key }}" + with_dict: {a: 0, b: 1, c: 2} + when: finish.stat.exists == False + + +# - name: delete broken machines +# command: "{{ oc }} -n openshift-machine-api delete machine -l machine.openshift.io/cluster-api-machine-role=master" +# when: finish.stat.exists == False + + +# - name: Fix master machine loadbalancer, securitygroup, subnet, ami and instanceprofile setting +# shell: | +# {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"loadBalancers":[{"name": "{{ cluster_infra_stack.stack_outputs.InternalApiLoadBalancerName.split('/')[1] }}", "type": "network"}{% if api_internal_only != true %},{"name": "{{ cluster_infra_stack.stack_outputs.ExternalApiLoadBalancerName.split('/')[1] }}", "type": "network"}{% endif %}]}}}}' --type merge +# {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"securityGroups":[{"filters":[{"name": "group-id", "values":["{{ cluster_security_stack.stack_outputs.MasterSecurityGroupId }}"]}]}]}}}}' --type merge +# {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"subnet":{"filters":[{"name": "subnet-id", "values":["{{ privatesubnets.split(',')[item | int] }}"]}]}}}}}' --type merge +# {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"iamInstanceProfile":{"id": "{{ cluster_security_stack.stack_outputs.MasterInstanceProfile }}"}}}}}' --type merge +# {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"ami":{"id": "{{ rhcos_ami }}"}}}}}' --type merge + + +# {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"loadBalancers":[{"name": "{{ cluster_infra_stack.stack_outputs.InternalApiLoadBalancerName.split('/')[1] }}", "type": "network"},{"name": "{{ cluster_infra_stack.stack_outputs.ExternalApiLoadBalancerName.split('/')[1] }}", "type": "network"}]}}}}' --type merge +# +# {{ oc }} -n openshift-machine-api patch machine {{ infraid }}-master-{{ item }} -p '{"spec":{"providerSpec":{"value":{"subnet":[{"filters":[{"name": "subnet-id", "values":["{{ privatesubnets.split(',')[item | int] }}"]}]}]}}}}' --type merge +# with_sequence: start=0 count=3 + + + - name: render DNS settings + template: + src: files/dnses.yaml.j2 + dest: /tmp/{{ clustername }}/dnses.yaml + when: finish.stat.exists == False + + - name: fix dns settings + command: "{{ oc }} replace -f /tmp/{{ clustername }}/dnses.yaml" + when: finish.stat.exists == False + + - name: Wait for ingress controller + command: "{{ oc }} get -n openshift-ingress-operator ingresscontroller/default" + register: ingress_controller + until: ingress_controller.rc == 0 + retries: 60 + delay: 10 + + - name: Scale ingress controller replicas + command: "{{ oc }} patch -n openshift-ingress-operator -p '{\"spec\":{\"replicas\":3}}' --type merge ingresscontroller/default" + when: finish.stat.exists == False + + - name: Wait until install is complete + shell: > + {{ openshift_install }} wait-for install-complete --dir=/tmp/{{ clustername }} && + touch /tmp/{{ clustername }}/finish + when: finish.stat.exists == False + + - name: Terminate bootstrap resources + block: + - name: Terminate bootstrap CloudFormation stack + cloudformation: + stack_name: "{{ infraid }}-cluster-bootstrap" + region: "{{ region }}" + state: absent + + - import_tasks: tasks/bootstrap-s3-bucket.yaml + vars: + state: absent + when: bootstrap_cleanup | bool + + - import_tasks: tasks/post_installation.yaml + + - debug: + msg: + - "OpenShift cluster {{ clustername }} has been installed successfully." + - "" + - "API: https://api.{{ clustername }}.{{ publiczonename }}:6443" + - "Web console: https://console-openshift-console.apps.{{ clustername }}.{{ publiczonename }}/" + + - debug: msg="Kubeadmin Password kubeadmin/{{lookup('file', '/tmp/{{ clustername }}/auth/kubeadmin-password') }}" +... diff --git a/inventory/group_vars/all b/inventory/group_vars/all new file mode 100644 index 0000000..9d08eec --- /dev/null +++ b/inventory/group_vars/all @@ -0,0 +1,104 @@ +--- +# Directory where the AWS CLI will be installed +awscli_venv_path: ~/virtualenv/awscli +aws: "{{ awscli_venv_path }}/bin/aws" + +# Directory where the openshift-install and oc binaries will be installed +bin_dir: ~/bin + +# OpenShift version to install +openshift_version: "4.2.7" + +openshift_install_url: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/{{ openshift_version }}/openshift-install-linux-{{ openshift_version }}.tar.gz +openshift_cli_url: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/{{ openshift_version }}/openshift-client-linux-{{ openshift_version }}.tar.gz + +openshift_install: "{{ bin_dir }}/openshift-install-{{ openshift_version }}" +oc: "{{ bin_dir }}/oc-{{ openshift_version }}" + +region: "ap-southeast-2" + +clustername: "ipi-test" + +publiczoneid: "Z2ORJM5PPC29C7" +publiczonename: "9cd5.sandbox1011.opentlc.com" + +privatezoneid: "Z04341453TFUYWVPR85NO" +privatezonename: "ipi-test.9cd5.sandbox1011.opentlc.com" + +# subnet order matters, needs to be az1,az2,az3,etc +privatesubnets: "subnet-04ff1ddf823a249c2,subnet-05cb1bf77be818c04,subnet-0add7a8f5bd253969" +publicsubnets: "subnet-0c7eb4e1da842d10f,subnet-0530e8e32568ee94f,subnet-02c38489901823742" + +vpcid: "vpc-04a5fc89bd6a946e3" +vpccidr: "10.0.0.0/16" + +clustercidr: "10.128.0.0/14" +servicecidr: "172.30.0.0/16" + +# Public AMI rhcos-410.8.20190417.1-hvm +rhcos_ami: "ami-0391e92574fb09e08" + +# This is used by create-encrypted-ami.yaml to tag the private AMI +# which we copy from the public AMI to enable disk +# encryption. install-upi.yaml uses this tag to find the AMI to use +# for master and worker nodes. +rhcos_version: "427" + +worker_instance: "m5.2xlarge" +master_instance: "m4.2xlarge" + +#ssh_key is optional either set or comment out +ssh_key: "" + +#get from try.openshift.com +pull_secret: '' + +# Additional tags to assign to the EC2 instances created by the +# installation playbook and by the OpenShift worker machine set. +# +# The "auto_shut_bool" tag is only an example, it will not do anything +# without a script that looks for this tag. AWS tags are strings, we +# have to quote the "True" value that otherwise would be sent to the +# AWS API as a boolean. +extra_instance_tags: + auto_shut_bool: 'True' + department: Engineering + +# AWS bootstrap resources are deleted after a successful installation +# if this is set to true. +bootstrap_cleanup: true + +# Enable this to only expose the API internally, requires ansible host to +# be on the same network as cluster +api_internal_only: false + +######################### +# LDAP +######################### +# Enable LDAP integration, disabled by default +enable_ldap_integration: false +# LDAP bind password in plain text, ideally should be in Ansible Vault or other secrets management tool +ldap_bindPassword: "SUPERSECRET" +# Name for the LDAP secret containing LDAP password +ldap_secret_name: "ldap-secret1" +# LDAP identity provider name +ldap_idp_name: "ldapidp1" +# Bind DN +bindDN: "a@X.test.example.au" +# LDAP URL +ldap_url: "ldap://X.test.example.au:389/OU=Users,OU=ad,DC=X,DC=test,DC=example,DC=au" + +######################### +# Cluster Monitoring +######################### +# Enable Cluster Monitoring Storage and Retention +enable_cluster_monitoring: false +# Required Storage Class +prometheus_storage_class: "gp2" +# Required Prometheus Storage +prometheus_storage_size: "20Gi" +# Required Prometheus Retention Period (h (hours), d (days), w (weeks), or y (years)) +prometheus_retention_time: "7d" +# Prometheus Replicas +prometheus_statefulset_replicas: "3" +... diff --git a/inventory/hosts b/inventory/hosts new file mode 100644 index 0000000..13cfabe --- /dev/null +++ b/inventory/hosts @@ -0,0 +1,2 @@ +[local] +localhost ansible_connection=local diff --git a/tasks/bootstrap-s3-bucket.yaml b/tasks/bootstrap-s3-bucket.yaml new file mode 100644 index 0000000..e26d443 --- /dev/null +++ b/tasks/bootstrap-s3-bucket.yaml @@ -0,0 +1,9 @@ +- name: "{{ 'Create' if (state is undefined or 'absent' not in state) else 'Terminate' }} S3 bucket ( bootstrap config )" + s3_bucket: + name: "{{ infraid }}-bootstrap" + region: "{{ region }}" + state: "{{ state | default('present') }}" + force: "{{ 'no' if (state is undefined or 'absent' not in state) else 'yes' }}" + tags: "{{ {'kubernetes.io/cluster/' ~ infraid: 'owned', 'clusterid': infraid} }}" + retries: 3 + delay: 3 diff --git a/tasks/download-ocp-binaries.yaml b/tasks/download-ocp-binaries.yaml new file mode 100644 index 0000000..1523b14 --- /dev/null +++ b/tasks/download-ocp-binaries.yaml @@ -0,0 +1,35 @@ +- name: Create bin directory + file: + path: "{{ bin_dir }}" + state: directory + +# Downloading the binaries is slow. Check if they are already +# installed. +- name: Check if OpenShift installer is already installed + stat: + path: "{{ openshift_install }}" + register: openshift_install_stat + +- block: + - name: Install OpenShift installer and CLI + unarchive: + src: "{{ item }}" + remote_src: yes + dest: "{{ bin_dir }}" + mode: '755' + exclude: + - README.md + loop: + - "{{ openshift_install_url }}" + - "{{ openshift_cli_url }}" + + - name: Copy OpenShift installer and CLI + file: + path: "{{ bin_dir }}/{{ item }}-{{ openshift_version }}" + src: "{{ bin_dir }}/{{ item }}" + state: hard + loop: + - openshift-install + - oc + - kubectl + when: not openshift_install_stat.stat.exists diff --git a/tasks/get_infraid.yaml b/tasks/get_infraid.yaml new file mode 100644 index 0000000..99e3e24 --- /dev/null +++ b/tasks/get_infraid.yaml @@ -0,0 +1,8 @@ +--- +- set_fact: + metadata: "{{ lookup('file', '/tmp/' ~ clustername ~ '/metadata.json') | from_json }}" + +- set_fact: + infraid: "{{ metadata.infraID }}" + clusterid: "{{ metadata.clusterID }}" +... diff --git a/tasks/installer-config-directory.yaml b/tasks/installer-config-directory.yaml new file mode 100644 index 0000000..11127bc --- /dev/null +++ b/tasks/installer-config-directory.yaml @@ -0,0 +1,5 @@ +- name: "{{ 'Create' if (state is undefined or 'absent' not in state) else 'Terminate' }} cluster install config directory" + file: + path: /tmp/{{ clustername }} + state: "{{ state | default('directory') }}" + mode: 0755 diff --git a/tasks/post_installation.yaml b/tasks/post_installation.yaml new file mode 100644 index 0000000..ad43ae1 --- /dev/null +++ b/tasks/post_installation.yaml @@ -0,0 +1,36 @@ +--- + +- block: + - name: 1.0 | Creating LDAP secrets | tasks/post_installation.yaml + k8s: + state: present + definition: "{{ lookup('template', 'files/LDAPsecret.yaml.j2') }}" + validate: + # Requires the kubernetes-validate python module + fail_on_error: no + strict: yes + + - name: 2.0 | Creating LDAP Custom Resource | tasks/post_installation.yaml + k8s: + state: present + definition: "{{ lookup('template', 'files/Oauth.yaml.j2') }}" + validate: + # Requires the kubernetes-validate python module + fail_on_error: no + strict: yes + when: enable_ldap_integration == true + +- block: + - name: 3.0 | Creating Cluster Monitoring | tasks/post_installation.yaml + k8s: + state: present + definition: "{{ lookup('template', 'files/cluster-monitoring-config.yaml.j2') }}" + validate: + fail_on_error: no + strict: yes + + - name: 4.0 | Scale the Replicas | tasks/post_installation.yaml + shell: + oc scale statefulset/prometheus-k8s --replicas="{{ prometheus_statefulset_replicas }}" -n openshift-monitoring + ignore_errors: true + when: enable_cluster_monitoring == true diff --git a/uninstall-upi.yaml b/uninstall-upi.yaml new file mode 100644 index 0000000..0b2276b --- /dev/null +++ b/uninstall-upi.yaml @@ -0,0 +1,249 @@ +--- +- name: Uninstall OpenShift 4 + gather_facts: false + hosts: localhost + vars: + state: absent + tasks: + # Allow users to pass the infraid and clusterid on the command + # line, in case they lost the /tmp/{{ clustername }} directory. + - import_tasks: tasks/get_infraid.yaml + when: infraid is not defined or clusterid is not defined + + - name: Get short infra ID + set_fact: + short_infraid: "{{ infraid | regex_replace('.*-([^-]+)$', '\\1') }}" + short_clusterid: "{{ clusterid | regex_replace('-', '') }}" + + - name: Terminate EC2 CloudFormation stacks + cloudformation: + stack_name: "{{ item }}" + region: "{{ region }}" + state: absent + loop: + - "{{ infraid }}-cluster-master-nodes" + - "{{ infraid }}-cluster-bootstrap" + + # Terminate EC2 instances created by OpenShift. + + - name: Get EC2 instances + ec2_instance_facts: + region: "{{ region }}" + filters: "tag:kubernetes.io/cluster/{{ infraid }}=owned" + register: ec2_instances + + # We check the kubernetes.io/cluster tag again to make sure that + # we don't delete the wrong EC2 instances. This is in case the + # "filters" parameter for ec2_instance_facts stops working in the + # future for some reason. + - name: Terminate EC2 instances + ec2: + instance_ids: "{{ item['instance_id'] }}" + region: "{{ region }}" + state: absent + wait: yes + when: + - "'kubernetes.io/cluster/' ~ infraid in item['tags']" + - item['tags']['kubernetes.io/cluster/' ~ infraid] == 'owned' + - item['state']['name'] != 'terminated' + loop: "{{ ec2_instances.instances }}" + loop_control: + label: "{{ item['tags']['Name'] | default(item['id']) }}" + + - name: List EBS volumes + ec2_vol: + state: list + region: "{{ region }}" + register: volumes + + - name: Terminate persistent volumes + ec2_vol: + id: "{{ item['id'] }}" + region: "{{ region }}" + state: absent + when: + - "'Name' in item['tags']" + - "'kubernetes-dynamic-pvc-' in item['tags']['Name']" + - "'kubernetes.io/cluster/' ~ infraid in item['tags']" + - item['tags']['kubernetes.io/cluster/' ~ infraid] == 'owned' + - item['status'] == 'available' + loop: "{{ volumes['volumes'] }}" + loop_control: + label: "{{ item['tags']['Name'] | default(item['id']) }}" + + # We need to delete records in the private Route53 zone that were + # created by OpenShift. Otherwise, the cluster-infra + # CloudFormation stack cannot be deleted. + + - set_fact: + privatezonename: "{{ clustername }}.{{ publiczonename }}" + + - name: Check private Route53 zone + route53_facts: + query: hosted_zone + register: hosted_zones + + - block: + - name: Get private Route53 record + route53: + state: get + zone: "{{ privatezonename }}" + private_zone: yes + record: "*.apps.{{ privatezonename }}" + type: A + register: route53_private + + - name: Terminate private Route53 record + route53: + zone: "{{ privatezonename }}" + private_zone: yes + record: "*.apps.{{ privatezonename }}" + type: A + value: "{{ route53_private.set.value }}" + ttl: "{{ route53_private.set.ttl }}" + alias: true + alias_hosted_zone_id: "{{ route53_private.set.alias_hosted_zone_id }}" + state: absent + when: "'value' in route53_private.set" + when: privatezonename ~ '.' in hosted_zones['HostedZones'] | map(attribute='Name') | list + + - name: Terminate other CloudFormation stacks + cloudformation: + stack_name: "{{ item }}" + region: "{{ region }}" + state: absent + loop: + - "{{ infraid }}-cluster-security" + - "{{ infraid }}-cluster-infra" + + # Terminate S3 buckets created by OpenShift. + + - import_tasks: tasks/bootstrap-s3-bucket.yaml + + - name: List S3 buckets + aws_s3_bucket_facts: + region: "{{ region }}" + register: s3 + + - name: Terminate S3 buckets + s3_bucket: + name: "{{ item }}" + state: absent + force: yes + when: item is match('image-registry-' ~ region ~ '-' ~ short_clusterid ~ '-[a-z0-9]+') + loop: "{{ s3.ansible_facts.buckets | map(attribute='name') | list }}" + + # Terminate load balancers and security groups created by + # OpenShift. + + - name: Get ELBs + ec2_elb_facts: + region: "{{ region }}" + register: ec2_elbs + + - name: Terminate ELBs + ec2_elb_lb: + name: "{{ item['name'] }}" + region: "{{ region }}" + state: absent + wait: yes + when: + - "'kubernetes.io/cluster/' ~ infraid in item['tags']" + - item['tags']['kubernetes.io/cluster/' ~ infraid] == 'owned' + loop: "{{ ec2_elbs.elbs }}" + loop_control: + label: "{{ item['name'] }}" + + - name: Get security groups + ec2_group_facts: + region: "{{ region }}" + filters: "tag:kubernetes.io/cluster/{{ infraid }}=owned" + register: ec2_groups + + # Even though we use the "wait" option when we terminate the ELBs, + # deleting the security groups sometimes fails because of + # dependent objects. We retry as a workaround. + + - name: Terminate security groups + ec2_group: + group_id: "{{ item['group_id'] }}" + region: "{{ region }}" + state: absent + register: task_result + retries: 3 + delay: 10 + until: task_result is success + when: + - "'kubernetes.io/cluster/' ~ infraid in item['tags']" + - item['tags']['kubernetes.io/cluster/' ~ infraid] == 'owned' + loop: "{{ ec2_groups.security_groups }}" + loop_control: + label: "{{ item['group_name'] }}" + + # The ID at the end of the IAM usernames is random, it does not + # match the cluster's infraid. There's no Ansible module that + # filters IAM users by tag. For now, we use the AWS CLI to list + # all IAM users and carefully check the username before deleting + # them. + + - name: Get IAM users + command: "{{ aws }} iam list-users" + register: all_iam_users + + - set_fact: + iam_users: "{{ (all_iam_users.stdout | from_json)['Users'] | selectattr('UserName', 'match', clustername ~ '-(cloud-credential-operator-iam-ro|openshift-image-registry|openshift-ingress|openshift-machine-api)-[a-z0-9]+') | map(attribute='UserName') | list }}" + + - name: Terminate IAM user policies + iam_policy: + iam_type: user + iam_name: "{{ item }}" + policy_name: "{{ item }}-policy" + state: absent + loop: "{{ iam_users }}" + + - name: Get IAM user access keys + command: "{{ aws }} iam list-access-keys --user-name {{ item }}" + register: access_keys + loop: "{{ iam_users }}" + + - name: Terminate IAM user access keys + iam: + iam_type: user + name: "{{ item['item'] }}" + state: update + access_key_ids: "{{ (item['stdout'] | from_json)['AccessKeyMetadata'] | map(attribute='AccessKeyId') | list }}" + access_key_state: remove + loop: "{{ access_keys['results'] }}" + + - name: Terminate IAM users + iam_user: + name: "{{ item }}" + state: absent + loop: "{{ iam_users }}" + + - name: Get public Route53 record + route53: + state: get + zone: "{{ publiczonename }}" + record: "*.apps.{{ clustername }}.{{ publiczonename }}" + type: A + register: route53_public + + - name: Terminate public Route53 records + route53: + zone: "{{ publiczonename }}" + record: "*.apps.{{ clustername }}.{{ publiczonename }}" + type: A + value: "{{ route53_public.set.value }}" + ttl: "{{ route53_public.set.ttl }}" + alias: true + alias_hosted_zone_id: "{{ route53_public.set.alias_hosted_zone_id }}" + state: absent + when: "'value' in route53_public.set" + + - name: Untag subnets + command: "{{ aws }} --region {{ region }} ec2 delete-tags --resources {{ item }} --tags Key=kubernetes.io/cluster/{{ infraid }}" + loop: "{{ publicsubnets.split(',') + privatesubnets.split(',') }}" + + - import_tasks: tasks/installer-config-directory.yaml +...