Merge pull request #5 from capturealpha/develop

#1 IPFS node terraform

.gitignore

@@ -11,5 +11,7 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
-# Dependency directories (remove the comment below to include it)
-# vendor/
+# Terraform secrets and provider files
+/terraform/**/*.terraform
+/terraform/**/*.env
+/terraform/**/*.tfvars

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "files.eol": "\n",
+}

README.md

@@ -1,2 +1,3 @@
-# gitblox
+# GitBlox
+
 Git IPFS Storage

assets/README.md

@@ -0,0 +1 @@
+# GitBlox Assets

docs/README.md

@@ -0,0 +1 @@
+# GitBlox Documentation

terraform/.gitignore

@@ -0,0 +1,2 @@
+.terraform
+*.tfvars

terraform/.terraform/environment

@@ -0,0 +1 @@
+develop

terraform/.terraform/terraform.tfstate

@@ -0,0 +1,52 @@
+{
+    "version": 3,
+    "serial": 1,
+    "lineage": "53b7c779-871b-621a-cee5-e3004c8d9292",
+    "backend": {
+        "type": "s3",
+        "config": {
+            "access_key": null,
+            "acl": null,
+            "assume_role_duration_seconds": null,
+            "assume_role_policy": null,
+            "assume_role_policy_arns": null,
+            "assume_role_tags": null,
+            "assume_role_transitive_tag_keys": null,
+            "bucket": "terraform-ca-state",
+            "dynamodb_endpoint": null,
+            "dynamodb_table": null,
+            "encrypt": true,
+            "endpoint": null,
+            "external_id": null,
+            "force_path_style": null,
+            "iam_endpoint": null,
+            "key": "terraform.tfstate",
+            "kms_key_id": null,
+            "max_retries": null,
+            "profile": null,
+            "region": "us-east-1",
+            "role_arn": null,
+            "secret_key": null,
+            "session_name": null,
+            "shared_credentials_file": null,
+            "skip_credentials_validation": null,
+            "skip_metadata_api_check": null,
+            "skip_region_validation": null,
+            "sse_customer_key": null,
+            "sts_endpoint": null,
+            "token": null,
+            "workspace_key_prefix": "gitblox"
+        },
+        "hash": 30241332
+    },
+    "modules": [
+        {
+            "path": [
+                "root"
+            ],
+            "outputs": {},
+            "resources": {},
+            "depends_on": []
+        }
+    ]
+}

terraform/README.md

@@ -0,0 +1,58 @@
+# GitBlox Infrastructure
+
+## Getting Started
+- Install terraform [Installation Instructions](https://learn.hashicorp.com/tutorials/terraform/install-cli)
+
+### Deployment
+
+>*This example will create the following resources in the specified AWS Region:*
+
+- Configure environment file in `config\ipfs-node\${workspace}.env`
+  - See `config\ipfs-node\env.example for required environment variables`
+- Configure `terraform.tfvars`
+  - See `terraform.tfvars.exmaple for required variables`
+  - If not configured terraform will prompt you for required values
+- `terraform workspace select $environment` (develop, stage, prod)
+- `terraform apply`
+
+Region is configured per workspace and is mapped in `variables.tf`
+
+- Virtual Private Cloud (10.0.0.0/16)
+- Internet Gateway
+- Route Table
+  - Egress all
+- Public Subnets (10.0.x.0/24)4
+  - One for each availability zone (depending on region)
+- Elastic Load Balancer
+  - LB https 443->8080 for IPFS Gateway
+- Security Group (ELB)
+  - Ingress
+    - 443 tcp all
+  - Egress
+    - all all all
+- Security Group (ipfs-node)
+  - Ingress
+    - all  all all - whitelisted IP addresses
+    - 8080 tcp ipfs-gateway - ELB
+    - 4001 tcp all - p2p
+  - Egress
+    - all   all all
+- Key Pair
+- EC2 Instance (Ubuntu Server 20.04.4 LTS)
+- Route53 (DNS)
+  - `ipfs-${count.index}.${terraform.workspace}.gitblox.io`
+    - ex: `ipfs-1.develop.gitblox.io`
+    - use for SSH or API access
+  - `ipfs-gateway.${terraform.workspace}.gitblox.io`
+    - ex: `ipfs-gateway.develop.gitblox.io`
+    - use for HTTPS gateway access via ELB
+    - [Example Link](https://ipfs-gateway.develop.gitblox.io/ipfs/QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc/readme)
+
+### Connect
+
+ - Run `./connect.sh ipfs_node #` with `#` being `0..n` node index if deploying multiple nodes
+ - SSH public key must exist on server under `~/.ssh/authorized_keys`
+   - Can be added to `terraform.tfvars` as `ssh_key_1` or `ssh_key_2`
+   - More can be added as needed
+ - IP address must be included to `terraform.tfvars` as `ip_whitelist`
+   - Must be in `address/cidr` format `ip_whitelist = ["x.x.x.x/32"]`

terraform/config/ipfs-node/.gitignore

@@ -0,0 +1 @@
+*.env

terraform/config/ipfs-node/env.example

@@ -0,0 +1 @@
+IPFS_VERSION=0.13.1

terraform/connect.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+source ./utilities/rainbow.sh
+
+USERNAME=`sed -e 's/^"//' -e 's/"$//' <<<$(terraform output prefix)`
+
+if [[ "${2}" != "" ]]; then
+	terraform output ${1}-ip | sed ':a;N;$!ba;s/,\n]/]/g' | jq -r ".[${2}]"
+	ssh -o "StrictHostKeyChecking no" $USERNAME@$(terraform output ${1}-ip | sed ':a;N;$!ba;s/,\n]/]/g' | jq -r ".[${2}]")
+elif [[ ! -z "${1}" ]]; then
+	ssh -o "StrictHostKeyChecking no" $USERNAME@$(terraform output ${1}-ip)
+else
+	echored "Missing ssh target instance!"
+fi

terraform/iam.tf

@@ -0,0 +1,99 @@
+resource "aws_iam_role" "role" {
+  name               = "${var.prefix}-${terraform.workspace}"
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+
+  tags = {
+    Name        = "${var.prefix}-${terraform.workspace}"
+    environment = terraform.workspace
+    group       = var.prefix
+  }
+}
+
+resource "aws_iam_instance_profile" "profile" {
+  name = "${var.prefix}-${terraform.workspace}"
+  role = aws_iam_role.role.name
+}
+
+resource "aws_iam_role_policy" "policy" {
+  name   = "${var.prefix}-${terraform.workspace}"
+  role   = aws_iam_role.role.id
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+		{
+			"Effect": "Allow",
+			"Action": [
+					"s3:*",
+					"ssm:DescribeAssociation",
+					"ssm:GetDeployablePatchSnapshotForInstance",
+					"ssm:GetDocument",
+					"ssm:DescribeDocument",
+					"ssm:GetManifest",
+					"ssm:GetParameters",
+					"ssm:ListAssociations",
+					"ssm:ListInstanceAssociations",
+					"ssm:PutInventory",
+					"ssm:PutComplianceItems",
+					"ssm:PutConfigurePackageResult",
+					"ssm:UpdateAssociationStatus",
+					"ssm:UpdateInstanceAssociationStatus",
+					"ssm:UpdateInstanceInformation",
+					"ssmmessages:CreateControlChannel",
+					"ssmmessages:CreateDataChannel",
+					"ssmmessages:OpenControlChannel",
+					"ssmmessages:OpenDataChannel",
+					"ec2:DetachVolume",
+          "ec2:AttachVolume",
+          "ec2:CopySnapshot",
+          "ec2:DeleteSnapshot",
+          "ec2:DescribeInstances",
+          "ec2:DeleteTags",
+          "ec2:DescribeTags",
+          "ec2:DescribeSnapshotAttribute",
+          "ec2:CreateTags",
+          "ec2:ResetSnapshotAttribute",
+          "ec2:ImportSnapshot",
+          "ec2:DescribeSnapshots",
+          "ec2:DescribeVolumeAttribute",
+          "ec2:DescribeImportSnapshotTasks",
+          "ec2:DescribeVolumeStatus",
+          "ec2:ModifySnapshotAttribute",
+          "ec2:DescribeVolumes",
+          "ec2:CreateSnapshot"
+			],
+			"Resource": "*"
+		},
+		{
+      "Effect":"Allow",
+      "Action":[
+        "ses:SendEmail",
+        "ses:SendRawEmail"
+      ],
+      "Resource":"*",
+      "Condition":{
+        "ForAllValues:StringLike":{
+          "ses:Recipients":[
+            "*@${var.root_domain}"
+          ]
+        }
+      }
+		}
+  ]
+}
+EOF
+}

terraform/ipfs-node-cloud-init.yml

@@ -0,0 +1,35 @@
+#cloud-config
+
+package_update: true
+package_upgrade: true
+users:
+    - name: ${prefix}
+      sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+      groups: sudo, docker
+      shell: /bin/bash
+      ssh_authorized_keys:
+          - "${ssh_key_1}"
+          - "${ssh_key_2}"
+packages:
+    - fail2ban
+    - jq
+    - prometheus-node-exporter
+    - python3-pip
+preserve_hostname: false
+fqdn: ${fqdn}
+hostname: ${fqdn}
+write_files:
+    - path: /etc/environment
+      content: |
+          IPFS_PATH="${ipfs_path}"
+          NODE_NUMBER=${ipfs_node_number}
+          REGION="${region}"
+          WORKSPACE="${workspace}"
+      append: true
+runcmd:
+    - pip3 install awscli
+    - sed -i -e '/^Port/s/^.*$/Port ${ssh_port}/' /etc/ssh/sshd_config
+    - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    - sed -i -e '$aAllowUsers ${prefix}' /etc/ssh/sshd_config
+    - echo '* soft nofile 512000' >> /etc/security/limits.conf
+    - echo '* hard nofile 512000' >> /etc/security/limits.conf

terraform/ipfs-node-ec2.tf

@@ -0,0 +1,73 @@
+
+
+resource "aws_instance" "ipfs_node" {
+  ami                    = data.aws_ami.ubuntu.id
+  count                  = var.ipfs_node_count[terraform.workspace]
+  iam_instance_profile   = aws_iam_instance_profile.profile.name
+  instance_type          = var.ipfs_node_instance_types[terraform.workspace]
+  key_name               = aws_key_pair.auth.id
+  subnet_id              = aws_subnet.public.0.id
+  vpc_security_group_ids = [aws_security_group.ipfs_node.id]
+  timeouts {
+    create = "30m"
+    delete = "10m"
+  }
+  user_data = templatefile("${abspath(path.root)}/ipfs-node-cloud-init.yml", {
+    fqdn             = "${var.prefix}-ipfs-node-${terraform.workspace}-${count.index + 1}.${var.root_domain}"
+    prefix           = var.prefix
+    ssh_port         = var.ssh_port
+    ssh_key_1        = var.ssh_key_1
+    ssh_key_2        = var.ssh_key_2
+    ipfs_node_number = "${count.index + 1}"
+    ipfs_path        = var.ipfs_path
+    region           = var.workspace_regions[terraform.workspace]
+    workspace        = terraform.workspace
+  })
+  connection {
+    type        = "ssh"
+    user        = var.prefix
+    port        = var.ssh_port
+    host        = self.public_ip
+    private_key = file(var.private_key_path)
+    agent       = false
+  }
+  root_block_device {
+    volume_size = var.ipfs_node_root_volume_size
+  }
+  ebs_block_device {
+    device_name = "/dev/sdf"
+    snapshot_id = length(data.aws_ebs_snapshot_ids.ipfs_data.ids) > 0 ? data.aws_ebs_snapshot_ids.ipfs_data.ids[0] : null
+    volume_size = var.ipfs_node_data_volume_size
+    volume_type = "gp2"
+  }
+  tags = {
+    Name        = "${var.prefix}-ipfs-node-${terraform.workspace}-${count.index + 1}"
+    environment = terraform.workspace
+    group       = var.prefix
+    type        = "ipfs-node"
+  }
+  volume_tags = {
+    Name        = "${var.prefix}-ipfs-node-${terraform.workspace}-${count.index + 1}"
+    environment = terraform.workspace
+    group       = var.prefix
+    type        = "ipfs-node"
+  }
+  provisioner "file" {
+    source      = "./ipfs-node"
+    destination = "/home/${var.prefix}/"
+  }
+  provisioner "file" {
+    source      = "./config/ipfs-node/${terraform.workspace}.env"
+    destination = "/home/${var.prefix}/ipfs-node/.env"
+  }
+  provisioner "file" {
+    source      = "./utilities"
+    destination = "/home/${var.prefix}/utilities"
+  }
+  provisioner "remote-exec" {
+    inline = ["cloud-init status --wait",
+      "find ~ -name '*.sh' | xargs  chmod +x",
+      "/home/${var.prefix}/ipfs-node/init.sh"
+    ]
+  }
+}