Merge pull request #9 from /issues/2/terraform-git

issues/2/terraform git

README.md

@@ -1,3 +1,27 @@
+<a href="http://capturealpha.com"><img src="https://capturealpha.com/wp-content/uploads/2022/07/GitBloxLogoLight.png" title="GitBlox" alt="GitBlox" width="500"></a>
+
 # GitBlox
 
-Git IPFS Storage
+> Decentralized Git Repository using IPFS<br><br>
+> Built for ETHGlobal Hack FS 2022<br><br>
+> Primary goal for this Hackathon is to use IPFS as a storage mechanism for Git Repositories. Part of this effort will be a decentralized compute architecture to handle the Git Smart Protocol responsible for packfiles.
+
+---
+
+## Team
+
+> Nate Bolam, Kensie Meredith, Carl
+
+---
+
+## Support
+
+Reach out to us!
+
+- Website at <a href="https://capturealpha.com" target="_blank">`capturealpha.com`</a>
+- Twitter at <a href="http://twitter.com/capture_alpha" target="_blank">`@capture_alpha`</a>
+- Discord at <a href="https://discord.gg/6K5e7hTK" target="_blank">`Capture Alpha Discord`</a>
+
+---
+
+- Copyright 2022 © <a href="http://capturealpha.com" target="_blank">Capture Alpha</a>.

terraform/git-server-cloud-init.yml

@@ -0,0 +1,40 @@
+#cloud-config
+
+package_update: true
+package_upgrade: true
+users:
+    - name: ${prefix}
+      sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+      groups: sudo, docker
+      shell: /bin/bash
+      ssh_authorized_keys:
+          - "${ssh_key_1}"
+          - "${ssh_key_2}"
+packages:
+    - fail2ban
+    - fcgiwrap
+    - git
+    - jq
+    - nginx
+    - prometheus-node-exporter
+    - python3-pip
+preserve_hostname: false
+fqdn: ${fqdn}
+hostname: ${fqdn}
+write_files:
+    - path: /etc/environment
+      content: |
+          DATA_DIR="${data_dir}"
+          DOMAIN="${domain}"
+          FQDN="${fqdn}"
+          NODE_NUMBER=${git_server_number}
+          REGION="${region}"
+          WORKSPACE="${workspace}"
+      append: true
+runcmd:
+    - pip3 install awscli
+    - sed -i -e '/^Port/s/^.*$/Port ${ssh_port}/' /etc/ssh/sshd_config
+    - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    - sed -i -e '$aAllowUsers ${prefix}' /etc/ssh/sshd_config
+    - echo '* soft nofile 512000' >> /etc/security/limits.conf
+    - echo '* hard nofile 512000' >> /etc/security/limits.conf

terraform/git-server-ec2.tf

@@ -0,0 +1,75 @@
+
+
+resource "aws_instance" "git_server" {
+  ami                    = data.aws_ami.ubuntu.id
+  count                  = var.git_server_count[terraform.workspace]
+  iam_instance_profile   = aws_iam_instance_profile.profile.name
+  instance_type          = var.git_server_instance_types[terraform.workspace]
+  key_name               = aws_key_pair.auth.id
+  subnet_id              = aws_subnet.public.0.id
+  vpc_security_group_ids = [aws_security_group.git_server.id]
+  timeouts {
+    create = "30m"
+    delete = "10m"
+  }
+  user_data = templatefile("${abspath(path.root)}/git-server-cloud-init.yml", {
+    data_dir          = var.git_path
+    domain            = var.root_domain
+    fqdn              = "${var.prefix}-git-server-${terraform.workspace}-${count.index + 1}.${var.root_domain}"
+    prefix            = var.prefix
+    ssh_port          = var.ssh_port
+    ssh_key_1         = var.ssh_key_1
+    ssh_key_2         = var.ssh_key_2
+    git_server_number = "${count.index + 1}"
+    git_path          = var.git_path
+    region            = var.workspace_regions[terraform.workspace]
+    workspace         = terraform.workspace
+  })
+  connection {
+    type        = "ssh"
+    user        = var.prefix
+    port        = var.ssh_port
+    host        = self.public_ip
+    private_key = file(var.private_key_path)
+    agent       = false
+  }
+  root_block_device {
+    volume_size = var.git_server_root_volume_size
+  }
+  ebs_block_device {
+    device_name = "/dev/sdf"
+    snapshot_id = length(data.aws_ebs_snapshot_ids.git_data.ids) > 0 ? data.aws_ebs_snapshot_ids.git_data.ids[0] : null
+    volume_size = var.git_server_data_volume_size
+    volume_type = "gp2"
+  }
+  tags = {
+    Name        = "${var.prefix}-git-server-${terraform.workspace}-${count.index + 1}"
+    environment = terraform.workspace
+    group       = var.prefix
+    type        = "git-server"
+  }
+  volume_tags = {
+    Name        = "${var.prefix}-git-server-${terraform.workspace}-${count.index + 1}"
+    environment = terraform.workspace
+    group       = var.prefix
+    type        = "git-server"
+  }
+  provisioner "file" {
+    source      = "./git-server"
+    destination = "/home/${var.prefix}/"
+  }
+  provisioner "file" {
+    source      = "./config/git-server/${terraform.workspace}.env"
+    destination = "/home/${var.prefix}/git-server/.env"
+  }
+  provisioner "file" {
+    source      = "./utilities"
+    destination = "/home/${var.prefix}/utilities"
+  }
+  provisioner "remote-exec" {
+    inline = ["cloud-init status --wait",
+      "find ~ -name '*.sh' | xargs  chmod +x",
+      "/home/${var.prefix}/git-server/init.sh"
+    ]
+  }
+}

terraform/git-server-elb.tf

@@ -0,0 +1,56 @@
+resource "aws_security_group" "git_server_elb" {
+  name        = "${var.prefix}-git-server-elb-${terraform.workspace}"
+  description = "${var.prefix} git-server ELB ${terraform.workspace}"
+  vpc_id      = aws_vpc.main.id
+
+  ingress {
+    description = "https"
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  egress {
+    description = "outbound internet access"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_elb" "git" {
+  name            = "${var.prefix}-git-${terraform.workspace}-${count.index + 1}"
+  count           = var.git_server_count[terraform.workspace]
+  subnets         = aws_subnet.public.*.id
+  security_groups = [aws_security_group.git_server_elb.id]
+  instances       = [aws_instance.git_server[count.index].id]
+
+  listener {
+    instance_port      = 80
+    instance_protocol  = "http"
+    lb_port            = 443
+    lb_protocol        = "https"
+    ssl_certificate_id = data.aws_acm_certificate.cert.arn
+  }
+
+  health_check {
+    healthy_threshold   = 2
+    unhealthy_threshold = 2
+    timeout             = 3
+    target              = "HTTP:80/test-repo/info/refs"
+    interval            = 30
+  }
+
+  cross_zone_load_balancing   = true
+  idle_timeout                = 400
+  connection_draining         = true
+  connection_draining_timeout = 400
+
+  tags = {
+    Name        = "git-${var.prefix}-${terraform.workspace}-${count.index + 1}"
+    environment = terraform.workspace
+    group       = var.prefix
+    type        = "git"
+  }
+}

terraform/git-server-sg.tf

@@ -0,0 +1,31 @@
+resource "aws_security_group" "git_server" {
+  name        = "${var.prefix}-git-server-${terraform.workspace}"
+  description = "${var.prefix} git server ${terraform.workspace}"
+  vpc_id      = aws_vpc.main.id
+  ingress {
+    description = "all"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = var.ip_whitelist
+  }
+  ingress {
+    description     = "${var.prefix} git server"
+    from_port       = 80
+    to_port         = 80
+    protocol        = "tcp"
+    security_groups = [aws_security_group.git_server_elb.id]
+  }
+  egress {
+    description = "outbound internet access"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  tags = {
+    environment = terraform.workspace
+    group       = var.prefix
+    type        = "git-server"
+  }
+}

terraform/git-server/init.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+SCRIPT_PATH=$(dirname $(realpath -s $0))
+cd ${SCRIPT_PATH}
+set -o allexport
+source .env
+source ../utilities/rainbow.sh
+set +o allexport
+
+~/utilities/create-data-volume.sh
+
+# Configure git repository
+if [ ! -d "${DATA_DIR}/test-repo" ]; then
+    mkdir -p ${DATA_DIR}/test-repo
+    cd ${DATA_DIR}/test-repo
+    git init . --bare --shared
+    git update-server-info
+    git config --bool http.receivepack true
+    sudo chmod -R ugo+rw .
+fi
+
+cd ${SCRIPT_PATH}
+if [ -f  "${SCRIPT_PATH}/nginx.conf" ]; then
+    sed -i s#%DATA_DIR%#${DATA_DIR}#g ./nginx.conf &&
+    sudo mv ./nginx.conf /etc/nginx/sites-available/default
+fi
+
+sudo systemctl enable fcgiwrap
+sudo systemctl enable nginx
+
+sudo systemctl restart fcgiwrap
+sudo systemctl restart nginx
+
+sleep 5
+
+if [ ! -d "${SCRIPT_PATH}/git-test" ]; then
+    mkdir git-test
+    cd git-test
+    git init
+    git remote add origin http://localhost/test-repo
+    git config --global user.email "[email protected]"
+    git config --global user.name "Gitblox Test"
+    mkdir test
+    echo "This is my first file" > test/test.txt
+    git add .
+    git commit -a -m "Add test file and directory"
+    git push --set-upstream origin master
+fi
+
+echogreen "deployment completed!"

terraform/git-server/nginx.conf

@@ -0,0 +1,13 @@
+server {
+    listen 80;
+    root %DATA_DIR%;
+
+    location / {
+        include /etc/nginx/fastcgi_params;
+        fastcgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend;
+        fastcgi_param GIT_HTTP_EXPORT_ALL "";
+        fastcgi_param GIT_PROJECT_ROOT %DATA_DIR%;
+        fastcgi_param PATH_INFO $uri;
+        fastcgi_pass unix:/var/run/fcgiwrap.socket;
+    }
+}

terraform/ipfs-node-cloud-init.yml

@@ -21,6 +21,7 @@ hostname: ${fqdn}
 write_files:
     - path: /etc/environment
       content: |
+          DATA_DIR="${data_dir}"
           IPFS_PATH="${ipfs_path}"
           NODE_NUMBER=${ipfs_node_number}
           REGION="${region}"

terraform/ipfs-node-ec2.tf

@@ -13,6 +13,7 @@ resource "aws_instance" "ipfs_node" {
     delete = "10m"
   }
   user_data = templatefile("${abspath(path.root)}/ipfs-node-cloud-init.yml", {
+    data_dir         = var.ipfs_path
     fqdn             = "${var.prefix}-ipfs-node-${terraform.workspace}-${count.index + 1}.${var.root_domain}"
     prefix           = var.prefix
     ssh_port         = var.ssh_port

terraform/ipfs-node-elb.tf

@@ -48,7 +48,7 @@ resource "aws_elb" "ipfs" {
   connection_draining_timeout = 400
 
   tags = {
-    Name        = "${var.prefix}-${terraform.workspace}-${count.index + 1}"
+    Name        = "ipfs-${var.prefix}-${terraform.workspace}-${count.index + 1}"
     environment = terraform.workspace
     group       = var.prefix
     type        = "ipfs"

terraform/ipfs-node/init.sh

@@ -16,9 +16,11 @@ sudo mv go-ipfs/ipfs /usr/local/bin &&
 rm -rf go-ipfs
 
 ipfs init &&
-sed -i s#127.0.0.1/tcp/8080#0.0.0.0/tcp/8080#g /data/gitblox/config #Global access to gateway
+sed -i s#127.0.0.1/tcp/8080#0.0.0.0/tcp/8080#g ${DATA_DIR}/config #Global access to gateway
 
-sudo cp ./ipfs.service /etc/systemd/system/
+sed -i s#%DATA_DIR%#${DATA_DIR}#g ./ipfs.service
+sed -i s#%USER%#${USER}#g ./ipfs.service
+sudo mv ./ipfs.service /etc/systemd/system/
 sudo systemctl enable ipfs
 sudo systemctl start ipfs
 

terraform/ipfs-node/ipfs.service

@@ -7,9 +7,9 @@ Type=simple
 Restart=always
 RestartSec=1
 StartLimitInterval=0
-User=gitblox
-Group=gitblox
-Environment="IPFS_PATH=/data/gitblox"
+User=%USER%
+Group=%USER%
+Environment="IPFS_PATH=%DATA_DIR%"
 ExecStart=/usr/local/bin/ipfs daemon --enable-gc
 LimitNOFILE=500000
 

terraform/main.tf

@@ -16,6 +16,13 @@ data "aws_ebs_snapshot_ids" "ipfs_data" {
   }
 }
 
+data "aws_ebs_snapshot_ids" "git_data" {
+  filter {
+    name   = "tag:Name"
+    values = ["${var.prefix}${var.git_snapshot_name}:${terraform.workspace}"]
+  }
+}
+
 data "aws_acm_certificate" "cert" {
   domain = "*.${terraform.workspace}.${var.root_domain}"
 }

terraform/outputs.tf

@@ -1,6 +1,9 @@
 output "ipfs-node-ip" {
   value = aws_instance.ipfs_node.*.public_ip
 }
+output "git-server-ip" {
+  value = aws_instance.git_server.*.public_ip
+}
 output "prefix" {
   value = var.prefix
 }