Skip to content
This repository has been archived by the owner on Oct 28, 2023. It is now read-only.

Commit

Permalink
Merge pull request #80 from aske-w/development
Browse files Browse the repository at this point in the history
Add swarm to production
  • Loading branch information
aske-w authored Apr 19, 2022
2 parents 54c3494 + 0e270a3 commit 292c44d
Show file tree
Hide file tree
Showing 30 changed files with 2,448 additions and 106 deletions.
117 changes: 75 additions & 42 deletions .github/workflows/cicd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ jobs:
docker build -t ${{ secrets.DOCKER_USERNAME }}/minitwit-client:latest . -f ./client/Dockerfile
docker push ${{ secrets.DOCKER_USERNAME }}/minitwit-client:latest
# Doesnt work ATM, see https://github.com/Azure/container-scan/issues/122
# Instead we have set up snyk.io to continuesly monitor our container directly from Docker Hub
scan:
name: "Scan the container"
needs: build
Expand All @@ -50,60 +52,91 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: copy file via ssh password
- name: copy file to manager node via ssh password
uses: appleboy/scp-action@master
with:
key: ${{ secrets.SSH_PRIVATE_KEY }}
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USER }}
key: ${{ secrets.XSSH_PRIVATE_KEY }}
host: ${{ secrets.XSERVER_IP }}
username: ${{ secrets.XSERVER_USER }}
port: 22
strip_components: 2
source: "./remote_files/docker-compose.yml, ./filebeat/filebeat.yml"
target: "/vagrant"
source: "./swarm/docker-compose.yml, ./swarm/filebeat.yml, ./swarm/grafana/, ./swarm/prometheus/"
target: "/root"
- name: copy file to worker node via ssh password
uses: appleboy/scp-action@master
with:
key: ${{ secrets.XSSH_PRIVATE_KEY }}
host: ${{ secrets.WORKER_IP }}
username: ${{ secrets.XSERVER_USER }}
port: 22
strip_components: 2
source: "./swarm/filebeat.yml"
target: "/root"

deploy:
name: "Deploy to droplet"
needs: copy
runs-on: ubuntu-latest
steps:
- name: Deploy to Digital Ocean droplet via SSH action
uses: appleboy/[email protected]
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
MYSQL_USER: ${{ secrets.MYSQL_USER }}
MYSQL_DATABASE: ${{ secrets.MYSQL_DATABASE }}
MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}
MYSQL_ADDRESS: ${{ secrets.MYSQL_ADDRESS }}
MYSQL_PORT: 25060
GRAFANA_ADMIN_USERNAME: ${{ secrets.GRAFANA_ADMIN_USERNAME }}
GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
ELK_USER: ${{ secrets.ELK_USER }}
ELASTIC_USER: ${{ secrets.ELASTIC_USER }}
ELASTIC_PASSWORD: ${{ secrets.ELASTIC_PASSWORD }}
with:
key: ${{ secrets.SSH_PRIVATE_KEY }}
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USER }}
port: 22
envs: DOCKER_USERNAME,MYSQL_USER,MYSQL_DATABASE,MYSQL_PASSWORD,MYSQL_ADDRESS,MYSQL_PORT,GRAFANA_ADMIN_USERNAME,GRAFANA_ADMIN_PASSWORD,ELK_USER,ELASTIC_USER,ELASTIC_PASSWORD

script: |
cd /vagrant
- name: Shh into worker node
uses: appleboy/[email protected]
with:
key: ${{ secrets.XSSH_PRIVATE_KEY }}
host: ${{ secrets.WORKER_IP }}
username: ${{ secrets.XSERVER_USER }}
port: 22
script: |
cd /root
chown root filebeat.yml
chmod go-w filebeat.yml
export DOCKER_USERNAME=$DOCKER_USERNAME
export MYSQL_USER=$MYSQL_USER
export MYSQL_DATABASE=$MYSQL_DATABASE
export MYSQL_PASSWORD=$MYSQL_PASSWORD
export MYSQL_ADDRESS=$MYSQL_ADDRESS
export MYSQL_PORT=$MYSQL_PORT
export GRAFANA_ADMIN_USERNAME=$GRAFANA_ADMIN_USERNAME
export GRAFANA_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD
export ELK_USER=$ELK_USER
export ELASTIC_USER=$ELASTIC_USER
export ELASTIC_PASSWORD=$ELASTIC_PASSWORD
- name: Deploy to Digital Ocean droplet via SSH action
uses: appleboy/[email protected]
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
MYSQL_USER: ${{ secrets.XMYSQL_USER }}
MYSQL_DATABASE: ${{ secrets.XMYSQL_DATABASE }}
MYSQL_PASSWORD: ${{ secrets.XMYSQL_PASSWORD }}
MYSQL_ADDRESS: ${{ secrets.XMYSQL_ADDRESS }}
MYSQL_PORT: 25060
GRAFANA_ADMIN_USERNAME: ${{ secrets.GRAFANA_ADMIN_USERNAME }}
GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
ELK_USER: ${{ secrets.ELK_USER }}
ELASTIC_USER: ${{ secrets.ELASTIC_USER }}
ELASTIC_PASSWORD: ${{ secrets.ELASTIC_PASSWORD }}
BUCKET_NAME: ${{ secrets.BUCKET_NAME }}
SECRET_KEY: ${{ secrets.SECRET_KEY }}
ACCESS_KEY: ${{ secrets.ACCESS_KEY }}
STACK_NAME: ${{ secrets.STACK_NAME }}
with:
key: ${{ secrets.XSSH_PRIVATE_KEY }}
host: ${{ secrets.XSERVER_IP }}
username: ${{ secrets.XSERVER_USER }}
port: 22
envs: DOCKER_USERNAME,MYSQL_USER,MYSQL_DATABASE,MYSQL_PORT,MYSQL_ADDRESS,MYSQL_PASSWORD,GRAFANA_ADMIN_USERNAME,GRAFANA_ADMIN_PASSWORD,ELK_USER,ELASTIC_USER,ELASTIC_PASSWORD,STACK_NAME,ACCESS_KEY,SECRET_KEY,BUCKET_NAME
script: |
cd /root
export DOCKER_USERNAME=$DOCKER_USERNAME
export MYSQL_USER=$MYSQL_USER
export MYSQL_DATABASE=$MYSQL_DATABASE
export MYSQL_PASSWORD=$MYSQL_PASSWORD
export MYSQL_ADDRESS=$MYSQL_ADDRESS
export MYSQL_PORT=$MYSQL_PORT
export GRAFANA_ADMIN_USERNAME=$GRAFANA_ADMIN_USERNAME
export GRAFANA_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD
export ELK_USER=$ELK_USER
export ELASTIC_USER=$ELASTIC_USER
export ELASTIC_PASSWORD=$ELASTIC_PASSWORD
export STACK_NAME=$STACK_NAME
export ACCESS_KEY=$ACCESS_KEY
export SECRET_KEY=$SECRET_KEY
export BUCKET_NAME=$BUCKET_NAME
chown root filebeat.yml
chmod go-w filebeat.yml
docker-compose pull
docker-compose up -d
docker-compose pull
docker stack deploy --compose-file docker-compose.yml $STACK_NAME
versioning:
name: "Tag and release"
Expand Down
7 changes: 4 additions & 3 deletions .github/workflows/static_analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@ jobs:
- name: Checkout Source
uses: actions/checkout@v2
- name: Install correct version of go
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: 1.x
# GOSEC and Lichen doesnt work with 1.18.x
go-version: '1.17.6'

- name: Format
working-directory: ./server
Expand All @@ -29,7 +30,7 @@ jobs:
- name: Examine source
working-directory: ./server
run: if [ "$(go vet)" -gt 0 ]; then exit 1; fi

- name: Download Gosec
uses: wei/wget@v1
with:
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,6 @@ config.monitoring
server/access.log
server/itu-minitwit
.htpasswd
runner-build-errors.log
runner-build

73 changes: 72 additions & 1 deletion README.MD
Original file line number Diff line number Diff line change
@@ -1,3 +1,46 @@
# Developing

## Setup

**First make sure the following is installed:**

- Docker & docker-compose or Docker Desktop
- Nodejs v14 or greater
- Go v1.17 or greater

In `./client` you need to run `npm install`

If you want automatic reloading in go (server development):

`go install github.com/pilu/fresh`

**Setting ENV:**

Make sure env files in the following directories are set:

_In each directory you'll find a `.env.example`. You need to create a file called `.env`_

- ./ (root)
- ./client
- ./server
- ./docker_db

## Running

In `./` run `docker compose up -d` (only if you need logging and the other services)

In `./docker_db` run `docker compose up -d`

In `./client` run `npm start`

If you have installed fresh:

- In `./server` run `fresh`

else:

- In `./server` run `go run main.go`

# Maintainability and Technical Debt estimation badges

## Sonarqube
Expand All @@ -11,7 +54,6 @@
[![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=aske-w_itu-minitwit&metric=ncloc)](https://sonarcloud.io/summary/new_code?id=aske-w_itu-minitwit)
[![Technical Debt](https://sonarcloud.io/api/project_badges/measure?project=aske-w_itu-minitwit&metric=sqale_index)](https://sonarcloud.io/summary/new_code?id=aske-w_itu-minitwit)


## Code Climate

[![Maintainability](https://api.codeclimate.com/v1/badges/305085e64353cb758288/maintainability)](https://codeclimate.com/github/aske-w/itu-minitwit/maintainability)
Expand Down Expand Up @@ -178,3 +220,32 @@ Under `Kibana/Index Patterns` do the following
Once the index pattern is created navigate `Analytics/Discover` in the sidebar. The select the newly created index pattern.

We can the filter by fields. For example if we only want to see the message of the logs we can under the `Avaliable fields` press the plus icon to filter by messages.


## Docker swarm


### Provisioning

In swarm directory run the following files in their respective order:

- prov.sh

-
### Nodes

Manager node ip: ```165.22.67.59```

Worker node ip: ```165.22.81.184```

### manager
username: manager
password:

### worker1
username: worker1
password:

### worker2
username: worker2
password:
2 changes: 1 addition & 1 deletion client/.env.production
Original file line number Diff line number Diff line change
@@ -1 +1 @@
REACT_APP_BASE_URL=http://67.207.75.163:8080/api
REACT_APP_BASE_URL=http://138.68.125.155:8080/api
2 changes: 1 addition & 1 deletion client/src/pages/Signup.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const Signup = () => {

api.post("/register", form)
.then(() => {
api.post("signin", {
api.post("/signin", {
username: form.username,
password: form.pwd,
}).then(response => {
Expand Down
1 change: 1 addition & 0 deletions joincommand.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
docker yeet
26 changes: 13 additions & 13 deletions remote_files/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@ networks:
name: itu-minitwit-network

services:
server:
image: ${DOCKER_USERNAME}/minitwitimage
container_name: minitwit
environment:
- MYSQL_USER=${MYSQL_USER:?err}
- MYSQL_DATABASE=${MYSQL_DATABASE:?err}
- MYSQL_PASSWORD=${MYSQL_PASSWORD:?err}
- MYSQL_ADDRESS=${MYSQL_ADDRESS:?err}
- MYSQL_PORT=${MYSQL_PORT:?err}
ports:
- 8080:8080 # host port : container port
networks:
- main
# server:
# image: ${DOCKER_USERNAME}/minitwitimage
# container_name: minitwit
# environment:
# - MYSQL_USER=${MYSQL_USER:?err}
# - MYSQL_DATABASE=${MYSQL_DATABASE:?err}
# - MYSQL_PASSWORD=${MYSQL_PASSWORD:?err}
# - MYSQL_ADDRESS=${MYSQL_ADDRESS:?err}
# - MYSQL_PORT=${MYSQL_PORT:?err}
# ports:
# - 8080:8080 # host port : container port
# networks:
# - main

client:
image: ${DOCKER_USERNAME}/minitwit-client
Expand Down
1 change: 1 addition & 0 deletions server/database/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func ConnectMySql() (*gorm.DB, error) {
address := os.Getenv("MYSQL_ADDRESS")
port := os.Getenv("MYSQL_PORT")
db_name := os.Getenv("MYSQL_DATABASE")

dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", user, password, address, port, db_name)

db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{
Expand Down
5 changes: 5 additions & 0 deletions server/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ require (
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/howeyc/fsnotify v0.9.0 // indirect
github.com/iris-contrib/go.uuid v2.0.0+incompatible // indirect
github.com/iris-contrib/jade v1.1.4 // indirect
github.com/iris-contrib/schema v0.0.6 // indirect
Expand All @@ -53,6 +54,8 @@ require (
github.com/kataras/tunnel v0.0.3 // indirect
github.com/klauspost/compress v1.14.4 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mediocregopher/radix/v3 v3.8.0 // indirect
github.com/microcosm-cc/bluemonday v1.0.18 // indirect
Expand All @@ -61,6 +64,8 @@ require (
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pilu/config v0.0.0-20131214182432-3eb99e6c0b9a // indirect
github.com/pilu/fresh v0.0.0-20190826141211-0fa698148017 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
Expand Down
11 changes: 11 additions & 0 deletions server/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk=
github.com/iris-contrib/go.uuid v2.0.0+incompatible h1:XZubAYg61/JwnJNbZilGjf3b3pB80+OQg2qf6c8BfWE=
Expand Down Expand Up @@ -229,6 +231,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mediocregopher/radix/v3 v3.8.0 h1:HI8EgkaM7WzsrFpYAkOXIgUKbjNonb2Ne7K6Le61Pmg=
Expand All @@ -254,6 +260,10 @@ github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pilu/config v0.0.0-20131214182432-3eb99e6c0b9a h1:Tg4E4cXPZSZyd3H1tJlYo6ZreXV0ZJvE/lorNqyw1AU=
github.com/pilu/config v0.0.0-20131214182432-3eb99e6c0b9a/go.mod h1:9Or9aIl95Kp43zONcHd5tLZGKXb9iLx0pZjau0uJ5zg=
github.com/pilu/fresh v0.0.0-20190826141211-0fa698148017 h1:XXDLZIIt9NqdeIEva0DM+z1npM0Tsx6h5TYqwNvXfP0=
github.com/pilu/fresh v0.0.0-20190826141211-0fa698148017/go.mod h1:2LLTtftTZSdAPR/iVyennXZDLZOYzyDn+T0qEKJ8eSw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -451,6 +461,7 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
Loading

0 comments on commit 292c44d

Please sign in to comment.