From 7fd6aaaf2e681ed16af59556d64d6f1a600cef34 Mon Sep 17 00:00:00 2001 From: Florian Meinel Date: Fri, 24 Nov 2017 13:38:24 +0100 Subject: [PATCH 01/16] add backup client (python) --- backup-client-python/.gitignore | 5 +++ backup-client-python/backup-client.py | 44 +++++++++++++++++++++++++++ backup-client-python/update_api.sh | 5 +++ 3 files changed, 54 insertions(+) create mode 100644 backup-client-python/.gitignore create mode 100644 backup-client-python/backup-client.py create mode 100755 backup-client-python/update_api.sh diff --git a/backup-client-python/.gitignore b/backup-client-python/.gitignore new file mode 100644 index 0000000..f64542a --- /dev/null +++ b/backup-client-python/.gitignore @@ -0,0 +1,5 @@ +__pycache__ +*.pyc + +fossildbapi_pb2_grpc.py +fossildbapi_pb2.py diff --git a/backup-client-python/backup-client.py b/backup-client-python/backup-client.py new file mode 100644 index 0000000..9a59fc4 --- /dev/null +++ b/backup-client-python/backup-client.py @@ -0,0 +1,44 @@ +import grpc +import sys +import argparse + +import fossildbapi_pb2 +import fossildbapi_pb2_grpc + +def main(): + + commands = {'backup': backup, 'restore': restore} + + + parser = argparse.ArgumentParser() + parser.add_argument('address', metavar='address', default='localhost', nargs='?', + help='address of the fossildb server (default: %(default)s)' ) + parser.add_argument('port', metavar='port', type=int, default=8090, nargs='?', + help='port of the fossildb server (default: %(default)s)') + parser.add_argument('command', metavar='command', + help='command to execute, one of {}'.format([key for key in commands.keys()])) + + args = parser.parse_args() + + full_address = '{}:{}'.format(args.address, args.port) + + print('Requesting RestoreFromBackup to FossilDB at', full_address) + + channel = grpc.insecure_channel(full_address) + stub = fossildbapi_pb2_grpc.FossilDBStub(channel) + + reply = commands[args.command](stub) + + print(reply) + if not reply.success: + sys.exit(1) + +def restore(stub): + return stub.RestoreFromBackup(fossildbapi_pb2.RestoreFromBackupRequest()) + +def backup(stub): + return stub.Backup(fossildbapi_pb2.BackupRequest()) + + +if __name__ == '__main__': + main() diff --git a/backup-client-python/update_api.sh b/backup-client-python/update_api.sh new file mode 100755 index 0000000..86cdba5 --- /dev/null +++ b/backup-client-python/update_api.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" + +python3 -m grpc_tools.protoc -I$SCRIPTPATH/../src/main/protobuf --python_out=$SCRIPTPATH --grpc_python_out=$SCRIPTPATH $SCRIPTPATH/../src/main/protobuf/fossildbapi.proto From 3fe9e1829729be9eed22b7434115e430abc36bf4 Mon Sep 17 00:00:00 2001 From: Florian Meinel Date: Fri, 24 Nov 2017 13:49:13 +0100 Subject: [PATCH 02/16] add requirements txt for pip --- backup-client-python/backup-client.py | 3 ++- backup-client-python/requirements.txt | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) mode change 100644 => 100755 backup-client-python/backup-client.py create mode 100644 backup-client-python/requirements.txt diff --git a/backup-client-python/backup-client.py b/backup-client-python/backup-client.py old mode 100644 new mode 100755 index 9a59fc4..70ff5b6 --- a/backup-client-python/backup-client.py +++ b/backup-client-python/backup-client.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import grpc import sys import argparse @@ -9,7 +11,6 @@ def main(): commands = {'backup': backup, 'restore': restore} - parser = argparse.ArgumentParser() parser.add_argument('address', metavar='address', default='localhost', nargs='?', help='address of the fossildb server (default: %(default)s)' ) diff --git a/backup-client-python/requirements.txt b/backup-client-python/requirements.txt new file mode 100644 index 0000000..4a328a4 --- /dev/null +++ b/backup-client-python/requirements.txt @@ -0,0 +1,2 @@ +argparse +grpcio-tools From 37dbe68f24f68b13ca23d226bfc87636b468fad6 Mon Sep 17 00:00:00 2001 From: "jonathan.striebel" Date: Fri, 24 Nov 2017 15:28:09 +0000 Subject: [PATCH 03/16] Create Dockerfile --- backup-client-python/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 backup-client-python/Dockerfile diff --git a/backup-client-python/Dockerfile b/backup-client-python/Dockerfile new file mode 100644 index 0000000..abf23e8 --- /dev/null +++ b/backup-client-python/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.6-alpine + +COPY . /fossildb + +RUN pip3 install -r requirements.txt +RUN /fossildb/update_api.sh + +ENTRYPOINT /fossildb/backup-client.py From 266178baf093d6106e12de3f95dc3c38f8a2f0f1 Mon Sep 17 00:00:00 2001 From: "jonathan.striebel" Date: Fri, 24 Nov 2017 15:39:26 +0000 Subject: [PATCH 04/16] add dockerbuild for backup client --- .circleci/config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 98caa47..74f28f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,9 +30,22 @@ jobs: docker push scalableminds/fossildb:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} docker push scalableminds/fossildb:${CIRCLE_BRANCH} docker logout + build_and_push_backup_client: + machine: + image: TODO + steps: + - checkout + - run: + command: | + docker build \ + -f backup-client-python/Dockerfile \ + -t scalableminds/fossildb-backup:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} \ + -t scalableminds/fossildb-backup:${CIRCLE_BRANCH} \ + . workflows: version: 2 default: jobs: - build_and_deploy + - build_and_push_backup_client From af825845ffda7d726a034454fb17c5e51e69155b Mon Sep 17 00:00:00 2001 From: "jonathan.striebel" Date: Fri, 24 Nov 2017 15:41:47 +0000 Subject: [PATCH 05/16] fix indentation --- .circleci/config.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 74f28f9..3b2d397 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,12 +36,13 @@ jobs: steps: - checkout - run: - command: | - docker build \ - -f backup-client-python/Dockerfile \ - -t scalableminds/fossildb-backup:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} \ - -t scalableminds/fossildb-backup:${CIRCLE_BRANCH} \ - . + name: build docker + command: | + docker build \ + -f backup-client-python/Dockerfile \ + -t scalableminds/fossildb-backup:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} \ + -t scalableminds/fossildb-backup:${CIRCLE_BRANCH} \ + . workflows: version: 2 From db32868d8215371785e8e7e79abe099ba5435cdf Mon Sep 17 00:00:00 2001 From: "jonathan.striebel" Date: Fri, 24 Nov 2017 15:43:27 +0000 Subject: [PATCH 06/16] fix image TODO --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b2d397..277eb5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: docker logout build_and_push_backup_client: machine: - image: TODO + image: circleci/classic:latest steps: - checkout - run: From 416dd096058be94fc7d54246fb42461f9757dae5 Mon Sep 17 00:00:00 2001 From: "jonathan.striebel" Date: Fri, 24 Nov 2017 16:34:30 +0000 Subject: [PATCH 07/16] client Dockerfile: fix requirements path --- backup-client-python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup-client-python/Dockerfile b/backup-client-python/Dockerfile index abf23e8..dd60397 100644 --- a/backup-client-python/Dockerfile +++ b/backup-client-python/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.6-alpine COPY . /fossildb -RUN pip3 install -r requirements.txt +RUN pip3 install -r /fossildb/requirements.txt RUN /fossildb/update_api.sh ENTRYPOINT /fossildb/backup-client.py From cf1251efbbe049e24c6a9fa32a23cc647da85e8f Mon Sep 17 00:00:00 2001 From: "jonathan.striebel" Date: Fri, 24 Nov 2017 16:41:18 +0000 Subject: [PATCH 08/16] client Dockerfile: fix paths --- backup-client-python/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backup-client-python/Dockerfile b/backup-client-python/Dockerfile index dd60397..87ef6ff 100644 --- a/backup-client-python/Dockerfile +++ b/backup-client-python/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.6-alpine COPY . /fossildb -RUN pip3 install -r /fossildb/requirements.txt -RUN /fossildb/update_api.sh +RUN pip3 install -r /fossildb/backup-client-python/requirements.txt +RUN /fossildb/backup-client-python/update_api.sh -ENTRYPOINT /fossildb/backup-client.py +ENTRYPOINT /fossildb/backup-client-python/backup-client.py From b2f7dc313ef4a0d3fe6e0e5335ab52d7471ee605 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Fri, 24 Nov 2017 17:58:48 +0100 Subject: [PATCH 09/16] client: use debian, alpine is not supported --- backup-client-python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup-client-python/Dockerfile b/backup-client-python/Dockerfile index 87ef6ff..d9598f2 100644 --- a/backup-client-python/Dockerfile +++ b/backup-client-python/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-alpine +FROM python:3.6-stretch COPY . /fossildb From 23d6597afb38ac73d1e38294700069228f2fd985 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Fri, 24 Nov 2017 18:06:58 +0100 Subject: [PATCH 10/16] bash flags --- backup-client-python/update_api.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backup-client-python/update_api.sh b/backup-client-python/update_api.sh index 86cdba5..872026a 100755 --- a/backup-client-python/update_api.sh +++ b/backup-client-python/update_api.sh @@ -1,4 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash +set -Eeuo pipefail SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" From d1ba580d13c9617bb1d20f77b1e9398a483aff1f Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Fri, 24 Nov 2017 18:07:12 +0100 Subject: [PATCH 11/16] client: python formatting --- backup-client-python/backup-client.py | 46 +++++++++++++++------------ 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/backup-client-python/backup-client.py b/backup-client-python/backup-client.py index 70ff5b6..d2006a7 100755 --- a/backup-client-python/backup-client.py +++ b/backup-client-python/backup-client.py @@ -7,39 +7,45 @@ import fossildbapi_pb2 import fossildbapi_pb2_grpc + def main(): - commands = {'backup': backup, 'restore': restore} + commands = {'backup': backup, 'restore': restore} + + parser = argparse.ArgumentParser() + parser.add_argument( + 'address', metavar='address', default='localhost', nargs='?', + help='address of the fossildb server (default: %(default)s)') + parser.add_argument( + 'port', metavar='port', type=int, default=8090, nargs='?', + help='port of the fossildb server (default: %(default)s)') + parser.add_argument( + 'command', metavar='command', + help='command to execute, one of {}'.format(list(commands.keys()))) - parser = argparse.ArgumentParser() - parser.add_argument('address', metavar='address', default='localhost', nargs='?', - help='address of the fossildb server (default: %(default)s)' ) - parser.add_argument('port', metavar='port', type=int, default=8090, nargs='?', - help='port of the fossildb server (default: %(default)s)') - parser.add_argument('command', metavar='command', - help='command to execute, one of {}'.format([key for key in commands.keys()])) + args = parser.parse_args() - args = parser.parse_args() + full_address = '{}:{}'.format(args.address, args.port) - full_address = '{}:{}'.format(args.address, args.port) + print('Requesting RestoreFromBackup to FossilDB at', full_address) - print('Requesting RestoreFromBackup to FossilDB at', full_address) + channel = grpc.insecure_channel(full_address) + stub = fossildbapi_pb2_grpc.FossilDBStub(channel) - channel = grpc.insecure_channel(full_address) - stub = fossildbapi_pb2_grpc.FossilDBStub(channel) + reply = commands[args.command](stub) - reply = commands[args.command](stub) + print(reply) + if not reply.success: + sys.exit(1) - print(reply) - if not reply.success: - sys.exit(1) def restore(stub): - return stub.RestoreFromBackup(fossildbapi_pb2.RestoreFromBackupRequest()) + return stub.RestoreFromBackup(fossildbapi_pb2.RestoreFromBackupRequest()) + def backup(stub): - return stub.Backup(fossildbapi_pb2.BackupRequest()) + return stub.Backup(fossildbapi_pb2.BackupRequest()) if __name__ == '__main__': - main() + main() From 311fe114b3df28bc6cf68eaface96a1250503952 Mon Sep 17 00:00:00 2001 From: "jonathan.striebel" Date: Mon, 27 Nov 2017 10:13:30 +0000 Subject: [PATCH 12/16] circleci: add push for client --- .circleci/config.yml | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 908ce0d..e81673d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2 jobs: - build_and_deploy: + build_and_push_db: machine: image: circleci/classic:latest environment: @@ -25,17 +25,20 @@ jobs: - "~/.sbt" - run: name: build docker - command: docker build -t scalableminds/fossildb:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} . + command: | + docker build \ + -t scalableminds/fossildb:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} \ + -t scalableminds/fossildb:${CIRCLE_BRANCH} \ + . - run: - name: deployment + name: push to dockerhub command: | set -x docker login -u $DOCKER_USER -p $DOCKER_PASS - docker tag scalableminds/fossildb:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} scalableminds/fossildb:${CIRCLE_BRANCH} docker push scalableminds/fossildb:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} docker push scalableminds/fossildb:${CIRCLE_BRANCH} docker logout - build_and_push_backup_client: + build_and_push_client: machine: image: circleci/classic:latest steps: @@ -45,13 +48,21 @@ jobs: command: | docker build \ -f backup-client-python/Dockerfile \ - -t scalableminds/fossildb-backup:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} \ - -t scalableminds/fossildb-backup:${CIRCLE_BRANCH} \ + -t scalableminds/fossildb-client:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} \ + -t scalableminds/fossildb-client:${CIRCLE_BRANCH} \ . + - run: + name: push to dockerhub + command: | + set -x + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker push scalableminds/fossildb-client:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} + docker push scalableminds/fossildb-client:${CIRCLE_BRANCH} + docker logout workflows: version: 2 default: jobs: - - build_and_deploy - - build_and_push_backup_client + - build_and_push_db + - build_and_push_client From cbac29c2121d11d114310095edf9b04bdbbf439b Mon Sep 17 00:00:00 2001 From: "jonathan.striebel" Date: Mon, 27 Nov 2017 10:30:31 +0000 Subject: [PATCH 13/16] copy only necessary parts --- backup-client-python/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backup-client-python/Dockerfile b/backup-client-python/Dockerfile index d9598f2..11a602e 100644 --- a/backup-client-python/Dockerfile +++ b/backup-client-python/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.6-stretch -COPY . /fossildb +COPY src/main/protobuf /fossildb/src/main/protobuf +COPY backup-client-python /fossildb/backup-client-python RUN pip3 install -r /fossildb/backup-client-python/requirements.txt RUN /fossildb/backup-client-python/update_api.sh From ed813ede57f1755a4f66eb11a0517ca7ce233087 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Mon, 27 Nov 2017 11:38:19 +0100 Subject: [PATCH 14/16] rename backup-client-python -> client --- .circleci/config.yml | 2 +- backup-client-python/Dockerfile | 9 --------- {backup-client-python => client}/.gitignore | 0 client/Dockerfile | 11 +++++++++++ {backup-client-python => client}/backup-client.py | 0 {backup-client-python => client}/requirements.txt | 0 {backup-client-python => client}/update_api.sh | 0 7 files changed, 12 insertions(+), 10 deletions(-) delete mode 100644 backup-client-python/Dockerfile rename {backup-client-python => client}/.gitignore (100%) create mode 100644 client/Dockerfile rename {backup-client-python => client}/backup-client.py (100%) rename {backup-client-python => client}/requirements.txt (100%) rename {backup-client-python => client}/update_api.sh (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index e81673d..a0fa773 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ jobs: name: build docker command: | docker build \ - -f backup-client-python/Dockerfile \ + -f client/Dockerfile \ -t scalableminds/fossildb-client:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} \ -t scalableminds/fossildb-client:${CIRCLE_BRANCH} \ . diff --git a/backup-client-python/Dockerfile b/backup-client-python/Dockerfile deleted file mode 100644 index 11a602e..0000000 --- a/backup-client-python/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3.6-stretch - -COPY src/main/protobuf /fossildb/src/main/protobuf -COPY backup-client-python /fossildb/backup-client-python - -RUN pip3 install -r /fossildb/backup-client-python/requirements.txt -RUN /fossildb/backup-client-python/update_api.sh - -ENTRYPOINT /fossildb/backup-client-python/backup-client.py diff --git a/backup-client-python/.gitignore b/client/.gitignore similarity index 100% rename from backup-client-python/.gitignore rename to client/.gitignore diff --git a/client/Dockerfile b/client/Dockerfile new file mode 100644 index 0000000..16c2a17 --- /dev/null +++ b/client/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.6-stretch + +COPY src/main/protobuf /fossildb/src/main/protobuf +COPY client /fossildb/client + +WORKDIR /fossildb/client + +RUN pip3 install -r requirements.txt +RUN update_api.sh + +ENTRYPOINT backup-client.py diff --git a/backup-client-python/backup-client.py b/client/backup-client.py similarity index 100% rename from backup-client-python/backup-client.py rename to client/backup-client.py diff --git a/backup-client-python/requirements.txt b/client/requirements.txt similarity index 100% rename from backup-client-python/requirements.txt rename to client/requirements.txt diff --git a/backup-client-python/update_api.sh b/client/update_api.sh similarity index 100% rename from backup-client-python/update_api.sh rename to client/update_api.sh From f441bcba14168d3ea6e93bd22f68768c5f5aba8f Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Mon, 27 Nov 2017 11:45:45 +0100 Subject: [PATCH 15/16] client Dockerfile: fix paths --- client/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/Dockerfile b/client/Dockerfile index 16c2a17..5cc183f 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -6,6 +6,6 @@ COPY client /fossildb/client WORKDIR /fossildb/client RUN pip3 install -r requirements.txt -RUN update_api.sh +RUN ./update_api.sh -ENTRYPOINT backup-client.py +ENTRYPOINT /fossildb/client/backup-client.py From a44fa7982a205e1a1ae2caaf1d6e66465bbb22aa Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Mon, 27 Nov 2017 15:37:07 +0100 Subject: [PATCH 16/16] Update config.yml --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0fa773..b422ac9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - restore_cache: key: cache-{{ .Branch }} - run: - name: build + name: Build server command: docker-compose run sbt sbt assembly - save_cache: key: cache-{{ .Branch }} @@ -24,14 +24,14 @@ jobs: - "~/.ivy2" - "~/.sbt" - run: - name: build docker + name: Build server docker image command: | docker build \ -t scalableminds/fossildb:${CIRCLE_BRANCH}__${CIRCLE_BUILD_NUM} \ -t scalableminds/fossildb:${CIRCLE_BRANCH} \ . - run: - name: push to dockerhub + name: Push to DockerHub command: | set -x docker login -u $DOCKER_USER -p $DOCKER_PASS @@ -44,7 +44,7 @@ jobs: steps: - checkout - run: - name: build docker + name: Build client docker image command: | docker build \ -f client/Dockerfile \ @@ -52,7 +52,7 @@ jobs: -t scalableminds/fossildb-client:${CIRCLE_BRANCH} \ . - run: - name: push to dockerhub + name: Push to DockerHub command: | set -x docker login -u $DOCKER_USER -p $DOCKER_PASS