From 0ae3b976c27c0ac945eb9ae8f93d371a2b6afec1 Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jun 2024 11:47:35 +0200 Subject: [PATCH 1/8] Initial staging-deploy.sh script --- README.md | 7 +++++++ staging-deploy.sh | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 staging-deploy.sh diff --git a/README.md b/README.md index cc62cdaa..a20c23cd 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,13 @@ This is a *new implementation*, [merged](https://github.com/tosdr/edit.tosdr.org Phoenix, when used, also connects to two other services: Atlassian, which runs [the widget](https://status.tosdr.org) that helps us to monitor the application's health, and jrbit.de, which is where our databases live as well as our system for reporting errors. +## Deploy + +To deploy this code to https://do-staging.tosdr.org: +* get an Ubuntu 24.04 server +* ssh into it as root +* download the `staging-deploy.sh` script from this repo, and run it + ## Development Requirements: diff --git a/staging-deploy.sh b/staging-deploy.sh new file mode 100644 index 00000000..9dcd22fd --- /dev/null +++ b/staging-deploy.sh @@ -0,0 +1,2 @@ +apt update +apt install docker.io From f244ac4935ef04fcadba9d982f9f0490c2c0a3f5 Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jun 2024 12:26:03 +0200 Subject: [PATCH 2/8] Install Docker, Phoenix, pyenv, next: Hypothesis --- staging-deploy.sh | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/staging-deploy.sh b/staging-deploy.sh index 9dcd22fd..8ac2ebf5 100644 --- a/staging-deploy.sh +++ b/staging-deploy.sh @@ -1,2 +1,36 @@ -apt update -apt install docker.io +# Docker +sudo apt-get update +sudo apt-get -yq install ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get install -yq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo service docker start +sudo docker run hello-world +echo Docker installed successfully. + +# edit.tosdr.org +git clone https://github.com/tosdr/edit.tosdr.org.git +cd edit.tosdr.org +docker compose build +docker network create elasticsearch +docker network create dbs +docker compose up -d +docker exec -it edittosdrorg-web-1 rails db:seed + +# pyenv +apt install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev +curl https://pyenv.run | bash +echo -e 'export PYENV_ROOT="$HOME/.pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc +echo -e 'eval "$(pyenv init --path)"\neval "$(pyenv init -)"' >> ~/.bashrc +source ~!/.bashrc + +# Hypothesis +pyenv install 3.8.12 +pyenv init +pyenv shell 3.8.12 From d28a0405220e3e2fa9a0c10610d2f9568c95b747 Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jun 2024 12:53:56 +0200 Subject: [PATCH 3/8] Fix line 32, add more Hypothesis lines --- staging-deploy.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/staging-deploy.sh b/staging-deploy.sh index 8ac2ebf5..7332aaeb 100644 --- a/staging-deploy.sh +++ b/staging-deploy.sh @@ -12,7 +12,6 @@ sudo apt-get update sudo apt-get install -yq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin sudo service docker start sudo docker run hello-world -echo Docker installed successfully. # edit.tosdr.org git clone https://github.com/tosdr/edit.tosdr.org.git @@ -22,15 +21,22 @@ docker network create elasticsearch docker network create dbs docker compose up -d docker exec -it edittosdrorg-web-1 rails db:seed +docker compose down +cd .. # pyenv apt install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev curl https://pyenv.run | bash echo -e 'export PYENV_ROOT="$HOME/.pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc echo -e 'eval "$(pyenv init --path)"\neval "$(pyenv init -)"' >> ~/.bashrc -source ~!/.bashrc +source ~/.bashrc # Hypothesis +git clone https://github.com/tosdr/h +cd h +echo Note this will take several minutes... pyenv install 3.8.12 pyenv init pyenv shell 3.8.12 +make services +make dev \ No newline at end of file From 01cde510a87d37ea2552dbd764c35c7122c3aeec Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jun 2024 16:04:53 +0200 Subject: [PATCH 4/8] refer to https://github.com/tosdr/h/pull/1 --- README.md | 10 ++++++---- staging-deploy.sh | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a20c23cd..d28d3b87 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ H is the Hypothesis web service and api. 1. To use it with Phoenix, clone [our fork of H](https://github.com/tosdr/h) into the same directory as the Phoenix clone, and `cd h/`. **The correct branch to work from is the *phoenix-integration* branch.** + **ATTENTION**: Due to [an issue with pip-tools](https://github.com/tosdr/edit.tosdr.org/issues/1156) you might need to use the *do-staging* branch instead. + **ATTENTION**: The official documentation for installing H is [here](https://h.readthedocs.io/en/latest/developing/install/). Please also consult these docs as needed. Note the prerequisites: @@ -134,8 +136,8 @@ H is the Hypothesis web service and api. **If pyenv has trouble finding the python binary**, you may need to add configuration to `.zshrc`, as documented [here](https://stackoverflow.com/questions/51863225/pyenv-python-command-not-found). -5. `make services`, which launches the docker services needed to run H. -7. `make dev` +2. `make services`, which launches the docker services needed to run H. +3. `make dev` If this is your first time, `make dev` will install the dependencies. To do so, it requires both *node* and *yarn*. @@ -148,12 +150,12 @@ H is the Hypothesis web service and api. You will have to exit and restart `tox` whenever changes are made to the code. Additionally, in debug mode, certain functionalities may be restricted. You will not be able to create and persist annotations from the client if H is running in debug mode, for example. To launch the shell and poke around in the database, run `make shell`. -9. Create an admin user from the shell +4. Create an admin user from the shell You will need an admin user to set up OAuth between H and the Hypothesis client. Follow [these instructions](https://h.readthedocs.io/en/latest/developing/administration/). -7. Log in as admin, and configure OAuth +5. Log in as admin, and configure OAuth Instructions [here](https://h.readthedocs.io/en/latest/developing/integrating-client/). diff --git a/staging-deploy.sh b/staging-deploy.sh index 7332aaeb..e24f38fe 100644 --- a/staging-deploy.sh +++ b/staging-deploy.sh @@ -34,9 +34,11 @@ source ~/.bashrc # Hypothesis git clone https://github.com/tosdr/h cd h +git checkout do-staging echo Note this will take several minutes... -pyenv install 3.8.12 +pyenv install 3.11.7 pyenv init -pyenv shell 3.8.12 +pyenv shell 3.11.7 +python -m pip install -rrequirements/dockercompose.txt make services make dev \ No newline at end of file From dba6255ee222c63c0a0e4d68030a1e56f89e8dc4 Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jun 2024 16:50:53 +0200 Subject: [PATCH 5/8] Make a start with https://github.com/tosdr/edit.tosdr.org/issues/1158 --- README.md | 2 +- staging-deploy.sh => staging/deploy.sh | 27 ++++++++++++++------ staging/nginxproxymanager/docker-compose.yml | 12 +++++++++ 3 files changed, 32 insertions(+), 9 deletions(-) rename staging-deploy.sh => staging/deploy.sh (88%) create mode 100644 staging/nginxproxymanager/docker-compose.yml diff --git a/README.md b/README.md index d28d3b87..79e25f11 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Phoenix, when used, also connects to two other services: Atlassian, which runs [ To deploy this code to https://do-staging.tosdr.org: * get an Ubuntu 24.04 server * ssh into it as root -* download the `staging-deploy.sh` script from this repo, and run it +* download the `./staging/deploy.sh` script from this repo, and run it ## Development diff --git a/staging-deploy.sh b/staging/deploy.sh similarity index 88% rename from staging-deploy.sh rename to staging/deploy.sh index e24f38fe..1f4b9c2f 100644 --- a/staging-deploy.sh +++ b/staging/deploy.sh @@ -13,7 +13,25 @@ sudo apt-get install -yq docker-ce docker-ce-cli containerd.io docker-buildx-plu sudo service docker start sudo docker run hello-world -# edit.tosdr.org +# pyenv +apt install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev +curl https://pyenv.run | bash +echo -e 'export PYENV_ROOT="$HOME/.pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc +echo -e 'eval "$(pyenv init --path)"\neval "$(pyenv init -)"' >> ~/.bashrc +source ~/.bashrc + +# Node.js +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash +source ~/.bashrc +nvm install 20 +nvm use 20 +corepack enable +yarn --version -q + +# Postgres +apt install -q libpq-dev + +# Phoenix git clone https://github.com/tosdr/edit.tosdr.org.git cd edit.tosdr.org docker compose build @@ -24,13 +42,6 @@ docker exec -it edittosdrorg-web-1 rails db:seed docker compose down cd .. -# pyenv -apt install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev -curl https://pyenv.run | bash -echo -e 'export PYENV_ROOT="$HOME/.pyenv"\nexport PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc -echo -e 'eval "$(pyenv init --path)"\neval "$(pyenv init -)"' >> ~/.bashrc -source ~/.bashrc - # Hypothesis git clone https://github.com/tosdr/h cd h diff --git a/staging/nginxproxymanager/docker-compose.yml b/staging/nginxproxymanager/docker-compose.yml new file mode 100644 index 00000000..a7b4bbd6 --- /dev/null +++ b/staging/nginxproxymanager/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3.8' +services: + app: + image: 'jc21/nginx-proxy-manager:latest' + restart: unless-stopped + ports: + - '80:80' + - '81:81' + - '443:443' + volumes: + - ./data:/data + - ./letsencrypt:/etc/letsencrypt \ No newline at end of file From eee338663bd59190724f537fee641e9680110e08 Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 9 Sep 2024 11:49:13 +0200 Subject: [PATCH 6/8] add nginx proxy manager --- staging/deploy.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/staging/deploy.sh b/staging/deploy.sh index 1f4b9c2f..a1c39cb2 100644 --- a/staging/deploy.sh +++ b/staging/deploy.sh @@ -34,13 +34,14 @@ apt install -q libpq-dev # Phoenix git clone https://github.com/tosdr/edit.tosdr.org.git cd edit.tosdr.org +git checkout staging-deploy docker compose build docker network create elasticsearch docker network create dbs docker compose up -d docker exec -it edittosdrorg-web-1 rails db:seed docker compose down -cd .. +cd ~ # Hypothesis git clone https://github.com/tosdr/h @@ -52,4 +53,10 @@ pyenv init pyenv shell 3.11.7 python -m pip install -rrequirements/dockercompose.txt make services -make dev \ No newline at end of file +make dev +cd ~ + +# Nginx Proxy Manager +cd edit.tosdr.org/staging/nginx-proxy-manager +docker compose up -d +cd ~ \ No newline at end of file From bd85cb6298c33b40a45006a854eacb78287e4237 Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 9 Sep 2024 11:53:14 +0200 Subject: [PATCH 7/8] Looks from #1157 that I meant to point to the branch, not the branch which doesn't seem to exist --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 79e25f11..b25d05c3 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ H is the Hypothesis web service and api. 1. To use it with Phoenix, clone [our fork of H](https://github.com/tosdr/h) into the same directory as the Phoenix clone, and `cd h/`. **The correct branch to work from is the *phoenix-integration* branch.** - **ATTENTION**: Due to [an issue with pip-tools](https://github.com/tosdr/edit.tosdr.org/issues/1156) you might need to use the *do-staging* branch instead. + **ATTENTION**: Due to [an issue with pip-tools](https://github.com/tosdr/edit.tosdr.org/issues/1156) you might need to use the *staging-deploy* branch instead. **ATTENTION**: The official documentation for installing H is [here](https://h.readthedocs.io/en/latest/developing/install/). Please also consult these docs as needed. From 9d9881cc10c4a4677a8cee52a7a05d59309f05c5 Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 9 Sep 2024 10:31:21 +0000 Subject: [PATCH 8/8] disable gitpod that seems related to the old crawler --- {.gitpod => disabled.gitpod}/crawler.docker-compose.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {.gitpod => disabled.gitpod}/crawler.docker-compose.yml (100%) diff --git a/.gitpod/crawler.docker-compose.yml b/disabled.gitpod/crawler.docker-compose.yml similarity index 100% rename from .gitpod/crawler.docker-compose.yml rename to disabled.gitpod/crawler.docker-compose.yml