From 3fbff329759873360542fdec49125c482b7c1ed6 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 6 May 2020 16:05:56 +0100 Subject: [PATCH 1/7] Dockerize the project for development - Dockerfile for app and datastore emulator - docker-compose.yml to orchestrate the containers --- docker-compose.yml | 25 +++++++++++++++++++++++++ docker/app/Dockerfile | 29 +++++++++++++++++++++++++++++ docker/app/run.sh | 15 +++++++++++++++ docker/datastore/Dockerfile | 8 ++++++++ docker/datastore/run.sh | 3 +++ 5 files changed, 80 insertions(+) create mode 100644 docker-compose.yml create mode 100644 docker/app/Dockerfile create mode 100755 docker/app/run.sh create mode 100644 docker/datastore/Dockerfile create mode 100755 docker/datastore/run.sh diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..58e1ad7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3' + +services: + app: + build: + context: . + dockerfile: ./docker/app/Dockerfile + image: app + depends_on: + - datastore + volumes: + - .:/app + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8082:8082" + - "127.0.0.1:8083:8083" + command: /run.sh + + datastore: + build: + context: . + dockerfile: ./docker/datastore/Dockerfile + image: datastore + ports: + - "127.0.0.1:8081:8081" diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile new file mode 100644 index 0000000..13de65e --- /dev/null +++ b/docker/app/Dockerfile @@ -0,0 +1,29 @@ +FROM python:2-slim + +RUN apt-get update \ + # dependencies for building Python packages + && apt-get install -y build-essential \ + && apt-get install -y python-dev \ + && apt-get install -y wget \ + && apt-get install -y zip unzip + +WORKDIR /usr/src/app + +COPY requirements ./requirements +COPY ./docker/app/run.sh ./run.sh +RUN pip install -r requirements/requirements.txt -t lib/ +RUN pip install -r requirements/local_requirements.txt + +# Install App engine SDK +RUN wget https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.90.zip -nv -P /usr/src/ +RUN unzip -q /usr/src/google_appengine_1.9.90.zip -d /usr/src/ +ENV SDK_LOCATION="/usr/src/google_appengine" + +# Install chromedriver +RUN wget https://chromedriver.storage.googleapis.com/81.0.4044.69/chromedriver_linux64.zip +RUN unzip -q chromedriver_linux64.zip +RUN cp chromedriver /usr/local/bin/ + +COPY . . + +ENTRYPOINT [ "./run.sh" ] diff --git a/docker/app/run.sh b/docker/app/run.sh new file mode 100755 index 0000000..01ac2a4 --- /dev/null +++ b/docker/app/run.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +HOST="0.0.0.0" +API_PORT="8082" +ADMIN_PORT="8083" +DATASTORE_EMULATOR_HOST_PORT=datastore:8081 + +python $SDK_LOCATION/dev_appserver.py --api_host 0.0.0.0 \ + --api_port "$API_PORT" \ + --admin_host "$HOST" \ + --admin_port "$ADMIN_PORT" \ + --host "$HOST" \ + --skip_sdk_update_check 1 . \ + --env_var DATASTORE_EMULATOR_HOST="$DATASTORE_EMULATOR_HOST_PORT" \ + --env_var DATASTORE_USE_PROJECT_ID_AS_APP_ID=true diff --git a/docker/datastore/Dockerfile b/docker/datastore/Dockerfile new file mode 100644 index 0000000..b9d640f --- /dev/null +++ b/docker/datastore/Dockerfile @@ -0,0 +1,8 @@ +FROM google/cloud-sdk:latest + +WORKDIR /usr/src/app +COPY ./docker/datastore/run.sh ./run.sh + +ENV CLOUDSDK_CORE_PROJECT=sympy-live-hrd + +ENTRYPOINT [ "./run.sh" ] diff --git a/docker/datastore/run.sh b/docker/datastore/run.sh new file mode 100755 index 0000000..950ecae --- /dev/null +++ b/docker/datastore/run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +gcloud beta emulators datastore start --host-port=0.0.0.0:8081 From 4559b7513093fede0fb40bc683f5e50c63ec9d29 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 6 May 2020 18:03:45 +0100 Subject: [PATCH 2/7] Install Chrome in the container for testing --- docker/app/Dockerfile | 9 +++++++++ tests/conftest.py | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 13de65e..dd7bf5a 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -19,6 +19,15 @@ RUN wget https://storage.googleapis.com/appengine-sdks/featured/google_appengine RUN unzip -q /usr/src/google_appengine_1.9.90.zip -d /usr/src/ ENV SDK_LOCATION="/usr/src/google_appengine" +# Install Google Chrome for running tests +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - +RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' +RUN apt-get -y update +RUN apt-get install -y google-chrome-stable + +# Set display port to avoid crash +ENV DISPLAY=:99 + # Install chromedriver RUN wget https://chromedriver.storage.googleapis.com/81.0.4044.69/chromedriver_linux64.zip RUN unzip -q chromedriver_linux64.zip diff --git a/tests/conftest.py b/tests/conftest.py index 7542c2c..660ff6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,10 @@ def browser(config): elif config['browser'] == 'Headless Chrome': opts = selenium.webdriver.ChromeOptions() opts.add_argument('headless') + opts.add_argument('--no-sandbox') + opts.add_argument('--window-size=1420,1080') + opts.add_argument('--disable-gpu') + b = selenium.webdriver.Chrome(options=opts) else: raise Exception('Browser "%s" is not supported' % config["browser"]) From 34c51e487664e460d5b05f87e6a2798648912ebd Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 6 May 2020 19:38:05 +0100 Subject: [PATCH 3/7] Support hot reloading for development --- .travis.yml | 36 ++++++++---------------------------- docker-compose.yml | 4 +++- docker/app/Dockerfile | 16 ++++++++-------- 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index 800531d..0c8d47f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,43 +2,22 @@ language: python python: - '2.7' -cache: - directories: - - "$HOME/google-cloud-sdk/" +services: + - docker -virtualenv: - system_site_packages: true - -addons: - chrome: stable - -# Stub: do nothing +# Do nothing before_install: - ls install: - - pip install -r requirements/requirements.txt -t lib/ - - pip install -r requirements/local_requirements.txt + - ls before_script: - - cd .. - - gcloud version || true - - if [ ! -d "$HOME/google-cloud-sdk/bin" ]; then rm -rf "$HOME/google-cloud-sdk"; curl https://sdk.cloud.google.com | bash > /dev/null; fi - - source /home/travis/google-cloud-sdk/path.bash.inc - - gcloud version - - gcloud components install cloud-datastore-emulator --quiet - - gcloud beta emulators datastore start & - - wget https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.90.zip -nv - - unzip -q google_appengine_1.9.90.zip - - export SDK_LOCATION="$(pwd)/google_appengine" - - wget https://chromedriver.storage.googleapis.com/81.0.4044.69/chromedriver_linux64.zip - - unzip -q chromedriver_linux64.zip - - cp chromedriver $HOME/bin/ - - cd $TRAVIS_BUILD_DIR - - python $SDK_LOCATION/dev_appserver.py --skip_sdk_update_check 1 . --env_var DATASTORE_EMULATOR_HOST=localhost:8081 --env_var DATASTORE_USE_PROJECT_ID_AS_APP_ID=true & + - docker-compose up script: - - pytest tests -v + - docker-compose exec app pytest tests/ -v before_deploy: + - docker cp app:/usr/src/app/lib lib - openssl aes-256-cbc -K $encrypted_2fd045226a67_key -iv $encrypted_2fd045226a67_iv -in client-secret.json.enc -out ../client-secret.json -d - version=$(if [ ! -z "$TRAVIS_TAG" ]; then echo $(cut -d'-' -f2 <<<"$TRAVIS_TAG"); @@ -62,6 +41,7 @@ after_deploy: - status_code=$(curl --write-out %{http_code} --silent --output /dev/null https://$version-dot-sympy-live-hrd.appspot.com/status) - echo "App status code => $status_code" - if [ "$status_code" != "200" ]; then travis_terminate; fi + - docker-compose down env: global: diff --git a/docker-compose.yml b/docker-compose.yml index 58e1ad7..98fcdae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,10 +6,11 @@ services: context: . dockerfile: ./docker/app/Dockerfile image: app + container_name: app depends_on: - datastore volumes: - - .:/app + - ./app/:/usr/src/app/app/ ports: - "127.0.0.1:8080:8080" - "127.0.0.1:8082:8082" @@ -21,5 +22,6 @@ services: context: . dockerfile: ./docker/datastore/Dockerfile image: datastore + container_name: datastore ports: - "127.0.0.1:8081:8081" diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index dd7bf5a..c3aa58c 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -10,8 +10,9 @@ RUN apt-get update \ WORKDIR /usr/src/app COPY requirements ./requirements -COPY ./docker/app/run.sh ./run.sh +COPY ./docker/app/run.sh /run.sh RUN pip install -r requirements/requirements.txt -t lib/ + RUN pip install -r requirements/local_requirements.txt # Install App engine SDK @@ -20,7 +21,7 @@ RUN unzip -q /usr/src/google_appengine_1.9.90.zip -d /usr/src/ ENV SDK_LOCATION="/usr/src/google_appengine" # Install Google Chrome for running tests -RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub -P /usr/src/ | apt-key add - RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' RUN apt-get -y update RUN apt-get install -y google-chrome-stable @@ -28,11 +29,10 @@ RUN apt-get install -y google-chrome-stable # Set display port to avoid crash ENV DISPLAY=:99 -# Install chromedriver -RUN wget https://chromedriver.storage.googleapis.com/81.0.4044.69/chromedriver_linux64.zip -RUN unzip -q chromedriver_linux64.zip -RUN cp chromedriver /usr/local/bin/ - COPY . . -ENTRYPOINT [ "./run.sh" ] +# Install chromedriver +RUN wget https://chromedriver.storage.googleapis.com/81.0.4044.69/chromedriver_linux64.zip -P /usr/src/ +RUN unzip -q /usr/src/chromedriver_linux64.zip -d /usr/local/bin/ + +ENTRYPOINT [ "/run.sh" ] From d38ad559ff76ebebae2af9f78e74e70a3168940e Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 6 May 2020 20:10:35 +0100 Subject: [PATCH 4/7] Run docker-compose in background --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0c8d47f..18c5a4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,9 @@ install: - ls before_script: - - docker-compose up + - docker-compose up -d script: + - sleep 2 # Wait for api to be up - docker-compose exec app pytest tests/ -v before_deploy: From 152b8b3c5b07f6965e2a329b5ed522ba266dbea0 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 6 May 2020 22:31:46 +0100 Subject: [PATCH 5/7] Add instructions for running via docker --- README.rst | 149 +++++++++++++---------------------------------------- 1 file changed, 35 insertions(+), 114 deletions(-) diff --git a/README.rst b/README.rst index d00f655..0d1cd94 100644 --- a/README.rst +++ b/README.rst @@ -20,87 +20,33 @@ evaluation request is a subject to this limit. There are also other limits related to memory consumption, output size, etc. (see Google App Engine documentation for details). -Installation ------------- - -Download 1.9.xx version of the original Google App Engine SDK for Python from -https://cloud.google.com/appengine/docs/python/download/ - it may be neccessary -to click ``Or, you can download the original App Engine SDK for Python.`` link -first to expand proper download section. Than move in the command line to the -location where you downloaded SDK and unpack the downloaded package. - - $ unzip google_appengine_1.9.xx.zip - -On the Mac, it is a disk image with an application, which you should -drag to your Applications folder. Open the program and install the -symlinks (it should ask you the first time you open the application, but -if it doesn't, choose "Make Symlinks..." from the -GoogleAppEngineLauncher menu). Note that you will have to do this again -each time you update the AppEngine program. - -Then clone sympy-live repository:: - - $ git clone git://github.com/sympy/sympy-live.git - $ cd sympy-live - -We use submodules to include external libraries in sympy-live:: - - $ git submodule init - $ git submodule update - -This is sufficient to clone appropriate repositories in correct versions -into sympy-live (see git documentation on submodules for information). - -You will need to install Datastore Emulator as well, which comes from gcloud's SDK, -install the Google Cloud SDK for your OS from here: https://cloud.google.com/sdk/install -Then run the following commands to install and run the datastore emulator in the background:: - - $ gcloud components install cloud-datastore-emulator --quiet - $ gcloud beta emulators datastore start & - -Install Dependencies --------------------- - -The project depends on some third-party libraries that are not on the list -of built-in libraries (in app.yaml) bundled with the runtime, to install them -run the following command.:: - - $ pip install -r requirements/requirements.txt -t lib/ - -Some libraries although available on app engine runtime, but needs to be -installed locally for development. - -Ref: https://cloud.google.com/appengine/docs/standard/python/tools/using-libraries-python-27#local_development :: - - $ pip install -r requirements/local_requirements.txt - -Development server +Development Server ------------------ -Now you are ready to run development web server:: +To setup the development environment and run the app locally, you +need ``docker`` and ``docker-compose``: - $ ../google_appengine/dev_appserver.py . +* https://docs.docker.com/get-docker/ +* https://docs.docker.com/compose/install/ -On the Mac, just run:: - - $ dev_appserver . +Now you are ready to run development web server:: -(make sure you installed the symlinks as described above). + $ docker-compose up -I couldn't figure out how to make it work in the GUI (it won't find the -sympy git submodule). If you figure out how to do it, please update -this file and send a patch describing how to do it. +This will build and run the image for app and datastore emulator. -This is a local server that runs on port 8080 (use ``--port`` option to -change this). Open a web browser and go to http://localhost:8080. You -should see GUI of SymPy Online Shell. +This will spin up a local server that runs on port ``8080``. +Open a web browser and go to http://localhost:8080. +You should see GUI of SymPy Online Shell. -Deploying to GAE ----------------- +Deploying to Google App Engine +------------------------------ Travis-CI is used to deploy automatically to the official server -via Github Releases. Go to https://github.com/sympy/sympy-live/releases -and click on create a release and name the release as `version-NN` +via Github Releases. + +* Go to https://github.com/sympy/sympy-live/releases +* Click on create a release and name the release as `version-NN` where `NN` is the release version. After this travis will automatically release version `NN`. @@ -118,6 +64,11 @@ the google cloud console for the project:: $ gcloud init +You need to to create ``lib`` (libraries) before deploying, make sure the development +server is up and running via ``docker-compose``, as mentioned above and create +libraries folder to package with the following command:: + + $ docker cp app:/usr/src/app/lib lib Assuming that sympy-live works properly (also across different mainstream web browsers), you can upload your changes to Google App Engine, replacing the @@ -160,14 +111,22 @@ Testing on the App Engine ------------------------- It's usually a good idea to test big changes on the App Engine itself before -deploying, as ``dev_appserver.py`` can only simulate the App Engine. There is -a semi-official testing server at sympy-live-tests.appspot.com. If you want +deploying, as ``dev_appserver.py`` can only simulate the App Engine. + +There is a semi-official testing server at sympy-live-tests.appspot.com. If you want write access to it, just ask Aaron Meurer. The convention there is to push to the version corresponding to the pull request (so if you have a branch that is pull request #55, you would push to version 55, and access it by 55-dot-sympy-live-tests.appspot.com). Alternately, you can set up your own testing server (it's free, though it requires a cell phone to set up). +You need to to create ``lib`` (libraries) before deploying, make sure the development +server is up and running via ``docker-compose``, as mentioned above and create +libraries folder to package with the following command:: + + $ docker cp app:/usr/src/app/lib lib + + Either way, to test, you will need to edit the Project ID in the deploy command mentioned above with your Project ID and the version you want to deploy to:: @@ -194,51 +153,13 @@ includes Chrome, Firefox, Safari and Internet Explorer. Be extra cautious about trailing commas in JavaScript object and arrays. IE doesn't allow them, so you have to remove them, if any were introduced. -GAE development server allows to use any Python interpreter, but Google -App Engine uses Python 2.5, so if the default Python isn't 2.5, then make -sure to test your changes to the server part, if it runs properly on 2.5. -Also don't use any modules that aren't supported by GAE. - Running Tests ------------- -To run the tests you need to have ``pytest`` and ``selenium`` installed. These are -already present in the ``local_requirements.txt``. If you you have installed -them you may skip this step. Otherwise to install testing requirements:: - - $ pip install selenium pytest - -You would also need latest version of one of the supported browsers: - -* Chrome: https://www.google.com/chrome/ -* Firefox: https://www.mozilla.org/firefox/new/ - -You need to download the driver for your browser as well: - -For chrome, get the driver for your system from here: -https://chromedriver.chromium.org/downloads - -Here is an example for Linux:: - - $ wget https://chromedriver.storage.googleapis.com/81.0.4044.69/chromedriver_linux64.zip - $ unzip -q chromedriver_linux64.zip - -For Firefox, get the driver for your system from here: -https://github.com/mozilla/geckodriver/releases - -After extracting the browser driver from the zip or tar file, for Unix based systems, -copy the driver to following directory:: - - $ mv /path/to/ChromeDriver /usr/local/bin - $ mv /path/to/geckodriver /usr/local/bin - -For other systems like Windows, you need to put it in PATH. - -Now run the application in the background, (See ``dev_appserver`` instructions above to start the app). - -Run selenium tests via the following command:: +To run tests you need to spinup the container as mentioned above +via ``docker-compose`` and run the following command:: - $ pytest tests -v + $ docker-compose exec app pytest tests/ -v Pulling changes --------------- From 2df3a7183010a180afd63a88f97b33a76a92757f Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 6 May 2020 22:50:59 +0100 Subject: [PATCH 6/7] Remove host from port mapping --- docker-compose.yml | 8 ++++---- docker/app/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 98fcdae..7e5de17 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,9 +12,9 @@ services: volumes: - ./app/:/usr/src/app/app/ ports: - - "127.0.0.1:8080:8080" - - "127.0.0.1:8082:8082" - - "127.0.0.1:8083:8083" + - "8080:8080" + - "8082:8082" + - "8083:8083" command: /run.sh datastore: @@ -24,4 +24,4 @@ services: image: datastore container_name: datastore ports: - - "127.0.0.1:8081:8081" + - "8081:8081" diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index c3aa58c..0467f19 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -11,8 +11,8 @@ WORKDIR /usr/src/app COPY requirements ./requirements COPY ./docker/app/run.sh /run.sh -RUN pip install -r requirements/requirements.txt -t lib/ +RUN pip install -r requirements/requirements.txt -t lib/ RUN pip install -r requirements/local_requirements.txt # Install App engine SDK From 6d304d901f810d50a03897db1b1030dff5f62a8c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 6 May 2020 23:03:04 +0100 Subject: [PATCH 7/7] Log response from github API --- bin/update_status_on_pr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/update_status_on_pr.py b/bin/update_status_on_pr.py index 7968388..dbac9e9 100644 --- a/bin/update_status_on_pr.py +++ b/bin/update_status_on_pr.py @@ -24,7 +24,7 @@ def get_branch_commit_sha(branch_name): if response.status_code == 200: response_json = response.json() else: - raise ValueError('Invalid response from github API') + raise ValueError('Invalid response from github API: %s', response.content) return response_json[0]['object']['sha']