From 376e24eed441792120371517db1784c088c7eb65 Mon Sep 17 00:00:00 2001 From: zach-blumenfeld Date: Thu, 29 Aug 2024 21:18:46 -0400 Subject: [PATCH] Adding GitHub Action for Running Notebooks --- .github/scripts/aura.py | 151 ++++++++++++++++++++++++++++ .github/workflows/run-notebooks.yml | 79 +++++++++++++++ genai-example-app-only.ipynb | 4 +- genai-workshop.ipynb | 36 +++---- ws.env.template | 11 +- 5 files changed, 261 insertions(+), 20 deletions(-) create mode 100644 .github/scripts/aura.py create mode 100644 .github/workflows/run-notebooks.yml diff --git a/.github/scripts/aura.py b/.github/scripts/aura.py new file mode 100644 index 0000000..e9773e6 --- /dev/null +++ b/.github/scripts/aura.py @@ -0,0 +1,151 @@ +import argparse +import os +import json +import time +import logging + +import requests + +logger = logging.getLogger(__name__) +logging.basicConfig(level='INFO') + + +class AuraAPI: + def __init__(self, url, tenant_id, token=None, **kwargs): + self.url = url + self.token = token + self.tenant_id = tenant_id + self.config = kwargs + + def status(self, instance_id): + headers = {"Content-Type": "application/json", "Authorization": self.token} + _url = os.path.join(self.url, instance_id) + response = requests.get(_url, headers=headers) + res = json.loads(response.content) + if not res.get('data'): + logger.info("Unable to retrieve instance Status : {}".format(instance_id)) + return 'Unknown' + status = res.get('data').get('status') + return status + + def create(self, params): + headers = {"Content-Type": "application/json", "Authorization": self.token} + params.update({ + 'tenant_id': self.tenant_id + }) + response = requests.post(self.url, headers=headers, json=params) + res = json.loads(response.content) + instance_details = res.get('data', {}) + errors = res.get('errors', {}) + if not instance_details: + logger.info("Instance creation not successful: {}".format(errors)) + return instance_details + + def delete(self, instance_id): + _url = os.path.join(self.url, instance_id) + headers = {"Content-Type": "application/json", "Authorization": self.token} + response = requests.delete(_url, headers=headers) + res = json.loads(response.content) + instance_details = res.get('data', {}) + errors = res.get('errors', {}) + if not instance_details: + logger.info("Instance not found or unable to delete: {}".format(errors)) + return dict() + return instance_details + + def generate_token(self, url, client_id, client_secret): + body = { + "grant_type": "client_credentials" + } + headers = {"Content-Type": "application/x-www-form-urlencoded"} + response = requests.post(url, auth=(client_id, client_secret), headers=headers, data=body) + data = json.loads(response.content) + token = data['access_token'] + return token + + def generate_token_if_expired(self): + auth_config = self.config['auth'] + auth_url = auth_config.get('endpoint') + client_id = auth_config.get('client_id') + client_secret = auth_config.get('client_secret') + if time.time() - auth_config.get('token_ttl') >= 3599: + self.token = self.generate_token(auth_url, client_id, client_secret) + self.config['auth']['access_token'] = self.token + self.config['auth']['token_ttl'] = time.time() + logger.info("Token Generation Successful: {}".format(time.ctime())) + return True + logger.info("Token is Valid") + return False + + def wait_for_status(self, instance_id, status=None, time_out=300): + start = time.time() + current_status = self.status(instance_id) + while current_status != status and time.time() - start <= time_out: + time.sleep(20) + current_status = self.status(instance_id) + logger.info("Waiting: {} {}".format(instance_id, status)) + return current_status + + +def cli(): + parser = argparse.ArgumentParser() + parser.add_argument('task', type=str, help='setup task', choices=['configure', 'delete']) + parser.add_argument('--tenant-id', type=str, help="Aura Tenant ID") + parser.add_argument('--client-id', type=str, help="Aura API Client ID") + parser.add_argument('--client-secret', type=str, help="Aura API Client Secret") + parser.add_argument('--region', type=str, help="Aura Region") + parser.add_argument('--cloud-provider', type=str, help="Aura Cloud Provider") + parser.add_argument('--instance-id', type=str, help="Aura Instance Id") + + return parser.parse_args() + + +def configure_instance(api, region, cloud_provider): + logger.info("Creating Aura instance") + data = api.create(params={ + "name": "gh-action-genai-workshop", + "version": "5", + "region": region, + "memory": "8GB", + "type": "enterprise-ds", + "cloud_provider": cloud_provider, + }) + instance_details = {k: v for k, v in data.items() if + k in ['id', 'connection_url', 'name', 'username', 'password']} + logger.info(f"Waiting for Aura instance {instance_details['id']} to come online") + api.wait_for_status(instance_details['id'], status="running", time_out=300) + + print(f""" +AURA_INSTANCEID={instance_details['id']} +NEO4J_URI={instance_details['connection_url']} +NEO4J_USERNAME={instance_details['username']} +NEO4J_PASSWORD={instance_details['password']} +AURA_DS=true +""") + + +def delete_instance(api, instance_id): + logger.info(f"Deleting Aura instance {instance_id}") + api.delete(instance_id) + + +if __name__ == '__main__': + args = cli() + + config = { + "auth": { + "endpoint": "https://api.neo4j.io/oauth/token", + "client_id": args.client_id, + "client_secret": args.client_secret, + "token_ttl": 0.0 + } + } + api = AuraAPI("https://api.neo4j.io/v1/instances", args.tenant_id, **config) + _ = api.generate_token_if_expired() + + task = args.task + if task == 'configure': + configure_instance(api, args.region, args.cloud_provider) + + if task == 'delete': + delete_instance(api, args.instance_id) diff --git a/.github/workflows/run-notebooks.yml b/.github/workflows/run-notebooks.yml new file mode 100644 index 0000000..a44c20d --- /dev/null +++ b/.github/workflows/run-notebooks.yml @@ -0,0 +1,79 @@ +name: Run Notebook and Commit Version With Output + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + +jobs: + run-notebooks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install jupyter nbconvert + + - name: Create env file + run: | + echo "${{ secrets.WORKSHOP_ENV }}" > ws.env + + - name: Create Aura instance + run: | + python .github/scripts/aura.py configure \ + --tenant-id $AURA_TENANT_ID \ + --client-id $AURA_CLIENT_ID \ + --client-secret $AURA_CLIENT_SECRET \ + --region $AURA_REGION \ + --cloud-provider $AURA_CLOUD_PROVIDER \ + >> ws.env + + - name: Run data loading notebook + run: | + jupyter nbconvert --to notebook --ExecutePreprocessor.timeout=1200 --execute data-load.ipynb + rm data-load.nbconvert.ipynb + env: + ENV_FILE: ws.env + + - name: Run and save workshop notebook + run: | + export AUTOMATED_RUN=true + jupyter nbconvert --to notebook --ExecutePreprocessor.timeout=1200 --execute genai-workshop.ipynb + mv genai-workshop.nbconvert.ipynb genai-workshop-w-outputs.ipynb + env: + ENV_FILE: ws.env + + - name: Run example-app-only notebook + run: | + export AUTOMATED_RUN=true + jupyter nbconvert --to notebook --ExecutePreprocessor.timeout=1200 --execute genai-example-app-only.ipynb + rm genai-example-app-only.nbconvert.ipynb + env: + ENV_FILE: ws.env + + - name: Delete Aura instance + run: | + source ws.env + python .github/scripts/aura.py delete \ + --tenant-id $AURA_TENANT_ID \ + --client-id $AURA_CLIENT_ID \ + --client-secret $AURA_CLIENT_SECRET \ + --instance-id $AURA_INSTANCEID + + - name: Commit and push notebook with outputs + run: | + git config --global user.name 'GitHub Action' + git config --global user.email 'action@github.com' + git add genai-workshop-w-outputs.ipynb + git commit -m "Auto-commit: Run notebook and update notebook with output file" + git push diff --git a/genai-example-app-only.ipynb b/genai-example-app-only.ipynb index a9f09ce..e6b1fe3 100644 --- a/genai-example-app-only.ipynb +++ b/genai-example-app-only.ipynb @@ -922,7 +922,9 @@ " outputs=message_result,\n", " examples=examples,\n", " title=\"🪄 Message Generator 🥳\")\n", - "demo.launch(share=True, debug=True)" + "\n", + "if not os.getenv('AUTOMATED_RUN') == \"true\":\n", + " demo.launch(share=True, debug=True)" ], "outputs": [ { diff --git a/genai-workshop.ipynb b/genai-workshop.ipynb index b106998..d0bb4c4 100644 --- a/genai-workshop.ipynb +++ b/genai-workshop.ipynb @@ -169,21 +169,21 @@ "outputs": [], "source": [ "# You can skip this cell if not using a ws.env file - alternative to above\n", - "# from dotenv import load_dotenv\n", - "# import os\n", + "from dotenv import load_dotenv\n", + "import os\n", "\n", - "# if os.path.exists('ws.env'):\n", - "# load_dotenv('ws.env', override=True)\n", + "if os.path.exists('ws.env'):\n", + " load_dotenv('ws.env', override=True)\n", "\n", - "# # Neo4j\n", - "# NEO4J_URI = os.getenv('NEO4J_URI')\n", - "# NEO4J_USERNAME = os.getenv('NEO4J_USERNAME')\n", - "# NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')\n", - "# AURA_DS = eval(os.getenv('AURA_DS').title())\n", + " # Neo4j\n", + " NEO4J_URI = os.getenv('NEO4J_URI')\n", + " NEO4J_USERNAME = os.getenv('NEO4J_USERNAME')\n", + " NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')\n", + " AURA_DS = eval(os.getenv('AURA_DS').title())\n", "\n", - "# # AI\n", - "# LLM = 'gpt-4o'\n", - "# OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')" + " # AI\n", + " LLM = 'gpt-4o'\n", + " OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')" ] }, { @@ -1529,7 +1529,9 @@ " outputs=message_result,\n", " examples=examples,\n", " title=\"🪄 Message Generator 🥳\")\n", - "demo.launch(share=True, debug=True) # NOTE - change share=False if you are running locally to use localhost" + "\n", + "if not os.getenv('AUTOMATED_RUN') == \"true\":\n", + " demo.launch(share=True, debug=True) # NOTE - change share=False if you are running locally to use localhost" ] }, { @@ -1542,13 +1544,11 @@ ] }, { + "metadata": {}, "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "nfODXiJUZACM" - }, "outputs": [], - "source": [] + "execution_count": null, + "source": "" } ], "metadata": { diff --git a/ws.env.template b/ws.env.template index fdd578d..799e637 100644 --- a/ws.env.template +++ b/ws.env.template @@ -9,4 +9,13 @@ NEO4J_PASSWORD= #***************************************************************** # AI #***************************************************************** -OPENAI_API_KEY=sk-... \ No newline at end of file +OPENAI_API_KEY=sk-... + +#***************************************************************** +# AURA - for GitHub Admin Only (GitHub Action CI/CD) +#***************************************************************** +AURA_TENANT_ID=<> +AURA_CLIENT_ID=<> +AURA_CLIENT_SECRET=<> +AURA_REGION=<> +AURA_CLOUD_PROVIDER=<> \ No newline at end of file