Adding GitHub Action for Running Notebooks

.github/scripts/aura.py

@@ -0,0 +1,151 @@
+import argparse
+import os
+import json
+import time
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level='INFO')
+
+
+class AuraAPI:
+    def __init__(self, url, tenant_id, token=None, **kwargs):
+        self.url = url
+        self.token = token
+        self.tenant_id = tenant_id
+        self.config = kwargs
+
+    def status(self, instance_id):
+        headers = {"Content-Type": "application/json", "Authorization": self.token}
+        _url = os.path.join(self.url, instance_id)
+        response = requests.get(_url, headers=headers)
+        res = json.loads(response.content)
+        if not res.get('data'):
+            logger.info("Unable to retrieve instance Status : {}".format(instance_id))
+            return 'Unknown'
+        status = res.get('data').get('status')
+        return status
+
+    def create(self, params):
+        headers = {"Content-Type": "application/json", "Authorization": self.token}
+        params.update({
+            'tenant_id': self.tenant_id
+        })
+        response = requests.post(self.url, headers=headers, json=params)
+        res = json.loads(response.content)
+        instance_details = res.get('data', {})
+        errors = res.get('errors', {})
+        if not instance_details:
+            logger.info("Instance creation not successful: {}".format(errors))
+        return instance_details
+
+    def delete(self, instance_id):
+        _url = os.path.join(self.url, instance_id)
+        headers = {"Content-Type": "application/json", "Authorization": self.token}
+        response = requests.delete(_url, headers=headers)
+        res = json.loads(response.content)
+        instance_details = res.get('data', {})
+        errors = res.get('errors', {})
+        if not instance_details:
+            logger.info("Instance not found or unable to delete: {}".format(errors))
+            return dict()
+        return instance_details
+
+    def generate_token(self, url, client_id, client_secret):
+        body = {
+            "grant_type": "client_credentials"
+        }
+        headers = {"Content-Type": "application/x-www-form-urlencoded"}
+        response = requests.post(url, auth=(client_id, client_secret), headers=headers, data=body)
+        data = json.loads(response.content)
+        token = data['access_token']
+        return token
+
+    def generate_token_if_expired(self):
+        auth_config = self.config['auth']
+        auth_url = auth_config.get('endpoint')
+        client_id = auth_config.get('client_id')
+        client_secret = auth_config.get('client_secret')
+        if time.time() - auth_config.get('token_ttl') >= 3599:
+            self.token = self.generate_token(auth_url, client_id, client_secret)
+            self.config['auth']['access_token'] = self.token
+            self.config['auth']['token_ttl'] = time.time()
+            logger.info("Token Generation Successful: {}".format(time.ctime()))
+            return True
+        logger.info("Token is Valid")
+        return False
+
+    def wait_for_status(self, instance_id, status=None, time_out=300):
+        start = time.time()
+        current_status = self.status(instance_id)
+        while current_status != status and time.time() - start <= time_out:
+            time.sleep(20)
+            current_status = self.status(instance_id)
+            logger.info("Waiting: {} {}".format(instance_id, status))
+        return current_status
+
+
+def cli():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('task', type=str, help='setup task', choices=['configure', 'delete'])
+    parser.add_argument('--tenant-id', type=str, help="Aura Tenant ID")
+    parser.add_argument('--client-id', type=str, help="Aura API Client ID")
+    parser.add_argument('--client-secret', type=str, help="Aura API Client Secret")
+    parser.add_argument('--region', type=str, help="Aura Region")
+    parser.add_argument('--cloud-provider', type=str, help="Aura Cloud Provider")
+    parser.add_argument('--instance-id', type=str, help="Aura Instance Id")
+
+    return parser.parse_args()
+
+
+def configure_instance(api, region, cloud_provider):
+    logger.info("Creating Aura instance")
+    data = api.create(params={
+        "name": "gh-action-genai-workshop",
+        "version": "5",
+        "region": region,
+        "memory": "8GB",
+        "type": "enterprise-ds",
+        "cloud_provider": cloud_provider,
+    })
+    instance_details = {k: v for k, v in data.items() if
+                        k in ['id', 'connection_url', 'name', 'username', 'password']}
+    logger.info(f"Waiting for Aura instance {instance_details['id']} to come online")
+    api.wait_for_status(instance_details['id'], status="running", time_out=300)
+
+    print(f"""
+AURA_INSTANCEID={instance_details['id']}
+NEO4J_URI={instance_details['connection_url']}
+NEO4J_USERNAME={instance_details['username']}
+NEO4J_PASSWORD={instance_details['password']}
+AURA_DS=true
+""")
+
+
+def delete_instance(api, instance_id):
+    logger.info(f"Deleting Aura instance {instance_id}")
+    api.delete(instance_id)
+
+
+if __name__ == '__main__':
+    args = cli()
+
+    config = {
+        "auth": {
+            "endpoint": "https://api.neo4j.io/oauth/token",
+            "client_id": args.client_id,
+            "client_secret": args.client_secret,
+            "token_ttl": 0.0
+        }
+    }
+    api = AuraAPI("https://api.neo4j.io/v1/instances", args.tenant_id, **config)
+    _ = api.generate_token_if_expired()
+
+    task = args.task
+    if task == 'configure':
+        configure_instance(api, args.region, args.cloud_provider)
+
+    if task == 'delete':
+        delete_instance(api, args.instance_id)

.github/workflows/run-notebooks.yml

@@ -0,0 +1,79 @@
+name: Run Notebook and Commit Version With Output
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - main
+
+jobs:
+  run-notebooks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install jupyter nbconvert
+
+      - name: Create env file
+        run: |
+          echo "${{ secrets.WORKSHOP_ENV }}" > ws.env
+
+      - name: Create Aura instance
+        run: |
+         python .github/scripts/aura.py configure \
+          --tenant-id $AURA_TENANT_ID \
+          --client-id $AURA_CLIENT_ID \
+          --client-secret $AURA_CLIENT_SECRET \
+          --region $AURA_REGION \
+          --cloud-provider $AURA_CLOUD_PROVIDER \
+          >> ws.env
+
+      - name: Run data loading notebook
+        run: |
+          jupyter nbconvert --to notebook --ExecutePreprocessor.timeout=1200 --execute data-load.ipynb
+          rm data-load.nbconvert.ipynb
+        env:
+          ENV_FILE: ws.env
+
+      - name: Run and save workshop notebook
+        run: |
+          export AUTOMATED_RUN=true
+          jupyter nbconvert --to notebook --ExecutePreprocessor.timeout=1200 --execute genai-workshop.ipynb
+          mv genai-workshop.nbconvert.ipynb genai-workshop-w-outputs.ipynb
+        env:
+          ENV_FILE: ws.env
+
+      - name: Run example-app-only notebook
+        run: |
+          export AUTOMATED_RUN=true
+          jupyter nbconvert --to notebook --ExecutePreprocessor.timeout=1200 --execute genai-example-app-only.ipynb
+          rm genai-example-app-only.nbconvert.ipynb
+        env:
+          ENV_FILE: ws.env
+
+      - name: Delete Aura instance
+        run: |
+          source ws.env
+          python .github/scripts/aura.py delete \
+           --tenant-id $AURA_TENANT_ID \
+           --client-id $AURA_CLIENT_ID \
+           --client-secret $AURA_CLIENT_SECRET \
+           --instance-id $AURA_INSTANCEID 
+
+      - name: Commit and push notebook with outputs
+        run: |
+          git config --global user.name 'GitHub Action'
+          git config --global user.email '[email protected]'
+          git add genai-workshop-w-outputs.ipynb
+          git commit -m "Auto-commit: Run notebook and update notebook with output file"
+          git push

genai-example-app-only.ipynb

@@ -922,7 +922,9 @@
     "                    outputs=message_result,\n",
     "                    examples=examples,\n",
     "                    title=\"🪄 Message Generator 🥳\")\n",
-    "demo.launch(share=True, debug=True)"
+    "\n",
+    "if not os.getenv('AUTOMATED_RUN') == \"true\":\n",
+    "    demo.launch(share=True, debug=True)"
    ],
    "outputs": [
     {

genai-workshop.ipynb

@@ -169,21 +169,21 @@
    "outputs": [],
    "source": [
     "# You can skip this cell if not using a ws.env file - alternative to above\n",
-    "# from dotenv import load_dotenv\n",
-    "# import os\n",
+    "from dotenv import load_dotenv\n",
+    "import os\n",
     "\n",
-    "# if os.path.exists('ws.env'):\n",
-    "#     load_dotenv('ws.env', override=True)\n",
+    "if os.path.exists('ws.env'):\n",
+    "     load_dotenv('ws.env', override=True)\n",
     "\n",
-    "#     # Neo4j\n",
-    "#     NEO4J_URI = os.getenv('NEO4J_URI')\n",
-    "#     NEO4J_USERNAME = os.getenv('NEO4J_USERNAME')\n",
-    "#     NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')\n",
-    "#     AURA_DS = eval(os.getenv('AURA_DS').title())\n",
+    "     # Neo4j\n",
+    "     NEO4J_URI = os.getenv('NEO4J_URI')\n",
+    "     NEO4J_USERNAME = os.getenv('NEO4J_USERNAME')\n",
+    "     NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')\n",
+    "     AURA_DS = eval(os.getenv('AURA_DS').title())\n",
     "\n",
-    "#     # AI\n",
-    "#     LLM = 'gpt-4o'\n",
-    "#     OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')"
+    "     # AI\n",
+    "     LLM = 'gpt-4o'\n",
+    "     OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')"
    ]
   },
   {
@@ -1529,7 +1529,9 @@
     "                    outputs=message_result,\n",
     "                    examples=examples,\n",
     "                    title=\"🪄 Message Generator 🥳\")\n",
-    "demo.launch(share=True, debug=True) # NOTE - change share=False if you are running locally to use localhost"
+    "\n",
+    "if not os.getenv('AUTOMATED_RUN') == \"true\":\n",
+    "    demo.launch(share=True, debug=True) # NOTE - change share=False if you are running locally to use localhost"
    ]
   },
   {
@@ -1542,13 +1544,11 @@
    ]
   },
   {
+   "metadata": {},
    "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "nfODXiJUZACM"
-   },
    "outputs": [],
-   "source": []
+   "execution_count": null,
+   "source": ""
   }
  ],
  "metadata": {

ws.env.template

@@ -9,4 +9,13 @@ NEO4J_PASSWORD=<password>
 #*****************************************************************
 # AI
 #*****************************************************************
-OPENAI_API_KEY=sk-...
+OPENAI_API_KEY=sk-...
+
+#*****************************************************************
+# AURA - for GitHub Admin Only (GitHub Action CI/CD)
+#*****************************************************************
+AURA_TENANT_ID=<>
+AURA_CLIENT_ID=<>
+AURA_CLIENT_SECRET=<>
+AURA_REGION=<>
+AURA_CLOUD_PROVIDER=<>