Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tests: add kfp_v2 notebook for testing v2 pipelines #53

Merged
merged 2 commits into from
Nov 22, 2023
Merged

tests: add kfp_v2 notebook for testing v2 pipelines #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
759a375
tests: add kfp_v2 notebook for testing v2 pipelines
DnPlas Nov 21, 2023
2954a09
Use separate names of each Notebook for KFP v1 and v2
kimwnasptd Nov 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
0 tests/notebooks/kfp/kfp-integration.ipynb → ...notebooks/kfp_v1/kfp-v1-integration.ipynb
View file
Open in desktop
File renamed without changes.
0 tests/notebooks/kfp/requirements.in → tests/notebooks/kfp_v1/requirements.in
View file
Open in desktop
File renamed without changes.
0 tests/notebooks/kfp/requirements.txt → tests/notebooks/kfp_v1/requirements.txt
View file
Open in desktop
File renamed without changes.
304 changes: 304 additions & 0 deletions tests/notebooks/kfp_v2/kfp-v2-integration.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,304 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "ba70a2ba-3645-419b-8f8d-3c75e5864af1",
"metadata": {},
"source": [
"# Test KFP Integration\n",
"\n",
"- create an experiment\n",
"- create a run\n",
"- check run passes\n",
"\n",
"This Notebook is based on the condition Kubeflow pipelines [example](https://github.com/kubeflow/pipelines/blob/master/samples/core/condition/condition_v2.py)."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "28f75e55-7bad-44e7-a65f-aedc81734a48",
"metadata": {
"tags": [
"pytest-skip"
]
},
"outputs": [],
"source": [
"# pin kfp to the latest v2 to ensure compatibility\n",
"# with the KFP API server version deployed in CKF 1.8\n",
"!pip install \"kfp>=2.4,<3.0\" tenacity -q"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "4cdd7548-bae9-4430-b548-f420d72a8aec",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import kfp\n",
"import os\n",
"\n",
"from kfp import dsl\n",
"from tenacity import retry, stop_after_attempt, wait_exponential"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "fd576641-1ff4-4fbb-9b3a-122abbd281ed",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/lib/python3.11/site-packages/kfp/client/client.py:159: FutureWarning: This client only works with Kubeflow Pipeline v2.0.0-beta.2 and later versions.\n",
" warnings.warn(\n"
]
}
],
"source": [
"client = kfp.Client()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "af70bb9d-3fea-40d7-acb9-649007b0bde6",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"EXPERIMENT_NAME = 'Flip a coin and output tails/heads pipeline' "
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "40a3a9e1-0645-474e-8451-92ccba88a122",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/lib/python3.11/site-packages/kfp/dsl/component_decorator.py:119: FutureWarning: Python 3.7 has reached end-of-life. The default base_image used by the @dsl.component decorator will switch from 'python:3.7' to 'python:3.8' on April 23, 2024. To ensure your existing components work with versions of the KFP SDK released after that date, you should provide an explicit base_image argument and ensure your component works as intended on Python 3.8.\n",
" return component_factory.create_component_from_func(\n"
]
}
],
"source": [
"@dsl.component()\n",
"def flip_coin(force_flip_result: str = '') -> str:\n",
" \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n",
" if force_flip_result:\n",
" return force_flip_result\n",
" import random\n",
" result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n",
" return result"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "1d134c8b-54a7-4d10-ae2f-321ff305600a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"@dsl.component()\n",
"def print_msg(msg: str):\n",
" \"\"\"Print a message.\"\"\"\n",
" print(msg)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "c8132d87-877c-4bfb-9127-e1f964fe3acb",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_356/2573238994.py:6: DeprecationWarning: dsl.Condition is deprecated. Please use dsl.If instead.\n",
" with dsl.Condition(flip1.output == 'heads'):\n"
]
}
],
"source": [
"@dsl.pipeline(name='condition-v2')\n",
"def condition_pipeline(text: str = 'condition test', force_flip_result: str = ''):\n",
" flip1 = flip_coin(force_flip_result=force_flip_result)\n",
" print_msg(msg=flip1.output)\n",
"\n",
" with dsl.Condition(flip1.output == 'heads'):\n",
" flip2 = flip_coin()\n",
" print_msg(msg=flip2.output)\n",
" print_msg(msg=text)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "b85cc961-b6cc-4434-a59d-31e4c8a6e175",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<a href=\"/pipeline/#/experiments/details/721a46c5-c6c9-4d28-af04-00a8503673ac\" target=\"_blank\" >Experiment details</a>."
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<a href=\"/pipeline/#/runs/details/398eb5c4-fc3a-46a3-b69d-2b06419db8c0\" target=\"_blank\" >Run details</a>."
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"run = client.create_run_from_pipeline_func(\n",
" condition_pipeline,\n",
" experiment_name=EXPERIMENT_NAME,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "37ebdc86-a16d-40a0-bc7e-33a2b90914f8",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"[{'created_at': datetime.datetime(2023, 11, 21, 10, 35, tzinfo=tzlocal()),\n",
" 'description': None,\n",
" 'display_name': 'Flip a coin and output tails/heads pipeline',\n",
" 'experiment_id': '721a46c5-c6c9-4d28-af04-00a8503673ac',\n",
" 'namespace': 'daniela',\n",
" 'storage_state': 'AVAILABLE'}]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.list_experiments().experiments"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "3226c13b-9d08-47e7-812f-47529c02d9dc",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"'SUCCEEDED'"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.get_run(run.run_id).state"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d945e7ba-dc63-46f5-93e4-a1edfe56aa81",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"@retry(\n",
" wait=wait_exponential(multiplier=2, min=1, max=10),\n",
" stop=stop_after_attempt(30),\n",
" reraise=True,\n",
")\n",
"def assert_run_succeeded(client, run_id):\n",
" \"\"\"Wait for the run to complete successfully.\"\"\"\n",
" status = client.get_run(run_id).state\n",
" assert status == \"SUCCEEDED\", f\"KFP run in {status} state.\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a9d0fa8-a825-4d0d-a384-3022dc451ba8",
"metadata": {
"tags": [
"raises-exception"
]
},
"outputs": [],
"source": [
"# fetch KFP experiment to ensure it exists\n",
"client.get_experiment(experiment_name=EXPERIMENT_NAME)\n",
"\n",
"assert_run_succeeded(client, run.run_id)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
4 changes: 4 additions & 0 deletions tests/notebooks/kfp_v2/requirements.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Fix kfp to a version that is compatible with the v2.0.3
# pipelines backend
kfp>=2.4,<3.0
tenacity
Loading