From 8e21f188d0c9f446b144915bd0f325a1a540d767 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 10 Aug 2024 07:16:37 +0800 Subject: [PATCH 1/9] add pythonpath . before loading modules Signed-off-by: Nelson Chen --- flytekit/bin/entrypoint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index a7fc1ed485..a9aad7f710 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -376,6 +376,8 @@ def _execute_task( dynamic_addl_distro, dynamic_dest_dir, ) as ctx: + import sys + sys.path.append(".") resolver_obj = load_object_from_module(resolver) # Use the resolver to load the actual task object _task_def = resolver_obj.load_task(loader_args=resolver_args) From 865d065b93b3e798be637e72f04e493b4ef639dd Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 10 Aug 2024 23:02:02 +0800 Subject: [PATCH 2/9] fix make lint failed Signed-off-by: Nelson Chen --- flytekit/bin/entrypoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index a9aad7f710..3a7bcd3d02 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -377,6 +377,7 @@ def _execute_task( dynamic_dest_dir, ) as ctx: import sys + sys.path.append(".") resolver_obj = load_object_from_module(resolver) # Use the resolver to load the actual task object From 2f0d01d6a5ef96454aa7e76483f2ce3add88b285 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Tue, 13 Aug 2024 22:39:18 +0800 Subject: [PATCH 3/9] improve code and remove pythonpath in Dockerfile.dev Signed-off-by: Nelson Chen --- Dockerfile.dev | 2 -- flytekit/bin/entrypoint.py | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 406740de27..de1e4fc037 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -51,7 +51,5 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \ && : -ENV PYTHONPATH "/flytekit:/flytekit/plugins/flytekit-k8s-pod:/flytekit/plugins/flytekit-deck-standard:" - # Switch to the 'flytekit' user for better security. USER flytekit diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 3a7bcd3d02..409c2b23b5 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -378,7 +378,8 @@ def _execute_task( ) as ctx: import sys - sys.path.append(".") + if "." not in sys.path: + sys.path.append(".") resolver_obj = load_object_from_module(resolver) # Use the resolver to load the actual task object _task_def = resolver_obj.load_task(loader_args=resolver_args) From de9d1dfe05dbacc91c4e6c96de2a5b47c68b4a18 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 17 Aug 2024 01:03:55 +0800 Subject: [PATCH 4/9] add pythonpath '.' if '.' isn't in sys.path Signed-off-by: Nelson Chen --- flytekit/bin/entrypoint.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 409c2b23b5..f8cc60fe6f 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -378,8 +378,10 @@ def _execute_task( ) as ctx: import sys - if "." not in sys.path: - sys.path.append(".") + working_dir = os.getcwd() + print(working_dir) + if all(os.path.realpath(path) != working_dir for path in sys.path): + sys.path.append(working_dir) resolver_obj = load_object_from_module(resolver) # Use the resolver to load the actual task object _task_def = resolver_obj.load_task(loader_args=resolver_args) From 89027078f92e4021174880d8a7d6dffbf61bedfe Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 17 Aug 2024 01:07:09 +0800 Subject: [PATCH 5/9] remove test Signed-off-by: Nelson Chen --- flytekit/bin/entrypoint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 3eaca22eaa..13eebc5b36 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -379,7 +379,6 @@ def _execute_task( import sys working_dir = os.getcwd() - print(working_dir) if all(os.path.realpath(path) != working_dir for path in sys.path): sys.path.append(working_dir) resolver_obj = load_object_from_module(resolver) From 659515c95c557d4d2b49ff04df3dc5815be5f468 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sun, 18 Aug 2024 17:02:23 +0800 Subject: [PATCH 6/9] change adding working_dir to dynamic_dest_dir Signed-off-by: Nelson Chen --- flytekit/bin/entrypoint.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 13eebc5b36..797bffe016 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -378,9 +378,8 @@ def _execute_task( ) as ctx: import sys - working_dir = os.getcwd() - if all(os.path.realpath(path) != working_dir for path in sys.path): - sys.path.append(working_dir) + if all(os.path.realpath(path) != dynamic_dest_dir for path in sys.path): + sys.path.append(dynamic_dest_dir) resolver_obj = load_object_from_module(resolver) # Use the resolver to load the actual task object _task_def = resolver_obj.load_task(loader_args=resolver_args) From c3c4bab131d4b96e40a0536ff5e16976ee39aa1d Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Mon, 19 Aug 2024 07:27:25 +0800 Subject: [PATCH 7/9] rollback changes back to previous Signed-off-by: Nelson Chen --- flytekit/bin/entrypoint.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 797bffe016..98e0d162e3 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -378,8 +378,9 @@ def _execute_task( ) as ctx: import sys - if all(os.path.realpath(path) != dynamic_dest_dir for path in sys.path): - sys.path.append(dynamic_dest_dir) + working_dir=os.getcwd() + if all(os.path.realpath(path) != working_dir for path in sys.path): + sys.path.append(working_dir) resolver_obj = load_object_from_module(resolver) # Use the resolver to load the actual task object _task_def = resolver_obj.load_task(loader_args=resolver_args) From 152d03349f5cecf7420cac5dae5c174f3998b43b Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Mon, 19 Aug 2024 07:32:42 +0800 Subject: [PATCH 8/9] lint Signed-off-by: Nelson Chen --- flytekit/bin/entrypoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 98e0d162e3..13eebc5b36 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -378,7 +378,7 @@ def _execute_task( ) as ctx: import sys - working_dir=os.getcwd() + working_dir = os.getcwd() if all(os.path.realpath(path) != working_dir for path in sys.path): sys.path.append(working_dir) resolver_obj = load_object_from_module(resolver) From 4afb6ae5992e4ed31fb320688b5bceb7b47f7b4a Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Tue, 20 Aug 2024 07:59:16 +0800 Subject: [PATCH 9/9] add working_dir in map task Signed-off-by: Nelson Chen --- flytekit/bin/entrypoint.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 13eebc5b36..edbd0c10ea 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -6,6 +6,7 @@ import pathlib import signal import subprocess +import sys import tempfile import traceback from sys import exit @@ -376,8 +377,6 @@ def _execute_task( dynamic_addl_distro, dynamic_dest_dir, ) as ctx: - import sys - working_dir = os.getcwd() if all(os.path.realpath(path) != working_dir for path in sys.path): sys.path.append(working_dir) @@ -429,6 +428,9 @@ def _execute_map_task( with setup_execution( raw_output_data_prefix, checkpoint_path, prev_checkpoint, dynamic_addl_distro, dynamic_dest_dir ) as ctx: + working_dir = os.getcwd() + if all(os.path.realpath(path) != working_dir for path in sys.path): + sys.path.append(working_dir) task_index = _compute_array_job_index() mtr = load_object_from_module(resolver)() map_task = mtr.load_task(loader_args=resolver_args, max_concurrency=max_concurrency)