Skip to content

Commit

Permalink
fix(cli): replace fix wait time with curl loop (#245)
Browse files Browse the repository at this point in the history
Because

- large model deployment might take long

This commit

- use curl loop to check ray cluster status
  • Loading branch information
heiruwu authored Nov 28, 2024
1 parent 8e6b3a7 commit 3f62de1
Showing 1 changed file with 16 additions and 3 deletions.
19 changes: 16 additions & 3 deletions instill/helpers/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import shutil
import subprocess
import tempfile
import time
import uuid

import ray
Expand All @@ -17,6 +16,12 @@
from instill.helpers.errors import ModelConfigException
from instill.utils.logger import Logger

bash_script = """
until curl -s -o /dev/null -w "%{{http_code}}" http://localhost:8265 | grep -q "200"; do
sleep 5
done
"""


def config_check_required_fields(c):
if "build" not in c or c["build"] is None:
Expand Down Expand Up @@ -276,7 +281,6 @@ def run(args):
docker_run = False
try:
name = uuid.uuid4()

Logger.i("[Instill] Starting model image...")
if not args.gpu:
subprocess.run(
Expand Down Expand Up @@ -307,7 +311,14 @@ def run(args):
stdout=subprocess.DEVNULL,
)
docker_run = True
time.sleep(10)
subprocess.run(
f"docker exec {str(name)} /bin/bash -c '{bash_script}'",
shell=True,
check=True,
stdout=subprocess.DEVNULL,
timeout=300,
)

Logger.i("[Instill] Deploying model...")
subprocess.run(
[
Expand Down Expand Up @@ -336,6 +347,8 @@ def run(args):
)
except subprocess.CalledProcessError:
Logger.e("[Instill] Run failed")
except subprocess.TimeoutExpired:
Logger.e("[Instill] Deployment timeout")
except Exception as e:
Logger.e("[Instill] Prepare failed")
Logger.e(e)
Expand Down

0 comments on commit 3f62de1

Please sign in to comment.