diff --git a/README.md b/README.md index c4d8bfd..3348bd7 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ import tes task = tes.Task( executors=[ Executor( - image_name="alpine", - cmd=["echo", "hello"] + image="alpine", + command=["echo", "hello"] ) ] ) diff --git a/tes/__init__.py b/tes/__init__.py index a7afe3f..7c774b9 100644 --- a/tes/__init__.py +++ b/tes/__init__.py @@ -3,9 +3,9 @@ from tes.client import HTTPClient from tes.utils import unmarshal from tes.models import ( - TaskParameter, + Input, + Output, Resources, - Ports, Executor, Task, ExecutorLog, @@ -22,9 +22,9 @@ __all__ = [ HTTPClient, unmarshal, - TaskParameter, + Input, + Output, Resources, - Ports, Executor, Task, ExecutorLog, @@ -38,4 +38,4 @@ ServiceInfo ] -__version__ = "0.1.6" +__version__ = "0.2.0" diff --git a/tes/models.py b/tes/models.py index 344761f..4c4c2ad 100644 --- a/tes/models.py +++ b/tes/models.py @@ -2,6 +2,7 @@ import dateutil.parser import json +import os import six from attr import asdict, attrs, attrib @@ -101,7 +102,7 @@ def as_json(self, drop_empty=True): @attrs -class TaskParameter(Base): +class Input(Base): url = attrib( default=None, convert=strconv, validator=optional(instance_of(str)) ) @@ -117,7 +118,26 @@ class TaskParameter(Base): description = attrib( default=None, convert=strconv, validator=optional(instance_of(str)) ) - contents = attrib( + content = attrib( + default=None, convert=strconv, validator=optional(instance_of(str)) + ) + + +@attrs +class Output(Base): + url = attrib( + default=None, convert=strconv, validator=optional(instance_of(str)) + ) + path = attrib( + default=None, convert=strconv, validator=optional(instance_of(str)) + ) + type = attrib( + default="FILE", validator=in_(["FILE", "DIRECTORY"]) + ) + name = attrib( + default=None, convert=strconv, validator=optional(instance_of(str)) + ) + description = attrib( default=None, convert=strconv, validator=optional(instance_of(str)) ) @@ -130,7 +150,7 @@ class Resources(Base): ram_gb = attrib( default=None, validator=optional(instance_of((float, int))) ) - size_gb = attrib( + disk_gb = attrib( default=None, validator=optional(instance_of((float, int))) ) preemptible = attrib( @@ -141,18 +161,12 @@ class Resources(Base): ) -@attrs -class Ports(Base): - container = attrib(validator=instance_of(int)) - host = attrib(default=0, validator=instance_of(int)) - - @attrs class Executor(Base): - image_name = attrib( + image = attrib( convert=strconv, validator=instance_of(str) ) - cmd = attrib( + command = attrib( convert=strconv, validator=list_of(str) ) workdir = attrib( @@ -167,10 +181,7 @@ class Executor(Base): stderr = attrib( default=None, convert=strconv, validator=optional(instance_of(str)) ) - ports = attrib( - default=None, validator=optional(list_of(Ports)) - ) - environ = attrib( + env = attrib( default=None, validator=optional(instance_of(dict)) ) @@ -196,12 +207,6 @@ class ExecutorLog(Base): exit_code = attrib( default=None, validator=optional(instance_of(int)) ) - host_ip = attrib( - default=None, convert=strconv, validator=optional(instance_of(str)) - ) - ports = attrib( - default=None, validator=optional(list_of(Ports)) - ) @attrs @@ -238,6 +243,9 @@ class TaskLog(Base): outputs = attrib( default=None, validator=optional(list_of(OutputFileLog)) ) + system_logs = attrib( + default=None, validator=optional(list_of(str)) + ) @attrs @@ -249,23 +257,20 @@ class Task(Base): default=None, validator=optional(in_( ["UKNOWN", "QUEUED", "INITIALIZING", "RUNNING", "COMPLETE", - "PAUSED", "CANCELED", "ERROR", "SYSTEM_ERROR"] + "CANCELED", "EXECUTOR_ERROR", "SYSTEM_ERROR"] )) ) name = attrib( default=None, convert=strconv, validator=optional(instance_of(str)) ) - project = attrib( - default=None, convert=strconv, validator=optional(instance_of(str)) - ) description = attrib( default=None, convert=strconv, validator=optional(instance_of(str)) ) inputs = attrib( - default=None, validator=optional(list_of(TaskParameter)) + default=None, validator=optional(list_of(Input)) ) outputs = attrib( - default=None, validator=optional(list_of(TaskParameter)) + default=None, validator=optional(list_of(Output)) ) resources = attrib( default=None, validator=optional(instance_of(Resources)) @@ -282,44 +287,74 @@ class Task(Base): logs = attrib( default=None, validator=optional(list_of(TaskLog)) ) + creation_time = attrib( + default=None, + convert=timestampconv, + validator=optional(instance_of(datetime)) + ) def is_valid(self): - if self.executors is None: - return False, TypeError("executors NoneType") + errs = [] + if self.executors is None or len(self.executors) == 0: + errs.append("Must provide one or more Executors") else: for e in self.executors: - if e.environ is not None: - for k, v in self.executors.environ: + if e.image is None: + errs.append("Executor image must be provided") + if len(e.command) == 0: + errs.append("Executor command must be provided") + if e.stdin is not None: + if not os.path.isabs(e.stdin): + errs.append("Executor stdin must be an absolute path") + if e.stdout is not None: + if not os.path.isabs(e.stdout): + errs.append("Executor stdout must be an absolute path") + if e.stderr is not None: + if not os.path.isabs(e.stderr): + errs.append("Executor stderr must be an absolute path") + if e.env is not None: + for k, v in self.executors.env: if not isinstance(k, str) and not isinstance(k, str): - return False, TypeError( - "keys and values of environ must be StrType" + errs.append( + "Executor env keys and values must be StrType" ) if self.inputs is not None: for i in self.inputs: - if i.url is None and i.contents is None: - return False, TypeError( - "TaskParameter url must be provided" - ) - if i.url is not None and i.contents is not None: - return False, TypeError( - "TaskParameter url and contents are mutually exclusive" - ) - if i.url is None and i.path is None: - return False, TypeError( - "TaskParameter url and path must be provided" - ) + if i.url is None and i.content is None: + errs.append("Input url must be provided") + if i.url is not None and i.content is not None: + errs.append("Input url and content are mutually exclusive") + if i.path is None: + errs.append("Input path must be provided") + elif not os.path.isabs(i.path): + errs.append("Input path must be absolute") if self.outputs is not None: for o in self.outputs: - if o.url is None or o.path is None: - return False, TypeError( - "TaskParameter url and path must be provided" - ) - if o.contents is not None: - return False, TypeError( - "Output TaskParameter instances do not have contents" + if o.url is None: + errs.append("Output url must be provided") + if o.path is None: + errs.append("Output path must be provided") + elif not os.path.isabs(i.path): + errs.append("Output path must be absolute") + + if self.volumes is not None: + if len(self.volumes) > 0: + for v in self.volumes: + if not os.path.isabs(v): + errs.append("Volume paths must be absolute") + + if self.tags is not None: + for k, v in self.tags: + if not isinstance(k, str) and not isinstance(k, str): + errs.append( + "Tag keys and values must be StrType" ) + + if len(errs) > 0: + return False, TypeError("\n".join(errs)) + return True, None diff --git a/tes/utils.py b/tes/utils.py index f755d38..b5471b8 100644 --- a/tes/utils.py +++ b/tes/utils.py @@ -4,7 +4,7 @@ import re from requests import HTTPError -from tes.models import (Task, TaskParameter, Resources, Executor, Ports, +from tes.models import (Task, Input, Output, Resources, Executor, TaskLog, ExecutorLog, OutputFileLog) @@ -44,10 +44,9 @@ def unmarshal(j, o, convert_camel_case=True): omap = { "tasks": Task, - "inputs": TaskParameter, - "outputs": (TaskParameter, OutputFileLog), + "inputs": Input, + "outputs": (Output, OutputFileLog), "logs": (TaskLog, ExecutorLog), - "ports": Ports, "resources": Resources, "executors": Executor } diff --git a/tests/test_client.py b/tests/test_client.py index 5e57304..6333013 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -12,8 +12,8 @@ class TestHTTPClient(unittest.TestCase): task = Task( executors=[ Executor( - image_name="alpine", - cmd=["echo", "hello"] + image="alpine", + command=["echo", "hello"] ) ] ) diff --git a/tests/test_models.py b/tests/test_models.py index e401ffc..2c8dd95 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,15 +1,15 @@ import json import unittest -from tes.models import Task, Executor, TaskParameter, strconv +from tes.models import Task, Executor, Input, Output, strconv class TestModels(unittest.TestCase): task = Task( executors=[ Executor( - image_name="alpine", - cmd=["echo", "hello"] + image="alpine", + command=["echo", "hello"] ) ] ) @@ -17,8 +17,8 @@ class TestModels(unittest.TestCase): expected = { "executors": [ { - "image_name": "alpine", - "cmd": ["echo", "hello"] + "image": "alpine", + "command": ["echo", "hello"] } ] } @@ -30,16 +30,16 @@ def test_strconv(self): self.assertTrue(strconv(1), 1) with self.assertRaises(TypeError): - TaskParameter( - url="s3:/some/path", path="foo", contents=123 + Input( + url="s3:/some/path", path="/opt/foo", content=123 ) def test_list_of(self): with self.assertRaises(TypeError): Task( inputs=[ - TaskParameter( - url="s3:/some/path", path="foo", contents="content" + Input( + url="s3:/some/path", path="/opt/foo" ), "foo" ] @@ -55,13 +55,13 @@ def test_is_valid(self): self.assertTrue(self.task.is_valid()[0]) task2 = self.task - task2.inputs = [TaskParameter(path="foo")] + task2.inputs = [Input(path="/opt/foo")] self.assertFalse(task2.is_valid()[0]) task3 = self.task task3.outputs = [ - TaskParameter( - url="s3:/some/path", path="foo", contents="content" + Output( + url="s3:/some/path", path="foo" ) ] self.assertFalse(task3.is_valid()[0]) diff --git a/tests/test_utils.py b/tests/test_utils.py index 2718192..9178d30 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,7 @@ import unittest from tes.utils import camel_to_snake, unmarshal, UnmarshalError -from tes.models import TaskParameter, Task, CreateTaskResponse +from tes.models import Input, Task, CreateTaskResponse class TestUtils(unittest.TestCase): @@ -30,10 +30,10 @@ def test_unmarshal(self): "type": "FILE" } test_simple_str = json.dumps(test_simple_dict) - o1 = unmarshal(test_simple_dict, TaskParameter) - o2 = unmarshal(test_simple_str, TaskParameter) - self.assertTrue(isinstance(o1, TaskParameter)) - self.assertTrue(isinstance(o2, TaskParameter)) + o1 = unmarshal(test_simple_dict, Input) + o2 = unmarshal(test_simple_str, Input) + self.assertTrue(isinstance(o1, Input)) + self.assertTrue(isinstance(o2, Input)) self.assertEqual(o1, o2) self.assertEqual(o1.as_dict(), test_simple_dict) self.assertEqual(o1.as_json(), test_simple_str) @@ -56,9 +56,9 @@ def test_unmarshal(self): ], "executors": [ { - "image_name": "alpine", - "cmd": ["echo", "hello"], - "ports": [{"host": 0, "container": 8000}] + "image": "alpine", + "command": ["echo", "hello"], + "env": {"HOME": "/home/"} } ], "logs": [ @@ -71,8 +71,6 @@ def test_unmarshal(self): "start_time": "2017-10-09T17:06:30.0Z", "end_time": "2017-10-09T17:39:50.0Z", "exit_code": 0, - "host_ip": "127.0.0.1", - "ports": [{"host": 8888, "container": 8000}] } ], "outputs": [ @@ -83,7 +81,8 @@ def test_unmarshal(self): } ] } - ] + ], + "creation_time": "2017-10-09T17:00:00.0Z" } test_complex_str = json.dumps(test_complex_dict) @@ -110,5 +109,8 @@ def test_unmarshal(self): expected["logs"][0]["logs"][0]["end_time"] = dateutil.parser.parse( expected["logs"][0]["logs"][0]["end_time"] ) + expected["creation_time"] = dateutil.parser.parse( + expected["creation_time"] + ) self.assertEqual(o1.as_dict(), expected)