update stable (#139)

* Bug: simplify secrets loading (#133)

* create aztklib object, move clusterlib and joblib to objects, update cli endpoints

* fix errors in joblib

* refactor SecretsConfig loading, refactor batch and blob client loading

* fix tests

* fix test and pylint errors

* rename clusterlib to cluster and rename joblib to job

* whitespace

* rename aztklib to aztk and fix whitespace

* fix get command

* fixed issues with docker_auth and tests

* Bug: Updated confusing cluster create success message (#80)

* updated confusing success message

* updated message and whitespace

* remove old file

* Update README.md (#136)

* Feature/yaml space note (#134)

* Update secrets.yaml.template

* added spaces

README.md

@@ -14,6 +14,12 @@ Currently, this toolkit is designed to run batch Spark jobs that require additio
 
 ## Setup
 1. Clone the repo
+```bash
+    git clone -b stable https://www.github.com/azure/aztk
+
+    # You can also clone directly from master to get the latest bits
+    git clone https://www.github.com/azure/aztk
+```
 2. Use pip to install required packages (requires python 3.5+ and pip 9.0.1+)
 ```bash
     pip install -r requirements.txt

aztk/aztklib.py

@@ -0,0 +1,37 @@
+from aztk.clusterlib import Cluster
+from aztk.joblib import Job
+import aztk.azure_api as azure_api
+import aztk.config as config
+import aztk.error as error
+
+
+class Aztk:
+    def __init__(self):
+        secrets_config = config.SecretsConfig()
+        secrets_config.load_secrets_config()
+
+        blob_config = azure_api.BlobConfig(
+            account_key=secrets_config.storage_account_key,
+            account_name=secrets_config.storage_account_name,
+            account_suffix=secrets_config.storage_account_suffix
+        )
+        batch_config = azure_api.BatchConfig(
+            account_key=secrets_config.batch_account_key,
+            account_name=secrets_config.batch_account_name,
+            account_url=secrets_config.batch_service_url
+        )
+
+        self.batch_client = azure_api.make_batch_client(batch_config)
+        self.blob_client = azure_api.make_blob_client(blob_config)
+
+        self.cluster = Cluster(
+            batch_client=self.batch_client,
+            blob_client=self.blob_client,
+            batch_config=batch_config,
+            blob_config=blob_config,
+            secrets_config=secrets_config
+        )
+        self.job = Job(
+            batch_client=self.batch_client,
+            blob_client=self.blob_client
+        )

aztk/azure_api.py

@@ -6,14 +6,6 @@
 from .version import __version__
 
 
-global_config = None
-
-batch_client = None
-batch_config = None
-blob_config = None
-blob_client = None
-
-
 class BatchConfig:
     def __init__(self, account_key: str, account_name: str, account_url: str):
         self.account_key = account_key
@@ -28,43 +20,35 @@ def __init__(self, account_key: str, account_name: str, account_suffix: str):
         self.account_suffix = account_suffix
 
 
-def get_batch_client():
-    """
-        :returns: the batch client singleton
-    """
-    if not batch_client:
-        __load_batch_client()
-    return batch_client
-
-
-def get_blob_client():
-    """
-        :returns: the blob client singleton
-    """
-    if not blob_client:
-        __load_blob_client()
-    return blob_client
+def _validate_batch_config(batch_config: BatchConfig):
+
+    if batch_config.account_key is None:
+        raise error.AzureApiInitError("Batch account key is not set in secrets.yaml config")
+    if batch_config.account_name is None:
+        raise error.AzureApiInitError("Batch account name is not set in secrets.yaml config")
+    if batch_config.account_url is None:
+        raise error.AzureApiInitError("Batch service url is not set in secrets.yaml config")
 
 
-def create_batch_client(
-        batch_account_key: str,
-        batch_account_name: str,
-        batch_service_url: str):
+def make_batch_client(batch_config: BatchConfig):
     """
         Creates a batch client object
         :param str batch_account_key: batch account key
         :param str batch_account_name: batch account name
         :param str batch_service_url: batch service url
     """
+    # Validate the given config
+    _validate_batch_config(batch_config)
+
     # Set up SharedKeyCredentials
     credentials = batch_auth.SharedKeyCredentials(
-        batch_account_name,
-        batch_account_key)
+        batch_config.account_name,
+        batch_config.account_key)
 
     # Set up Batch Client
     batch_client = batch.BatchServiceClient(
         credentials,
-        base_url=batch_service_url)
+        base_url=batch_config.account_url)
 
     # Set retry policy
     batch_client.config.retry_policy.retries = 5
@@ -73,105 +57,29 @@ def create_batch_client(
     return batch_client
 
 
-def create_blob_client(
-        storage_account_key: str,
-        storage_account_name: str,
-        storage_account_suffix: str):
+def _validate_blob_config(blob_config: BlobConfig):
+    if blob_config.account_key is None:
+        raise error.AzureApiInitError("Storage account key is not set in secrets.yaml config")
+    if blob_config.account_name is None:
+        raise error.AzureApiInitError("Storage account name is not set in secrets.yaml config")
+    if blob_config.account_suffix is None:
+        raise error.AzureApiInitError("Storage account suffix is not set in secrets.yaml config")
+
+
+def make_blob_client(blob_config: BlobConfig):
     """
         Creates a blob client object
         :param str storage_account_key: storage account key
         :param str storage_account_name: storage account name
         :param str storage_account_suffix: storage account suffix
     """
+    # Validate Blob config
+    _validate_blob_config(blob_config)
+
     # Set up BlockBlobStorage
     blob_client = blob.BlockBlobService(
-        account_name=storage_account_name,
-        account_key=storage_account_key,
-        endpoint_suffix=storage_account_suffix)
-
-    return blob_client
-
-
-def get_batch_config() -> BatchConfig:
-    if not batch_config:
-        __load_batch_config()
-
-    return batch_config
-
-
-def __load_batch_config():
-    secrets_config = config.SecretsConfig()
-    secrets_config.load_secrets_config()
-
-    global batch_config
-
-    if secrets_config.batch_account_key is None:
-        raise error.AzureApiInitError("Batch account key is not set in secrets.yaml config")
-    if secrets_config.batch_account_name is None:
-        raise error.AzureApiInitError("Batch account name is not set in secrets.yaml config")
-    if secrets_config.batch_service_url is None:
-        raise error.AzureApiInitError("Batch service url is not set in secrets.yaml config")
-
-    # Get configuration
-    account_key = secrets_config.batch_account_key
-    account_name = secrets_config.batch_account_name
-    account_url = secrets_config.batch_service_url
-
-    batch_config = BatchConfig(
-        account_key=account_key, account_name=account_name, account_url=account_url)
-
-
-def __load_batch_client():
-    global batch_client
-
-    config = get_batch_config()
-
-    # create batch client
-    batch_client = create_batch_client(
-        config.account_key,
-        config.account_name,
-        config.account_url)
-
-
-def get_blob_config() -> BlobConfig:
-    if not blob_config:
-        __load_blob_config()
-
-    return blob_config
-
-def __load_blob_config():
-    secrets_config = config.SecretsConfig()
-    secrets_config.load_secrets_config()
-
-    global blob_config
-
-    if secrets_config.storage_account_key is None:
-        raise error.AzureApiInitError("Storage account key is not set in secrets.yaml config")
-    if secrets_config.storage_account_name is None:
-        raise error.AzureApiInitError("Storage account name is not set in secrets.yaml config")
-    if secrets_config.storage_account_suffix is None:
-        raise error.AzureApiInitError("Storage account suffix is not set in secrets.yaml config")
-
-    # Get configuration
-    storage_account_key = secrets_config.storage_account_key
-    storage_account_name = secrets_config.storage_account_name
-    storage_account_suffix = secrets_config.storage_account_suffix
-
-    blob_config = BlobConfig(
-        account_key=storage_account_key,
-        account_name=storage_account_name,
-        account_suffix=storage_account_suffix)
-
-
-def __load_blob_client():
-    global blob_client
-
-    blob_config = get_blob_config()
-
-    # create storage client
-    blob_client = create_blob_client(
-        blob_config.account_key,
-        blob_config.account_name,
-        blob_config.account_suffix)
-
+        account_name=blob_config.account_name,
+        account_key=blob_config.account_key,
+        endpoint_suffix=blob_config.account_suffix)
 
+    return blob_client