diff --git a/config/dxtools.conf b/config/dxtools.conf index 9f02d3d..a388f1c 100644 --- a/config/dxtools.conf +++ b/config/dxtools.conf @@ -1,12 +1,13 @@ { "myve2": [ { - "ip_address": "18.207.2.95", - "username": "delphix_admin", - "password": "delphix", - "use_https": "True", - "default": "True", - "hostname": "myve2" + "ip_address": "HOSTNAME OF ENGINE", + "username": "admin", + "password": "Delphix", + "use_https": true, + "default": true, + "hostname": "myve2", + "isEncrypted": false } ] } diff --git a/delphix_admin_setup.py b/delphix_admin_setup.py new file mode 100755 index 0000000..ce5472a --- /dev/null +++ b/delphix_admin_setup.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +import getopt +import logging +import signal +import sys +import time +import traceback +from os.path import basename + +from delphixpy.v1_11_16.delphix_engine import DelphixEngine +from delphixpy.v1_11_16.exceptions import HttpError +from delphixpy.v1_11_16.exceptions import JobError +from delphixpy.v1_11_16.web import user +from delphixpy.v1_11_16.web import system +from delphixpy.v1_11_16.web import domain +from delphixpy.v1_11_16.web import vo +#from delphixpy.v1_16_11.web.vo import PasswordCredential +#from delphixpy.v1_16_11.web.vo import User + +from lib import dlpx_exceptions +from lib import dx_logging as log +from lib import get_references +from lib import get_session +from lib import run_job +from lib.run_async import run_async + +""" +This script configures the delphix_admin user after domain0 is configured +Will come back and properly throw this with logging, etc +""" +VERSION = "v.3.0.002" + + +def serversess(f_engine_address, f_engine_username, f_engine_password, system_init=True): + """ + Function to grab the server session + """ + if system_init is True: + server_session = DelphixEngine( + f_engine_address, f_engine_username, f_engine_password, "SYSTEM" + ) + else: + server_session = DelphixEngine( + f_engine_address, f_engine_username, f_engine_password, "DOMAIN" + ) + return server_session + + +def help(): + print( + "\n" + + basename(__file__) + + " [-e ] [-o - Engine must be up, unconfigured, and console screen must be green" + ) + print( + "-o - will use this password to initially access the system" + ) + print( + "-p - will set the delphix_admin user to this password" + ) + sys.exit(2) + + +def on_exit(sig, func=None): + log.print_info("Shutdown Command Received") + log.print_info("Shutting down prime_setup.py") + sys.exit(0) + + +def set_exit_handler(func): + signal.signal(signal.SIGTERM, func) + + +def time_elapsed(time_start): + elapsed_minutes = round((time.time() - time_start) / 60, +1) + return elapsed_minutes + + +def main(argv): + try: + # ADD OPTION FOR LOG + log.logging_est("delphix_admin.log") + time_start = time.time() + engine_ip = "" + engine_pass = "" + old_engine_pass = "" + try: + opts, args = getopt.getopt(argv, "t:e:o:p:hv") + except getopt.GetoptError: + help() + for opt, arg in opts: + if opt == "-h": + help() + elif opt == "-e": + engine_ip = arg + elif opt == "-o": + old_engine_pass = arg + elif opt == "-p": + engine_pass = arg + elif opt == "-t": + engine_type = arg.upper() +# if not engine_ip or not engine_pass or not old_engine_pass: +# help() + + server = serversess(engine_ip, "sysadmin", old_engine_pass) + init_engine = vo.SystemInitializationParameters() + engine_system = vo.SystemInfo() + if engine_type == "MASKING": + engine_system.engine_type = "MASKING" + system.start_masking(server) + elif engine_type == "VIRTUALIZATION": + system.stop_masking(server) + engine_system.engine_type = "VIRTUALIZATION" + system.set(server, system_info=engine_system) + init_engine.default_user = "admin" + init_engine.default_password = engine_pass + init_engine.default_email = "spam@delphix.com" + init_engine.devices = ["STORAGE_DEVICE-xvdb"] + domain.initialize_system(server, init_engine) + domain_server = serversess(engine_ip, "admin", "delphix", False) + #if user.get(server, "USER-2").email_address is None: + time.sleep(120) + if user.get(domain_server, "USER-2").email_address is None: + log.print_debug("Setting admin's email address") + admin_user = vo.User() + admin_user.email_address = "spam@delphix.com" + user.update(domain_server, "USER-2", admin_user) + log.print_debug("Setting admin's password") + admin_credupdate = vo.CredentialUpdateParameters() + admin_credupdate.new_credential = vo.PasswordCredential() + admin_credupdate.new_credential.password = engine_pass + user.update_credential(domain_server, "USER-2", delphix_admin_credupdate) + else: + log.print_info("The delphix_admin user has already been setup") + + + except SystemExit as e: + sys.exit(e) + except HttpError as err: + log.print_exception("Connection failed to the Delphix Engine") + log.print_exception("Please check the ERROR message below") + log.print_exception(err) + sys.exit(2) + except JobError as e: + log.print_exception("A job failed in the Delphix Engine") + log.print_exception(e.job) + elapsed_minutes = time_elapsed() + log.print_info("took " + str(elapsed_minutes) + " minutes to get this far.") + except KeyboardInterrupt: + log.print_debug("You sent a CTRL+C to interrupt the process") + elapsed_minutes = time_elapsed() + log.print_info("took " + str(elapsed_minutes) + " minutes to get this far.") + except: + log.print_exception(sys.exc_info()[0]) + log.print_exception(traceback.format_exc()) + elapsed_minutes = time_elapsed(time_start) + log.print_info("took " + str(elapsed_minutes) + " minutes to get this far.") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/dx_provision_vdb.py b/dx_provision_vdb.py old mode 100755 new mode 100644 index c5365f8..3d0ebca --- a/dx_provision_vdb.py +++ b/dx_provision_vdb.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -# Adam Bowen - Apr 2016 # This script provisions a vdb or dSource -# Updated by Corey Brune Aug 2016 # --- Create vFiles VDB # requirements # pip install docopt delphixpy @@ -118,8 +116,7 @@ from lib import run_job from lib.run_async import run_async -VERSION = "v.0.3.007" - +VERSION = 'v.0.3.007' def create_ase_vdb( dlpx_obj, @@ -488,6 +485,7 @@ def create_oracle_si_vdb( dlpx_obj.jobs[dlpx_obj.server_session.address] = dlpx_obj.server_session.last_job + def create_oracle_mt_vdb( dlpx_obj, group_ref, @@ -532,6 +530,7 @@ def create_oracle_mt_vdb( """ engine_name = list(dlpx_obj.dlpx_ddps)[0] cdb_obj = get_references.find_obj_by_name( + dlpx_obj.server_session, sourceconfig, ARGUMENTS["--source"] ) try: diff --git a/dx_replication.py b/dx_replication.py new file mode 100755 index 0000000..dc0b7ee --- /dev/null +++ b/dx_replication.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +# Description: +# This script will setup replication between two hosts. +# +# Requirements +# pip install docopt delphixpy + +# The below doc follows the POSIX compliant standards and allows us to use +# this doc to also define our ARGUMENTS for the script. +"""Description +Usage: + dx_replication.py --rep_name --target_host --target_user --target_pw --rep_objs [--schedule --bandwidth --num_cons --enabled] + [--engine | --all] [--single_thread ] + [--debug] [--parallel ] [--poll ] + [--config ] [--logdir ] + dx_replication.py --delete + [--config ] [--logdir ] + dx_replication.py --execute + [--config ] [--logdir ] + dx_replication.py --list + [--config ] [--logdir ] + + dx_replication.py -h | --help | -v | --version + +Description +Setup replication between two hosts. +Examples: +dx_replication.py --rep_name mytest --target_host 172.16.169.141 --target_user admin --target_pw delphix --rep_objs mytest1 --schedule '55 0 19 * * ?' --enabled +dx_replication.py --rep_name mytest --target_host 172.16.169.141 --target_user admin --target_pw delphix --rep_objs mytest1 --schedule '0 40 20 */4 * ?' --bandwidth 5 --num_cons 2 --enabled + +dx_replication.py --delete mytest + +Options: + --rep_name Name of the replication job. + --target_host Name / IP of the target replication host. + --target_user Username for the replication target host. + --target_pw Password for the user. + --schedule Schedule of the replication job in crontab format. (seconds, minutes, hours, day of month, month) + [default: 0 0 0 * * ?] + --rep_objs Comma delimited list of objects to replicate. + --delete Name of the replication job to delete. + --bandwidth Limit bandwidth to MB/s. + --num_cons Number of network connections for the replication job. + --list List all of the replication jobs. + --execute Name of the replication job to execute. + --single_thread Run as a single thread? True or False + [default: False] + --engine Alt Identifier of Delphix engine in dxtools.conf. + [default: default] + --all Run against all engines. + --debug Enable debug logging + --parallel Limit number of jobs to maxjob + --poll The number of seconds to wait between job polls + [default: 10] + --config The path to the dxtools.conf file + [default: ./dxtools.conf] + --logdir The path to the logfile you want to use. + [default: ./dx_replication.log] + -h --help Show this screen. + -v --version Show version. +""" +from __future__ import print_function + +VERSION = "v.0.1.002" + +import sys +from os.path import basename +from time import sleep +import time +from tabulate import tabulate +import docopt + +from delphixpy.v1_10_2 import exceptions +from delphixpy.v1_10_2.web import host +from delphixpy.v1_10_2.web import vo +from delphixpy.v1_10_2.web.replication import spec + +from lib import dlpx_exceptions +from lib import dx_logging +from lib import get_references +from lib import get_session +from lib import run_job +from lib.run_async import run_async + +def create_replication_job(dx_session_obj): + """ + Create a replication job + :return: Reference to the spec object + """ + rep_spec = vo.ReplicationSpec() + rep_spec.name = ARGUMENTS["--rep_name"] + rep_spec.target_host = ARGUMENTS["--target_host"] + rep_spec.target_principal = ARGUMENTS["--target_user"] + rep_spec.target_credential = { + "type": "PasswordCredential", + "password": ARGUMENTS["--target_pw"], + } + rep_spec.object_specification = vo.ReplicationSecureList() + rep_spec.schedule = ARGUMENTS["--schedule"] + rep_spec.encrypted = True + + if ARGUMENTS["--num_cons"]: + rep_spec.number_of_connections = int(ARGUMENTS["--num_cons"]) + if ARGUMENTS["--bandwidth"]: + rep_spec.bandwidth_limit = int(ARGUMENTS["--bandwidth"]) + if ARGUMENTS["--enabled"]: + rep_spec.enabled = True + try: + rep_spec.object_specification.containers = get_references.find_obj_specs( + dx_session_obj.server_session, ARGUMENTS["--rep_objs"].split(",") + ) + #rep_spec.object_specification.objects = get_references.find_obj_specs( + # dx_session_obj.server_session, ARGUMENTS["--rep_objs"].split(",") + #) + + ref = spec.create(dx_session_obj.server_session, rep_spec) + if dx_session_obj.server_session.last_job: + dx_session_obj.jobs[ + dx_session_obj.server_session.address + ] = dx_session_obj.server_session.last_job + dx_logging.print_info( + "Successfully created {} with reference " + "{}\n".format(ARGUMENTS["--rep_name"], ref) + ) + + except (exceptions.HttpError, exceptions.RequestError, dlpx_exceptions.DlpxException) as e: + dx_logging.print_exception( + "Could not create replication job {}:\n{}".format( + ARGUMENTS["--rep_name"], e + ) + ) + + +def delete_replication_job(dx_session_obj): + """ + Delete a replication job. + :return: Reference to the spec object + """ + try: + spec.delete( + dx_session_obj.server_session, + get_references.find_obj_by_name( + dx_session_obj.server_session, spec, ARGUMENTS["--delete"] + ).reference, + ) + if dx_session_obj.server_session.last_job: + dx_session_obj.jobs[ + dx_session_obj.server_session.address + ] = dx_session_obj.server_session.last_job + dx_logging.print_info("Successfully deleted {}.\n".format(ARGUMENTS["--delete"])) + + except (exceptions.HttpError, exceptions.RequestError, dlpx_exceptions.DlpxException) as e: + print_exception( + "Was not able to delete {}:\n{}".format(ARGUMENTS["--delete"], e) + ) + + +def list_replication_jobs(dx_session_obj): + """ + List the replication jobs on a given engine + """ + table_lst = [] + final_lst = [] + #import pdb;pdb.set_trace() + for rep_job in spec.get_all(dx_session_obj.server_session): + table_lst = [rep_job.name, + rep_job.reference, + rep_job.schedule, + rep_job.bandwidth_limit, + ] + final_lst.append(table_lst) + print (tabulate(final_lst, + headers=["Name", "Reference", + "Schedule", "Bandwidth Limit"])) + #for obj_spec_ref in rep_job.object_specification: + # obj_names_lst.append( + # database.get(dx_session_obj.server_session, obj_spec_ref).name + # ) + #print ( + # "Name: {}\nReplicated Objects: {}\nEnabled: {}\nEncrypted: {}\n" + # "Reference: {}\nSchedule: {}\nTarget Host: {}\n\n".format( + # rep_job.name, + # ", ".join(obj_names_lst), + # rep_job.enabled, + # rep_job.encrypted, + # rep_job.reference, + # rep_job.schedule, + # rep_job.target_host, + # ) + #) + + +def execute_replication_job(obj_name, dx_session_obj): + """ + Execute a replication job immediately. + :param obj_name: name of object to execute. + """ + try: + spec.execute( + dx_session_obj.server_session, + get_references.find_obj_by_name(dx_session_obj.server_session, spec, obj_name).reference, + ) + if dx_session_obj.server_session.last_job: + dx_session_obj.jobs[ + dx_session_obj.server_session.address + ] = dx_session_obj.server_session.last_job + dx_logging.print_info("Successfully executed {}.\n".format(obj_name)) + except (exceptions.HttpError, exceptions.RequestError, dlpx_exceptions.DlpxException, exceptions.JobError) as e: + print_exception("Could not execute job {}:\n{}".format(obj_name, e)) + + +@run_async +def main_workflow(engine, dlpx_obj, single_thread): + """ + This function is where we create our main workflow. + Use the @run_async decorator to run this function asynchronously. + The @run_async decorator allows us to run against multiple Delphix Engine + simultaneously + :param engine: Dictionary of engines + :type engine: dictionary + :param dlpx_obj: DDP session object + :type dlpx_obj: lib.GetSession.GetSession object + :param single_thread: True - run single threaded, False - run multi-thread + :type single_thread: bool + """ + try: + # Setup the connection to the Delphix DDP + dlpx_obj.dlpx_session( + engine["ip_address"], engine["username"], engine["password"], + engine["use_https"] + ) + except dlpx_exceptions.DlpxException as err: + dx_logging.print_exception( + f"ERROR: dx_environment encountered an error authenticating to " + f' {engine["ip_address"]} :\n{err}' + ) + try: + with dlpx_obj.job_mode(single_thread): + if ARGUMENTS["--rep_name"]: + create_replication_job(dlpx_obj) + elif ARGUMENTS["--delete"]: + delete_replication_job(dlpx_obj) + elif ARGUMENTS["--list"]: + list_replication_jobs(dlpx_obj) + elif ARGUMENTS["--execute"]: + execute_replication_job(ARGUMENTS["--execute"], dlpx_obj) + except ( + dlpx_exceptions.DlpxException, + exceptions.RequestError, + exceptions.JobError, + exceptions.HttpError, + ) as err: + dx_logging.print_exception( + f"Error in dx_environment for engine:" + f'{engine["ip_address"]}: Error Message: {err}' + ) + except (exceptions.HttpError, + exceptions.RequestError, + exceptions.JobError, + dlpx_exceptions.DlpxException + ) as err: + print_exception( + "ERROR: Could not complete replication" " operation:{}".format(err) + ) + + +def main(): + """ + main function - creates session and runs jobs + """ + time_start = time.time() + try: + dx_session_obj = get_session.GetSession() + dx_logging.logging_est(ARGUMENTS["--logdir"]) + config_file_path = ARGUMENTS["--config"] + single_thread = ARGUMENTS["--single_thread"] + engine = ARGUMENTS["--engine"] + dx_session_obj.get_config(config_file_path) + for each in run_job.run_job_mt( + main_workflow, dx_session_obj, engine, single_thread + ): + each.join() + elapsed_minutes = run_job.time_elapsed(time_start) + dx_logging.print_info( + f"dx_replication took {elapsed_minutes} minutes to complete." + ) + except SystemExit as err: + # This is what we use to handle our sys.exit(#) + sys.exit(err) + except dlpx_exceptions.DlpxException as err: + # Handle an error occurs in a function call. + dx_logging.print_exception( + f"ERROR: Please check the ERROR message below:\n {err.error}" + ) + sys.exit(2) + except exceptions.HttpError as err: + dx_logging.print_exception( + f"ERROR: Connection failed to the Delphix DDP. Please check " + f"the ERROR message below:\n{err.status}" + ) + sys.exit(2) + except exceptions.JobError as err: + # If a job fails in Delphix so that we have actionable data + elapsed_minutes = run_job.time_elapsed(time_start) + dx_logging.print_exception( + f"A job failed in the Delphix Engine:\n{err.job}." + f"{basename(__file__)} took {elapsed_minutes} minutes to " + f"complete" + ) + sys.exit(3) + except KeyboardInterrupt: + # Gracefully handle ctrl+c exits + dx_logging.print_debug("You sent a CTRL+C to interrupt the process") + elapsed_minutes = run_job.time_elapsed(time_start) + dx_logging.print_info( + f"{basename(__file__)} took {elapsed_minutes} " f"minutes to complete." + ) + + +if __name__ == "__main__": + # Grab our ARGUMENTS from the doc at the top of the script + ARGUMENTS = docopt.docopt(__doc__, version=basename(__file__) + " " + VERSION) + # Feed our ARGUMENTS to the main function, and off we go! + main() + diff --git a/dx_snapshot_db.py b/dx_snapshot_db.py index 888ef20..c822971 100755 --- a/dx_snapshot_db.py +++ b/dx_snapshot_db.py @@ -1,17 +1,12 @@ #!/usr/bin/env python3 -# Adam Bowen - Apr 2016 # This script snapshots a vdb or dSource -# Corey Brune - March 2017 -# Updated to allow backup of Sybase -# requirements -# pip install docopt delphixpy # The below doc follows the POSIX compliant standards and allows us to use # This doc to also define our ARGUMENTS for the script. """Snapshot dSources and VDB's Usage: - dx_snapshot_db.py (--group |--name | --all_dbs ) + dx_snapshot_db.py (--group |--name | --all_dbs | --list) [--engine ] [--usebackup --bck_file --parallel ] [--poll --create_bckup --single_thread ] @@ -32,6 +27,7 @@ --engine Alt Identifier of Delphix engine in dxtools.conf. [default: default] --all_dbs Run against all database objects + --list List all snapshots --single_thread Run as a single thread. False if running multiple threads. [default: False] @@ -65,6 +61,7 @@ from delphixpy.v1_10_2 import exceptions from delphixpy.v1_10_2.web import database +from delphixpy.v1_10_2.web import snapshot from delphixpy.v1_10_2.web import source from delphixpy.v1_10_2.web import vo from lib import dlpx_exceptions @@ -74,8 +71,19 @@ from lib import run_job from lib.run_async import run_async -VERSION = "v.0.3.002" +VERSION = "v.0.3.003" +def list_snapshots(dlpx_obj, db_name=None): + """ + """ + if db_name: + snapshots = snapshot.get_all(dlpx_obj, database=db_name) + for snap in snapshots: + print(snap) + elif db_name is None: + snapshots = snapshot.get_all(dlpx_obj) + for snap in snapshots: + print(snap) def snapshot_database( dlpx_obj, @@ -160,6 +168,8 @@ def snapshot_database( dlpx_obj.jobs[dlpx_obj.server_session.address].append( dlpx_obj.server_session.last_job ) + print(container_obj_ref) + return(dlpx_obj.server_session.last_job) @run_async @@ -192,7 +202,7 @@ def main_workflow(engine, dlpx_obj, single_thread): try: with dlpx_obj.job_mode(single_thread): if ARGUMENTS["--name"] is not None: - snapshot_database( + last_job = snapshot_database( dlpx_obj, ARGUMENTS["--name"], None, @@ -231,7 +241,18 @@ def main_workflow(engine, dlpx_obj, single_thread): ARGUMENTS["--bck_file"], ARGUMENTS["--create_bckup"], ) + elif ARGUMENTS["--list"]: + list_snapshots(dlpx_obj.server_session) run_job.track_running_jobs(engine, dlpx_obj) + snap_name = run_job.find_snapshot_ref_jobid( + dlpx_obj.server_session, dlpx_obj.server_session.last_job + ) + if snap_name: + snap_obj = get_references.find_obj_by_name( + dlpx_obj.server_session, snapshot, f'{snap_name}' + ) + print(snap_obj.name) + except ( dlpx_exceptions.DlpxObjectNotFound, exceptions.RequestError, diff --git a/dxtools.conf b/dxtools.conf new file mode 100644 index 0000000..a0da221 --- /dev/null +++ b/dxtools.conf @@ -0,0 +1,13 @@ +{ + "myve2": [ + { + "ip_address": "VIRTUALIZATION_IP_OR_NAME`", + "username": "USERNAME", + "password": "PASSWORD", + "use_https": true, + "default": true, + "hostname": "myve2", + "isEncrypted": false + } + ] +} diff --git a/lib/dx_config.py b/lib/dx_config.py new file mode 100755 index 0000000..4c295f9 --- /dev/null +++ b/lib/dx_config.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import json +from cryptography.fernet import Fernet +import sys + +class DxConfig(): + """ + """ + def __init__(self): + self.key = b'mB6l0679MPRgyJ3RNFzFDX9q7FG_rOxioeKgEv3c03e=' + + def _encrypt(self, string): + encode_str = string.encode() + f = Fernet(self.key) + return f.encrypt(encode_str) + + def _decrypt(self, encrypt_str): + """ + """ + f = Fernet(self.key) + return f.decrypt(encrypt_str).decode() + + def encrypt_json(self, json_obj): + """ + """ + enc_password = self._encrypt(json_obj["password"]).decode("utf-8") + enc_username = self._encrypt(json_obj["username"]).decode("utf-8") + json_obj["password"] = enc_password + json_obj["username"] = enc_username + return json_obj + + def decrypt_cred(self, encrypted_cred): + """ + """ + bytes_cred = encrypted_cred.encode("utf-8") + return self._decrypt(bytes_cred) + diff --git a/lib/dx_timeflow.py b/lib/dx_timeflow.py index 3591238..30f1f15 100644 --- a/lib/dx_timeflow.py +++ b/lib/dx_timeflow.py @@ -159,7 +159,8 @@ def find_snapshot(self, snap_name): for snapshot_obj in snapshots: if str(snapshot_obj.name).startswith(snap_name): return snapshot_obj.reference - elif str(snapshot_obj.latest_change_point.timestamp).startswith(snap_name): + elif str(snapshot_obj.latest_change_point.timestamp).startswith( + snap_name): return snapshot_obj.reference def set_timeflow_point( diff --git a/lib/get_references.py b/lib/get_references.py index fc662dc..ccf5e5a 100644 --- a/lib/get_references.py +++ b/lib/get_references.py @@ -46,6 +46,19 @@ def convert_timestamp(engine, timestamp): return None +def find_obj_specs(engine, obj_lst): + """ + Function to find objects for replication + engine: Delphix Virtualization session object + obj_lst: List of names for replication + :return: List of references for the given object names + """ + rep_lst = [] + for obj in obj_lst: + rep_lst.append(find_obj_by_name(engine, database, obj).reference) + return rep_lst + + def get_running_job(engine, object_ref): """ Function to find a running job from the DB target reference. diff --git a/lib/get_session.py b/lib/get_session.py index 4f548c8..bc71860 100644 --- a/lib/get_session.py +++ b/lib/get_session.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# Corey Brune - Oct 2016 # This class handles the config file and authentication to a DDP # requirements # pip install docopt delphixpy @@ -19,8 +18,9 @@ from delphixpy.v1_10_2.delphix_engine import DelphixEngine from lib import dlpx_exceptions from lib import dx_logging +from lib import dx_config -VERSION = "v.0.3.001" +VERSION = "v.0.4.000" class GetSession: @@ -33,7 +33,7 @@ def __init__(self): self.dlpx_ddps = {} self.jobs = {} - def get_config(self, config_file_path="./config/dxtools.conf"): + def get_config(self, config_file_path="./config/dxtools.conf", engine="default"): """ This method reads in the dxtools.conf file @@ -41,6 +41,8 @@ def get_config(self, config_file_path="./config/dxtools.conf"): :type config_file_path: str :return: dict containing engine information """ + eng_dct = {} + new_eng = {} # First test to see that the file is there and we can open it try: with open(config_file_path) as config_file: @@ -57,19 +59,29 @@ def get_config(self, config_file_path="./config/dxtools.conf"): f"again.\n {err}" ) for each in config.keys(): - temp_config = config[each].pop() - use_https = temp_config["use_https"] - if use_https and use_https.lower() == "true": - temp_config["use_https"] = True - else: - temp_config["use_https"] = False - self.dlpx_ddps[each] = temp_config + for eng in config[each]: + if eng["isEncrypted"] is False: + config_obj = dx_config.DxConfig() + new_eng = config_obj.encrypt_json(eng) + new_eng["isEncrypted"] = True + eng = new_eng + hostname = eng["hostname"] + self.dlpx_ddps[hostname] = eng + if new_eng: + try: + with open(config_file_path, "w", encoding="utf-8") as f: + json.dump(config, f, indent=4) + except IOError: + raise dlpx_exceptions.DlpxException( + f"\nERROR: Was unable to open {config_file_path}. Please " + f"check the path and permissions, and try again.\n" + ) def dlpx_session( self, f_engine_address, f_engine_username, - f_engine_password=None, + f_engine_password, enable_https=True, ): """ @@ -84,17 +96,20 @@ def dlpx_session( :type enable_https: bool :return: delphixpy.v1_10_2.delphix_engine.DelphixEngine object """ + config_obj = dx_config.DxConfig() f_engine_namespace = "DOMAIN" # Remove the next 3 lines if using in a production environment. if not os.environ.get("PYTHONHTTPSVERIFY", "") and getattr( ssl, "_create_unverified_context", None ): ssl._create_default_https_context = ssl._create_unverified_context + unenc_pass = config_obj.decrypt_cred(f_engine_password) + unenc_user = config_obj.decrypt_cred(f_engine_username) try: self.server_session = DelphixEngine( f_engine_address, - f_engine_username, - f_engine_password, + unenc_user, + unenc_pass, f_engine_namespace, enable_https, ) @@ -108,7 +123,7 @@ def dlpx_session( f"ERROR: An error occurred while authenticating to " f"{f_engine_address}:\n {err}\n" ) - except (TimeoutError) as err: + except TimeoutError as err: raise dlpx_exceptions.DlpxException( f"ERROR: Timeout while authenticating to " f"{f_engine_address}:\n {err}\n" diff --git a/lib/run_job.py b/lib/run_job.py index c438ab0..13587ee 100644 --- a/lib/run_job.py +++ b/lib/run_job.py @@ -1,6 +1,7 @@ """ Runs jobs passing a function as an argument. Thread safe. """ +import re import time from delphixpy.v1_10_2 import exceptions @@ -8,7 +9,7 @@ from lib import dlpx_exceptions from lib import dx_logging -VERSION = "v.0.3.004" +VERSION = "v.0.3.005" def run_job(main_func, dx_obj, engine="default", single_thread=True): @@ -28,6 +29,7 @@ def run_job(main_func, dx_obj, engine="default", single_thread=True): """ threads = [] # if engine ="all", run against every engine in config_file + import pdb;pdb.set_trace() if engine == "all": dx_logging.print_info(f"Executing against all Delphix Engines") try: @@ -109,7 +111,7 @@ def run_job_mt(main_func, dx_obj, engine="default", single_thread=True): try: for delphix_ddp in dx_obj.dlpx_ddps.keys(): is_default = dx_obj.dlpx_ddps[delphix_ddp]["default"] - if is_default and is_default.lower() == "true": + if is_default is True: dx_obj_default = dx_obj dx_obj_default.dlpx_ddps = { delphix_ddp: dx_obj.dlpx_ddps[delphix_ddp] @@ -244,6 +246,25 @@ def find_job_state_by_jobid(engine, dx_obj, job_id, poll=20): return job_obj.job_state +def find_snapshot_ref_jobid(dx_obj, job_id): + """ + Retrieves snapshot ref + :param engine: Dictionary containing info on the DDP (IP, username, etc.) + :param dx_obj: Delphix session object from config + :type dx_obj: lib.get_session.GetSession object + :param job_id: Job ID to check the state + :return: + """ + # get the job object + job_obj = job.get(dx_obj, job_id) + snapshot_details = job_obj.events[7].message_details + try: + snapshot_name = re.search('@.*Z', snapshot_details) + return(snapshot_name.group()) + except AttributeError: + return False + + def time_elapsed(time_start): """ This function calculates the time elapsed since the beginning of the script. diff --git a/requirements.txt b/requirements.txt index 1c0861d..63e6ed2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pip python-dateutil setuptools untangle +cryptography diff --git a/ss_container.py b/ss_container.py old mode 100755 new mode 100644 index ac8f6ce..c2ee643 --- a/ss_container.py +++ b/ss_container.py @@ -112,7 +112,8 @@ def create_container(dlpx_obj, template_name, container_name, database_name): container_ds_lst = [] for data_set in database_name.split(":"): container_ds_lst.append( - get_references.build_data_source_params(dlpx_obj, database, data_set) + get_references.build_data_source_params(dlpx_obj, database, + data_set) ) try: ss_template_ref = get_references.find_obj_by_name( @@ -122,7 +123,8 @@ def create_container(dlpx_obj, template_name, container_name, database_name): ss_container_params.timeline_point_parameters = ( vo.JSTimelinePointLatestTimeInput() ) - ss_container_params.timeline_point_parameters.sourceDataLayout = ss_template_ref + ss_container_params.timeline_point_parameters.sourceDataLayout = \ + ss_template_ref ss_container_params.data_sources = container_ds_lst ss_container_params.name = container_name container_ref = selfservice.container.create( @@ -169,8 +171,8 @@ def remove_owner(dlpx_obj, owner_name, container_name): exceptions.HttpError, ) as err: dx_logging.print_exception( - f"The user was not added to container " - f"{container_name}. The error was:\n{err}" + f'The user was not added to container ' + f'{container_name}. The error was:\n{err}' ) @@ -208,7 +210,7 @@ def restore_container(dlpx_obj, container_name, bookmark_name): exceptions.RequestError, exceptions.HttpError, ) as err: - dx_logging.print_exception(f"The container was not restored:\n{err}") + dx_logging.print_exception(f'The container was not restored:\n{err}') def add_owner(dlpx_obj, owner_name, container_name): @@ -290,7 +292,8 @@ def delete_container(dlpx_obj, container_name, keep_vdbs=False): selfservice.container.delete( dlpx_obj.server_session, get_references.find_obj_by_name( - dlpx_obj.server_session, selfservice.container, container_name + dlpx_obj.server_session, selfservice.container, + container_name ).reference, ss_container_params, ) @@ -298,7 +301,8 @@ def delete_container(dlpx_obj, container_name, keep_vdbs=False): selfservice.container.delete( dlpx_obj.server_session, get_references.find_obj_by_name( - dlpx_obj.server_session, selfservice.container, container_name + dlpx_obj.server_session, selfservice.container, + container_name ).reference, ) except ( @@ -326,9 +330,9 @@ def list_containers(dlpx_obj): dlpx_obj.server_session, ss_container.last_updated[:-5] ) dx_logging.print_info( - f"{ss_container.name}, {ss_container.active_branch}, " - f"{ss_container.owner}, {ss_container.reference}," - f"{ss_container.template}, {last_updated}" + f'{ss_container.name}, {ss_container.active_branch}, ' + f'{ss_container.owner}, {ss_container.reference},' + f'{ss_container.template}, {last_updated}' ) except ( dlpx_exceptions.DlpxException, @@ -380,15 +384,15 @@ def list_hierarchy(dlpx_obj, container_name): dlpx_obj.server_session, database, data_source.container ) - if hasattr(data_source.runtime, "instance_jdbc_string"): + if hasattr(data_source.runtime, 'instance_jdbc_string'): database_dct[db_name] = data_source.runtime.instance_jdbc_string else: database_dct[db_name] = None try: dx_logging.print_info( - f"Container: {container_name}\n" - f"Related VDBs: " - f"{convert_dct_str(database_dct)}\n" + f'Container: {container_name}\n' + f'Related VDBs: ' + f'{convert_dct_str(database_dct)}\n' ) except AttributeError as err: dx_logging.print_exception(err) @@ -445,52 +449,52 @@ def main_workflow(engine, dlpx_obj, single_thread): ) try: with dlpx_obj.job_mode(single_thread): - if ARGUMENTS["--create_container"]: + if ARGUMENTS['--create_container']: create_container( dlpx_obj, - ARGUMENTS["--template_name"], - ARGUMENTS["--create_container"], - ARGUMENTS["--database"], + ARGUMENTS['--template_name'], + ARGUMENTS['--create_container'], + ARGUMENTS['--database'], ) dx_logging.print_info( f'Self Service Container {ARGUMENTS["--create_container"]}' - f"was created successfully." + f'was created successfully.' ) - elif ARGUMENTS["--delete_container"]: + elif ARGUMENTS['--delete_container']: delete_container( dlpx_obj, - ARGUMENTS["--delete_container"], - ARGUMENTS["--keep_vdbs"], + ARGUMENTS['--delete_container'], + ARGUMENTS['--keep_vdbs'], ) - elif ARGUMENTS["--list"]: + elif ARGUMENTS['--list']: list_containers(dlpx_obj) - elif ARGUMENTS["--remove_owner"]: + elif ARGUMENTS['--remove_owner']: remove_owner( dlpx_obj, - ARGUMENTS["--remove_owner"], - ARGUMENTS["--container_name"], + ARGUMENTS['--remove_owner'], + ARGUMENTS['--container_name'], ) dx_logging.print_info( f'User {ARGUMENTS["--remove_owner"]} had ' - f"access revoked from " + f'access revoked from ' f'{ARGUMENTS["--container_name"]}' ) - elif ARGUMENTS["--restore_container"]: + elif ARGUMENTS['--restore_container']: restore_container( dlpx_obj, - ARGUMENTS["--restore_container"], - ARGUMENTS["--bookmark_name"], + ARGUMENTS['--restore_container'], + ARGUMENTS['--bookmark_name'], ) dx_logging.print_info( f'Container {ARGUMENTS["--restore_container"]} ' - f"was restored successfully with bookmark " + f'was restored successfully with bookmark ' f'{ARGUMENTS["--bookmark_name"]}' ) - elif ARGUMENTS["--add_owner"]: + elif ARGUMENTS['--add_owner']: add_owner( dlpx_obj, - ARGUMENTS["--add_owner"], - ARGUMENTS["--container_name"], + ARGUMENTS['--add_owner'], + ARGUMENTS['--container_name'], ) dx_logging.print_info( f'User {ARGUMENTS["--add_owner"]} was granted ' @@ -500,12 +504,12 @@ def main_workflow(engine, dlpx_obj, single_thread): refresh_container(engine, dlpx_obj, ARGUMENTS["--refresh_container"]) dx_logging.print_info( f'The container {ARGUMENTS["--refresh_container"]}' - f" was refreshed." + f' was refreshed.' ) - elif ARGUMENTS["--list_hierarchy"]: - list_hierarchy(dlpx_obj, ARGUMENTS["--list_hierarchy"]) - elif ARGUMENTS["--reset_container"]: - reset_container(dlpx_obj, ARGUMENTS["--reset_container"]) + elif ARGUMENTS['--list_hierarchy']: + list_hierarchy(dlpx_obj, ARGUMENTS['--list_hierarchy']) + elif ARGUMENTS['--reset_container']: + reset_container(dlpx_obj, ARGUMENTS['--reset_container']) print(f'Container {ARGUMENTS["--reset_container"]} was reset.') except ( dlpx_exceptions.DlpxException, @@ -526,10 +530,10 @@ def main(): time_start = time.time() try: dx_session_obj = get_session.GetSession() - dx_logging.logging_est(ARGUMENTS["--logdir"]) - config_file_path = ARGUMENTS["--config"] + dx_logging.logging_est(ARGUMENTS['--logdir']) + config_file_path = ARGUMENTS['--config'] single_thread = ARGUMENTS["--single_thread"] - engine = ARGUMENTS["--engine"] + engine = ARGUMENTS['--engine'] dx_session_obj.get_config(config_file_path) for each in run_job.run_job_mt( main_workflow, dx_session_obj, engine, single_thread @@ -557,7 +561,7 @@ def main(): except exceptions.JobError as err: # We use this exception handler when a job fails in Delphix so that we # have actionable data - print(f"A job failed in the Delphix Engine:\n{err.job}") + print(f'A job failed in the Delphix Engine:\n{err.job}') elapsed_minutes = run_job.time_elapsed(time_start) dx_logging.print_info( f"{basename(__file__)} took {elapsed_minutes} minutes to get this far." @@ -565,15 +569,15 @@ def main(): sys.exit(3) except KeyboardInterrupt: # We use this exception handler to gracefully handle ctrl+c exits - dx_logging.print_debug("You sent a CTRL+C to interrupt the process") + dx_logging.print_debug('You sent a CTRL+C to interrupt the process') elapsed_minutes = run_job.time_elapsed(time_start) dx_logging.print_info( f"{basename(__file__)} took {elapsed_minutes} minutes to get this far." ) -if __name__ == "__main__": +if __name__ == '__main__': # Grab our ARGUMENTS from the doc at the top of the script - ARGUMENTS = docopt(__doc__, version=basename(__file__) + " " + VERSION) + ARGUMENTS = docopt(__doc__, version=basename(__file__) + ' ' + VERSION) # Feed our ARGUMENTS to the main function, and off we go! main()