From 380b0be51c513b2762c945a46de06814e7c6fe50 Mon Sep 17 00:00:00 2001 From: Bartosz Kosciug Date: Wed, 22 Feb 2023 13:43:33 +0100 Subject: [PATCH 01/28] Adding deleting unknown resources feature. --- aws_cleanup | 58 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/aws_cleanup b/aws_cleanup index 5e9a1833c..81aad7679 100755 --- a/aws_cleanup +++ b/aws_cleanup @@ -12,6 +12,7 @@ except ImportError: print('You need to pip install boto3==1.12.13') sys.exit(1) +TEST_GENERATED = '*testpytest*' NOT_TERMINATED = [ 'pending', 'running', 'shutting-down', 'stopping', 'stopped' ] @@ -164,6 +165,16 @@ def get_vpcs(client, owner): ], )['Vpcs'] +def get_all_vpcs(client): + vpcs = [] + for vpc in client.describe_vpcs( + Filters = [ + {'Name' : 'tag:CreatedBy' , 'Values' : [TEST_GENERATED]} + ] + )['Vpcs']: + if not 'Name' in [ k['Key'] for k in vpc['Tags'] ]: + vpcs.append(vpc) + return vpcs def get_subnets(client, vpc): return client.describe_subnets( @@ -422,8 +433,36 @@ def delete_resources(client, resources_by_test): # investigate by hand if the script can't handle it. client.delete_vpc(VpcId=vpc) +def collect_resources_from_unknown_test(client, owner): + vpcs = get_all_vpcs(client) + for i, _ in enumerate(vpcs): + vpcs[i]['Tags'].append( { 'Key' : 'Name', 'Value' : 'unknowntest'+str(i) + "_" \ + + str(datetime.now().strftime('%Y%m%d%H%M%S'))} ) -def main(older_than, newer_than, owner, delete): + resources_by_test = {} + for i, vpc in enumerate(vpcs): + vpc_id = vpc['VpcId'] + instances = get_instances(client, vpc_id) + interfaces = get_interfaces(client, vpc_id) + subnets = get_subnets(client, vpc_id) + route_tables = get_route_tables(client, vpc_id) + security_groups = get_security_groups(client, vpc_id) + internet_gateways = get_internet_gateways(client, vpc_id) + + group_by_previous_tests(instances, interfaces, + subnets, route_tables, + internet_gateways, + security_groups, + vpc, resources_by_test) + + elastic_ips = get_elastic_ips(client, owner) + group_non_vpc_entities_by_previous_tests(elastic_ips, 'elastic ips', + resources_by_test) + + output_resources_by_test(resources_by_test) + return resources_by_test + +def main(older_than, newer_than, owner, delete, unknown): conf = load_config() client = get_ec2_client(conf) @@ -433,6 +472,7 @@ def main(older_than, newer_than, owner, delete): resources_by_test = {} vpcs = get_vpcs(client, owner) + for vpc in vpcs: vpc_id = vpc['VpcId'] instances = get_instances(client, vpc_id) @@ -441,7 +481,7 @@ def main(older_than, newer_than, owner, delete): route_tables = get_route_tables(client, vpc_id) security_groups = get_security_groups(client, vpc_id) internet_gateways = get_internet_gateways(client, vpc_id) - + group_by_previous_tests(instances, interfaces, subnets, route_tables, internet_gateways, @@ -453,13 +493,15 @@ def main(older_than, newer_than, owner, delete): resources_by_test) filter_test_groups(resources_by_test, older_than, newer_than) - output_resources_by_test(resources_by_test) - + if delete: print('Deleting') delete_resources(client, resources_by_test) - + if unknown: + print('Deleting resources from unkown tests') + unknown_resources = collect_resources_from_unknown_test(client, owner) + delete_resources(client, unknown_resources) if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -492,6 +534,12 @@ if __name__ == '__main__': action='store_true', default=False, ) + parser.add_argument( + '-u', '--unknown', + help='Delete discovered resources from unkown tests.', + action='store_true', + default=False, + ) args = parser.parse_args() From 4a8a890a08238b4c509537bfcf036a74f7f0716e Mon Sep 17 00:00:00 2001 From: Bartosz Kosciug Date: Wed, 22 Feb 2023 16:32:14 +0100 Subject: [PATCH 02/28] remove ips from unknown deletion. --- aws_cleanup | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/aws_cleanup b/aws_cleanup index 81aad7679..d82a43775 100755 --- a/aws_cleanup +++ b/aws_cleanup @@ -12,7 +12,7 @@ except ImportError: print('You need to pip install boto3==1.12.13') sys.exit(1) -TEST_GENERATED = '*testpytest*' +TEST_GENERATED = '*pytest*' NOT_TERMINATED = [ 'pending', 'running', 'shutting-down', 'stopping', 'stopped' ] @@ -438,10 +438,10 @@ def collect_resources_from_unknown_test(client, owner): for i, _ in enumerate(vpcs): vpcs[i]['Tags'].append( { 'Key' : 'Name', 'Value' : 'unknowntest'+str(i) + "_" \ + str(datetime.now().strftime('%Y%m%d%H%M%S'))} ) - resources_by_test = {} for i, vpc in enumerate(vpcs): vpc_id = vpc['VpcId'] + instances = get_instances(client, vpc_id) interfaces = get_interfaces(client, vpc_id) subnets = get_subnets(client, vpc_id) @@ -454,10 +454,6 @@ def collect_resources_from_unknown_test(client, owner): internet_gateways, security_groups, vpc, resources_by_test) - - elastic_ips = get_elastic_ips(client, owner) - group_non_vpc_entities_by_previous_tests(elastic_ips, 'elastic ips', - resources_by_test) output_resources_by_test(resources_by_test) return resources_by_test @@ -543,5 +539,4 @@ if __name__ == '__main__': args = parser.parse_args() - main(older_than=args.older, newer_than=args.newer, owner=args.owner, - delete=args.delete) + main(older_than=args.older, newer_than=args.newer, owner=args.owner, delete=args.delete, unknown=args.unknown) From 888d25c670ce5fdd80828fb46c1d040ad2df09bf Mon Sep 17 00:00:00 2001 From: opencm Date: Wed, 8 Mar 2023 10:08:47 +0000 Subject: [PATCH 03/28] Bump version --- .../blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml | 2 +- .../resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml | 2 +- .../resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml | 2 +- .../blueprints/cli/test-os-fileserver-vm-blueprint.yaml | 2 +- .../resources/blueprints/cli/test-os-linux-vm-blueprint.yaml | 2 +- requirements.txt | 2 +- setup.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml index 6603be65e..b89d8906c 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml @@ -1,7 +1,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml inputs: diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml index f5071e01e..0dca5f9ff 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml @@ -1,7 +1,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml inputs: diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml index 5ed0dd8e1..04fc3ff7f 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml @@ -2,7 +2,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - http://www.getcloudify.org/spec/aws-plugin/1.4.10/plugin.yaml inputs: diff --git a/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml index 024f60613..94f984572 100644 --- a/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml @@ -2,7 +2,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - plugin:cloudify-openstack-plugin inputs: diff --git a/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml index ac5bee496..4e82df3ae 100644 --- a/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml @@ -2,7 +2,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - - http://www.getcloudify.org/spec/cloudify/7.0.0/types.yaml + - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml - plugin:cloudify-openstack-plugin inputs: diff --git a/requirements.txt b/requirements.txt index cdd09b836..bf44de57a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ charset-normalizer==2.1.1 # via # aiohttp # requests -cloudify-common @ https://github.com/cloudify-cosmo/cloudify-common/archive/7.0.0-build.zip +cloudify-common @ https://github.com/cloudify-cosmo/cloudify-common/archive/master.zip # via -r requirements.in cryptography==38.0.4 # via diff --git a/setup.py b/setup.py index 5df8e99b1..17317c110 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='cloudify-system-tests', - version='7.0.0', + version='7.1.0.dev1', author='Cloudify', author_email='cosmo-admin@cloudify.co', packages=['cosmo_tester'], From 2f54b93a35b5a13275c3d8854b0aae6438adf86d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Maksymczuk?= Date: Tue, 4 Apr 2023 12:49:45 +0200 Subject: [PATCH 04/28] RND-293 agent upgrade test: only assert on the STARTED agent (#1535) The agent list contains the old "UPGRADED"/"STOPPED" agent, and a new "STARTED" agent. Only assert on the new one. --- cosmo_tester/test_suites/agent/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cosmo_tester/test_suites/agent/__init__.py b/cosmo_tester/test_suites/agent/__init__.py index 9c9318d16..0a54e1f3a 100644 --- a/cosmo_tester/test_suites/agent/__init__.py +++ b/cosmo_tester/test_suites/agent/__init__.py @@ -1,3 +1,5 @@ +from cloudify.models_states import AgentState + from cosmo_tester.framework.test_hosts import Hosts, VM @@ -25,8 +27,13 @@ def get_test_prerequisites(ssh_key, module_tmpdir, test_config, logger, def validate_agent(manager, example, test_config, broken_system=False, install_method='remote', upgrade=False): - agents = list(manager.client.agents.list(tenant_name=example.tenant, - _all_tenants=True)) + agents = list( + manager.client.agents.list( + tenant_name=example.tenant, + state=AgentState.STARTED, + _all_tenants=True, + ) + ) instances = list( manager.client.node_instances.list( tenant_name=example.tenant, node_id='vm', From aac92e33418be9a9dafc64081cfcbcddcb570c17 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Tue, 4 Apr 2023 14:00:32 +0300 Subject: [PATCH 05/28] RD-7139 Add upgrade agents to cluster inplace upgrade tests (#1537) Co-authored-by: glukhman --- .../cluster/cfy_cluster_manager_shared.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py index 4e53664e6..7dbf8b5fd 100644 --- a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py +++ b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py @@ -6,6 +6,7 @@ import yaml from cosmo_tester.framework import util +from cosmo_tester.test_suites.snapshots import upgrade_agents CLUSTER_MANAGER_RESOURCES_PATH = pkg_resources.resource_filename( @@ -149,8 +150,52 @@ def _cluster_upgrade_test(test_config, base_version, nodes, 'echo -e \\\\nservice_management: systemd ' '| sudo tee -a {}'.format(conf) ) + _install_deployment_with_agent(manager, test_config, ssh_key, logger) _upgrade_cluster(nodes_list, manager, test_config, logger) + upgrade_agents(manager, logger, test_config) + + +def _install_deployment_with_agent(manager, test_config, ssh_key, logger): + dep_name = 'example' + + # copy test plugin + example blueprint + manager.run_command('yum install -y unzip', use_sudo=True) + manager.put_remote_file( + '/tmp/test_plugin.zip', + util.get_resource_path('plugin/test_plugin-1.0.0.zip'), + ) + manager.run_command('mkdir -p /tmp/example_bp') + manager.put_remote_file( + f'/tmp/{dep_name}_bp/{dep_name}.yaml', + util.get_resource_path('blueprints/compute/example.yaml'), + ) + manager.run_command('unzip /tmp/test_plugin.zip -d /tmp/test_plugin') + + # upload test plugin + blueprint + manager.run_command( + 'cfy plugins upload /tmp/test_plugin/*.wgn ' + '-y /tmp/test_plugin/plugin.yaml' + ) + manager.run_command( + f'cfy blueprints upload /tmp/{dep_name}_bp/{dep_name}.yaml' + f' -b {dep_name}' + ) + + # create and install deployment + inputs = { + 'server_ip': manager.ip_address, + 'path': f'/home/{manager.username}/test_file', + 'content': 'Test', + 'agent_user': manager.username + } + with open(ssh_key.private_key_path) as key_handle: + ssh_key_string = key_handle.read() + manager.run_command(f'cfy secrets create agent_key -s "{ssh_key_string}"') + manager.run_command( + f'cfy deployments create {dep_name} -b {dep_name} -i "{inputs}"') + manager.run_command( + f'cfy execution start install -d {dep_name}') def _get_config_dict(node_count, test_config, vm_user): From 405c539a524c5b38092c38659a86f626eef42604 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:41:22 +0300 Subject: [PATCH 06/28] Fix summary error message check after RND-390 (#1541) Co-authored-by: glukhman --- cosmo_tester/test_suites/summary/summary_error_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cosmo_tester/test_suites/summary/summary_error_test.py b/cosmo_tester/test_suites/summary/summary_error_test.py index fc1f9f880..430fdf97c 100644 --- a/cosmo_tester/test_suites/summary/summary_error_test.py +++ b/cosmo_tester/test_suites/summary/summary_error_test.py @@ -57,7 +57,7 @@ def test_bad_field_selected(prepared_manager, summary_type): '{endpoint}.'.format(endpoint=summary_type) ) except CloudifyClientError as err: - components = ['notafield', 'not', 'summary', 'Valid'] + components = ['notafield', 'summary', 'Invalid'] components.extend(SUPPORTED_FIELDS[summary_type]) if not all(component in str(err) for component in components): raise AssertionError( @@ -96,7 +96,7 @@ def test_bad_subfield_selected(prepared_manager, summary_type): '{endpoint}.'.format(endpoint=summary_type) ) except CloudifyClientError as err: - components = ['notafield', 'not', 'summary', 'Valid'] + components = ['notafield', 'summary', 'Invalid'] components.extend(SUPPORTED_FIELDS[summary_type]) if not all(component in str(err) for component in components): raise AssertionError( From 20b86af7e6676e4a1c80e5c6b3f05ceda8ea5899 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Wed, 19 Apr 2023 15:22:29 +0300 Subject: [PATCH 07/28] Give AMQP service more time to stop/start (#1542) Co-authored-by: glukhman --- cosmo_tester/test_suites/general/status_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmo_tester/test_suites/general/status_test.py b/cosmo_tester/test_suites/general/status_test.py index c529b331d..427e6267f 100644 --- a/cosmo_tester/test_suites/general/status_test.py +++ b/cosmo_tester/test_suites/general/status_test.py @@ -24,7 +24,7 @@ def test_status(image_based_manager, logger): # Allow time for services to start when we restart them -@retrying.retry(stop_max_attempt_number=10, wait_fixed=1000) +@retrying.retry(stop_max_attempt_number=20, wait_fixed=1000) def _check_status(manager, logger, healthy=True): if healthy: logger.info('Checking for healthy status') From 205f98b55a35e6b068b7771ad036756d8cff4a14 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:18:04 +0300 Subject: [PATCH 08/28] gnore agent dir + add agent uninstall assert (#1544) Co-authored-by: glukhman --- .../test_suites/bootstrap/teardown_test.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cosmo_tester/test_suites/bootstrap/teardown_test.py b/cosmo_tester/test_suites/bootstrap/teardown_test.py index 967e92a1a..d525f6f33 100644 --- a/cosmo_tester/test_suites/bootstrap/teardown_test.py +++ b/cosmo_tester/test_suites/bootstrap/teardown_test.py @@ -1,3 +1,5 @@ +import json + from cosmo_tester.framework.examples import get_example_deployment from cosmo_tester.framework.util import substitute_testing_version @@ -42,17 +44,14 @@ def _test_teardown(function_scoped_manager, ssh_key, logger, test_config): # When the example deployment installs the agent this group will exist. # It shouldn't be deleted afterwards in case files on the system are in # that group. + # We also don't remove the agents dir for possible subsequent installs expected_diffs['os groups'] = {'cfyagent'} + expected_diffs['folders in /opt'] = {'cloudify-agent'} - function_scoped_manager.teardown(kill_certs=False) + assert json.loads(function_scoped_manager.run_command( + 'cfy agents list -a --json').stdout) == [] - # The agents dir should be empty, so let's remove it. - function_scoped_manager.run_command( - 'rmdir /opt/cloudify-agent-{}'.format( - test_config['testing_version'].replace('-ga', '') - ), - use_sudo=True, - ) + function_scoped_manager.teardown(kill_certs=False) current_state = _get_system_state(function_scoped_manager) diffs = {} From 53168b8d502cc6e18a79ec41fafaa2aac0a75067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Maksymczuk?= Date: Mon, 15 May 2023 14:37:37 +0200 Subject: [PATCH 09/28] Do not pass external_cert/key/ca (#1547) Currently, separate external certs are only supported when public_ip is a domain name, not an ip, and these tests always provide ips. This is because both internal and external listen on 443, and SNI only works for domain names, not ips. --- .../config_files_templates/manager_config.yaml | 3 --- cosmo_tester/test_suites/cluster/conftest.py | 3 --- cosmo_tester/test_suites/snapshots/inplace_restore_test.py | 4 ---- 3 files changed, 10 deletions(-) diff --git a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/config_files_templates/manager_config.yaml b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/config_files_templates/manager_config.yaml index bec1b2fd3..dcf3f37a3 100644 --- a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/config_files_templates/manager_config.yaml +++ b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/config_files_templates/manager_config.yaml @@ -39,14 +39,11 @@ validations: ssl_inputs: - external_cert_path: {{ node.cert_path }} - external_key_path: {{ node.key_path }} internal_cert_path: {{ node.cert_path }} internal_key_path: {{ node.key_path }} postgresql_client_cert_path: {{ node.cert_path }} postgresql_client_key_path: {{ node.key_path }} ca_cert_path: {{ ca_path }} - external_ca_cert_path: {{ ca_path }} prometheus: credentials: diff --git a/cosmo_tester/test_suites/cluster/conftest.py b/cosmo_tester/test_suites/cluster/conftest.py index 5ec867586..151f096f3 100644 --- a/cosmo_tester/test_suites/cluster/conftest.py +++ b/cosmo_tester/test_suites/cluster/conftest.py @@ -716,12 +716,9 @@ def _bootstrap_manager_node(node, mgr_num, dbs, brokers, skip_bootstrap_list, if high_security: node.install_config['ssl_inputs'] = { - 'external_cert_path': node.remote_cert, - 'external_key_path': node.remote_key, 'internal_cert_path': node.remote_cert, 'internal_key_path': node.remote_key, 'ca_cert_path': node.remote_ca, - 'external_ca_cert_path': node.remote_ca, } node.install_config['manager']['security'][ 'ssl_enabled'] = True diff --git a/cosmo_tester/test_suites/snapshots/inplace_restore_test.py b/cosmo_tester/test_suites/snapshots/inplace_restore_test.py index c04d923f8..5cd6c90aa 100644 --- a/cosmo_tester/test_suites/snapshots/inplace_restore_test.py +++ b/cosmo_tester/test_suites/snapshots/inplace_restore_test.py @@ -80,13 +80,9 @@ def test_inplace_restore(manager_and_vm, 'key_path': '/tmp/ssl_backup/monitoring_key.pem', } manager.install_config['ssl_inputs'] = { - 'external_cert_path': '/tmp/ssl_backup/cloudify_external_cert.pem', - 'external_key_path': '/tmp/ssl_backup/cloudify_external_key.pem', 'internal_cert_path': '/tmp/ssl_backup/cloudify_internal_cert.pem', 'internal_key_path': '/tmp/ssl_backup/cloudify_internal_key.pem', 'ca_cert_path': '/tmp/ssl_backup/cloudify_internal_ca_cert.pem', - 'external_ca_cert_path': - '/tmp/ssl_backup/cloudify_internal_ca_cert.pem', } manager.bootstrap() upload_snapshot(manager, snapshot_path, snapshot_name, logger) From 735e30530567c7370b8177c0e295c2ab675ee835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Maksymczuk?= Date: Tue, 16 May 2023 10:59:37 +0200 Subject: [PATCH 10/28] Update default manager-under-test to 7.1.0 (#1548) --- cosmo_tester/config_schemas/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmo_tester/config_schemas/base.yaml b/cosmo_tester/config_schemas/base.yaml index 571ffdecd..47ad269ba 100644 --- a/cosmo_tester/config_schemas/base.yaml +++ b/cosmo_tester/config_schemas/base.yaml @@ -7,4 +7,4 @@ target_platform: valid_values: [openstack, aws] testing_version: description: Which manager version we're testing. Note that this is expected to be in the form -. Bad things may happen without the hyphen. - default: 7.0.0-.dev1 + default: 7.1.0-.dev1 From e60703cba1600d74cd9ab49f793bf5d778d4ee71 Mon Sep 17 00:00:00 2001 From: Mateusz Neumann Date: Fri, 16 Jun 2023 10:07:19 +0200 Subject: [PATCH 11/28] Upgrade blueprints to DSL version 1.5 (#1549) * Update most of the blueprints to DSL 1.5 * Update AWS infrastructure blueprints: use cloudify/types/types.yaml --- .../blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml | 2 +- .../resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml | 2 +- .../blueprints/cli/test-ec2-windows-vm-blueprint.yaml | 2 +- .../blueprints/cli/test-os-fileserver-vm-blueprint.yaml | 2 +- .../resources/blueprints/cli/test-os-linux-vm-blueprint.yaml | 2 +- .../resources/blueprints/compute/central_executor.yaml | 2 +- .../resources/blueprints/compute/central_executor_4_3_3.yaml | 2 +- cosmo_tester/resources/blueprints/compute/example.yaml | 2 +- .../resources/blueprints/compute/example_2_files.yaml | 2 +- cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml | 2 +- cosmo_tester/resources/blueprints/scale/groups.yaml | 2 +- cosmo_tester/resources/blueprints/scale/nodes.yaml | 2 +- .../blueprints/service_composition/capable_file.yaml | 2 +- .../resources/blueprints/service_composition/component.yaml | 2 +- .../resources/blueprints/service_composition/fake_vm.yaml | 2 +- .../blueprints/service_composition/nesting_component.yaml | 2 +- .../blueprints/service_composition/shared_resource.yaml | 2 +- cosmo_tester/resources/blueprints/summary/multivm.yaml | 2 +- cosmo_tester/resources/blueprints/summary/small.yaml | 2 +- cosmo_tester/resources/blueprints/summary/withrelations.yaml | 2 +- .../infrastructure_blueprints/aws/infrastructure-ipv6.yaml | 4 ++-- .../aws/infrastructure-multi-net.yaml | 4 ++-- .../infrastructure_blueprints/aws/infrastructure.yaml | 4 ++-- .../resources/infrastructure_blueprints/aws/vm-ipv6.yaml | 4 ++-- .../resources/infrastructure_blueprints/aws/vm-multi-net.yaml | 4 ++-- cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml | 4 ++-- .../openstack/infrastructure-multi-net.yaml | 2 +- .../infrastructure_blueprints/openstack/infrastructure.yaml | 2 +- .../infrastructure_blueprints/openstack/vm-multi-net.yaml | 2 +- .../resources/infrastructure_blueprints/openstack/vm.yaml | 2 +- 30 files changed, 36 insertions(+), 36 deletions(-) diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml index b89d8906c..a30d41f96 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml index 0dca5f9ff..6902c5c4f 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml index 04fc3ff7f..f753bafb2 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml @@ -1,5 +1,5 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml index 94f984572..58709b15d 100644 --- a/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml @@ -1,5 +1,5 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml index 4e82df3ae..8e54973ee 100644 --- a/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml @@ -1,5 +1,5 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/compute/central_executor.yaml b/cosmo_tester/resources/blueprints/compute/central_executor.yaml index 962141a36..4c6f68532 100644 --- a/cosmo_tester/resources/blueprints/compute/central_executor.yaml +++ b/cosmo_tester/resources/blueprints/compute/central_executor.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Run test functions on the central executor instead of an agent. diff --git a/cosmo_tester/resources/blueprints/compute/central_executor_4_3_3.yaml b/cosmo_tester/resources/blueprints/compute/central_executor_4_3_3.yaml index a80de53ab..559d1bbc5 100644 --- a/cosmo_tester/resources/blueprints/compute/central_executor_4_3_3.yaml +++ b/cosmo_tester/resources/blueprints/compute/central_executor_4_3_3.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Run test functions on the central executor instead of an agent. diff --git a/cosmo_tester/resources/blueprints/compute/example.yaml b/cosmo_tester/resources/blueprints/compute/example.yaml index 9acba2339..000576680 100644 --- a/cosmo_tester/resources/blueprints/compute/example.yaml +++ b/cosmo_tester/resources/blueprints/compute/example.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Create a file in an arbitrary location on a pre-existing VM using a plugin. diff --git a/cosmo_tester/resources/blueprints/compute/example_2_files.yaml b/cosmo_tester/resources/blueprints/compute/example_2_files.yaml index 5e19e3250..4aafbc593 100644 --- a/cosmo_tester/resources/blueprints/compute/example_2_files.yaml +++ b/cosmo_tester/resources/blueprints/compute/example_2_files.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Create a file in an arbitrary location on a pre-existing VM using a plugin. diff --git a/cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml b/cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml index 9b1570a1f..d5bc44c86 100644 --- a/cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml +++ b/cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/scale/groups.yaml b/cosmo_tester/resources/blueprints/scale/groups.yaml index 3a009cb5b..d46dc1f62 100644 --- a/cosmo_tester/resources/blueprints/scale/groups.yaml +++ b/cosmo_tester/resources/blueprints/scale/groups.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/scale/nodes.yaml b/cosmo_tester/resources/blueprints/scale/nodes.yaml index a2a1bfb2a..82e9bd9b7 100644 --- a/cosmo_tester/resources/blueprints/scale/nodes.yaml +++ b/cosmo_tester/resources/blueprints/scale/nodes.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/service_composition/capable_file.yaml b/cosmo_tester/resources/blueprints/service_composition/capable_file.yaml index b7ea86de3..882543d70 100644 --- a/cosmo_tester/resources/blueprints/service_composition/capable_file.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/capable_file.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Create a file on a vm owned by another deployment, using capabilities. diff --git a/cosmo_tester/resources/blueprints/service_composition/component.yaml b/cosmo_tester/resources/blueprints/service_composition/component.yaml index 251fc0dd1..efec28ec8 100644 --- a/cosmo_tester/resources/blueprints/service_composition/component.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/component.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Create an app using another deployment to deploy a VM (as a component) diff --git a/cosmo_tester/resources/blueprints/service_composition/fake_vm.yaml b/cosmo_tester/resources/blueprints/service_composition/fake_vm.yaml index 9293f29e2..3768e37a3 100644 --- a/cosmo_tester/resources/blueprints/service_composition/fake_vm.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/fake_vm.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Create compute resource for use in service composition tests. diff --git a/cosmo_tester/resources/blueprints/service_composition/nesting_component.yaml b/cosmo_tester/resources/blueprints/service_composition/nesting_component.yaml index 54409400f..99cb282ba 100644 --- a/cosmo_tester/resources/blueprints/service_composition/nesting_component.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/nesting_component.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Create a VM which depends on an app deployed using another deployment diff --git a/cosmo_tester/resources/blueprints/service_composition/shared_resource.yaml b/cosmo_tester/resources/blueprints/service_composition/shared_resource.yaml index 641538387..25527fb96 100644 --- a/cosmo_tester/resources/blueprints/service_composition/shared_resource.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/shared_resource.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 description: > Create an app using a VM from a pre-existing deployment as a resource diff --git a/cosmo_tester/resources/blueprints/summary/multivm.yaml b/cosmo_tester/resources/blueprints/summary/multivm.yaml index b37b2cee8..fcf4003e8 100644 --- a/cosmo_tester/resources/blueprints/summary/multivm.yaml +++ b/cosmo_tester/resources/blueprints/summary/multivm.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/summary/small.yaml b/cosmo_tester/resources/blueprints/summary/small.yaml index 0d076e129..62102cb28 100644 --- a/cosmo_tester/resources/blueprints/summary/small.yaml +++ b/cosmo_tester/resources/blueprints/summary/small.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/summary/withrelations.yaml b/cosmo_tester/resources/blueprints/summary/withrelations.yaml index 0f4698b80..6e029502c 100644 --- a/cosmo_tester/resources/blueprints/summary/withrelations.yaml +++ b/cosmo_tester/resources/blueprints/summary/withrelations.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-ipv6.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-ipv6.yaml index c0e8e9ec2..255fc0cb0 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-ipv6.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-ipv6.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - - https://cloudify.co/spec/cloudify/5.1.0/types.yaml + - cloudify/types/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-multi-net.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-multi-net.yaml index f7e91fb6a..ac2e083cc 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-multi-net.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-multi-net.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - - https://cloudify.co/spec/cloudify/5.1.0/types.yaml + - cloudify/types/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure.yaml index d6f97198e..baea66bdb 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - - https://cloudify.co/spec/cloudify/5.1.0/types.yaml + - cloudify/types/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/vm-ipv6.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/vm-ipv6.yaml index 766112c38..b236a1449 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/vm-ipv6.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/vm-ipv6.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - - https://cloudify.co/spec/cloudify/5.1.0/types.yaml + - cloudify/types/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/vm-multi-net.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/vm-multi-net.yaml index c65b00c57..0920001fa 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/vm-multi-net.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/vm-multi-net.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - - https://cloudify.co/spec/cloudify/5.1.0/types.yaml + - cloudify/types/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml index a99755b66..44cba6a6f 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - - https://cloudify.co/spec/cloudify/5.1.0/types.yaml + - cloudify/types/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure-multi-net.yaml b/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure-multi-net.yaml index 4cfcfacea..8006bd173 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure-multi-net.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure-multi-net.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/5.0.5/types.yaml diff --git a/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure.yaml b/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure.yaml index 0b1bcabf3..77bc947e8 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/5.0.5/types.yaml diff --git a/cosmo_tester/resources/infrastructure_blueprints/openstack/vm-multi-net.yaml b/cosmo_tester/resources/infrastructure_blueprints/openstack/vm-multi-net.yaml index f6883cba4..25cd5996a 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/openstack/vm-multi-net.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/openstack/vm-multi-net.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/5.0.5/types.yaml diff --git a/cosmo_tester/resources/infrastructure_blueprints/openstack/vm.yaml b/cosmo_tester/resources/infrastructure_blueprints/openstack/vm.yaml index 5c8b2e046..d16b29afc 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/openstack/vm.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/openstack/vm.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_3 +tosca_definitions_version: cloudify_dsl_1_5 imports: - http://www.getcloudify.org/spec/cloudify/5.0.5/types.yaml From 7dc73d7ae4e7cc816fc32f17a2e6b1cdc3fac8e5 Mon Sep 17 00:00:00 2001 From: Mateusz Neumann Date: Mon, 19 Jun 2023 14:48:31 +0200 Subject: [PATCH 12/28] RND-895 Proxy 443 port along 53333 in test_agent_via_proxy (#1550) * RND-895 Proxy 443 port along 53333 in test_agent_via_proxy * Revert "Upgrade blueprints to DSL version 1.5 (#1549)" This reverts commit e60703cba1600d74cd9ab49f793bf5d778d4ee71. This is a temporary solution before we decide what to do about incompatibility of the Cloudify Manager used in system tests: https://cloudifysource.atlassian.net/browse/RND-896 --- .../blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml | 2 +- .../resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml | 2 +- .../blueprints/cli/test-ec2-windows-vm-blueprint.yaml | 2 +- .../blueprints/cli/test-os-fileserver-vm-blueprint.yaml | 2 +- .../resources/blueprints/cli/test-os-linux-vm-blueprint.yaml | 2 +- .../resources/blueprints/compute/central_executor.yaml | 2 +- .../resources/blueprints/compute/central_executor_4_3_3.yaml | 2 +- cosmo_tester/resources/blueprints/compute/example.yaml | 2 +- .../resources/blueprints/compute/example_2_files.yaml | 2 +- cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml | 2 +- cosmo_tester/resources/blueprints/scale/groups.yaml | 2 +- cosmo_tester/resources/blueprints/scale/nodes.yaml | 2 +- .../blueprints/service_composition/capable_file.yaml | 2 +- .../resources/blueprints/service_composition/component.yaml | 2 +- .../resources/blueprints/service_composition/fake_vm.yaml | 2 +- .../blueprints/service_composition/nesting_component.yaml | 2 +- .../blueprints/service_composition/shared_resource.yaml | 2 +- cosmo_tester/resources/blueprints/summary/multivm.yaml | 2 +- cosmo_tester/resources/blueprints/summary/small.yaml | 2 +- cosmo_tester/resources/blueprints/summary/withrelations.yaml | 2 +- .../infrastructure_blueprints/aws/infrastructure-ipv6.yaml | 4 ++-- .../aws/infrastructure-multi-net.yaml | 4 ++-- .../infrastructure_blueprints/aws/infrastructure.yaml | 4 ++-- .../resources/infrastructure_blueprints/aws/vm-ipv6.yaml | 4 ++-- .../resources/infrastructure_blueprints/aws/vm-multi-net.yaml | 4 ++-- cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml | 4 ++-- .../openstack/infrastructure-multi-net.yaml | 2 +- .../infrastructure_blueprints/openstack/infrastructure.yaml | 2 +- .../infrastructure_blueprints/openstack/vm-multi-net.yaml | 2 +- .../resources/infrastructure_blueprints/openstack/vm.yaml | 2 +- cosmo_tester/test_suites/multi_net/multi_network_test.py | 2 +- 31 files changed, 37 insertions(+), 37 deletions(-) diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml index a30d41f96..b89d8906c 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-fileserver-vm-blueprint.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml index 6902c5c4f..0dca5f9ff 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-linux-vm-blueprint.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml index f753bafb2..04fc3ff7f 100644 --- a/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-ec2-windows-vm-blueprint.yaml @@ -1,5 +1,5 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml index 58709b15d..94f984572 100644 --- a/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-os-fileserver-vm-blueprint.yaml @@ -1,5 +1,5 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml b/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml index 8e54973ee..4e82df3ae 100644 --- a/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml +++ b/cosmo_tester/resources/blueprints/cli/test-os-linux-vm-blueprint.yaml @@ -1,5 +1,5 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/7.1.0.dev1/types.yaml diff --git a/cosmo_tester/resources/blueprints/compute/central_executor.yaml b/cosmo_tester/resources/blueprints/compute/central_executor.yaml index 4c6f68532..962141a36 100644 --- a/cosmo_tester/resources/blueprints/compute/central_executor.yaml +++ b/cosmo_tester/resources/blueprints/compute/central_executor.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Run test functions on the central executor instead of an agent. diff --git a/cosmo_tester/resources/blueprints/compute/central_executor_4_3_3.yaml b/cosmo_tester/resources/blueprints/compute/central_executor_4_3_3.yaml index 559d1bbc5..a80de53ab 100644 --- a/cosmo_tester/resources/blueprints/compute/central_executor_4_3_3.yaml +++ b/cosmo_tester/resources/blueprints/compute/central_executor_4_3_3.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Run test functions on the central executor instead of an agent. diff --git a/cosmo_tester/resources/blueprints/compute/example.yaml b/cosmo_tester/resources/blueprints/compute/example.yaml index 000576680..9acba2339 100644 --- a/cosmo_tester/resources/blueprints/compute/example.yaml +++ b/cosmo_tester/resources/blueprints/compute/example.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Create a file in an arbitrary location on a pre-existing VM using a plugin. diff --git a/cosmo_tester/resources/blueprints/compute/example_2_files.yaml b/cosmo_tester/resources/blueprints/compute/example_2_files.yaml index 4aafbc593..5e19e3250 100644 --- a/cosmo_tester/resources/blueprints/compute/example_2_files.yaml +++ b/cosmo_tester/resources/blueprints/compute/example_2_files.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Create a file in an arbitrary location on a pre-existing VM using a plugin. diff --git a/cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml b/cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml index d5bc44c86..9b1570a1f 100644 --- a/cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml +++ b/cosmo_tester/resources/blueprints/proxy/proxy_blueprint.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/scale/groups.yaml b/cosmo_tester/resources/blueprints/scale/groups.yaml index d46dc1f62..3a009cb5b 100644 --- a/cosmo_tester/resources/blueprints/scale/groups.yaml +++ b/cosmo_tester/resources/blueprints/scale/groups.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/scale/nodes.yaml b/cosmo_tester/resources/blueprints/scale/nodes.yaml index 82e9bd9b7..a2a1bfb2a 100644 --- a/cosmo_tester/resources/blueprints/scale/nodes.yaml +++ b/cosmo_tester/resources/blueprints/scale/nodes.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/service_composition/capable_file.yaml b/cosmo_tester/resources/blueprints/service_composition/capable_file.yaml index 882543d70..b7ea86de3 100644 --- a/cosmo_tester/resources/blueprints/service_composition/capable_file.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/capable_file.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Create a file on a vm owned by another deployment, using capabilities. diff --git a/cosmo_tester/resources/blueprints/service_composition/component.yaml b/cosmo_tester/resources/blueprints/service_composition/component.yaml index efec28ec8..251fc0dd1 100644 --- a/cosmo_tester/resources/blueprints/service_composition/component.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/component.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Create an app using another deployment to deploy a VM (as a component) diff --git a/cosmo_tester/resources/blueprints/service_composition/fake_vm.yaml b/cosmo_tester/resources/blueprints/service_composition/fake_vm.yaml index 3768e37a3..9293f29e2 100644 --- a/cosmo_tester/resources/blueprints/service_composition/fake_vm.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/fake_vm.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Create compute resource for use in service composition tests. diff --git a/cosmo_tester/resources/blueprints/service_composition/nesting_component.yaml b/cosmo_tester/resources/blueprints/service_composition/nesting_component.yaml index 99cb282ba..54409400f 100644 --- a/cosmo_tester/resources/blueprints/service_composition/nesting_component.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/nesting_component.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Create a VM which depends on an app deployed using another deployment diff --git a/cosmo_tester/resources/blueprints/service_composition/shared_resource.yaml b/cosmo_tester/resources/blueprints/service_composition/shared_resource.yaml index 25527fb96..641538387 100644 --- a/cosmo_tester/resources/blueprints/service_composition/shared_resource.yaml +++ b/cosmo_tester/resources/blueprints/service_composition/shared_resource.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 description: > Create an app using a VM from a pre-existing deployment as a resource diff --git a/cosmo_tester/resources/blueprints/summary/multivm.yaml b/cosmo_tester/resources/blueprints/summary/multivm.yaml index fcf4003e8..b37b2cee8 100644 --- a/cosmo_tester/resources/blueprints/summary/multivm.yaml +++ b/cosmo_tester/resources/blueprints/summary/multivm.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/summary/small.yaml b/cosmo_tester/resources/blueprints/summary/small.yaml index 62102cb28..0d076e129 100644 --- a/cosmo_tester/resources/blueprints/summary/small.yaml +++ b/cosmo_tester/resources/blueprints/summary/small.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/blueprints/summary/withrelations.yaml b/cosmo_tester/resources/blueprints/summary/withrelations.yaml index 6e029502c..0f4698b80 100644 --- a/cosmo_tester/resources/blueprints/summary/withrelations.yaml +++ b/cosmo_tester/resources/blueprints/summary/withrelations.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/6.3.0/types.yaml diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-ipv6.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-ipv6.yaml index 255fc0cb0..c0e8e9ec2 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-ipv6.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-ipv6.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - - cloudify/types/types.yaml + - https://cloudify.co/spec/cloudify/5.1.0/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-multi-net.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-multi-net.yaml index ac2e083cc..f7e91fb6a 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-multi-net.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure-multi-net.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - - cloudify/types/types.yaml + - https://cloudify.co/spec/cloudify/5.1.0/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure.yaml index baea66bdb..d6f97198e 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/infrastructure.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - - cloudify/types/types.yaml + - https://cloudify.co/spec/cloudify/5.1.0/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/vm-ipv6.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/vm-ipv6.yaml index b236a1449..766112c38 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/vm-ipv6.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/vm-ipv6.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - - cloudify/types/types.yaml + - https://cloudify.co/spec/cloudify/5.1.0/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/vm-multi-net.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/vm-multi-net.yaml index 0920001fa..c65b00c57 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/vm-multi-net.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/vm-multi-net.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - - cloudify/types/types.yaml + - https://cloudify.co/spec/cloudify/5.1.0/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml b/cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml index 44cba6a6f..a99755b66 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/aws/vm.yaml @@ -1,7 +1,7 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - - cloudify/types/types.yaml + - https://cloudify.co/spec/cloudify/5.1.0/types.yaml - plugin:cloudify-aws-plugin inputs: diff --git a/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure-multi-net.yaml b/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure-multi-net.yaml index 8006bd173..4cfcfacea 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure-multi-net.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure-multi-net.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/5.0.5/types.yaml diff --git a/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure.yaml b/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure.yaml index 77bc947e8..0b1bcabf3 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/openstack/infrastructure.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/5.0.5/types.yaml diff --git a/cosmo_tester/resources/infrastructure_blueprints/openstack/vm-multi-net.yaml b/cosmo_tester/resources/infrastructure_blueprints/openstack/vm-multi-net.yaml index 25cd5996a..f6883cba4 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/openstack/vm-multi-net.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/openstack/vm-multi-net.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/5.0.5/types.yaml diff --git a/cosmo_tester/resources/infrastructure_blueprints/openstack/vm.yaml b/cosmo_tester/resources/infrastructure_blueprints/openstack/vm.yaml index d16b29afc..5c8b2e046 100644 --- a/cosmo_tester/resources/infrastructure_blueprints/openstack/vm.yaml +++ b/cosmo_tester/resources/infrastructure_blueprints/openstack/vm.yaml @@ -1,4 +1,4 @@ -tosca_definitions_version: cloudify_dsl_1_5 +tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/5.0.5/types.yaml diff --git a/cosmo_tester/test_suites/multi_net/multi_network_test.py b/cosmo_tester/test_suites/multi_net/multi_network_test.py index 50765d051..2fbb2bc10 100644 --- a/cosmo_tester/test_suites/multi_net/multi_network_test.py +++ b/cosmo_tester/test_suites/multi_net/multi_network_test.py @@ -213,7 +213,7 @@ def proxy_prepare_hosts(instances, logger): # setup the proxy - simple socat services that forward all TCP connections # to the manager proxy.run_command('yum install socat -y', use_sudo=True) - for port in [5671, 53333, 15671]: + for port in [443, 5671, 53333, 15671]: service = 'proxy_{0}'.format(port) filename = '/usr/lib/systemd/system/{0}.service'.format(service) logger.info('Deploying proxy service file') From 6e960b43b0174fa4afc3b1413006c314c18515e6 Mon Sep 17 00:00:00 2001 From: Mateusz Neumann Date: Tue, 20 Jun 2023 11:54:21 +0200 Subject: [PATCH 13/28] RND-895 Don't proxy 53333 - it's no longer necessary (#1551) --- .../multi_net/multi_network_test.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cosmo_tester/test_suites/multi_net/multi_network_test.py b/cosmo_tester/test_suites/multi_net/multi_network_test.py index 2fbb2bc10..8ca01457f 100644 --- a/cosmo_tester/test_suites/multi_net/multi_network_test.py +++ b/cosmo_tester/test_suites/multi_net/multi_network_test.py @@ -213,7 +213,7 @@ def proxy_prepare_hosts(instances, logger): # setup the proxy - simple socat services that forward all TCP connections # to the manager proxy.run_command('yum install socat -y', use_sudo=True) - for port in [443, 5671, 53333, 15671]: + for port in [443, 5671, 15671]: service = 'proxy_{0}'.format(port) filename = '/usr/lib/systemd/system/{0}.service'.format(service) logger.info('Deploying proxy service file') @@ -242,18 +242,16 @@ def test_agent_via_proxy(proxy_hosts, # to make sure that the agents go through the proxy, and not connect to # the manager directly, we block all communication on the manager's - # rabbitmq and internal REST endpoint, except from the proxy (and from - # localhost) + # rabbitmq endpoint, except from the proxy (and from localhost) manager_ip = manager.private_ip_address proxy_ip = proxy.private_ip_address - for port in [5671, 53333]: + manager.run_command( + 'iptables -I INPUT -p tcp -s 0.0.0.0/0 --dport 5671 -j DROP', + use_sudo=True) + for ip in [proxy_ip, manager_ip, '127.0.0.1']: manager.run_command( - 'iptables -I INPUT -p tcp -s 0.0.0.0/0 --dport {0} -j DROP' - .format(port), use_sudo=True) - for ip in [proxy_ip, manager_ip, '127.0.0.1']: - manager.run_command( - 'iptables -I INPUT -p tcp -s {0} --dport {1} -j ACCEPT' - .format(ip, port), use_sudo=True) + f'iptables -I INPUT -p tcp -s {ip} --dport 5671 -j ACCEPT', + use_sudo=True) example = get_example_deployment( manager, ssh_key, logger, 'agent_via_proxy', test_config, vm) From 546fc1d90b80ee5bb034fde502772f6083829ca4 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Thu, 22 Jun 2023 15:49:27 +0300 Subject: [PATCH 14/28] Rd 6678 test upgrade ext db (#1521) * RD-6678 Add external db upgrade test --- .../3+ext_db_nodes_config.yaml | 75 +++++++++++++++++++ .../external_component_cluster_test.py | 70 +++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/3+ext_db_nodes_config.yaml diff --git a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/3+ext_db_nodes_config.yaml b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/3+ext_db_nodes_config.yaml new file mode 100644 index 000000000..7635a47fa --- /dev/null +++ b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_resources/3+ext_db_nodes_config.yaml @@ -0,0 +1,75 @@ +ssh_key_path: '' # Filled by the test +ssh_user: '' # Filled by the test +cloudify_license_path: '' # Filled by the test +manager_rpm_path: '' # Filled by the test + +ldap: + server: '' + domain: '' + is_active_directory: '' + ca_cert: '' + username: '' + password: '' + dn_extra: '' + +ca_cert_path: '' + +load_balancer_ip: '' + +existing_vms: + node-1: + private_ip: '' + public_ip: '' + hostname: '' + cert_path: '' + key_path: '' + config_path: + manager_config_path: '' + postgresql_config_path: '' + rabbitmq_config_path: '' + + node-2: + private_ip: '' + public_ip: '' + hostname: '' + cert_path: '' + key_path: '' + config_path: + manager_config_path: '' + postgresql_config_path: '' + rabbitmq_config_path: '' + + node-3: + private_ip: '' + public_ip: '' + hostname: '' + cert_path: '' + key_path: '' + config_path: + manager_config_path: '' + postgresql_config_path: '' + rabbitmq_config_path: '' + +external_db_configuration: + host: '' + ca_path: '' + server_db_name: postgres + server_username: postgres + server_password: '' + cloudify_db_name: cloudify_db + cloudify_username: cloudify + cloudify_password: cloudify + +credentials: + manager: + admin_username: 'admin' + admin_password: 'admin' + + rabbitmq: + username: '' + password: '' + erlang_cookie: '' + + prometheus: + username: '' + password: '' diff --git a/cosmo_tester/test_suites/cluster/external_component_cluster_test.py b/cosmo_tester/test_suites/cluster/external_component_cluster_test.py index f6c489362..54724f0fc 100644 --- a/cosmo_tester/test_suites/cluster/external_component_cluster_test.py +++ b/cosmo_tester/test_suites/cluster/external_component_cluster_test.py @@ -1,3 +1,4 @@ +import os import pytest from cosmo_tester.test_suites.cluster import check_managers @@ -6,6 +7,12 @@ create_snapshot, restore_snapshot, ) +from .cfy_cluster_manager_shared import ( + _get_config_dict, + _set_rpm_path, + _install_cluster, + _upgrade_cluster, +) # This is to confirm that we work with a single DB endpoint set (e.g. on a @@ -32,3 +39,66 @@ def test_cluster_single_db(cluster_with_single_db, logger, ssh_key, admin_password=mgr1.mgr_password) check_managers(mgr1, mgr2, example) + + +@pytest.mark.three_vms +@pytest.mark.upgrade +@pytest.mark.parametrize('base_version', ['6.3.2-ga', '6.4.1-ga']) +def test_upgrade_external_db( + base_version, three_vms, logger, ssh_key, test_config): + ext_db_fqdn = os.environ["EXTDB_FQDN"] + ext_db_password = os.environ["EXTDB_PSWD"] + + nodes_list = [node for node in three_vms] + mgr = nodes_list[0] + + _cleanup_external_db(ext_db_fqdn, ext_db_password, mgr) + + # download the ca-cert bundle for the external db + mgr.run_command('curl https://truststore.pki.rds.amazonaws.com/' + 'eu-west-1/eu-west-1-bundle.pem ' + '-o /tmp/eu-west-1-bundle.pem') + + config_dict = _get_config_dict('3+ext_db', test_config, mgr.username) + _set_rpm_path(config_dict, test_config, base_version) + _update_3_nodes_ext_db_config_dict_vms( + config_dict, nodes_list, ext_db_fqdn, ext_db_password) + _install_cluster(mgr, three_vms, config_dict, test_config, + ssh_key, logger) + _upgrade_cluster(nodes_list, mgr, test_config, logger) + + +def _cleanup_external_db(ext_db_fqdn, ext_db_password, node): + cleanup_script = f''' +if PGPASSWORD={ext_db_password} psql -U postgres -h {ext_db_fqdn} -p 5432 \ + -c "\\l" | grep -q "cloudify"; then + PGPASSWORD=cloudify psql -U cloudify postgres -h {ext_db_fqdn} -p 5432 \ + -c "drop database cloudify_db" + PGPASSWORD=cloudify psql -U cloudify postgres -h {ext_db_fqdn} -p 5432 \ + -c "drop database stage" + PGPASSWORD=cloudify psql -U cloudify postgres -h {ext_db_fqdn} -p 5432 \ + -c "drop database composer" + PGPASSWORD={ext_db_password} psql -U postgres -h {ext_db_fqdn} -p 5432 \ + -c "drop user cloudify" +fi + ''' + node.run_command('yum install -y postgresql', use_sudo=True) + node.run_command(cleanup_script) + node.run_command('yum remove -y postgresql', use_sudo=True) + + +def _update_3_nodes_ext_db_config_dict_vms( + config_dict, existing_vms_list, ext_db_fqdn, ext_db_password): + for i, node in enumerate(existing_vms_list, start=1): + config_dict['existing_vms']['node-{0}'.format(i)].update({ + 'private_ip': str(node.private_ip_address), + 'public_ip': f'{node.private_ip_address}.example.com' + }) + # Just put some FQDN, because when public_ip is an IP address, + # the external certificate cannot be provided. + # Public IPs are not used in this test. + config_dict['external_db_configuration']['host'] = ext_db_fqdn + config_dict['external_db_configuration']['ca_path'] = \ + '/tmp/eu-west-1-bundle.pem' + config_dict['external_db_configuration']['server_password'] = \ + ext_db_password From 88a62aa37baae7c512102f5b5522b9296e0ee4f0 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:49:19 +0300 Subject: [PATCH 15/28] RD-6678 Add documentation on external DB variables (#1552) --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index cf2c61eb9..60f3b2cfb 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,12 @@ pytest --pdb -s cosmo_tester/test_suites/general/simple_deployment_test.py::test --pdb is recommended for manual test runs as this will pause test execution on failure and may allow you to gain valuable insight into the cause of the failure. -s is recommended in order to ensure all test output is shown. +#### Tests which use an external DB +Some of our tests (currently only `cosmo_tester/test_suites/cluster/external_component_cluster_test.py::test_upgrade_external_db`) use an external database. +The FQDN and password of the database should be provided as environment variables `EXTDB_FQDN=...` and `EXTDB_PSWD=...` when running the test manually. +Developers may refer to the `pipelines-k8s/system-tests` pipeline in our `cloudify-build-system` repo for the values of our test-db on RDS. + + ## Using the config in tests There are two supported ways of accessing the config within tests. From 2a3d7e1cda8b7e27df451d622224b61e9ece6828 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Tue, 27 Jun 2023 15:27:43 +0300 Subject: [PATCH 16/28] Cluster manager tests: public IPs -> FQDNs (#1553) Co-authored-by: glukhman --- .../test_suites/cluster/cfy_cluster_manager_shared.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py index 7dbf8b5fd..2cd7d1877 100644 --- a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py +++ b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py @@ -30,7 +30,7 @@ def _update_three_nodes_config_dict_vms(config_dict, existing_vms_list): for i, node in enumerate(existing_vms_list, start=1): config_dict['existing_vms']['node-{0}'.format(i)].update({ 'private_ip': str(node.private_ip_address), - 'public_ip': str(node.ip_address) + 'public_ip': f'{node.ip_address}.example.com' }) @@ -46,7 +46,7 @@ def _update_nine_nodes_config_dict_vms(config_dict, existing_vms_list): config_dict['existing_vms'][node_name].update({ 'private_ip': str(node.private_ip_address), - 'public_ip': str(node.ip_address) + 'public_ip': f'{node.ip_address}.example.com' }) From a3ebb12d75057e7d6da45ca7bcd56991cd78ecb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Maksymczuk?= Date: Thu, 29 Jun 2023 09:52:57 +0200 Subject: [PATCH 17/28] RND-901 Download new rest cert: re-establish the http session (#1554) * RND-901 Download new rest cert: re-establish the http session Close the session after fetching the cert, so that next request re-loads the cert, instead of using the old cert and throwing an error. Alternatively, we could just send a dummy request, that would fail with a SSLError - that also causes a reload, so the next request would pass. But that would be silly :) * delete_deployment: exit fast if already done If there's no deployments at all, we're done, we don't need to do the 40 iterations then :P --- cosmo_tester/framework/test_hosts.py | 3 +++ cosmo_tester/framework/util.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/cosmo_tester/framework/test_hosts.py b/cosmo_tester/framework/test_hosts.py index 0c4839806..6357c3a31 100644 --- a/cosmo_tester/framework/test_hosts.py +++ b/cosmo_tester/framework/test_hosts.py @@ -854,6 +854,9 @@ def download_rest_ca(self, force=False): '/etc/cloudify/ssl/cloudify_internal_ca_cert.pem', self.api_ca_path, ) + # close the current restclient session to force making a new connection + # using the new certificate, on first use after this call + self.client._client._session.close() @only_manager def clean_local_rest_ca(self): diff --git a/cosmo_tester/framework/util.py b/cosmo_tester/framework/util.py index 63f9a0a7c..58842f3bd 100644 --- a/cosmo_tester/framework/util.py +++ b/cosmo_tester/framework/util.py @@ -598,6 +598,8 @@ def delete_deployment(client, deployment_id, logger): for _ in range(40): found = False deployments = client.deployments.list() + if not deployments: + break for deployment in deployments: if deployment['id'] == deployment_id: found = True From 177b2fe4fb695602027d18163804f2c0b5ab9ebb Mon Sep 17 00:00:00 2001 From: Mateusz Neumann Date: Mon, 3 Jul 2023 15:11:16 +0200 Subject: [PATCH 18/28] RND-959 Preserve ownership/permissions when copying (#1555) Fixes at least one test: cosmo_tester/test_suites/auth/test_saml.py::test_saml_auth --- cosmo_tester/test_suites/auth/test_saml.py | 2 +- cosmo_tester/test_suites/cluster/broker_management_test.py | 4 ++-- .../test_suites/snapshots/multi_tenant_snapshot_test.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cosmo_tester/test_suites/auth/test_saml.py b/cosmo_tester/test_suites/auth/test_saml.py index 30a1006d1..cc526a8da 100644 --- a/cosmo_tester/test_suites/auth/test_saml.py +++ b/cosmo_tester/test_suites/auth/test_saml.py @@ -61,7 +61,7 @@ def test_saml_auth(image_based_manager, logger): def _activate_saml_auth(manager, logger): logger.info('Putting SAML cert in location and restarting rest service') - manager.run_command(f'sudo cp {INTERNAL_CERT} {SAML_CERT}') + manager.run_command(f'sudo cp -a {INTERNAL_CERT} {SAML_CERT}') manager.run_command('sudo supervisorctl restart cloudify-restservice') logger.info('Waiting for manager service to finish restarting.') manager.wait_for_manager() diff --git a/cosmo_tester/test_suites/cluster/broker_management_test.py b/cosmo_tester/test_suites/cluster/broker_management_test.py index f9af93001..a5d9c1a7c 100644 --- a/cosmo_tester/test_suites/cluster/broker_management_test.py +++ b/cosmo_tester/test_suites/cluster/broker_management_test.py @@ -45,12 +45,12 @@ def add_to_hosts(target_broker, new_entry_brokers): def backup_hosts(brokers_list): for broker in brokers_list: - broker.run_command('sudo cp /etc/hosts /etc/hosts.bak') + broker.run_command('sudo cp -a /etc/hosts /etc/hosts.bak') def restore_hosts(brokers_list): for broker in brokers_list: - broker.run_command('sudo cp /etc/hosts.bak /etc/hosts') + broker.run_command('sudo cp -a /etc/hosts.bak /etc/hosts') broker.run_command('sudo rm /etc/hosts.bak') diff --git a/cosmo_tester/test_suites/snapshots/multi_tenant_snapshot_test.py b/cosmo_tester/test_suites/snapshots/multi_tenant_snapshot_test.py index ce3500992..fe35531f3 100644 --- a/cosmo_tester/test_suites/snapshots/multi_tenant_snapshot_test.py +++ b/cosmo_tester/test_suites/snapshots/multi_tenant_snapshot_test.py @@ -213,7 +213,7 @@ def prepare_old_manager_resources(manager, logger, ssh_key, test_config, 'cloudify-windows-agent.exe' ) manager.run_command('curl -Lo {} {}'.format(tmp_path, agent_url)) - manager.run_command('sudo cp {} {}'.format( + manager.run_command('sudo cp -a {} {}'.format( tmp_path, agent_destination)) for tenant in INSTALL_TENANTS: From 184dce0de64e3d8c4f82bb07c0e755e987ad5f1f Mon Sep 17 00:00:00 2001 From: Mateusz Neumann Date: Wed, 5 Jul 2023 09:22:59 +0200 Subject: [PATCH 19/28] RND-969 Update default AMI for ubuntu_16_04_image (#1556) --- cosmo_tester/config_schemas/platform_aws.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmo_tester/config_schemas/platform_aws.yaml b/cosmo_tester/config_schemas/platform_aws.yaml index f1dfb7aaa..c172818ad 100644 --- a/cosmo_tester/config_schemas/platform_aws.yaml +++ b/cosmo_tester/config_schemas/platform_aws.yaml @@ -28,7 +28,7 @@ ubuntu_14_04_image: default: ami-005af4c3162f495fa ubuntu_16_04_image: description: Image to use for Ubuntu 16.04 on this platform. - default: ami-0a9aac550bc5711d3 + default: ami-0f29c8402f8cce65c ubuntu_18_04_image: description: Image to use for Ubuntu 18.04 on this platform. default: ami-02f0341ac93c96375 From 74513d6464ec689f1b2faaea628ded801357fed4 Mon Sep 17 00:00:00 2001 From: Mateusz Neumann Date: Wed, 5 Jul 2023 09:58:37 +0200 Subject: [PATCH 20/28] RND-974 Close HTTP session only if client initialized (#1557) --- cosmo_tester/framework/test_hosts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cosmo_tester/framework/test_hosts.py b/cosmo_tester/framework/test_hosts.py index 6357c3a31..4795daa97 100644 --- a/cosmo_tester/framework/test_hosts.py +++ b/cosmo_tester/framework/test_hosts.py @@ -856,7 +856,8 @@ def download_rest_ca(self, force=False): ) # close the current restclient session to force making a new connection # using the new certificate, on first use after this call - self.client._client._session.close() + if self.client: + self.client._client._session.close() @only_manager def clean_local_rest_ca(self): From 5e857f4c2b051cd9b94bceba8b0122f02f567f46 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Wed, 5 Jul 2023 20:01:06 +0300 Subject: [PATCH 21/28] RND-946 Fix framework bugs to allow testing on RHEL-8 (#1558) Co-authored-by: glukhman --- cosmo_tester/framework/test_hosts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cosmo_tester/framework/test_hosts.py b/cosmo_tester/framework/test_hosts.py index 4795daa97..fff1f96dd 100644 --- a/cosmo_tester/framework/test_hosts.py +++ b/cosmo_tester/framework/test_hosts.py @@ -926,7 +926,8 @@ def _is_manager_image_type(self): def _is_rhel8_supported(self): if self.image_type == 'master': return True - if parse_version(self.image_type) >= parse_version('6.4.0'): + if parse_version(self.image_type.split('-')[0]) \ + >= parse_version('6.4.0'): return True return False From 420d4761348dbf297d43c615988bae802d18c9a7 Mon Sep 17 00:00:00 2001 From: Mateusz Neumann Date: Thu, 6 Jul 2023 12:27:19 +0200 Subject: [PATCH 22/28] Make all certs readable for slapd (#1559) --- cosmo_tester/test_suites/auth/test_openldap.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cosmo_tester/test_suites/auth/test_openldap.py b/cosmo_tester/test_suites/auth/test_openldap.py index 45c8b62e2..87df9e4a0 100644 --- a/cosmo_tester/test_suites/auth/test_openldap.py +++ b/cosmo_tester/test_suites/auth/test_openldap.py @@ -243,6 +243,13 @@ def _configure_openldap(host, logger): replace: olcTLSCACertificateFile olcTLSCACertificateFile: /etc/ssl/certs/ldapcacert.crt''', ) + host.run_command( + 'chgrp ldap' + ' /etc/openldap/certs/ldap-key.pem' + ' /etc/openldap/certs/ldap-cert.pem' + ' /etc/ssl/certs/ldapcacert.crt', + use_sudo=True, + ) host.run_command( 'ldapmodify -Y EXTERNAL -H ldapi:/// -f /tmp/certs.ldif', use_sudo=True, From db0b1b9697ad383f6871811c821ee006f7204703 Mon Sep 17 00:00:00 2001 From: Mateusz Neumann Date: Fri, 7 Jul 2023 15:36:00 +0200 Subject: [PATCH 23/28] RND-948 Make /cfy_backup/ readable to all (#1560) This solves the issue of teardown process not being able to restore rsync-ed backups in environment with less lax umask settings. --- cosmo_tester/framework/test_hosts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cosmo_tester/framework/test_hosts.py b/cosmo_tester/framework/test_hosts.py index fff1f96dd..4b833b66b 100644 --- a/cosmo_tester/framework/test_hosts.py +++ b/cosmo_tester/framework/test_hosts.py @@ -991,6 +991,7 @@ def rsync_backup(self): 'Creating Rsync backup for host {}. Might take up to 5 ' 'minutes...'.format(self.deployment_id)) self.run_command("mkdir /cfy_backup", use_sudo=True) + self.run_command("chmod o+r /cfy_backup", use_sudo=True) rsync_backup_file = self._tmpdir / 'rsync_backup_{0}'.format( self.ip_address) locations = ' '.join(RSYNC_LOCATIONS) From 22ed3f466442af0bb4e648b2c5560f688ad72462 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:56:06 +0300 Subject: [PATCH 24/28] RND-984 Use FQDN instead of public IP (#1561) Co-authored-by: glukhman --- cosmo_tester/framework/test_hosts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cosmo_tester/framework/test_hosts.py b/cosmo_tester/framework/test_hosts.py index 4b833b66b..4cf7b1d4d 100644 --- a/cosmo_tester/framework/test_hosts.py +++ b/cosmo_tester/framework/test_hosts.py @@ -1618,6 +1618,11 @@ def _update_instance(self, server_index, node_instance): runtime_props = node_instance['runtime_properties'] public_ip_address = runtime_props['public_ip_address'] + if self._test_config['target_platform'] == 'aws': + zone = runtime_props['resource']['Placement']['AvailabilityZone'] + public_ip_address = f"ec2-{public_ip_address.replace('.', '-')}."\ + f"{zone[:-1]}.compute.amazonaws.com" + private_ip_address = runtime_props['ipv6_address'] if self.ipv6_net \ else runtime_props['ip'] From 7c87bdd5bae2d9e69450c0412e75a7a65c59f019 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Tue, 11 Jul 2023 17:49:38 +0300 Subject: [PATCH 25/28] RND-984 workaround old test plugin file deletion (#1562) Co-authored-by: glukhman --- cosmo_tester/framework/examples.py | 5 +++-- .../resources/plugin/old_test_plugin.zip | Bin 3281 -> 0 bytes .../test_suites/general/cfy_manager_test.py | 9 ++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 cosmo_tester/resources/plugin/old_test_plugin.zip diff --git a/cosmo_tester/framework/examples.py b/cosmo_tester/framework/examples.py index b1ac7766f..921fc1201 100644 --- a/cosmo_tester/framework/examples.py +++ b/cosmo_tester/framework/examples.py @@ -144,9 +144,10 @@ def install(self): self.execute('install') self.installed = True - def uninstall(self, check_files_are_deleted=True, delete_dep=True): + def uninstall(self, check_files_are_deleted=True, delete_dep=True, + parameters=None): self.logger.info('Cleaning up example.') - self.execute('uninstall') + self.execute('uninstall', parameters=parameters) self.installed = False if check_files_are_deleted: self.check_all_test_files_deleted() diff --git a/cosmo_tester/resources/plugin/old_test_plugin.zip b/cosmo_tester/resources/plugin/old_test_plugin.zip deleted file mode 100644 index fd43096c9932a633d1527a418098283e58852986..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3281 zcma)t+UR#=iasMde_^&JpT{BFTcl72S{`W0058yf}kCSUmXNc2?PK@0ucZ} z319_y!hKv^J%s%1p2J}#-?U8+THMy>6(xXxXd4Is{PXCBwtIZzq-{Pl8zilE zoX}P{8#RXI&paGw2NQOJ7=W+27GR%3MgWof59VxMKByFw_`!at5V74qb^+N7y=(Vb zv_@NaGCahHfx!pl!@7m@tQUD+UCze!hSicTMpJ70PRA2+dq_#%fLMPC{=v=<`{m1A-rq&T z-$x2=C8imTS35hGYfC}JuY&kze3C@CM&E+LOr{Z#@HL5VT!n8@hEZmpEJ8BR5f3npxlbyG(Nv#HQg$1)F$^8MiK8p_i?^R}Pq}k!o+Os_z?)diS*dSU6dhnk zU7by|mm*qjvYB?wuj@ZqHo;mqmD@6{j@g{fs~z(!4ug}&=jnz)CN3vp(+P!OCkFsbC;@(M3t3s^wl$3SSSpiQVu+W^5jv9E?*y45#M zv%xw5Je4NlxjB8p^NJP-fbo=H3b1n7c}5zJqgOxT`w(0821ro_45*~Ocf($TxZD;} z?vg42BObRU(gS4vBn70ME9_a4*BcWG^b7O}gf3ox+W58GJ)TLEmZzs?HpPnX2Jpb; zhp%QdGl(^MZwDy%6@P(fAswbLC`=EzSt)TGUN7>?TdK%wNxi9;6zN5BfwxJrX6$p& zQWX6Zwf6{ID>bUK*P86+KZ6=dXqhO3q%B(rjeH>}&P^KK>e8vLZK8O!4z~D8Ese(g zStzN+@GAAkhrn?}#NosH4(raD95KQMFsR$%Pdc;kA$eifOiu@u{Va$Y2XpM4m&<0T zTEjgE-8Tg~rjfwa6neQNVYEC@(O5TQp>%VKRHM75yvw&mK%43ooV)In@(hyK=l^6! zp_BS3WY{R-_VJ^48}Wxy?-)cnP|VE!yK@^R3zc5UKj%u-9SB)>nPy^4XohzeF4F5c z+miDdVnI1{JcLV7W&4rws=NyMqN@}8Q$+E3;sr5QTv5Ul+wI9egWokvsGs3oQmrWJ z*O0_6^VCoHmT0Qu7!vnRkhV;|6qw%+(tf>{YHdpP&H$uc#*CaNLr-Q5e=CJ5W@S&M zMBE(Fk>`x*p$|S1dyONuyf;Rcr1ks65il!!O_hrp$GQl?G$VANb&_lymap}KOjw+4 z8O$EJ2TGP%{9%$*M&Uh*VC=2b-vya%4S@$EJ-=84V=HA1G);1w9bY{R z^kcH1(ah$O(GTjtl^!iY<8qp&jg2Bg@2@4NM(tR-x;6+VKV$exBBi9XUlKLNb`Y@Q z8QBzD@VQX~qpl<~0%j}nem#2a;sG)3C9*qcy$xE2^m-SynZ9Dy%dOv`blWIc+(2vR zhmT6PiK@P$QgcwFcCV782-&k}X`SpLKV|4%oSs*q36$2)RTj3Pbk{C$tX_!|Fw}N! z*qO5-)vw_O&Et*$B88DL4Z{OEjO=Uc%pAV~7}rpg@^MhBy*s0GBLd(2 zl`4~x+cB+t$53C{Gp_S;w2$&ZV5DBnh10Te;$?MYJ$xgSEE;xacO@)1CR(7^A`(-S5f+Xt zzMoRL(zoqdZ)?}BofAE1R*N}6b&BM+|FQHkW3X%>C_f0jO1=(;{P2iCPE%!S0{aJb z24wosD}_asQGAceBK+Tprpv}T>K8s>=WdnKC()VZ`sR;Tj9u#NW%6Dg?EOHSWFEy5 zu>6%rRsV<;^f1OM#FMo`IG23)@pOpO0fs zGiFW!`@)cete3|6e$Hplh~T0{Tjh2amHb~$H6?bVXz&GgkDXWbXcn>wl~Cc5iciXo zZns9J`$O^?r+86ff4)UhoIy{l;>7PmAS@9i)UqO^_2Z zGt`mBWxTZO?s-3xSixN=*d~WALjubLW#2mjeJgwrihCD@D&yt3|AeQh?Wq=5xaykO zO>?gf85L{?*m4ZuzYUWauV*5IMP-D8G75w;ZIha#pHSMroetT`wVWCQ7ILt2 z-%a%9Ef(X`bub{E%UEw*Kw4QxHNaYO(tS*!5`b(0B<&2tQ>tR$NB3;$%HM#q$nuZh zb}y8x6olH}y1TsNsz&-3Idw0bJ^t~;3VBmVA0|HQ^J@rx-q77_dOCd?>%PQ>Xg4EE zvS2iiADP1A4U)w=G@Z7*s`!QU0${Z%9HE}%&KdT$Cj$NL`Cggl@(+gTu_`lgH1=)+~AuDA(`Qf&6N-ZLH zq+9Exhx4vZ^BabD15LSW@B-~bW`B7UHKme%Q<&41HY6=>}}Oq)BnJ< z%q$e{#}LzLFy_gsjHEmQ$;DGRK?vGwPEkF~gpG@-cuFl8J>w<8oaR8M1hS5jBH3cb zS{=+-;*lB^1NH4R>o6kILo+XQb+U(iCRcL5@~XGZOA=pr^r2|Fy??^;p?qUXX(Np} z-gF~E>p}mgRPhJqVV1KNby=Km|43UGoR{kMCoDfpG(z~-_sn9{7!@m;9bAY3Y9XML z1K$~|{WR7Mgzc!uGN0e*u}~@kZx9a=4CX8{MJd69gO*d1>4hLGbGr50I^@m%*1`@K zJQG-f@5v)1r#@fkMZaAuu)koJNA3H z-D@&Cq>DfKo?kq%79&)1J^`A_$j8X#JM7gpI^7q_fPQ*$5?GuV!}wlEX1e10s43Xm zK*0D}e^s6F3LG=$dObT`v@2+5rHEW?PWaM{QDQKsysgdwrlNm;;NA*TUS$pb;Kx%6 z&ug^UNI;n^t>pGvN8wuJ#Q53sdMH5s-FijZ`tB!xjd-p$+oi1yL=4YLW>pYSe&?!b z{UJ4W6L<`i@@Z+lnOO^!XT(Q&UniawcU~vF`pLIOlM6gxT12#mV7+pb9@&IL&J@mW zF!C$N#cS28GKCuY5uAeJv%4!Ca<}RxUbK#eu`ZMCEtqvleSZIa;b7TNhk%eB_pz5m_-`TjA2RUo!2fQZe*=G`s>9}gasOXE^w*%@J@mf@5tIA| P5&l{YzitAm-@ktVO}H6( diff --git a/cosmo_tester/test_suites/general/cfy_manager_test.py b/cosmo_tester/test_suites/general/cfy_manager_test.py index ab3ad7550..e96e6f8a3 100644 --- a/cosmo_tester/test_suites/general/cfy_manager_test.py +++ b/cosmo_tester/test_suites/general/cfy_manager_test.py @@ -167,7 +167,14 @@ def test_cfy_manager_upgrade(base_manager, ssh_key, logger, test_config): upgrade_agents(base_manager, logger, test_config) validate_cluster_status_and_agents(base_manager, example.tenant, logger) - example.uninstall() + # We'd like to ignore a failure of file.delete due to a possible old + # version of the test plugin + if base_manager.image_type.startswith('5') or \ + base_manager.image_type.startswith('6'): + example.uninstall(parameters={'ignore_failure': True}, + check_files_are_deleted=False) + else: + example.uninstall() def _edit_security_config(manager): From 770afceeac1549ee7064121560875bc4cc024024 Mon Sep 17 00:00:00 2001 From: glukhman <47591609+glukhman@users.noreply.github.com> Date: Wed, 26 Jul 2023 18:38:11 +0300 Subject: [PATCH 26/28] Add a flag for using FQDN instead of public ip (#1563) Co-authored-by: glukhman --- cosmo_tester/framework/test_hosts.py | 11 ++-- .../cluster/cfy_cluster_manager_shared.py | 4 +- ...ompact_cfy_cluster_manager_upgrade_test.py | 6 +-- cosmo_tester/test_suites/cluster/conftest.py | 53 +++++++++++++++++++ .../external_component_cluster_test.py | 2 +- .../full_cfy_cluster_manager_upgrade_test.py | 4 +- 6 files changed, 67 insertions(+), 13 deletions(-) diff --git a/cosmo_tester/framework/test_hosts.py b/cosmo_tester/framework/test_hosts.py index 4cf7b1d4d..64baffd86 100644 --- a/cosmo_tester/framework/test_hosts.py +++ b/cosmo_tester/framework/test_hosts.py @@ -1140,7 +1140,7 @@ def __init__(self, else: self.server_flavor = self._test_config.platform['linux_size'] - def create(self): + def create(self, use_fqdn=False): """Creates the infrastructure for a Cloudify manager.""" self._logger.info('Creating image based cloudify instances: ' '[number_of_instances=%d]', len(self.instances)) @@ -1179,7 +1179,7 @@ def create(self): test_identifier, instance.is_manager, instance.image_type) - self._finish_deploy_test_vms() + self._finish_deploy_test_vms(use_fqdn=use_fqdn) for instance in self.instances: if instance.is_manager and not instance.bootstrappable: @@ -1571,7 +1571,7 @@ def _populate_aws_platform_properties(self): self._platform_resource_ids = resource_ids - def _finish_deploy_test_vms(self): + def _finish_deploy_test_vms(self, use_fqdn=False): node_instances = {} for vm_id, details in self._test_vm_installs.items(): execution, index = details @@ -1586,6 +1586,7 @@ def _finish_deploy_test_vms(self): self._update_instance( index, node_instance, + use_fqdn=use_fqdn, ) node_instances[index] = node_instance @@ -1613,12 +1614,12 @@ def _finish_undeploy_test_vms(self): util.delete_deployment(self._infra_client, vm_id, self._logger) - def _update_instance(self, server_index, node_instance): + def _update_instance(self, server_index, node_instance, use_fqdn=False): instance = self.instances[server_index] runtime_props = node_instance['runtime_properties'] public_ip_address = runtime_props['public_ip_address'] - if self._test_config['target_platform'] == 'aws': + if use_fqdn and self._test_config['target_platform'] == 'aws': zone = runtime_props['resource']['Placement']['AvailabilityZone'] public_ip_address = f"ec2-{public_ip_address.replace('.', '-')}."\ f"{zone[:-1]}.compute.amazonaws.com" diff --git a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py index 2cd7d1877..1b6fbf5a6 100644 --- a/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py +++ b/cosmo_tester/test_suites/cluster/cfy_cluster_manager_shared.py @@ -30,7 +30,7 @@ def _update_three_nodes_config_dict_vms(config_dict, existing_vms_list): for i, node in enumerate(existing_vms_list, start=1): config_dict['existing_vms']['node-{0}'.format(i)].update({ 'private_ip': str(node.private_ip_address), - 'public_ip': f'{node.ip_address}.example.com' + 'public_ip': f'{node.ip_address}' }) @@ -46,7 +46,7 @@ def _update_nine_nodes_config_dict_vms(config_dict, existing_vms_list): config_dict['existing_vms'][node_name].update({ 'private_ip': str(node.private_ip_address), - 'public_ip': f'{node.ip_address}.example.com' + 'public_ip': f'{node.ip_address}' }) diff --git a/cosmo_tester/test_suites/cluster/compact_cfy_cluster_manager_upgrade_test.py b/cosmo_tester/test_suites/cluster/compact_cfy_cluster_manager_upgrade_test.py index 07ef7d8ee..928a2e54d 100644 --- a/cosmo_tester/test_suites/cluster/compact_cfy_cluster_manager_upgrade_test.py +++ b/cosmo_tester/test_suites/cluster/compact_cfy_cluster_manager_upgrade_test.py @@ -7,8 +7,8 @@ @pytest.mark.three_vms @pytest.mark.upgrade @pytest.mark.parametrize('base_version', SUPPORTED_FOR_RPM_UPGRADE) -def test_three_nodes_cluster_upgrade(base_version, three_vms, test_config, - ssh_key, logger): +def test_three_nodes_cluster_upgrade(base_version, three_vms_fqdns, + test_config, ssh_key, logger): """Tests the command cfy_cluster_manager upgrade on a 3 nodes cluster.""" - _cluster_upgrade_test(test_config, base_version, three_vms, ssh_key, + _cluster_upgrade_test(test_config, base_version, three_vms_fqdns, ssh_key, logger) diff --git a/cosmo_tester/test_suites/cluster/conftest.py b/cosmo_tester/test_suites/cluster/conftest.py index 151f096f3..49d26c261 100644 --- a/cosmo_tester/test_suites/cluster/conftest.py +++ b/cosmo_tester/test_suites/cluster/conftest.py @@ -34,6 +34,21 @@ def three_session_vms(request, ssh_key, session_tmpdir, test_config, hosts.destroy() +@pytest.fixture(scope='session') +def three_session_vms_fqdns(request, ssh_key, session_tmpdir, test_config, + session_logger): + hosts = Hosts(ssh_key, session_tmpdir, test_config, + session_logger, request, bootstrappable=True, + number_of_instances=3) + try: + hosts.create(use_fqdn=True) + if len(request.session.items) > 1: + hosts.rsync_backup() + yield hosts.instances + finally: + hosts.destroy() + + @pytest.fixture(scope='session') def three_ipv6_session_vms(request, ssh_key, session_tmpdir, test_config, session_logger): @@ -94,6 +109,21 @@ def nine_session_vms(request, ssh_key, session_tmpdir, test_config, hosts.destroy() +@pytest.fixture(scope='session') +def nine_session_vms_fqdns(request, ssh_key, session_tmpdir, test_config, + session_logger): + hosts = Hosts(ssh_key, session_tmpdir, test_config, + session_logger, request, bootstrappable=True, + number_of_instances=9) + try: + hosts.create(use_fqdn=True) + if len(request.session.items) > 1: + hosts.rsync_backup() + yield hosts.instances + finally: + hosts.destroy() + + @pytest.fixture(scope='function') def brokers(three_session_vms, test_config, logger, request): util.reboot_if_required(three_session_vms) @@ -237,6 +267,17 @@ def three_vms(three_session_vms, test_config, logger, request): util.rsync_restore(three_session_vms) +@pytest.fixture(scope='function') +def three_vms_fqdns(three_session_vms_fqdns, test_config, logger, request): + util.reboot_if_required(three_session_vms_fqdns) + for vm in three_session_vms_fqdns: + _ensure_installer_not_installed(vm) + yield _get_hosts(three_session_vms_fqdns, test_config, logger, + three_nodes_cluster=True, bootstrap=False) + if len(request.session.items) > 1: + util.rsync_restore(three_session_vms_fqdns) + + @pytest.fixture(scope='function') def three_vms_ipv6(three_ipv6_session_vms, test_config, logger, request): util.reboot_if_required(three_ipv6_session_vms) @@ -260,6 +301,18 @@ def nine_vms(nine_session_vms, test_config, logger, request): util.rsync_restore(nine_session_vms) +@pytest.fixture(scope='function') +def nine_vms_fqdns(nine_session_vms_fqdns, test_config, logger, request): + util.reboot_if_required(nine_session_vms) + for vm in nine_session_vms_fqdns: + _ensure_installer_not_installed(vm) + yield _get_hosts(nine_session_vms_fqdns, test_config, logger, + broker_count=3, db_count=3, + manager_count=3, bootstrap=False) + if len(request.session.items) > 1: + util.rsync_restore(nine_session_vms_fqdns) + + def _ensure_installer_not_installed(vm): vm.wait_for_ssh() vm.run_command( diff --git a/cosmo_tester/test_suites/cluster/external_component_cluster_test.py b/cosmo_tester/test_suites/cluster/external_component_cluster_test.py index 54724f0fc..2a2408b18 100644 --- a/cosmo_tester/test_suites/cluster/external_component_cluster_test.py +++ b/cosmo_tester/test_suites/cluster/external_component_cluster_test.py @@ -92,7 +92,7 @@ def _update_3_nodes_ext_db_config_dict_vms( for i, node in enumerate(existing_vms_list, start=1): config_dict['existing_vms']['node-{0}'.format(i)].update({ 'private_ip': str(node.private_ip_address), - 'public_ip': f'{node.private_ip_address}.example.com' + 'public_ip': f'{node.private_ip_address}' }) # Just put some FQDN, because when public_ip is an IP address, # the external certificate cannot be provided. diff --git a/cosmo_tester/test_suites/cluster/full_cfy_cluster_manager_upgrade_test.py b/cosmo_tester/test_suites/cluster/full_cfy_cluster_manager_upgrade_test.py index 8cc2d1321..98da1635f 100644 --- a/cosmo_tester/test_suites/cluster/full_cfy_cluster_manager_upgrade_test.py +++ b/cosmo_tester/test_suites/cluster/full_cfy_cluster_manager_upgrade_test.py @@ -8,8 +8,8 @@ @pytest.mark.nine_vms @pytest.mark.upgrade @pytest.mark.parametrize('base_version', SUPPORTED_FOR_RPM_UPGRADE) -def test_nine_nodes_cluster_upgrade(base_version, nine_vms, test_config, +def test_nine_nodes_cluster_upgrade(base_version, nine_vms_fqdns, test_config, ssh_key, logger): """Tests the command cfy_cluster_manager upgrade on a 9 nodes cluster.""" - _cluster_upgrade_test(test_config, base_version, nine_vms, + _cluster_upgrade_test(test_config, base_version, nine_vms_fqdns, ssh_key, logger) From 88782b2f7138cabf0dabbfb6b0246670cc39114b Mon Sep 17 00:00:00 2001 From: glukhman Date: Tue, 15 Aug 2023 11:39:44 +0000 Subject: [PATCH 27/28] Fix-nine-nodes-fqdns-typo --- cosmo_tester/test_suites/cluster/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cosmo_tester/test_suites/cluster/conftest.py b/cosmo_tester/test_suites/cluster/conftest.py index 49d26c261..4ade0794d 100644 --- a/cosmo_tester/test_suites/cluster/conftest.py +++ b/cosmo_tester/test_suites/cluster/conftest.py @@ -303,7 +303,7 @@ def nine_vms(nine_session_vms, test_config, logger, request): @pytest.fixture(scope='function') def nine_vms_fqdns(nine_session_vms_fqdns, test_config, logger, request): - util.reboot_if_required(nine_session_vms) + util.reboot_if_required(nine_session_vms_fqdns) for vm in nine_session_vms_fqdns: _ensure_installer_not_installed(vm) yield _get_hosts(nine_session_vms_fqdns, test_config, logger, @@ -877,3 +877,4 @@ def _remove_cluster(node): # yum clean all doesn't clean all, so let's be more forceful node.log_action('Cleaning yum cache') node.run_command('sudo rm -rf /var/cache/yum') + From a0476a3d9b794d217b7c815cda6a40c229f8d853 Mon Sep 17 00:00:00 2001 From: glukhman Date: Tue, 15 Aug 2023 11:41:27 +0000 Subject: [PATCH 28/28] Update cluster-manager to newest version --- cosmo_tester/config_schemas/cfy_cluster_manager.yaml | 3 ++- cosmo_tester/test_suites/cluster/conftest.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cosmo_tester/config_schemas/cfy_cluster_manager.yaml b/cosmo_tester/config_schemas/cfy_cluster_manager.yaml index 70c00c866..e9f0a85e2 100644 --- a/cosmo_tester/config_schemas/cfy_cluster_manager.yaml +++ b/cosmo_tester/config_schemas/cfy_cluster_manager.yaml @@ -1,4 +1,5 @@ namespace: cfy_cluster_manager rpm_path: description: The cfy_cluster_manager RPM path to install. - default: 'https://repository.cloudifysource.org/cloudify/cloudify-cluster-manager/1.1.4/ga-release/cloudify-cluster-manager-1.1.4-ga.el7.x86_64.rpm' + default: 'https://repository.cloudifysource.org/cloudify/cloudify-cluster-manager/1.1.6/ga-release/cloudify-cluster-manager-1.1.6-ga.el7.x86_64.rpm' + diff --git a/cosmo_tester/test_suites/cluster/conftest.py b/cosmo_tester/test_suites/cluster/conftest.py index 4ade0794d..07c2fb4e8 100644 --- a/cosmo_tester/test_suites/cluster/conftest.py +++ b/cosmo_tester/test_suites/cluster/conftest.py @@ -877,4 +877,3 @@ def _remove_cluster(node): # yum clean all doesn't clean all, so let's be more forceful node.log_action('Cleaning yum cache') node.run_command('sudo rm -rf /var/cache/yum') -