SEC-19136 | DAR-2360 | don't force a default value for the --aws-prof…

…ile if not provided in spark-run (#3901) * DAR-2360 | don't force a default value for the --aws-profile if not provided in spark-run * SEC-19136 | detailed message for preferring --aws-profile over iam_role * SEC-19136 | bump up service_configuration_lib version --------- Co-authored-by: Sameer Sharma <[email protected]>
Yelp · Jul 15, 2024 · deed6ed · deed6ed
1 parent 6372be1
commit deed6ed
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 11 deletions.
diff --git a/paasta_tools/cli/cmds/spark_run.py b/paasta_tools/cli/cmds/spark_run.py
@@ -72,8 +72,6 @@
 # Extra room for memory overhead and for any other running inside container
 DOCKER_RESOURCE_ADJUSTMENT_FACTOR = 2
 
-DEFAULT_AWS_PROFILE = "default"
-
 DEPRECATED_OPTS = {
     "j": "spark.jars",
     "jars": "spark.jars",
@@ -361,7 +359,6 @@ def add_subparser(subparsers):
         "--aws-credentials-yaml is not specified and --service is either "
         "not specified or the service does not have credentials in "
         "/etc/boto_cfg",
-        default=DEFAULT_AWS_PROFILE,
     )
 
     aws_group.add_argument(
@@ -1062,24 +1059,23 @@ def update_args_from_tronfig(args: argparse.Namespace) -> Optional[Dict[str, str
             if field_name == "spark_args":
                 value = " ".join([f"{k}={v}" for k, v in dict(value).items()])
 
-            # Befutify for printing
+            # Beautify for printing
             arg_name_str = (f"--{arg_name.replace('_', '-')}").ljust(31, " ")
-            field_name_str = field_name.ljust(28)
 
             # Only load iam_role value if --aws-profile is not set
-            if field_name == "iam_role" and args.aws_profile != DEFAULT_AWS_PROFILE:
+            if field_name == "iam_role" and args.aws_profile is not None:
                 print(
                     PaastaColors.yellow(
-                        f"Overwriting args with Tronfig: {arg_name_str} => {field_name_str} : IGNORE, "
-                        "since --aws-profile is provided"
+                        f"Ignoring Tronfig: `{field_name} : {value}`, since `--aws-profile` is provided. "
+                        f"We are giving higher priority to `--aws-profile` in case of paasta spark-run adhoc runs."
                     ),
                 )
                 continue
 
             if hasattr(args, arg_name):
                 print(
                     PaastaColors.yellow(
-                        f"Overwriting args with Tronfig: {arg_name_str} => {field_name_str} : {value}"
+                        f"Overwriting args with Tronfig: {arg_name_str} => {field_name} : {value}"
                     ),
                 )
             setattr(args, arg_name, value)

diff --git a/requirements-minimal.txt b/requirements-minimal.txt
@@ -54,7 +54,7 @@ requests-cache >= 0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib >= 2.18.17
+service-configuration-lib >= 2.18.19
 signalfx
 slackclient >= 1.2.1
 sticht >= 1.1.0

diff --git a/requirements.txt b/requirements.txt
@@ -89,7 +89,7 @@ rsa==4.7.2
 ruamel.yaml==0.15.96
 s3transfer==0.10.0
 sensu-plugin==0.3.1
-service-configuration-lib==2.18.17
+service-configuration-lib==2.18.19
 setuptools==39.0.1
 signalfx==1.0.17
 simplejson==3.10.0