Merge branch 'master' into temp-jammy-fork

# Conflicts:
#	Makefile
#	cluster_itests/docker-compose.yml
#	debian/rules
#	requirements.txt
#	tox.ini
#	tron/core/actionrun.py

.readthedocs.yaml

@@ -0,0 +1,39 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# RTD defaults as of 2023-11-08
+build:
+  # TODO: Bump to jammy+3.8 once master branch updated
+  os: ubuntu-20.04
+  tools:
+    python: "3.6"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Also provide downloadable zip
+formats: [htmlzip]
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: requirements-docs.txt

debian/changelog

@@ -1,3 +1,42 @@
+tron (1.27.5) jammy; urgency=medium
+
+  * 1.27.5 tagged with 'make release'
+    Commit: Allow Tron to use more than 10k file descriptors (#923)
+    There appears to be an fd leak somewhere in tron, we're not in
+    danger of hitting any host-level limits - so let's update the unit
+    file to have a bigger limit in the meantime
+
+ -- Luis Perez <[email protected]>  Wed, 11 Oct 2023 12:11:22 -0700
+
+tron (1.27.4) jammy; urgency=medium
+
+  * 1.27.4 tagged with 'make release'
+    Commit: Revert 'Link tronweb_url ' (#928)  Reverts #911 as it did
+    not actually result in a clickable link in Slack
+
+ -- Luis Perez <[email protected]>  Tue, 10 Oct 2023 12:26:07 -0700
+
+tron (1.27.3) jammy; urgency=medium
+
+  * 1.27.3 tagged with 'make release'
+    Commit: Catch all exceptions in k8s submit_command() (#926)  It's
+    entirely possible that creating a task_processing task (and/or
+    submitting one) can result in an exception. At the moment, this
+    results in the affected ActionRun getting stuck in the Starting
+    state - but this is a lie and means that the normal
+    monitoring/alerting on failed runs does not kick in.
+
+ -- Luis Perez <[email protected]>  Mon, 21 Aug 2023 12:27:26 -0700
+
+tron (1.27.2) jammy; urgency=medium
+
+  * 1.27.2 tagged with 'make release'
+    Commit: Merge pull request #924 from Yelp/u/vit/fix-
+    configsecretvolume  Fix item assignment issue with
+    ConfigSecretVolume
+
+ -- Vincent Thibault <[email protected]>  Mon, 14 Aug 2023 13:36:37 -0700
+
 tron (1.27.1) jammy; urgency=medium
 
   * 1.27.1 tagged with 'make release'

debian/tron.service

@@ -9,7 +9,7 @@ ExecStart=/usr/bin/trond --lock-file=${LOCKFILE:-$PIDFILE} --working-dir=${WORKI
 ExecStopPost=/usr/bin/logger -t tron_exit_status "SERVICE_RESULT:${SERVICE_RESULT} EXIT_CODE:${EXIT_CODE} EXIT_STATUS:${EXIT_STATUS}"
 TimeoutStopSec=20
 Restart=always
-LimitNOFILE=10000
+LimitNOFILE=100000
 
 [Install]
 WantedBy=multi-user.target

docs/jobs.rst

@@ -143,12 +143,9 @@ Optional Fields
 
 **max_runtime** (default **None**)
     A time interval (ex: "2 hours") that limits the duration of each job run.
-    If the job run is still running after this duration, all of it's actions
+    If the job run is still running after this duration, all of its actions
     are sent SIGTERM.
 
-    Note: This requires an :ref:`action_runners` to be configured. If
-    `action_runner` is none max_runtime does nothing.
-
 **time_zone** (default **None**)
     Time zone used for calculating when a job should run. Defaults to
     None, which means it will use the default time_zone set in the master

tron/__init__.py

@@ -1,4 +1,4 @@
-__version_info__ = (1, 27, 1)
+__version_info__ = (1, 27, 5)
 __version__ = ".".join("%s" % v for v in __version_info__)
 __author__ = "Yelp <[email protected]>"
 __credits__ = [

tron/bin/check_tron_jobs.py

@@ -119,7 +119,7 @@ def compute_check_result_for_job_runs(client, job, job_content, url_index, hide_
         if _skip_sensu_failure_logging:
             job_run_url = "/".join(job_run_id.rsplit(".", 1))
             tronweb_url = f"http://y/tron-{get_superregion()}/#job/{job_run_url}"
-            stderr_default = f"Please visit [{tronweb_url}]({tronweb_url}) for stderr details."
+            stderr_default = f"Please visit {tronweb_url} for stderr details."
             action_run_details = {}
         else:
             stderr_default = "(No stderr available)"

tron/core/actionrun.py

@@ -1178,7 +1178,14 @@ def submit_command(self, attempt: ActionRunAttempt) -> Optional[KubernetesTask]:
 
         # Watch before submitting, in case submit causes a transition
         self.watch(task)
-        k8s_cluster.submit(task)
+
+        try:
+            k8s_cluster.submit(task)
+        except Exception:
+            log.exception(f"Unable to submit task for ActionRun {self.id}")
+            self.fail(exitcode.EXIT_KUBERNETES_TASK_INVALID)
+            return None
+
         return task
 
     def recover(self) -> Optional[KubernetesTask]:

yelp_package/bionic/Dockerfile

@@ -37,5 +37,5 @@ RUN echo "deb http://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list
 RUN echo "deb http://deb.nodesource.com/node_10.x bionic main" > /etc/apt/sources.list.d/nodesource.list
 RUN apt-get -q update && apt-get -q install -y --no-install-recommends yarn nodejs
 
-RUN pip3 install --index-url ${PIP_INDEX_URL} virtualenv==16.7.5
+RUN pip3 install --trusted-host 169.254.255.254 --index-url ${PIP_INDEX_URL} virtualenv==16.7.5
 WORKDIR /work

yelp_package/itest_dockerfiles/tronmaster/Dockerfile

@@ -0,0 +1,40 @@
+FROM ubuntu:bionic
+
+RUN apt-get -q update && \
+    DEBIAN_FRONTEND=noninteractive apt-get -q install -y --no-install-recommends \
+        coffeescript \
+        debhelper \
+        devscripts \
+        dh-virtualenv \
+        dpkg-dev \
+        gcc \
+        gdebi-core \
+        git \
+        help2man \
+        libffi-dev \
+        libgpgme11 \
+        libssl-dev \
+        libdb5.3-dev \
+        libyaml-dev \
+        libssl-dev \
+        libffi-dev \
+        python3.6-dev \
+        python3-pip \
+        python-tox \
+        wget \
+    && apt-get -q clean
+
+ARG PIP_INDEX_URL
+ARG NPM_CONFIG_REGISTRY
+ENV PIP_INDEX_URL=${PIP_INDEX_URL:-https://pypi.python.org/simple}
+ENV NPM_CONFIG_REGISTRY=${NPM_CONFIG_REGISTRY:-https://npm.yelpcorp.com}
+
+# Get yarn and node
+RUN wget --quiet -O - https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+RUN wget --quiet -O - https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
+RUN echo "deb http://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list
+RUN echo "deb http://deb.nodesource.com/node_10.x bionic main" > /etc/apt/sources.list.d/nodesource.list
+RUN apt-get -q update && apt-get -q install -y --no-install-recommends yarn nodejs
+
+RUN pip3 install --trusted-host 169.254.255.254 --index-url ${PIP_INDEX_URL} virtualenv==16.7.5
+WORKDIR /work