External Test Runner Support (#263)

* External test runner support

* Use default test runner for projects missing from workspace.edn

* Keep running tests from other projects even if one of the projects do not have anything to run

* Added more tests to check enrich-settings

* Updated docstrings for TestRunner and ExternalTestRunner

* Fixed test runner usage example

* Fixed a typo in setup/teardown executor functions

* Fixed failing workspace tests

* Updated test runner examples

* Move test runner related enrichment to the workspace map from workspace-clj to workspace component

components/test-runner-contract/src/polylith/clj/core/test_runner_contract/interface.clj

@@ -1,48 +1,143 @@
 (ns polylith.clj.core.test-runner-contract.interface)
 
 (defprotocol TestRunner
-  "Implement to supply a custom test runner
+  "Implement this protocol to supply a custom test runner.
 
-  `test-runner-name`
-    - should return a printable name that the test orchestrator can print out for information purposes
+  Runner options:
+  `is-verbose`      -> A boolean indicating if we are running in verbose mode
+                       or not. TestRunner can use this to print additional
+                       information about the test run.
+  `color-mode`      -> The color-mode that the poly tool is currently running with.
+                       TestRunner is expected to respect the color mode.
+  `project`         -> A map containing the project information.
+  `all-paths`       -> A vector of all paths necessary to create a classpath for
+                       running the tests in isolation within the context of the
+                       current project.
+  `setup-fn`        -> An optional setup function for tests defined in the
+                       workspace config. The poly tool will run this function
+                       before calling run-tests only if this is an in-process
+                       TestRunner. If this is an ExternalTestRunner, the external
+                       test runner should run the setup-fn.
+  `teardown-fn`     -> An optional teardown function for tests defined in the
+                       workspace config. The poly tool will run this function
+                       after the run-tests function completes (exception or not),
+                       only if this is an in-process TestRunner. If this is an
+                       ExternalTestRunner, the external test runner should run
+                       the teardown-fn.
 
-  `test-sources-present?`
-    - called first
-    - if falsey, we short-circuit, not even the project classloader will be created
+  Additional options for in-process TestRunner:
+  `class-loader`    -> The isolated classloader created from the `all-paths`.
+                       This classloader will be used to evaluate statements within
+                       the project's context. Use this if you need more granular
+                       access. `eval-in-project` should be sufficient for most
+                       cases.
+  `eval-in-project` -> A function that takes a single form as its argument and
+                       evaluates it within the project's classloader. It returns
+                       the result of the evaluation. This is the primary interface
+                       for running tests in the project's isolated context.
 
-  `tests-present?`
-    - if falsey, run-tests won't be called; can eval forms in the project context
+  Additional options for ExternalTestRunner:
+  `process-ns`      -> The main namespace of the external test runner. This
+                       namespace will be invoked as a Java subprocess.
 
-  `run-tests`
-    - should throw if the test run is considered failed
+  Usage:
+  Create a constructor function that returns an instance of TestRunner or
+  ExternalTestRunner:
 
-  Special args:
+  ```
+  (defn create [{:keys [workspace project test-settings is-verbose color-mode changes]}]
+    ...
 
-  `eval-in-project`
-    - a function that takes a single form which it evaluates in the project classloader and returns its result
-    - this is the primary interface for running tests in the project's context
+    (reify
+      test-runner-contract/TestRunner
+      (test-runner-name [this] ...)
 
-  `class-loader`
-    - the project classloader in case more granular access is needed to it
+      (test-sources-present? [this] ...)
 
+      (tests-present? [this runner-opts] ...)
 
-   To use a custom test runner, create a constructor that returns an instance of it:
+      (run-tests [this runner-opts] ...)
 
-   (defn create [{:keys [workspace project changes test-settings]}]
-     ,,,
-     (reify TestRunner ,,,))
+      ; Optional, only if you want an external test runner
+      test-runner-contract/ExternalTestRunner
+      (external-process-namespace [this] ...)))
+  ```
 
-   `workspace` passed to the constructor will contain `:user-input`, which
-   can be used to receive additional parameters for runtime configuration.
+  `workspace` passed to the constructor will contain `:user-input`, which
+  can be used to receive additional parameters for runtime configuration.
 
-   And in workspace.edn:
+  Add your constructor function in the workspace.edn. To add a single global
+  test runner, use the `:test` key:
 
-   {:test {:create-test-runner my.namespace/create} ;; to use it globally
+  {:test {:create-test-runner my.namespace/create}
 
-    :projects {\"project-a\" {:test {:create-test-runner my.namespace/create}} ;; to use it only for a project
-               \"project-b\" {:test {:create-test-runner :default}} ;; to reset the global setting to default
-               }}"
-  (test-runner-name [this])
-  (test-sources-present? [this])
-  (tests-present? [this {:keys [class-loader eval-in-project] :as opts}])
-  (run-tests [this {:keys [class-loader color-mode eval-in-project is-verbose] :as opts}]))
+   :projects {\"project-a\" {:alias \"a\"}
+              \"project-b\" {:alias \"b\"}}}
+
+  To add a multiple global test runners, use the vector variant inside the
+  `:test` key. The following example will add three test runners globally
+  where the last one is the default test runner.
+
+  {:test {:create-test-runner [my.namespace/create se.example/create :default]}
+
+   :projects {\"project-a\" {:alias \"a\"}
+              \"project-b\" {:alias \"b\"}}}
+
+  To add a custom test runner for a specific project, use the `:test` key
+  in the project configuration. You can also add multiple test runners with
+  using the vector variant.
+
+  {:projects {\"project-a\" {:alias \"a\"
+                             :test {:create-test-runner my.namespace/create}}
+              \"project-b\" {:alias \"b\"
+                             :test {:create-test-runner [my.namespace/create
+                                                         :default]}}}}
+
+  Adding a test runner definition to a project will override the global test
+  runner. The project-a will use the global test runner, `my.namespace/create`
+  whereas project-b will use the default test runner.
+
+  {:test {:create-test-runner my.namespace/create}
+
+   :projects {\"project-a\" {:alias \"a\"}
+              \"project-b\" {:alias \"b\"
+                             :test {:create-test-runner :default}}}}"
+
+  (test-runner-name [this]
+    "Returns a printable name that the poly tool can print out for
+    information purposes")
+
+  (test-sources-present? [this]
+    "The poly tool calls this first before attempting to run any tests. If
+    it returns a falsy value, we short-circuit. Not even the project
+    classloader will be created")
+
+  (tests-present? [this runner-opts]
+    "The poly tool calls this before calling the run-tests. If it returns a
+    falsy value, run-tests won't be called. The runner-opts passed to this
+    function is identical to the one passed to the run-tests. It can evaluate
+    forms in the project's context.")
+
+  (run-tests [this runner-opts]
+    "It should run the tests and throw an exception if the test run is considered
+     failed."))
+
+(defprotocol ExternalTestRunner
+  "Extends the `TestRunner` protocol to provide an external process namespace
+  for a test runner. Polylith uses a classloader approach to run tests in
+  isolation by default. `ExternalTestRunner` skips the classloaders and uses
+  Java subprocesses."
+
+  (external-process-namespace [this]
+    "Returns a symbol or string identifying the main namespace of an external
+    test runner. If it returns nil (default), the test runner will be an
+    in-process test runner and the tests will run in an isolated classloader
+    within the same process.
+
+    When an external test runner is used, the poly tool will not create a
+    classloader. The external test runner implementation should use the
+    `all-paths` argument passed to the run-tests function to create a classpath
+    for the Java subprocesses.
+
+    The setup-fn and teardown-fn must be run by the external test runner
+    instead of the poly tool."))

components/test-runner-contract/src/polylith/clj/core/test_runner_contract/interface/verifiers.clj

@@ -7,5 +7,8 @@
 (defn valid-test-runner? [candidate]
   (core/valid-test-runner? candidate))
 
+(defn valid-external-test-runner? [candidate]
+  (core/valid-external-test-runner? candidate))
+
 (defn ensure-valid-test-runner [candidate]
   (core/ensure-valid-test-runner candidate))

components/test-runner-contract/src/polylith/clj/core/test_runner_contract/verifiers.clj

@@ -16,6 +16,9 @@
 (defn valid-test-runner? [candidate]
   (satisfies? test-runner-contract/TestRunner candidate))
 
+(defn valid-external-test-runner? [candidate]
+  (satisfies? test-runner-contract/ExternalTestRunner candidate))
+
 (defn ensure-valid-test-runner [candidate]
   (when-not (valid-test-runner? candidate)
     (throw (ex-info "Test runners must satisfy the TestRunner protocol" {:candidate candidate})))

components/test-runner-orchestrator/deps.edn

@@ -1,4 +1,4 @@
 {:paths ["src"]
  :deps {}
- :aliases {:test {:extra-paths []
+ :aliases {:test {:extra-paths ["test"]
                   :extra-deps {}}}}

components/test-runner-orchestrator/src/polylith/clj/core/test_runner_orchestrator/core.clj

@@ -56,19 +56,48 @@
                           " project: " e))
             (throw e))))))
 
+(defn execute-setup-fn [project color-mode {:keys [setup-fn process-ns class-loader]}]
+  ;; DO NOT run setup-fn if the current test runner is an external process test runner.
+  ;; The external test runner will run setup and teardown functions itself.
+  (if (or process-ns (nil? setup-fn))
+    true
+    (execute-fn setup-fn "setup" (:name project) class-loader color-mode)))
+
+(defn execute-teardown-fn [project color-mode {:keys [teardown-fn process-ns class-loader]} throw?]
+  ;; DO NOT run teardown-fn if the current test runner is an external process test runner.
+  ;; The external test runner will run setup and teardown functions itself.
+  (when-not process-ns
+    (when teardown-fn
+      (when-not (execute-fn teardown-fn "teardown" (:name project) class-loader color-mode)
+        (when throw?
+          (throw (ex-info "Test terminated due to teardown failure"
+                          {:project project
+                           :teardown-failed? true})))))))
+
 (defn run-tests-for-project-with-test-runner
-  [{:keys [test-runner setup-delay teardown-delay color-mode runner-opts project]}]
+  [{:keys [test-runner color-mode runner-opts project]}]
   (let [for-project-using-runner
         (str "for the " (color/project (:name project) color-mode)
              " project using test runner: "
              (test-runner-contract/test-runner-name test-runner))]
     (if-not (test-runner-contract/tests-present? test-runner runner-opts)
       (println (str "No tests to run " for-project-using-runner "."))
-      (if (deref setup-delay)
+      (if (execute-setup-fn (:name project) color-mode runner-opts)
         (try
           (println (str "Running tests " for-project-using-runner "..."))
           (test-runner-contract/run-tests test-runner runner-opts)
-          (catch Throwable e (deref teardown-delay) (throw e)))
+
+          ;; Run the teardown function and throw if it fails.
+          (execute-teardown-fn project color-mode runner-opts true)
+
+          (catch Throwable e
+            ;; If this exception is due to running teardown function, skip running
+            ;; teardown function again.
+            (when-not (-> e ex-data :teardown-failed?)
+              ;; Run the teardown function and DO NOT throw if it fails. We want to
+              ;; throw the actual exception received from the test runner.
+              (execute-teardown-fn project color-mode runner-opts false))
+            (throw e)))
         (throw (ex-info (str "Test terminated due to setup failure") {:project project}))))))
 
 (defn ex-causes [ex]
@@ -84,12 +113,12 @@
                    {:form form}
                    e))))))
 
-(defn test-opts [workspace
-                 settings
-                 changes
-                 {:keys [name] :as project}
-                 is-verbose
-                 color-mode]
+(defn ->test-opts [workspace
+                   settings
+                   changes
+                   {:keys [name] :as project}
+                   is-verbose
+                   color-mode]
   {:workspace workspace
    :project project
    :changes changes
@@ -99,7 +128,7 @@
 
 (defn run-tests-for-project [{:keys [workspace project test-settings is-verbose color-mode] :as opts}]
   (let [{:keys [settings]} workspace
-        {:keys [name paths]} project
+        {:keys [paths]} project
         {:keys [create-test-runner setup-fn teardown-fn]} test-settings
         test-runners-seeing-test-sources
         (into []
@@ -108,26 +137,26 @@
               create-test-runner)]
     (when (seq test-runners-seeing-test-sources)
       (let [lib-paths (resolve-deps project settings is-verbose color-mode)
-            all-paths (into #{} cat [(:src paths) (:test paths) lib-paths])
-            class-loader (common/create-class-loader all-paths color-mode)
-            setup!* (delay (execute-fn setup-fn "setup" name class-loader color-mode))
-            setup-failed? #(and (realized? setup!*) (not (deref setup!*)))
-            setup-succeeded? #(and (realized? setup!*) (deref setup!*))
-            teardown!* (delay (execute-fn teardown-fn "teardown" name class-loader color-mode))
-            runner-opts (merge opts
-                               {:class-loader class-loader
-                                :eval-in-project (->eval-in-project class-loader)})]
+            all-paths (into [] cat [(:src paths) (:test paths) lib-paths])
+            class-loader-delay (delay (common/create-class-loader all-paths color-mode))]
         (when is-verbose (println (str "# paths:\n" all-paths "\n")))
         (doseq [current-test-runner test-runners-seeing-test-sources]
-          (when-not (setup-failed?)
+          (let [process-ns (when (test-runner-verifiers/valid-external-test-runner? current-test-runner)
+                             (test-runner-contract/external-process-namespace current-test-runner))
+                runner-opts (cond-> {:setup-fn setup-fn
+                                     :teardown-fn teardown-fn
+                                     :all-paths all-paths}
+
+                                    process-ns
+                                    (assoc :process-ns process-ns)
+
+                                    (not process-ns)
+                                    (assoc :class-loader @class-loader-delay
+                                           :eval-in-project (->eval-in-project @class-loader-delay)))]
             (->> {:test-runner current-test-runner
-                  :setup-delay setup!*
-                  :teardown-delay teardown!*
-                  :runner-opts runner-opts}
+                  :runner-opts (merge opts runner-opts)}
                  (merge opts)
-                 (run-tests-for-project-with-test-runner))))
-        (when (setup-succeeded?)
-          (deref teardown!*))))))
+                 (run-tests-for-project-with-test-runner))))))))
 
 (defn affected-by-changes? [{:keys [name]} {:keys [project-to-bricks-to-test project-to-projects-to-test]}]
   (seq (concat (project-to-bricks-to-test name)
@@ -180,12 +209,9 @@
           (print-projects-to-test projects-to-test color-mode)
           (print-bricks-to-test component-names base-names bricks-to-test color-mode)
           (println)
-          (transduce
-            (comp (map #(test-opts workspace settings changes % is-verbose color-mode))
-                  (map run-tests-for-project)
-                  (map #(do (System/gc) %)))
-            (completing (fn [_ x] (cond-> x (not x) (reduced))))
-            true
-            projects-to-test)))
+          (doseq [project projects-to-test]
+            (let [test-opts (->test-opts workspace settings changes project is-verbose color-mode)]
+              (run-tests-for-project test-opts)
+              (System/gc)))))
       (print-execution-time start-time)
       true)))