From 5c400d3f45d689178d7f54d543dab2c4878175eb Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 10 Sep 2024 14:21:52 +0900 Subject: [PATCH] Update the latest CoreAssetion for assert_linear_performance --- tool/lib/core_assertions.rb | 130 +++++++++--- tool/lib/envutil.rb | 81 ++++++-- tool/lib/test/unit.rb | 347 ++++++++++++++++++++++--------- tool/lib/test/unit/assertions.rb | 23 +- tool/lib/test/unit/parallel.rb | 27 ++- tool/lib/test/unit/testcase.rb | 22 +- 6 files changed, 450 insertions(+), 180 deletions(-) diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 5ae9019c46a0f9..361b1a697d4495 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -1,8 +1,49 @@ # frozen_string_literal: true module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + module Unit module Assertions + def assert_raises(*exp, &b) + raise NoMethodError, "use assert_raise", caller + end + def _assertions= n # :nodoc: @_assertions = n end @@ -33,6 +74,11 @@ def message msg = nil, ending = nil, &default module CoreAssertions require_relative 'envutil' require 'pp' + begin + require '-test-/asan' + rescue LoadError + end + nil.pretty_inspect def mu_pp(obj) #:nodoc: @@ -107,8 +153,13 @@ def syntax_check(code, fname, line) end def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) - # TODO: consider choosing some appropriate limit for MJIT and stop skipping this once it does not randomly fail + # TODO: consider choosing some appropriate limit for RJIT and stop skipping this once it does not randomly fail + pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + # For previous versions which implemented MJIT pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + # ASAN has the same problem - its shadow memory greatly increases memory usage + # (plus asan has better ways to detect memory leaks than this assertion) + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if defined?(Test::ASAN) && Test::ASAN.enabled? require_relative 'memory_status' raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) @@ -244,7 +295,11 @@ def separated_runner(token, out = nil) at_exit { out.puts "#{token}", [Marshal.dump($!)].pack('m'), "#{token}", "#{token}assertions=#{self._assertions}" } - Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) if defined?(Test::Unit::Runner) + if defined?(Test::Unit::Runner) + Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) + elsif defined?(Test::Unit::AutoRunner) + Test::Unit::AutoRunner.need_auto_run = false + end end def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **opt) @@ -254,7 +309,7 @@ def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **o line ||= loc.lineno end capture_stdout = true - unless /mswin|mingw/ =~ RUBY_PLATFORM + unless /mswin|mingw/ =~ RbConfig::CONFIG['host_os'] capture_stdout = false opt[:out] = Test::Unit::Runner.output if defined?(Test::Unit::Runner) res_p, res_c = IO.pipe @@ -531,11 +586,11 @@ def assert_not_respond_to(obj, (meth, *priv), msg = nil) refute_respond_to(obj, meth, msg) end - # pattern_list is an array which contains regexp and :*. + # pattern_list is an array which contains regexp, string and :*. # :* means any sequence. # # pattern_list is anchored. - # Use [:*, regexp, :*] for non-anchored match. + # Use [:*, regexp/string, :*] for non-anchored match. def assert_pattern_list(pattern_list, actual, message=nil) rest = actual anchored = true @@ -544,11 +599,13 @@ def assert_pattern_list(pattern_list, actual, message=nil) anchored = false else if anchored - match = /\A#{pattern}/.match(rest) + match = rest.rindex(pattern, 0) else - match = pattern.match(rest) + match = rest.index(pattern) end - unless match + if match + post_match = $~ ? $~.post_match : rest[match+pattern.size..-1] + else msg = message(msg) { expect_msg = "Expected #{mu_pp pattern}\n" if /\n[^\n]/ =~ rest @@ -565,7 +622,7 @@ def assert_pattern_list(pattern_list, actual, message=nil) } assert false, msg end - rest = match.post_match + rest = post_match anchored = true end } @@ -592,14 +649,14 @@ def assert_warn(*args) def assert_deprecated_warning(mesg = /deprecated/) assert_warning(mesg) do - Warning[:deprecated] = true + Warning[:deprecated] = true if Warning.respond_to?(:[]=) yield end end def assert_deprecated_warn(mesg = /deprecated/) assert_warn(mesg) do - Warning[:deprecated] = true + Warning[:deprecated] = true if Warning.respond_to?(:[]=) yield end end @@ -691,7 +748,7 @@ def assert_join_threads(threads, message = nil) msg = "exceptions on #{errs.length} threads:\n" + errs.map {|t, err| "#{t.inspect}:\n" + - RUBY_VERSION >= "2.5.0" ? err.full_message(highlight: false, order: :top) : err.message + (err.respond_to?(:full_message) ? err.full_message(highlight: false, order: :top) : err.message) }.join("\n---\n") if message msg = "#{message}\n#{msg}" @@ -726,35 +783,60 @@ def assert_all_assertions_foreach(msg = nil, *keys, &block) end alias all_assertions_foreach assert_all_assertions_foreach + %w[ + CLOCK_THREAD_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID + CLOCK_MONOTONIC + ].find do |c| + if Process.const_defined?(c) + [c.to_sym, Process.const_get(c)].find do |clk| + begin + Process.clock_gettime(clk) + rescue + # Constants may be defined but not implemented, e.g., mingw. + else + PERFORMANCE_CLOCK = clk + end + end + end + end + # Expect +seq+ to respond to +first+ and +each+ methods, e.g., # Array, Range, Enumerator::ArithmeticSequence and other # Enumerable-s, and each elements should be size factors. # # :yield: each elements of +seq+. def assert_linear_performance(seq, rehearsal: nil, pre: ->(n) {n}) + pend "No PERFORMANCE_CLOCK found" unless defined?(PERFORMANCE_CLOCK) + + # Timeout testing generally doesn't work when RJIT compilation happens. + rjit_enabled = defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + measure = proc do |arg, message| + st = Process.clock_gettime(PERFORMANCE_CLOCK) + yield(*arg) + t = (Process.clock_gettime(PERFORMANCE_CLOCK) - st) + assert_operator 0, :<=, t, message unless rjit_enabled + t + end + first = seq.first *arg = pre.call(first) times = (0..(rehearsal || (2 * first))).map do - st = Process.clock_gettime(Process::CLOCK_MONOTONIC) - yield(*arg) - t = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - st) - assert_operator 0, :<=, t - t.nonzero? + measure[arg, "rehearsal"].nonzero? end times.compact! tmin, tmax = times.minmax - tmax *= tmax / tmin - tmax = 10**Math.log10(tmax).ceil + + # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) + tbase = 10 * tmax * [(tmax / tmin) ** 2 / 4, 1].max + info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" seq.each do |i| next if i == first - t = tmax * i.fdiv(first) + t = tbase * i.fdiv(first) *arg = pre.call(i) - message = "[#{i}]: in #{t}s" + message = "[#{i}]: in #{t}s #{info}" Timeout.timeout(t, Timeout::Error, message) do - st = Process.clock_gettime(Process::CLOCK_MONOTONIC) - yield(*arg) - assert_operator (Process.clock_gettime(Process::CLOCK_MONOTONIC) - st), :<=, t, message + measure[arg, message] end end end diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index 0391b90c1cc9de..642965047f7c32 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -15,23 +15,22 @@ module EnvUtil def rubybin if ruby = ENV["RUBY"] - return ruby - end - ruby = "ruby" - exeext = RbConfig::CONFIG["EXEEXT"] - rubyexe = (ruby + exeext if exeext and !exeext.empty?) - 3.times do - if File.exist? ruby and File.executable? ruby and !File.directory? ruby - return File.expand_path(ruby) - end - if rubyexe and File.exist? rubyexe and File.executable? rubyexe - return File.expand_path(rubyexe) - end - ruby = File.join("..", ruby) - end - if defined?(RbConfig.ruby) + ruby + elsif defined?(RbConfig.ruby) RbConfig.ruby else + ruby = "ruby" + exeext = RbConfig::CONFIG["EXEEXT"] + rubyexe = (ruby + exeext if exeext and !exeext.empty?) + 3.times do + if File.exist? ruby and File.executable? ruby and !File.directory? ruby + return File.expand_path(ruby) + end + if rubyexe and File.exist? rubyexe and File.executable? rubyexe + return File.expand_path(rubyexe) + end + ruby = File.join("..", ruby) + end "ruby" end end @@ -53,7 +52,14 @@ def capture_global_values @original_internal_encoding = Encoding.default_internal @original_external_encoding = Encoding.default_external @original_verbose = $VERBOSE - @original_warning = defined?(Warning.[]) ? %i[deprecated experimental].to_h {|i| [i, Warning[i]]} : nil + @original_warning = + if defined?(Warning.categories) + Warning.categories.to_h {|i| [i, Warning[i]]} + elsif defined?(Warning.[]) # 2.7+ + %i[deprecated experimental performance].to_h do |i| + [i, begin Warning[i]; rescue ArgumentError; end] + end.compact + end end end @@ -152,7 +158,12 @@ def invoke_ruby(args, stdin_data = "", capture_stdout = false, capture_stderr = if RUBYLIB and lib = child_env["RUBYLIB"] child_env["RUBYLIB"] = [lib, RUBYLIB].join(File::PATH_SEPARATOR) end - child_env['ASAN_OPTIONS'] = ENV['ASAN_OPTIONS'] if ENV['ASAN_OPTIONS'] + + # remain env + %w(ASAN_OPTIONS RUBY_ON_BUG).each{|name| + child_env[name] = ENV[name] if !child_env.key?(name) and ENV.key?(name) + } + args = [args] if args.kind_of?(String) pid = spawn(child_env, *precommand, rubybin, *args, opt) in_c.close @@ -241,6 +252,24 @@ def under_gc_stress(stress = true) end module_function :under_gc_stress + def under_gc_compact_stress(val = :empty, &block) + raise "compaction doesn't work well on s390x. Omit the test in the caller." if RUBY_PLATFORM =~ /s390x/ # https://github.com/ruby/ruby/pull/5077 + auto_compact = GC.auto_compact + GC.auto_compact = val + under_gc_stress(&block) + ensure + GC.auto_compact = auto_compact + end + module_function :under_gc_compact_stress + + def without_gc + prev_disabled = GC.disable + yield + ensure + GC.enable unless prev_disabled + end + module_function :without_gc + def with_default_external(enc) suppress_warning { Encoding.default_external = enc } yield @@ -292,16 +321,24 @@ def self.diagnostic_reports(signame, pid, now) cmd = @ruby_install_name if "ruby-runner#{RbConfig::CONFIG["EXEEXT"]}" == cmd path = DIAGNOSTIC_REPORTS_PATH timeformat = DIAGNOSTIC_REPORTS_TIMEFORMAT - pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.crash" + pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.{crash,ips}" first = true 30.times do first ? (first = false) : sleep(0.1) Dir.glob(pat) do |name| log = File.read(name) rescue next - if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log - File.unlink(name) - File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil - return log + case name + when /\.crash\z/ + if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log + File.unlink(name) + File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil + return log + end + when /\.ips\z/ + if /^ *"pid" *: *#{pid},/ =~ log + File.unlink(name) + return log + end end end end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 3bb2692b43cb4d..30f30df62e32cc 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -1,5 +1,20 @@ # frozen_string_literal: true +# Enable deprecation warnings for test-all, so deprecated methods/constants/functions are dealt with early. +Warning[:deprecated] = true + +if ENV['BACKTRACE_FOR_DEPRECATION_WARNINGS'] + Warning.extend Module.new { + def warn(message, category: nil, **kwargs) + if category == :deprecated and $stderr.respond_to?(:puts) + $stderr.puts nil, message, caller, nil + else + super + end + end + } +end + require_relative '../envutil' require_relative '../colorize' require_relative '../leakchecker' @@ -9,42 +24,6 @@ # See Test::Unit module Test - class << self - ## - # Filter object for backtraces. - - attr_accessor :backtrace_filter - end - - class BacktraceFilter # :nodoc: - def filter bt - return ["No backtrace"] unless bt - - new_bt = [] - pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] - - unless $DEBUG then - bt.each do |line| - break if pattern.match?(line) - new_bt << line - end - - new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? - new_bt = bt.dup if new_bt.empty? - else - new_bt = bt.dup - end - - new_bt - end - end - - self.backtrace_filter = BacktraceFilter.new - - def self.filter_backtrace bt # :nodoc: - backtrace_filter.filter bt - end - ## # Test::Unit is an implementation of the xUnit testing framework for Ruby. module Unit @@ -58,6 +37,26 @@ class AssertionFailedError < Exception; end class PendedError < AssertionFailedError; end + class << self + ## + # Extract the location where the last assertion method was + # called. Returns "" if _e_ does not have backtrace, or + # an empty string if no assertion method location was found. + + def location e + last_before_assertion = nil + + return '' unless e&.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /:in \W(?:.*\#)?(?:assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + return "" unless last_before_assertion + /:in / =~ last_before_assertion ? $` : last_before_assertion + end + end + module Order class NoSort def initialize(seed) @@ -74,17 +73,7 @@ def group(list) end end - module JITFirst - def group(list) - # JIT first - jit, others = list.partition {|e| /test_jit/ =~ e} - jit + others - end - end - class Alpha < NoSort - include JITFirst - def sort_by_name(list) list.sort_by(&:name) end @@ -97,8 +86,6 @@ def sort_by_string(list) # shuffle test suites based on CRC32 of their names Shuffle = Struct.new(:seed, :salt) do - include JITFirst - def initialize(seed) self.class::CRC_TBL ||= (0..255).map {|i| (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } @@ -116,6 +103,10 @@ def sort_by_string(list) list.sort_by {|e| randomize_key(e)} end + def group(list) + list + end + private def crc32(str, crc32 = 0xffffffff) @@ -199,6 +190,7 @@ def process_args(args = []) @help = "\n" + orig_args.map { |s| " " + (s =~ /[\s|&<>$()]/ ? s.inspect : s) }.join("\n") + @options = options end @@ -270,10 +262,16 @@ def non_options(files, options) @jobserver = nil makeflags = ENV.delete("MAKEFLAGS") if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(\d+),(\d+)/ =~ makeflags + /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags begin - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end rescue r.close if r nil @@ -281,10 +279,10 @@ def non_options(files, options) r.close_on_exec = true w.close_on_exec = true @jobserver = [r, w] - options[:parallel] ||= 1 + options[:parallel] ||= 256 # number of tokens to acquire first end end - @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 180) + @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) super end @@ -324,7 +322,8 @@ def setup_options(opts, options) options[:retry] = false end - opts.on '--ruby VAL', "Path to ruby which is used at -j option" do |a| + opts.on '--ruby VAL', "Path to ruby which is used at -j option", + "Also used as EnvUtil.rubybin by some assertion methods" do |a| options[:ruby] = a.split(/ /).reject(&:empty?) end @@ -407,16 +406,18 @@ def close rescue IOError end - def quit + def quit(reason = :normal) return if @io.closed? @quit_called = true - @io.puts "quit" + @io.puts "quit #{reason}" rescue Errno::EPIPE => e warn "#{@pid}:#{@status.to_s.ljust(7)}:#{@file}: #{e.message}" end def kill - Process.kill(:KILL, @pid) + signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV + Process.kill(signal, @pid) + warn "worker #{to_s} does not respond; #{signal} is sent" rescue Errno::ESRCH end @@ -470,8 +471,8 @@ def after_worker_down(worker, e=nil, c=false) real_file = worker.real_file and warn "running file: #{real_file}" @need_quit = true warn "" - warn "Some worker was crashed. It seems ruby interpreter's bug" - warn "or, a bug of test/unit/parallel.rb. try again without -j" + warn "A test worker crashed. It might be an interpreter bug or" + warn "a bug in test/unit/parallel.rb. Try again without the -j" warn "option." warn "" if File.exist?('core') @@ -534,15 +535,15 @@ def quit_workers(&cond) @workers.reject! do |worker| next unless cond&.call(worker) begin - Timeout.timeout(1) do - worker.quit + Timeout.timeout(5) do + worker.quit(cond ? :timeout : :normal) end rescue Errno::EPIPE rescue Timeout::Error end closed&.push worker begin - Timeout.timeout(0.2) do + Timeout.timeout(1) do worker.close end rescue Timeout::Error @@ -555,7 +556,7 @@ def quit_workers(&cond) return if (closed ||= @workers).empty? pids = closed.map(&:pid) begin - Timeout.timeout(0.2 * closed.size) do + Timeout.timeout(1 * closed.size) do Process.waitall end rescue Timeout::Error @@ -674,14 +675,22 @@ def _run_parallel suites, type, result @ios = [] # Array of worker IOs @job_tokens = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver begin - [@tasks.size, @options[:parallel]].min.times {launch_worker} - while true - timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0) + @worker_timeout, 1].max + newjobs = [@tasks.size, @options[:parallel]].min - @workers.size + if newjobs > 0 + if @jobserver + t = @jobserver[0].read_nonblock(newjobs, exception: false) + @job_tokens << t if String === t + newjobs = @job_tokens.size + 1 - @workers.size + end + newjobs.times {launch_worker} + end + + timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0), 0].max + @worker_timeout if !(_io = IO.select(@ios, nil, nil, timeout)) timeout = Time.now - @worker_timeout - quit_workers {|w| w.response_at < timeout}&.map {|w| + quit_workers {|w| w.response_at&.<(timeout) }&.map {|w| rep << {file: w.real_file, result: nil, testcase: w.current[0], error: w.current} } elsif _io.first.any? {|io| @@ -691,15 +700,9 @@ def _run_parallel suites, type, result } break end - break if @tasks.empty? and @workers.empty? - if @jobserver and @job_tokens and !@tasks.empty? and - ((newjobs = [@tasks.size, @options[:parallel]].min) > @workers.size or - !@workers.any? {|x| x.status == :ready}) - t = @jobserver[0].read_nonblock(newjobs, exception: false) - if String === t - @job_tokens << t - t.size.times {launch_worker} - end + if @tasks.empty? + break if @workers.empty? + next # wait for all workers to finish end end rescue Interrupt => ex @@ -707,7 +710,7 @@ def _run_parallel suites, type, result return result ensure if file = @options[:timetable_data] - open(file, 'w'){|f| + File.open(file, 'w'){|f| @records.each{|(worker, suite), (st, ed)| f.puts '[' + [worker.dump, suite.dump, st.to_f * 1_000, ed.to_f * 1_000].join(", ") + '],' } @@ -735,14 +738,34 @@ def _run_parallel suites, type, result del_status_line or puts error, suites = suites.partition {|r| r[:error]} unless suites.empty? - puts "\n""Retrying..." + puts "\n" + @failed_output.puts "Failed tests:" + suites.each {|r| + r[:report].each {|c, m, e| + @failed_output.puts "#{c}##{m}: #{e&.class}: #{e&.message&.slice(/\A.*/)}" + } + } + @failed_output.puts "\n" + puts "Retrying..." @verbose = options[:verbose] suites.map! {|r| ::Object.const_get(r[:testcase])} _run_suites(suites, type) end unless error.empty? puts "\n""Retrying hung up testcases..." - error.map! {|r| ::Object.const_get(r[:testcase])} + error = error.map do |r| + begin + ::Object.const_get(r[:testcase]) + rescue NameError + # testcase doesn't specify the correct case, so show `r` for information + require 'pp' + + $stderr.puts "Retrying is failed because the file and testcase is not consistent:" + PP.pp r, $stderr + @errors += 1 + nil + end + end.compact verbose = @verbose job_status = options[:job_status] options[:verbose] = @verbose = true @@ -759,7 +782,7 @@ def _run_parallel suites, type, result unless rep.empty? rep.each do |r| if r[:error] - puke(*r[:error], Timeout::Error) + puke(*r[:error], Timeout::Error.new) next end r[:report]&.each do |f| @@ -779,6 +802,7 @@ def _run_parallel suites, type, result warn "" @warnings.uniq! {|w| w[1].message} @warnings.each do |w| + @errors += 1 warn "#{w[0]}: #{w[1].message} (#{w[1].class})" end warn "" @@ -848,7 +872,7 @@ def update_list(list, rec, max) end end - def record(suite, method, assertions, time, error) + def record(suite, method, assertions, time, error, source_location = nil) if @options.values_at(:longest, :most_asserted).any? @tops ||= {} rec = [suite.name, method, assertions, time, error] @@ -946,7 +970,7 @@ def output end def _prepare_run(suites, type) - options[:job_status] ||= :replace if @tty && !@verbose + options[:job_status] ||= @tty ? :replace : :normal unless @verbose case options[:color] when :always color = true @@ -962,11 +986,14 @@ def _prepare_run(suites, type) @output = Output.new(self) unless @options[:testing] filter = options[:filter] type = "#{type}_methods" - total = if filter - suites.inject(0) {|n, suite| n + suite.send(type).grep(filter).size} - else - suites.inject(0) {|n, suite| n + suite.send(type).size} - end + total = suites.sum {|suite| + methods = suite.send(type) + if filter + methods.count {|method| filter === "#{suite}##{method}"} + else + methods.size + end + } @test_count = 0 @total_tests = total.to_s(10) end @@ -1004,7 +1031,7 @@ def failed(s) end first, msg = msg.split(/$/, 2) first = sprintf("%3d) %s", @report_count += 1, first) - $stdout.print(sep, @colorize.decorate(first, color), msg, "\n") + @failed_output.print(sep, @colorize.decorate(first, color), msg, "\n") sep = nil end report.clear @@ -1060,7 +1087,7 @@ def print(s) runner.add_status(" = #$1") when /\A\.+\z/ runner.succeed - when /\A\.*[EFS][EFS.]*\z/ + when /\A\.*[EFST][EFST.]*\z/ runner.failed(s) else $stdout.print(s) @@ -1165,6 +1192,28 @@ def non_options(files, options) end end + module OutputOption # :nodoc: all + def setup_options(parser, options) + super + parser.separator "output options:" + + options[:failed_output] = $stdout + parser.on '--stderr-on-failure', 'Use stderr to print failure messages' do + options[:failed_output] = $stderr + end + parser.on '--stdout-on-failure', 'Use stdout to print failure messages', '(default)' do + options[:failed_output] = $stdout + end + end + + def process_args(args = []) + return @options if @options + options = super + @failed_output = options[:failed_output] + options + end + end + module GCOption # :nodoc: all def setup_options(parser, options) super @@ -1226,8 +1275,13 @@ def non_options(files, options) puts "#{f}: #{$!}" end } + @load_failed = errors.size.nonzero? result end + + def run(*) + super or @load_failed + end end module RepeatOption # :nodoc: all @@ -1329,6 +1383,95 @@ def non_options(files, options) end end + module LaunchableOption + module Nothing + private + def setup_options(opts, options) + super + opts.define_tail 'Launchable options:' + # This is expected to be called by Test::Unit::Worker. + opts.on_tail '--launchable-test-reports=PATH', String, 'Do nothing' + end + end + + def record(suite, method, assertions, time, error, source_location = nil) + if writer = @options[:launchable_test_reports] + if loc = (source_location || suite.instance_method(method).source_location) + path, lineno = loc + # Launchable JSON schema is defined at + # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code. + e = case error + when nil + status = 'TEST_PASSED' + nil + when Test::Unit::PendedError + status = 'TEST_SKIPPED' + "Skipped:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Test::Unit::AssertionFailedError + status = 'TEST_FAILED' + "Failure:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Timeout::Error + status = 'TEST_FAILED' + "Timeout:\n#{suite.name}##{method}\n" + else + status = 'TEST_FAILED' + bt = Test::filter_backtrace(error.backtrace).join "\n " + "Error:\n#{suite.name}##{method}:\n#{error.class}: #{error.message.b}\n #{bt}\n" + end + repo_path = File.expand_path("#{__dir__}/../../../") + relative_path = path.delete_prefix("#{repo_path}/") + # The test path is a URL-encoded representation. + # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18 + test_path = {file: relative_path, class: suite.name, testcase: method}.map{|key, val| + "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}" + }.join('#') + end + end + super + ensure + if writer && test_path && status + # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified. + # In such cases, we proceed to execute the operation here. + writer.write_object( + { + testPath: test_path, + status: status, + duration: time, + createdAt: Time.now.to_s, + stderr: e, + stdout: nil, + data: { + lineNumber: lineno + } + } + ) + end + end + + private + def setup_options(opts, options) + super + opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path| + require_relative '../launchable' + options[:launchable_test_reports] = writer = Launchable::JsonStreamWriter.new(path) + writer.write_array('testCases') + main_pid = Process.pid + at_exit { + # This block is executed when the fork block in a test is completed. + # Therefore, we need to verify whether all tests have been completed. + stack = caller + if stack.size == 0 && main_pid == Process.pid && $!.is_a?(SystemExit) + writer.close + end + } + end + + def encode_test_path_component component + component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26') + end + end + end + class Runner # :nodoc: all attr_accessor :report, :failures, :errors, :skips # :nodoc: @@ -1518,7 +1661,7 @@ def _run_suite suite, type _start_method(inst) inst._assertions = 0 - print "#{suite}##{method} = " if @verbose + print "#{suite}##{method.inspect.sub(/\A:/, '')} = " if @verbose start_time = Time.now if @verbose result = @@ -1533,9 +1676,7 @@ def _run_suite suite, type puts if @verbose $stdout.flush - unless defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # compiler process is wrongly considered as leak - leakchecker.check("#{inst.class}\##{inst.__name__}") - end + leakchecker.check("#{inst.class}\##{inst.__name__}") _end_method(inst) @@ -1566,19 +1707,11 @@ def _end_method(inst) # failure or error in teardown, it will be sent again with the # error or failure. - def record suite, method, assertions, time, error + def record suite, method, assertions, time, error, source_location = nil end def location e # :nodoc: - last_before_assertion = "" - - return '' unless e.backtrace # SystemStackError can return nil. - - e.backtrace.reverse_each do |s| - break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ - last_before_assertion = s - end - last_before_assertion.sub(/:in .*$/, '') + Test::Unit.location e end ## @@ -1624,7 +1757,7 @@ def _run args = [] break unless report.empty? end - return failures + errors if self.test_count > 0 # or return nil... + return (failures + errors).nonzero? # or return nil... rescue Interrupt abort 'Interrupted' end @@ -1650,12 +1783,14 @@ def status io = self.output prepend Test::Unit::Statistics prepend Test::Unit::Skipping prepend Test::Unit::GlobOption + prepend Test::Unit::OutputOption prepend Test::Unit::RepeatOption prepend Test::Unit::LoadPathOption prepend Test::Unit::GCOption prepend Test::Unit::ExcludesOption prepend Test::Unit::TimeoutOption prepend Test::Unit::RunCount + prepend Test::Unit::LaunchableOption::Nothing ## # Begins the full test run. Delegates to +runner+'s #_run method. @@ -1691,6 +1826,9 @@ def puke klass, meth, e when Test::Unit::AssertionFailedError then @failures += 1 "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Timeout::Error + @errors += 1 + "Timeout:\n#{klass}##{meth}\n" else @errors += 1 bt = Test::filter_backtrace(e.backtrace).join "\n " @@ -1709,6 +1847,7 @@ def puke klass, meth, e class AutoRunner # :nodoc: all class Runner < Test::Unit::Runner include Test::Unit::RequireFiles + include Test::Unit::LaunchableOption end attr_accessor :to_run, :options diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index dcf3e6fcb9add5..fe3351107ec407 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -522,18 +522,16 @@ def refute_same exp, act, msg = nil # Skips the current test. Gets listed at the end of the run but # doesn't cause a failure exit code. - def pend msg = nil, bt = caller + def pend msg = nil, bt = caller, &_ msg ||= "Skipped, no message given" @skip = true raise Test::Unit::PendedError, msg, bt end alias omit pend - # TODO: Removed this and enabled to raise NoMethodError with skip - alias skip pend - # def skip(msg = nil, bt = caller) - # raise NoMethodError, "use omit or pend", caller - # end + def skip(msg = nil, bt = caller) + raise NoMethodError, "use omit or pend", caller + end ## # Was this testcase skipped? Meant for #teardown. @@ -565,10 +563,6 @@ def assert_block(*msgs) assert yield, *msgs end - def assert_raises(*exp, &b) - raise NoMethodError, "use assert_raise", caller - end - # :call-seq: # assert_nothing_thrown( failure_message = nil, &block ) # @@ -774,7 +768,14 @@ def assert_syntax_error(code, error, *args, **opt) e = assert_raise(SyntaxError, mesg) do syntax_check(src, fname, line) end - assert_match(error, e.message, mesg) + + # Prism adds ANSI escape sequences to syntax error messages to + # colorize and format them. We strip them out here to make them easier + # to match against in tests. + message = e.message + message.gsub!(/\e\[.*?m/, "") + + assert_match(error, message, mesg) e end end diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb index b3a8957f26e21b..188a0d1a191a2b 100644 --- a/tool/lib/test/unit/parallel.rb +++ b/tool/lib/test/unit/parallel.rb @@ -1,12 +1,6 @@ # frozen_string_literal: true -$LOAD_PATH.unshift "#{__dir__}/../.." -require_relative '../../test/unit' -require_relative '../../profile_test_all' if ENV.key?('RUBY_TEST_ALL_PROFILE') -require_relative '../../tracepointchecker' -require_relative '../../zombie_hunter' -require_relative '../../iseq_loader_checker' -require_relative '../../gc_checker' +require_relative "../../../test/init" module Test module Unit @@ -133,7 +127,18 @@ def run(args = []) # :nodoc: else _report "ready" end - when /^quit$/ + when /^quit (.+?)$/, "quit" + if $1 == "timeout" + err = ["", "!!! worker #{$$} killed due to timeout:"] + Thread.list.each do |th| + err << "#{ th.inspect }:" + th.backtrace.each do |s| + err << " #{ s }" + end + end + err << "" + STDERR.puts err.join("\n") + end _report "bye" exit end @@ -186,7 +191,7 @@ def record(suite, method, assertions, time, error) # :nodoc: else error = ProxyError.new(error) end - _report "record", Marshal.dump([suite.name, method, assertions, time, error]) + _report "record", Marshal.dump([suite.name, method, assertions, time, error, suite.instance_method(method).source_location]) super end end @@ -208,5 +213,9 @@ def self.on_parallel_worker? end end require 'rubygems' + begin + require 'rake' + rescue LoadError + end Test::Unit::Worker.new.run(ARGV) end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb index 44d9ba7fdbf978..51ffff37ebc376 100644 --- a/tool/lib/test/unit/testcase.rb +++ b/tool/lib/test/unit/testcase.rb @@ -137,6 +137,9 @@ class TestCase attr_reader :__name__ # :nodoc: + # Method name of this test. + alias method_name __name__ + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, Interrupt, SystemExit] # :nodoc: @@ -144,8 +147,7 @@ class TestCase # Runs the tests reporting the status to +runner+ def run runner - @options = runner.options - + @__runner_options__ = runner.options trap "INFO" do runner.report.each_with_index do |msg, i| warn "\n%3d) %s" % [i + 1, msg] @@ -161,7 +163,7 @@ def run runner result = "" begin - @passed = nil + @__passed__ = nil self.before_setup self.setup self.after_setup @@ -169,11 +171,11 @@ def run runner result = "." unless io? time = Time.now - start_time runner.record self.class, self.__name__, self._assertions, time, nil - @passed = true + @__passed__ = true rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - @passed = Test::Unit::PendedError === e + @__passed__ = Test::Unit::PendedError === e time = Time.now - start_time runner.record self.class, self.__name__, self._assertions, time, e result = runner.puke self.class, self.__name__, e @@ -184,7 +186,7 @@ def run runner rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - @passed = false + @__passed__ = false runner.record self.class, self.__name__, self._assertions, time, e result = runner.puke self.class, self.__name__, e end @@ -206,12 +208,12 @@ def run_test(name) def initialize name # :nodoc: @__name__ = name @__io__ = nil - @passed = nil - @@current = self # FIX: make thread local + @__passed__ = nil + @@__current__ = self # FIX: make thread local end def self.current # :nodoc: - @@current # FIX: make thread local + @@__current__ # FIX: make thread local end ## @@ -263,7 +265,7 @@ def self.test_methods # :nodoc: # Returns true if the test passed. def passed? - @passed + @__passed__ end ##