From 7311bf9536e6201558ff5e29eff2a6bf39be3d6e Mon Sep 17 00:00:00 2001 From: Nathan Broadbent Date: Tue, 3 Nov 2026 11:53:47 +1300 Subject: [PATCH] Explain what to do when a worker process dies When a forked worker exits without reporting back (killed for running out of memory, or crashed in a native extension), Parallel raises `Parallel::DeadWorker` and Tapioca printed only its backtrace. Raise a `Tapioca::Error` instead, saying what usually kills a worker, where to look for a crash report, and that `--workers=1` runs the work in one process. On macOS with the pg gem loaded and `PGGSSENCMODE` unset, also suggest turning off libpq's GSSAPI encryption, which is the fix maintainers have pointed people to in #1555, #2241 and #2390. See #1566. --- lib/tapioca/executor.rb | 35 +++++++++++++++++++++++++++++++++++ spec/executor_spec.rb | 25 +++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/lib/tapioca/executor.rb b/lib/tapioca/executor.rb index 13243ad38..ccef7f14c 100644 --- a/lib/tapioca/executor.rb +++ b/lib/tapioca/executor.rb @@ -21,10 +21,45 @@ def run_in_parallel(&block) # To have the parallel gem run jobs in the parent process, you must pass 0 as the number of processes number_of_processes = @number_of_workers == 1 ? 0 : @number_of_workers Parallel.map(@queue, { in_processes: number_of_processes }, &block) + rescue Parallel::DeadWorker + raise Tapioca::Error, dead_worker_message end private + # A worker that raises sends its exception back to the parent, which re-raises it. `Parallel::DeadWorker` means one + # exited without doing that: killed (often for running out of memory) or crashed in a native extension, which + # prints its own crash report to stderr. + #: -> String + def dead_worker_message + message = <<~MESSAGE + A worker process exited before it finished. Tapioca forks #{@number_of_workers} worker processes, and a worker usually + dies because the system ran out of memory, or because a native extension crashed in the forked process + (look above for a crash report, such as `[BUG] Segmentation fault`). + + To do the work in a single process, which shows the error if there is one, run the command again with + `--workers=1`. + MESSAGE + + if pg_gssapi_crash_possible? + message += <<~MESSAGE + + On macOS, a crash in the pg gem while it connects to PostgreSQL comes from libpq trying GSSAPI + encryption. Turn it off with `PGGSSENCMODE=disable` in the environment, or with + `gssencmode: disable` in config/database.yml. + MESSAGE + end + + message + end + + # libpq tries GSSAPI encryption on each new connection by default, and on macOS the system's Kerberos library + # crashes when it's used in a process forked after it was loaded. + #: -> bool + def pg_gssapi_crash_possible? + !!defined?(::PG) && RUBY_PLATFORM.include?("darwin") && ENV["PGGSSENCMODE"].nil? + end + #: -> Integer def max_processors env_max_processors = ENV["PARALLEL_PROCESSOR_COUNT"].to_i diff --git a/spec/executor_spec.rb b/spec/executor_spec.rb index 82925d9c5..3a9d20be1 100644 --- a/spec/executor_spec.rb +++ b/spec/executor_spec.rb @@ -61,6 +61,31 @@ class ExecutorSpec < Minitest::Spec end end + it "explains how to find out why a worker died" do + executor = Executor.new(@queue, number_of_workers: 2) + + error = assert_raises(Tapioca::Error) do + executor.run_in_parallel { Process.kill(:KILL, Process.pid) } + end + + assert_includes(error.message, "A worker process exited before it finished. Tapioca forks 2 worker processes") + assert_includes(error.message, "run the command again with\n`--workers=1`") + end + + it "suggests turning off GSSAPI encryption when a worker may have crashed in libpq on macOS" do + [true, false].each do |possible| + executor = Executor.new(@queue, number_of_workers: 2) + + error = T.unsafe(executor).stub(:pg_gssapi_crash_possible?, possible) do + assert_raises(Tapioca::Error) do + executor.run_in_parallel { Process.kill(:KILL, Process.pid) } + end + end + + assert_equal(possible, error.message.include?("`PGGSSENCMODE=disable`")) + end + end + #: (Integer expected_count) -> ^(untyped _arg1, untyped _arg2) -> Array[Integer] def assert_parallel_count(expected_count) ->(_, options) {