Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions lib/tapioca/executor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,45 @@ def run_in_parallel(&block)
# To have the parallel gem run jobs in the parent process, you must pass 0 as the number of processes
number_of_processes = @number_of_workers == 1 ? 0 : @number_of_workers
Parallel.map(@queue, { in_processes: number_of_processes }, &block)
rescue Parallel::DeadWorker
raise Tapioca::Error, dead_worker_message
end

private

# A worker that raises sends its exception back to the parent, which re-raises it. `Parallel::DeadWorker` means one
# exited without doing that: killed (often for running out of memory) or crashed in a native extension, which
# prints its own crash report to stderr.
#: -> String
def dead_worker_message
message = <<~MESSAGE
A worker process exited before it finished. Tapioca forks #{@number_of_workers} worker processes, and a worker usually
dies because the system ran out of memory, or because a native extension crashed in the forked process
(look above for a crash report, such as `[BUG] Segmentation fault`).

To do the work in a single process, which shows the error if there is one, run the command again with
`--workers=1`.
MESSAGE

if pg_gssapi_crash_possible?
message += <<~MESSAGE

On macOS, a crash in the pg gem while it connects to PostgreSQL comes from libpq trying GSSAPI
encryption. Turn it off with `PGGSSENCMODE=disable` in the environment, or with
`gssencmode: disable` in config/database.yml.
MESSAGE
end

message
end

# libpq tries GSSAPI encryption on each new connection by default, and on macOS the system's Kerberos library
# crashes when it's used in a process forked after it was loaded.
#: -> bool
def pg_gssapi_crash_possible?
!!defined?(::PG) && RUBY_PLATFORM.include?("darwin") && ENV["PGGSSENCMODE"].nil?
end

#: -> Integer
def max_processors
env_max_processors = ENV["PARALLEL_PROCESSOR_COUNT"].to_i
Expand Down
25 changes: 25 additions & 0 deletions spec/executor_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,31 @@ class ExecutorSpec < Minitest::Spec
end
end

it "explains how to find out why a worker died" do
executor = Executor.new(@queue, number_of_workers: 2)

error = assert_raises(Tapioca::Error) do
executor.run_in_parallel { Process.kill(:KILL, Process.pid) }
end

assert_includes(error.message, "A worker process exited before it finished. Tapioca forks 2 worker processes")
assert_includes(error.message, "run the command again with\n`--workers=1`")
end

it "suggests turning off GSSAPI encryption when a worker may have crashed in libpq on macOS" do
[true, false].each do |possible|
executor = Executor.new(@queue, number_of_workers: 2)

error = T.unsafe(executor).stub(:pg_gssapi_crash_possible?, possible) do
assert_raises(Tapioca::Error) do
executor.run_in_parallel { Process.kill(:KILL, Process.pid) }
end
end

assert_equal(possible, error.message.include?("`PGGSSENCMODE=disable`"))
end
end

#: (Integer expected_count) -> ^(untyped _arg1, untyped _arg2) -> Array[Integer]
def assert_parallel_count(expected_count)
->(_, options) {
Expand Down
Loading