diff --git a/README.md b/README.md index f77c4d5a1..abd6f04ce 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,23 @@ Here's an overview of the different options: It is recommended to set this value less than or equal to the queue database's connection pool size minus 2, as each worker thread uses one connection, and two additional connections are reserved for polling and heartbeat. - `processes`: this is the number of worker processes that will be forked by the supervisor with the settings given. By default, this is `1`, just a single process. This setting is useful if you want to dedicate more than one CPU core to a queue or queues with the same configuration. Only workers have this setting. **Note**: this option will be ignored if [running in `async` mode](#fork-vs-async-mode). - `concurrency_maintenance`: whether the dispatcher will perform the concurrency maintenance work. This is `true` by default, and it's useful if you don't use any [concurrency controls](#concurrency-controls) and want to disable it or if you run multiple dispatchers and want some of them to just dispatch jobs without doing anything else. +- `name`: an optional, human-readable name for the process. By default, processes are named with their kind and a random hex string, such as `worker-5a3b1c0d9e8f7a6b5c4d`, which makes it hard to tell them apart in `ps` output or in the processes table of the dashboard. Setting a name replaces that random name, and it's also reflected in the process title, for example `solid-queue-worker[pipeline-1](1.3.1): waiting for jobs in background`. When a worker also sets `processes` to a value greater than `1`, each of the forked processes gets a numbered suffix, so `name: pipeline` with `processes: 3` yields `pipeline-1`, `pipeline-2` and `pipeline-3`. With a single process, the name is used as-is. For example: + + ```yml + production: + dispatchers: + - polling_interval: 1 + batch_size: 500 + name: main-dispatcher + workers: + - queues: background + threads: 3 + processes: 3 + name: pipeline + - queues: default + threads: 1 + name: housekeeping + ``` ### Optional scheduler configuration diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index e63a000ca..9c1ad95a4 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -131,7 +131,17 @@ def workers 1 end - processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) } + base_name = worker_options[:name] + + processes.times.map do |index| + options = worker_options.with_defaults(WORKER_DEFAULTS) + + if base_name + options = options.merge(name: processes > 1 ? "#{base_name}-#{index + 1}" : base_name) + end + + Process.new(:worker, options) + end end end diff --git a/lib/solid_queue/processes/base.rb b/lib/solid_queue/processes/base.rb index 8eec3cebf..6e13f5194 100644 --- a/lib/solid_queue/processes/base.rb +++ b/lib/solid_queue/processes/base.rb @@ -9,11 +9,16 @@ class Base attr_reader :name - def initialize(*) - @name = generate_name + def initialize(name: nil, **) + @name = name || generate_name + @custom_name = name.present? @stopped = false end + def custom_name? + @custom_name + end + def kind self.class.name.demodulize end diff --git a/lib/solid_queue/processes/procline.rb b/lib/solid_queue/processes/procline.rb index f9caea41f..1abd1740e 100644 --- a/lib/solid_queue/processes/procline.rb +++ b/lib/solid_queue/processes/procline.rb @@ -4,8 +4,11 @@ module SolidQueue::Processes module Procline # Sets the procline ($0) # solid-queue-supervisor(0.1.0): + # solid-queue-worker[pipeline-1](0.1.0): def procline(string) - $0 = "solid-queue-#{self.class.name.demodulize.underscore.dasherize}(#{SolidQueue::VERSION}): #{string}" + process_kind = self.class.name.demodulize.underscore.dasherize + label = custom_name? ? "#{process_kind}[#{name}]" : process_kind + $0 = "solid-queue-#{label}(#{SolidQueue::VERSION}): #{string}" end end end diff --git a/lib/solid_queue/supervisor.rb b/lib/solid_queue/supervisor.rb index ae17ec95c..1ddf15186 100644 --- a/lib/solid_queue/supervisor.rb +++ b/lib/solid_queue/supervisor.rb @@ -29,7 +29,7 @@ def initialize(configuration) @configured_processes = {} @process_instances = {} - super + super() end def start diff --git a/test/dummy/config/named_workers_configuration.yml b/test/dummy/config/named_workers_configuration.yml new file mode 100644 index 000000000..0bb2f1597 --- /dev/null +++ b/test/dummy/config/named_workers_configuration.yml @@ -0,0 +1,19 @@ +default: &default + workers: + - queues: background + threads: 3 + processes: 3 + name: pipeline + - queues: default + threads: 1 + name: housekeeping + dispatchers: + - polling_interval: 1 + batch_size: 500 + name: main-dispatcher + +development: + <<: *default + +test: + <<: *default diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index 34f69658b..257c6b213 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -134,6 +134,58 @@ class ConfigurationTest < ActiveSupport::TestCase end end + test "named workers with multiple processes get sequential names" do + configuration = SolidQueue::Configuration.new( + workers: [ { queues: "background", threads: 3, processes: 3, name: "pipeline" } ], + skip_recurring: true + ) + + assert_processes configuration, :worker, 3, + queues: "background", + name: [ "pipeline-1", "pipeline-2", "pipeline-3" ] + end + + test "named worker with single process gets name without suffix" do + configuration = SolidQueue::Configuration.new( + workers: [ { queues: "default", threads: 1, name: "housekeeping" } ], + skip_recurring: true + ) + + assert_processes configuration, :worker, 1, name: "housekeeping" + end + + test "named dispatcher gets name passed through" do + configuration = SolidQueue::Configuration.new( + dispatchers: [ { batch_size: 500, name: "main-dispatcher" } ], + skip_recurring: true + ) + + assert_processes configuration, :dispatcher, 1, name: "main-dispatcher" + end + + test "workers without name still get no name in config attributes" do + configuration = SolidQueue::Configuration.new( + workers: [ { queues: "background", processes: 2 } ], + skip_recurring: true + ) + + assert_processes configuration, :worker, 2 + configuration.configured_processes.select { |p| p.kind == :worker }.each do |p| + assert_nil p.attributes[:name] + end + end + + test "read named configuration from provided file" do + configuration = SolidQueue::Configuration.new( + config_file: config_file_path(:named_workers_configuration), + skip_recurring: true + ) + + assert_processes configuration, :worker, 4, + name: [ "pipeline-1", "pipeline-2", "pipeline-3", "housekeeping" ] + assert_processes configuration, :dispatcher, 1, name: "main-dispatcher" + end + test "validate configuration" do # Valid and invalid recurring tasks configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_invalid)) diff --git a/test/unit/dispatcher_test.rb b/test/unit/dispatcher_test.rb index 7df0591f5..25d89bbfb 100644 --- a/test/unit/dispatcher_test.rb +++ b/test/unit/dispatcher_test.rb @@ -23,6 +23,27 @@ class DispatcherTest < ActiveSupport::TestCase assert_metadata process, { polling_interval: 0.1, batch_size: 10, concurrency_maintenance_interval: 600 } end + test "dispatcher with custom name registers with that name" do + dispatcher = SolidQueue::Dispatcher.new(polling_interval: 0.1, batch_size: 10, name: "main-dispatcher") + assert_equal "main-dispatcher", dispatcher.name + assert dispatcher.custom_name? + + dispatcher.start + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + assert_equal "Dispatcher", process.kind + assert_equal "main-dispatcher", process.name + ensure + dispatcher&.stop + wait_for_registered_processes(0, timeout: 1.second) + end + + test "dispatcher without custom name generates a random name" do + assert_match /\Adispatcher-[0-9a-f]{20}\z/, @dispatcher.name + assert_not @dispatcher.custom_name? + end + test "concurrency maintenance is optional" do no_concurrency_maintenance_dispatcher = SolidQueue::Dispatcher.new(polling_interval: 0.1, batch_size: 10, concurrency_maintenance: false) no_concurrency_maintenance_dispatcher.start diff --git a/test/unit/worker_test.rb b/test/unit/worker_test.rb index 3d692404b..23a7df967 100644 --- a/test/unit/worker_test.rb +++ b/test/unit/worker_test.rb @@ -22,6 +22,27 @@ class WorkerTest < ActiveSupport::TestCase assert_metadata process, { queues: "background", polling_interval: 0.2, thread_pool_size: 3 } end + test "worker with custom name registers with that name" do + worker = SolidQueue::Worker.new(queues: "background", threads: 3, polling_interval: 0.2, name: "pipeline-1") + assert_equal "pipeline-1", worker.name + assert worker.custom_name? + + worker.start + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + assert_equal "Worker", process.kind + assert_equal "pipeline-1", process.name + ensure + worker&.stop + wait_for_registered_processes(0, timeout: 1.second) + end + + test "worker without custom name generates a random name" do + assert_match /\Aworker-[0-9a-f]{20}\z/, @worker.name + assert_not @worker.custom_name? + end + test "errors on polling are passed to on_thread_error and re-raised" do errors = Concurrent::Array.new