diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eef17fa5..95638773 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: - name: Setup Ruby and install gems uses: ruby/setup-ruby@v1 with: - ruby-version: 3.0 + ruby-version: 3.4 bundler-cache: true - name: Run rubocop run: | diff --git a/.rubocop.yml b/.rubocop.yml index 999ad5f5..75df1173 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,5 +6,4 @@ inherit_gem: { rubocop-rails-omakase: rubocop.yml } AllCops: TargetRubyVersion: 3.3 Exclude: - - "test/dummy/db/schema.rb" - - "test/dummy/db/queue_schema.rb" + - "**/*_schema.rb" diff --git a/Gemfile.lock b/Gemfile.lock index 36c358be..8d6371f1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - solid_queue (1.1.0) + solid_queue (1.1.1) activejob (>= 7.1) activerecord (>= 7.1) concurrent-ruby (>= 1.3.1) @@ -86,11 +86,11 @@ GEM mutex_m (0.3.0) mysql2 (0.5.6) nio4r (2.7.4) - nokogiri (1.16.7-arm64-darwin) + nokogiri (1.18.0-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-darwin) + nokogiri (1.18.0-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.7-x86_64-linux) + nokogiri (1.18.0-x86_64-linux-gnu) racc (~> 1.4) parallel (1.26.3) parser (3.3.6.0) diff --git a/README.md b/README.md index 1328635b..d4c68bed 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,31 @@ Besides regular job enqueuing and processing, Solid Queue supports delayed jobs, Solid Queue can be used with SQL databases such as MySQL, PostgreSQL or SQLite, and it leverages the `FOR UPDATE SKIP LOCKED` clause, if available, to avoid blocking and waiting on locks when polling jobs. It relies on Active Job for retries, discarding, error handling, serialization, or delays, and it's compatible with Ruby on Rails's multi-threading. +## Table of contents + +- [Installation](#installation) + - [Single database configuration](#single-database-configuration) + - [Incremental adoption](#incremental-adoption) + - [High performance requirements](#high-performance-requirements) +- [Configuration](#configuration) + - [Workers, dispatchers and scheduler](#workers-dispatchers-and-scheduler) + - [Queue order and priorities](#queue-order-and-priorities) + - [Queues specification and performance](#queues-specification-and-performance) + - [Threads, processes and signals](#threads-processes-and-signals) + - [Database configuration](#database-configuration) + - [Other configuration settings](#other-configuration-settings) +- [Lifecycle hooks](#lifecycle-hooks) +- [Errors when enqueuing](#errors-when-enqueuing) +- [Concurrency controls](#concurrency-controls) +- [Failed jobs and retries](#failed-jobs-and-retries) + - [Error reporting on jobs](#error-reporting-on-jobs) +- [Puma plugin](#puma-plugin) +- [Jobs and transactional integrity](#jobs-and-transactional-integrity) +- [Recurring tasks](#recurring-tasks) +- [Inspiration](#inspiration) +- [License](#license) + + ## Installation Solid Queue is configured by default in new Rails 8 applications. But if you're running an earlier version, you can add it manually following these steps: @@ -43,8 +68,6 @@ production: migrations_paths: db/queue_migrate ``` -Note: Calling `bin/rails solid_queue:install` will automatically add `config.solid_queue.connects_to = { database: { writing: :queue } }` to `config/environments/production.rb`, so no additional configuration is needed there (although you must make sure that you use the `queue` name in `database.yml` for this to match!). But if you want to use Solid Queue in a different environment (like staging or even development), you'll have to manually add that `config.solid_queue.connects_to` line to the respective environment file. And, as always, make sure that the name you're using for the database in `config/database.yml` matches the name you use in `config.solid_queue.connects_to`. - Then run `db:prepare` in production to ensure the database is created and the schema is loaded. Now you're ready to start processing jobs by running `bin/jobs` on the server that's doing the work. This will start processing jobs in all queues using the default configuration. See [below](#configuration) to learn more about configuring Solid Queue. @@ -53,6 +76,72 @@ For small projects, you can run Solid Queue on the same machine as your webserve **Note**: future changes to the schema will come in the form of regular migrations. +### Usage in development and other non-production environments + +Calling `bin/rails solid_queue:install` will automatically add `config.solid_queue.connects_to = { database: { writing: :queue } }` to `config/environments/production.rb`. In order to use Solid Queue in other environments (such as development or staging), you'll need to add a similar configuration(s). + +For example, if you're using SQLite in development, update `database.yml` as follows: + +```diff +development: + primary: + <<: *default + database: storage/development.sqlite3 ++ queue: ++ <<: *default ++ database: storage/development_queue.sqlite3 ++ migrations_paths: db/queue_migrate +``` + +Next, add the following to `development.rb` + +```ruby + # Use Solid Queue in Development. + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } +``` + +Once you've added this, run `db:prepare` to create the Solid Queue database and load the schema. + +Finally, in order for jobs to be processed, you'll need to have Solid Queue running. In Development, this can be done via [the Puma plugin](#puma-plugin) as well. In `puma.rb` update the following line: + +```ruby +# You can either set the env var, or check for development +plugin :solid_queue if ENV["SOLID_QUEUE_IN_PUMA"] || Rails.env.development? +``` + +You can also just use `bin/jobs`, but in this case you might want to [set a different logger for Solid Queue](#other-configuration-settings) because the default logger will log to `log/development.log` and you won't see anything when you run `bin/jobs`. For example: +```ruby +config.solid_queue.logger = ActiveSupport::Logger.new(STDOUT) +``` + +**Note about Action Cable**: If you use Action Cable (or anything dependent on Action Cable, such as Turbo Streams), you will also need to update it to use a database. + +In `config/cable.yml` + +```diff +development: +- adapter: async ++ adapter: solid_cable ++ connects_to: ++ database: ++ writing: cable ++ polling_interval: 0.1.seconds ++ message_retention: 1.day +``` + +In `config/database.yml` + +```diff +development: + primary: + <<: *default + database: storage/development.sqlite3 ++ cable: ++ <<: *default ++ database: storage/development_cable.sqlite3 ++ migrations_paths: db/cable_migrate +``` ### Single database configuration @@ -64,7 +153,7 @@ Running Solid Queue in a separate database is recommended, but it's also possibl You won't have multiple databases, so `database.yml` doesn't need to have primary and queue database. -## Incremental adoption +### Incremental adoption If you're planning to adopt Solid Queue incrementally by switching one job at the time, you can do so by leaving the `config.active_job.queue_adapter` set to your old backend, and then set the `queue_adapter` directly in the jobs you're moving: @@ -77,7 +166,7 @@ class MyJob < ApplicationJob end ``` -## High performance requirements +### High performance requirements Solid Queue was designed for the highest throughput when used with MySQL 8+ or PostgreSQL 9.5+, as they support `FOR UPDATE SKIP LOCKED`. You can use it with older versions, but in that case, you might run into lock waits if you run multiple workers for the same queue. You can also use it with SQLite on smaller applications. @@ -86,6 +175,7 @@ Solid Queue was designed for the highest throughput when used with MySQL 8+ or P ### Workers, dispatchers and scheduler We have several types of actors in Solid Queue: + - _Workers_ are in charge of picking jobs ready to run from queues and processing them. They work off the `solid_queue_ready_executions` table. - _Dispatchers_ are in charge of selecting jobs scheduled to run in the future that are due and _dispatching_ them, which is simply moving them from the `solid_queue_scheduled_executions` table over to the `solid_queue_ready_executions` table so that workers can pick them up. On top of that, they do some maintenance work related to [concurrency controls](#concurrency-controls). - The _scheduler_ manages [recurring tasks](#recurring-tasks), enqueuing jobs for them when they're due. @@ -99,7 +189,6 @@ By default, Solid Queue will try to find your configuration under `config/queue. bin/jobs -c config/calendar.yml ``` - This is what this configuration looks like: ```yml @@ -153,6 +242,7 @@ Here's an overview of the different options: Check the sections below on [how queue order behaves combined with priorities](#queue-order-and-priorities), and [how the way you specify the queues per worker might affect performance](#queues-specification-and-performance). - `threads`: this is the max size of the thread pool that each worker will have to run jobs. Each worker will fetch this number of jobs from their queue(s), at most and will post them to the thread pool to be run. By default, this is `3`. Only workers have this setting. +It is recommended to set this value less than or equal to the queue database's connection pool size minus 2, as each worker thread uses one connection, and two additional connections are reserved for polling and heartbeat. - `processes`: this is the number of worker processes that will be forked by the supervisor with the settings given. By default, this is `1`, just a single process. This setting is useful if you want to dedicate more than one CPU core to a queue or queues with the same configuration. Only workers have this setting. - `concurrency_maintenance`: whether the dispatcher will perform the concurrency maintenance work. This is `true` by default, and it's useful if you don't use any [concurrency controls](#concurrency-controls) and want to disable it or if you run multiple dispatchers and want some of them to just dispatch jobs without doing anything else. @@ -250,6 +340,32 @@ You can configure the database used by Solid Queue via the `config.solid_queue.c All the options available to Active Record for multiple databases can be used here. +### Other configuration settings + +_Note_: The settings in this section should be set in your `config/application.rb` or your environment config like this: `config.solid_queue.silence_polling = true` + +There are several settings that control how Solid Queue works that you can set as well: +- `logger`: the logger you want Solid Queue to use. Defaults to the app logger. +- `app_executor`: the [Rails executor](https://guides.rubyonrails.org/threading_and_code_execution.html#executor) used to wrap asynchronous operations, defaults to the app executor +- `on_thread_error`: custom lambda/Proc to call when there's an error within a Solid Queue thread that takes the exception raised as argument. Defaults to + + ```ruby + -> (exception) { Rails.error.report(exception, handled: false) } + ``` + + **This is not used for errors raised within a job execution**. Errors happening in jobs are handled by Active Job's `retry_on` or `discard_on`, and ultimately will result in [failed jobs](#failed-jobs-and-retries). This is for errors happening within Solid Queue itself. + +- `use_skip_locked`: whether to use `FOR UPDATE SKIP LOCKED` when performing locking reads. This will be automatically detected in the future, and for now, you'd only need to set this to `false` if your database doesn't support it. For MySQL, that'd be versions < 8, and for PostgreSQL, versions < 9.5. If you use SQLite, this has no effect, as writes are sequential. +- `process_heartbeat_interval`: the heartbeat interval that all processes will follow—defaults to 60 seconds. +- `process_alive_threshold`: how long to wait until a process is considered dead after its last heartbeat—defaults to 5 minutes. +- `shutdown_timeout`: time the supervisor will wait since it sent the `TERM` signal to its supervised processes before sending a `QUIT` version to them requesting immediate termination—defaults to 5 seconds. +- `silence_polling`: whether to silence Active Record logs emitted when polling for both workers and dispatchers—defaults to `true`. +- `supervisor_pidfile`: path to a pidfile that the supervisor will create when booting to prevent running more than one supervisor in the same host, or in case you want to use it for a health check. It's `nil` by default. +- `preserve_finished_jobs`: whether to keep finished jobs in the `solid_queue_jobs` table—defaults to `true`. +- `clear_finished_jobs_after`: period to keep finished jobs around, in case `preserve_finished_jobs` is true—defaults to 1 day. **Note:** Right now, there's no automatic cleanup of finished jobs. You'd need to do this by periodically invoking `SolidQueue::Job.clear_finished_in_batches`, which can be configured as [a recurring task](#recurring-tasks). +- `default_concurrency_control_period`: the value to be used as the default for the `duration` parameter in [concurrency controls](#concurrency-controls). It defaults to 3 minutes. + + ## Lifecycle hooks In Solid queue, you can hook into two different points in the supervisor's life: @@ -277,30 +393,6 @@ SolidQueue.on_stop { stop_metrics_server } These can be called several times to add multiple hooks, but it needs to happen before Solid Queue is started. An initializer would be a good place to do this. -### Other configuration settings - -_Note_: The settings in this section should be set in your `config/application.rb` or your environment config like this: `config.solid_queue.silence_polling = true` - -There are several settings that control how Solid Queue works that you can set as well: -- `logger`: the logger you want Solid Queue to use. Defaults to the app logger. -- `app_executor`: the [Rails executor](https://guides.rubyonrails.org/threading_and_code_execution.html#executor) used to wrap asynchronous operations, defaults to the app executor -- `on_thread_error`: custom lambda/Proc to call when there's an error within a Solid Queue thread that takes the exception raised as argument. Defaults to - - ```ruby - -> (exception) { Rails.error.report(exception, handled: false) } - ``` - - **This is not used for errors raised within a job execution**. Errors happening in jobs are handled by Active Job's `retry_on` or `discard_on`, and ultimately will result in [failed jobs](#failed-jobs-and-retries). This is for errors happening within Solid Queue itself. - -- `use_skip_locked`: whether to use `FOR UPDATE SKIP LOCKED` when performing locking reads. This will be automatically detected in the future, and for now, you'd only need to set this to `false` if your database doesn't support it. For MySQL, that'd be versions < 8, and for PostgreSQL, versions < 9.5. If you use SQLite, this has no effect, as writes are sequential. -- `process_heartbeat_interval`: the heartbeat interval that all processes will follow—defaults to 60 seconds. -- `process_alive_threshold`: how long to wait until a process is considered dead after its last heartbeat—defaults to 5 minutes. -- `shutdown_timeout`: time the supervisor will wait since it sent the `TERM` signal to its supervised processes before sending a `QUIT` version to them requesting immediate termination—defaults to 5 seconds. -- `silence_polling`: whether to silence Active Record logs emitted when polling for both workers and dispatchers—defaults to `true`. -- `supervisor_pidfile`: path to a pidfile that the supervisor will create when booting to prevent running more than one supervisor in the same host, or in case you want to use it for a health check. It's `nil` by default. -- `preserve_finished_jobs`: whether to keep finished jobs in the `solid_queue_jobs` table—defaults to `true`. -- `clear_finished_jobs_after`: period to keep finished jobs around, in case `preserve_finished_jobs` is true—defaults to 1 day. **Note:** Right now, there's no automatic cleanup of finished jobs. You'd need to do this by periodically invoking `SolidQueue::Job.clear_finished_in_batches`, but this will happen automatically in the near future. -- `default_concurrency_control_period`: the value to be used as the default for the `duration` parameter in [concurrency controls](#concurrency-controls). It defaults to 3 minutes. ## Errors when enqueuing @@ -412,6 +504,12 @@ plugin :solid_queue ``` to your `puma.rb` configuration. +If you're using Puma in development but you don't want to use Solid Queue in development, make sure you avoid the plugin being used, for example using an environment variable like this: +```ruby +plugin :solid_queue if ENV["SOLID_QUEUE_IN_PUMA"] +``` +that you set in production only. This is what Rails 8's default Puma config looks like. Otherwise, if you're using Puma in development but not Solid Queue, starting Puma would start also Solid Queue supervisor and it'll most likely fail because it won't be properly configured. + ## Jobs and transactional integrity :warning: Having your jobs in the same ACID-compliant database as your application data enables a powerful yet sharp tool: taking advantage of transactional integrity to ensure some action in your app is not committed unless your job is also committed and vice versa, and ensuring that your job won't be enqueued until the transaction within which you're enqueuing it is committed. This can be very powerful and useful, but it can also backfire if you base some of your logic on this behaviour, and in the future, you move to another active job backend, or if you simply move Solid Queue to its own database, and suddenly the behaviour changes under you. Because this can be quite tricky and many people shouldn't need to worry about it, by default Solid Queue is configured in a different database as the main app. @@ -477,9 +575,15 @@ MyJob.perform_later(42, status: "custom_status") - `priority`: a numeric priority value to be used when enqueuing the job. - Tasks are enqueued at their corresponding times by the scheduler, and each task schedules the next one. This is pretty much [inspired by what GoodJob does](https://github.com/bensheldon/good_job/blob/994ecff5323bf0337e10464841128fda100750e6/lib/good_job/cron_manager.rb). +For recurring tasks defined as a `command`, you can also change the job class that runs them as follows: +```ruby +Rails.application.config.after_initialize do # or to_prepare + SolidQueue::RecurringTask.default_job_class = MyRecurringCommandJob +end +``` + It's possible to run multiple schedulers with the same `recurring_tasks` configuration, for example, if you have multiple servers for redundancy, and you run the `scheduler` in more than one of them. To avoid enqueuing duplicate tasks at the same time, an entry in a new `solid_queue_recurring_executions` table is created in the same transaction as the job is enqueued. This table has a unique index on `task_key` and `run_at`, ensuring only one entry per task per time will be created. This only works if you have `preserve_finished_jobs` set to `true` (the default), and the guarantee applies as long as you keep the jobs around. **Note**: a single recurring schedule is supported, so you can have multiple schedulers using the same schedule, but not multiple schedulers using different configurations. diff --git a/app/models/solid_queue/job/clearable.rb b/app/models/solid_queue/job/clearable.rb index cc954d69..555026a6 100644 --- a/app/models/solid_queue/job/clearable.rb +++ b/app/models/solid_queue/job/clearable.rb @@ -10,9 +10,10 @@ module Clearable end class_methods do - def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago, class_name: nil) + def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago, class_name: nil, sleep_between_batches: 0) loop do records_deleted = clearable(finished_before: finished_before, class_name: class_name).limit(batch_size).delete_all + sleep(sleep_between_batches) if sleep_between_batches > 0 break if records_deleted == 0 end end diff --git a/app/models/solid_queue/recurring_task.rb b/app/models/solid_queue/recurring_task.rb index 54777531..5363f0a7 100644 --- a/app/models/solid_queue/recurring_task.rb +++ b/app/models/solid_queue/recurring_task.rb @@ -12,6 +12,8 @@ class RecurringTask < Record scope :static, -> { where(static: true) } + has_many :recurring_executions, foreign_key: :task_key, primary_key: :key + mattr_accessor :default_job_class self.default_job_class = RecurringJob @@ -53,6 +55,18 @@ def next_time parsed_schedule.next_time.utc end + def previous_time + parsed_schedule.previous_time.utc + end + + def last_enqueued_time + if recurring_executions.loaded? + recurring_executions.map(&:run_at).max + else + recurring_executions.maximum(:run_at) + end + end + def enqueue(at:) SolidQueue.instrument(:enqueue_recurring_task, task: key, at: at) do |payload| active_job = if using_solid_queue_adapter? diff --git a/lib/solid_queue/app_executor.rb b/lib/solid_queue/app_executor.rb index da0976fe..0580213f 100644 --- a/lib/solid_queue/app_executor.rb +++ b/lib/solid_queue/app_executor.rb @@ -4,7 +4,7 @@ module SolidQueue module AppExecutor def wrap_in_app_executor(&block) if SolidQueue.app_executor - SolidQueue.app_executor.wrap(&block) + SolidQueue.app_executor.wrap(source: "application.solid_queue", &block) else yield end diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 96e732c3..bd238dcc 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -2,6 +2,12 @@ module SolidQueue class Configuration + include ActiveModel::Model + + validate :ensure_configured_processes + validate :ensure_valid_recurring_tasks + validate :ensure_correctly_sized_thread_pool + class Process < Struct.new(:kind, :attributes) def instantiate "SolidQueue::#{kind.to_s.titleize}".safe_constantize.new(**attributes) @@ -36,14 +42,46 @@ def configured_processes end end - def max_number_of_threads - # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task - workers_options.map { |options| options[:threads] }.max + 2 + def error_messages + if configured_processes.none? + "No workers or processed configured. Exiting..." + else + error_messages = invalid_tasks.map do |task| + all_messages = task.errors.full_messages.map { |msg| "\t#{msg}" }.join("\n") + "#{task.key}:\n#{all_messages}" + end + .join("\n") + + "Invalid processes configured:\n#{error_messages}" + end end private attr_reader :options + def ensure_configured_processes + unless configured_processes.any? + errors.add(:base, "No processes configured") + end + end + + def ensure_valid_recurring_tasks + unless skip_recurring_tasks? || invalid_tasks.none? + error_messages = invalid_tasks.map do |task| + "- #{task.key}: #{task.errors.full_messages.join(", ")}" + end + + errors.add(:base, "Invalid recurring tasks:\n#{error_messages.join("\n")}") + end + end + + def ensure_correctly_sized_thread_pool + if (db_pool_size = SolidQueue::Record.connection_pool&.size) && db_pool_size < estimated_number_of_threads + errors.add(:base, "Solid Queue is configured to use #{estimated_number_of_threads} threads but the " + + "database connection pool is #{db_pool_size}. Increase it in `config/database.yml`") + end + end + def default_options { config_file: Rails.root.join(ENV["SOLID_QUEUE_CONFIG"] || DEFAULT_CONFIG_FILE_PATH), @@ -54,6 +92,10 @@ def default_options } end + def invalid_tasks + recurring_tasks.select(&:invalid?) + end + def only_work? options[:only_work] end @@ -100,7 +142,7 @@ def dispatchers_options def recurring_tasks @recurring_tasks ||= recurring_tasks_config.map do |id, options| RecurringTask.from_configuration(id, **options) - end.select(&:valid?) + end end def processes_config @@ -147,5 +189,11 @@ def load_config_from_file(file) {} end end + + def estimated_number_of_threads + # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task + thread_count = workers_options.map { |options| options.fetch(:threads, WORKER_DEFAULTS[:threads]) }.max + (thread_count || 1) + 2 + end end end diff --git a/lib/solid_queue/dispatcher.rb b/lib/solid_queue/dispatcher.rb index a22a82d8..fb988075 100644 --- a/lib/solid_queue/dispatcher.rb +++ b/lib/solid_queue/dispatcher.rb @@ -24,7 +24,8 @@ def metadata private def poll batch = dispatch_next_batch - batch.size + + batch.size.zero? ? polling_interval : 0.seconds end def dispatch_next_batch diff --git a/lib/solid_queue/log_subscriber.rb b/lib/solid_queue/log_subscriber.rb index 3d2ec02c..96fb19bf 100644 --- a/lib/solid_queue/log_subscriber.rb +++ b/lib/solid_queue/log_subscriber.rb @@ -145,6 +145,7 @@ def unhandled_signal_error(event) end def replace_fork(event) + supervisor_pid = event.payload[:supervisor_pid] status = event.payload[:status] attributes = event.payload.slice(:pid).merge \ status: (status.exitstatus || "no exit status set"), @@ -155,7 +156,7 @@ def replace_fork(event) if replaced_fork = event.payload[:fork] info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname, name: replaced_fork.name)) - else + elsif supervisor_pid != 1 # Running Docker, possibly having some processes that have been reparented warn formatted_event(event, action: "Tried to replace forked process but it had already died", **attributes) end end diff --git a/lib/solid_queue/processes/interruptible.rb b/lib/solid_queue/processes/interruptible.rb index 09c027b6..3bff1dd9 100644 --- a/lib/solid_queue/processes/interruptible.rb +++ b/lib/solid_queue/processes/interruptible.rb @@ -12,13 +12,17 @@ def interrupt queue << true end + # Sleeps for 'time'. Can be interrupted asynchronously and return early via wake_up. + # @param time [Numeric] the time to sleep. 0 returns immediately. + # @return [true, nil] + # * returns `true` if an interrupt was requested via #wake_up between the + # last call to `interruptible_sleep` and now, resulting in an early return. + # * returns `nil` if it slept the full `time` and was not interrupted. def interruptible_sleep(time) - # Invoking from the main thread can result in a 35% slowdown (at least when running the test suite). - # Using some form of Async (Futures) addresses this performance issue. + # Invoking this from the main thread may result in significant slowdown. + # Utilizing asynchronous execution (Futures) addresses this performance issue. Concurrent::Promises.future(time) do |timeout| - if timeout > 0 && queue.pop(timeout:) - queue.clear - end + queue.pop(timeout:).tap { queue.clear } end.value end diff --git a/lib/solid_queue/processes/poller.rb b/lib/solid_queue/processes/poller.rb index bf5a7450..75df6104 100644 --- a/lib/solid_queue/processes/poller.rb +++ b/lib/solid_queue/processes/poller.rb @@ -25,11 +25,11 @@ def start_loop loop do break if shutting_down? - wrap_in_app_executor do - unless poll > 0 - interruptible_sleep(polling_interval) - end + delay = wrap_in_app_executor do + poll end + + interruptible_sleep(delay) end ensure SolidQueue.instrument(:shutdown_process, process: self) do diff --git a/lib/solid_queue/supervisor.rb b/lib/solid_queue/supervisor.rb index 9ef736e4..e8f075eb 100644 --- a/lib/solid_queue/supervisor.rb +++ b/lib/solid_queue/supervisor.rb @@ -10,10 +10,10 @@ def start(**options) SolidQueue.supervisor = true configuration = Configuration.new(**options) - if configuration.configured_processes.any? + if configuration.valid? new(configuration).tap(&:start) else - abort "No workers or processed configured. Exiting..." + abort configuration.errors.full_messages.join("\n") + "\nExiting..." end end end diff --git a/lib/solid_queue/version.rb b/lib/solid_queue/version.rb index 557ba5c0..bcd5f370 100644 --- a/lib/solid_queue/version.rb +++ b/lib/solid_queue/version.rb @@ -1,3 +1,3 @@ module SolidQueue - VERSION = "1.1.0" + VERSION = "1.1.1" end diff --git a/lib/solid_queue/worker.rb b/lib/solid_queue/worker.rb index fc203774..f34a14f0 100644 --- a/lib/solid_queue/worker.rb +++ b/lib/solid_queue/worker.rb @@ -7,6 +7,7 @@ class Worker < Processes::Poller after_boot :run_start_hooks before_shutdown :run_stop_hooks + attr_accessor :queues, :pool def initialize(**options) @@ -29,7 +30,7 @@ def poll pool.post(execution) end - executions.size + pool.idle? ? polling_interval : 10.minutes end end diff --git a/test/dummy/app/models/sharded_job_result.rb b/test/dummy/app/models/sharded_job_result.rb new file mode 100644 index 00000000..001bb3b7 --- /dev/null +++ b/test/dummy/app/models/sharded_job_result.rb @@ -0,0 +1,2 @@ +class ShardedJobResult < ShardedRecord +end diff --git a/test/dummy/app/models/sharded_record.rb b/test/dummy/app/models/sharded_record.rb new file mode 100644 index 00000000..87d4bcaf --- /dev/null +++ b/test/dummy/app/models/sharded_record.rb @@ -0,0 +1,8 @@ +class ShardedRecord < ApplicationRecord + self.abstract_class = true + + connects_to shards: { + shard_one: { writing: :shard_one }, + shard_two: { writing: :shard_two } + } +end diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml index 027f6706..fdb186a5 100644 --- a/test/dummy/config/database.yml +++ b/test/dummy/config/database.yml @@ -10,7 +10,7 @@ <% if ENV["TARGET_DB"] == "sqlite" %> default: &default adapter: sqlite3 - pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 50 } %> + pool: 50 retries: 100 <% elsif ENV["TARGET_DB"] == "postgres" %> @@ -18,7 +18,7 @@ default: &default adapter: postgresql encoding: unicode username: postgres - pool: 5 + pool: 20 host: "127.0.0.1" port: 55432 gssencmode: disable # https://github.com/ged/ruby-pg/issues/311 @@ -27,7 +27,7 @@ default: &default default: &default adapter: mysql2 username: root - pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + pool: 20 host: "127.0.0.1" port: 33060 <% end %> @@ -36,6 +36,14 @@ development: primary: <<: *default database: <%= database_name_from("development") %> + shard_one: + <<: *default + database: <%= database_name_from("development_shard_one") %> + migrations_paths: db/migrate_shards + shard_two: + <<: *default + database: <%= database_name_from("development_shard_two") %> + migrations_paths: db/migrate_shards queue: <<: *default database: <%= database_name_from("development_queue") %> @@ -44,10 +52,16 @@ development: test: primary: <<: *default - pool: 20 database: <%= database_name_from("test") %> + shard_one: + <<: *default + database: <%= database_name_from("test_shard_one") %> + migrations_paths: db/migrate_shards + shard_two: + <<: *default + database: <%= database_name_from("test_shard_two") %> + migrations_paths: db/migrate_shards queue: <<: *default - pool: 20 database: <%= database_name_from("test_queue") %> migrations_paths: db/queue_migrate diff --git a/test/dummy/config/recurring_with_invalid.yml b/test/dummy/config/recurring_with_invalid.yml new file mode 100644 index 00000000..69dacf6f --- /dev/null +++ b/test/dummy/config/recurring_with_invalid.yml @@ -0,0 +1,8 @@ +periodic_invalid_class: + class: StoreResultJorrrrrrb + queue: default + args: [42, { status: "custom_status" }] + schedule: every second +periodic_incorrect_schedule: + class: StoreResultJob + schedule: every 1.minute diff --git a/test/dummy/db/migrate_shards/20241205195735_create_sharded_job_results.rb b/test/dummy/db/migrate_shards/20241205195735_create_sharded_job_results.rb new file mode 100644 index 00000000..e2ded0c9 --- /dev/null +++ b/test/dummy/db/migrate_shards/20241205195735_create_sharded_job_results.rb @@ -0,0 +1,9 @@ +class CreateShardedJobResults < ActiveRecord::Migration[7.1] + def change + create_table :sharded_job_results do |t| + t.string :value + + t.timestamps + end + end +end diff --git a/test/dummy/db/schema.rb b/test/dummy/db/schema.rb index a52fb820..fd3db435 100644 --- a/test/dummy/db/schema.rb +++ b/test/dummy/db/schema.rb @@ -18,132 +18,4 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false end - - create_table "solid_queue_blocked_executions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.bigint "job_id", null: false - t.string "queue_name", null: false - t.integer "priority", default: 0, null: false - t.string "concurrency_key", null: false - t.datetime "expires_at", null: false - t.datetime "created_at", null: false - t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release" - t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance" - t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true - end - - create_table "solid_queue_claimed_executions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.bigint "job_id", null: false - t.bigint "process_id" - t.datetime "created_at", null: false - t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true - t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" - end - - create_table "solid_queue_failed_executions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.bigint "job_id", null: false - t.text "error" - t.datetime "created_at", null: false - t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true - end - - create_table "solid_queue_jobs", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.string "queue_name", null: false - t.string "class_name", null: false - t.text "arguments" - t.integer "priority", default: 0, null: false - t.string "active_job_id" - t.datetime "scheduled_at" - t.datetime "finished_at" - t.string "concurrency_key" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id" - t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name" - t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at" - t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering" - t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting" - end - - create_table "solid_queue_pauses", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.string "queue_name", null: false - t.datetime "created_at", null: false - t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true - end - - create_table "solid_queue_processes", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.string "kind", null: false - t.datetime "last_heartbeat_at", null: false - t.bigint "supervisor_id" - t.integer "pid", null: false - t.string "hostname" - t.text "metadata" - t.datetime "created_at", null: false - t.string "name", null: false - t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" - t.index ["name", "supervisor_id"], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true - t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" - end - - create_table "solid_queue_ready_executions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.bigint "job_id", null: false - t.string "queue_name", null: false - t.integer "priority", default: 0, null: false - t.datetime "created_at", null: false - t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true - t.index ["priority", "job_id"], name: "index_solid_queue_poll_all" - t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue" - end - - create_table "solid_queue_recurring_executions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.bigint "job_id", null: false - t.string "task_key", null: false - t.datetime "run_at", null: false - t.datetime "created_at", null: false - t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true - t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true - end - - create_table "solid_queue_recurring_tasks", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.string "key", null: false - t.string "schedule", null: false - t.string "command", limit: 2048 - t.string "class_name" - t.text "arguments" - t.string "queue_name" - t.integer "priority", default: 0 - t.boolean "static", default: true, null: false - t.text "description" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["key"], name: "index_solid_queue_recurring_tasks_on_key", unique: true - t.index ["static"], name: "index_solid_queue_recurring_tasks_on_static" - end - - create_table "solid_queue_scheduled_executions", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.bigint "job_id", null: false - t.string "queue_name", null: false - t.integer "priority", default: 0, null: false - t.datetime "scheduled_at", null: false - t.datetime "created_at", null: false - t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true - t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all" - end - - create_table "solid_queue_semaphores", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| - t.string "key", null: false - t.integer "value", default: 1, null: false - t.datetime "expires_at", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at" - t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value" - t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true - end - - add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade end diff --git a/test/dummy/db/shard_one_schema.rb b/test/dummy/db/shard_one_schema.rb new file mode 100644 index 00000000..2a6d3806 --- /dev/null +++ b/test/dummy/db/shard_one_schema.rb @@ -0,0 +1,20 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[7.1].define(version: 2024_12_05_195735) do + create_table "sharded_job_results", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.string "value" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + +end diff --git a/test/dummy/db/shard_two_schema.rb b/test/dummy/db/shard_two_schema.rb new file mode 100644 index 00000000..2a6d3806 --- /dev/null +++ b/test/dummy/db/shard_two_schema.rb @@ -0,0 +1,20 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[7.1].define(version: 2024_12_05_195735) do + create_table "sharded_job_results", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t| + t.string "value" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + +end diff --git a/test/models/solid_queue/job_test.rb b/test/models/solid_queue/job_test.rb index ed8ba090..17a658d7 100644 --- a/test/models/solid_queue/job_test.rb +++ b/test/models/solid_queue/job_test.rb @@ -68,6 +68,16 @@ class NonOverlappingGroupedJob2 < NonOverlappingJob assert_equal solid_queue_job.scheduled_at, execution.scheduled_at end + test "enqueue jobs within a connected_to block for the primary DB" do + ShardedRecord.connected_to(role: :writing, shard: :shard_two) do + ShardedJobResult.create!(value: "in shard two") + AddToBufferJob.perform_later("enqueued within block") + end + + job = SolidQueue::Job.last + assert_equal "enqueued within block", job.arguments.dig("arguments", 0) + end + test "enqueue jobs without concurrency controls" do active_job = AddToBufferJob.perform_later(1) assert_nil active_job.concurrency_limit diff --git a/test/test_helper.rb b/test/test_helper.rb index 5f1bebbb..539f67bc 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -28,7 +28,7 @@ class ExpectedTestError < RuntimeError; end class ActiveSupport::TestCase - include ProcessesTestHelper, JobsTestHelper + include ConfigurationTestHelper, ProcessesTestHelper, JobsTestHelper setup do # Could be cleaner with one several minitest gems, but didn't want to add new dependency diff --git a/test/test_helpers/configuration_test_helper.rb b/test/test_helpers/configuration_test_helper.rb new file mode 100644 index 00000000..24b95e6b --- /dev/null +++ b/test/test_helpers/configuration_test_helper.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module ConfigurationTestHelper + def config_file_path(name) + Rails.root.join("config/#{name}.yml") + end +end diff --git a/test/test_helpers/processes_test_helper.rb b/test/test_helpers/processes_test_helper.rb index 729216bd..9a6d0f65 100644 --- a/test/test_helpers/processes_test_helper.rb +++ b/test/test_helpers/processes_test_helper.rb @@ -7,6 +7,16 @@ def run_supervisor_as_fork(**options) end end + def run_supervisor_as_fork_with_captured_io(**options) + pid = nil + out, err = capture_subprocess_io do + pid = run_supervisor_as_fork(**options) + wait_for_registered_processes(4) + end + + [ pid, out, err ] + end + def wait_for_registered_processes(count, timeout: 1.second) wait_while_with_timeout(timeout) { SolidQueue::Process.count != count } end diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index 556a4930..87b8726e 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -54,11 +54,6 @@ class ConfigurationTest < ActiveSupport::TestCase assert_processes configuration, :worker, 2 end - test "max number of threads" do - configuration = SolidQueue::Configuration.new - assert 7, configuration.max_number_of_threads - end - test "mulitple workers with the same configuration" do background_worker = { queues: "background", polling_interval: 10, processes: 3 } configuration = SolidQueue::Configuration.new(workers: [ background_worker ]) @@ -90,6 +85,32 @@ class ConfigurationTest < ActiveSupport::TestCase assert_processes configuration, :dispatcher, 1, polling_interval: 0.1, recurring_tasks: nil end + test "validate configuration" do + # Valid and invalid recurring tasks + configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_invalid)) + assert_not configuration.valid? + assert configuration.errors.full_messages.one? + error = configuration.errors.full_messages.first + + assert error.include?("Invalid recurring tasks") + assert error.include?("periodic_invalid_class: Class name doesn't correspond to an existing class") + assert error.include?("periodic_incorrect_schedule: Schedule is not a supported recurring schedule") + + assert SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:empty_recurring)).valid? + assert SolidQueue::Configuration.new(skip_recurring: true).valid? + + # No processes + configuration = SolidQueue::Configuration.new(skip_recurring: true, dispatchers: [], workers: []) + assert_not configuration.valid? + assert_equal [ "No processes configured" ], configuration.errors.full_messages + + # Not enough DB connections + configuration = SolidQueue::Configuration.new(workers: [ { queues: "background", threads: 50, polling_interval: 10 } ]) + assert_not configuration.valid? + assert_match /Solid Queue is configured to use \d+ threads but the database connection pool is \d+. Increase it in `config\/database.yml`/, + configuration.errors.full_messages.first + end + private def assert_processes(configuration, kind, count, **attributes) processes = configuration.configured_processes.select { |p| p.kind == kind } @@ -121,8 +142,4 @@ def assert_equal_value(expected_value, value) assert_equal expected_value, value end end - - def config_file_path(name) - Rails.root.join("config/#{name}.yml") - end end diff --git a/test/unit/dispatcher_test.rb b/test/unit/dispatcher_test.rb index 42d57c92..5bca7743 100644 --- a/test/unit/dispatcher_test.rb +++ b/test/unit/dispatcher_test.rb @@ -92,6 +92,30 @@ class DispatcherTest < ActiveSupport::TestCase another_dispatcher&.stop end + test "sleeps `0.seconds` between polls if there are ready to dispatch jobs" do + dispatcher = SolidQueue::Dispatcher.new(polling_interval: 10, batch_size: 1) + dispatcher.expects(:interruptible_sleep).with(0.seconds).at_least(3) + dispatcher.expects(:interruptible_sleep).with(dispatcher.polling_interval).at_least_once + + 3.times { AddToBufferJob.set(wait: 0.1).perform_later("I'm scheduled") } + assert_equal 3, SolidQueue::ScheduledExecution.count + sleep 0.1 + + dispatcher.start + wait_while_with_timeout(1.second) { SolidQueue::ScheduledExecution.any? } + + assert_equal 0, SolidQueue::ScheduledExecution.count + assert_equal 3, SolidQueue::ReadyExecution.count + end + + test "sleeps `polling_interval` between polls if there are no un-dispatched jobs" do + dispatcher = SolidQueue::Dispatcher.new(polling_interval: 10, batch_size: 1) + dispatcher.expects(:interruptible_sleep).with(0.seconds).never + dispatcher.expects(:interruptible_sleep).with(dispatcher.polling_interval).at_least_once + dispatcher.start + sleep 0.1 + end + private def with_polling(silence:) old_silence_polling, SolidQueue.silence_polling = SolidQueue.silence_polling, silence diff --git a/test/unit/supervisor_test.rb b/test/unit/supervisor_test.rb index d4919070..c430544a 100644 --- a/test/unit/supervisor_test.rb +++ b/test/unit/supervisor_test.rb @@ -41,11 +41,22 @@ class SupervisorTest < ActiveSupport::TestCase end test "start with empty configuration" do - pid = run_supervisor_as_fork(workers: [], dispatchers: []) + pid, _out, error = run_supervisor_as_fork_with_captured_io(workers: [], dispatchers: []) sleep(0.5) assert_no_registered_processes assert_not process_exists?(pid) + assert_match %r{No processes configured}, error + end + + test "start with invalid recurring tasks" do + pid, _out, error = run_supervisor_as_fork_with_captured_io(recurring_schedule_file: config_file_path(:recurring_with_invalid), skip_recurring: false) + + sleep(0.5) + assert_no_registered_processes + + assert_not process_exists?(pid) + assert_match %r{Invalid recurring tasks}, error end test "create and delete pidfile" do @@ -66,11 +77,12 @@ class SupervisorTest < ActiveSupport::TestCase FileUtils.mkdir_p(File.dirname(@pidfile)) File.write(@pidfile, ::Process.pid.to_s) - pid = run_supervisor_as_fork + pid, _out, err = run_supervisor_as_fork_with_captured_io wait_for_registered_processes(4) assert File.exist?(@pidfile) assert_not_equal pid, File.read(@pidfile).strip.to_i + assert_match %r{A Solid Queue supervisor is already running}, err wait_for_process_termination_with_timeout(pid, exitstatus: 1) end diff --git a/test/unit/worker_test.rb b/test/unit/worker_test.rb index d511cf74..52b0d8e8 100644 --- a/test/unit/worker_test.rb +++ b/test/unit/worker_test.rb @@ -171,6 +171,26 @@ class WorkerTest < ActiveSupport::TestCase SolidQueue.process_heartbeat_interval = old_heartbeat_interval end + test "sleeps `10.minutes` if at capacity" do + 3.times { |i| StoreResultJob.perform_later(i, pause: 1.second) } + + @worker.expects(:interruptible_sleep).with(10.minutes).at_least_once + @worker.expects(:interruptible_sleep).with(@worker.polling_interval).never + + @worker.start + sleep 1.second + end + + test "sleeps `polling_interval` if worker not at capacity" do + 2.times { |i| StoreResultJob.perform_later(i, pause: 1.second) } + + @worker.expects(:interruptible_sleep).with(@worker.polling_interval).at_least_once + @worker.expects(:interruptible_sleep).with(10.minutes).never + + @worker.start + sleep 1.second + end + private def with_polling(silence:) old_silence_polling, SolidQueue.silence_polling = SolidQueue.silence_polling, silence