diff --git a/Gemfile.lock b/Gemfile.lock index 4403329..bd5b9fa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,6 +53,7 @@ GEM drb (2.1.1) ruby2_keywords erubi (1.12.0) + extralite (2.2) i18n (1.14.1) concurrent-ruby (~> 1.0) io-console (0.6.0) @@ -65,11 +66,11 @@ GEM loofah (2.21.3) crass (~> 1.0.2) nokogiri (>= 1.12.0) + mini_portile2 (2.8.5) minitest (5.20.0) mutex_m (0.1.2) - nokogiri (1.15.4-arm64-darwin) - racc (~> 1.4) - nokogiri (1.15.4-x86_64-linux) + nokogiri (1.15.4) + mini_portile2 (~> 2.8.2) racc (~> 1.4) parallel (1.23.0) parser (3.2.2.3) @@ -126,8 +127,8 @@ GEM rubocop-ast (>= 0.4.0) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) - sqlite3 (1.6.6-arm64-darwin) - sqlite3 (1.6.6-x86_64-linux) + sqlite3 (1.6.6) + mini_portile2 (~> 2.8.0) standard (1.30.1) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.0) @@ -151,11 +152,14 @@ GEM PLATFORMS arm64-darwin-21 + arm64-darwin-22 + arm64-darwin-23 x86_64-linux DEPENDENCIES activerecord-enhancedsqlite3-adapter! combustion (~> 1.3) + extralite minitest (~> 5.0) railties rake (~> 13.0) diff --git a/activerecord-enhancedsqlite3-adapter.gemspec b/activerecord-enhancedsqlite3-adapter.gemspec index 3574153..604a06e 100644 --- a/activerecord-enhancedsqlite3-adapter.gemspec +++ b/activerecord-enhancedsqlite3-adapter.gemspec @@ -36,6 +36,7 @@ Gem::Specification.new do |spec| spec.add_dependency "sqlite3", "~> 1.6" spec.add_development_dependency "combustion", "~> 1.3" + spec.add_development_dependency "extralite" spec.add_development_dependency "railties" spec.add_development_dependency "minitest" spec.add_development_dependency "rake" diff --git a/lib/enhanced_sqlite3/adapter.rb b/lib/enhanced_sqlite3/adapter.rb index 87c7ef3..965b526 100644 --- a/lib/enhanced_sqlite3/adapter.rb +++ b/lib/enhanced_sqlite3/adapter.rb @@ -7,28 +7,68 @@ require "active_record/connection_adapters/sqlite3_adapter" require "enhanced_sqlite3/supports_virtual_columns" require "enhanced_sqlite3/supports_deferrable_constraints" +require "enhanced_sqlite3/extralite/database_compatibility" +require "enhanced_sqlite3/extralite/adapter_compatibility" module EnhancedSQLite3 module Adapter + module ClassMethods + def new_client(config) + if config[:client] == "extralite" + new_client_extralite(config) + else + super + end + end + + def new_client_extralite(config) + config.delete(:results_as_hash) + + if config[:strict] == true + raise ArgumentError, "The :strict option is not supported by the SQLite3 adapter using Extralite" + end + + unsupported_configuration_keys = config.keys - %i[database readonly client adapter strict timeout pool] + if unsupported_configuration_keys.any? + Rails.logger.warn "Unsupported configuration options for SQLite3 adapter using Extralite: #{unsupported_configuration_keys}" + end + + ::Extralite::Database.new(config[:database].to_s, read_only: config[:readonly]).tap do |database| + database.singleton_class.prepend EnhancedSQLite3::Extralite::DatabaseCompatibility + end + rescue Errno::ENOENT => error + if error.message.include?("No such file or directory") + raise ActiveRecord::NoDatabaseError + else + raise + end + end + end + # Setup the Rails SQLite3 adapter instance. # # extends https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L90 def initialize(...) super - # Ensure that all connections default to immediate transaction mode. - # This is necessary to prevent SQLite from deadlocking when concurrent processes open write transactions. - # By default, SQLite opens transactions in deferred mode, which means that a transactions acquire - # a shared lock on the database, but will attempt to upgrade that lock to an exclusive lock if/when - # a write is attempted. Because SQLite is in the middle of a transaction, it cannot retry the transaction - # if a BUSY exception is raised, and so it will immediately raise a SQLITE_BUSY exception without calling - # the `busy_handler`. Because Rails only wraps writes in transactions, this means that all transactions - # will attempt to acquire an exclusive lock on the database. Thus, under any concurrent load, you are very - # likely to encounter a SQLITE_BUSY exception. - # By setting the default transaction mode to immediate, SQLite will instead attempt to acquire - # an exclusive lock as soon as the transaction is opened. If the lock cannot be acquired, it will - # immediately call the `busy_handler` to retry the transaction. This allows concurrent processes to - # coordinate and linearize their transactions, avoiding deadlocks. - @connection_parameters.merge!(default_transaction_mode: :immediate) + + if @config[:client] == "extralite" + singleton_class.prepend EnhancedSQLite3::Extralite::AdapterCompatibility + else + # Ensure that all connections default to immediate transaction mode. + # This is necessary to prevent SQLite from deadlocking when concurrent processes open write transactions. + # By default, SQLite opens transactions in deferred mode, which means that a transactions acquire + # a shared lock on the database, but will attempt to upgrade that lock to an exclusive lock if/when + # a write is attempted. Because SQLite is in the middle of a transaction, it cannot retry the transaction + # if a BUSY exception is raised, and so it will immediately raise a SQLITE_BUSY exception without calling + # the `busy_handler`. Because Rails only wraps writes in transactions, this means that all transactions + # will attempt to acquire an exclusive lock on the database. Thus, under any concurrent load, you are very + # likely to encounter a SQLITE_BUSY exception. + # By setting the default transaction mode to immediate, SQLite will instead attempt to acquire + # an exclusive lock as soon as the transaction is opened. If the lock cannot be acquired, it will + # immediately call the `busy_handler` to retry the transaction. This allows concurrent processes to + # coordinate and linearize their transactions, avoiding deadlocks. + @connection_parameters.merge!(default_transaction_mode: :immediate) + end end # Perform any necessary initialization upon the newly-established @@ -53,10 +93,16 @@ def configure_connection private def configure_busy_handler_timeout - return unless @config.key?(:timeout) + return unless @config[:timeout] timeout = self.class.type_cast_config_to_integer(@config[:timeout]) timeout_seconds = timeout.fdiv(1000) + + if @config[:client] == "extralite" + @raw_connection.busy_timeout(timeout_seconds) + return + end + retry_interval = 6e-5 # 60 microseconds @raw_connection.busy_handler do |count| @@ -111,7 +157,9 @@ def configure_pragmas end def configure_extensions - @raw_connection.enable_load_extension(true) + # NOTE: Extralite enables extension loading by default and doesn't provide an API to toggle it. + @raw_connection.enable_load_extension(true) if @raw_connection.is_a?(::SQLite3::Database) + @config.fetch(:extensions, []).each do |extension_name| require extension_name extension_classname = extension_name.camelize @@ -122,7 +170,7 @@ def configure_extensions rescue NameError Rails.logger.error("Failed to find the SQLite extension class: #{extension_classname}. Skipping...") end - @raw_connection.enable_load_extension(false) + @raw_connection.enable_load_extension(false) if @raw_connection.is_a?(::SQLite3::Database) end end end diff --git a/lib/enhanced_sqlite3/extralite/adapter_compatibility.rb b/lib/enhanced_sqlite3/extralite/adapter_compatibility.rb new file mode 100644 index 0000000..4c94ea9 --- /dev/null +++ b/lib/enhanced_sqlite3/extralite/adapter_compatibility.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# Makes some methods of the ActiveRecord SQLite3 adapter compatible with Extralite::Database. + +module EnhancedSQLite3 + module Extralite + module AdapterCompatibility + def internal_exec_query(sql, name = nil, binds = [], prepare: false, async: false) + sql = transform_query(sql) + check_if_write_query(sql) + + mark_transaction_written_if_write(sql) + + type_casted_binds = type_casted_binds(binds) + + log(sql, name, binds, type_casted_binds, async: async) do + with_raw_connection do |conn| + unless prepare + stmt = conn.prepare(sql) + begin + cols = stmt.columns + unless without_prepared_statement?(binds) + stmt.bind(*type_casted_binds) # Added splat since Extralite doesn't accept an array argument for #bind + end + records = stmt.to_a_ary # Extralite uses to_a_ary rather than to_a to get a Array[Array] result + ensure + stmt.close + end + else + stmt = @statements[sql] ||= conn.prepare(sql) + cols = stmt.columns + stmt.reset # Extralite uses reset rather than reset! + stmt.bind(*type_casted_binds) # Added splat since Extralite doesn't accept an array argument for #bind + records = stmt.to_a_ary # Extralite uses to_a_ary rather than to_a to get a Array[Array] result + end + + # Extralite defaults to returning symbols for columns but #build_result expects strings + build_result(columns: cols.map(&:to_s), rows: records) + end + end + end + + private + + def raw_execute(sql, name, async: false, allow_retry: false, materialize_transactions: false) + log(sql, name, async: async) do + with_raw_connection(allow_retry: allow_retry, materialize_transactions: materialize_transactions) do |conn| + conn.query(sql) # Extralite::Database#execute doesn't return results so use #query instead + end + end + end + end + end +end diff --git a/lib/enhanced_sqlite3/extralite/database_compatibility.rb b/lib/enhanced_sqlite3/extralite/database_compatibility.rb new file mode 100644 index 0000000..7fda7bc --- /dev/null +++ b/lib/enhanced_sqlite3/extralite/database_compatibility.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# Implements some methods to make Extralite::Database compatible enough with +# SQLite3::Database to be used by ActiveRecord's SQLite adapter. + +module EnhancedSQLite3 + module Extralite + module DatabaseCompatibility + def transaction(mode = :deferred) + execute "BEGIN #{mode.to_s.upcase} TRANSACTION" + + if block_given? + abort = false + begin + yield self + rescue StandardError + abort = true + raise + ensure + abort and rollback or commit + end + end + + true + end + + def commit + execute "COMMIT TRANSACTION" + end + + def rollback + execute "ROLLBACK TRANSACTION" + end + + # NOTE: Extralite only supports UTF-8 encoding while the sqlite3 gem can use UTF-16 + # if utf16: true is passed to the Database initializer. + def encoding + "UTF-8" + end + + # NOTE: The sqlite3 gem appears to support both busy_timeout= and busy_timeout + # The ActiveRecord adapter #configure_connection method uses the latter, which + # could potentially be changed, allowing us to get rid of the monkey patch. + def busy_timeout(timeout) + self.busy_timeout = timeout + end + + def readonly? + read_only? + end + end + end +end diff --git a/lib/enhanced_sqlite3/railtie.rb b/lib/enhanced_sqlite3/railtie.rb index de2e702..573a6a5 100644 --- a/lib/enhanced_sqlite3/railtie.rb +++ b/lib/enhanced_sqlite3/railtie.rb @@ -8,9 +8,10 @@ class Railtie < ::Rails::Railtie # Enhance the SQLite3 ActiveRecord adapter with optimized defaults initializer "enhanced_sqlite3.enhance_active_record_sqlite3adapter" do |app| ActiveSupport.on_load(:active_record_sqlite3adapter) do - # self refers to `SQLite3Adapter` here, + # self refers to `ActiveRecord::ConnectionAdapters::SQLite3Adapter` here, # so we can call .prepend prepend EnhancedSQLite3::Adapter + singleton_class.prepend EnhancedSQLite3::Adapter::ClassMethods end end end diff --git a/test/combustion/config/database.yml b/test/combustion/config/database.yml index 2c715ed..0ecdf69 100644 --- a/test/combustion/config/database.yml +++ b/test/combustion/config/database.yml @@ -1,3 +1,4 @@ test: adapter: sqlite3 - database: database.sqlite \ No newline at end of file + database: database.sqlite + client: <%= "extralite" if ENV["EXTRALITE"] == "true" %> \ No newline at end of file diff --git a/test/test_helper.rb b/test/test_helper.rb index 1afbef8..6595bf7 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -12,7 +12,12 @@ require "minitest/autorun" require "combustion" -# require "sqlite3" + +if ENV["EXTRALITE"] == "true" + puts "Testing in Extralite client mode..." + require "extralite" +end + Combustion.path = "test/combustion" Combustion.initialize! :active_record