From c6825d385889dd62f0ad90c0e7da8aa7c9b6b212 Mon Sep 17 00:00:00 2001 From: Chris Beer Date: Wed, 11 May 2022 07:17:18 -0700 Subject: [PATCH 1/2] Use ActionController::Parameters to filter out non-search parameters --- .rubocop.yml | 1 + lib/blacklight/configuration.rb | 9 +- lib/blacklight/parameters.rb | 85 ++++++++++++++++++- lib/blacklight/search_builder.rb | 2 +- lib/blacklight/search_state.rb | 74 +++------------- lib/blacklight/search_state/filter_field.rb | 18 ++-- ...render_constraints_helper_behavior_spec.rb | 2 +- spec/lib/blacklight/parameters_spec.rb | 84 ++++++++++++++++++ .../search_state/filter_field_spec.rb | 15 +--- spec/lib/blacklight/search_state_spec.rb | 15 +--- .../blacklight/solr/search_builder_spec.rb | 4 +- 11 files changed, 206 insertions(+), 103 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 3c1af4ec2f..0e312ffff5 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -39,6 +39,7 @@ Metrics/ClassLength: - "lib/blacklight/configuration.rb" - "lib/blacklight/search_builder.rb" - "lib/blacklight/search_state.rb" + - "lib/blacklight/search_state/filter_field.rb" Layout/LineLength: Max: 200 diff --git a/lib/blacklight/configuration.rb b/lib/blacklight/configuration.rb index 8c2da23246..e003c9ccce 100644 --- a/lib/blacklight/configuration.rb +++ b/lib/blacklight/configuration.rb @@ -296,7 +296,7 @@ def default_per_page property :enable_search_bar_autofocus, default: false BASIC_SEARCH_PARAMETERS = [:q, :qt, :page, :per_page, :search_field, :sort, :controller, :action, :'facet.page', :'facet.prefix', :'facet.sort', :rows, :format].freeze - ADVANCED_SEARCH_PARAMETERS = [:clause, :op].freeze + ADVANCED_SEARCH_PARAMETERS = [{ clause: {} }, :op].freeze # List the request parameters that compose the SearchState. # If you use a plugin that adds to the search state, then you can add the parameters # by modifiying this field. @@ -305,6 +305,13 @@ def default_per_page # @return [Array] property :search_state_fields, default: BASIC_SEARCH_PARAMETERS + ADVANCED_SEARCH_PARAMETERS + # Have SearchState filter out unknown request parameters + # + # @!attribute filter_search_state_fields + # @since v8.0.0 + # @return [Boolean] + property :filter_search_state_fields, default: false + ## # Create collections of solr field configurations. # This will create array-like accessor methods for diff --git a/lib/blacklight/parameters.rb b/lib/blacklight/parameters.rb index ec4b46b796..f9587672d4 100644 --- a/lib/blacklight/parameters.rb +++ b/lib/blacklight/parameters.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Blacklight - module Parameters + class Parameters ## # Sanitize the search parameters by removing unnecessary parameters # from the provided parameters. @@ -9,5 +9,88 @@ def self.sanitize params params.reject { |_k, v| v.nil? } .except(:action, :controller, :id, :commit, :utf8) end + + # rubocop:disable Naming/MethodParameterName + # Merge two Rails strong_params-style permissions into a single list of permitted parameters, + # deep-merging complex values as needed. + # @param [Array] a + # @param [Array] b + # @return [Array] + def self.deep_merge_permitted_params(a, b) + a = [a] if a.is_a? Hash + b = [b] if b.is_a? Hash + + complex_params_from_a, scalar_params_from_a = a.flatten.uniq.partition { |x| x.is_a? Hash } + complex_params_from_a = complex_params_from_a.inject({}) { |tmp, h| _deep_merge_permitted_param_hashes(h, tmp) } + complex_params_from_b, scalar_params_from_b = b.flatten.uniq.partition { |x| x.is_a? Hash } + complex_params_from_b = complex_params_from_b.inject({}) { |tmp, h| _deep_merge_permitted_param_hashes(h, tmp) } + + (scalar_params_from_a + scalar_params_from_b + [_deep_merge_permitted_param_hashes(complex_params_from_a, complex_params_from_b)]).reject(&:blank?).uniq + end + + private_class_method def self._deep_merge_permitted_param_hashes(h1, h2) + h1.merge(h2) do |_key, old_value, new_value| + if (old_value.is_a?(Hash) && old_value.empty?) || (new_value.is_a?(Hash) && new_value.empty?) + {} + elsif old_value.is_a?(Hash) && new_value.is_a?(Hash) + _deep_merge_permitted_param_hashes(old_value, new_value) + elsif old_value.is_a?(Array) || new_value.is_a?(Array) + deep_merge_permitted_params(old_value, new_value) + else + new_value + end + end + end + # rubocop:enable Naming/MethodParameterName + + attr_reader :params, :search_state + + delegate :blacklight_config, :filter_fields, to: :search_state + + def initialize(params, search_state) + @params = params.is_a?(Hash) ? params.with_indifferent_access : params + @search_state = search_state + end + + # @param [Hash] params with unknown structure (not declared in the blacklight config or filters) stripped out + def permit_search_params + # if the parameters were generated internally, we can (probably) trust that they're fine + return params unless params.is_a?(ActionController::Parameters) + + # if the parameters were permitted already, we should be able to trust them + return params if params.permitted? + + permitted_params = filter_fields.inject(blacklight_config.search_state_fields) do |allowlist, filter| + Blacklight::Parameters.deep_merge_permitted_params(allowlist, filter.permitted_params) + end + + deep_unmangle_params!(params, permitted_params) + + if blacklight_config.filter_search_state_fields + params.permit(*permitted_params) + else + params.deep_dup.permit! + end + end + + private + + # Facebook's crawler turns array query parameters into a hash with numeric keys. Once we know + # the expected parameter structure, we can unmangle those parameters to match our expected values. + def deep_unmangle_params!(params, permitted_params) + permitted_params.select { |p| p.is_a?(Hash) }.each do |permission| + permission.each do |key, permitted_value| + if params[key].is_a?(ActionController::Parameters) && permitted_value.is_a?(Hash) + deep_unmangle_params!(params[key], [permitted_value]) + elsif permitted_value.is_a?(Array) && permitted_value.empty? + if params[key].is_a?(ActionController::Parameters) && params[key]&.keys&.all? { |k| k.to_s =~ /\A\d+\z/ } + params[key] = params[key].values + elsif params[key].is_a?(String) + params[key] = Array(params[key]) + end + end + end + end + end end end diff --git a/lib/blacklight/search_builder.rb b/lib/blacklight/search_builder.rb index 5ea9a800d0..bd371fbced 100644 --- a/lib/blacklight/search_builder.rb +++ b/lib/blacklight/search_builder.rb @@ -48,7 +48,7 @@ def where(conditions) Deprecation.warn(Blacklight::SearchBuilder, "SearchBuilder#where must be called with a hash, received #{conditions.inspect}.") unless conditions.is_a? Hash params_will_change! @search_state = @search_state.reset(@search_state.params.merge(q: conditions)) - @blacklight_params = @search_state.params.dup + @blacklight_params = @search_state.params @additional_filters = conditions self end diff --git a/lib/blacklight/search_state.rb b/lib/blacklight/search_state.rb index 84d3b6d6ff..7e2343fb6b 100644 --- a/lib/blacklight/search_state.rb +++ b/lib/blacklight/search_state.rb @@ -19,71 +19,17 @@ class SearchState delegate :facet_configuration_for_field, to: :blacklight_config - def self.modifiable_params(params) - if params.respond_to?(:to_unsafe_h) - # This is the typical (not-ActionView::TestCase) code path. - params = params.to_unsafe_h - # In Rails 5 to_unsafe_h returns a HashWithIndifferentAccess, in Rails 4 it returns Hash - params = params.with_indifferent_access if params.instance_of? Hash - elsif params.is_a? Hash - # This is an ActionView::TestCase workaround for Rails 4.2. - params = params.dup.with_indifferent_access - else - params = params.dup.to_h.with_indifferent_access - end - params - end - # @param [ActionController::Parameters] params # @param [Blacklight::Config] blacklight_config # @param [ApplicationController] controller used for the routing helpers def initialize(params, blacklight_config, controller = nil) @blacklight_config = blacklight_config @controller = controller - @params = SearchState.modifiable_params(params) - normalize_params! if needs_normalization? - end - - def needs_normalization? - return false if params.blank? - return true if (params.keys.map(&:to_s) - permitted_fields.map(&:to_s)).present? - - !!filters.detect { |filter| filter.values.detect { |value| filter.needs_normalization?(value) } } - end - - def normalize_params! - @params = normalize_params - end - - def normalize_params - return params unless needs_normalization? - - base_params = params.slice(*blacklight_config.search_state_fields) - normal_state = blacklight_config.facet_fields.each_value.inject(reset(base_params)) do |working_state, filter_key| - f = filter(filter_key) - next working_state unless f.any? - - filter_values = f.values(except: [:inclusive_filters]).inject([]) do |memo, filter_value| - # flatten arrays that had been mangled into integer-indexed hashes - memo.concat([f.normalize(filter_value)].flatten) - end - filter_values = f.values(except: [:filters, :missing]).inject(filter_values) do |memo, filter_value| - memo << f.normalize(filter_value) - end - filter_values.inject(working_state) do |memo, filter_value| - memo.filter(filter_key).add(filter_value) - end - end - normal_state.params - end - - def permitted_fields - filter_keys = filter_fields.inject(Set.new) { |memo, filter| memo.merge [filter.filters_key, filter.inclusive_filters_key] } - blacklight_config.search_state_fields + filter_keys.subtract([nil, '']).to_a + @params = Blacklight::Parameters.new(params, self).permit_search_params.to_h.with_indifferent_access end def to_hash - @params.deep_dup + params.deep_dup end alias to_h to_hash @@ -132,7 +78,7 @@ def filter_params # @return [Blacklight::SearchState] def reset(params = nil) - self.class.new(params || ActionController::Parameters.new, blacklight_config, controller) + self.class.new(params || {}, blacklight_config, controller) end # @return [Blacklight::SearchState] @@ -162,6 +108,10 @@ def remove_query_params p end + def filter_fields + blacklight_config.facet_fields.each_value.map { |value| filter(value) } + end + def filters @filters ||= filter_fields.select(&:any?) end @@ -192,7 +142,7 @@ def add_facet_params(field, item) # catalog/index with their new facet choice. def add_facet_params_and_redirect(field, item) new_params = Deprecation.silence(self.class) do - add_facet_params(field, item) + add_facet_params(field, item).to_h.with_indifferent_access end # Delete any request params from facet-specific action, needed @@ -229,7 +179,7 @@ def has_facet?(config, value: nil) # @yield [params] The merged parameters hash before being sanitized def params_for_search(params_to_merge = {}) # params hash we'll return - my_params = params.dup.merge(self.class.new(params_to_merge, blacklight_config, controller)) + my_params = to_h.merge(self.class.new(params_to_merge, blacklight_config, controller)) if block_given? yield my_params @@ -297,11 +247,7 @@ def facet_request_keys # and need to be reset when e.g. constraints change # @return [ActionController::Parameters] def reset_search_params - Parameters.sanitize(params).except(:page, :counter) - end - - def filter_fields - blacklight_config.facet_fields.each_value.map { |value| filter(value) } + Parameters.sanitize(to_h).except(:page, :counter) end end end diff --git a/lib/blacklight/search_state/filter_field.rb b/lib/blacklight/search_state/filter_field.rb index 4b20183043..9e3a698ad5 100644 --- a/lib/blacklight/search_state/filter_field.rb +++ b/lib/blacklight/search_state/filter_field.rb @@ -159,12 +159,18 @@ def include?(item) end end - def needs_normalization?(value_params) - value_params&.is_a?(Hash) && value_params != Blacklight::SearchState::FilterField::MISSING - end - - def normalize(value_params) - needs_normalization?(value_params) ? value_params.values : value_params + def permitted_params + if config.pivot + { + filters_key => config.pivot.each_with_object({}) { |key, filter| filter.merge(key => [], "-#{key}" => []) }, + inclusive_filters_key => config.pivot.each_with_object({}) { |key, filter| filter.merge(key => []) } + } + else + { + filters_key => { config.key => [], "-#{config.key}" => [] }, + inclusive_filters_key => { config.key => [] } + } + end end private diff --git a/spec/helpers/blacklight/render_constraints_helper_behavior_spec.rb b/spec/helpers/blacklight/render_constraints_helper_behavior_spec.rb index e505dd758b..891ea65273 100644 --- a/spec/helpers/blacklight/render_constraints_helper_behavior_spec.rb +++ b/spec/helpers/blacklight/render_constraints_helper_behavior_spec.rb @@ -27,7 +27,7 @@ let(:params) { ActionController::Parameters.new(q: 'foobar', f: { type: 'journal' }) } it "has a link relative to the current url" do - expect(subject).to have_link 'Remove constraint', href: '/catalog?f%5Btype%5D=journal' + expect(subject).to have_link 'Remove constraint', href: '/catalog?f%5Btype%5D%5B%5D=journal' end end diff --git a/spec/lib/blacklight/parameters_spec.rb b/spec/lib/blacklight/parameters_spec.rb index ecf25518d5..3eab91cc85 100644 --- a/spec/lib/blacklight/parameters_spec.rb +++ b/spec/lib/blacklight/parameters_spec.rb @@ -21,4 +21,88 @@ end end end + + describe '.deep_merge_permitted_params' do + it 'merges scalar values' do + expect(described_class.deep_merge_permitted_params([:a], [:b])).to eq [:a, :b] + end + + it 'appends complex values' do + expect(described_class.deep_merge_permitted_params([:a], { b: [] })).to eq [:a, { b: [] }] + end + + it 'merges lists of scalar values' do + expect(described_class.deep_merge_permitted_params({ f: [:a, :b] }, { f: [:b, :c] })).to eq [{ f: [:a, :b, :c] }] + end + + it 'merges complex value data structures' do + expect(described_class.deep_merge_permitted_params([{ f: { field1: [] } }], { f: { field2: [] } })).to eq [{ f: { field1: [], field2: [] } }] + end + + it 'takes the most permissive value' do + expect(described_class.deep_merge_permitted_params([{ f: {} }], { f: { field2: [] } })).to eq [{ f: {} }] + expect(described_class.deep_merge_permitted_params([{ f: {} }], { f: [:some_value] })).to eq [{ f: {} }] + end + end + + describe '#permit_search_params' do + subject(:params) { described_class.new(query_params, search_state) } + + let(:query_params) { ActionController::Parameters.new(a: 1, b: 2, c: []) } + let(:search_state) { Blacklight::SearchState.new(query_params, blacklight_config) } + let(:blacklight_config) { Blacklight::Configuration.new } + + context 'with facebooks badly mangled query parameters' do + let(:query_params) do + ActionController::Parameters.new( + f: { field: { '0': 'first', '1': 'second' } }, + f_inclusive: { field: { '0': 'first', '1': 'second' } } + ) + end + + before do + blacklight_config.add_facet_field 'field' + end + + it 'normalizes the facets to the expected format' do + expect(params.permit_search_params.to_h.with_indifferent_access).to include f: { field: %w[first second] }, f_inclusive: { field: %w[first second] } + end + end + + context 'with filter_search_state_fields set to false' do + let(:blacklight_config) { Blacklight::Configuration.new(filter_search_state_fields: false) } + + it 'allows all params' do + expect(params.permit_search_params.to_h.with_indifferent_access).to include(a: 1, b: 2, c: []) + end + end + + context 'with filter_search_state_fields set to true' do + let(:blacklight_config) { Blacklight::Configuration.new(filter_search_state_fields: true) } + + it 'rejects unknown params' do + expect(params.permit_search_params.to_h).to be_empty + end + + context 'with some search parameters' do + let(:query_params) { ActionController::Parameters.new(q: 'abc', page: 5, f: { facet_field: %w[a b], unknown_field: ['a'] }) } + + before do + blacklight_config.add_facet_field 'facet_field' + end + + it 'allows scalar params' do + expect(params.permit_search_params.to_h.with_indifferent_access).to include(q: 'abc', page: 5) + end + + it 'allows facet params' do + expect(params.permit_search_params.to_h.with_indifferent_access).to include(f: { facet_field: %w[a b] }) + end + + it 'removes unknown facet fields parameters' do + expect(params.permit_search_params.to_h.with_indifferent_access[:f]).not_to include(:unknown_field) + end + end + end + end end diff --git a/spec/lib/blacklight/search_state/filter_field_spec.rb b/spec/lib/blacklight/search_state/filter_field_spec.rb index 5d997c288b..3f2a10de57 100644 --- a/spec/lib/blacklight/search_state/filter_field_spec.rb +++ b/spec/lib/blacklight/search_state/filter_field_spec.rb @@ -6,6 +6,7 @@ let(:params) { { f: { some_field: %w[1 2], another_field: ['3'] } } } let(:blacklight_config) do Blacklight::Configuration.new.configure do |config| + config.add_facet_field 'new_field' config.add_facet_field 'another_field', single: true simple_facet_fields.each { |simple_facet_field| config.add_facet_field simple_facet_field } config.search_state_fields = config.search_state_fields + additional_search_fields @@ -23,14 +24,6 @@ expect(new_state.filter('some_field').values).to eq %w[1 2 4] end - it 'creates new parameter as needed' do - filter = search_state.filter('unknown_field') - new_state = filter.add('4') - - expect(new_state.filter('unknown_field').values).to eq %w[4] - expect(new_state.params[:f]).to include(:unknown_field) - end - context 'without any parameters in the url' do let(:params) { {} } @@ -205,10 +198,4 @@ expect(search_state.filter('some_field').include?(OpenStruct.new(value: '1'))).to eq true end end - - describe '#needs_normalization?' do - it 'returns false for Blacklight::SearchState::FilterField::MISSING' do - expect(search_state.filter('some_field').needs_normalization?(Blacklight::SearchState::FilterField::MISSING)).to be false - end - end end diff --git a/spec/lib/blacklight/search_state_spec.rb b/spec/lib/blacklight/search_state_spec.rb index 37d548d68c..d8208fa8fa 100644 --- a/spec/lib/blacklight/search_state_spec.rb +++ b/spec/lib/blacklight/search_state_spec.rb @@ -54,19 +54,6 @@ end end - context 'with facebooks badly mangled query parameters' do - let(:simple_facet_fields) { [:field] } - let(:params) do - { f: { field: { '0': 'first', '1': 'second' } }, - f_inclusive: { field: { '0': 'first', '1': 'second' } } } - end - - it 'normalizes the facets to the expected format' do - expect(search_state.to_h).to include f: { field: %w[first second] } - expect(search_state.to_h).to include f_inclusive: { field: %w[first second] } - end - end - context 'deleting item from to_h' do let(:additional_search_fields) { [:q_1] } let(:params) { { q: 'foo', q_1: 'bar' } } @@ -81,7 +68,7 @@ end context 'deleting deep item from to_h' do - let(:additional_search_fields) { [:foo] } + let(:additional_search_fields) { [{ foo: {} }] } let(:params) { { foo: { bar: [] } } } it 'does not mutate search_state to deep mutate search_state.to_h' do diff --git a/spec/models/blacklight/solr/search_builder_spec.rb b/spec/models/blacklight/solr/search_builder_spec.rb index af206d0c7f..e1262b78e5 100644 --- a/spec/models/blacklight/solr/search_builder_spec.rb +++ b/spec/models/blacklight/solr/search_builder_spec.rb @@ -216,7 +216,9 @@ expect(subject["spellcheck.q"]).to be_blank single_facet.each_value do |value| - expect(subject[:fq]).to include("{!term f=#{single_facet.keys[0]}}#{value}") + Array(value).each do |v| + expect(subject[:fq]).to include("{!term f=#{single_facet.keys[0]}}#{v}") + end end end end From 9644976c01ad7ab4d785ca88b78607a60613cdfa Mon Sep 17 00:00:00 2001 From: Chris Beer Date: Wed, 11 May 2022 13:34:36 -0700 Subject: [PATCH 2/2] Add a deprecation warning when using unpermitted parameters --- blacklight.gemspec | 1 + lib/blacklight/parameters.rb | 13 +++++++++++++ spec/lib/blacklight/parameters_spec.rb | 5 ++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/blacklight.gemspec b/blacklight.gemspec index 42940a288b..510b31085a 100644 --- a/blacklight.gemspec +++ b/blacklight.gemspec @@ -33,6 +33,7 @@ Gem::Specification.new do |s| s.add_dependency "i18n", '>= 1.7.0' # added named parameters s.add_dependency "ostruct", '>= 0.3.2' s.add_dependency "view_component", '~> 2.43' + s.add_dependency 'hashdiff' s.add_development_dependency "rsolr", ">= 1.0.6", "< 3" # Library for interacting with rSolr. s.add_development_dependency "rspec-rails", "~> 5.0" diff --git a/lib/blacklight/parameters.rb b/lib/blacklight/parameters.rb index f9587672d4..12e6552dc4 100644 --- a/lib/blacklight/parameters.rb +++ b/lib/blacklight/parameters.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true + +require 'hashdiff' + module Blacklight class Parameters + extend Deprecation + ## # Sanitize the search parameters by removing unnecessary parameters # from the provided parameters. @@ -69,12 +74,20 @@ def permit_search_params if blacklight_config.filter_search_state_fields params.permit(*permitted_params) else + warn_about_deprecated_parameter_handling(params, permitted_params) params.deep_dup.permit! end end private + def warn_about_deprecated_parameter_handling(params, permitted_params) + diff = Hashdiff.diff(params.to_unsafe_h, params.permit(*permitted_params).to_h) + return if diff.empty? + + Deprecation.warn(Blacklight::Parameters, "Blacklight 8 will filter out non-search parameter, including: #{diff.map { |_op, key, *| key }.to_sentence}") + end + # Facebook's crawler turns array query parameters into a hash with numeric keys. Once we know # the expected parameter structure, we can unmangle those parameters to match our expected values. def deep_unmangle_params!(params, permitted_params) diff --git a/spec/lib/blacklight/parameters_spec.rb b/spec/lib/blacklight/parameters_spec.rb index 3eab91cc85..722f620f12 100644 --- a/spec/lib/blacklight/parameters_spec.rb +++ b/spec/lib/blacklight/parameters_spec.rb @@ -72,8 +72,11 @@ context 'with filter_search_state_fields set to false' do let(:blacklight_config) { Blacklight::Configuration.new(filter_search_state_fields: false) } - it 'allows all params' do + it 'allows all params, but warns about the behavior' do + allow(Deprecation).to receive(:warn) expect(params.permit_search_params.to_h.with_indifferent_access).to include(a: 1, b: 2, c: []) + + expect(Deprecation).to have_received(:warn).with(described_class, /including: a, b, and c/).at_least(:once) end end