diff --git a/README.md b/README.md index 245cb6305..55744dcc7 100644 --- a/README.md +++ b/README.md @@ -1090,6 +1090,27 @@ end A parent with several such children sorts by the smallest value ascending and the largest descending. `:score => :min`, `:max`, `:avg` or `:total` combines them another way. Solr compares the values as floats, so times less than about two minutes apart, and integers above 2**24, can tie. Solr also reads a negative value as zero, so times before 1970 all sort as 1970, and negative numbers as zero. A later `order_by` breaks the tie. +#### Faceting on a child field + +**Solr 7.7 and above**, for the `uniqueBlock` count: Solr 7.7 has it and 7.3 doesn't. The specs run it on Solr 8 and above. + +`child_facet` facets on a field of the children, counting the parents that have each value rather than the children, and ranking the values by that count, so `:limit` keeps the values most parents have. It counts only the children of parents matching the search, and only those meeting the block's restrictions, the same restrictions `with_child` takes. A `with_child` on the search doesn't restrict the facet's children, so repeat any condition they need in the block. + +```ruby +# The names of the milestones Ana owns on active projects, each with its number of projects +search = Project.search do + with :status, 'active' + child_facet :milestones, :name, :matches => 'des.*' do + with :owner_names, 'ana' + end +end +search.facet(:name).rows.map { |row| [row.value, row.count] } # => [["design", 1]] +``` + +* The facet is read with `facet`, by the field's name or by `:name`. A search with another facet of the same name raises `ArgumentError` unless `child_facet` has a `:name` of its own. +* Options: `:limit`, `:offset`, `:prefix`, `:missing`, `:all_buckets`, `:sort` (`:count`, the default, which ranks by the number of parents, or `:index`) with `:sort_type`, `:minimum_count`, counted in parents, and `:matches`. Any other option raises `ArgumentError`. +* `:matches` is a Lucene regular expression, not a Ruby one, matched against the whole value of a string field, case-sensitively. `Sunspot::Util.escape_lucene_regexp` escapes text to match it literally, e.g. text a user typed. On a multiple-value field, `:matches` selects children, so a matching child's other values are counted too. + #### Schema The schema needs a `_root_` field with the same type as `id`. The bundled configset has one. diff --git a/sunspot/lib/sunspot/dsl/field_query.rb b/sunspot/lib/sunspot/dsl/field_query.rb index 52933253e..11287f436 100644 --- a/sunspot/lib/sunspot/dsl/field_query.rb +++ b/sunspot/lib/sunspot/dsl/field_query.rb @@ -388,6 +388,41 @@ def json_facet(*field_names) end end + # + # Facet on a field of the children in a nested association, counting the + # documents that have each value rather than the children, and ranking + # the values by that count. Only the children of documents matching the + # search are counted, and only those that satisfy every restriction in + # the block, which takes the same restrictions as DSL::Scope#with_child. + # + # The facet is read as search.facet(field_name), or by its + # +:name+, which it needs when the search has another facet of that + # name. Options: +:limit+, +:offset+, +:prefix+, +:missing+, + # +:all_buckets+, +:sort+ (+:count+, the default, or +:index+) with + # +:sort_type+, +:minimum_count+ (of documents), and +:matches+: a + # Lucene regular expression the child's value of a string field must + # match as a whole, case-sensitively. On a multiple-value field, + # +:matches+ selects children, so a matching child's other values are + # counted too. Util.escape_lucene_regexp escapes text to match literally. + # + # ==== Example + # + # search = Sunspot.search(Project) do + # with :status, 'active' + # child_facet :milestones, :name, :matches => 'des.*' do + # with :owner_names, 'ana' + # end + # end + # search.facet(:name).rows # each row's count is a number of projects + # + def child_facet(name, field_name, options = {}, &block) + block_join, child_setup = build_block_join(false, name, &block) + field = child_setup.field(field_name.to_sym) + facet = Sunspot::Query::ChildFieldJsonFacet.new(field, options, child_setup, block_join) + @search.add_child_json_facet(field, options) + @query.add_query_facet(facet) + end + def stats(*field_names, &block) options = Sunspot::Util.extract_options_from(field_names) diff --git a/sunspot/lib/sunspot/query.rb b/sunspot/lib/sunspot/query.rb index 6cdae57d7..c4d1437c8 100644 --- a/sunspot/lib/sunspot/query.rb +++ b/sunspot/lib/sunspot/query.rb @@ -1,5 +1,5 @@ %w(filter abstract_field_facet abstract_json_field_facet connective boost_query date_field_facet field_json_facet - range_facet range_json_facet date_field_json_facet abstract_fulltext dismax join block_join + range_facet range_json_facet date_field_json_facet abstract_fulltext dismax join block_join child_field_json_facet field_list field_facet highlighting pagination restriction common_query spellcheck standard_query more_like_this more_like_this_query geo geofilt bbox query_facet scope sort sort_composite text_field_boost function_query field_stats diff --git a/sunspot/lib/sunspot/query/abstract_json_field_facet.rb b/sunspot/lib/sunspot/query/abstract_json_field_facet.rb index 34e094973..8c6d8c2e4 100644 --- a/sunspot/lib/sunspot/query/abstract_json_field_facet.rb +++ b/sunspot/lib/sunspot/query/abstract_json_field_facet.rb @@ -1,3 +1,5 @@ +require 'json' # to_params serializes the facet with to_json + module Sunspot module Query class AbstractJsonFieldFacet diff --git a/sunspot/lib/sunspot/query/block_join.rb b/sunspot/lib/sunspot/query/block_join.rb index 47f158aac..7910896ae 100644 --- a/sunspot/lib/sunspot/query/block_join.rb +++ b/sunspot/lib/sunspot/query/block_join.rb @@ -43,12 +43,14 @@ def negate self.class.new(@nested_setup, !negated?, @scope, @paths) end - private - + # The query matching the children this block join is about: those in + # its association that satisfy every restriction in #scope. def child_query @scope.to_boolean_phrase end + private + def escape(value) self.class.escape(value) end diff --git a/sunspot/lib/sunspot/query/child_field_json_facet.rb b/sunspot/lib/sunspot/query/child_field_json_facet.rb new file mode 100644 index 000000000..d19de9093 --- /dev/null +++ b/sunspot/lib/sunspot/query/child_field_json_facet.rb @@ -0,0 +1,55 @@ +module Sunspot + module Query + # + # A terms facet over the children of the matching documents: the children + # in one nested association that satisfy a block join's restrictions. Each + # row counts the documents with such a child, not the children, and the + # rows are ranked by that count. + # + # With +:matches+, only children whose value of the field matches it are + # counted. It is a Lucene regular expression, matched against the whole + # indexed value, case-sensitively. Util.escape_lucene_regexp escapes text + # to match literally. + # + class ChildFieldJsonFacet < FieldJsonFacet + OPTIONS = [:name, :limit, :offset, :sort, :sort_type, :minimum_count, :prefix, :missing, :all_buckets, :matches].freeze + + # The count each row reports: the number of parents with such a child + PARENT_COUNT = 'distinct'.freeze + + attr_reader :name + + def initialize(field, options, setup, block_join) + unknown = options.keys - OPTIONS + raise ArgumentError, "child_facet doesn't take #{unknown.map(&:inspect).join(', ')}" if unknown.any? + unless field.type.is_a?(Sunspot::Type::StringType) || !options.key?(:matches) + raise ArgumentError, "child_facet's :matches needs a string field, but #{field.name} isn't one" + end + raise ArgumentError, "child_facet's :matches takes a Lucene regular expression as a String" if options.key?(:matches) && !options[:matches].is_a?(String) + + super(field, options, setup) + @block_join = block_join + @name = (options[:name] || field.name).to_sym + end + + def field_name_with_local_params + params = init_params.merge(:type => 'terms', :field => @field.indexed_name) + # Solr ranks buckets by its own count, the children, unless told to use the parent count + params[:sort] = { PARENT_COUNT => @options[:sort_type] || 'desc' } if @options[:sort].nil? || @options[:sort] == :count + params.delete(:mincount) # Solr's mincount counts children; rows apply :minimum_count to parents + + filters = [@block_join.child_query] + filters << "#{@field.indexed_name}:/#{@options[:matches].gsub('/', '\\/')}/" if @options[:matches] + + { @name => params.merge( + :domain => { :blockChildren => BlockJoin::ROOTS, :filter => filters }, + :facet => { PARENT_COUNT => 'uniqueBlock(_root_)' } + ) } + end + + def get_params + field_name_with_local_params + end + end + end +end diff --git a/sunspot/lib/sunspot/search.rb b/sunspot/lib/sunspot/search.rb index 945fb7c4f..c07776c19 100644 --- a/sunspot/lib/sunspot/search.rb +++ b/sunspot/lib/sunspot/search.rb @@ -1,5 +1,5 @@ %w(abstract_search standard_search more_like_this_search query_facet - field_facet field_json_facet date_facet range_facet json_facet_stats + field_facet field_json_facet child_field_json_facet date_facet range_facet json_facet_stats facet_row json_facet_row hit highlight field_group group hit_enumerable stats_row stats_json_row field_stats stats_facet query_group).each do |file| require File.join(File.dirname(__FILE__), 'search', file) diff --git a/sunspot/lib/sunspot/search/abstract_search.rb b/sunspot/lib/sunspot/search/abstract_search.rb index ad71d1c3b..b0cdffb8f 100644 --- a/sunspot/lib/sunspot/search/abstract_search.rb +++ b/sunspot/lib/sunspot/search/abstract_search.rb @@ -268,6 +268,12 @@ def add_json_facet(field, options = {}) add_facet(name, FieldJsonFacet.new(field, self, options)) end + def add_child_json_facet(field, options = {}) #:nodoc: + name = (options[:name] || field.name).to_sym + raise ArgumentError, "The search already has a facet named #{name}; give child_facet a :name of its own" if @facets_by_name.key?(name) + add_facet(name, ChildFieldJsonFacet.new(field, self, options)) + end + def highlights_for(doc) #:nodoc: if @solr_result['highlighting'] @solr_result['highlighting'][doc['id']] diff --git a/sunspot/lib/sunspot/search/child_field_json_facet.rb b/sunspot/lib/sunspot/search/child_field_json_facet.rb new file mode 100644 index 000000000..94fca276a --- /dev/null +++ b/sunspot/lib/sunspot/search/child_field_json_facet.rb @@ -0,0 +1,28 @@ +module Sunspot + module Search + # + # The rows of a Query::ChildFieldJsonFacet, read by its name. Each row's + # count is a number of parents, and the rows come in Solr's order: by that + # count, unless the facet was sorted by :index. + # + class ChildFieldJsonFacet < FieldJsonFacet + def initialize(field, search, options) + super + @name = (options[:name] || field.name).to_sym + end + + def rows + @rows ||= begin + data = no_data? ? [] : @search.json_facet_response[@name.to_s]['buckets'] + rows = data.map {|bucket| JsonFacetRow.new(bucket, self) } + rows = rows.select {|row| row.count >= @options[:minimum_count] } if @options[:minimum_count] + rows + end + end + + def no_data? + @search.json_facet_response[@name.to_s].nil? + end + end + end +end diff --git a/sunspot/lib/sunspot/util.rb b/sunspot/lib/sunspot/util.rb index e83be0378..d244f06b8 100644 --- a/sunspot/lib/sunspot/util.rb +++ b/sunspot/lib/sunspot/util.rb @@ -188,6 +188,15 @@ def escape(value) RSolr.solr_escape(value).gsub(/([\s\.])/, '\\\\\1') end + # + # Escapes text for a Lucene regular expression, such as child_facet's + # +:matches+, so it matches literally. Leaves a slash, which child_facet + # escapes itself. + # + def escape_lucene_regexp(value) + value.to_s.gsub(/[.?+*|{}\[\]()"\\#@&<>~^$]/) {|char| "\\#{char}" } + end + def parse_json_facet(field_name, options, setup) field = setup.field(field_name) if options[:range] || options[:time_range] diff --git a/sunspot/spec/api/nested_documents_spec.rb b/sunspot/spec/api/nested_documents_spec.rb index 15f457154..f3d5fd072 100644 --- a/sunspot/spec/api/nested_documents_spec.rb +++ b/sunspot/spec/api/nested_documents_spec.rb @@ -334,6 +334,68 @@ def last_search end end + describe 'faceting on a child field' do + def json_facet + JSON.parse(connection.searches.last['json.facet']) + end + + it 'facets on the children of the matching parents, ranked by the number of parents' do + session.search(Project) { child_facet(:milestones, :name) { with :owner_names, 'ana' } } + expect(json_facet).to eq('name' => { + 'type' => 'terms', 'field' => 'name_s', 'sort' => { 'distinct' => 'desc' }, + 'domain' => { 'blockChildren' => roots, 'filter' => ['(_sunspot_nested_path_s:"Project.milestones" AND owner_names_sm:ana)'] }, + 'facet' => { 'distinct' => 'uniqueBlock(_root_)' } + }) + end + + it 'is keyed by its :name' do + session.search(Project) { child_facet(:milestones, :name, :name => :milestone_name) } + expect(json_facet.keys).to eq(['milestone_name']) + end + + it 'ranks by the number of parents when sorted by count, and keeps an index sort' do + session.search(Project) { child_facet(:milestones, :name, :sort => :count, :sort_type => 'asc') } + expect(json_facet['name']['sort']).to eq('distinct' => 'asc') + + session.search(Project) { child_facet(:milestones, :name, :sort => :index, :sort_type => 'asc') } + expect(json_facet['name']['sort']).to eq('index' => 'asc') + end + + it 'leaves a minimum count to the rows, since Solr would apply it to the children' do + session.search(Project) { child_facet(:milestones, :name, :minimum_count => 2) } + expect(json_facet['name']).not_to have_key('mincount') + end + + it 'filters the children by :matches, escaping a slash' do + session.search(Project) { child_facet(:milestones, :name, :matches => 'a/b.*') } + expect(json_facet['name']['domain']['filter'].last).to eq('name_s:/a\/b.*/') + end + + it 'reads the children of every searched class that declares the association' do + session.search(Project, Program) { child_facet(:milestones, :name) } + expect(json_facet['name']['domain']['filter'].first).to include('Project.milestones', 'Program.milestones') + end + + it 'raises for an option it does not support' do + expect { session.search(Project) { child_facet(:milestones, :name, :range => [1, 10]) } }.to raise_error(ArgumentError, /:range/) + end + + it 'raises for :matches on a field that is not a string, or given anything but a String' do + expect { session.search(Project) { child_facet(:milestones, :started_at, :matches => '2026.*') } }.to raise_error(ArgumentError, /string field/) + expect { session.search(Project) { child_facet(:milestones, :name, :matches => /des/) } }.to raise_error(ArgumentError, /String/) + end + + it 'raises for a name the search already has a facet by' do + expect { session.search(Project) { json_facet :name; child_facet(:milestones, :name) } }.to raise_error(ArgumentError, /:name of its own/) + end + end + + describe 'escaping a Lucene regular expression' do + it 'escapes every character the syntax reserves, but a slash' do + expect(Sunspot::Util.escape_lucene_regexp('a.b (c) "d" f&g ~h @i #j \\ k/l')).to eq('a\.b \(c\) \"d\" \ f\&g \~h \@i \#j \\\\ k/l') + end + end + describe 'removal' do it 'removes the whole block when removing a parent' do project = Project.new diff --git a/sunspot/spec/integration/nested_documents_spec.rb b/sunspot/spec/integration/nested_documents_spec.rb index 595217e0c..a6d3b57ed 100644 --- a/sunspot/spec/integration/nested_documents_spec.rb +++ b/sunspot/spec/integration/nested_documents_spec.rb @@ -105,6 +105,69 @@ def solr_count(query) end end + describe 'child_facet' do + before { skip_before_solr 8, 'The parent count needs uniqueBlock, which Solr 7.7 has and 7.3 lacks, and skip_before_solr compares major versions' } + + def facet_rows(&block) + Sunspot.search(Project, &block).facet(:name).rows.map { |row| [row.value, row.count] } + end + + it 'counts the parents with a child of each value, not the children' do + Sunspot.index!(Project.new(:name => 'built twice', :status => 'active', :milestones => [milestone('build', Time.utc(2026, 3, 1)), milestone('build', Time.utc(2026, 5, 1))])) + + expect(facet_rows { child_facet(:milestones, :name) }).to match_array([['design', 2], ['build', 2]]) + end + + it 'counts only the children meeting every condition in the block' do + expect(facet_rows { child_facet(:milestones, :name) { with :owner_names, 'ana' } }).to match_array([['design', 1], ['build', 1]]) + end + + it 'counts only the children of parents matching the search' do + expect(facet_rows { with :name, 'designed earlier'; child_facet(:milestones, :name) }).to match_array([['design', 1], ['build', 1]]) + end + + it 'counts only the children whose value matches the regular expression' do + expect(facet_rows { child_facet(:milestones, :name, :matches => 'des.*') }).to eq([['design', 2]]) + end + + it 'matches a value containing a slash' do + Sunspot.index!(Project.new(:name => 'slashed', :status => 'active', :milestones => [milestone('a/b', Time.utc(2026, 3, 1))])) + + expect(facet_rows { child_facet(:milestones, :name, :matches => 'a/b') }).to eq([['a/b', 1]]) + end + + it 'ranks the values by their number of parents, so a limit keeps the values most parents have' do + Sunspot.index!(Project.new(:name => 'reviewed thrice', :status => 'active', :milestones => [milestone('review', Time.utc(2026, 3, 1)), milestone('review', Time.utc(2026, 4, 1)), milestone('review', Time.utc(2026, 5, 1))])) + + expect(facet_rows { child_facet(:milestones, :name, :limit => 1) }).to eq([['design', 2]]) + end + + it 'applies a minimum count to the number of parents' do + Sunspot.index!(Project.new(:name => 'built thrice', :status => 'active', :milestones => [milestone('build', Time.utc(2026, 3, 1)), milestone('build', Time.utc(2026, 4, 1)), milestone('build', Time.utc(2026, 5, 1))])) + + expect(facet_rows { child_facet(:milestones, :name, :minimum_count => 3) }).to eq([]) + expect(facet_rows { child_facet(:milestones, :name, :minimum_count => 2) }).to match_array([['design', 2], ['build', 2]]) + end + + it 'keeps two facets on the same field apart by their :name' do + search = Sunspot.search(Project) do + child_facet(:milestones, :name, :name => :anas) { with :owner_names, 'ana' } + child_facet(:milestones, :name, :name => :bens) { with :owner_names, 'ben' } + end + + expect(search.facet(:anas).rows.map { |row| [row.value, row.count] }).to match_array([['design', 1], ['build', 1]]) + expect(search.facet(:bens).rows.map { |row| [row.value, row.count] }).to eq([['design', 1]]) + end + + it 'counts the other values of a multiple-value field on a child that matches' do + Sunspot.index!(Project.new(:name => 'shared', :status => 'active', :milestones => [milestone('plan', Time.utc(2026, 3, 1), :owner_names => %w(cy dee))])) + + rows = Sunspot.search(Project) { child_facet(:milestones, :owner_names, :matches => 'd.*') }.facet(:owner_names).rows + + expect(rows.map(&:value)).to match_array(['cy', 'dee']) + end + end + describe 'order_by_child' do before { skip_before_solr 8, 'Solr 6 and earlier reject the sort, and Solr 7 orders some parents differently' }