Kouhei Sutou
null+****@clear*****
Mon Jan 22 15:40:45 JST 2018
Kouhei Sutou 2018-01-22 15:40:45 +0900 (Mon, 22 Jan 2018) New Revision: 7522fab9cc63f0894aa573f00216af27f6f2bf85 https://github.com/groonga/groonga/commit/7522fab9cc63f0894aa573f00216af27f6f2bf85 Message: logical_range_filter: add post_filter Added files: test/command/suite/sharding/logical_range_filter/post_filter/filtered_column.expected test/command/suite/sharding/logical_range_filter/post_filter/filtered_column.test test/command/suite/sharding/logical_range_filter/post_filter/no_filter.expected test/command/suite/sharding/logical_range_filter/post_filter/no_filter.test test/command/suite/sharding/logical_range_filter/post_filter/no_filtered_column.expected test/command/suite/sharding/logical_range_filter/post_filter/no_filtered_column.test Modified files: plugins/sharding/logical_range_filter.rb Modified: plugins/sharding/logical_range_filter.rb (+25 -1) =================================================================== --- plugins/sharding/logical_range_filter.rb 2018-01-22 14:27:57 +0900 (cbb1545b7) +++ plugins/sharding/logical_range_filter.rb 2018-01-22 15:40:45 +0900 (2435bc706) @@ -15,6 +15,7 @@ module Groonga "limit", "output_columns", "use_range_index", + "post_filter", ]) def run_body(input) @@ -71,6 +72,7 @@ module Groonga key << "#{input[:limit]}\0" key << "#{input[:output_columns]}\0" key << "#{input[:use_range_index]}\0" + key << "#{input[:post_filter]}\0" dynamic_columns = DynamicColumns.parse(input) key << dynamic_columns.cache_key key @@ -90,6 +92,7 @@ module Groonga attr_reader :unsorted_result_sets attr_reader :temporary_tables attr_reader :threshold + attr_reader :post_filter def initialize(input) @input = input @use_range_index = parse_use_range_index(@input[:use_range_index]) @@ -109,6 +112,8 @@ module Groonga @temporary_tables = [] @threshold = compute_threshold + + @post_filter = @input[:post_filter] end def close @@ -210,6 +215,7 @@ module Groonga @target_table =****@shard***** @filter =****@conte***** + @post_filter =****@conte*****_filter @result_sets =****@conte*****_sets @unsorted_result_sets =****@conte*****_result_sets @@ -305,6 +311,11 @@ module Groonga return decide_use_range_index(false, reason, __LINE__, __method__) end + unles****@conte*****_filter.nil? + reason = "post_filter is used" + return decide_use_range_index(false, reason, __LINE__, __method__) + end + current_limit =****@conte*****_limit if current_limit < 0 reason = "limit is negative: <#{current_limit}>" @@ -482,7 +493,7 @@ module Groonga def filter_shard_all(range_index, expression_builder) table = @target_table if****@filte*****? - if table.size <=****@conte*****_offset + if @post_filter.nil? and table.size <=****@conte*****_offset @context.current_offset -= table.size return end @@ -632,6 +643,13 @@ module Groonga end end + def apply_post_filter(table) + create_expression(table) do |expression| + expression.parse(@post_filter) + table.select(expression) + end + end + def sort_result_set(result_set) @context.temporary_tables.delete(result_set) @@ -647,6 +665,12 @@ module Groonga dynamic_column.apply(result_set) end + unless @post_filter.nil? + filtered_table = result_set + result_set = apply_post_filter(filtered_table) + @context.temporary_tables << filtered_table + end + if result_set.size <=****@conte*****_offset @context.current_offset -= result_set.size result_set.close if result_set.temporary? Added: test/command/suite/sharding/logical_range_filter/post_filter/filtered_column.expected (+94 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/post_filter/filtered_column.expected 2018-01-22 15:40:45 +0900 (1c7c4b13b) @@ -0,0 +1,94 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --columns[filtered_id].stage filtered --columns[filtered_id].type UInt32 --columns[filtered_id].flags COLUMN_SCALAR --columns[filtered_id].value '_id' --filter 'price <= 900' --post_filter 'filtered_id > 1' --offset 1 --limit 3 --output_columns _id,filtered_id,price +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + "_id", + "UInt32" + ], + [ + "filtered_id", + "UInt32" + ], + [ + "price", + "UInt32" + ] + ], + [ + 3, + 3, + 300 + ], + [ + 2, + 2, + 520 + ], + [ + 3, + 3, + 110 + ] + ] +] +#|d| [logical_range_filter][select] <Logs_20170315>: dynamic columns are used +#|d| [logical_range_filter][select] <Logs_20170316>: dynamic columns are used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/post_filter/filtered_column.test (+57 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/post_filter/filtered_column.test 2018-01-22 15:40:45 +0900 (d0f943358) @@ -0,0 +1,57 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --columns[filtered_id].stage filtered \ + --columns[filtered_id].type UInt32 \ + --columns[filtered_id].flags COLUMN_SCALAR \ + --columns[filtered_id].value '_id' \ + --filter 'price <= 900' \ + --post_filter 'filtered_id > 1' \ + --offset 1 \ + --limit 3 \ + --output_columns _id,filtered_id,price +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/post_filter/no_filter.expected (+58 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/post_filter/no_filter.expected 2018-01-22 15:40:45 +0900 (c556a2ffb) @@ -0,0 +1,58 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --post_filter '_id < 3' --offset 4 --limit 1 --output_columns _id,price +[[0,0.0,0.0],[[["_id","UInt32"],["price","UInt32"]],[1,800]]] +#|d| [logical_range_filter][select] <Logs_20170315>: post_filter is used +#|d| [logical_range_filter][select] <Logs_20170316>: post_filter is used +#|d| [logical_range_filter][select] <Logs_20170317>: post_filter is used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/post_filter/no_filter.test (+52 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/post_filter/no_filter.test 2018-01-22 15:40:45 +0900 (5d43fb054) @@ -0,0 +1,52 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --post_filter '_id < 3' \ + --offset 4 \ + --limit 1 \ + --output_columns _id,price +log_level --level notice +#@remove-important-log-levels debug Added: test/command/suite/sharding/logical_range_filter/post_filter/no_filtered_column.expected (+57 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/post_filter/no_filtered_column.expected 2018-01-22 15:40:45 +0900 (56243c345) @@ -0,0 +1,57 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_range_filter Logs --shard_key timestamp --filter 'price <= 900' --post_filter '_id > 1' --offset 1 --limit 3 --output_columns _id,price +[[0,0.0,0.0],[[["_id","UInt32"],["price","UInt32"]],[3,300],[2,520],[3,110]]] +#|d| [logical_range_filter][select] <Logs_20170315>: post_filter is used +#|d| [logical_range_filter][select] <Logs_20170316>: post_filter is used +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_range_filter/post_filter/no_filtered_column.test (+53 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_range_filter/post_filter/no_filtered_column.test 2018-01-22 15:40:45 +0900 (5fec7fca6) @@ -0,0 +1,53 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_range_filter Logs \ + --shard_key timestamp \ + --filter 'price <= 900' \ + --post_filter '_id > 1' \ + --offset 1 \ + --limit 3 \ + --output_columns _id,price +log_level --level notice +#@remove-important-log-levels debug -------------- next part -------------- HTML����������������������������... URL: https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20180122/f16f7c07/attachment-0001.htm
