Kouhei Sutou 2018-11-13 10:53:33 +0900 (Tue, 13 Nov 2018) Revision: 62f6458b2524db1476e3d2fec7c60dc284d1d48f https://github.com/groonga/groonga/commit/62f6458b2524db1476e3d2fec7c60dc284d1d48f Message: TokenDelimit: support tokenized delimiter again Modified files: lib/tokenizers.c Modified: lib/tokenizers.c (+9 -0) =================================================================== --- lib/tokenizers.c 2018-11-12 18:57:53 +0900 (27e881253) +++ lib/tokenizers.c 2018-11-13 10:53:33 +0900 (2f599c3e9) @@ -276,6 +276,8 @@ delimit_next(grn_ctx *ctx, if (tokenizer->have_tokenized_delimiter) { unsigned int rest_length; + grn_obj *status; + grn_obj *data; rest_length = tokenizer->end - tokenizer->next; tokenizer->next = (unsigned char *)grn_tokenizer_tokenized_delimiter_next( @@ -284,6 +286,13 @@ delimit_next(grn_ctx *ctx, (const char *)tokenizer->next, rest_length, tokenizer->encoding); + status = grn_ctx_pop(ctx); + data = grn_ctx_pop(ctx); + grn_token_set_data(ctx, + token, + GRN_TEXT_VALUE(data), + GRN_TEXT_LEN(data)); + grn_token_set_status(ctx, token, GRN_UINT32_VALUE(status)); } else { size_t cl; const unsigned char *p = tokenizer->next, *r; -------------- next part -------------- An HTML attachment was scrubbed... URL: <https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20181113/5edc7199/attachment.html>
