Kouhei Sutou
null+****@clear*****
Tue Nov 1 01:17:30 JST 2016
Kouhei Sutou 2016-11-01 01:17:30 +0900 (Tue, 01 Nov 2016) New Revision: 7e2bbb0cef3c78d606beb3a0d6888c4d2af1d29f https://github.com/pgroonga/pgroonga/commit/7e2bbb0cef3c78d606beb3a0d6888c4d2af1d29f Message: wal: support jsonb Modified files: src/pgrn_global.c src/pgrn_global.h src/pgrn_jsonb.c src/pgrn_jsonb.h src/pgrn_wal.c src/pgrn_wal.h src/pgroonga.c Modified: src/pgrn_global.c (+2 -0) =================================================================== --- src/pgrn_global.c 2016-10-31 16:32:03 +0900 (4d26dbb) +++ src/pgrn_global.c 2016-11-01 01:17:30 +0900 (6d420cc) @@ -17,6 +17,7 @@ PGrnInitializeBuffers(void) GRN_UINT64_INIT(&(PGrnBuffers.ctid), 0); GRN_FLOAT_INIT(&(PGrnBuffers.score), 0); GRN_RECORD_INIT(&(PGrnBuffers.sourceIDs), GRN_OBJ_VECTOR, GRN_ID_NIL); + GRN_UINT64_INIT(&(PGrnBuffers.jsonbValueKeys), GRN_OBJ_VECTOR); GRN_UINT64_INIT(&(PGrnBuffers.walPosition), 0); GRN_VOID_INIT(&(PGrnBuffers.walValue)); GRN_TEXT_INIT(&(PGrnBuffers.head), 0); @@ -35,6 +36,7 @@ PGrnFinalizeBuffers(void) GRN_OBJ_FIN(ctx, &(PGrnBuffers.ctid)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.score)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.sourceIDs)); + GRN_OBJ_FIN(ctx, &(PGrnBuffers.jsonbValueKeys)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.walPosition)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.walValue)); GRN_OBJ_FIN(ctx, &(PGrnBuffers.head)); Modified: src/pgrn_global.h (+1 -0) =================================================================== --- src/pgrn_global.h 2016-10-31 16:32:03 +0900 (79a2e4a) +++ src/pgrn_global.h 2016-11-01 01:17:30 +0900 (a4bf3c1) @@ -11,6 +11,7 @@ struct PGrnBuffers grn_obj ctid; grn_obj score; grn_obj sourceIDs; + grn_obj jsonbValueKeys; grn_obj walPosition; grn_obj walValue; grn_obj head; Modified: src/pgrn_jsonb.c (+305 -99) =================================================================== --- src/pgrn_jsonb.c 2016-10-31 16:32:03 +0900 (fc17eb9) +++ src/pgrn_jsonb.c 2016-11-01 01:17:30 +0900 (1807b88) @@ -33,6 +33,7 @@ typedef struct typedef struct PGrnJSONBInsertData { + Relation index; grn_obj *pathsTable; grn_obj *valuesTable; grn_obj *pathColumn; @@ -43,11 +44,12 @@ typedef struct PGrnJSONBInsertData grn_obj *sizeColumn; grn_obj *typeColumn; grn_obj *valueIDs; - PGrnWALData *walData; grn_obj key; grn_obj components; + grn_obj valueKey; grn_obj path; grn_obj pathIDs; + grn_obj paths; grn_obj value; grn_obj type; } PGrnJSONBInsertData; @@ -168,65 +170,118 @@ PGrnJSONBLookupSizeLexicon(Relation index, } static grn_obj * -PGrnJSONBCreatePathsTable(const char *name) +PGrnJSONBCreateTable(Relation index, + const char *name, + grn_table_flags flags, + grn_obj *type, + grn_obj *tokenizer, + grn_obj *normalizer) +{ + grn_obj *table; + + table = PGrnCreateTable(name, flags, type); + PGrnWALCreateTable(index, name, flags, type, tokenizer, normalizer); + + if (tokenizer) + grn_obj_set_info(ctx, table, GRN_INFO_DEFAULT_TOKENIZER, tokenizer); + if (normalizer) + grn_obj_set_info(ctx, table, GRN_INFO_NORMALIZER, normalizer); + + return table; +} + +static grn_obj * +PGrnJSONBCreatePathsTable(Relation index, const char *name) { - return PGrnCreateTable(name, - GRN_OBJ_TABLE_PAT_KEY, - grn_ctx_at(ctx, GRN_DB_SHORT_TEXT)); + return PGrnJSONBCreateTable(index, + name, + GRN_OBJ_TABLE_PAT_KEY, + grn_ctx_at(ctx, GRN_DB_SHORT_TEXT), + NULL, + NULL); } static grn_obj * -PGrnJSONBCreateTypesTable(const char *name) +PGrnJSONBCreateTypesTable(Relation index, const char *name) { - return PGrnCreateTable(name, - GRN_OBJ_TABLE_PAT_KEY, - grn_ctx_at(ctx, GRN_DB_SHORT_TEXT)); + return PGrnJSONBCreateTable(index, + name, + GRN_OBJ_TABLE_PAT_KEY, + grn_ctx_at(ctx, GRN_DB_SHORT_TEXT), + NULL, + NULL); } static grn_obj * -PGrnJSONBCreateValuesTable(const char *name) +PGrnJSONBCreateValuesTable(Relation index, const char *name) { - return PGrnCreateTable(name, - GRN_OBJ_TABLE_HASH_KEY, - grn_ctx_at(ctx, GRN_DB_UINT64)); + return PGrnJSONBCreateTable(index, + name, + GRN_OBJ_TABLE_HASH_KEY, + grn_ctx_at(ctx, GRN_DB_UINT64), + NULL, + NULL); +} + +static grn_obj * +PGrnJSONBCreateColumn(Relation index, + grn_obj *table, + const char *name, + grn_column_flags flags, + grn_obj *type) +{ + grn_obj *column; + + column = PGrnCreateColumn(table, name, flags, type); + PGrnWALCreateColumn(index, table, name, flags, type); + + return column; } static void -PGrnJSONBCreateDataColumns(PGrnJSONBCreateData *jsonbData) +PGrnJSONBCreateDataColumns(Relation index, + PGrnJSONBCreateData *jsonbData) { - PGrnCreateColumn(jsonbData->valuesTable, - "path", - GRN_OBJ_COLUMN_SCALAR, - jsonbData->pathsTable); - PGrnCreateColumn(jsonbData->valuesTable, - "paths", - GRN_OBJ_COLUMN_VECTOR, - jsonbData->pathsTable); + PGrnJSONBCreateColumn(index, + jsonbData->valuesTable, + "path", + GRN_OBJ_COLUMN_SCALAR, + jsonbData->pathsTable); + PGrnJSONBCreateColumn(index, + jsonbData->valuesTable, + "paths", + GRN_OBJ_COLUMN_VECTOR, + jsonbData->pathsTable); { - grn_obj_flags flags = 0; + grn_column_flags flags = 0; if (PGrnIsLZ4Available) flags |= GRN_OBJ_COMPRESS_LZ4; - PGrnCreateColumn(jsonbData->valuesTable, - "string", - flags, - grn_ctx_at(ctx, GRN_DB_LONG_TEXT)); + PGrnJSONBCreateColumn(index, + jsonbData->valuesTable, + "string", + flags, + grn_ctx_at(ctx, GRN_DB_LONG_TEXT)); } - PGrnCreateColumn(jsonbData->valuesTable, - "number", - 0, - grn_ctx_at(ctx, GRN_DB_FLOAT)); - PGrnCreateColumn(jsonbData->valuesTable, - "boolean", - 0, - grn_ctx_at(ctx, GRN_DB_BOOL)); - PGrnCreateColumn(jsonbData->valuesTable, - "size", - 0, - grn_ctx_at(ctx, GRN_DB_UINT32)); - PGrnCreateColumn(jsonbData->valuesTable, - "type", - 0, - jsonbData->typesTable); + PGrnJSONBCreateColumn(index, + jsonbData->valuesTable, + "number", + 0, + grn_ctx_at(ctx, GRN_DB_FLOAT)); + PGrnJSONBCreateColumn(index, + jsonbData->valuesTable, + "boolean", + 0, + grn_ctx_at(ctx, GRN_DB_BOOL)); + PGrnJSONBCreateColumn(index, + jsonbData->valuesTable, + "size", + 0, + grn_ctx_at(ctx, GRN_DB_UINT32)); + PGrnJSONBCreateColumn(index, + jsonbData->valuesTable, + "type", + 0, + jsonbData->typesTable); } static void @@ -328,9 +383,11 @@ PGrnJSONBInsertDataInit(PGrnJSONBInsertData *data) GRN_TEXT_INIT(&(data->key), 0); GRN_TEXT_INIT(&(data->components), GRN_OBJ_VECTOR); + GRN_UINT64_INIT(&(data->valueKey), 0); GRN_TEXT_INIT(&(data->path), 0); GRN_RECORD_INIT(&(data->pathIDs), GRN_OBJ_VECTOR, grn_obj_id(ctx, data->pathsTable)); + GRN_TEXT_INIT(&(data->paths), GRN_OBJ_VECTOR); GRN_VOID_INIT(&(data->value)); GRN_TEXT_INIT(&(data->type), GRN_OBJ_DO_SHALLOW_COPY); } @@ -340,8 +397,10 @@ PGrnJSONBInsertDataFin(PGrnJSONBInsertData *data) { GRN_OBJ_FIN(ctx, &(data->type)); GRN_OBJ_FIN(ctx, &(data->value)); + GRN_OBJ_FIN(ctx, &(data->paths)); GRN_OBJ_FIN(ctx, &(data->pathIDs)); GRN_OBJ_FIN(ctx, &(data->path)); + GRN_OBJ_FIN(ctx, &(data->valueKey)); GRN_OBJ_FIN(ctx, &(data->components)); GRN_OBJ_FIN(ctx, &(data->key)); } @@ -480,6 +539,9 @@ PGrnJSONBInsertValueSet(PGrnJSONBInsertData *data, uint64_t key; grn_id valueID; int added; + size_t nColumns = 3; /* _key, path?, paths, column?, type */ + bool setPath; + PGrnWALData *walData = NULL; key = PGrnJSONBInsertGenerateKey(data, column != NULL, typeName); valueID = grn_table_add(ctx, data->valuesTable, @@ -489,27 +551,76 @@ PGrnJSONBInsertValueSet(PGrnJSONBInsertData *data, if (!added) return; + if (column) + nColumns++; + GRN_BULK_REWIND(&(data->path)); PGrnJSONGenerateCompletePath(&(data->components), &(data->path)); - if (GRN_TEXT_LEN(&(data->path)) < GRN_TABLE_MAX_KEY_SIZE) + setPath = (GRN_TEXT_LEN(&(data->path)) < GRN_TABLE_MAX_KEY_SIZE); + if (setPath) + nColumns++; + + if (data->index) + walData = PGrnWALStart(data->index); + + PGrnWALInsertStart(walData, data->valuesTable, nColumns); + if (walData) + { + GRN_UINT64_SET(ctx, &(data->valueKey), key); + PGrnWALInsertKey(walData, &(data->valueKey)); + } + + if (setPath) { - /* TODO: WAL */ grn_obj_set_value(ctx, data->pathColumn, valueID, &(data->path), GRN_OBJ_SET); + PGrnWALInsertColumn(walData, data->pathColumn, &(data->path)); } PGrnJSONBInsertGeneratePaths(data); - /* TODO: WAL */ grn_obj_set_value(ctx, data->pathsColumn, valueID, &(data->pathIDs), GRN_OBJ_SET); + if (walData) + { + unsigned int i, n; + + GRN_BULK_REWIND(&(data->paths)); + n = GRN_BULK_VSIZE(&(data->pathIDs)) / sizeof(grn_id); + for (i = 0; i < n; i++) + { + grn_id pathID; + char path[GRN_TABLE_MAX_KEY_SIZE]; + int pathSize; + + pathID = GRN_RECORD_VALUE_AT(&(data->pathIDs), i); + pathSize = grn_table_get_key(ctx, + data->pathsTable, + pathID, + path, + GRN_TABLE_MAX_KEY_SIZE); + grn_vector_add_element(ctx, + &(data->paths), + path, + pathSize, + 0, + GRN_DB_SHORT_TEXT); + } + PGrnWALInsertColumn(walData, data->pathsColumn, &(data->paths)); + } if (column) + { grn_obj_set_value(ctx, column, valueID, &(data->value), GRN_OBJ_SET); + PGrnWALInsertColumn(walData, column, &(data->value)); + } GRN_TEXT_SETS(ctx, &(data->type), typeName); - /* TODO: WAL */ grn_obj_set_value(ctx, data->typeColumn, valueID, &(data->type), GRN_OBJ_SET); + PGrnWALInsertColumn(walData, data->typeColumn, &(data->type)); + + PGrnWALInsertFinish(walData); + PGrnWALFinish(walData); } static void PGrnJSONBInsertContainer(JsonbIterator **iter, @@ -535,10 +646,13 @@ PGrnJSONBInsertValue(JsonbIterator **iter, break; case jbvNumeric: { - Datum numericInString = + Datum numericInString; + const char *numericInCString; + + numericInString = DirectFunctionCall1(numeric_out, NumericGetDatum(value->val.numeric)); - const char *numericInCString = DatumGetCString(numericInString); + numericInCString = DatumGetCString(numericInString); grn_obj_reinit(ctx, &(data->value), GRN_DB_TEXT, GRN_OBJ_DO_SHALLOW_COPY); GRN_TEXT_SETS(ctx, &(data->value), numericInCString); @@ -629,14 +743,14 @@ PGrnInitializeJSONB(void) #ifdef PGRN_SUPPORT_JSONB PGrnJSONBCreateData data; - tmpPathsTable = PGrnJSONBCreatePathsTable(NULL); - tmpTypesTable = PGrnJSONBCreateTypesTable(NULL); - tmpValuesTable = PGrnJSONBCreateValuesTable(NULL); + tmpPathsTable = PGrnJSONBCreatePathsTable(InvalidRelation, NULL); + tmpTypesTable = PGrnJSONBCreateTypesTable(InvalidRelation, NULL); + tmpValuesTable = PGrnJSONBCreateValuesTable(InvalidRelation, NULL); data.pathsTable = tmpPathsTable; data.typesTable = tmpTypesTable; data.valuesTable = tmpValuesTable; - PGrnJSONBCreateDataColumns(&data); + PGrnJSONBCreateDataColumns(InvalidRelation, &data); #endif } @@ -660,7 +774,8 @@ PGrnJSONBCreateTables(PGrnCreateData *data, snprintf(jsonPathsTableName, sizeof(jsonPathsTableName), PGrnJSONPathsTableNameFormat, data->relNode, data->i); - jsonbData->pathsTable = PGrnJSONBCreatePathsTable(jsonPathsTableName); + jsonbData->pathsTable = + PGrnJSONBCreatePathsTable(data->index, jsonPathsTableName); GRN_PTR_PUT(ctx, data->supplementaryTables, jsonbData->pathsTable); } @@ -669,7 +784,8 @@ PGrnJSONBCreateTables(PGrnCreateData *data, snprintf(jsonTypesTableName, sizeof(jsonTypesTableName), PGrnJSONTypesTableNameFormat, data->relNode, data->i); - jsonbData->typesTable = PGrnJSONBCreateTypesTable(jsonTypesTableName); + jsonbData->typesTable = + PGrnJSONBCreateTypesTable(data->index, jsonTypesTableName); GRN_PTR_PUT(ctx, data->supplementaryTables, jsonbData->typesTable); } @@ -678,7 +794,8 @@ PGrnJSONBCreateTables(PGrnCreateData *data, snprintf(jsonValuesTableName, sizeof(jsonValuesTableName), PGrnJSONValuesTableNameFormat, data->relNode, data->i); - jsonbData->valuesTable = PGrnJSONBCreateValuesTable(jsonValuesTableName); + jsonbData->valuesTable = + PGrnJSONBCreateValuesTable(data->index, jsonValuesTableName); GRN_PTR_PUT(ctx, data->supplementaryTables, jsonbData->valuesTable); } } @@ -690,7 +807,11 @@ PGrnJSONBCreateFullTextSearchIndexColumn(PGrnCreateData *data, const char *tokenizerName = PGRN_DEFAULT_TOKENIZER; const char *normalizerName = PGRN_DEFAULT_NORMALIZER; char lexiconName[GRN_TABLE_MAX_KEY_SIZE]; + grn_table_flags flags = GRN_OBJ_TABLE_PAT_KEY; + grn_obj *type; grn_obj *lexicon; + grn_obj *tokenizer; + grn_obj *normalizer = NULL; PGrnApplyOptionValues(data->index, &tokenizerName, &normalizerName); @@ -700,23 +821,23 @@ PGrnJSONBCreateFullTextSearchIndexColumn(PGrnCreateData *data, snprintf(lexiconName, sizeof(lexiconName), PGrnJSONValueLexiconNameFormat, "FullTextSearch", data->relNode, data->i); - lexicon = PGrnCreateTable(lexiconName, - GRN_OBJ_TABLE_PAT_KEY, - grn_ctx_at(ctx, GRN_DB_SHORT_TEXT)); - GRN_PTR_PUT(ctx, data->lexicons, lexicon); - - grn_obj_set_info(ctx, lexicon, GRN_INFO_DEFAULT_TOKENIZER, - PGrnLookup(tokenizerName, ERROR)); + type = grn_ctx_at(ctx, GRN_DB_SHORT_TEXT); + tokenizer = PGrnLookup(tokenizerName, ERROR); if (!PGrnIsNoneValue(normalizerName)) - { - grn_obj_set_info(ctx, lexicon, GRN_INFO_NORMALIZER, - PGrnLookup(normalizerName, ERROR)); - } + normalizer = PGrnLookup(normalizerName, ERROR); + lexicon = PGrnJSONBCreateTable(data->index, + lexiconName, + flags, + type, + tokenizer, + normalizer); + GRN_PTR_PUT(ctx, data->lexicons, lexicon); - PGrnCreateColumn(lexicon, - PGrnIndexColumnName, - GRN_OBJ_COLUMN_INDEX | GRN_OBJ_WITH_POSITION, - jsonbData->valuesTable); + PGrnJSONBCreateColumn(data->index, + lexicon, + PGrnIndexColumnName, + GRN_OBJ_COLUMN_INDEX | GRN_OBJ_WITH_POSITION, + jsonbData->valuesTable); } static void @@ -732,26 +853,34 @@ PGrnJSONBCreateIndexColumn(PGrnCreateData *data, snprintf(lexiconName, sizeof(lexiconName), PGrnJSONValueLexiconNameFormat, typeName, data->relNode, data->i); - lexicon = PGrnCreateTable(lexiconName, tableType, type); + lexicon = PGrnJSONBCreateTable(data->index, + lexiconName, + tableType, + type, + NULL, + NULL); GRN_PTR_PUT(ctx, data->lexicons, lexicon); - PGrnCreateColumn(lexicon, - PGrnIndexColumnName, - GRN_OBJ_COLUMN_INDEX, - jsonbData->valuesTable); + PGrnJSONBCreateColumn(data->index, + lexicon, + PGrnIndexColumnName, + GRN_OBJ_COLUMN_INDEX, + jsonbData->valuesTable); } static void PGrnJSONBCreateIndexColumns(PGrnCreateData *data, PGrnJSONBCreateData *jsonbData) { - PGrnCreateColumn(jsonbData->valuesTable, - PGrnIndexColumnName, - GRN_OBJ_COLUMN_INDEX, - data->sourcesTable); - PGrnCreateColumn(jsonbData->pathsTable, - PGrnIndexColumnName, - GRN_OBJ_COLUMN_INDEX | GRN_OBJ_WITH_SECTION, - jsonbData->valuesTable); + PGrnJSONBCreateColumn(data->index, + jsonbData->valuesTable, + PGrnIndexColumnName, + GRN_OBJ_COLUMN_INDEX, + data->sourcesTable); + PGrnJSONBCreateColumn(data->index, + jsonbData->pathsTable, + PGrnIndexColumnName, + GRN_OBJ_COLUMN_INDEX | GRN_OBJ_WITH_SECTION, + jsonbData->valuesTable); /* TODO: 4KiB over string value can't be searched. */ PGrnJSONBCreateIndexColumn(data, @@ -805,7 +934,7 @@ PGrnJSONBCreate(PGrnCreateData *data) } PGrnJSONBCreateTables(data, &jsonbData); - PGrnJSONBCreateDataColumns(&jsonbData); + PGrnJSONBCreateDataColumns(data->index, &jsonbData); PGrnJSONBCreateIndexColumns(data, &jsonbData); data->attributeTypeID = grn_obj_id(ctx, jsonbData.valuesTable); data->attributeFlags = GRN_OBJ_VECTOR; @@ -845,6 +974,7 @@ PGrnJSONBValueSetSource(Relation index, source = PGrnLookupColumn(jsonValuesTable, columnName, ERROR); PGrnIndexColumnSetSource(indexColumn, source); + PGrnWALSetSource(index, indexColumn, source); grn_obj_unlink(ctx, source); grn_obj_unlink(ctx, indexColumn); @@ -862,21 +992,23 @@ PGrnJSONBSetSources(Relation index, { grn_obj *source; grn_obj *indexColumn; + grn_obj *sourceIDs; - GRN_BULK_REWIND(&(buffers->sourceIDs)); + sourceIDs = &(buffers->sourceIDs); + GRN_BULK_REWIND(sourceIDs); source = PGrnLookupColumn(jsonValuesTable, "path", ERROR); - GRN_RECORD_PUT(ctx, &(buffers->sourceIDs), grn_obj_id(ctx, source)); + GRN_RECORD_PUT(ctx, sourceIDs, grn_obj_id(ctx, source)); grn_obj_unlink(ctx, source); source = PGrnLookupColumn(jsonValuesTable, "paths", ERROR); - GRN_RECORD_PUT(ctx, &(buffers->sourceIDs), grn_obj_id(ctx, source)); + GRN_RECORD_PUT(ctx, sourceIDs, grn_obj_id(ctx, source)); grn_obj_unlink(ctx, source); indexColumn = PGrnLookupColumn(jsonPathsTable, PGrnIndexColumnName, ERROR); - grn_obj_set_info(ctx, indexColumn, GRN_INFO_SOURCE, - &(buffers->sourceIDs)); + grn_obj_set_info(ctx, indexColumn, GRN_INFO_SOURCE, sourceIDs); + PGrnWALSetSources(index, indexColumn, sourceIDs); grn_obj_unlink(ctx, indexColumn); } @@ -945,11 +1077,11 @@ pgroonga_match_jsonb(PG_FUNCTION_ARGS) grn_obj *result = NULL; bool matched = false; + data.index = NULL; data.pathsTable = tmpPathsTable; data.valuesTable = tmpValuesTable; GRN_PTR_INIT(&valueIDs, GRN_OBJ_VECTOR, grn_obj_id(ctx, data.valuesTable)); data.valueIDs = &valueIDs; - data.walData = NULL; PGrnJSONBInsertDataInit(&data); iter = JsonbIteratorInit(&(jsonb->root)); PGrnJSONBInsertContainer(&iter, &data); @@ -994,31 +1126,105 @@ pgroonga_match_jsonb(PG_FUNCTION_ARGS) PG_RETURN_BOOL(matched); } + +static void +PGrnJSONBInsertJSONB(PGrnJSONBInsertData *data, + Jsonb *jsonb) +{ + JsonbIterator *iter; + + iter = JsonbIteratorInit(&(jsonb->root)); + PGrnJSONBInsertContainer(&iter, data); +} + +static void +PGrnJSONBInsertRecord(Relation index, + grn_obj *sourcesTable, + grn_obj *sourcesCtidColumn, + uint64_t packedCtid, + PGrnJSONBInsertData *data) +{ + TupleDesc desc = RelationGetDescr(index); + grn_id id; + PGrnWALData *walData; + Form_pg_attribute attribute; + NameData *attributeName; + grn_obj *column; + + id = grn_table_add(ctx, sourcesTable, NULL, 0, NULL); + + walData = PGrnWALStart(index); + PGrnWALInsertStart(walData, NULL, desc->natts + 1); + + GRN_UINT64_SET(ctx, &(buffers->ctid), packedCtid); + grn_obj_set_value(ctx, sourcesCtidColumn, id, &(buffers->ctid), GRN_OBJ_SET); + PGrnWALInsertColumn(walData, sourcesCtidColumn, &(buffers->ctid)); + + attribute = desc->attrs[0]; + attributeName = &(attribute->attname); + column = PGrnLookupColumn(sourcesTable, attributeName->data, ERROR); + grn_obj_set_value(ctx, column, id, data->valueIDs, GRN_OBJ_SET); + PGrnCheck("pgroonga: failed to set column value"); + + if (walData) + { + grn_obj *valueKeys = &(buffers->jsonbValueKeys); + size_t i, nValueIDs; + + GRN_BULK_REWIND(valueKeys); + nValueIDs = GRN_BULK_VSIZE(data->valueIDs) / sizeof(grn_id); + for (i = 0; i < nValueIDs; i++) + { + grn_id valueID; + uint64_t key; + + valueID = GRN_RECORD_VALUE_AT(data->valueIDs, i); + grn_table_get_key(ctx, + data->valuesTable, + valueID, + &key, + sizeof(uint64_t)); + GRN_UINT64_PUT(ctx, valueKeys, key); + } + PGrnWALInsertColumn(walData, column, valueKeys); + } + + PGrnWALInsertFinish(walData); + PGrnWALFinish(walData); +} #endif void PGrnJSONBInsert(Relation index, + grn_obj *sourcesTable, + grn_obj *sourcesCtidColumn, Datum *values, - unsigned int nthValue, - grn_obj *valueIDs, - PGrnWALData *walData) + bool *isnull, + uint64_t packedCtid) { #ifdef PGRN_SUPPORT_JSONB PGrnJSONBInsertData data; + unsigned int nthValue = 0; Jsonb *jsonb; - JsonbIterator *iter; + data.index = index; data.pathsTable = PGrnJSONBLookupPathsTable(index, nthValue, ERROR); data.valuesTable = PGrnJSONBLookupValuesTable(index, nthValue, ERROR); - data.valueIDs = valueIDs; - data.walData = walData; + data.valueIDs = &(buffers->general); grn_obj_reinit(ctx, data.valueIDs, grn_obj_id(ctx, data.valuesTable), GRN_OBJ_VECTOR); PGrnJSONBInsertDataInit(&data); + jsonb = DatumGetJsonb(values[nthValue]); - iter = JsonbIteratorInit(&(jsonb->root)); - PGrnJSONBInsertContainer(&iter, &data); + PGrnJSONBInsertJSONB(&data, jsonb); + + PGrnJSONBInsertRecord(index, + sourcesTable, + sourcesCtidColumn, + packedCtid, + &data); + PGrnJSONBInsertDataFin(&data); #endif } Modified: src/pgrn_jsonb.h (+4 -3) =================================================================== --- src/pgrn_jsonb.h 2016-10-31 16:32:03 +0900 (a410c44) +++ src/pgrn_jsonb.h 2016-11-01 01:17:30 +0900 (bf408e0) @@ -42,10 +42,11 @@ grn_obj *PGrnJSONBLookupSizeLexicon(Relation index, grn_obj *PGrnJSONBSetSource(Relation index, unsigned int i); void PGrnJSONBInsert(Relation index, + grn_obj *sourcesTable, + grn_obj *sourcesCtidColumn, Datum *values, - unsigned int nthValue, - grn_obj *valueIDs, - PGrnWALData *walData); + bool *isnull, + uint64_t packedCtid); bool PGrnJSONBBuildSearchCondition(PGrnSearchData *data, ScanKey key, Modified: src/pgrn_wal.c (+127 -38) =================================================================== --- src/pgrn_wal.c 2016-10-31 16:32:03 +0900 (84e5e7e) +++ src/pgrn_wal.c 2016-11-01 01:17:30 +0900 (b692cf5) @@ -154,8 +154,8 @@ PGrnWALPackPosition(BlockNumber block, OffsetNumber offset) static void PGrnWALUnpackPosition(uint64_t position, - BlockNumber *block, - OffsetNumber *offset) + BlockNumber *block, + OffsetNumber *offset) { *block = (BlockNumber)(position >> 32); *offset = (OffsetNumber)(position & ((1 << 16) - 1)); @@ -163,8 +163,8 @@ PGrnWALUnpackPosition(uint64_t position, static void PGrnWALUpdateStatus(Relation index, - BlockNumber block, - OffsetNumber offset) + BlockNumber block, + OffsetNumber offset) { grn_obj *statusesTable; grn_obj *currentColumn; @@ -342,6 +342,9 @@ PGrnWALStart(Relation index) if (!PGrnWALEnabled) return NULL; + if (!RelationIsValid(index)) + return NULL; + data = palloc(sizeof(PGrnWALData)); data->index = index; @@ -397,16 +400,35 @@ PGrnWALAbort(PGrnWALData *data) void PGrnWALInsertStart(PGrnWALData *data, - size_t nColumns) + grn_obj *table, + size_t nColumns) { #ifdef PGRN_SUPPORT_WAL msgpack_packer *packer; + size_t nElements = nColumns; - if (!PGrnWALEnabled) + if (!data) return; + if (table) + nElements++; + packer = &(data->packer); - msgpack_pack_map(packer, nColumns); + msgpack_pack_map(packer, nElements); + + if (table) + { + char tableName[GRN_TABLE_MAX_KEY_SIZE]; + int tableNameSize; + + tableNameSize = grn_obj_name(ctx, + table, + tableName, + GRN_TABLE_MAX_KEY_SIZE); + msgpack_pack_cstr(packer, "_table"); + msgpack_pack_str(packer, tableNameSize); + msgpack_pack_str_body(packer, tableName, tableNameSize); + } #endif } @@ -417,18 +439,17 @@ PGrnWALInsertFinish(PGrnWALData *data) void PGrnWALInsertColumnStart(PGrnWALData *data, - const char *name) + const char *name, + size_t nameSize) { #ifdef PGRN_SUPPORT_WAL msgpack_packer *packer; - size_t nameSize; - if (!PGrnWALEnabled) + if (!data) return; packer = &(data->packer); - nameSize = strlen(name); msgpack_pack_str(packer, nameSize); msgpack_pack_str_body(packer, name, nameSize); #endif @@ -443,6 +464,7 @@ PGrnWALInsertColumnFinish(PGrnWALData *data) static void PGrnWALInsertColumnValueRaw(PGrnWALData *data, const char *name, + size_t nameSize, grn_id domain, const char *value, size_t valueSize) @@ -512,8 +534,8 @@ PGrnWALInsertColumnValueRaw(PGrnWALData *data, ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("pgroonga: WAL: insert: unsupported type: " - "<%s>: <%.*s>", - name, + "<%.*s>: <%.*s>", + (int)nameSize, name, domainNameSize, domainName))); } break; @@ -523,10 +545,12 @@ PGrnWALInsertColumnValueRaw(PGrnWALData *data, static void PGrnWALInsertColumnValueBulk(PGrnWALData *data, const char *name, + size_t nameSize, grn_obj *value) { PGrnWALInsertColumnValueRaw(data, name, + nameSize, value->header.domain, GRN_BULK_HEAD(value), GRN_BULK_VSIZE(value)); @@ -535,6 +559,7 @@ PGrnWALInsertColumnValueBulk(PGrnWALData *data, static void PGrnWALInsertColumnValueVector(PGrnWALData *data, const char *name, + size_t nameSize, grn_obj *value) { msgpack_packer *packer; @@ -556,36 +581,83 @@ PGrnWALInsertColumnValueVector(PGrnWALData *data, &element, NULL, &domain); - PGrnWALInsertColumnValueRaw(data, name, domain, element, elementSize); + PGrnWALInsertColumnValueRaw(data, + name, + nameSize, + domain, + element, + elementSize); + } +} + +static void +PGrnWALInsertColumnUValueVector(PGrnWALData *data, + const char *name, + size_t nameSize, + grn_obj *value) +{ + msgpack_packer *packer; + grn_id domain; + unsigned int elementSize; + unsigned int i, n; + + packer = &(data->packer); + + domain = value->header.domain; + elementSize = grn_uvector_element_size(ctx, value); + n = grn_uvector_size(ctx, value); + msgpack_pack_array(packer, n); + for (i = 0; i < n; i++) + { + const char *element; + + element = GRN_BULK_HEAD(value) + (elementSize * i); + PGrnWALInsertColumnValueRaw(data, + name, + nameSize, + domain, + element, + elementSize); } } #endif void PGrnWALInsertColumn(PGrnWALData *data, - const char *name, + grn_obj *column, grn_obj *value) { #ifdef PGRN_SUPPORT_WAL - if (!PGrnWALEnabled) + char name[GRN_TABLE_MAX_KEY_SIZE]; + int nameSize; + + if (!data) + return; + + nameSize = grn_column_name(ctx, column, name, GRN_TABLE_MAX_KEY_SIZE); + + if (!data) return; - PGrnWALInsertColumnStart(data, name); + PGrnWALInsertColumnStart(data, name, nameSize); switch (value->header.type) { case GRN_BULK: - PGrnWALInsertColumnValueBulk(data, name, value); + PGrnWALInsertColumnValueBulk(data, name, nameSize, value); break; case GRN_VECTOR: - PGrnWALInsertColumnValueVector(data, name, value); + PGrnWALInsertColumnValueVector(data, name, nameSize, value); + break; + case GRN_UVECTOR: + PGrnWALInsertColumnUValueVector(data, name, nameSize, value); break; default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("pgroonga: WAL: not bulk value isn't supported yet: " - "<%s>: <%s>", - name, + "<%.*s>: <%s>", + (int)nameSize, name, grn_obj_type_to_string(value->header.type)))); break; } @@ -595,6 +667,26 @@ PGrnWALInsertColumn(PGrnWALData *data, } void +PGrnWALInsertKey(PGrnWALData *data, grn_obj *key) +{ +#ifdef PGRN_SUPPORT_WAL + msgpack_packer *packer; + + if (!data) + return; + + packer = &(data->packer); + + PGrnWALInsertColumnStart(data, + GRN_COLUMN_NAME_KEY, + GRN_COLUMN_NAME_KEY_LEN); + msgpack_pack_bin(packer, GRN_BULK_VSIZE(key)); + msgpack_pack_bin_body(packer, GRN_BULK_HEAD(key), GRN_BULK_VSIZE(key)); + PGrnWALInsertColumnFinish(data); +#endif +} + +void PGrnWALCreateTable(Relation index, const char *name, grn_table_flags flags, @@ -607,10 +699,9 @@ PGrnWALCreateTable(Relation index, msgpack_packer *packer; size_t nElements = 6; - if (!PGrnWALEnabled) - return; - data = PGrnWALStart(index); + if (!data) + return; packer = &(data->packer); msgpack_pack_map(packer, nElements); @@ -649,10 +740,9 @@ PGrnWALCreateColumn(Relation index, msgpack_packer *packer; size_t nElements = 5; - if (!PGrnWALEnabled) - return; - data = PGrnWALStart(index); + if (!data) + return; packer = &(data->packer); msgpack_pack_map(packer, nElements); @@ -686,10 +776,9 @@ PGrnWALSetSource(Relation index, msgpack_packer *packer; size_t nElements = 3; - if (!PGrnWALEnabled) - return; - data = PGrnWALStart(index); + if (!data) + return; packer = &(data->packer); msgpack_pack_map(packer, nElements); @@ -718,10 +807,9 @@ PGrnWALSetSources(Relation index, msgpack_packer *packer; size_t nElements = 3; - if (!PGrnWALEnabled) - return; - data = PGrnWALStart(index); + if (!data) + return; packer = &(data->packer); msgpack_pack_map(packer, nElements); @@ -775,8 +863,8 @@ PGrnWALApplyNeeded(PGrnWALApplyData *data) GRN_BULK_REWIND(position); grn_obj_get_value(ctx, data->currentColumn, data->statusID, position); PGrnWALUnpackPosition(GRN_UINT64_VALUE(position), - ¤tBlock, - ¤tOffset); + ¤tBlock, + ¤tOffset); } nBlocks = RelationGetNumberOfBlocks(data->index); @@ -1074,16 +1162,17 @@ PGrnWALApplyInsert(PGrnWALApplyData *data, msgpack_object_kv *kv; kv = &(map->ptr[currentElement]); - if (PGrnWALApplyKeyEqual(context, &(kv->key), "_key")) + if (PGrnWALApplyKeyEqual(context, &(kv->key), GRN_COLUMN_NAME_KEY)) { if (kv->val.type != MSGPACK_OBJECT_BIN) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("pgroonga: WAL: apply: %s: " - "_key value must be binary: " + "%s value must be binary: " "<%#x>", context, + GRN_COLUMN_NAME_KEY, kv->val.type))); } key = kv->val.via.bin.ptr; @@ -1113,7 +1202,7 @@ PGrnWALApplyInsert(PGrnWALApplyData *data, key->type))); } - column = PGrnLookupColumnWithSize(data->sources, + column = PGrnLookupColumnWithSize(table, key->via.str.ptr, key->via.str.size, ERROR); Modified: src/pgrn_wal.h (+9 -3) =================================================================== --- src/pgrn_wal.h 2016-10-31 16:32:03 +0900 (b25f1f8) +++ src/pgrn_wal.h 2016-11-01 01:17:30 +0900 (63930f1) @@ -15,13 +15,19 @@ PGrnWALData *PGrnWALStart(Relation index); void PGrnWALFinish(PGrnWALData *data); void PGrnWALAbort(PGrnWALData *data); -void PGrnWALInsertStart(PGrnWALData *data, size_t nColumns); +void PGrnWALInsertStart(PGrnWALData *data, + grn_obj *table, + size_t nColumns); void PGrnWALInsertFinish(PGrnWALData *data); -void PGrnWALInsertColumnStart(PGrnWALData *data, const char *name); +void PGrnWALInsertColumnStart(PGrnWALData *data, + const char *name, + size_t nameSize); void PGrnWALInsertColumnFinish(PGrnWALData *data); void PGrnWALInsertColumn(PGrnWALData *data, - const char *name, + grn_obj *column, grn_obj *value); +void PGrnWALInsertKey(PGrnWALData *data, + grn_obj *key); void PGrnWALCreateTable(Relation index, const char *name, Modified: src/pgroonga.c (+18 -19) =================================================================== --- src/pgroonga.c 2016-10-31 16:32:03 +0900 (8bf7781) +++ src/pgroonga.c 2016-11-01 01:17:30 +0900 (70dbdc0) @@ -1934,14 +1934,25 @@ PGrnInsert(Relation index, PGrnWALData *walData; unsigned int i; + if (desc->natts == 1 && PGrnAttributeIsJSONB(desc->attrs[0]->atttypid)) + { + PGrnJSONBInsert(index, + sourcesTable, + sourcesCtidColumn, + values, + isnull, + CtidToUInt64(ht_ctid)); + return; + } + id = grn_table_add(ctx, sourcesTable, NULL, 0, NULL); walData = PGrnWALStart(index); - PGrnWALInsertStart(walData, desc->natts + 1); + PGrnWALInsertStart(walData, NULL, desc->natts + 1); GRN_UINT64_SET(ctx, &(buffers->ctid), CtidToUInt64(ht_ctid)); grn_obj_set_value(ctx, sourcesCtidColumn, id, &(buffers->ctid), GRN_OBJ_SET); - PGrnWALInsertColumn(walData, "ctid", &(buffers->ctid)); + PGrnWALInsertColumn(walData, sourcesCtidColumn, &(buffers->ctid)); for (i = 0; i < desc->natts; i++) { @@ -1957,24 +1968,12 @@ PGrnInsert(Relation index, continue; dataColumn = PGrnLookupColumn(sourcesTable, name->data, ERROR); - buffer = &(buffers->general); - if (PGrnAttributeIsJSONB(attribute->atttypid)) - { - /* PGrnWALInsertColumnStart(walData, name->data); */ - PGrnJSONBInsert(index, values, i, buffer, walData); - grn_obj_set_value(ctx, dataColumn, id, buffer, GRN_OBJ_SET); - /* PGrnWALInsertColumnFinish(walData); */ - PGrnWALInsertColumn(walData, name->data, buffer); - } - else - { - domain = PGrnGetType(index, i, &flags); - grn_obj_reinit(ctx, buffer, domain, flags); - PGrnConvertFromData(values[i], attribute->atttypid, buffer); - grn_obj_set_value(ctx, dataColumn, id, buffer, GRN_OBJ_SET); - PGrnWALInsertColumn(walData, name->data, buffer); - } + domain = PGrnGetType(index, i, &flags); + grn_obj_reinit(ctx, buffer, domain, flags); + PGrnConvertFromData(values[i], attribute->atttypid, buffer); + grn_obj_set_value(ctx, dataColumn, id, buffer, GRN_OBJ_SET); + PGrnWALInsertColumn(walData, dataColumn, buffer); grn_obj_unlink(ctx, dataColumn); if (!PGrnCheck("pgroonga: failed to set column value")) { continue; -------------- next part -------------- HTML����������������������������...ダウンロード
