Update to version v3.24.0

.github/workflows/test.yml

@@ -139,12 +139,6 @@ jobs:
       CXX: ${{matrix.cxx}}
       TEST: ${{matrix.test}}
     steps:
-      - name: Fix kernel mmap rnd bits
-        if: ${{ matrix.os != 'macos-latest' && matrix.sanitizer != '' }}
-        # Asan provided in ubuntu 22.04 is incompatible with
-        # high-entropy ASLR in much newer kernels that GitHub runners are
-        # using leading to random crashes: https://reviews.llvm.org/D148280
-        run: sudo sysctl vm.mmap_rnd_bits=28
       - name: Checkout repository
         if: ${{ matrix.os != 'macos-latest' || matrix.test == 'GO' }}
         uses: actions/checkout@v4

CMakeLists.txt

@@ -1,5 +1,5 @@
 project(reindexer)
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.10)
 enable_testing()
 set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)

bindings.go

@@ -42,7 +42,7 @@ func (db *reindexerImpl) modifyItem(ctx context.Context, namespace string, ns *r
 			return
 		}
 
-		out, err := db.binding.ModifyItem(ctx, ns.nsHash, ns.name, format, ser.Bytes(), mode, precepts, stateToken)
+		out, err := db.binding.ModifyItem(ctx, ns.name, format, ser.Bytes(), mode, precepts, stateToken)
 
 		if err != nil {
 			rerr, ok := err.(bindings.Error)
@@ -236,14 +236,16 @@ func (db *reindexerImpl) rawResultToJson(rawResult []byte, jsonName string, tota
 }
 
 func (db *reindexerImpl) prepareQuery(ctx context.Context, q *Query, asJson bool) (result bindings.RawBuffer, err error) {
-
+	// Ordering in q.nsArray is matter ad must correspond to the ordering in C++
 	if ns, err := db.getNS(q.Namespace); err == nil {
 		q.nsArray = append(q.nsArray, nsArrayEntry{ns, ns.cjsonState.Copy()})
 	} else {
 		return nil, err
 	}
 
 	ser := q.ser
+	ser.PutVarCUInt(queryEnd)
+
 	for _, sq := range q.mergedQueries {
 		if ns, err := db.getNS(sq.Namespace); err == nil {
 			q.nsArray = append(q.nsArray, nsArrayEntry{ns, ns.cjsonState.Copy()})
@@ -258,20 +260,7 @@ func (db *reindexerImpl) prepareQuery(ctx context.Context, q *Query, asJson bool
 		} else {
 			return nil, err
 		}
-	}
 
-	for _, mq := range q.mergedQueries {
-		for _, sq := range mq.joinQueries {
-			if ns, err := db.getNS(sq.Namespace); err == nil {
-				q.nsArray = append(q.nsArray, nsArrayEntry{ns, ns.cjsonState.Copy()})
-			} else {
-				return nil, err
-			}
-		}
-	}
-
-	ser.PutVarCUInt(queryEnd)
-	for _, sq := range q.joinQueries {
 		ser.PutVarCUInt(sq.joinType)
 		ser.Append(sq.ser)
 		ser.PutVarCUInt(queryEnd)
@@ -281,7 +270,14 @@ func (db *reindexerImpl) prepareQuery(ctx context.Context, q *Query, asJson bool
 		ser.PutVarCUInt(merge)
 		ser.Append(mq.ser)
 		ser.PutVarCUInt(queryEnd)
+
 		for _, sq := range mq.joinQueries {
+			if ns, err := db.getNS(sq.Namespace); err == nil {
+				q.nsArray = append(q.nsArray, nsArrayEntry{ns, ns.cjsonState.Copy()})
+			} else {
+				return nil, err
+			}
+
 			ser.PutVarCUInt(sq.joinType)
 			ser.Append(sq.ser)
 			ser.PutVarCUInt(queryEnd)
@@ -378,7 +374,7 @@ func (db *reindexerImpl) deleteQuery(ctx context.Context, q *Query) (int, error)
 		return 0, err
 	}
 
-	result, err := db.binding.DeleteQuery(ctx, ns.nsHash, q.ser.Bytes())
+	result, err := db.binding.DeleteQuery(ctx, q.ser.Bytes())
 	if err != nil {
 		return 0, err
 	}
@@ -418,7 +414,7 @@ func (db *reindexerImpl) updateQuery(ctx context.Context, q *Query) *Iterator {
 		return errIterator(err)
 	}
 
-	result, err := db.binding.UpdateQuery(ctx, ns.nsHash, q.ser.Bytes())
+	result, err := db.binding.UpdateQuery(ctx, q.ser.Bytes())
 	if err != nil {
 		return errIterator(err)
 	}

bindings/builtin/builtin.go

@@ -267,7 +267,7 @@ func (binding *Builtin) awaitLimiter(ctx context.Context) (withLimiter bool, err
 	return
 }
 
-func (binding *Builtin) ModifyItem(ctx context.Context, nsHash int, namespace string, format int, data []byte, mode int, precepts []string, stateToken int) (bindings.RawBuffer, error) {
+func (binding *Builtin) ModifyItem(ctx context.Context, namespace string, format int, data []byte, mode int, precepts []string, stateToken int) (bindings.RawBuffer, error) {
 	if withLimiter, err := binding.awaitLimiter(ctx); err != nil {
 		return nil, err
 	} else if withLimiter {
@@ -470,6 +470,29 @@ func (binding *Builtin) DropIndex(ctx context.Context, namespace, index string)
 	return err2go(C.reindexer_drop_index(binding.rx, str2c(namespace), str2c(index), ctxInfo.cCtx))
 }
 
+func (binding *Builtin) EnumMeta(ctx context.Context, namespace string) ([]string, error) {
+	ctxInfo, err := binding.StartWatchOnCtx(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer binding.ctxWatcher.StopWatchOnCtx(ctxInfo)
+
+	data, err := ret2go(C.reindexer_enum_meta(binding.rx, str2c(namespace), ctxInfo.cCtx))
+	if err != nil {
+		return nil, err
+	}
+	defer data.Free()
+
+	ser := cjson.NewSerializer(data.GetBuf())
+
+	keyCnt := ser.GetVarUInt()
+	keys := make([]string, keyCnt)
+	for i := 0; i < int(keyCnt); i++ {
+		keys[i] = ser.GetVString()
+	}
+	return keys, nil
+}
+
 func (binding *Builtin) PutMeta(ctx context.Context, namespace, key, data string) error {
 	ctxInfo, err := binding.StartWatchOnCtx(ctx)
 	if err != nil {
@@ -490,6 +513,16 @@ func (binding *Builtin) GetMeta(ctx context.Context, namespace, key string) (bin
 	return ret2go(C.reindexer_get_meta(binding.rx, str2c(namespace), str2c(key), ctxInfo.cCtx))
 }
 
+func (binding *Builtin) DeleteMeta(ctx context.Context, namespace, key string) error {
+	ctxInfo, err := binding.StartWatchOnCtx(ctx)
+	if err != nil {
+		return err
+	}
+	defer binding.ctxWatcher.StopWatchOnCtx(ctxInfo)
+
+	return err2go(C.reindexer_delete_meta(binding.rx, str2c(namespace), str2c(key), ctxInfo.cCtx))
+}
+
 func (binding *Builtin) Select(ctx context.Context, query string, asJson bool, ptVersions []int32, fetchCount int) (bindings.RawBuffer, error) {
 	if withLimiter, err := binding.awaitLimiter(ctx); err != nil {
 		return nil, err
@@ -560,7 +593,7 @@ func (binding *Builtin) SelectQuery(ctx context.Context, data []byte, asJson boo
 	return ret2go(C.reindexer_select_query(binding.rx, buf2c(data), bool2cint(asJson), (*C.int32_t)(unsafe.Pointer(&ptVersions[0])), C.int(len(ptVersions)), ctxInfo.cCtx))
 }
 
-func (binding *Builtin) DeleteQuery(ctx context.Context, nsHash int, data []byte) (bindings.RawBuffer, error) {
+func (binding *Builtin) DeleteQuery(ctx context.Context, data []byte) (bindings.RawBuffer, error) {
 	if withLimiter, err := binding.awaitLimiter(ctx); err != nil {
 		return nil, err
 	} else if withLimiter {
@@ -576,7 +609,7 @@ func (binding *Builtin) DeleteQuery(ctx context.Context, nsHash int, data []byte
 	return ret2go(C.reindexer_delete_query(binding.rx, buf2c(data), ctxInfo.cCtx))
 }
 
-func (binding *Builtin) UpdateQuery(ctx context.Context, nsHash int, data []byte) (bindings.RawBuffer, error) {
+func (binding *Builtin) UpdateQuery(ctx context.Context, data []byte) (bindings.RawBuffer, error) {
 	if withLimiter, err := binding.awaitLimiter(ctx); err != nil {
 		return nil, err
 	} else if withLimiter {

bindings/builtinserver/builtinserver.go

@@ -180,6 +180,10 @@ func (server *BuiltinServer) DropIndex(ctx context.Context, namespace, index str
 	return server.builtin.DropIndex(ctx, namespace, index)
 }
 
+func (server *BuiltinServer) EnumMeta(ctx context.Context, namespace string) ([]string, error) {
+	return server.builtin.EnumMeta(ctx, namespace)
+}
+
 func (server *BuiltinServer) PutMeta(ctx context.Context, namespace, key, data string) error {
 	return server.builtin.PutMeta(ctx, namespace, key, data)
 }
@@ -188,8 +192,12 @@ func (server *BuiltinServer) GetMeta(ctx context.Context, namespace, key string)
 	return server.builtin.GetMeta(ctx, namespace, key)
 }
 
-func (server *BuiltinServer) ModifyItem(ctx context.Context, nsHash int, namespace string, format int, data []byte, mode int, percepts []string, stateToken int) (bindings.RawBuffer, error) {
-	return server.builtin.ModifyItem(ctx, nsHash, namespace, format, data, mode, percepts, stateToken)
+func (server *BuiltinServer) DeleteMeta(ctx context.Context, namespace, key string) error {
+	return server.builtin.DeleteMeta(ctx, namespace, key)
+}
+
+func (server *BuiltinServer) ModifyItem(ctx context.Context, namespace string, format int, data []byte, mode int, percepts []string, stateToken int) (bindings.RawBuffer, error) {
+	return server.builtin.ModifyItem(ctx, namespace, format, data, mode, percepts, stateToken)
 }
 
 func (server *BuiltinServer) BeginTx(ctx context.Context, namespace string) (bindings.TxCtx, error) {
@@ -229,12 +237,12 @@ func (server *BuiltinServer) SelectQuery(ctx context.Context, rawQuery []byte, a
 	return server.builtin.SelectQuery(ctx, rawQuery, asJson, ptVersions, fetchCount)
 }
 
-func (server *BuiltinServer) DeleteQuery(ctx context.Context, nsHash int, rawQuery []byte) (bindings.RawBuffer, error) {
-	return server.builtin.DeleteQuery(ctx, nsHash, rawQuery)
+func (server *BuiltinServer) DeleteQuery(ctx context.Context, rawQuery []byte) (bindings.RawBuffer, error) {
+	return server.builtin.DeleteQuery(ctx, rawQuery)
 }
 
-func (server *BuiltinServer) UpdateQuery(ctx context.Context, nsHash int, rawQuery []byte) (bindings.RawBuffer, error) {
-	return server.builtin.UpdateQuery(ctx, nsHash, rawQuery)
+func (server *BuiltinServer) UpdateQuery(ctx context.Context, rawQuery []byte) (bindings.RawBuffer, error) {
+	return server.builtin.UpdateQuery(ctx, rawQuery)
 }
 
 func (server *BuiltinServer) Commit(ctx context.Context, namespace string) error {

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.23.1"
+const ReindexerVersion = "v3.24.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

bindings/cproto/connection.go

@@ -70,6 +70,7 @@ const (
 	cmdSelectSQL         = 49
 	cmdFetchResults      = 50
 	cmdCloseResults      = 51
+	cmdDeleteMeta        = 63
 	cmdGetMeta           = 64
 	cmdPutMeta           = 65
 	cmdEnumMeta          = 66

bindings/cproto/cproto.go

@@ -252,7 +252,7 @@ func (binding *NetCProto) UpdateQueryTx(txCtx *bindings.TxCtx, rawQuery []byte)
 	return netBuffer.conn.rpcCallNoResults(txCtx.UserCtx, cmdUpdateQueryTx, uint32(binding.timeouts.RequestTimeout/time.Second), rawQuery, int64(txCtx.Id))
 }
 
-func (binding *NetCProto) ModifyItem(ctx context.Context, nsHash int, namespace string, format int, data []byte, mode int, precepts []string, stateToken int) (bindings.RawBuffer, error) {
+func (binding *NetCProto) ModifyItem(ctx context.Context, namespace string, format int, data []byte, mode int, precepts []string, stateToken int) (bindings.RawBuffer, error) {
 
 	var packedPercepts []byte
 	if len(precepts) != 0 {
@@ -336,6 +336,20 @@ func (binding *NetCProto) DropIndex(ctx context.Context, namespace, index string
 	return binding.rpcCallNoResults(ctx, opWr, cmdDropIndex, namespace, index)
 }
 
+func (binding *NetCProto) EnumMeta(ctx context.Context, namespace string) ([]string, error) {
+	buf, err := binding.rpcCall(ctx, opRd, cmdEnumMeta, namespace)
+	if err != nil {
+		return nil, err
+	}
+	defer buf.Free()
+
+	keys := make([]string, len(buf.args))
+	for i, item := range buf.args {
+		keys[i] = string(item.([]byte))
+	}
+	return keys, nil
+}
+
 func (binding *NetCProto) PutMeta(ctx context.Context, namespace, key, data string) error {
 	return binding.rpcCallNoResults(ctx, opWr, cmdPutMeta, namespace, key, data)
 }
@@ -344,6 +358,10 @@ func (binding *NetCProto) GetMeta(ctx context.Context, namespace, key string) (b
 	return binding.rpcCall(ctx, opRd, cmdGetMeta, namespace, key)
 }
 
+func (binding *NetCProto) DeleteMeta(ctx context.Context, namespace, key string) error {
+	return binding.rpcCallNoResults(ctx, opWr, cmdDeleteMeta, namespace, key)
+}
+
 func (binding *NetCProto) Select(ctx context.Context, query string, asJson bool, ptVersions []int32, fetchCount int) (bindings.RawBuffer, error) {
 	flags := 0
 	if asJson {
@@ -390,11 +408,11 @@ func (binding *NetCProto) SelectQuery(ctx context.Context, data []byte, asJson b
 	return buf, err
 }
 
-func (binding *NetCProto) DeleteQuery(ctx context.Context, nsHash int, data []byte) (bindings.RawBuffer, error) {
+func (binding *NetCProto) DeleteQuery(ctx context.Context, data []byte) (bindings.RawBuffer, error) {
 	return binding.rpcCall(ctx, opWr, cmdDeleteQuery, data)
 }
 
-func (binding *NetCProto) UpdateQuery(ctx context.Context, nsHash int, data []byte) (bindings.RawBuffer, error) {
+func (binding *NetCProto) UpdateQuery(ctx context.Context, data []byte) (bindings.RawBuffer, error) {
 	return binding.rpcCall(ctx, opWr, cmdUpdateQuery, data)
 }
 

bindings/interface.go

@@ -192,13 +192,15 @@ type RawBinding interface {
 	DeleteQueryTx(txCtx *TxCtx, rawQuery []byte) error
 	UpdateQueryTx(txCtx *TxCtx, rawQuery []byte) error
 
+	EnumMeta(ctx context.Context, namespace string) ([]string, error)
 	PutMeta(ctx context.Context, namespace, key, data string) error
 	GetMeta(ctx context.Context, namespace, key string) (RawBuffer, error)
-	ModifyItem(ctx context.Context, nsHash int, namespace string, format int, data []byte, mode int, percepts []string, stateToken int) (RawBuffer, error)
+	DeleteMeta(ctx context.Context, namespace, key string) error
+	ModifyItem(ctx context.Context, namespace string, format int, data []byte, mode int, percepts []string, stateToken int) (RawBuffer, error)
 	Select(ctx context.Context, query string, asJson bool, ptVersions []int32, fetchCount int) (RawBuffer, error)
 	SelectQuery(ctx context.Context, rawQuery []byte, asJson bool, ptVersions []int32, fetchCount int) (RawBuffer, error)
-	DeleteQuery(ctx context.Context, nsHash int, rawQuery []byte) (RawBuffer, error)
-	UpdateQuery(ctx context.Context, nsHash int, rawQuery []byte) (RawBuffer, error)
+	DeleteQuery(ctx context.Context, rawQuery []byte) (RawBuffer, error)
+	UpdateQuery(ctx context.Context, rawQuery []byte) (RawBuffer, error)
 	Commit(ctx context.Context, namespace string) error
 	EnableLogger(logger Logger)
 	DisableLogger()

changelog.md

@@ -1,3 +1,40 @@
+# Version 3.24.0 (12.04.2024)
+
+## Core
+- [fea] Added `array_remove` and `array_remove_once` functions for the UPDATE-queries. [Details](readme.md#remove-array-elements-by-values)
+- [fea] Increased limit for IdSet join preselect from 10'000 to 20'000 documents
+- [fea] Added C++ benchmarks for `sparse`-indexes
+- [fea] Disabled comparators for `sparse`-indexes. From now `sparse`-indexes always use more effective IdSets instead
+- [fea] Added `DeleteMeta` method and simplified meta storing/reading logic
+- [fix] Fixed race condition in the JOINS cache
+- [fix] Fixed results serialization for MERGE-queries with multiple JOINS in cases when some of the JOIN-queries have not joined any items
+- [fix] Fixed `distinct` result in cases when `WHERE`-condition contains duplicate values (e.g. `distinct(id) WHERE id IN (1,1,1,3)`)
+
+## Fulltext
+- [fea] Increased max `merge_limit` value (new values range is `[1, 0x1FFFFFFF]`)
+- [fix] Fixed [phrase search](fulltext.md#phrase-search) behavior for the phrases containig single word
+
+## Go connector
+- [fea] Added `EnumMeta` and `DeleteMeta` functions
+- [fix] Fixed `RenameNamespace` function - now upsert after namespace's renaming works properly
+
+## Reindexer server
+- [fea] Add `DELETE`-method for the `/metabykey` HTTP-endpoint
+
+## Deploy
+- [fea] Added [APT-repo](cpp_src/readme.md#altlinux) for `AltLinux P10`
+
+## Face
+- [fea] Added possibility to delete namespace's Meta data
+- [fea] Added Git documentation link for the Bm25 config
+- [fea] Added explicit null-values in the Grid view
+- [fix] Fixed the inform window that appeared on the Cancel button on the NS Config page
+- [fix] Fixed extra data uploading on the Performace page
+- [fix] Changed Git documentation link in the Main menu
+- [fix] Fixed console issues on the add/edit indexes
+- [fix] Fixed caching of the NS config 
+- [fix] Fixed layout issues on the Index form
+
 # Version 3.23.1 (20.03.2024)
 
 ## Core