Merge branch 'unstable' into ptype

.github/workflows/pika.yml

@@ -2,9 +2,9 @@ name: Pika
 
 on:
   push:
-    branches: ["unstable"]
+    branches: [ "unstable" ]
   pull_request:
-    branches: ["unstable"]
+    branches: [ "unstable" ]
 
 env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
@@ -24,13 +24,19 @@ jobs:
         uses: actions/cache@v2
         id: cache
         with:
-          path: ${{ github.workspace }}/${{ env.INSTALL_LOCATION }}
+          path: |
+            ${{ github.workspace }}/${{ env.INSTALL_LOCATION }}
+            ~/.cache/pip
           key: ${{ runner.os }}-dependencies
 
       - name: install Deps
         if: ${{ steps.cache.output.cache-hit != 'true' }}
         run: |
           sudo apt install autoconf libprotobuf-dev protobuf-compiler -y
+          sudo apt-get install -y clang-tidy-12
+          sudo apt-get install -y python3-pip
+          python3 -m pip install --upgrade pip
+          python3 -m pip install redis
 
       - name: Configure CMake
         # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
@@ -52,6 +58,19 @@ jobs:
         run: |
           ./pikatests.sh all
 
+        # master on port 9221, slave on port 9231, all with 2 db
+      - name: Start pika master and slave
+        working-directory: ${{github.workspace}}/build
+        run: |
+          chmod +x ../tests/integration/start_master_and_slave.sh
+          ../tests/integration/start_master_and_slave.sh
+
+      - name: Run Python E2E Tests
+        working-directory: ${{github.workspace}}/build
+        run: |
+          python3 ../tests/integration/rpoplpush_replication_test.py
+
+
   build_on_centos:
     # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac.
     # You can convert this to a matrix build if you need cross-platform coverage.
@@ -67,6 +86,8 @@ jobs:
         run: |
           yum install -y wget git autoconf centos-release-scl
           yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-make devtoolset-10-bin-util
+          yum install -y llvm-toolset-7
+          yum install -y llvm-toolset-7-clang
           source /opt/rh/devtoolset-10/enable
           gcc --version
           make --version

.gitignore

@@ -71,3 +71,4 @@ deps
 # include codis fe javascript lib files
 !codis/cmd/fe/assets/**
 
+tests/tmp

CMakeLists.txt

@@ -97,6 +97,8 @@ if (${AUTOCONF} MATCHES AUTOCONF-NOTFOUND)
   message(FATAL_ERROR "not find autoconf on localhost")
 endif()
 
+#set(CLANG_SEARCH_PATH "/usr/local/bin" "/usr/bin" "/usr/local/opt/llvm/bin"
+#                      "/usr/local/opt/llvm@12/bin")
 find_program(CLANG_TIDY_BIN
         NAMES clang-tidy clang-tidy-12
         HINTS ${CLANG_SEARCH_PATH})

README.md

@@ -151,7 +151,7 @@ docker run -d \
   -v <db_dir>:/pika/db \
   -v <dump_dir>:/pika/dump \
   -v <dbsync_dir>:/pika/dbsync \
-  pikadb/pika:v3.3.6 
+  pikadb/pika:v3.3.6
 
 redis-cli -p 9221 "info"
 
@@ -165,13 +165,13 @@ Meaning of dirs:
 
 
 ### Build Image
-If you want to build the image yourself, we have provided a script `build_docker.sh` to simplify this process. 
+If you want to build the image yourself, we have provided a script `build_docker.sh` to simplify this process.
 
 The script accepts several optional arguments:
 
 - `-t tag`: Specify the Docker tag for the image. By default, the tag is `pikadb/pika:<git tag>`.
-- `-p platform`: Specify the platform for the Docker image. By default is current docker's platform. `all`, `linux/amd64`, `linux/arm`, `linux/arm64`. 
-- `--proxy`: Use a proxy to download packages to speed up the build process. This is particularly useful if you are in China. 
+- `-p platform`: Specify the platform for the Docker image. By default is current docker's platform. `all`, `linux/amd64`, `linux/arm`, `linux/arm64`.
+- `--proxy`: Use a proxy to download packages to speed up the build process. This is particularly useful if you are in China.
 - `--help`: Display help information.
 
 Here is an example usage of the script:
@@ -196,5 +196,7 @@ More details on [Performance](docs/benchmark/performance.md).
 
 ![](docs/images/pika-wechat.png)
 
+* [Slack Channel](https://join.slack.com/t/w1687838400-twm937235/shared_invite/zt-1y72dch5d-~9CuERHYUSmfeJZh32Z~qQ)
+
 QQ group: 294254078
 

README_CN.md

@@ -138,7 +138,7 @@ docker run -d \
   -v <db_dir>:/pika/db \
   -v <dump_dir>:/pika/dump \
   -v <dbsync_dir>:/pika/dbsync \
-  pikadb/pika:v3.3.6 
+  pikadb/pika:v3.3.6
 
 redis-cli -p 9221 "info"
 ```
@@ -154,7 +154,7 @@ redis-cli -p 9221 "info"
 - `--help`: 显示帮助信息。
 
 这是脚本的一个示例使用：
-  
+
 ```bash
 ./build_docker.sh -p linux/amd64 -t private_registry/pika:latest
 ```
@@ -398,4 +398,6 @@ Pika与Redis的极限QPS对比。
 
 ![](docs/images/pika-wechat-cn.png)
 
-QQ群：294254078
+* [Slack Channel](https://join.slack.com/t/w1687838400-twm937235/shared_invite/zt-1y72dch5d-~9CuERHYUSmfeJZh32Z~qQ)
+
+* QQ群：294254078

codis/cmd/admin/dashboard.go

@@ -68,13 +68,6 @@ func (t *cmdDashboard) Main(d map[string]interface{}) {
 	case d["--promote-server"].(bool):
 		t.handleGroupCommand(d)
 
-	case d["--sentinel-add"].(bool):
-		fallthrough
-	case d["--sentinel-del"].(bool):
-		fallthrough
-	case d["--sentinel-resync"].(bool):
-		t.handleSentinelCommand(d)
-
 	case d["--sync-action"].(bool):
 		t.handleSyncActionCommand(d)
 
@@ -632,44 +625,6 @@ func (t *cmdDashboard) handleGroupCommand(d map[string]interface{}) {
 	}
 }
 
-func (t *cmdDashboard) handleSentinelCommand(d map[string]interface{}) {
-	c := t.newTopomClient()
-
-	switch {
-
-	case d["--sentinel-add"].(bool):
-
-		addr := utils.ArgumentMust(d, "--addr")
-
-		log.Debugf("call rpc add-sentinel to dashboard %s", t.addr)
-		if err := c.AddSentinel(addr); err != nil {
-			log.PanicErrorf(err, "call rpc add-sentinel to dashboard %s failed", t.addr)
-		}
-		log.Debugf("call rpc add-sentinel OK")
-
-	case d["--sentinel-del"].(bool):
-
-		addr := utils.ArgumentMust(d, "--addr")
-
-		force := d["--force"].(bool)
-
-		log.Debugf("call rpc del-sentinel to dashboard %s", t.addr)
-		if err := c.DelSentinel(addr, force); err != nil {
-			log.PanicErrorf(err, "call rpc del-sentinel to dashboard %s failed", t.addr)
-		}
-		log.Debugf("call rpc del-sentinel OK")
-
-	case d["--sentinel-resync"].(bool):
-
-		log.Debugf("call rpc resync-sentinels to dashboard %s", t.addr)
-		if err := c.ResyncSentinels(); err != nil {
-			log.PanicErrorf(err, "call rpc resync-sentinels to dashboard %s failed", t.addr)
-		}
-		log.Debugf("call rpc resync-sentinels OK")
-
-	}
-}
-
 func (t *cmdDashboard) handleSyncActionCommand(d map[string]interface{}) {
 	c := t.newTopomClient()
 

codis/cmd/fe/assets/index.html

@@ -654,107 +654,6 @@ <h4 style="padding-left:30px; padding-right:20px; display: inline;">Group</h4>
                 </table>
             </div>
         </div>
-
-        <div class="row" style="min-width: 1200px">
-            <div class="col-md-12"
-                 style="margin-bottom: 10px; margin-top: 30px; padding-bottom: 10px; border-bottom: solid 1px lightgray;">
-                <form class="form-inline">
-                    <h4 style="padding-left:30px; padding-right:20px; display: inline;">Sentinels</h4>
-                </form>
-            </div>
-            <div class="col-md-12"
-                 style="padding-bottom: 10px">
-                <form class="form-inline">
-                    <span ng-if="codis_addr != 'NA'">
-                        <button class="btn btn-warning btn-sm active" style="width: 120px; font-size: 14px; padding: 2px;"
-                                ng-click="addSentinel(new_sentinel_addr)"
-                                ng-disabled="!new_sentinel_addr">Add Sentinel
-                        </button>
-                        <input style="height: 28px; width: 250px; " type="text" class="form-control"
-                               placeholder="Redis Sentinel Address"
-                               ng-model="new_sentinel_addr">
-                    </span>
-                </form>
-            </div>
-            <div class="col-md-4"
-                 style="padding-bottom: 10px">
-                <table class="table table-bordered table-striped table-hover table-condensed" style="white-space: nowrap">
-                    <thead>
-                    <tr>
-                        <th style="width: 85px; text-align: center">
-                            <span ng-switch="sentinel_out_of_sync">
-                                <span ng-switch-when="true">
-                                    <button class="btn btn-danger btn-xs active"
-                                            data-toggle="tooltip" data-placement="right" title="RESYNC ALL SENTINELS & PROXIES"
-                                            style="width: 70px;"
-                                            ng-click="resyncSentinels()">
-                                        SYNC
-                                    </button>
-                                </span>
-                                <span ng-switch-default>
-                                    <button class="btn btn-primary btn-xs active"
-                                            data-toggle="tooltip" data-placement="right" title="RESYNC ALL SENTINELS & PROXIES"
-                                            style="width: 70px;"
-                                            ng-click="resyncSentinels()">
-                                        SYNC
-                                    </button>
-                                </span>
-                            </span>
-                        </th>
-                        <th style="min-width: 220px;">
-                            Sentinels
-                            <span ng-if="sentinel_out_of_sync" style="color: white; background-color: red">
-                                (OUT OF SYNC)
-                            </span>
-                        </th>
-                        <th style="min-width: 35px;"/>
-                        <th style="min-width: 350px;">
-                            Status
-                        </th>
-                        <th style="min-width: 35px;"/>
-                        <th style="min-width: 35px;"/>
-                    </tr>
-                    </thead>
-                    <tr ng-repeat="sentinel in sentinel_servers">
-                        <td style="text-align: center">
-                            <a ng-href="http://[[codis_addr]]/api/topom/sentinels/info/[[sentinel.server]]/monitored" style="width: 70px;" target="_blank" class="btn btn-default btn-xs active" role="button">WATCHED</a>
-                        </td>
-                        <td>
-                            <a ng-href="http://[[codis_addr]]/api/topom/sentinels/info/[[sentinel.server]]" target="_blank" class="btn btn-default btn-xs active" role="button">S</a>
-                            <span>
-                                [[sentinel.server]]
-                            </span>
-                        </td>
-                        <td class="button_tight_column" style="text-align: center">
-                            <button class="btn btn-danger btn-xs active" ng-if="sentinel.status == 'ERROR'"
-                                data-toggle="tooltip" data-placement="right" title="FORCE REMOVE SENTINEL [[sentinel.server]]"
-                                ng-click="delSentinel(sentinel, true)">
-                                <span class="glyphicon glyphicon-remove"></span>
-                            </button>
-                        </td>
-                        <td ng-switch="sentinel.status">
-                            <span ng-switch-when="ERROR" class="status_label_error">[[sentinel.status]]</span>
-                            <span ng-switch-when="TIMEOUT" class="status_label_warning">[[sentinel.status]]</span>
-                            <span ng-switch-when="PENDING" class="status_label_pending">[[sentinel.status]]</span>
-                            <span ng-switch-default>[[sentinel.status_text]]</span>
-                        </td>
-                        <td ng-switch="sentinel.runid_error != ''">
-                            <span ng-switch-when="true" style="color: red">[[sentinel.runid_error]]</span>
-                            <span ng-switch-default/>
-                        </td>
-                        <td class="button_tight_column">
-                            <button class="btn btn-danger btn-xs active"
-                                data-toggle="tooltip" data-placement="right" title="REMOVE SENTINEL [[sentinel.server]]"
-                                ng-click="delSentinel(sentinel, false)">
-                                <span class="glyphicon glyphicon-minus"></span>
-                            </button>
-                        </td>
-                    </tr>
-                    </tbody>
-                </table>
-            </div>
-
-        </div>
     </div>
 </div>
 

codis/config/dashboard.toml

@@ -30,6 +30,10 @@ migration_async_numkeys = 500
 migration_timeout = "30s"
 
 # Set configs for redis sentinel.
+sentinel_check_server_state_interval = "5s"
+sentinel_check_master_failover_interval = "1s"
+sentinel_master_dead_check_times = 5
+
 sentinel_client_timeout = "10s"
 sentinel_quorum = 2
 sentinel_parallel_syncs = 1

codis/config/proxy.toml

@@ -75,7 +75,7 @@ backend_replica_parallel = 1
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 6
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/deploy/root/opt/codis/etc/10.4.10.200_19000/proxy.toml

@@ -58,7 +58,7 @@ backend_replica_parallel = 1
 backend_keepalive_period = "75s"
 
 # Set number of databases of backend.
-backend_number_databases = 16
+backend_number_databases = 6
 
 # If there is no request from client for a long time, the connection will be closed. (0 to disable)
 # Set session recv buffer size & timeout.

codis/pkg/models/client.go

@@ -6,9 +6,9 @@ package models
 import (
 	"time"
 
-	"pika/codis/v2/pkg/models/etcd"
-	"pika/codis/v2/pkg/models/fs"
-	"pika/codis/v2/pkg/models/zk"
+	etcdclient "pika/codis/v2/pkg/models/etcd"
+	fsclient "pika/codis/v2/pkg/models/fs"
+	zkclient "pika/codis/v2/pkg/models/zk"
 	"pika/codis/v2/pkg/utils/errors"
 )
 

codis/pkg/models/fs/fsclient.go

@@ -141,7 +141,7 @@ func (c *Client) writeFile(realpath string, data []byte, noexists bool) error {
 	if noexists {
 		_, err := os.Stat(realpath)
 		if err == nil {
-			return errors.Errorf("file already exists")
+			return errors.Errorf("file %s already exists", realpath)
 		} else if !os.IsNotExist(err) {
 			return errors.Trace(err)
 		}

codis/pkg/models/group.go

@@ -17,6 +17,22 @@ type Group struct {
 	OutOfSync bool `json:"out_of_sync"`
 }
 
+func (g *Group) GetServersMap() map[string]*GroupServer {
+	results := make(map[string]*GroupServer)
+	for _, server := range g.Servers {
+		results[server.Addr] = server
+	}
+	return results
+}
+
+type GroupServerState int8
+
+const (
+	GroupServerStateNormal GroupServerState = iota
+	GroupServerStateSubjectiveOffline
+	GroupServerStateOffline
+)
+
 type GroupServer struct {
 	Addr       string `json:"server"`
 	DataCenter string `json:"datacenter"`
@@ -26,6 +42,16 @@ type GroupServer struct {
 		State string `json:"state,omitempty"`
 	} `json:"action"`
 
+	// master or slave
+	Role string `json:"role"`
+	// If it is a master node, take the master_repl_offset field, otherwise take the slave_repl_offset field
+	ReplyOffset int `json:"reply_offset"`
+	// Monitoring status, 0 normal, 1 subjective offline, 2 actual offline
+	// If marked as 2 , no service is provided
+	State GroupServerState `json:"state"`
+
+	ReCallTimes int8 `json:"recall_times"`
+
 	ReplicaGroup bool `json:"replica_group"`
 }