From 22c2939b2ab7b0e4abaad4997a8c0dc7d2840047 Mon Sep 17 00:00:00 2001
From: leoppro <zhaoyilin@pingcap.com>
Date: Thu, 22 Aug 2019 16:00:36 +0800
Subject: [PATCH] Avoid repeated called of NewIterator which might be slow
 (#718)

# Conflicts:
#	pump/storage/storage.go
---
 pump/storage/storage.go | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/pump/storage/storage.go b/pump/storage/storage.go
index 79588b358..c84bc62b2 100644
--- a/pump/storage/storage.go
+++ b/pump/storage/storage.go
@@ -33,6 +33,7 @@ import (
 	"github.com/pingcap/tidb/store/tikv/oracle"
 	pb "github.com/pingcap/tipb/go-binlog"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
 	"golang.org/x/sys/unix"
@@ -594,7 +595,13 @@ func (a *Append) doGCTS(ts int64) {
 		Start: encodeTSKey(0),
 		Limit: encodeTSKey(ts + 1),
 	}
-	iter := a.metadata.NewIterator(irange, nil)
+	var iter iterator.Iterator
+	defer func() {
+		if iter != nil {
+			iter.Release()
+			iter = nil
+		}
+	}()
 
 	for {
 		nStr, err := a.metadata.GetProperty("leveldb.num-files-at-level0")
@@ -611,6 +618,10 @@ func (a *Append) doGCTS(ts int64) {
 
 		if l0Num >= l0Trigger {
 			log.Infof("wait some time to gc cause too many L0 file, files: %d", l0Num)
+			if iter != nil {
+				iter.Release()
+				iter = nil
+			}
 			time.Sleep(5 * time.Second)
 			continue
 		}
@@ -618,6 +629,13 @@ func (a *Append) doGCTS(ts int64) {
 		deleteBatch := 0
 		var lastKey []byte
 
+		if iter == nil {
+			log.Infof("New LevelDB iterator created for GC, start: %d, limit: %d",
+				decodeTSKey(irange.Start),
+				decodeTSKey(irange.Limit))
+			iter = a.metadata.NewIterator(irange, nil)
+		}
+
 		for iter.Next() && deleteBatch < 100 {
 			batch.Delete(iter.Key())
 			deleteNum++
@@ -647,6 +665,7 @@ func (a *Append) doGCTS(ts int64) {
 		}
 
 		if len(lastKey) > 0 {
+			irange.Start = lastKey
 			a.vlog.gcTS(decodeTSKey(lastKey))
 			doneGcTSGauge.Set(float64(oracle.ExtractPhysical(uint64(decodeTSKey(lastKey)))))
 		}
@@ -654,8 +673,6 @@ func (a *Append) doGCTS(ts int64) {
 		log.Infof("has delete %d number", deleteNum)
 	}
 
-	iter.Release()
-
 	a.vlog.gcTS(ts)
 	doneGcTSGauge.Set(float64(oracle.ExtractPhysical(uint64(ts))))
 	log.Infof("finish gc, ts: %d delete num: %d", ts, deleteNum)