ddl: limit the count of getting ddlhistory jobs

docs/tidb_http_api.md

@@ -565,13 +565,13 @@ timezone.*
 
     **Note**: If you request a TiDB that is not ddl owner, the response will be `This node is not a ddl owner, can't be resigned.`
 
-25. Get all TiDB DDL job history information.
+25. Get the TiDB DDL job history information.
 
      ```shell
      curl http://{TiDBIP}:10080/ddl/history
      ```
 
-     **Note**: When the DDL history is very very long, it may consume a lot memory and even cause OOM. Consider adding `start_job_id` and `limit`.
+     **Note**: When the DDL history is very very long, it may contains too many jobs. This interface will get 1024 history ddl jobs by default. If you want get more jobs, consider adding `start_job_id` and `limit`.
 
 26. Get count {number} TiDB DDL job history information.
 

pkg/ddl/ddl_history.go

@@ -37,6 +37,8 @@ import (
 const (
 	DefNumHistoryJobs   = 10
 	batchNumHistoryJobs = 128
+	// DefNumGetDDLHistoryJobs is the max count for getting the ddl history once.
+	DefNumGetDDLHistoryJobs = 1024
 )
 
 // AddHistoryDDLJob record the history job.
@@ -150,9 +152,15 @@ func GetAllHistoryDDLJobs(m *meta.Meta) ([]*model.Job, error) {
 func ScanHistoryDDLJobs(m *meta.Meta, startJobID int64, limit int) ([]*model.Job, error) {
 	var iter meta.LastJobIterator
 	var err error
+
 	if startJobID == 0 {
+		// if 'start_job_id' == 0 and 'limit' == 0(default value), get the last 1024 ddl history job by defaultly.
+		if limit == 0 {
+			limit = DefNumGetDDLHistoryJobs
+		}
 		iter, err = m.GetLastHistoryDDLJobsIterator()
 	} else {
+		// if 'start_job_id' > 0, it must set value to 'limit'
 		if limit == 0 {
 			return nil, errors.New("when 'start_job_id' is specified, it must work with a 'limit'")
 		}

pkg/ddl/ddl_history_test.go

@@ -33,6 +33,10 @@ import (
 )
 
 func TestDDLHistoryBasic(t *testing.T) {
+	var (
+		ddlHistoryJobCount = 0
+	)
+
 	store := testkit.CreateMockStore(t)
 	rs := pools.NewResourcePool(func() (pools.Resource, error) {
 		newTk := testkit.NewTestKit(t, store)
@@ -78,8 +82,9 @@ func TestDDLHistoryBasic(t *testing.T) {
 
 	err = kv.RunInNewTxn(ctx, store, false, func(ctx context.Context, txn kv.Transaction) error {
 		m := meta.NewMeta(txn)
-		_, err := ddl.GetAllHistoryDDLJobs(m)
+		jobs, err := ddl.GetAllHistoryDDLJobs(m)
 		require.NoError(t, err)
+		ddlHistoryJobCount = len(jobs)
 		return nil
 	})
 
@@ -96,4 +101,32 @@ func TestDDLHistoryBasic(t *testing.T) {
 	})
 
 	require.NoError(t, err)
+
+	err = kv.RunInNewTxn(ctx, store, false, func(ctx context.Context, txn kv.Transaction) error {
+		m := meta.NewMeta(txn)
+		jobs, err := ddl.ScanHistoryDDLJobs(m, 0, 0)
+		require.NoError(t, err)
+		if ddlHistoryJobCount <= ddl.DefNumGetDDLHistoryJobs {
+			require.Equal(t, ddlHistoryJobCount, len(jobs))
+		} else {
+			require.Equal(t, ddl.DefNumGetDDLHistoryJobs, len(jobs))
+		}
+		require.True(t, len(jobs) > 2)
+		require.Equal(t, int64(2), jobs[ddlHistoryJobCount-2].ID)
+		require.Equal(t, int64(1), jobs[ddlHistoryJobCount-1].ID)
+		return nil
+	})
+
+	require.NoError(t, err)
+}
+
+func TestScanHistoryDDLJobsWithErrorLimit(t *testing.T) {
+	var (
+		m                = &meta.Meta{}
+		startJobID int64 = 10
+		limit            = 0
+	)
+
+	_, err := ddl.ScanHistoryDDLJobs(m, startJobID, limit)
+	require.ErrorContains(t, err, "when 'start_job_id' is specified, it must work with a 'limit'")
 }

pkg/server/handler/tests/http_handler_test.go

@@ -16,6 +16,7 @@ package tests
 
 import (
 	"bytes"
+	"cmp"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
@@ -1063,6 +1064,11 @@ func TestAllHistory(t *testing.T) {
 	data, err := ddl.GetAllHistoryDDLJobs(txnMeta)
 	require.NoError(t, err)
 	err = decoder.Decode(&jobs)
+	require.True(t, len(jobs) < ddl.DefNumGetDDLHistoryJobs)
+	// sort job.
+	slices.SortFunc(jobs, func(i, j *model.Job) int {
+		return cmp.Compare(i.ID, j.ID)
+	})
 
 	require.NoError(t, err)
 	require.NoError(t, resp.Body.Close())

pkg/server/handler/tikvhandler/tikv_handler.go

@@ -1125,8 +1125,11 @@ func (h *TableHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 // ServeHTTP handles request of ddl jobs history.
 func (h DDLHistoryJobHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	var jobID, limitID int
-	var err error
+	var (
+		jobID   = 0
+		limitID = 0
+		err     error
+	)
 	if jobValue := req.FormValue(handler.JobID); len(jobValue) > 0 {
 		jobID, err = strconv.Atoi(jobValue)
 		if err != nil {
@@ -1144,8 +1147,9 @@ func (h DDLHistoryJobHandler) ServeHTTP(w http.ResponseWriter, req *http.Request
 			handler.WriteError(w, err)
 			return
 		}
-		if limitID < 1 {
-			handler.WriteError(w, errors.New("ddl history limit must be greater than 0"))
+		if limitID < 1 || limitID > ddl.DefNumGetDDLHistoryJobs {
+			handler.WriteError(w,
+				errors.Errorf("ddl history limit must be greater than 0 and less than or equal to %v", ddl.DefNumGetDDLHistoryJobs))
 			return
 		}
 	}
@@ -1165,11 +1169,7 @@ func (h DDLHistoryJobHandler) getHistoryDDL(jobID, limit int) (jobs []*model.Job
 	}
 	txnMeta := meta.NewMeta(txn)
 
-	if jobID == 0 && limit == 0 {
-		jobs, err = ddl.GetAllHistoryDDLJobs(txnMeta)
-	} else {
-		jobs, err = ddl.ScanHistoryDDLJobs(txnMeta, int64(jobID), limit)
-	}
+	jobs, err = ddl.ScanHistoryDDLJobs(txnMeta, int64(jobID), limit)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}