refactor: remove Bytes()

base/array.go

@@ -14,8 +14,6 @@
 
 package base
 
-import "reflect"
-
 const batchSize = 1024 * 1024
 
 type Array[T any] struct {
@@ -39,16 +37,3 @@ func (a *Array[T]) Append(val T) {
 	}
 	a.Data[len(a.Data)-1] = append(a.Data[len(a.Data)-1], val)
 }
-
-func (a *Array[T]) Bytes() int {
-	// The memory usage of Array[T] consists of:
-	// 1. struct
-	// 2. slices in s.Data[*]
-	// 3. elements in s.Data[*][*]
-	bytes := reflect.TypeOf(a).Elem().Size()
-	if len(a.Data) > 0 {
-		bytes += reflect.TypeOf(a.Data).Elem().Size() * uintptr(cap(a.Data))
-		bytes += reflect.TypeOf(a.Data).Elem().Elem().Size() * uintptr(len(a.Data)) * batchSize
-	}
-	return int(bytes)
-}

base/array_test.go

@@ -1,8 +1,9 @@
 package base
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestArray(t *testing.T) {
@@ -15,5 +16,4 @@ func TestArray(t *testing.T) {
 	for i := 0; i < 123; i++ {
 		assert.Equal(t, int32(i), a.Get(i))
 	}
-	assert.Equal(t, 48+4*batchSize, a.Bytes())
 }

base/encoding/encoding.go

@@ -20,10 +20,8 @@ import (
 	"encoding/gob"
 	"fmt"
 	"io"
-	"reflect"
 	"strconv"
 
-	"github.com/chewxy/math32"
 	"github.com/juju/errors"
 )
 
@@ -128,38 +126,3 @@ func ReadGob(r io.Reader, v interface{}) error {
 func FormatFloat32(val float32) string {
 	return strconv.FormatFloat(float64(val), 'f', -1, 32)
 }
-
-func ParseFloat32(val string) float32 {
-	f, err := strconv.ParseFloat(val, 32)
-	if err != nil {
-		return math32.NaN()
-	}
-	return float32(f)
-}
-
-func ArrayBytes[T any](val []T) uintptr {
-	byteCount := reflect.TypeOf(val).Size()
-	if len(val) > 0 {
-		byteCount += reflect.TypeOf(val).Elem().Size() * uintptr(len(val))
-	}
-	return byteCount
-}
-
-func MatrixBytes[T any](val [][]T) uintptr {
-	byteCount := reflect.TypeOf(val).Size()
-	if len(val) > 0 {
-		rowSize := len(val[0])
-		byteCount += reflect.TypeOf(val).Elem().Size() * uintptr(len(val))
-		byteCount += reflect.TypeOf(val).Elem().Elem().Size() * uintptr(len(val)*rowSize)
-	}
-	return byteCount
-}
-
-func StringsBytes(val []string) uintptr {
-	byteCount := reflect.TypeOf(val).Size()
-	byteCount += reflect.TypeOf(val).Elem().Size() * uintptr(len(val))
-	for _, s := range val {
-		byteCount += 2 * uintptr(len(s))
-	}
-	return byteCount
-}

base/encoding/encoding_test.go

@@ -17,8 +17,9 @@ package encoding
 import (
 	"bytes"
 	"fmt"
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestHex(t *testing.T) {
@@ -57,37 +58,3 @@ func TestWriteGob(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, a, b)
 }
-
-func TestFloat32(t *testing.T) {
-	a := FormatFloat32(1.23)
-	assert.Equal(t, "1.23", a)
-	b := ParseFloat32("1.23")
-	assert.Equal(t, float32(1.23), b)
-}
-
-func TestArrayBytes(t *testing.T) {
-	a := []int64{1, 2, 3, 4}
-	assert.Equal(t, 7*8, int(ArrayBytes(a)))
-	b := []int32{1, 2, 3, 4}
-	assert.Equal(t, 3*8+4*4, int(ArrayBytes(b)))
-	c := []int16{1, 2, 3, 4}
-	assert.Equal(t, 3*8+2*4, int(ArrayBytes(c)))
-	d := []int8{1, 2, 3, 4}
-	assert.Equal(t, 3*8+4, int(ArrayBytes(d)))
-}
-
-func TestMatrixBytes(t *testing.T) {
-	a := [][]int64{{1}, {2}, {3}, {4}}
-	assert.Equal(t, 5*24+4*8, int(MatrixBytes(a)))
-	b := [][]int32{{1}, {2}, {3}, {4}}
-	assert.Equal(t, 5*24+4*4, int(MatrixBytes(b)))
-	c := [][]int16{{1}, {2}, {3}, {4}}
-	assert.Equal(t, 5*24+4*2, int(MatrixBytes(c)))
-	d := [][]int8{{1}, {2}, {3}, {4}}
-	assert.Equal(t, 5*24+4, int(MatrixBytes(d)))
-}
-
-func TestStringsBytes(t *testing.T) {
-	a := []string{"abc", "de", "f"}
-	assert.Equal(t, 24+16*3+12, int(StringsBytes(a)))
-}

base/index.go

@@ -16,11 +16,11 @@ package base
 
 import (
 	"encoding/binary"
-	"github.com/juju/errors"
-	"github.com/zhenghaoz/gorse/base/encoding"
 	"io"
-	"reflect"
 	"strconv"
+
+	"github.com/juju/errors"
+	"github.com/zhenghaoz/gorse/base/encoding"
 )
 
 // Index keeps the mapping between names (string) and indices (integer).
@@ -32,7 +32,6 @@ type Index interface {
 	GetNames() []string
 	Marshal(w io.Writer) error
 	Unmarshal(r io.Reader) error
-	Bytes() int
 }
 
 const (
@@ -89,9 +88,8 @@ func UnmarshalIndex(r io.Reader) (Index, error) {
 // a user ID or item ID. The dense index is the internal user index or item index
 // optimized for faster parameter access and less memory usage.
 type MapIndex struct {
-	Numbers    map[string]int32 // sparse ID -> dense index
-	Names      []string         // dense index -> sparse ID
-	Characters int
+	Numbers map[string]int32 // sparse ID -> dense index
+	Names   []string         // dense index -> sparse ID
 }
 
 // NotId represents an ID doesn't exist.
@@ -118,7 +116,6 @@ func (idx *MapIndex) Add(name string) {
 	if _, exist := idx.Numbers[name]; !exist {
 		idx.Numbers[name] = int32(len(idx.Names))
 		idx.Names = append(idx.Names, name)
-		idx.Characters += len(name)
 	}
 }
 
@@ -178,24 +175,6 @@ func (idx *MapIndex) Unmarshal(r io.Reader) error {
 	return nil
 }
 
-func (idx *MapIndex) Bytes() int {
-	// The memory usage of MapIndex consists of:
-	// 1. struct
-	// 2. string in idx.Names
-	// 3. int32 (value) in idx.Numbers
-	// 4. string (key) in idx.Numbers
-	// 5. rune in string
-	// The cost of map is omitted.
-	bytes := reflect.TypeOf(idx).Elem().Size()
-	if idx.Len() > 0 {
-		bytes += reflect.TypeOf(idx.Names).Elem().Size() * uintptr(cap(idx.Names))
-		bytes += reflect.TypeOf(idx.Numbers).Elem().Size() * uintptr(len(idx.Numbers))
-		bytes += reflect.TypeOf(idx.Numbers).Key().Size() * uintptr(len(idx.Numbers))
-		bytes += reflect.TypeOf(rune(0)).Size() * uintptr(idx.Characters)
-	}
-	return int(bytes)
-}
-
 // DirectIndex means that the name and its index is the same. For example,
 // the index of "1" is 1, vice versa.
 type DirectIndex struct {
@@ -261,7 +240,3 @@ func (idx *DirectIndex) Marshal(w io.Writer) error {
 func (idx *DirectIndex) Unmarshal(r io.Reader) error {
 	return binary.Read(r, binary.LittleEndian, &idx.Limit)
 }
-
-func (idx *DirectIndex) Bytes() int {
-	return int(reflect.TypeOf(idx).Elem().Size())
-}

base/index_test.go

@@ -2,9 +2,10 @@ package base
 
 import (
 	"bytes"
-	"github.com/stretchr/testify/assert"
 	"strconv"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestMapIndex(t *testing.T) {
@@ -38,8 +39,6 @@ func TestMapIndex(t *testing.T) {
 	indexCopy, err := UnmarshalIndex(buf)
 	assert.NoError(t, err)
 	assert.Equal(t, index, indexCopy)
-	// Index size
-	assert.Equal(t, 200, index.Bytes())
 }
 
 func TestDirectIndex(t *testing.T) {
@@ -77,6 +76,4 @@ func TestDirectIndex(t *testing.T) {
 	indexCopy, err := UnmarshalIndex(buf)
 	assert.NoError(t, err)
 	assert.Equal(t, index, indexCopy)
-	// Byte size
-	assert.Equal(t, 4, index.Bytes())
 }

common/sizeof/size_test.go

@@ -0,0 +1,64 @@
+// Copyright 2025 gorse Project Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sizeof
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCyclic(t *testing.T) {
+	type V struct {
+		Z int
+		E *V
+	}
+
+	v := &V{Z: 25}
+	want := DeepSize(v)
+	v.E = v // induce a cycle
+	got := DeepSize(v)
+	if got != want {
+		t.Errorf("Cyclic size: got %d, want %d", got, want)
+	}
+}
+
+func TestDeepSize(t *testing.T) {
+	// matrix
+	a := [][]int64{{1}, {2}, {3}, {4}}
+	assert.Equal(t, 5*24+4*8, DeepSize(a))
+	b := [][]int32{{1}, {2}, {3}, {4}}
+	assert.Equal(t, 5*24+4*4, DeepSize(b))
+	c := [][]int16{{1}, {2}, {3}, {4}}
+	assert.Equal(t, 5*24+4*2, DeepSize(c))
+	d := [][]int8{{1}, {2}, {3}, {4}}
+	assert.Equal(t, 5*24+4, DeepSize(d))
+
+	// strings
+	e := []string{"abc", "de", "f"}
+	assert.Equal(t, 24+16*3+6, DeepSize(e))
+	f := []string{"♥♥♥", "♥♥", "♥"}
+	assert.Equal(t, 24+16*3+18, DeepSize(f))
+
+	// slice
+	g := []int64{1, 2, 3, 4}
+	assert.Equal(t, 7*8, DeepSize(g))
+	h := []int32{1, 2, 3, 4}
+	assert.Equal(t, 3*8+4*4, DeepSize(h))
+	i := []int16{1, 2, 3, 4}
+	assert.Equal(t, 3*8+2*4, DeepSize(i))
+	j := []int8{1, 2, 3, 4}
+	assert.Equal(t, 3*8+4, DeepSize(j))
+}

master/master.go

@@ -34,8 +34,8 @@ import (
 	"github.com/zhenghaoz/gorse/base/log"
 	"github.com/zhenghaoz/gorse/base/parallel"
 	"github.com/zhenghaoz/gorse/base/progress"
-	"github.com/zhenghaoz/gorse/base/sizeof"
 	"github.com/zhenghaoz/gorse/base/task"
+	"github.com/zhenghaoz/gorse/common/sizeof"
 	"github.com/zhenghaoz/gorse/common/util"
 	"github.com/zhenghaoz/gorse/config"
 	"github.com/zhenghaoz/gorse/model"
@@ -201,7 +201,7 @@ func (m *Master) Serve() {
 		CollaborativeFilteringPrecision10.Set(float64(m.rankingScore.Precision))
 		CollaborativeFilteringRecall10.Set(float64(m.rankingScore.Recall))
 		CollaborativeFilteringNDCG10.Set(float64(m.rankingScore.NDCG))
-		MemoryInUseBytesVec.WithLabelValues("collaborative_filtering_model").Set(float64(m.RankingModel.Bytes()))
+		MemoryInUseBytesVec.WithLabelValues("collaborative_filtering_model").Set(float64(sizeof.DeepSize(m.RankingModel)))
 	}
 	if m.localCache.ClickModel != nil {
 		log.Logger().Info("load cached click model",

master/rest.go

@@ -38,10 +38,10 @@ import (
 	"github.com/rakyll/statik/fs"
 	"github.com/samber/lo"
 	"github.com/zhenghaoz/gorse/base"
-	"github.com/zhenghaoz/gorse/base/encoding"
 	"github.com/zhenghaoz/gorse/base/log"
 	"github.com/zhenghaoz/gorse/base/progress"
 	"github.com/zhenghaoz/gorse/cmd/version"
+	"github.com/zhenghaoz/gorse/common/util"
 	"github.com/zhenghaoz/gorse/config"
 	"github.com/zhenghaoz/gorse/model/click"
 	"github.com/zhenghaoz/gorse/model/ranking"
@@ -584,7 +584,10 @@ func (m *Master) getStats(request *restful.Request, response *restful.Response)
 		if temp, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.MatchingIndexRecall)).String(); err != nil {
 			log.ResponseLogger(response).Warn("failed to get matching index recall", zap.Error(err))
 		} else {
-			status.MatchingIndexRecall = encoding.ParseFloat32(temp)
+			status.MatchingIndexRecall, err = util.ParseFloat[float32](temp)
+			if err != nil {
+				log.ResponseLogger(response).Warn("failed to parse matching index recall", zap.Error(err))
+			}
 		}
 	}
 	server.Ok(response, status)