Added statfs for block-cache (#1470)

* Added statfs for block_cache

CHANGELOG.md

@@ -1,8 +1,12 @@
 ## 2.4.1 (Unreleased)
 **Bug Fixes**
 - Create block pool only in the child process.
+- Correct statFS results to reflect block-cache in memory cache status.
 
-## 2.4.0 (Unreleased)
+**Other Changes**
+- Optimized listing operation on HNS account to support symlinks.
+
+## 2.4.0 (2024-12-03)
 **Features**
 - Added 'gen-config' command to auto generate the recommended blobfuse2 config file based on computing resources and memory available on the node. Command details can be found with `blobfuse2 gen-config --help`.
 - Added option to set Entry cache to hold directory listing results in cache for a given timeout. This will reduce REST calls going to storage and enables faster access across multiple applications that use Blobfuse on the same node.
@@ -20,6 +24,7 @@
 - `Stream` option automatically replaced with "Stream with Block-cache" internally for optimized performance.
 - Login via Managed Identify is supported with Object-ID for all versions of blobfuse except 2.3.0 and 2.3.2.To use Object-ID for these two versions, use AzCLI or utilize Application/Client-ID or Resource ID base authentication..
 - Version check is now moved to a static website hosted on a public container.
+- 'df' command output will present memory availability in case of block-cache if disk is not configured.
 
 ## 2.3.2 (2024-09-03)
 **Bug Fixes**

component/block_cache/block_cache.go

@@ -1759,6 +1759,36 @@ func (bc *BlockCache) SyncFile(options internal.SyncFileOptions) error {
 	return nil
 }
 
+func (bc *BlockCache) StatFs() (*syscall.Statfs_t, bool, error) {
+	var maxCacheSize uint64
+	if bc.diskSize > 0 {
+		maxCacheSize = bc.diskSize
+	} else {
+		maxCacheSize = bc.memSize
+	}
+
+	if maxCacheSize == 0 {
+		return nil, false, nil
+	}
+
+	usage, _ := common.GetUsage(bc.tmpPath)
+	usage = usage * float64(_1MB)
+
+	available := (float64)(maxCacheSize) - usage
+	statfs := &syscall.Statfs_t{}
+	err := syscall.Statfs("/", statfs)
+	if err != nil {
+		log.Debug("BlockCache::StatFs : statfs err [%s].", err.Error())
+		return nil, false, err
+	}
+	statfs.Frsize = int64(bc.blockSize)
+	statfs.Blocks = uint64(maxCacheSize) / uint64(bc.blockSize)
+	statfs.Bavail = uint64(math.Max(0, available)) / uint64(bc.blockSize)
+	statfs.Bfree = statfs.Bavail
+
+	return statfs, true, nil
+}
+
 // ------------------------- Factory -------------------------------------------
 // Pipeline will call this method to create your object, initialize your variables here
 // << DO NOT DELETE ANY AUTO GENERATED CODE HERE >>

component/block_cache/block_cache_test.go

@@ -47,6 +47,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -236,6 +237,58 @@ func (suite *blockCacheTestSuite) TestFreeDiskSpace() {
 	suite.assert.LessOrEqual(difference, tolerance)
 }
 
+func (suite *blockCacheTestSuite) TestStatfsMemory() {
+	emptyConfig := "read-only: true\n\nblock_cache:\n  block-size-mb: 16\n"
+	tobj, err := setupPipeline(emptyConfig)
+	defer tobj.cleanupPipeline()
+
+	suite.assert.Nil(err)
+	suite.assert.Equal(tobj.blockCache.Name(), "block_cache")
+	cmd := exec.Command("bash", "-c", "free -b | grep Mem | awk '{print $4}'")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err = cmd.Run()
+	suite.assert.Nil(err)
+	free, err := strconv.Atoi(strings.TrimSpace(out.String()))
+	suite.assert.Nil(err)
+	expected := uint64(0.8 * float64(free))
+	stat, ret, err := tobj.blockCache.StatFs()
+	suite.assert.Equal(ret, true)
+	suite.assert.Equal(err, nil)
+	suite.assert.NotEqual(stat, &syscall.Statfs_t{})
+	actual := tobj.blockCache.memSize
+	difference := math.Abs(float64(actual) - float64(expected))
+	tolerance := 0.10 * float64(math.Max(float64(actual), float64(expected)))
+	suite.assert.LessOrEqual(difference, tolerance)
+}
+
+func (suite *blockCacheTestSuite) TestStatfsDisk() {
+	disk_cache_path := getFakeStoragePath("fake_storage")
+	config := fmt.Sprintf("read-only: true\n\nblock_cache:\n  block-size-mb: 1\n  path: %s", disk_cache_path)
+	tobj, err := setupPipeline(config)
+	defer tobj.cleanupPipeline()
+
+	suite.assert.Nil(err)
+	suite.assert.Equal(tobj.blockCache.Name(), "block_cache")
+
+	cmd := exec.Command("bash", "-c", fmt.Sprintf("df -B1 %s | awk 'NR==2{print $4}'", disk_cache_path))
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err = cmd.Run()
+	suite.assert.Nil(err)
+	freeDisk, err := strconv.Atoi(strings.TrimSpace(out.String()))
+	suite.assert.Nil(err)
+	expected := uint64(0.8 * float64(freeDisk))
+	stat, ret, err := tobj.blockCache.StatFs()
+	suite.assert.Equal(ret, true)
+	suite.assert.Equal(err, nil)
+	suite.assert.NotEqual(stat, &syscall.Statfs_t{})
+	actual := tobj.blockCache.diskSize
+	difference := math.Abs(float64(actual) - float64(expected))
+	tolerance := 0.10 * float64(math.Max(float64(actual), float64(expected)))
+	suite.assert.LessOrEqual(difference, tolerance)
+}
+
 func (suite *blockCacheTestSuite) TestInvalidPrefetchCount() {
 	cfg := "read-only: true\n\nblock_cache:\n  block-size-mb: 16\n  mem-size-mb: 500\n  prefetch: 8\n  parallelism: 10\n  path: abcd\n  disk-size-mb: 100\n  disk-timeout-sec: 5"
 	tobj, err := setupPipeline(cfg)

component/file_cache/file_cache.go

@@ -298,9 +298,9 @@ func (c *FileCache) Configure(_ bool) error {
 	err = syscall.Statfs(c.tmpPath, &stat)
 	if err != nil {
 		log.Err("FileCache::Configure : config error %s [%s]. Assigning a default value of 4GB or if any value is assigned to .disk-size-mb in config.", c.Name(), err.Error())
-		c.maxCacheSize = 4192 * MB
+		c.maxCacheSize = 4192
 	} else {
-		c.maxCacheSize = 0.8 * float64(stat.Bavail) * float64(stat.Bsize)
+		c.maxCacheSize = (0.8 * float64(stat.Bavail) * float64(stat.Bsize)) / (MB)
 	}
 
 	if config.IsSet(compName+".max-size-mb") && conf.MaxSizeMB != 0 {

component/file_cache/file_cache_test.go

@@ -208,7 +208,7 @@ func (suite *fileCacheTestSuite) TestDefaultCacheSize() {
 	freeDisk, err := strconv.Atoi(strings.TrimSpace(out.String()))
 	suite.assert.Nil(err)
 	expected := uint64(0.8 * float64(freeDisk))
-	actual := suite.fileCache.maxCacheSize
+	actual := suite.fileCache.maxCacheSize * MB
 	difference := math.Abs(float64(actual) - float64(expected))
 	tolerance := 0.10 * float64(math.Max(float64(actual), float64(expected)))
 	suite.assert.LessOrEqual(difference, tolerance, "mssg:", actual, expected)