From 3a81114fe3fa221214e78d8ea9259226b8e64b9a Mon Sep 17 00:00:00 2001
From: Sam Calder-Mason <sam@puritydev.io>
Date: Thu, 10 Oct 2024 12:05:19 +1000
Subject: [PATCH 1/3] feat(disk): Refactor disk usage calc
---
pkg/exporter/disk/disk.go | 43 +++++++++++++++------------------------
1 file changed, 16 insertions(+), 27 deletions(-)
diff --git a/pkg/exporter/disk/disk.go b/pkg/exporter/disk/disk.go
index 95b9365..06283b8 100644
--- a/pkg/exporter/disk/disk.go
+++ b/pkg/exporter/disk/disk.go
@@ -3,6 +3,7 @@ package disk
import (
"context"
"os"
+ "path/filepath"
"time"
"github.com/sirupsen/logrus"
@@ -51,15 +52,17 @@ func (d *diskUsage) GetUsage(ctx context.Context, directories []string) ([]Usage
var diskUsed []Usage
for _, directory := range directories {
- info, err := os.Lstat(directory)
+ _, err := os.Lstat(directory)
if err != nil {
d.log.WithField("directory", directory).Warn("Directory does not exist")
+
continue
}
- used, err := getDiskUsed(directory, info)
+ used, err := getDiskUsed(directory)
if err != nil {
d.log.WithField("directory", directory).WithError(err).Error("Failed to get usage")
+
continue
}
@@ -76,33 +79,19 @@ func (d *diskUsage) GetUsage(ctx context.Context, directories []string) ([]Usage
return diskUsed, nil
}
-func getDiskUsed(currentPath string, info os.FileInfo) (int64, error) {
- size := info.Size()
- if !info.IsDir() {
- return size, nil
- }
-
- dir, err := os.Open(currentPath)
-
- if err != nil {
- return size, err
- }
- defer dir.Close()
-
- files, err := dir.Readdir(-1)
- if err != nil {
- return size, err
- }
-
- for _, file := range files {
- if file.Name() == "." || file.Name() == ".." {
- continue
+func getDiskUsed(path string) (int64, error) {
+ var size int64
+ err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
}
- s, _ := getDiskUsed(currentPath+"/"+file.Name(), file)
+ if !info.IsDir() {
+ size += info.Size()
+ }
- size += s
- }
+ return err
+ })
- return size, nil
+ return size, err
}
From 185b9d4b8252747dc3223692e79b0cf17ee64636 Mon Sep 17 00:00:00 2001
From: Sam Calder-Mason <sam@puritydev.io>
Date: Thu, 10 Oct 2024 12:15:02 +1000
Subject: [PATCH 2/3] feat(disk): Refactor disk usage calc
---
example_config.yaml | 1 +
pkg/exporter/config.go | 7 +++++--
pkg/exporter/disk/disk.go | 13 +++++++++++--
pkg/exporter/exporter.go | 24 +++++++++++++++++++++---
4 files changed, 38 insertions(+), 7 deletions(-)
diff --git a/example_config.yaml b/example_config.yaml
index 4db96b8..57d37da 100644
--- a/example_config.yaml
+++ b/example_config.yaml
@@ -13,5 +13,6 @@ execution:
- "txpool"
diskUsage:
enabled: false
+ interval: 30m
directories:
- /data/ethereum
\ No newline at end of file
diff --git a/pkg/exporter/config.go b/pkg/exporter/config.go
index 7a213ae..e3a923c 100644
--- a/pkg/exporter/config.go
+++ b/pkg/exporter/config.go
@@ -1,5 +1,7 @@
package exporter
+import "github.com/ethpandaops/beacon/pkg/human"
+
// Config holds the configuration for the ethereum sync status tool.
type Config struct {
// Execution is the execution node to use.
@@ -29,8 +31,9 @@ type ExecutionNode struct {
// DiskUsage configures the exporter to expose disk usage stats for these directories.
type DiskUsage struct {
- Enabled bool `yaml:"enabled"`
- Directories []string `yaml:"directories"`
+ Enabled bool `yaml:"enabled"`
+ Directories []string `yaml:"directories"`
+ Interval human.Duration `yaml:"interval"`
}
// PairConfig holds the config for a Pair of Execution and Consensus Clients
diff --git a/pkg/exporter/disk/disk.go b/pkg/exporter/disk/disk.go
index 06283b8..3b8e24e 100644
--- a/pkg/exporter/disk/disk.go
+++ b/pkg/exporter/disk/disk.go
@@ -21,24 +21,33 @@ type diskUsage struct {
log logrus.FieldLogger
metrics Metrics
directories []string
+ interval time.Duration
}
// NewUsage returns a new DiskUsage instance.
-func NewUsage(ctx context.Context, log logrus.FieldLogger, namespace string, directories []string) (UsageMetrics, error) {
+func NewUsage(ctx context.Context, log logrus.FieldLogger, namespace string, directories []string, interval time.Duration) (UsageMetrics, error) {
return &diskUsage{
log: log,
metrics: NewMetrics(log, namespace),
directories: directories,
+ interval: interval,
}, nil
}
func (d *diskUsage) StartAsync(ctx context.Context) {
+ d.log.WithField("directories", d.directories).Info("Starting disk usage metrics...")
+
+ _, err := d.GetUsage(ctx, d.directories)
+ if err != nil {
+ d.log.WithError(err).Error("Failed to get disk usage")
+ }
+
go func() {
for {
select {
case <-ctx.Done():
return
- case <-time.After(time.Second * 60):
+ case <-time.After(d.interval):
if _, err := d.GetUsage(ctx, d.directories); err != nil {
d.log.WithError(err).Error("Failed to get disk usage")
}
diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go
index 8a0c950..b45437b 100644
--- a/pkg/exporter/exporter.go
+++ b/pkg/exporter/exporter.go
@@ -53,7 +53,14 @@ func (e *exporter) Init(ctx context.Context) error {
if e.config.Execution.Enabled {
e.log.WithField("modules", strings.Join(e.config.Execution.Modules, ", ")).Info("Initializing execution...")
- executionNode, err := execution.NewExecutionNode(ctx, e.log.WithField("exporter", "execution"), fmt.Sprintf("%s_exe", e.namespace), e.config.Execution.Name, e.config.Execution.URL, e.config.Execution.Modules)
+ executionNode, err := execution.NewExecutionNode(
+ ctx,
+ e.log.WithField("exporter", "execution"),
+ fmt.Sprintf("%s_exe", e.namespace),
+ e.config.Execution.Name,
+ e.config.Execution.URL,
+ e.config.Execution.Modules,
+ )
if err != nil {
return err
}
@@ -68,7 +75,18 @@ func (e *exporter) Init(ctx context.Context) error {
if e.config.DiskUsage.Enabled {
e.log.Info("Initializing disk usage...")
- diskUsage, err := disk.NewUsage(ctx, e.log.WithField("exporter", "disk"), fmt.Sprintf("%s_disk", e.namespace), e.config.DiskUsage.Directories)
+ interval := e.config.DiskUsage.Interval.Duration
+ if interval == 0 {
+ interval = 30 * time.Minute
+ }
+
+ diskUsage, err := disk.NewUsage(
+ ctx,
+ e.log.WithField("exporter", "disk"),
+ fmt.Sprintf("%s_disk", e.namespace),
+ e.config.DiskUsage.Directories,
+ interval,
+ )
if err != nil {
return err
}
@@ -130,7 +148,7 @@ func (e *exporter) Serve(ctx context.Context, port int) error {
return nil
}
-func (e *exporter) bootstrapConsensusClients(ctx context.Context) error {
+func (e *exporter) bootstrapConsensusClients(_ context.Context) error {
opts := *beacon.DefaultOptions().
EnableDefaultBeaconSubscription().
EnablePrometheusMetrics()
From d6d8486d8ef9368b824ab1b81b7d0db655f13882 Mon Sep 17 00:00:00 2001
From: Sam Calder-Mason <sam@puritydev.io>
Date: Thu, 10 Oct 2024 12:17:21 +1000
Subject: [PATCH 3/3] fix: Update disk usage interval to 60 minutes
---
example_config.yaml | 2 +-
pkg/exporter/config.go | 9 ++++++++-
pkg/exporter/exporter.go | 2 +-
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/example_config.yaml b/example_config.yaml
index 57d37da..87d496c 100644
--- a/example_config.yaml
+++ b/example_config.yaml
@@ -13,6 +13,6 @@ execution:
- "txpool"
diskUsage:
enabled: false
- interval: 30m
+ interval: 60m
directories:
- /data/ethereum
\ No newline at end of file
diff --git a/pkg/exporter/config.go b/pkg/exporter/config.go
index e3a923c..0109b3e 100644
--- a/pkg/exporter/config.go
+++ b/pkg/exporter/config.go
@@ -1,6 +1,10 @@
package exporter
-import "github.com/ethpandaops/beacon/pkg/human"
+import (
+ "time"
+
+ "github.com/ethpandaops/beacon/pkg/human"
+)
// Config holds the configuration for the ethereum sync status tool.
type Config struct {
@@ -58,6 +62,9 @@ func DefaultConfig() *Config {
DiskUsage: DiskUsage{
Enabled: false,
Directories: []string{},
+ Interval: human.Duration{
+ Duration: 60 * time.Minute,
+ },
},
Pair: PairConfig{
Enabled: true,
diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go
index b45437b..1536b1f 100644
--- a/pkg/exporter/exporter.go
+++ b/pkg/exporter/exporter.go
@@ -77,7 +77,7 @@ func (e *exporter) Init(ctx context.Context) error {
interval := e.config.DiskUsage.Interval.Duration
if interval == 0 {
- interval = 30 * time.Minute
+ interval = 60 * time.Minute
}
diskUsage, err := disk.NewUsage(