From d4e6ab7f6d7ce1e102d6dd61eea4bad4a74b83c2 Mon Sep 17 00:00:00 2001 From: Ed Schouten Date: Wed, 3 Feb 2021 07:41:54 +0100 Subject: [PATCH] Tie the remote output service into bb_clientd This change extends the pathname structure offered by bb_clientd. Instead of letting the top-level directory provide immediate access to the CAS, we move this feature to a subdirectory named "cas". Two new top-level directories are added: - "outputs", which provides an implementation of the Bazel Remote Output Service. Details: https://github.com/bazelbuild/bazel/pull/12823 - "scratch", which provides a simple scratch space to test the behaviour of the FUSE file system without requiring Bazel to create an output directory for you. --- BUILD.bazel | 2 + cmd/bb_clientd/BUILD.bazel | 3 + cmd/bb_clientd/main.go | 90 ++++++++++++++++--- .../configuration/bb_clientd/BUILD.bazel | 2 + .../configuration/bb_clientd/bb_clientd.proto | 27 +++++- 5 files changed, 106 insertions(+), 18 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 6cf8dee..e898468 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -5,6 +5,8 @@ load("@bazel_gazelle//:def.bzl", "gazelle") # gazelle:resolve proto go pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore # gazelle:resolve proto pkg/proto/configuration/builder/builder.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/builder:builder_proto # gazelle:resolve proto go pkg/proto/configuration/builder/builder.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/builder +# gazelle:resolve proto pkg/proto/configuration/filesystem/filesystem.proto @com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/filesystem:filesystem_proto +# gazelle:resolve proto go pkg/proto/configuration/filesystem/filesystem.proto @com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/filesystem # gazelle:resolve proto pkg/proto/configuration/fuse/fuse.proto @com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/fuse:fuse_proto # gazelle:resolve proto go pkg/proto/configuration/fuse/fuse.proto @com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/fuse # gazelle:resolve proto pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto diff --git a/cmd/bb_clientd/BUILD.bazel b/cmd/bb_clientd/BUILD.bazel index fc6808a..ab6c1df 100644 --- a/cmd/bb_clientd/BUILD.bazel +++ b/cmd/bb_clientd/BUILD.bazel @@ -18,7 +18,9 @@ go_library( "//pkg/proto/configuration/bb_clientd", "@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution", "@com_github_buildbarn_bb_remote_execution//pkg/cas", + "@com_github_buildbarn_bb_remote_execution//pkg/filesystem", "@com_github_buildbarn_bb_remote_execution//pkg/filesystem/fuse", + "@com_github_buildbarn_bb_remote_execution//pkg/proto/remoteoutputservice", "@com_github_buildbarn_bb_storage//pkg/blobstore", "@com_github_buildbarn_bb_storage//pkg/blobstore/configuration", "@com_github_buildbarn_bb_storage//pkg/blobstore/grpcservers", @@ -27,6 +29,7 @@ go_library( "@com_github_buildbarn_bb_storage//pkg/filesystem/path", "@com_github_buildbarn_bb_storage//pkg/global", "@com_github_buildbarn_bb_storage//pkg/grpc", + "@com_github_buildbarn_bb_storage//pkg/random", "@com_github_buildbarn_bb_storage//pkg/util", "@com_github_hanwen_go_fuse_v2//fuse", "@go_googleapis//google/bytestream:bytestream_go_proto", diff --git a/cmd/bb_clientd/main.go b/cmd/bb_clientd/main.go index b8829e0..27b748f 100644 --- a/cmd/bb_clientd/main.go +++ b/cmd/bb_clientd/main.go @@ -10,7 +10,9 @@ import ( cd_fuse "github.com/buildbarn/bb-clientd/pkg/filesystem/fuse" "github.com/buildbarn/bb-clientd/pkg/proto/configuration/bb_clientd" re_cas "github.com/buildbarn/bb-remote-execution/pkg/cas" + re_filesystem "github.com/buildbarn/bb-remote-execution/pkg/filesystem" re_fuse "github.com/buildbarn/bb-remote-execution/pkg/filesystem/fuse" + "github.com/buildbarn/bb-remote-execution/pkg/proto/remoteoutputservice" blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" "github.com/buildbarn/bb-storage/pkg/builder" @@ -18,6 +20,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/filesystem/path" "github.com/buildbarn/bb-storage/pkg/global" bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc" + "github.com/buildbarn/bb-storage/pkg/random" "github.com/buildbarn/bb-storage/pkg/util" "github.com/hanwen/go-fuse/v2/fuse" @@ -57,6 +60,12 @@ func main() { log.Fatal(err) } + // Storage of files created through the FUSE file system. + filePool, err := re_filesystem.NewFilePoolFromConfiguration(configuration.FilePool) + if err != nil { + log.Fatal("Failed to create file pool: ", err) + } + // Factories for FUSE nodes corresponding to plain files, // executable files, directories and trees. // @@ -65,6 +74,9 @@ func main() { // directory and tree objects. Let's not address this for the // time being, as we mainly care about accessing individual // files. + indexedTreeFetcher := cd_cas.NewBlobAccessIndexedTreeFetcher( + contentAddressableStorage, + int(configuration.MaximumMessageSizeBytes)) globalFileContext := NewGlobalFileContext( context.Background(), contentAddressableStorage, @@ -74,11 +86,7 @@ func main() { re_cas.NewBlobAccessDirectoryFetcher( contentAddressableStorage, int(configuration.MaximumMessageSizeBytes))) - globalTreeContext := NewGlobalTreeContext( - globalFileContext, - cd_cas.NewBlobAccessIndexedTreeFetcher( - contentAddressableStorage, - int(configuration.MaximumMessageSizeBytes))) + globalTreeContext := NewGlobalTreeContext(globalFileContext, indexedTreeFetcher) // Factory function for per instance name "blobs" directories // that give access to arbitrary files, directories and trees. @@ -133,21 +141,73 @@ func main() { return d, nil } - // Top-level directory that parses instance names until a - // "blobs" pathname component is accessed. - rootDirectoryInodeNumberTree := re_fuse.NewRandomInodeNumberTree() - rootDirectory := cd_fuse.NewInstanceNameParsingDirectory( - rootDirectoryInodeNumberTree, - map[path.Component]cd_fuse.InstanceNameLookupFunc{ - path.MustNewComponent("blobs"): blobsDirectoryLookupFunc, + // Implementation of the Remote Output Service. The Remote + // Output Service allows Bazel to place its bazel-out/ + // directories on a FUSE file system, thereby allowing data to + // be loaded lazily. + var serverCallbacks re_fuse.SimpleRawFileSystemServerCallbacks + outputsInodeNumber := random.FastThreadSafeGenerator.Uint64() + outputsDirectory := cd_fuse.NewRemoteOutputServiceDirectory( + outputsInodeNumber, + random.NewFastSingleThreadedGenerator(), + serverCallbacks.EntryNotify, + func(errorLogger util.ErrorLogger, inodeNumber uint64) re_fuse.PrepopulatedDirectory { + return re_fuse.NewInMemoryPrepopulatedDirectory( + re_fuse.NewPoolBackedFileAllocator( + filePool, + errorLogger, + random.FastThreadSafeGenerator), + errorLogger, + inodeNumber, + random.FastThreadSafeGenerator, + serverCallbacks.EntryNotify) + }, + contentAddressableStorage, + indexedTreeFetcher) + + // Construct the top-level directory of the FUSE mount. It contains + // three subdirectories: + // + // - "cas": raw access to the Content Addressable Storage. + // - "outputs": outputs of builds performed using Bazel. + // - "scratch": a writable directory for testing. + rootInodeNumber := random.FastThreadSafeGenerator.Uint64() + casInodeNumberTree := re_fuse.NewRandomInodeNumberTree() + scratchInodeNumber := random.FastThreadSafeGenerator.Uint64() + rootDirectory := cd_fuse.NewStaticDirectory( + rootInodeNumber, + map[path.Component]cd_fuse.StaticDirectoryEntry{ + path.MustNewComponent("cas"): { + Child: cd_fuse.NewInstanceNameParsingDirectory( + casInodeNumberTree, + map[path.Component]cd_fuse.InstanceNameLookupFunc{ + path.MustNewComponent("blobs"): blobsDirectoryLookupFunc, + }), + InodeNumber: casInodeNumberTree.Get(), + }, + path.MustNewComponent("outputs"): { + Child: outputsDirectory, + InodeNumber: outputsInodeNumber, + }, + path.MustNewComponent("scratch"): { + Child: re_fuse.NewInMemoryPrepopulatedDirectory( + re_fuse.NewPoolBackedFileAllocator( + filePool, + util.DefaultErrorLogger, + random.FastThreadSafeGenerator), + util.DefaultErrorLogger, + scratchInodeNumber, + random.FastThreadSafeGenerator, + serverCallbacks.EntryNotify), + InodeNumber: scratchInodeNumber, + }, }) // Expose the FUSE file system. - var serverCallbacks re_fuse.SimpleRawFileSystemServerCallbacks if err := re_fuse.NewMountFromConfiguration( configuration.Fuse, rootDirectory, - rootDirectoryInodeNumberTree.Get(), + rootInodeNumber, &serverCallbacks, "bb_clientd"); err != nil { log.Fatal("Failed to mount FUSE file system: ", err) @@ -177,6 +237,8 @@ func main() { 1<<16)) remoteexecution.RegisterCapabilitiesServer(s, buildQueue) remoteexecution.RegisterExecutionServer(s, buildQueue) + + remoteoutputservice.RegisterRemoteOutputServiceServer(s, outputsDirectory) })) }() diff --git a/pkg/proto/configuration/bb_clientd/BUILD.bazel b/pkg/proto/configuration/bb_clientd/BUILD.bazel index 9b56d93..064c357 100644 --- a/pkg/proto/configuration/bb_clientd/BUILD.bazel +++ b/pkg/proto/configuration/bb_clientd/BUILD.bazel @@ -7,6 +7,7 @@ proto_library( srcs = ["bb_clientd.proto"], visibility = ["//visibility:public"], deps = [ + "@com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/filesystem:filesystem_proto", "@com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/fuse:fuse_proto", "@com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto", "@com_github_buildbarn_bb_storage//pkg/proto/configuration/builder:builder_proto", @@ -21,6 +22,7 @@ go_proto_library( proto = ":bb_clientd_proto", visibility = ["//visibility:public"], deps = [ + "@com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/filesystem", "@com_github_buildbarn_bb_remote_execution//pkg/proto/configuration/fuse", "@com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore", "@com_github_buildbarn_bb_storage//pkg/proto/configuration/builder", diff --git a/pkg/proto/configuration/bb_clientd/bb_clientd.proto b/pkg/proto/configuration/bb_clientd/bb_clientd.proto index 7ddccfe..6e3dbbe 100644 --- a/pkg/proto/configuration/bb_clientd/bb_clientd.proto +++ b/pkg/proto/configuration/bb_clientd/bb_clientd.proto @@ -4,6 +4,7 @@ package buildbarn.configuration.bb_clientd; import "pkg/proto/configuration/blobstore/blobstore.proto"; import "pkg/proto/configuration/builder/builder.proto"; +import "pkg/proto/configuration/filesystem/filesystem.proto"; import "pkg/proto/configuration/fuse/fuse.proto"; import "pkg/proto/configuration/global/global.proto"; import "pkg/proto/configuration/grpc/grpc.proto"; @@ -25,15 +26,28 @@ message ApplicationConfiguration { // Storage (CAS) are exposed for reading. The following pathname // schemes are supported: // - // - ${instance_name}/blobs/directory/${hash}-${size_bytes}: + // - cas/${instance_name}/blobs/directory/${hash}-${size_bytes}/: // View the contents of a Directory object. - // - ${instance_name}/blobs/executable/${hash}-${size_bytes}: + // - cas/${instance_name}/blobs/executable/${hash}-${size_bytes}: // Access a single file, having the executable bit (+x) set. - // - ${instance_name}/blobs/file/${hash}-${size_bytes}: + // - cas/${instance_name}/blobs/file/${hash}-${size_bytes}: // Access a single file, having the executable bit (+x) clear. - // - ${instance_name}/blobs/tree/${hash}-${size_bytes}: + // - cas/${instance_name}/blobs/tree/${hash}-${size_bytes}/: // View the contents of a Tree object. // + // - outputs/${output_base}/: + // Location where Bazel may store the per-workspace bazel-out/ + // directory. Files yielded by remote build actions are loaded + // lazily. Using this feature gives the same performance + // improvements as --remote_download_minimal, with the added + // advantage that remote output files remain accessible locally. + // More details: https://github.com/bazelbuild/bazel/pull/12823 + // + // - scratch/: + // A writable directory where arbitrary path layouts may be + // constructed. Files that reference CAS objects may be created by + // hardlinking them from the "cas" directory. + // // Instance names containing slashes are permitted. It automatically // causes intermediate directories to be created. "blobs" directories // exist at every level. @@ -46,4 +60,9 @@ message ApplicationConfiguration { // the key corresponds to the instance name prefix. map schedulers = 6; + + // Location where files are stored that are created in the "outputs" + // and "scratch" directories. Files that are hardlinked from the "cas" + // directory don't take up any space in the file pool. + buildbarn.configuration.filesystem.FilePoolConfiguration file_pool = 7; }