diff --git a/Makefile b/Makefile index d19a6b31..e6381e66 100644 --- a/Makefile +++ b/Makefile @@ -12,3 +12,4 @@ gen: cd plugin && buf generate cd plugin/pluginexec && buf generate cd plugin/plugingroup && buf generate + cd internal/hproto && buf generate diff --git a/internal/engine/engine.go b/internal/engine/engine.go index d17e936d..b0656141 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/hephbuild/heph/internal/hcore/hlog" "github.com/hephbuild/heph/internal/hcore/hstep" "github.com/hephbuild/heph/internal/hfs" @@ -65,6 +67,7 @@ type Engine struct { Drivers []pluginv1connect.DriverClient DriversHandle map[pluginv1connect.DriverClient]PluginHandle DriversByName map[string]pluginv1connect.DriverClient + DriversConfig map[string]*pluginv1.ConfigResponse } func New(ctx context.Context, root string, cfg Config) (*Engine, error) { diff --git a/internal/engine/local_cache.go b/internal/engine/local_cache.go index 327f1827..39047cee 100644 --- a/internal/engine/local_cache.go +++ b/internal/engine/local_cache.go @@ -3,11 +3,9 @@ package engine import ( "context" "encoding/hex" - "encoding/json" "errors" "fmt" "io" - "os" "time" "github.com/hephbuild/heph/internal/hartifact" @@ -109,15 +107,13 @@ func (e *Engine) CacheLocally(ctx context.Context, def *LightLinkedTarget, hashi }) } - manifestfs := hfs.At(cachedir, ArtifactManifestName) - - m := Manifest{ + m := hartifact.Manifest{ Version: "v1", CreatedAt: time.Now(), Hashin: hashin, } for _, artifact := range cacheArtifacts { - m.Artifacts = append(m.Artifacts, ManifestArtifact{ + m.Artifacts = append(m.Artifacts, hartifact.ManifestArtifact{ Hashout: artifact.Hashout, Group: artifact.Group, Name: artifact.Name, @@ -126,32 +122,18 @@ func (e *Engine) CacheLocally(ctx context.Context, def *LightLinkedTarget, hashi }) } - b, err := json.Marshal(m) //nolint:musttag - if err != nil { - return nil, err - } - - err = hfs.WriteFile(manifestfs, "", b, os.ModePerm) + manifestArtifact, err := hartifact.NewManifestArtifact(cachedir, m) if err != nil { return nil, err } cacheArtifacts = append(cacheArtifacts, ExecuteResultArtifact{ - Artifact: manifestV1Artifact(cachedir), + Artifact: manifestArtifact, }) return cacheArtifacts, nil } -func manifestV1Artifact(fs hfs.OS) *pluginv1.Artifact { - return &pluginv1.Artifact{ - Name: ArtifactManifestName, - Type: pluginv1.Artifact_TYPE_MANIFEST_V1, - Encoding: pluginv1.Artifact_ENCODING_NONE, - Uri: "file://" + hfs.At(fs, ArtifactManifestName).Path(), - } -} - func (e *Engine) ResultFromLocalCache(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { multi := hlocks.NewMulti() @@ -182,7 +164,7 @@ func (e *Engine) resultFromLocalCacheInner( dirfs := hfs.At(e.Cache, def.Ref.GetPackage(), "__"+def.Ref.GetName(), hashin) { - l := hlocks.NewFlock2(dirfs, "", ArtifactManifestName, false) + l := hlocks.NewFlock2(dirfs, "", hartifact.ManifestName, false) err := l.RLock(ctx) if err != nil { return nil, false, err @@ -190,18 +172,12 @@ func (e *Engine) resultFromLocalCacheInner( locks.Add(l.RUnlock) } - mainfestb, err := hfs.ReadFile(dirfs, ArtifactManifestName) + manifest, err := hartifact.ManifestFromFS(dirfs) if err != nil { return nil, false, err } - var manifest Manifest - err = json.Unmarshal(mainfestb, &manifest) //nolint:musttag - if err != nil { - return nil, false, err - } - - var artifacts []ManifestArtifact + var artifacts []hartifact.ManifestArtifact for _, output := range outputs { outputArtifacts := manifest.GetArtifacts(output) @@ -231,8 +207,13 @@ func (e *Engine) resultFromLocalCacheInner( }) } + manifestArtifact, err := hartifact.NewManifestArtifact(dirfs, manifest) + if err != nil { + return nil, false, err + } + execArtifacts = append(execArtifacts, ExecuteResultArtifact{ - Artifact: manifestV1Artifact(dirfs), + Artifact: manifestArtifact, }) return &ExecuteResult{ diff --git a/internal/engine/manifest.go b/internal/engine/manifest.go deleted file mode 100644 index 55624f0e..00000000 --- a/internal/engine/manifest.go +++ /dev/null @@ -1,38 +0,0 @@ -package engine - -import ( - "time" - - pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" -) - -var ArtifactManifestName = "manifest.v1.json" - -type ManifestArtifact struct { - Hashout string - - Group string - Name string - Type pluginv1.Artifact_Type - Encoding pluginv1.Artifact_Encoding -} - -type Manifest struct { - Version string - CreatedAt time.Time - Hashin string - Artifacts []ManifestArtifact -} - -func (m Manifest) GetArtifacts(output string) []ManifestArtifact { - a := make([]ManifestArtifact, 0) - for _, artifact := range m.Artifacts { - if artifact.Group != output { - continue - } - - a = append(a, artifact) - } - - return a -} diff --git a/internal/engine/plugins.go b/internal/engine/plugins.go index 4ce2d51e..99df68c3 100644 --- a/internal/engine/plugins.go +++ b/internal/engine/plugins.go @@ -182,10 +182,14 @@ func (e *Engine) RegisterDriver(ctx context.Context, handler pluginv1connect.Dri if e.DriversHandle == nil { e.DriversHandle = map[pluginv1connect.DriverClient]PluginHandle{} } + if e.DriversConfig == nil { + e.DriversConfig = map[string]*pluginv1.ConfigResponse{} + } e.Drivers = append(e.Drivers, client) e.DriversByName[res.Msg.GetName()] = client e.DriversHandle[client] = pluginh + e.DriversConfig[res.Msg.GetName()] = res.Msg err = e.initPlugin(ctx, handler) if err != nil { diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index f44277e2..21b55dcf 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -13,6 +13,8 @@ import ( "sync" "time" + "github.com/hephbuild/heph/internal/hproto" + "connectrpc.com/connect" "github.com/dlsniper/debugger" "github.com/hephbuild/heph/internal/hcore/hlog" @@ -266,8 +268,15 @@ func (e *Engine) hashin(ctx context.Context, def *LightLinkedTarget, results []* return "", err } - // TODO support fieldmask of things to include in hashin - b, err = proto.Marshal(def.Def) + defHash := def.Def + if ignoreFromHash := e.DriversConfig[def.Ref.GetDriver()].GetIgnoreFromHash(); len(ignoreFromHash) > 0 { + defHash, err = hproto.RemoveMasked(defHash, ignoreFromHash) + if err != nil { + return "", err + } + } + + b, err = proto.Marshal(defHash) if err != nil { return "", err } diff --git a/internal/enginee2e/hash_deps_test.go b/internal/enginee2e/hash_deps_test.go new file mode 100644 index 00000000..67755244 --- /dev/null +++ b/internal/enginee2e/hash_deps_test.go @@ -0,0 +1,83 @@ +package enginee2e + +import ( + "context" + "os" + "testing" + "time" + + "github.com/hephbuild/heph/internal/engine" + "github.com/hephbuild/heph/internal/hartifact" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/hephbuild/heph/plugin/pluginexec" + "github.com/hephbuild/heph/plugin/pluginstaticprovider" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestHashDeps(t *testing.T) { + ctx := context.Background() + + dir, err := os.MkdirTemp("", "") + require.NoError(t, err) + defer os.RemoveAll(dir) + + e, err := engine.New(ctx, dir, engine.Config{}) + require.NoError(t, err) + + staticprovider := pluginstaticprovider.NewFunc(func() []pluginstaticprovider.Target { + return []pluginstaticprovider.Target{ + { + Spec: &pluginv1.TargetSpec{ + Ref: &pluginv1.TargetRef{ + Package: "some/package", + Name: "sometarget", + Driver: "sh", + }, + Config: map[string]*structpb.Value{ + "run": newValueMust([]any{`echo hello > out`}), + "out": newValueMust([]any{"out"}), + "runtime_env": newValueMust(map[string]any{ + "VAR1": time.Now().String(), + }), + }, + }, + }, + } + }) + + _, err = e.RegisterProvider(ctx, staticprovider) + require.NoError(t, err) + + _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) + require.NoError(t, err) + + var at time.Time + { + ch := e.Result(ctx, "some/package", "sometarget", []string{""}, engine.ResultOptions{}) + + res := <-ch + require.NoError(t, res.Err) + + require.Len(t, res.Artifacts, 2) + + m, err := hartifact.ManifestFromArtifact(ctx, res.Artifacts[1].Artifact) + require.NoError(t, err) + + at = m.CreatedAt + } + + { + ch := e.Result(ctx, "some/package", "sometarget", []string{""}, engine.ResultOptions{}) + + res := <-ch + require.NoError(t, res.Err) + + require.Len(t, res.Artifacts, 2) + + m, err := hartifact.ManifestFromArtifact(ctx, res.Artifacts[1].Artifact) + require.NoError(t, err) + + require.Equal(t, at, m.CreatedAt) + } +} diff --git a/internal/enginee2e/env_test.go b/internal/enginee2e/src_out_env_test.go similarity index 99% rename from internal/enginee2e/env_test.go rename to internal/enginee2e/src_out_env_test.go index 80d659f1..5bf15845 100644 --- a/internal/enginee2e/env_test.go +++ b/internal/enginee2e/src_out_env_test.go @@ -14,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) -func TestEnv(t *testing.T) { +func TestSrcOutEnv(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "") diff --git a/internal/hartifact/manifest.go b/internal/hartifact/manifest.go new file mode 100644 index 00000000..1dea3273 --- /dev/null +++ b/internal/hartifact/manifest.go @@ -0,0 +1,93 @@ +package hartifact + +import ( + "context" + "encoding/json" + "io" + "os" + "time" + + "github.com/hephbuild/heph/internal/hfs" + + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" +) + +var ManifestName = "manifest.v1.json" + +type ManifestArtifact struct { + Hashout string + + Group string + Name string + Type pluginv1.Artifact_Type + Encoding pluginv1.Artifact_Encoding +} + +type Manifest struct { + Version string + CreatedAt time.Time + Hashin string + Artifacts []ManifestArtifact +} + +func (m Manifest) GetArtifacts(output string) []ManifestArtifact { + a := make([]ManifestArtifact, 0) + for _, artifact := range m.Artifacts { + if artifact.Group != output { + continue + } + + a = append(a, artifact) + } + + return a +} + +func NewManifestArtifact(fs hfs.FS, m Manifest) (*pluginv1.Artifact, error) { + b, err := json.Marshal(m) //nolint:musttag + if err != nil { + return nil, err + } + + err = hfs.WriteFile(fs, ManifestName, b, os.ModePerm) + if err != nil { + return nil, err + } + + return &pluginv1.Artifact{ + Name: ManifestName, + Type: pluginv1.Artifact_TYPE_MANIFEST_V1, + Encoding: pluginv1.Artifact_ENCODING_NONE, + Uri: "file://" + hfs.At(fs, ManifestName).Path(), + }, nil +} + +func ManifestFromArtifact(ctx context.Context, a *pluginv1.Artifact) (Manifest, error) { + r, err := Reader(ctx, a) + if err != nil { + return Manifest{}, err + } + defer r.Close() + + return openManifest(r) +} + +func ManifestFromFS(fs hfs.FS) (Manifest, error) { + f, err := hfs.Open(fs, ManifestName) + if err != nil { + return Manifest{}, err + } + defer f.Close() + + return openManifest(f) +} + +func openManifest(r io.Reader) (Manifest, error) { + var manifest Manifest + err := json.NewDecoder(r).Decode(&manifest) //nolint:musttag + if err != nil { + return Manifest{}, err + } + + return manifest, nil +} diff --git a/internal/hproto/.gitignore b/internal/hproto/.gitignore new file mode 100644 index 00000000..d1da35d0 --- /dev/null +++ b/internal/hproto/.gitignore @@ -0,0 +1 @@ +internal/gen diff --git a/internal/hproto/buf.gen.yaml b/internal/hproto/buf.gen.yaml new file mode 100644 index 00000000..be8c8935 --- /dev/null +++ b/internal/hproto/buf.gen.yaml @@ -0,0 +1,14 @@ +version: v2 +clean: true +managed: + enabled: true + override: + - file_option: go_package_prefix + value: github.com/hephbuild/heph/internal/hproto/internal/gen +plugins: + - remote: buf.build/protocolbuffers/go + out: internal/gen + opt: paths=source_relative +inputs: + - directory: internal/proto + diff --git a/internal/hproto/internal/proto/heph/hproto/v1/sample.proto b/internal/hproto/internal/proto/heph/hproto/v1/sample.proto new file mode 100644 index 00000000..bbdb7cae --- /dev/null +++ b/internal/hproto/internal/proto/heph/hproto/v1/sample.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; +package heph.hproto.v1; + +message Sample { + message Nested { + repeated string included = 1; + repeated string excluded = 2; + } + + repeated string included = 1; + repeated string excluded = 2; + Nested nested = 3; + repeated Nested repeated_nested = 4; +} diff --git a/internal/hproto/proto.go b/internal/hproto/proto.go index 68bf4553..45ae9bc3 100644 --- a/internal/hproto/proto.go +++ b/internal/hproto/proto.go @@ -1,7 +1,63 @@ package hproto -import "google.golang.org/protobuf/proto" +import ( + "fmt" + "slices" + "strings" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protopath" + "google.golang.org/protobuf/reflect/protorange" + "google.golang.org/protobuf/reflect/protoreflect" +) func Clone[T proto.Message](m T) T { return proto.Clone(m).(T) //nolint:errcheck } + +func protoPathValueToDotPath(p protopath.Values) string { + segments := make([]string, 0, len(p.Path)) + for _, step := range p.Path { + switch step.Kind() { //nolint:exhaustive,gocritic + case protopath.FieldAccessStep: + segments = append(segments, step.FieldDescriptor().TextName()) + } + } + + return strings.Join(segments, ".") +} + +func RemoveMasked[T proto.Message](m T, paths []string) (T, error) { + m = Clone(m) + + err := protorange.Range(m.ProtoReflect(), func(p protopath.Values) error { + if !slices.Contains(paths, protoPathValueToDotPath(p)) { + return nil + } + + last := p.Index(-1) + + beforeLast := p.Index(-2) + switch last.Step.Kind() { //nolint:exhaustive + case protopath.FieldAccessStep: + m := beforeLast.Value.Message() + fd := last.Step.FieldDescriptor() + m.Clear(fd) + case protopath.ListIndexStep: + ls := beforeLast.Value.List() + i := last.Step.ListIndex() + // TODO: figure out how to remove + ls.Set(i, protoreflect.ValueOfMessage(ls.Get(i).Message().Type().New())) + case protopath.MapIndexStep: + ms := beforeLast.Value.Map() + k := last.Step.MapIndex() + ms.Clear(k) + default: + return fmt.Errorf("unsupported field access step: %v", last.Step.Kind()) + } + + return nil + }) + + return m, err +} diff --git a/internal/hproto/remove_mask_test.go b/internal/hproto/remove_mask_test.go new file mode 100644 index 00000000..2f6753fb --- /dev/null +++ b/internal/hproto/remove_mask_test.go @@ -0,0 +1,40 @@ +package hproto + +import ( + "testing" + + "google.golang.org/protobuf/encoding/protojson" + + hprotov1 "github.com/hephbuild/heph/internal/hproto/internal/gen/heph/hproto/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRemoveMasked(t *testing.T) { + m := &hprotov1.Sample{ + Included: []string{"hello"}, + Excluded: []string{"world"}, + Nested: &hprotov1.Sample_Nested{ + Included: []string{"hello"}, + Excluded: []string{"world"}, + }, + RepeatedNested: []*hprotov1.Sample_Nested{{ + Included: []string{"hello"}, + Excluded: []string{"world"}, + }}, + } + + m2, err := RemoveMasked(m, []string{"excluded", "nested.excluded", "repeated_nested.excluded"}) + require.NoError(t, err) + + b, err := protojson.Marshal(m2) + require.NoError(t, err) + assert.JSONEq(t, `{"included":["hello"],"nested":{"included":["hello"]},"repeatedNested":[{"included":["hello"]}]}`, string(b)) +} + +func TestRemoveNil(t *testing.T) { + m := &hprotov1.Sample{} + + _, err := RemoveMasked(m, []string{"excluded", "nested.excluded", "repeated_nested.excluded"}) + require.NoError(t, err) +} diff --git a/plugin/pluginexec/plugin.go b/plugin/pluginexec/plugin.go index bc467860..7f901bb8 100644 --- a/plugin/pluginexec/plugin.go +++ b/plugin/pluginexec/plugin.go @@ -78,6 +78,11 @@ func (p *Plugin) Config(ctx context.Context, c *connect.Request[pluginv1.ConfigR return connect.NewResponse(&pluginv1.ConfigResponse{ Name: p.name, TargetSchema: desc, + IgnoreFromHash: []string{ + "runtime_deps", + "runtime_env", + "runtime_pass_env", + }, }), nil } @@ -90,8 +95,14 @@ func (p *Plugin) Parse(ctx context.Context, req *connect.Request[pluginv1.ParseR } s := &execv1.Target{ - Run: targetSpec.Run, - Deps: map[string]*execv1.Target_Dep{}, + Run: targetSpec.Run, + Deps: map[string]*execv1.Target_Dep{}, + HashDeps: map[string]*execv1.Target_Dep{}, + RuntimeDeps: map[string]*execv1.Target_Dep{}, + Env: targetSpec.Env, + RuntimeEnv: targetSpec.RuntimeEnv, + PassEnv: targetSpec.PassEnv, + RuntimePassEnv: targetSpec.RuntimePassEnv, } var allOutputPaths []string diff --git a/plugin/pluginexec/proto/heph/plugin/exec/v1/plugin.proto b/plugin/pluginexec/proto/heph/plugin/exec/v1/plugin.proto index 653b0189..f9fed0b6 100644 --- a/plugin/pluginexec/proto/heph/plugin/exec/v1/plugin.proto +++ b/plugin/pluginexec/proto/heph/plugin/exec/v1/plugin.proto @@ -19,5 +19,11 @@ message Target { repeated string run = 1; map deps = 2; - repeated Output outputs = 3; + map hash_deps = 3; + map runtime_deps = 4; + repeated Output outputs = 5; + map env = 6; + map runtime_env = 7; + repeated string pass_env = 8; + repeated string runtime_pass_env = 9; } diff --git a/plugin/pluginexec/spec.go b/plugin/pluginexec/spec.go index cd2558b1..29e0660c 100644 --- a/plugin/pluginexec/spec.go +++ b/plugin/pluginexec/spec.go @@ -51,10 +51,16 @@ func (s *SpecOutputs) MapstructureDecode(v any) error { } type Spec struct { - Run SpecStrings `mapstructure:"run"` - Deps SpecDeps `mapstructure:"deps"` - Out SpecOutputs `mapstructure:"out"` - Cache bool `mapstructure:"cache"` - Pty bool `mapstructure:"pty"` - Codegen string `mapstructure:"codegen"` + Run SpecStrings `mapstructure:"run"` + Deps SpecDeps `mapstructure:"deps"` + HashDeps SpecDeps `mapstructure:"hash_deps"` + RuntimeDeps SpecDeps `mapstructure:"runtime_deps"` + Out SpecOutputs `mapstructure:"out"` + Cache bool `mapstructure:"cache"` + Pty bool `mapstructure:"pty"` + Codegen string `mapstructure:"codegen"` + Env map[string]string `mapstructure:"env"` + RuntimeEnv map[string]string `mapstructure:"runtime_env"` + PassEnv []string `mapstructure:"pass_env"` + RuntimePassEnv []string `mapstructure:"runtime_pass_env"` } diff --git a/plugin/pluginstaticprovider/plugin.go b/plugin/pluginstaticprovider/plugin.go index 695a5052..73c6899c 100644 --- a/plugin/pluginstaticprovider/plugin.go +++ b/plugin/pluginstaticprovider/plugin.go @@ -15,17 +15,25 @@ type Target struct { } type Plugin struct { - targets []Target + f func() []Target } func New(targets []Target) *Plugin { return &Plugin{ - targets: targets, + func() []Target { + return targets + }, + } +} + +func NewFunc(f func() []Target) *Plugin { + return &Plugin{ + f: f, } } func (p *Plugin) List(ctx context.Context, req *connect.Request[pluginv1.ListRequest], res *connect.ServerStream[pluginv1.ListResponse]) error { - for _, target := range p.targets { + for _, target := range p.f() { if req.Msg.GetPackage() != "" { if req.Msg.GetDeep() { if !strings.HasPrefix(target.Spec.GetRef().GetPackage(), req.Msg.GetPackage()) { @@ -50,7 +58,7 @@ func (p *Plugin) List(ctx context.Context, req *connect.Request[pluginv1.ListReq } func (p *Plugin) Get(ctx context.Context, req *connect.Request[pluginv1.GetRequest]) (*connect.Response[pluginv1.GetResponse], error) { - for _, target := range p.targets { + for _, target := range p.f() { if target.Spec.GetRef().GetPackage() != req.Msg.GetRef().GetPackage() || target.Spec.GetRef().GetName() != req.Msg.GetRef().GetName() { continue } diff --git a/plugin/proto/heph/plugin/v1/driver.proto b/plugin/proto/heph/plugin/v1/driver.proto index 21db7d3e..0318bb68 100644 --- a/plugin/proto/heph/plugin/v1/driver.proto +++ b/plugin/proto/heph/plugin/v1/driver.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package heph.plugin.v1; import "google/protobuf/struct.proto"; -import "google/protobuf/any.proto"; import "google/protobuf/descriptor.proto"; import "heph/plugin/v1/plugin.proto"; @@ -59,6 +58,7 @@ message ConfigRequest {} message ConfigResponse { string name = 1; google.protobuf.DescriptorProto target_schema = 2; + repeated string ignore_from_hash = 9; } message PipeRequest {} diff --git a/plugin/proto/heph/plugin/v1/plugin.proto b/plugin/proto/heph/plugin/v1/plugin.proto index 2f094022..b369aa8c 100644 --- a/plugin/proto/heph/plugin/v1/plugin.proto +++ b/plugin/proto/heph/plugin/v1/plugin.proto @@ -3,6 +3,7 @@ package heph.plugin.v1; import "google/protobuf/struct.proto"; import "google/protobuf/any.proto"; +import "google/protobuf/field_mask.proto"; message TargetRef { string package = 1;