internal/cuetxtar: allow using a fallback golden set

This allows different implementations of the same functionality
to share the same golden file set, if overlapping.

This introduces a version type for the evaluator used by TestEvalAlpha.
TestEvalAlpha is added as a test case for this. As the version has no
effect as of now, it tests that the fallback mechanism works properly
for all current tests.

Signed-off-by: Marcel van Lohuizen <[email protected]>
Change-Id: I71484daa6eb3817e575d3ada2320e4608b2a0f17
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1167861
Reviewed-by: Daniel Martí <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>

Author: mpvl

internal/core/adt/context.go

@@ -197,6 +197,16 @@ func New(v *Vertex, cfg *Config) *OpContext {
 	return ctx
 }
 
+type EvaluatorVersion int
+
+const (
+	DefaultVersion EvaluatorVersion = iota
+
+	// The DevVersion is used for new implementations of the evaluator that
+	// do not cover all features of the CUE language yet.
+	DevVersion
+)
+
 // An OpContext implements CUE's unification operation. It only
 // operates on values that are created with the Runtime with which an OpContext
 // is associated. An OpContext is not goroutine safe and only one goroutine may
@@ -205,6 +215,8 @@ type OpContext struct {
 	Runtime
 	Format func(Node) string
 
+	Version EvaluatorVersion
+
 	nest int
 
 	stats        stats.Counts

internal/core/adt/eval_test.go

@@ -24,6 +24,7 @@ import (
 
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/cuecontext"
+	"cuelang.org/go/cue/errors"
 	"cuelang.org/go/internal/core/adt"
 	"cuelang.org/go/internal/core/debug"
 	"cuelang.org/go/internal/core/eval"
@@ -51,7 +52,7 @@ func TestEval(t *testing.T) {
 	}
 
 	test.Run(t, func(tc *cuetxtar.Test) {
-		runEvalTest(tc)
+		runEvalTest(tc, adt.DefaultVersion)
 	})
 }
 
@@ -63,7 +64,29 @@ var needFix = map[string]string{
 	"DIR/NAME": "reason",
 }
 
-func runEvalTest(t *cuetxtar.Test) {
+var todoAlpha = map[string]string{
+	"DIR/NAME": "reason",
+}
+
+func TestEvalAlpha(t *testing.T) {
+	test := cuetxtar.TxTarTest{
+		Root:     "../../../cue/testdata",
+		Name:     "evalalpha",
+		Fallback: "eval", // Allow eval golden files to pass these tests.
+		Skip:     alwaysSkip,
+		ToDo:     todoAlpha,
+	}
+
+	if *todo {
+		test.ToDo = nil
+	}
+
+	test.Run(t, func(t *cuetxtar.Test) {
+		runEvalTest(t, adt.DevVersion)
+	})
+}
+
+func runEvalTest(t *cuetxtar.Test, version adt.EvaluatorVersion) {
 	a := t.Instance()
 	r := runtime.New()
 
@@ -75,6 +98,7 @@ func runEvalTest(t *cuetxtar.Test) {
 
 	e := eval.New(r)
 	ctx := e.NewContext(v)
+	ctx.Version = version
 	v.Finalize(ctx)
 
 	stats := ctx.Stats()
@@ -103,8 +127,11 @@ func runEvalTest(t *cuetxtar.Test) {
 
 // TestX is for debugging. Do not delete.
 func TestX(t *testing.T) {
-	verbosity := 0
-	verbosity = 1 // uncomment to turn logging off.
+	var verbosity int
+	verbosity = 1 // comment to turn logging off.
+
+	var version adt.EvaluatorVersion
+	version = adt.DevVersion // comment to use default implementation.
 
 	in := `
 -- cue.mod/module.cue --
@@ -136,11 +163,15 @@ module: "mod.test"
 
 	e := eval.New(r)
 	ctx := e.NewContext(v)
+	ctx.Version = version
 	v.Finalize(ctx)
 	adt.Verbosity = 0
 
-	// b := validate.Validate(ctx, v, &validate.Config{Concrete: true})
-	// t.Log(errors.Details(b.Err, nil))
+	if b := validate.Validate(ctx, v, &validate.Config{
+		AllErrors: true,
+	}); b != nil {
+		t.Log(errors.Details(b.Err, nil))
+	}
 
 	t.Error(debug.NodeString(r, v, nil))
 

internal/cuetxtar/txtar.go

@@ -49,6 +49,12 @@ type TxTarTest struct {
 	// TODO: by default derive from the current base directory name.
 	Name string
 
+	// Fallback allows the golden tests named by Fallback to pass tests in
+	// case the golden file corresponding to Name does not exist.
+	// The feature can be used to have two implementations of the same
+	// functionality share the same test sets.
+	Fallback string
+
 	// Skip is a map of tests to skip; the key is the test name; the value is the
 	// skip message.
 	Skip map[string]string
@@ -92,6 +98,7 @@ type Test struct {
 	*testing.T
 
 	prefix   string
+	fallback string
 	buf      *bytes.Buffer // the default buffer
 	outFiles []file
 
@@ -109,14 +116,15 @@ type Test struct {
 func (t *Test) Write(b []byte) (n int, err error) {
 	if t.buf == nil {
 		t.buf = &bytes.Buffer{}
-		t.outFiles = append(t.outFiles, file{t.prefix, t.buf})
+		t.outFiles = append(t.outFiles, file{t.prefix, t.fallback, t.buf})
 	}
 	return t.buf.Write(b)
 }
 
 type file struct {
-	name string
-	buf  *bytes.Buffer
+	name     string
+	fallback string
+	buf      *bytes.Buffer
 }
 
 // HasTag reports whether the tag with the given key is defined
@@ -201,10 +209,13 @@ func (t *Test) WriteFile(f *ast.File) {
 // in the txtar file. If name is empty, data will be written to the test
 // output and checked against "out/\(testName)".
 func (t *Test) Writer(name string) io.Writer {
+	var fallback string
 	switch name {
 	case "":
 		name = t.prefix
+		fallback = t.fallback
 	default:
+		fallback = path.Join(t.fallback, name)
 		name = path.Join(t.prefix, name)
 	}
 
@@ -215,7 +226,7 @@ func (t *Test) Writer(name string) io.Writer {
 	}
 
 	w := &bytes.Buffer{}
-	t.outFiles = append(t.outFiles, file{name, w})
+	t.outFiles = append(t.outFiles, file{name, fallback, w})
 
 	if name == t.prefix {
 		t.buf = w
@@ -326,6 +337,11 @@ func (x *TxTarTest) Run(t *testing.T, f func(tc *Test)) {
 				prefix:     path.Join("out", x.Name),
 				LoadConfig: x.LoadConfig,
 			}
+			if x.Fallback != "" {
+				tc.fallback = path.Join("out", x.Fallback)
+			} else {
+				tc.fallback = tc.prefix
+			}
 
 			if tc.HasTag("skip") {
 				t.Skip()
@@ -341,13 +357,14 @@ func (x *TxTarTest) Run(t *testing.T, f func(tc *Test)) {
 			update := false
 
 			for i, f := range a.Files {
-
-				if strings.HasPrefix(f.Name, tc.prefix) && (f.Name == tc.prefix || f.Name[len(tc.prefix)] == '/') {
+				hasPrefix := func(s string) bool {
 					// It's either "\(tc.prefix)" or "\(tc.prefix)/..." but not some other name
 					// that happens to start with tc.prefix.
-					tc.hasGold = true
+					return strings.HasPrefix(f.Name, s) && (f.Name == s || f.Name[len(s)] == '/')
 				}
 
+				tc.hasGold = hasPrefix(tc.prefix) || hasPrefix(tc.fallback)
+
 				// Format CUE files as required
 				if tc.HasTag("noformat") || !strings.HasSuffix(f.Name, ".cue") {
 					continue
@@ -377,6 +394,10 @@ func (x *TxTarTest) Run(t *testing.T, f func(tc *Test)) {
 					k = i
 					break
 				}
+				if i, ok := index[sub.fallback]; ok {
+					k = i
+					break
+				}
 			}
 
 			files := a.Files[:k:k]
@@ -394,6 +415,15 @@ func (x *TxTarTest) Run(t *testing.T, f func(tc *Test)) {
 					if bytes.Equal(gold.Data, result) {
 						continue
 					}
+				} else if i, ok := index[sub.fallback]; ok {
+					gold.Data = a.Files[i].Data
+
+					// Use the golden file of the fallback set if it matches.
+					if bytes.Equal(gold.Data, result) {
+						gold.Name = sub.fallback
+						delete(index, sub.fallback)
+						continue
+					}
 				}
 
 				if cuetest.UpdateGoldenFiles {