internal/cueimports: export API and fix for CUE syntax

The original code still needed some adapting from the original
Go version. Specifically:
- it looked for semicolons not commas
- it recognized the Go backquoted string syntax
- it recognized the Go /*...*/ comment syntax

Also, make it a little less general, as currently we don't
seem to need the import-saving functionality (we can use
`parser.ParseFile` with `CommentsOnly` which will keep us
honest, even at a little bit of runtime cost), and because
of that, we don't need to try to detect syntax errors.
We can optimize at a later date if need be.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: I276087961c73d04222a115b462e01e130b4cd4e3
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1172690
TryBot-Result: CUEcueckoo <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>

Author: rogpeppe

internal/cueimports/read.go

@@ -1,6 +1,4 @@
-//go:build ignore
-
-// Copyright 2018 The CUE Authors
+// Copyright 2023 The CUE Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package load
+// Package cueimports provides support for reading the import
+// section of a CUE file without needing to read the rest of it.
+package cueimports
 
 import (
 	"bufio"
@@ -76,7 +76,7 @@ func (r *importReader) readByte() byte {
 func (r *importReader) peekByte(skipSpace bool) byte {
 	if r.err != nil {
 		if r.nerr++; r.nerr > 10000 {
-			panic("go/build: import reader looping")
+			panic("import reader looping")
 		}
 		return 0
 	}
@@ -90,10 +90,10 @@ func (r *importReader) peekByte(skipSpace bool) byte {
 	}
 	for r.err == nil && !r.eof {
 		if skipSpace {
-			// For the purposes of this reader, semicolons are never necessary to
+			// For the purposes of this reader, commas are never necessary to
 			// understand the input and are treated as spaces.
 			switch c {
-			case ' ', '\f', '\t', '\r', '\n', ';':
+			case ' ', '\f', '\t', '\r', '\n', ',':
 				c = r.readByte()
 				continue
 
@@ -103,14 +103,6 @@ func (r *importReader) peekByte(skipSpace bool) byte {
 					for c != '\n' && r.err == nil && !r.eof {
 						c = r.readByte()
 					}
-				} else if c == '*' {
-					var c1 byte
-					for (c != '*' || c1 != '/') && r.err == nil {
-						if r.eof {
-							r.syntaxError()
-						}
-						c, c1 = c1, r.readByte()
-					}
 				} else {
 					r.syntaxError()
 				}
@@ -161,29 +153,15 @@ func (r *importReader) readIdent() {
 
 // readString reads a quoted string literal from the input.
 // If an identifier is not present, readString records a syntax error.
-func (r *importReader) readString(save *[]string) {
+func (r *importReader) readString() {
 	switch r.nextByte(true) {
-	case '`':
-		start := len(r.buf) - 1
-		for r.err == nil {
-			if r.nextByte(false) == '`' {
-				if save != nil {
-					*save = append(*save, string(r.buf[start:]))
-				}
-				break
-			}
-			if r.eof {
-				r.syntaxError()
-			}
-		}
 	case '"':
-		start := len(r.buf) - 1
+		// Note: although the syntax in the specification only allows
+		// a simple string literal here, the cue/parser package also
+		// allows #"..."# and """ literals, so there's some impedance-mismatch here.
 		for r.err == nil {
 			c := r.nextByte(false)
 			if c == '"' {
-				if save != nil {
-					*save = append(*save, string(r.buf[start:]))
-				}
 				break
 			}
 			if r.eof || c == '\n' {
@@ -200,19 +178,17 @@ func (r *importReader) readString(save *[]string) {
 
 // readImport reads an import clause - optional identifier followed by quoted string -
 // from the input.
-func (r *importReader) readImport(imports *[]string) {
+func (r *importReader) readImport() {
 	c := r.peekByte(true)
-	if c == '.' {
-		r.peek = 0
-	} else if isIdent(c) {
+	if isIdent(c) {
 		r.readIdent()
 	}
-	r.readString(imports)
+	r.readString()
 }
 
-// readImports is like io.ReadAll, except that it expects a CUE file as
+// Read is like io.ReadAll, except that it expects a CUE file as
 // input and stops reading the input once the imports have completed.
-func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte, errors.Error) {
+func Read(f io.Reader) ([]byte, errors.Error) {
 	r := &importReader{b: bufio.NewReader(f)}
 
 	r.readKeyword("package")
@@ -222,11 +198,11 @@ func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte
 		if r.peekByte(true) == '(' {
 			r.nextByte(false)
 			for r.peekByte(true) != ')' && r.err == nil {
-				r.readImport(imports)
+				r.readImport()
 			}
 			r.nextByte(false)
 		} else {
-			r.readImport(imports)
+			r.readImport()
 		}
 	}
 
@@ -237,8 +213,8 @@ func readImports(f io.Reader, reportSyntaxError bool, imports *[]string) ([]byte
 	}
 
 	// If we stopped for a syntax error, consume the whole file so that
-	// we are sure we don't change the errors that go/parser returns.
-	if r.err == errSyntax && !reportSyntaxError {
+	// we are sure we don't change the errors that the cue/parser returns.
+	if r.err == errSyntax {
 		r.err = nil
 		for r.err == nil && !r.eof {
 			r.readByte()

internal/cueimports/read_test.go

@@ -1,5 +1,3 @@
-//go:build ignore
-
 // Copyright 2018 The CUE Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package load
+package cueimports
 
 import (
 	"io"
@@ -24,8 +22,6 @@ import (
 	"cuelang.org/go/cue/errors"
 )
 
-const quote = "`"
-
 type readTest struct {
 	// Test input contains ℙ where readImports should stop.
 	in  string
@@ -38,15 +34,15 @@ var readImportsTests = []readTest{
 		"",
 	},
 	{
-		`package p; import "x"`,
+		`package p, import "x"`,
 		"",
 	},
 	{
-		`package p; import . "x"`,
+		`package p, import . "x"`,
 		"",
 	},
 	{
-		`package p; import "x";ℙvar x = 1`,
+		`package p, import "x",ℙvar x = 1`,
 		"",
 	},
 	{
@@ -62,15 +58,12 @@ var readImportsTests = []readTest{
 			"x"
 			_ "x"
 			a "x" // comment
-			` + quote + `x` + quote + `
-			_ ` + quote + `x` + quote + `
-			a ` + quote + `x` + quote + `
 		)
 		import (
 		)
 		import ()
 		import()import()import()
-		import();import();import()
+		import(),import(),import()
 
 		ℙvar x = 1
 		`,
@@ -116,97 +109,80 @@ func testRead(t *testing.T, tests []readTest, read func(io.Reader) ([]byte, erro
 
 func TestReadImports(t *testing.T) {
 	testRead(t, readImportsTests, func(r io.Reader) ([]byte, errors.Error) {
-		return readImports(r, true, nil)
+		return Read(r)
 	})
 }
 
 var readFailuresTests = []readTest{
 	{
 		`package`,
-		"syntax error",
+		"",
 	},
 	{
 		"package p\n\x00\nimport `math`\n",
 		"unexpected NUL in input",
 	},
 	{
-		`package p; import`,
-		"syntax error",
+		`package p, import`,
+		"",
 	},
 	{
-		`package p; import "`,
-		"syntax error",
+		`package p, import "`,
+		"",
 	},
 	{
-		"package p; import ` \n\n",
-		"syntax error",
+		"package p, import ` \n\n",
+		"",
 	},
 	{
-		`package p; import "x`,
-		"syntax error",
+		`package p, import "x`,
+		"",
 	},
 	{
-		`package p; import _`,
-		"syntax error",
+		`package p, import _`,
+		"",
 	},
 	{
-		`package p; import _ "`,
-		"syntax error",
+		`package p, import _ "`,
+		"",
 	},
 	{
-		`package p; import _ "x`,
-		"syntax error",
+		`package p, import _ "x`,
+		"",
 	},
 	{
-		`package p; import .`,
-		"syntax error",
+		`package p, import .`,
+		"",
 	},
 	{
-		`package p; import . "`,
-		"syntax error",
+		`package p, import . "`,
+		"",
 	},
 	{
-		`package p; import . "x`,
-		"syntax error",
+		`package p, import . "x`,
+		"",
 	},
 	{
-		`package p; import (`,
-		"syntax error",
+		`package p, import (`,
+		"",
 	},
 	{
-		`package p; import ("`,
-		"syntax error",
+		`package p, import ("`,
+		"",
 	},
 	{
-		`package p; import ("x`,
-		"syntax error",
+		`package p, import ("x`,
+		"",
 	},
 	{
-		`package p; import ("x"`,
-		"syntax error",
+		`package p, import ("x"`,
+		"",
 	},
 }
 
 func TestReadFailures(t *testing.T) {
 	// Errors should be reported (true arg to readImports).
 	testRead(t, readFailuresTests, func(r io.Reader) ([]byte, errors.Error) {
-		return readImports(r, true, nil)
-	})
-}
-
-func TestReadFailuresIgnored(t *testing.T) {
-	// Syntax errors should not be reported (false arg to readImports).
-	// Instead, entire file should be the output and no error.
-	// Convert tests not to return syntax errors.
-	tests := make([]readTest, len(readFailuresTests))
-	copy(tests, readFailuresTests)
-	for i := range tests {
-		tt := &tests[i]
-		if !strings.Contains(tt.err, "NUL") {
-			tt.err = ""
-		}
-	}
-	testRead(t, tests, func(r io.Reader) ([]byte, errors.Error) {
-		return readImports(r, false, nil)
+		return Read(r)
 	})
 }