all: Windows fixes (don't listen on file descriptors in test.World, etc)

test/integration: don't listen on file descriptors.
make.go: unrelated, but options to make it much faster.
internal/images: t.Skip on HEIC dependency failures

Fixes #1140
Updates golang/go#25210

Change-Id: I8092155411826d6ed1f8d85230b753d1369044af

internal/images/images.go

@@ -656,6 +656,10 @@ func localImageMagick() string {
 	return ""
 }
 
+type NoHEICTOJPEGError struct {
+	error
+}
+
 // HEIFToJPEG converts the HEIF file in fr to JPEG. It optionally resizes it
 // to the given maxSize argument, if any. It returns the contents of the JPEG file.
 func HEIFToJPEG(fr io.Reader, maxSize *Dimensions) ([]byte, error) {
@@ -665,7 +669,7 @@ func HEIFToJPEG(fr io.Reader, maxSize *Dimensions) ([]byte, error) {
 	bin := localImageMagick()
 	if bin == "" {
 		if err := setUpThumbnailContainer(); err != nil {
-			return nil, fmt.Errorf("recent ImageMagick magick binary not found in PATH, and could not fallback on docker image because %v. Install a modern ImageMagick or install docker.", err)
+			return nil, NoHEICTOJPEGError{fmt.Errorf("recent ImageMagick magick binary not found in PATH, and could not fallback on docker image because %v. Install a modern ImageMagick or install docker.", err)}
 		}
 		bin = "docker"
 		useDocker = true

internal/images/images_test.go

@@ -391,6 +391,9 @@ func TestHEIFToJPEG(t *testing.T) {
 	max := 1008
 	data, err := HEIFToJPEG(f, &Dimensions{MaxWidth: max, MaxHeight: max})
 	if err != nil {
+		if _, ok := err.(NoHEICTOJPEGError); ok {
+			t.Skipf("skipping test; missing program: %v", err)
+		}
 		t.Fatal(err)
 	}
 	conf, tp, err := image.DecodeConfig(bytes.NewReader(data))

internal/osutil/paths.go

@@ -337,6 +337,9 @@ func LookPathGopath(binName string) (string, error) {
 		return "", fmt.Errorf("command %q not found in $PATH, and could not look in $GOPATH/bin because %v", binName, err)
 	}
 	binPath = filepath.Join(binDir, binName)
+	if runtime.GOOS == "windows" {
+		binPath += ".exe"
+	}
 	if _, err := os.Stat(binPath); err != nil {
 		return "", err
 	}

make.go

@@ -52,7 +52,6 @@ var haveSQLite = checkHaveSQLite()
 var (
 	embedResources = flag.Bool("embed_static", true, "Whether to embed resources needed by the UI such as images, css, and javascript.")
 	sqlFlag        = flag.String("sqlite", "false", "Whether you want SQLite in your build: true, false, or auto.")
-	all            = flag.Bool("all", false, "Force rebuild of everything (go install -a)")
 	race           = flag.Bool("race", false, "Build race-detector version of binaries (they will run slowly)")
 	verbose        = flag.Bool("v", strings.Contains(os.Getenv("CAMLI_DEBUG_X"), "makego"), "Verbose mode")
 	targets        = flag.String("targets", "", "Optional comma-separated list of targets (i.e go packages) to build and install. '*' builds everything.  Empty builds defaults for this platform. Example: perkeep.org/server/perkeepd,perkeep.org/cmd/pk-put")
@@ -64,6 +63,7 @@ var (
 	website        = flag.Bool("website", false, "Just build the website.")
 	camnetdns      = flag.Bool("camnetdns", false, "Just build perkeep.org/server/camnetdns.")
 	static         = flag.Bool("static", false, "Build a static binary, so it can run in an empty container.")
+	skipGopherJS   = flag.Bool("skip_gopherjs", false, "skip building/running GopherJS, even if building perkeepd/etc")
 )
 
 var (
@@ -146,25 +146,25 @@ func main() {
 		}
 	}
 
-	withCamlistored := stringListContains(targs, "perkeep.org/server/perkeepd")
+	withPerkeepd := stringListContains(targs, "perkeep.org/server/perkeepd")
 	withPublisher := stringListContains(targs, "perkeep.org/app/publisher")
-	if withCamlistored || withPublisher {
+	if (withPerkeepd || withPublisher) && !*skipGopherJS {
 		if err := buildReactGen(); err != nil {
 			log.Fatal(err)
 		}
-		if withCamlistored {
+		if withPerkeepd {
 			if err := genWebUIReact(); err != nil {
 				log.Fatal(err)
 			}
 		}
 		// gopherjs has to run before doEmbed since we need all the javascript
 		// to be generated before embedding happens.
-		if err := makeJS(withCamlistored, withPublisher); err != nil {
+		if err := makeJS(withPerkeepd, withPublisher); err != nil {
 			log.Fatal(err)
 		}
 	}
 
-	if *embedResources && withCamlistored {
+	if *embedResources && withPerkeepd {
 		doEmbed()
 	}
 
@@ -180,9 +180,6 @@ func main() {
 		tags = append(tags, "with_sqlite")
 	}
 	baseArgs := []string{"install", "-v"}
-	if *all {
-		baseArgs = append(baseArgs, "-a")
-	}
 	if *race {
 		baseArgs = append(baseArgs, "-race")
 	}
@@ -199,7 +196,10 @@ func main() {
 		}
 		ldFlags += "-X \"perkeep.org/pkg/buildinfo.GitInfo=" + version + "\""
 	}
-	baseArgs = append(baseArgs, "--ldflags="+ldFlags, "--tags="+strings.Join(tags, " "))
+	if ldFlags != "" {
+		baseArgs = append(baseArgs, "--ldflags="+ldFlags)
+	}
+	baseArgs = append(baseArgs, "--tags="+strings.Join(tags, " "))
 
 	// First install command: build just the final binaries, installed to a GOBIN
 	// under <perkeep_root>/bin:
@@ -588,9 +588,6 @@ func genEmbeds() error {
 	} {
 		embeds := fullSrcPath(embeds)
 		var args []string
-		if *all {
-			args = append(args, "-all")
-		}
 		args = append(args, embeds)
 		cmd := exec.Command(cmdName, args...)
 		cmd.Stdout = os.Stdout
@@ -627,9 +624,6 @@ func buildBin(pkg string) error {
 	pkgBase := pathpkg.Base(pkg)
 
 	args := []string{"install", "-v"}
-	if *all {
-		args = append(args, "-a")
-	}
 	args = append(args,
 		filepath.FromSlash(pkg),
 	)

pkg/blobserver/diskpacked/testdata/.gitattributes

@@ -0,0 +1 @@
+*.blobs binary

pkg/client/upload.go

@@ -29,6 +29,7 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	"perkeep.org/internal/hashutil"
@@ -41,10 +42,6 @@ import (
 	"perkeep.org/pkg/schema"
 )
 
-// multipartOverhead is how many extra bytes mime/multipart's
-// Writer adds around content
-var multipartOverhead = calculateMultipartOverhead()
-
 // UploadHandle contains the parameters is a request to upload a blob.
 type UploadHandle struct {
 	// BlobRef is the required blobref of the blob to upload.
@@ -91,16 +88,26 @@ type statResponse struct {
 
 type ResponseFormatError error
 
-func calculateMultipartOverhead() int64 {
-	var b bytes.Buffer
-	w := multipart.NewWriter(&b)
-	part, _ := w.CreateFormFile("0", "0")
+var (
+	multipartOnce     sync.Once
+	multipartOverhead int64
+)
 
-	dummyContents := []byte("0")
-	part.Write(dummyContents)
+// multipartOverhead is how many extra bytes mime/multipart's
+// Writer adds around content
+func getMultipartOverhead() int64 {
+	multipartOnce.Do(func() {
+		var b bytes.Buffer
+		w := multipart.NewWriter(&b)
+		part, _ := w.CreateFormFile("0", "0")
 
-	w.Close()
-	return int64(b.Len()) - 3 // remove what was added
+		dummyContents := []byte("0")
+		part.Write(dummyContents)
+
+		w.Close()
+		multipartOverhead = int64(b.Len()) - 3 // remove what was added
+	})
+	return multipartOverhead
 }
 
 func parseStatResponse(res *http.Response) (*statResponse, error) {
@@ -365,7 +372,7 @@ func (c *Client) Upload(ctx context.Context, h *UploadHandle) (*PutResult, error
 		req.Header.Add("X-Camlistore-Vivify", "1")
 	}
 	req.Body = ioutil.NopCloser(pipeReader)
-	req.ContentLength = multipartOverhead + bodySize + int64(len(blobrefStr))*2
+	req.ContentLength = getMultipartOverhead() + bodySize + int64(len(blobrefStr))*2
 	resp, err := c.doReqGated(req)
 	if err != nil {
 		return errorf("upload http error: %v", err)

pkg/test/integration/camlistore_test.go

@@ -190,10 +190,10 @@ func TestAndroidCamputFile(t *testing.T) {
 	//   CAMPUT_ANDROID_OUTPUT=1
 	cacheDir, clean := mustTempDir(t)
 	defer clean()
-	env := []string{
+	env := append(os.Environ(),
 		"CAMPUT_ANDROID_OUTPUT=1",
-		"CAMLI_CACHE_DIR=" + cacheDir,
-	}
+		"CAMLI_CACHE_DIR="+cacheDir,
+	)
 	cmd := w.CmdWithEnv("pk-put",
 		env,
 		"--server="+w.ServerBaseURL(),

pkg/test/testdata/.gitattributes

@@ -0,0 +1,2 @@
+# Uploaded by tests and expected to be byte identical, so:
+*.json binary