OCI build from Dockerfile

CHANGELOG.md

@@ -20,6 +20,13 @@
   commands, allows individual layers to be preserved when an OCI-SIF image is
   created from an OCI source. Multi layer OCI-SIF images can be run with
   SingularityCE 4.1 and later.
+- Singularity will now build OCI-SIF images from Dockerfiles, if the `--oci`
+  flag is used with the `build` command. Provide a Dockerfile as the final
+  argument to `build`, instead of a Singularity definition (.def) file. Supports
+  `--build-arg` / `--build-arg-file` options, `--arch` for cross-architecture
+  builds, `--authfile` and other authentication options, and more. See the [user
+  guide](https://docs.sylabs.io/guides/latest/user-guide/build_a_container.html#dockerfile)
+  for more information.
 
 ### Bug Fixes
 

cmd/internal/cli/build.go

@@ -373,7 +373,11 @@ var buildCmd = &cobra.Command{
 
 func preRun(cmd *cobra.Command, _ []string) {
 	if isOCI {
-		sylog.Fatalf("Builds are not yet supported in OCI-mode. Omit --oci, or use --no-oci, to build a non-OCI Singularity container.")
+		if buildArgs.remote {
+			sylog.Fatalf("Remote OCI builds from Dockerfiles are not supported.")
+		}
+
+		return
 	}
 
 	if buildArgs.noSetgroups && !buildArgs.fakeroot {

cmd/internal/cli/build_linux.go

@@ -18,10 +18,12 @@ import (
 	"strings"
 	"syscall"
 
+	ocitypes "github.com/containers/image/v5/types"
 	"github.com/spf13/cobra"
 	keyclient "github.com/sylabs/scs-key-client/client"
 	"github.com/sylabs/singularity/v4/internal/pkg/build"
 	"github.com/sylabs/singularity/v4/internal/pkg/build/args"
+	bkclient "github.com/sylabs/singularity/v4/internal/pkg/build/buildkit/client"
 	"github.com/sylabs/singularity/v4/internal/pkg/build/remotebuilder"
 	"github.com/sylabs/singularity/v4/internal/pkg/buildcfg"
 	"github.com/sylabs/singularity/v4/internal/pkg/cache"
@@ -111,30 +113,45 @@ func runBuild(cmd *cobra.Command, args []string) {
 		if buildArgs.remote {
 			sylog.Fatalf("--nv option is not supported for remote build")
 		}
+		if isOCI {
+			sylog.Fatalf("--nv option is not supported for OCI builds from Dockerfiles")
+		}
 		os.Setenv("SINGULARITY_NV", "1")
 	}
 	if buildArgs.nvccli {
 		if buildArgs.remote {
 			sylog.Fatalf("--nvccli option is not supported for remote build")
 		}
+		if isOCI {
+			sylog.Fatalf("--nvccli option is not supported for OCI builds from Dockerfiles")
+		}
 		os.Setenv("SINGULARITY_NVCCLI", "1")
 	}
 	if buildArgs.rocm {
 		if buildArgs.remote {
 			sylog.Fatalf("--rocm option is not supported for remote build")
 		}
+		if isOCI {
+			sylog.Fatalf("--rocm option is not supported for OCI builds from Dockerfiles")
+		}
 		os.Setenv("SINGULARITY_ROCM", "1")
 	}
 	if len(buildArgs.bindPaths) > 0 {
 		if buildArgs.remote {
 			sylog.Fatalf("-B/--bind option is not supported for remote build")
 		}
+		if isOCI {
+			sylog.Fatalf("-B/--bind option is not supported for OCI builds from Dockerfiles")
+		}
 		os.Setenv("SINGULARITY_BINDPATH", strings.Join(buildArgs.bindPaths, ","))
 	}
 	if len(buildArgs.mounts) > 0 {
 		if buildArgs.remote {
 			sylog.Fatalf("--mount option is not supported for remote build")
 		}
+		if isOCI {
+			sylog.Fatalf("--mount option is not supported for OCI builds from Dockerfiles")
+		}
 		os.Setenv("SINGULARITY_MOUNT", strings.Join(buildArgs.mounts, "\n"))
 	}
 	if buildArgs.writableTmpfs {
@@ -144,22 +161,25 @@ func runBuild(cmd *cobra.Command, args []string) {
 		if buildArgs.fakeroot {
 			sylog.Fatalf("--writable-tmpfs option is not supported for fakeroot build")
 		}
+		if isOCI {
+			sylog.Fatalf("--writable-tmpfs option is not supported for OCI builds from Dockerfiles")
+		}
 		os.Setenv("SINGULARITY_WRITABLE_TMPFS", "1")
 	}
 
 	if cmd.Flags().Lookup("authfile").Changed && buildArgs.remote {
 		sylog.Fatalf("Custom authfile is not supported for remote build")
 	}
 
-	if buildArgs.arch != runtime.GOARCH && !buildArgs.remote {
+	if buildArgs.arch != runtime.GOARCH && !(buildArgs.remote || isOCI) {
 		sylog.Fatalf("Requested architecture (%s) does not match host (%s). Cannot build locally.", buildArgs.arch, runtime.GOARCH)
 	}
 
 	dest := args[0]
 	spec := args[1]
 
 	// Non-remote build with def file as source
-	rootNeeded := !buildArgs.remote && fs.IsFile(spec) && !isImage(spec)
+	rootNeeded := !buildArgs.remote && fs.IsFile(spec) && !isImage(spec) && !isOCI
 
 	if rootNeeded && syscall.Getuid() != 0 && !buildArgs.fakeroot {
 		prootPath, err := bin.FindBin("proot")
@@ -177,9 +197,31 @@ func runBuild(cmd *cobra.Command, args []string) {
 
 	if buildArgs.remote {
 		runBuildRemote(cmd.Context(), cmd, dest, spec)
+		return
+	}
+
+	authConf, err := makeDockerCredentials(cmd)
+	if err != nil {
+		sylog.Fatalf("While creating Docker credentials: %v", err)
+	}
+
+	if isOCI {
+		reqArch := ""
+		if cmd.Flags().Lookup("arch").Changed {
+			reqArch = buildArgs.arch
+		}
+		bkOpts := &bkclient.Opts{
+			AuthConf:        authConf,
+			ReqAuthFile:     reqAuthFile,
+			BuildVarArgs:    buildArgs.buildVarArgs,
+			BuildVarArgFile: buildArgs.buildVarArgFile,
+			ReqArch:         reqArch,
+		}
+		bkclient.Run(cmd.Context(), bkOpts, dest, spec)
 	} else {
-		runBuildLocal(cmd.Context(), cmd, dest, spec)
+		runBuildLocal(cmd.Context(), authConf, cmd, dest, spec)
 	}
+
 	sylog.Infof("Build complete: %s", dest)
 }
 
@@ -307,7 +349,7 @@ func runBuildRemote(ctx context.Context, cmd *cobra.Command, dst, spec string) {
 	}
 }
 
-func runBuildLocal(ctx context.Context, cmd *cobra.Command, dst, spec string) {
+func runBuildLocal(ctx context.Context, authConf *ocitypes.DockerAuthConfig, cmd *cobra.Command, dst, spec string) {
 	var keyInfo *cryptkey.KeyInfo
 	if buildArgs.encrypt || promptForPassphrase || cmd.Flags().Lookup("pem-path").Changed {
 		if os.Getuid() != 0 {
@@ -337,11 +379,6 @@ func runBuildLocal(ctx context.Context, cmd *cobra.Command, dst, spec string) {
 		sylog.Fatalf("Could not check build sections: %v", err)
 	}
 
-	authConf, err := makeDockerCredentials(cmd)
-	if err != nil {
-		sylog.Fatalf("While creating Docker credentials: %v", err)
-	}
-
 	// parse definition to determine build source
 	buildArgsMap, err := args.ReadBuildArgs(buildArgs.buildVarArgs, buildArgs.buildVarArgFile)
 	if err != nil {

e2e/imgbuild/imgbuild.go

@@ -1742,7 +1742,7 @@ cat /proc/$$/cmdline`
 	)
 }
 
-func (c imgBuildTests) buildDefinitionWithBuildArgs(t *testing.T) {
+func (c imgBuildTests) buildWithBuildArgs(t *testing.T) {
 	busyboxSIF := e2e.BusyboxSIF(t)
 	fileContent := `HOME=/root
 DEMO=demo=with===equals==signs
@@ -1941,6 +1941,74 @@ DEMO=demo=with===equals==signs
 	})
 }
 
+func (c imgBuildTests) buildDockerfile(t *testing.T) {
+	profiles := []e2e.Profile{e2e.UserProfile, e2e.RootProfile}
+	for _, profile := range profiles {
+		t.Run(profile.String(), func(t *testing.T) {
+			tmpdir, tmpdirCleanup := e2e.MakeTempDir(t, "", "build_dockerfile_e2e_", "dir")
+			t.Cleanup(func() {
+				if !t.Failed() {
+					tmpdirCleanup(t)
+				}
+			})
+
+			outputImgPath := filepath.Join(tmpdir, "image.oci.sif")
+
+			tests := []struct {
+				name            string
+				buildArgs       []string
+				actCmd          string
+				actArgs         []string
+				buildExpectExit int
+				buildExpects    []e2e.SingularityCmdResultOp
+				actExpectExit   int
+				actExpects      []e2e.SingularityCmdResultOp
+			}{
+				{
+					name:            "simple",
+					buildArgs:       []string{outputImgPath, filepath.Join("..", "test", "defs", "Dockerfile.simple")},
+					actCmd:          "exec",
+					actArgs:         []string{outputImgPath, "/bin/true"},
+					buildExpectExit: 0,
+					actExpectExit:   0,
+				},
+				{
+					name:            "broken",
+					buildArgs:       []string{outputImgPath, filepath.Join("..", "test", "defs", "Dockerfile.broken")},
+					buildExpectExit: 255,
+				},
+			}
+
+			for _, tt := range tests {
+				t.Run(tt.name, func(t *testing.T) {
+					if len(tt.buildArgs) > 0 {
+						buildArgs := append([]string{"-F", "--oci"}, tt.buildArgs...)
+						c.env.RunSingularity(
+							t,
+							e2e.AsSubtest("build"),
+							e2e.WithProfile(profile),
+							e2e.WithCommand("build"),
+							e2e.WithArgs(buildArgs...),
+							e2e.ExpectExit(tt.buildExpectExit, tt.buildExpects...),
+						)
+					}
+					if len(tt.actArgs) > 0 {
+						actArgs := append([]string{"--oci"}, tt.actArgs...)
+						c.env.RunSingularity(
+							t,
+							e2e.AsSubtest("act"),
+							e2e.WithProfile(profile),
+							e2e.WithCommand(tt.actCmd),
+							e2e.WithArgs(actArgs...),
+							e2e.ExpectExit(tt.actExpectExit, tt.actExpects...),
+						)
+					}
+				})
+			}
+		})
+	}
+}
+
 func (c imgBuildTests) buildWithAuth(t *testing.T) {
 	e2e.EnsureImage(t, c.env)
 
@@ -1964,9 +2032,10 @@ func (c imgBuildTests) buildWithAuth(t *testing.T) {
 func (c imgBuildTests) buildWithAuthTester(t *testing.T, withCustomAuthFile bool, profile e2e.Profile) {
 	e2e.EnsureImage(t, c.env)
 
+	privImgNoPrefix := strings.TrimPrefix(c.env.TestRegistryPrivImage, "docker://")
 	simpleDef := e2e.PrepareDefFile(e2e.DefFileDetails{
 		Bootstrap: "docker",
-		From:      strings.TrimPrefix(c.env.TestRegistryPrivImage, "docker://"),
+		From:      privImgNoPrefix,
 	})
 	t.Cleanup(func() {
 		if !t.Failed() {
@@ -1981,13 +2050,24 @@ func (c imgBuildTests) buildWithAuthTester(t *testing.T, withCustomAuthFile bool
 		}
 	})
 
+	dockerfile, err := e2e.WriteTempFile(tmpdir, "Dockerfile", fmt.Sprintf(
+		`
+FROM %s
+CMD /bin/true
+`,
+		privImgNoPrefix,
+	))
+	if err != nil {
+		t.Fatalf("while trying to generate test dockerfile: %v", err)
+	}
+
 	prevCwd, err := os.Getwd()
 	if err != nil {
-		t.Fatalf("could not get current working directory: %s", err)
+		t.Fatalf("could not get current working directory: %v", err)
 	}
 	defer os.Chdir(prevCwd)
 	if err = os.Chdir(tmpdir); err != nil {
-		t.Fatalf("could not change cwd to %q: %s", tmpdir, err)
+		t.Fatalf("could not change cwd to %q: %v", tmpdir, err)
 	}
 
 	localAuthFileName := ""
@@ -2034,6 +2114,18 @@ func (c imgBuildTests) buildWithAuthTester(t *testing.T, withCustomAuthFile bool
 			whileLoggedIn: true,
 			expectExit:    0,
 		},
+		{
+			name:          "privimg df logged out",
+			args:          []string{"-F", "--oci", "--no-https", "--disable-cache", "./my_image_file.oci.sif", dockerfile},
+			whileLoggedIn: false,
+			expectExit:    255,
+		},
+		{
+			name:          "privimg df logged in",
+			args:          []string{"-F", "--oci", "--no-https", "--disable-cache", "./my_image_file.oci.sif", dockerfile},
+			whileLoggedIn: true,
+			expectExit:    0,
+		},
 	}
 
 	for _, tt := range tests {
@@ -2101,39 +2193,40 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 	np := testhelper.NoParallel
 
 	return testhelper.Tests{
-		"bad path":                        c.badPath,                      // try to build from a non existent path
-		"build encrypt with PEM file":     c.buildEncryptPemFile,          // build encrypted images with certificate
-		"build encrypted with passphrase": c.buildEncryptPassphrase,       // build encrypted images with passphrase
-		"definition":                      c.buildDefinition,              // builds from definition template
-		"from local image":                c.buildLocalImage,              // build and image from an existing image
-		"from":                            c.buildFrom,                    // builds from definition file and URI
-		"multistage":                      c.buildMultiStageDefinition,    // multistage build from definition templates
-		"non-root build":                  c.nonRootBuild,                 // build sifs from non-root
-		"build and update sandbox":        c.buildUpdateSandbox,           // build/update sandbox
-		"fingerprint check":               c.buildWithFingerprint,         // definition file includes fingerprint check
-		"build with bind mount":           c.buildBindMount,               // build image with bind mount
-		"test with writable tmpfs":        c.testWritableTmpfs,            // build image, using writable tmpfs in the test step
-		"library host":                    c.buildLibraryHost,             // build image with hostname in library URI
-		"proot":                           c.buildProot,                   // build image as an unpriv user with proot
-		"customShebang":                   c.buildCustomShebang,           // build image with custom #! in %test and %runscript
-		"no-setgroups":                    c.buildNoSetgroups,             // build with --fakeroot --no-setgroups
-		"buildArgs":                       c.buildDefinitionWithBuildArgs, // builds from definition with build args (build arg file) support
-		"auth":                            np(c.buildWithAuth),            // build with custom auth file
-		"issue 3848":                      c.issue3848,                    // https://github.com/hpcng/singularity/issues/3848
-		"issue 4203":                      c.issue4203,                    // https://github.com/sylabs/singularity/issues/4203
-		"issue 4407":                      c.issue4407,                    // https://github.com/sylabs/singularity/issues/4407
-		"issue 4583":                      c.issue4583,                    // https://github.com/sylabs/singularity/issues/4583
-		"issue 4820":                      c.issue4820,                    // https://github.com/sylabs/singularity/issues/4820
-		"issue 4837":                      c.issue4837,                    // https://github.com/sylabs/singularity/issues/4837
-		"issue 4967":                      c.issue4967,                    // https://github.com/sylabs/singularity/issues/4967
-		"issue 4969":                      c.issue4969,                    // https://github.com/sylabs/singularity/issues/4969
-		"issue 5166":                      c.issue5166,                    // https://github.com/sylabs/singularity/issues/5166
-		"issue 5250":                      c.issue5250,                    // https://github.com/sylabs/singularity/issues/5250
-		"issue 5315":                      c.issue5315,                    // https://github.com/sylabs/singularity/issues/5315
-		"issue 5435":                      c.issue5435,                    // https://github.com/hpcng/singularity/issues/5435
-		"issue 5668":                      c.issue5668,                    // https://github.com/hpcng/singularity/issues/5435
-		"issue 5690":                      c.issue5690,                    // https://github.com/hpcng/singularity/issues/5690
-		"issue 1273":                      c.issue1273,                    // https://github.com/sylabs/singularity/issues/1273
-		"issue 1812":                      c.issue1812,                    // https://github.com/sylabs/singularity/issues/1812
+		"bad path":                        c.badPath,                   // try to build from a non existent path
+		"build encrypt with PEM file":     c.buildEncryptPemFile,       // build encrypted images with certificate
+		"build encrypted with passphrase": c.buildEncryptPassphrase,    // build encrypted images with passphrase
+		"definition":                      c.buildDefinition,           // builds from definition template
+		"from local image":                c.buildLocalImage,           // build and image from an existing image
+		"from":                            c.buildFrom,                 // builds from definition file and URI
+		"multistage":                      c.buildMultiStageDefinition, // multistage build from definition templates
+		"non-root build":                  c.nonRootBuild,              // build sifs from non-root
+		"build and update sandbox":        c.buildUpdateSandbox,        // build/update sandbox
+		"fingerprint check":               c.buildWithFingerprint,      // definition file includes fingerprint check
+		"build with bind mount":           c.buildBindMount,            // build image with bind mount
+		"test with writable tmpfs":        c.testWritableTmpfs,         // build image, using writable tmpfs in the test step
+		"library host":                    c.buildLibraryHost,          // build image with hostname in library URI
+		"proot":                           c.buildProot,                // build image as an unpriv user with proot
+		"customShebang":                   c.buildCustomShebang,        // build image with custom #! in %test and %runscript
+		"no-setgroups":                    c.buildNoSetgroups,          // build with --fakeroot --no-setgroups
+		"buildArgs":                       c.buildWithBuildArgs,        // builds from definition with build args (build arg file) support
+		"dockerfile":                      c.buildDockerfile,           // build OCI-SIF image from Dockerfile
+		"auth":                            np(c.buildWithAuth),         // build with custom auth file
+		"issue 3848":                      c.issue3848,                 // https://github.com/hpcng/singularity/issues/3848
+		"issue 4203":                      c.issue4203,                 // https://github.com/sylabs/singularity/issues/4203
+		"issue 4407":                      c.issue4407,                 // https://github.com/sylabs/singularity/issues/4407
+		"issue 4583":                      c.issue4583,                 // https://github.com/sylabs/singularity/issues/4583
+		"issue 4820":                      c.issue4820,                 // https://github.com/sylabs/singularity/issues/4820
+		"issue 4837":                      c.issue4837,                 // https://github.com/sylabs/singularity/issues/4837
+		"issue 4967":                      c.issue4967,                 // https://github.com/sylabs/singularity/issues/4967
+		"issue 4969":                      c.issue4969,                 // https://github.com/sylabs/singularity/issues/4969
+		"issue 5166":                      c.issue5166,                 // https://github.com/sylabs/singularity/issues/5166
+		"issue 5250":                      c.issue5250,                 // https://github.com/sylabs/singularity/issues/5250
+		"issue 5315":                      c.issue5315,                 // https://github.com/sylabs/singularity/issues/5315
+		"issue 5435":                      c.issue5435,                 // https://github.com/hpcng/singularity/issues/5435
+		"issue 5668":                      c.issue5668,                 // https://github.com/hpcng/singularity/issues/5435
+		"issue 5690":                      c.issue5690,                 // https://github.com/hpcng/singularity/issues/5690
+		"issue 1273":                      c.issue1273,                 // https://github.com/sylabs/singularity/issues/1273
+		"issue 1812":                      c.issue1812,                 // https://github.com/sylabs/singularity/issues/1812
 	}
 }