Add support for reading in device extension files for container creat…

…e hcs document

Signed-off-by: Kathryn Baldauf <[email protected]>

internal/hcs/schema2/container.go

@@ -31,4 +31,6 @@ type Container struct {
 	RegistryChanges *RegistryChanges `json:"RegistryChanges,omitempty"`
 
 	AssignedDevices []Device `json:"AssignedDevices,omitempty"`
+
+	AdditionalDeviceNamespace *ContainerDefinitionDevice `json:"AdditionalDeviceNamespace,omitempty"`
 }

internal/hcs/schema2/device.go

@@ -12,16 +12,16 @@ package hcsschema
 type DeviceType string
 
 const (
-	ClassGUID      DeviceType = "ClassGuid"
-	DeviceInstance DeviceType = "DeviceInstance"
-	GPUMirror      DeviceType = "GpuMirror"
+	ClassGUID        DeviceType = "ClassGuid"
+	DeviceInstanceID DeviceType = "DeviceInstance"
+	GPUMirror        DeviceType = "GpuMirror"
 )
 
 type Device struct {
 	//  The type of device to assign to the container.
 	Type DeviceType `json:"Type,omitempty"`
 	//  The interface class guid of the device interfaces to assign to the  container.  Only used when Type is ClassGuid.
 	InterfaceClassGuid string `json:"InterfaceClassGuid,omitempty"`
-	//  The location path of the device to assign to the container.  Only used when Type is DeviceInstance.
+	//  The location path of the device to assign to the container.  Only used when Type is DeviceInstanceID.
 	LocationPath string `json:"LocationPath,omitempty"`
 }

internal/hcs/schema2/model_container_definition_device.go

@@ -0,0 +1,14 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ContainerDefinitionDevice struct {
+	DeviceExtension []DeviceExtension `json:"device_extension,omitempty"`
+}

internal/hcs/schema2/model_device_category.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type DeviceCategory struct {
+	Name           string           `json:"name,omitempty"`
+	InterfaceClass []InterfaceClass `json:"interface_class,omitempty"`
+}

internal/hcs/schema2/model_device_extension.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type DeviceExtension struct {
+	DeviceCategory *DeviceCategory           `json:"device_category,omitempty"`
+	Namespace      *DeviceExtensionNamespace `json:"namespace,omitempty"`
+}

internal/hcs/schema2/model_device_instance.go

@@ -0,0 +1,17 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type DeviceInstance struct {
+	Id             string           `json:"id,omitempty"`
+	LocationPath   string           `json:"location_path,omitempty"`
+	PortName       string           `json:"port_name,omitempty"`
+	InterfaceClass []InterfaceClass `json:"interface_class,omitempty"`
+}

internal/hcs/schema2/model_device_namespace.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type DeviceNamespace struct {
+	RequiresDriverstore bool             `json:"requires_driverstore,omitempty"`
+	DeviceCategory      []DeviceCategory `json:"device_category,omitempty"`
+	DeviceInstance      []DeviceInstance `json:"device_instance,omitempty"`
+}

internal/hcs/schema2/model_interface_class.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type InterfaceClass struct {
+	Type_      string `json:"type,omitempty"`
+	Identifier string `json:"identifier,omitempty"`
+	Recurse    bool   `json:"recurse,omitempty"`
+}

internal/hcs/schema2/model_namespace.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type DeviceExtensionNamespace struct {
+	Ob     *ObjectNamespace `json:"ob,omitempty"`
+	Device *DeviceNamespace `json:"device,omitempty"`
+}

internal/hcs/schema2/model_object_directory.go

@@ -0,0 +1,18 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ObjectDirectory struct {
+	Name    string            `json:"name,omitempty"`
+	Clonesd string            `json:"clonesd,omitempty"`
+	Shadow  string            `json:"shadow,omitempty"`
+	Symlink []ObjectSymlink   `json:"symlink,omitempty"`
+	Objdir  []ObjectDirectory `json:"objdir,omitempty"`
+}

internal/hcs/schema2/model_object_namespace.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ObjectNamespace struct {
+	Shadow  string            `json:"shadow,omitempty"`
+	Symlink []ObjectSymlink   `json:"symlink,omitempty"`
+	Objdir  []ObjectDirectory `json:"objdir,omitempty"`
+}

internal/hcs/schema2/model_object_symlink.go

@@ -0,0 +1,18 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type ObjectSymlink struct {
+	Name        string `json:"name,omitempty"`
+	Path        string `json:"path,omitempty"`
+	Scope       string `json:"scope,omitempty"`
+	Pathtoclone string `json:"pathtoclone,omitempty"`
+	AccessMask  int32  `json:"access_mask,omitempty"`
+}

internal/hcsoci/devices.go

@@ -2,12 +2,14 @@ package hcsoci
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/Microsoft/hcsshim/internal/devices"
+	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oci"
 	"github.com/Microsoft/hcsshim/internal/resources"
@@ -20,14 +22,8 @@ const deviceUtilExeName = "device-util.exe"
 
 // getAssignedDeviceKernelDrivers gets any device drivers specified on the spec.
 // Drivers are optional, therefore do not return an error if none are on the spec.
-//
-// See comment on oci.AnnotationAssignedDeviceKernelDrivers for expected format.
 func getAssignedDeviceKernelDrivers(annotations map[string]string) ([]string, error) {
-	csDrivers, ok := annotations[oci.AnnotationAssignedDeviceKernelDrivers]
-	if !ok || csDrivers == "" {
-		return nil, nil
-	}
-	drivers := strings.Split(csDrivers, ",")
+	drivers := oci.ParseAnnotationCommaSeparated(oci.AnnotationAssignedDeviceKernelDrivers, annotations)
 	for _, driver := range drivers {
 		if _, err := os.Stat(driver); err != nil {
 			return nil, errors.Wrapf(err, "failed to find path to drivers at %s", driver)
@@ -36,11 +32,47 @@ func getAssignedDeviceKernelDrivers(annotations map[string]string) ([]string, er
 	return drivers, nil
 }
 
+// getDeviceExtensionPaths gets any device extensions paths specified on the spec.
+// device extensions are optional, therefore if none are on the spec, do not return an error.
+func getDeviceExtensionPaths(annotations map[string]string) ([]string, error) {
+	extensions := oci.ParseAnnotationCommaSeparated(oci.AnnotationDeviceExtensions, annotations)
+	for _, ext := range extensions {
+		if _, err := os.Stat(ext); err != nil {
+			return nil, errors.Wrapf(err, "failed to find path to driver extensions at %s", ext)
+		}
+	}
+	return extensions, nil
+}
+
 // getDeviceUtilHostPath is a simple helper function to find the host path of the device-util tool
 func getDeviceUtilHostPath() string {
 	return filepath.Join(filepath.Dir(os.Args[0]), deviceUtilExeName)
 }
 
+// getDeviceExtensions is a helper function to read the files at `extensionPaths` and unmarshal the contents
+// into a `hcsshema.DeviceExtension` to be added to a container's hcs create document.
+func getDeviceExtensions(annotations map[string]string) (*hcsschema.ContainerDefinitionDevice, error) {
+	extensionPaths, err := getDeviceExtensionPaths(annotations)
+	if err != nil {
+		return nil, err
+	}
+	results := &hcsschema.ContainerDefinitionDevice{
+		DeviceExtension: []hcsschema.DeviceExtension{},
+	}
+	for _, extensionPath := range extensionPaths {
+		data, err := ioutil.ReadFile(extensionPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to read extension file at %s", extensionPath)
+		}
+		extension := hcsschema.DeviceExtension{}
+		if err := json.Unmarshal(data, &extension); err != nil {
+			return nil, errors.Wrapf(err, "failed to unmarshal extension file at %s", extensionPath)
+		}
+		results.DeviceExtension = append(results.DeviceExtension, extension)
+	}
+	return results, nil
+}
+
 // handleAssignedDevicesWindows does all of the work to setup the hosting UVM, assign in devices
 // specified on the spec, and install any necessary, specified kernel drivers into the UVM.
 //
@@ -91,18 +123,5 @@ func handleAssignedDevicesWindows(ctx context.Context, vm *uvm.UtilityVM, annota
 		}
 	}
 
-	// get the spec specified kernel drivers and install them on the UVM
-	drivers, err := getAssignedDeviceKernelDrivers(annotations)
-	if err != nil {
-		return nil, closers, err
-	}
-	for _, d := range drivers {
-		driverCloser, err := devices.InstallWindowsDriver(ctx, vm, d)
-		if err != nil {
-			return nil, closers, err
-		}
-		closers = append(closers, driverCloser)
-	}
-
 	return resultDevs, closers, nil
 }

internal/hcsoci/hcsdoc_wcow.go

@@ -367,6 +367,13 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter
 		return nil, nil, err
 	}
 
+	// add any device extensions
+	extensions, err := getDeviceExtensions(coi.Spec.Annotations)
+	if err != nil {
+		return nil, nil, err
+	}
+	v2Container.AdditionalDeviceNamespace = extensions
+
 	return v1, v2Container, nil
 }
 
@@ -383,7 +390,7 @@ func parseAssignedDevices(ctx context.Context, coi *createOptionsInternal, v2 *h
 		switch d.IDType {
 		case uvm.VPCILocationPathIDType:
 			v2Dev.LocationPath = d.ID
-			v2Dev.Type = hcsschema.DeviceInstance
+			v2Dev.Type = hcsschema.DeviceInstanceID
 		case uvm.VPCIClassGUIDTypeLegacy:
 			v2Dev.InterfaceClassGuid = d.ID
 		case uvm.VPCIClassGUIDType:

internal/hcsoci/resources_wcow.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/Microsoft/hcsshim/internal/credentials"
+	"github.com/Microsoft/hcsshim/internal/devices"
 	"github.com/Microsoft/hcsshim/internal/layers"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/resources"
@@ -102,6 +103,21 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r
 		coi.Spec.Windows.Devices = windowsDevices
 	}
 
+	if coi.HostingSystem != nil {
+		// get the spec specified kernel drivers and install them on the UVM
+		drivers, err := getAssignedDeviceKernelDrivers(coi.Spec.Annotations)
+		if err != nil {
+			return err
+		}
+		for _, d := range drivers {
+			driverCloser, err := devices.InstallWindowsDriver(ctx, coi.HostingSystem, d)
+			if err != nil {
+				return err
+			}
+			r.Add(driverCloser)
+		}
+	}
+
 	return nil
 }
 

internal/oci/uvm.go

@@ -87,6 +87,9 @@ const (
 	// files and information needed to install given driver(s). This may include .sys,
 	// .inf, .cer, and/or other files used during standard installation with pnputil.
 	AnnotationAssignedDeviceKernelDrivers = "io.microsoft.assigneddevice.kerneldrivers"
+	// AnnotationDeviceExtensions contains a comma separated list of full paths to device extension files.
+	// The content of these are added to a container's hcs create document.
+	AnnotationDeviceExtensions = "io.microsoft.container.wcow.deviceextensions"
 	// AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process
 	// container as and instead inherit the user token from the executable that is launching the container process.
 	AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user"
@@ -182,6 +185,17 @@ func parseAnnotationsBool(ctx context.Context, a map[string]string, key string,
 	return def
 }
 
+// ParseAnnotationCommaSeparated searches `annotations` for `annotation` corresponding to a
+// list of comma separated strings
+func ParseAnnotationCommaSeparated(annotation string, annotations map[string]string) []string {
+	cs, ok := annotations[annotation]
+	if !ok || cs == "" {
+		return nil
+	}
+	results := strings.Split(cs, ",")
+	return results
+}
+
 // ParseAnnotationsCPUCount searches `s.Annotations` for the CPU annotation. If
 // not found searches `s` for the Windows CPU section. If neither are found
 // returns `def`.