Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

V2 API support for win-overlay CNI #725

Merged
merged 1 commit into from
Apr 14, 2022
Merged

V2 API support for win-overlay CNI #725

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions pkg/hns/endpoint_windows.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type EndpointInfo struct {
NetworkId string
Gateway net.IP
IpAddress net.IP
MacAddress string
}

// GetSandboxContainerID returns the sandbox ID of this pod.
Expand Down Expand Up @@ -248,6 +249,7 @@ func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndp
Minor: 0,
},
Name: epInfo.EndpointName,
MacAddress: epInfo.MacAddress,
HostComputeNetwork: epInfo.NetworkId,
Dns: hcn.Dns{
Domain: epInfo.DNS.Domain,
Expand Down Expand Up @@ -280,6 +282,16 @@ func RemoveHcnEndpoint(epName string) error {
}
return errors.Annotatef(err, "failed to find HostComputeEndpoint %s", epName)
}
epNamespace, err := hcn.GetNamespaceByID(hcnEndpoint.HostComputeNamespace)
if err != nil && !hcn.IsNotFoundError(err) {
return errors.Annotatef(err, "failed to get HostComputeNamespace %s", epName)
}
if epNamespace != nil {
err = hcn.RemoveNamespaceEndpoint(hcnEndpoint.HostComputeNamespace, hcnEndpoint.Id)
if err != nil && !hcn.IsNotFoundError(err) {
return errors.Annotatef(err,"error removing endpoint: %s from namespace", epName)
}
}

err = hcnEndpoint.Delete()
if err != nil {
Expand Down
46 changes: 46 additions & 0 deletions plugins/main/windows/win-overlay/sample-v2.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{
"cniVersion": "0.2.0",
"name": "OVNKubernetesHybridOverlayNetwork",
"type": "win-overlay",
"ipam": {
"type": "host-local",
"subnet": "10.132.0.0/24"
},
"apiVersion": 2,
"capabilities": {
"portMappings": true,
"dns": true
},
"policies": [
{
"name": "EndpointPolicy",
"value": {
"Type": "OutBoundNAT",
"Settings": {
"Exceptions": [
"172.30.0.0/16"
]
}
}
},
{
"name": "EndpointPolicy",
"value": {
"Type": "SDNRoute",
"Settings": {
"DestinationPrefix": "172.30.0.0/16",
"NeedEncap": true
}
}
},
{
"name": "EndpointPolicy",
"value": {
"Type": "ProviderAddress",
"Settings": {
"ProviderAddress": "10.0.133.170"
}
}
}
]
}
141 changes: 128 additions & 13 deletions plugins/main/windows/win-overlay/win-overlay_windows.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@ package main
import (
"encoding/json"
"fmt"
"net"
"runtime"
"strings"

"github.com/Microsoft/hcsshim"

"github.com/Microsoft/hcsshim/hcn"
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
current "github.com/containernetworking/cni/pkg/types/100"
Expand Down Expand Up @@ -55,16 +57,106 @@ func loadNetConf(bytes []byte) (*NetConf, string, error) {
return n, n.CNIVersion, nil
}

func cmdAdd(args *skel.CmdArgs) error {
success := false
n, cniVersion, err := loadNetConf(args.StdinData)
func processEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, error) {
epInfo := new(hns.EndpointInfo)
epInfo.NetworkName = n.Name
epInfo.EndpointName = hns.ConstructEndpointName(args.ContainerID, args.Netns, epInfo.NetworkName)

if n.IPAM.Type != "" {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tip: FYI If desirable, we could expand this to fallback to HNS to provision an IP address and MAC address for you, instead of requiring on an IPAM plugin. This is just an FYI, it is also a perfectly valid decision to require IPAM.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To be clear, this is not blocking.

r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return nil, errors.Annotatef(err, "error while executing IPAM addition")
}

// convert whatever the IPAM result was into the current result
result, err := current.NewResultFromResult(r)
if err != nil {
return nil, errors.Annotatef(err, "error while converting the result from IPAM addition")
}
if len(result.IPs) == 0 {
return nil, fmt.Errorf("IPAM plugin return is missing IP config")
}
epInfo.IpAddress = result.IPs[0].Address.IP.To4()
if epInfo.IpAddress == nil {
return nil, fmt.Errorf("IPAM plugin return is missing valid IP Address")
}
epInfo.MacAddress = fmt.Sprintf("%v-%02x-%02x-%02x-%02x", n.EndpointMacPrefix, epInfo.IpAddress[0], epInfo.IpAddress[1], epInfo.IpAddress[2], epInfo.IpAddress[3])

}
epInfo.DNS = n.GetDNS()
if n.LoopbackDSR {
n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress)
}
return epInfo, nil
}

func cmdHcnAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) {
if len(n.EndpointMacPrefix) != 0 {
if len(n.EndpointMacPrefix) != 5 || n.EndpointMacPrefix[2] != '-' {
return nil, fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", n.EndpointMacPrefix)
}
} else {
n.EndpointMacPrefix = "0E-2A"
}

networkName := n.Name
hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit Feels like we should be able to avoid making a query to get network representation through HNS APIs since the result from HCN APIs should be sufficient... Or is there a reason we are getting the same network using both HCN and HNS?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To be clear, this is a nit-pick and not blocking.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@daschott I was running into an issue where I was not able to get the exact gateway IP from the HCN network object. The way win-bridge CNI gets the gateway IP address is returning me a "different IP" when I tried to use it in win-overlay. That is the only reason I am still using the HNS network object. I already discussed it with @mansikulkarni96 and will take it as a separate improvement PR. I need to dig and find the right way to get the default IP.
If you are aware of getting the correct gateway from the HCN network object, pls do let me know.

hcnNetwork, err := hcn.GetNetworkByName(networkName)
if err != nil {
return nil, errors.Annotatef(err, "error while hcn.GetNetworkByName(%s)", networkName)
}
if hcnNetwork == nil {
return nil, fmt.Errorf("network %v is not found", networkName)
}
if hnsNetwork == nil {
return nil, fmt.Errorf("network %v not found", networkName)
}

if !strings.EqualFold(string (hcnNetwork.Type), "Overlay") {
return nil, fmt.Errorf("network %v is of an unexpected type: %v", networkName, hcnNetwork.Type)
}

epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)

hcnEndpoint, err := hns.AddHcnEndpoint(epName, hcnNetwork.Id, args.Netns, func() (*hcn.HostComputeEndpoint, error) {
epInfo, err := processEndpointArgs(args, n)
if err != nil {
return nil, errors.Annotate(err, "error while processing endpoint args")
}
epInfo.NetworkId = hcnNetwork.Id
gatewayAddr := net.ParseIP(hnsNetwork.Subnets[0].GatewayAddress)
epInfo.Gateway = gatewayAddr.To4()
n.ApplyDefaultPAPolicy(hnsNetwork.ManagementIP)
if n.IPMasq {
n.ApplyOutboundNatPolicy(hnsNetwork.Subnets[0].AddressPrefix)
}
hcnEndpoint, err := hns.GenerateHcnEndpoint(epInfo, &n.NetConf)

if err != nil {
return nil, errors.Annotate(err, "error while generating HostComputeEndpoint")
}
return hcnEndpoint, nil
})
if err != nil {
return errors.Annotate(err, "error while loadNetConf")
return nil, errors.Annotate(err, "error while adding HostComputeEndpoint")
}

result, err := hns.ConstructHcnResult(hcnNetwork, hcnEndpoint)

if err != nil {
ipam.ExecDel(n.IPAM.Type, args.StdinData)
return nil, errors.Annotate(err, "error while constructing HostComputeEndpoint addition result")
}

return result, nil

}
func cmdHnsAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) {
success := false

if len(n.EndpointMacPrefix) != 0 {
if len(n.EndpointMacPrefix) != 5 || n.EndpointMacPrefix[2] != '-' {
return fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", n.EndpointMacPrefix)
return nil, fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", n.EndpointMacPrefix)
}
} else {
n.EndpointMacPrefix = "0E-2A"
Expand All @@ -73,15 +165,15 @@ func cmdAdd(args *skel.CmdArgs) error {
networkName := n.Name
hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
if err != nil {
return errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName)
return nil, errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName)
}

if hnsNetwork == nil {
return fmt.Errorf("network %v not found", networkName)
return nil, fmt.Errorf("network %v not found", networkName)
}

if !strings.EqualFold(hnsNetwork.Type, "Overlay") {
return fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type)
return nil, fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type)
}

epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
Expand Down Expand Up @@ -140,15 +232,34 @@ func cmdAdd(args *skel.CmdArgs) error {
}
}()
if err != nil {
return errors.Annotatef(err, "error while AddHnsEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID)
return nil, errors.Annotatef(err, "error while AddHnsEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID)
}

result, err := hns.ConstructHnsResult(hnsNetwork, hnsEndpoint)
if err != nil {
return errors.Annotatef(err, "error while constructResult")
return nil, errors.Annotatef(err, "error while constructResult")
}

success = true
return result, nil
}
func cmdAdd(args *skel.CmdArgs) error {
n, cniVersion, err := loadNetConf(args.StdinData)
if err != nil {
return err
}

var result *current.Result
if n.ApiVersion == 2 {
result, err = cmdHcnAdd(args, n)
} else {
result, err = cmdHnsAdd(args, n)
}
if err != nil {
ipam.ExecDel(n.IPAM.Type, args.StdinData)
return err
}

return types.PrintResult(result, cniVersion)
}

Expand All @@ -158,12 +269,16 @@ func cmdDel(args *skel.CmdArgs) error {
return err
}

if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
return err
if n.IPAM.Type != "" {
if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
return err
}
}

epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)

if n.ApiVersion == 2 {
return hns.RemoveHcnEndpoint(epName)
}
return hns.RemoveHnsEndpoint(epName, args.Netns, args.ContainerID)
}

Expand Down