diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 8768ffa377..a5af29a64c 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -192,6 +192,13 @@ type VhostUserEndpoint struct { EndpointType EndpointType } +type BridgedMacvlanEndpoint struct { + NetPair NetworkInterfacePair + EndpointProperties NetworkInfo + Physical bool + EndpointType EndpointType +} + // Properties returns properties for the veth interface in the network pair. func (endpoint *VirtualEndpoint) Properties() NetworkInfo { return endpoint.EndpointProperties @@ -227,7 +234,7 @@ func networkLogger() *logrus.Entry { func (endpoint *VirtualEndpoint) Attach(h hypervisor) error { networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint") - if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil { + if err := xconnectVMNetwork(&(endpoint.NetPair), true, endpoint.EndpointType); err != nil { networkLogger().WithError(err).Error("Error bridging virtual endpoint") return err } @@ -247,14 +254,80 @@ func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) err networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint") return doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(&(endpoint.NetPair), false) + return xconnectVMNetwork(&(endpoint.NetPair), false, endpoint.EndpointType) + }) +} + +// Macvlan + +// Properties returns properties of the interface. +func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// Name returns name of the veth interface in the network pair. +func (endpoint *BridgedMacvlanEndpoint) Name() string { + return endpoint.NetPair.VirtIface.Name +} + +// HardwareAddr returns the mac address that is assigned to the tap interface +// in th network pair. +func (endpoint *BridgedMacvlanEndpoint) HardwareAddr() string { + return endpoint.NetPair.TAPIface.HardAddr +} + +// Type identifies the endpoint as a virtual endpoint. +func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// SetProperties sets the properties for the endpoint. +func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// Attach for virtual endpoint bridges the network pair and adds the +// tap interface of the network pair to the hypervisor. +func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error { + networkLogger().Info("Attaching macvlan endpoint") + if err := xconnectVMNetwork(&(endpoint.NetPair), true, endpoint.EndpointType); err != nil { + networkLogger().WithError(err).Error("Error bridging virtual ep") + return err + } + + return h.addDevice(endpoint, netDev) +} + +// Detach for the virtual endpoint tears down the tap and bridge +// created for the veth interface. +func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath string) error { + // The network namespace would have been deleted at this point + // if it has not been created by virtcontainers. + if !netNsCreated { + return nil + } + + networkLogger().Info("Detaching virtual endpoint") + + return doNetNS(netNsPath, func(_ ns.NetNS) error { + return xconnectVMNetwork(&(endpoint.NetPair), false, endpoint.EndpointType) }) } +// HotAttach for physical endpoint not supported yet +func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach") +} + +// HotDetach for physical endpoint not supported yet +func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach") +} + // HotAttach for the virtual endpoint uses hot plug device func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error { networkLogger().Info("Hot attaching virtual endpoint") - if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil { + if err := xconnectVMNetwork(&(endpoint.NetPair), true, endpoint.EndpointType); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } @@ -273,7 +346,7 @@ func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netN } networkLogger().Info("Hot detaching virtual endpoint") if err := doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(&(endpoint.NetPair), false) + return xconnectVMNetwork(&(endpoint.NetPair), false, endpoint.EndpointType) }); err != nil { networkLogger().WithError(err).Error("Error abridging virtual ep") return err @@ -439,6 +512,8 @@ const ( // VhostUserEndpointType is the vhostuser network interface. VhostUserEndpointType EndpointType = "vhost-user" + + BridgedMacvlanEndpointType EndpointType = "macvlan" ) // Set sets an endpoint type based on the input string. @@ -453,6 +528,9 @@ func (endpointType *EndpointType) Set(value string) error { case "vhost-user": *endpointType = VhostUserEndpointType return nil + case "macvlan": + *endpointType = BridgedMacvlanEndpointType + return nil default: return fmt.Errorf("Unknown endpoint type %s", value) } @@ -467,6 +545,8 @@ func (endpointType *EndpointType) String() string { return string(VirtualEndpointType) case VhostUserEndpointType: return string(VhostUserEndpointType) + case BridgedMacvlanEndpointType: + return string(BridgedMacvlanEndpointType) default: return "" } @@ -584,6 +664,16 @@ func (n *NetworkNamespace) UnmarshalJSON(b []byte) error { "endpoint-type": "vhostuser", }).Info("endpoint unmarshalled") + case BridgedMacvlanEndpointType: + var endpoint BridgedMacvlanEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return err + } + + endpoints = append(endpoints, &endpoint) + virtLog.Infof("Macvlan endpoint unmarshalled [%v]", endpoint) + default: networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type") } @@ -878,6 +968,10 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. if l, ok := link.(*netlink.Macvtap); ok { return l, nil } + case (&netlink.Macvlan{}).Type(): + if l, ok := link.(*netlink.Macvlan); ok { + return l, nil + } default: return nil, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) } @@ -886,7 +980,7 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. } // The endpoint type should dictate how the connection needs to be made -func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool) error { +func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, endpointType EndpointType) error { if netPair.NetInterworkingModel == NetXConnectDefaultModel { netPair.NetInterworkingModel = DefaultNetInterworkingModel } @@ -894,15 +988,15 @@ func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool) error { case NetXConnectBridgedModel: netPair.NetInterworkingModel = NetXConnectBridgedModel if connect { - return bridgeNetworkPair(netPair) + return bridgeNetworkPair(netPair, endpointType) } - return unBridgeNetworkPair(*netPair) + return unBridgeNetworkPair(*netPair, endpointType) case NetXConnectMacVtapModel: netPair.NetInterworkingModel = NetXConnectMacVtapModel if connect { - return tapNetworkPair(netPair) + return tapNetworkPair(netPair, endpointType) } - return untapNetworkPair(*netPair) + return untapNetworkPair(*netPair, endpointType) case NetXConnectEnlightenedModel: return fmt.Errorf("Unsupported networking model") default: @@ -989,18 +1083,30 @@ func setIPs(link netlink.Link, addrs []netlink.Addr) error { return nil } -func tapNetworkPair(netPair *NetworkInterfacePair) error { +func tapNetworkPair(netPair *NetworkInterfacePair, endpointType EndpointType) error { netHandle, err := netlink.NewHandle() if err != nil { return err } defer netHandle.Delete() - vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) - if err != nil { - return fmt.Errorf("Could not get veth interface: %s: %s", netPair.VirtIface.Name, err) + var link netlink.Link + + if endpointType == VirtualEndpointType { + link, err = getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) + if err != nil { + return fmt.Errorf("Could not get veth interface: %s: %s", netPair.VirtIface.Name, err) + } + } else if endpointType == BridgedMacvlanEndpointType { + link, err = getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Macvlan{}) + if err != nil { + return fmt.Errorf("Could not get macvlan interface: %s: %s", netPair.VirtIface.Name, err) + } + } else { + return fmt.Errorf("Unxepected Endpoint type : %s", endpointType) } - vethLinkAttrs := vethLink.Attrs() + + attrs := link.Attrs() // Attach the macvtap interface to the underlying container // interface. Also picks relevant attributes from the parent @@ -1008,8 +1114,8 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error { &netlink.Macvtap{ Macvlan: netlink.Macvlan{ LinkAttrs: netlink.LinkAttrs{ - TxQLen: vethLinkAttrs.TxQLen, - ParentIndex: vethLinkAttrs.Index, + TxQLen: attrs.TxQLen, + ParentIndex: attrs.Index, }, }, }) @@ -1023,18 +1129,18 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error { // the one inside the VM in order to avoid any firewall issues. The // bridge created by the network plugin on the host actually expects // to see traffic from this MAC address and not another one. - tapHardAddr := vethLinkAttrs.HardwareAddr - netPair.TAPIface.HardAddr = vethLinkAttrs.HardwareAddr.String() + tapHardAddr := attrs.HardwareAddr + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - if err := netHandle.LinkSetMTU(tapLink, vethLinkAttrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", vethLinkAttrs.MTU, err) + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) } hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) if err != nil { return err } - if err := netHandle.LinkSetHardwareAddr(vethLink, hardAddr); err != nil { + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) } @@ -1049,16 +1155,16 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error { } // Clear the IP addresses from the veth interface to prevent ARP conflict - netPair.VirtIface.Addrs, err = netlink.AddrList(vethLink, netlink.FAMILY_V4) + netPair.VirtIface.Addrs, err = netlink.AddrList(link, netlink.FAMILY_V4) if err != nil { return fmt.Errorf("Unable to obtain veth IP addresses: %s", err) } - if err := clearIPs(vethLink, netPair.VirtIface.Addrs); err != nil { + if err := clearIPs(link, netPair.VirtIface.Addrs); err != nil { return fmt.Errorf("Unable to clear veth IP addresses: %s", err) } - if err := netHandle.LinkSetUp(vethLink); err != nil { + if err := netHandle.LinkSetUp(link); err != nil { return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) } @@ -1087,7 +1193,7 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error { return nil } -func bridgeNetworkPair(netPair *NetworkInterfacePair) error { +func bridgeNetworkPair(netPair *NetworkInterfacePair, endpointType EndpointType) error { netHandle, err := netlink.NewHandle() if err != nil { return err @@ -1106,29 +1212,41 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair) error { } netPair.VhostFds = vhostFds - vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) - if err != nil { - return fmt.Errorf("Could not get veth interface %s : %s", netPair.VirtIface.Name, err) + var attrs *netlink.LinkAttrs + var link netlink.Link + + if endpointType == VirtualEndpointType { + link, err = getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) + if err != nil { + return fmt.Errorf("Could not get veth interface %s : %s", netPair.VirtIface.Name, err) + } + } else if endpointType == BridgedMacvlanEndpointType { + link, err = getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Macvlan{}) + if err != nil { + return fmt.Errorf("Could not get veth interface %s : %s", netPair.VirtIface.Name, err) + } + } else { + return fmt.Errorf("Unexpected endpointType %s", endpointType) } - vethLinkAttrs := vethLink.Attrs() + attrs = link.Attrs() // Save the veth MAC address to the TAP so that it can later be used // to build the hypervisor command line. This MAC address has to be // the one inside the VM in order to avoid any firewall issues. The // bridge created by the network plugin on the host actually expects // to see traffic from this MAC address and not another one. - netPair.TAPIface.HardAddr = vethLinkAttrs.HardwareAddr.String() + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - if err := netHandle.LinkSetMTU(tapLink, vethLinkAttrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", vethLinkAttrs.MTU, err) + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) } hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) if err != nil { return err } - if err := netHandle.LinkSetHardwareAddr(vethLink, hardAddr); err != nil { + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) } @@ -1148,12 +1266,12 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair) error { return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) } - if err := netHandle.LinkSetMaster(vethLink, bridgeLink.(*netlink.Bridge)); err != nil { + if err := netHandle.LinkSetMaster(link, bridgeLink.(*netlink.Bridge)); err != nil { return fmt.Errorf("Could not attach veth %s to the bridge %s: %s", netPair.VirtIface.Name, netPair.Name, err) } - if err := netHandle.LinkSetUp(vethLink); err != nil { + if err := netHandle.LinkSetUp(link); err != nil { return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) } @@ -1164,7 +1282,7 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair) error { return nil } -func untapNetworkPair(netPair NetworkInterfacePair) error { +func untapNetworkPair(netPair NetworkInterfacePair, endpointType EndpointType) error { netHandle, err := netlink.NewHandle() if err != nil { return err @@ -1180,22 +1298,33 @@ func untapNetworkPair(netPair NetworkInterfacePair) error { return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) } - vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) - if err != nil { - // The veth pair is not totally managed by virtcontainers - networkLogger().WithError(err).WithField("veth-name", netPair.VirtIface.Name).Warn("Could not get veth interface") - } else { - if err := netHandle.LinkSetDown(vethLink); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + var link netlink.Link + + if endpointType == VirtualEndpointType { + link, err = getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) + if err != nil { + // The veth pair is not totally managed by virtcontainers + networkLogger().WithError(err).WithField("veth-name", netPair.VirtIface.Name).Warn("Could not get veth interface") + } + } else if endpointType == BridgedMacvlanEndpointType { + link, err = getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Macvlan{}) + if err != nil { + return fmt.Errorf("Could not get veth interface: %s: %s", netPair.VirtIface.Name, err) } + } else { + return fmt.Errorf("Unxepected Endpoint type : %s", endpointType) + } + + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) } // Restore the IPs that were cleared - err = setIPs(vethLink, netPair.VirtIface.Addrs) + err = setIPs(link, netPair.VirtIface.Addrs) return err } -func unBridgeNetworkPair(netPair NetworkInterfacePair) error { +func unBridgeNetworkPair(netPair NetworkInterfacePair, endpointType EndpointType) error { netHandle, err := netlink.NewHandle() if err != nil { return err @@ -1232,19 +1361,29 @@ func unBridgeNetworkPair(netPair NetworkInterfacePair) error { return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) } - vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) - if err != nil { - // The veth pair is not totally managed by virtcontainers - networkLogger().WithError(err).Warn("Could not get veth interface") - } else { - if err := netHandle.LinkSetDown(vethLink); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) - } + var link netlink.Link + if endpointType == VirtualEndpointType { + link, err = getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) - if err := netHandle.LinkSetNoMaster(vethLink); err != nil { - return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) + if err != nil { + // The veth pair is not totally managed by virtcontainers + networkLogger().WithError(err).Warn("Could not get veth interface") + } + } else if endpointType == BridgedMacvlanEndpointType { + link, err = getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Macvlan{}) + if err != nil { + return fmt.Errorf("Could not get veth interface: %s: %s", netPair.VirtIface.Name, err) } + } else { + return fmt.Errorf("Unxepected Endpoint type : %s", endpointType) + } + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + } + + if err := netHandle.LinkSetNoMaster(link); err != nil { + return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) } return nil @@ -1311,29 +1450,36 @@ func createVirtualNetworkEndpoint(idx int, ifName string, interworkingModel NetI return &VirtualEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) } - uniqueID := uuid.Generate().String() - - hardAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, byte(idx >> 8), byte(idx)} + netPair := createNetworkInterfacePair(idx, ifName, interworkingModel) endpoint := &VirtualEndpoint{ // TODO This is too specific. We may need to create multiple // end point types here and then decide how to connect them // at the time of hypervisor attach and not here - NetPair: NetworkInterfacePair{ - ID: uniqueID, - Name: fmt.Sprintf("br%d", idx), - VirtIface: NetworkInterface{ - Name: fmt.Sprintf("eth%d", idx), - HardAddr: hardAddr.String(), - }, - TAPIface: NetworkInterface{ - Name: fmt.Sprintf("tap%d", idx), - }, - NetInterworkingModel: interworkingModel, - }, + NetPair: netPair, EndpointType: VirtualEndpointType, } + if ifName != "" { + endpoint.NetPair.VirtIface.Name = ifName + } + + return endpoint, nil +} + +func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) { + if idx < 0 { + return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } + netPair := createNetworkInterfacePair(idx, ifName, interworkingModel) + + endpoint := &BridgedMacvlanEndpoint{ + // TODO This is too specific. We may need to create multiple + // end point types here and then decide how to connect them + // at the time of hypervisor attach and not here + NetPair: netPair, + EndpointType: BridgedMacvlanEndpointType, + } if ifName != "" { endpoint.NetPair.VirtIface.Name = ifName } @@ -1428,6 +1574,28 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*grpc.Interface, return ifaces, routes, nil } +func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) NetworkInterfacePair { + uniqueID := uuid.Generate().String() + + hardAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, byte(idx >> 8), byte(idx)} + + netPair := NetworkInterfacePair{ + ID: uniqueID, + Name: fmt.Sprintf("br%d", idx), + VirtIface: NetworkInterface{ + Name: fmt.Sprintf("eth%d", idx), + HardAddr: hardAddr.String(), + }, + TAPIface: NetworkInterface{ + Name: fmt.Sprintf("tap%d", idx), + }, + NetInterworkingModel: interworkingModel, + } + + return netPair + +} + func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo, error) { addrs, err := handle.AddrList(link, netlink.FAMILY_ALL) if err != nil { @@ -1520,6 +1688,10 @@ func createEndpointsFromScan(networkNSPath string, config NetworkConfig) ([]Endp if socketPath != "" { networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") endpoint, err = createVhostUserEndpoint(netInfo, socketPath) + } else if macvlan, ok := link.(*netlink.Macvlan); ok { + networkLogger().Infof("macvlan interface found: %v", macvlan) + endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, config.InterworkingModel) + return nil } else { endpoint, err = createVirtualNetworkEndpoint(idx, netInfo.Iface.Name, config.InterworkingModel) } diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index d65bc9cb72..d8fc983585 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -441,6 +441,23 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi }, ) q.networkIndex++ + case *BridgedMacvlanEndpoint: + devices = append(devices, + govmmQemu.NetDevice{ + Type: networkModelToQemuType(ep.NetPair.NetInterworkingModel), + Driver: govmmQemu.VirtioNetPCI, + ID: fmt.Sprintf("network-%d", q.networkIndex), + IFName: ep.NetPair.TAPIface.Name, + MACAddress: ep.NetPair.TAPIface.HardAddr, + DownScript: "no", + Script: "no", + VHost: true, + DisableModern: q.nestedRun, + FDs: ep.NetPair.VMFds, + VhostFDs: ep.NetPair.VhostFds, + }, + ) + q.networkIndex++ } return devices