USHIFT-7187: C2CC Dual Stack support & tests

cmd/generate-config/config/config-openapi-spec.json

@@ -232,8 +232,11 @@
                 "type": "string"
               },
               "nextHop": {
-                "description": "IP address of the remote cluster's node, used as next-hop for routing.",
-                "type": "string"
+                "description": "IP addresses of the remote cluster's node, used as next-hop for routing.\nAt most one IPv4 and one IPv6 address. Dual-stack clusters need both.",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
               },
               "serviceNetwork": {
                 "description": "Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes.",

docs/user/howto_config.md

@@ -38,7 +38,7 @@ clusterToCluster:
     remoteClusters:
         - clusterNetwork: []
           domain: ""
-          nextHop: ""
+          nextHop: []
           serviceNetwork: []
     routing:
         routeTableID: 0
@@ -207,7 +207,7 @@ clusterToCluster:
     remoteClusters:
         - clusterNetwork: []
           domain: ""
-          nextHop: ""
+          nextHop: []
           serviceNetwork: []
     routing:
         routeTableID: 200

packaging/microshift/config.yaml

@@ -62,8 +62,9 @@ clusterToCluster:
           # Services are reachable as <svc>.<ns>.svc.<domain>.
           # Optional — if empty, no DNS forwarding is configured for this remote.
           domain: ""
-          # IP address of the remote cluster's node, used as next-hop for routing.
-          nextHop: ""
+          # IP addresses of the remote cluster's node, used as next-hop for routing.
+          # At most one IPv4 and one IPv6 address. Dual-stack clusters need both.
+          nextHop: []
           # Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes.
           serviceNetwork: []
     # Linux policy routing table settings for C2CC routes.

pkg/config/c2cc.go

@@ -68,8 +68,9 @@ type C2CC struct {
 }
 
 type RemoteCluster struct {
-	// IP address of the remote cluster's node, used as next-hop for routing.
-	NextHop string `json:"nextHop"`
+	// IP addresses of the remote cluster's node, used as next-hop for routing.
+	// At most one IPv4 and one IPv6 address. Dual-stack clusters need both.
+	NextHop []string `json:"nextHop"`
 	// Pod CIDRs of the remote cluster. Must not overlap with local cluster or other remotes.
 	ClusterNetwork []string `json:"clusterNetwork"`
 	// Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes.
@@ -81,14 +82,40 @@ type RemoteCluster struct {
 }
 
 type ResolvedRemoteCluster struct {
-	NextHop        net.IP
+	NextHops       map[int]net.IP // key: netlink.FAMILY_V4 or netlink.FAMILY_V6
 	ClusterNetwork []*net.IPNet
 	ServiceNetwork []*net.IPNet
 	Domain         string
 	DNSIP          string // 10th IP of ServiceNetwork[0], computed during validation when Domain is set
 	ProbeIP        string // 11th IP of ServiceNetwork[0], deterministic probe service ClusterIP
 }
 
+func (rc *ResolvedRemoteCluster) NextHopForFamily(family int) (net.IP, bool) {
+	ip, ok := rc.NextHops[family]
+	return ip, ok
+}
+
+func (rc *ResolvedRemoteCluster) PrimaryNextHop() net.IP {
+	if len(rc.ClusterNetwork) == 0 {
+		return nil
+	}
+	return rc.NextHops[ipFamilyOfIPNet(rc.ClusterNetwork[0])]
+}
+
+func ipFamilyOfIPNet(ipNet *net.IPNet) int {
+	if ipNet.IP.To4() != nil {
+		return netlink.FAMILY_V4
+	}
+	return netlink.FAMILY_V6
+}
+
+func familyName(family int) string {
+	if family == netlink.FAMILY_V4 {
+		return "IPv4"
+	}
+	return "IPv6"
+}
+
 func (rc *ResolvedRemoteCluster) AllCIDRs() []*net.IPNet {
 	all := make([]*net.IPNet, 0, len(rc.ClusterNetwork)+len(rc.ServiceNetwork))
 	all = append(all, rc.ClusterNetwork...)
@@ -114,7 +141,7 @@ func (c *C2CC) AllRemoteCIDRStrings() []string {
 }
 
 func (rc *RemoteCluster) isEmpty() bool {
-	return rc.NextHop == "" && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == ""
+	return len(rc.NextHop) == 0 && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == ""
 }
 
 func (c *C2CC) stripEmptyRemoteClusters() {
@@ -155,11 +182,26 @@ func (c *C2CC) parseRemoteClusters() ([]ResolvedRemoteCluster, []error) {
 		rc := &c.RemoteClusters[i]
 		label := fmt.Sprintf("remoteClusters[%d]", i)
 
-		ip := net.ParseIP(rc.NextHop)
-		if ip == nil {
-			errs = append(errs, fmt.Errorf("%s.nextHop %q is not a valid IP address", label, rc.NextHop))
+		if len(rc.NextHop) == 0 {
+			errs = append(errs, fmt.Errorf("%s.nextHop must not be empty", label))
+		}
+		resolved[i].NextHops = make(map[int]net.IP, len(rc.NextHop))
+		for j, hopStr := range rc.NextHop {
+			ip := net.ParseIP(hopStr)
+			if ip == nil {
+				errs = append(errs, fmt.Errorf("%s.nextHop[%d] %q is not a valid IP address", label, j, hopStr))
+				continue
+			}
+			family := netlink.FAMILY_V4
+			if ip.To4() == nil {
+				family = netlink.FAMILY_V6
+			}
+			if _, dup := resolved[i].NextHops[family]; dup {
+				errs = append(errs, fmt.Errorf("%s.nextHop has multiple %s addresses (max 1 per family)", label, familyName(family)))
+				continue
+			}
+			resolved[i].NextHops[family] = ip
 		}
-		resolved[i].NextHop = ip
 
 		if len(rc.ClusterNetwork) == 0 {
 			errs = append(errs, fmt.Errorf("%s.clusterNetwork must not be empty", label))
@@ -359,14 +401,16 @@ func validateRemoteCluster(
 	label := fmt.Sprintf("remoteClusters[%d]", i)
 	var errs []error
 
-	normalizedNextHop := res.NextHop.String()
-	if res.NextHop.Equal(nodeIP) || (nodeIPv6 != nil && res.NextHop.Equal(nodeIPv6)) {
-		errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalizedNextHop))
-	}
-	if prev, ok := seenNextHops[normalizedNextHop]; ok {
-		errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalizedNextHop, prev))
-	} else {
-		seenNextHops[normalizedNextHop] = i
+	for _, hop := range res.NextHops {
+		normalized := hop.String()
+		if hop.Equal(nodeIP) || (nodeIPv6 != nil && hop.Equal(nodeIPv6)) {
+			errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalized))
+		}
+		if prev, ok := seenNextHops[normalized]; ok {
+			errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalized, prev))
+		} else {
+			seenNextHops[normalized] = i
+		}
 	}
 
 	for j, cidrNet := range res.ClusterNetwork {
@@ -406,6 +450,7 @@ func validateRemoteCluster(
 	errs = append(errs, validateNetworkShapeNets(res.ClusterNetwork, res.ServiceNetwork, label)...)
 	errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ClusterNetwork, label)...)
 	errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ServiceNetwork, label)...)
+	errs = append(errs, validateNextHopCoverage(res.NextHops, res.ClusterNetwork, res.ServiceNetwork, label)...)
 
 	return errs
 }
@@ -462,6 +507,23 @@ func validateRemoteIPFamilyCompatibility(localV4, localV6 bool, remoteCIDRs []*n
 	return errs
 }
 
+func validateNextHopCoverage(nextHops map[int]net.IP, clusterNetwork, serviceNetwork []*net.IPNet, label string) []error {
+	needed := make(map[int]bool)
+	for _, cidr := range clusterNetwork {
+		needed[ipFamilyOfIPNet(cidr)] = true
+	}
+	for _, cidr := range serviceNetwork {
+		needed[ipFamilyOfIPNet(cidr)] = true
+	}
+	var errs []error
+	for family := range needed {
+		if _, ok := nextHops[family]; !ok {
+			errs = append(errs, fmt.Errorf("%s has %s CIDRs but no %s nextHop", label, familyName(family), familyName(family)))
+		}
+	}
+	return errs
+}
+
 func checkCIDRConflicts(cidr *net.IPNet, cidrStr, label string, seenCIDRs []labeledCIDR, hostIPs []net.IP) []error {
 	var errs []error
 	for _, existing := range seenCIDRs {