Merge pull request kubernetes-sigs#196 from jweite-amazon/multi-vm-affinity-e2e

Multi-VM affinity e2e

Author: jweite-amazon

Makefile

@@ -280,12 +280,13 @@ e2e-essentials: $(GINKGO_V1) $(KUBECTL) e2e-cluster-templates kind-cluster ## Fu
 	IMG=$(IMG_LOCAL) make generate-manifests docker-build docker-push
 
 JOB ?= .*
+E2E_CONFIG ?= ${REPO_ROOT}/test/e2e/config/cloudstack.yaml
 run-e2e: e2e-essentials ## Run e2e testing. JOB is an optional REGEXP to select certainn test cases to run. e.g. JOB=PR-Blocking, JOB=Conformance
 	$(KUBECTL) apply -f cloud-config.yaml && \
 	cd test/e2e && \
 	$(GINKGO_V1) -v -trace -tags=e2e -focus=$(JOB) -skip=Conformance -skipPackage=helpers -nodes=1 -noColor=false ./... -- \
 	    -e2e.artifacts-folder=${REPO_ROOT}/_artifacts \
-	    -e2e.config=${REPO_ROOT}/test/e2e/config/cloudstack.yaml \
+	    -e2e.config=${E2E_CONFIG} \
 	    -e2e.skip-resource-cleanup=false -e2e.use-existing-cluster=true
 	kind delete clusters capi-test
 

test/e2e/README.md

@@ -33,16 +33,26 @@ The first step to running the e2e tests is setting up the required environment v
 | `CLOUDSTACK_TEMPLATE_NAME`                  | The machine template for both control plane and worke node VM instances          | `kube-v1.20.10/ubuntu-2004` |
 | `CLOUDSTACK_SSH_KEY_NAME`                   | The name of SSH key added to the VM instances                                    | `CAPCKeyPair6`              |
 
-You also have to export `CLOUDSTACK_B64ENCODED_SECRET` environment variable using this command `export CLOUDSTACK_B64ENCODED_SECRET=$(base64 -i cloud-config)` after creating `cloud-config` file with the following format.
+Default values for these variables are defined in *config/cloudstack.yaml*.  This cloudstack.yaml can be completely overridden 
+by providing make with the *fully qualified* path of another cloudstack.yaml via environment variable `E2E_CONFIG`
+
+You will also have to define a k8s secret in a *cloud-config.yaml* file in the project root, containing a pointer to and 
+credentials for the CloudStack backend that will be used for the test:
 
 ```
-[Global]
-api-key    = XXXXX
-secret-key = XXXXX
-api-url    = http://192.168.1.96:8080/client/api
-verify-ssl = true or false
+apiVersion: v1
+kind: Secret
+metadata:
+  name: secret1
+  namespace: default
+type: Opaque
+stringData:
+  api-key: XXXX
+  secret-key: XXXX
+  api-url: http://1.2.3.4:8080/client/api
+  verify-ssl: "false"
 ```
-
+This will be applied to the kind cluster that hosts CAPI/CAPC for the test, allowing CAPC to access the cluster. 
 The api-key and secret-key can be found or generated at Home > Accounts > admin > Users > admin of the ACS management UI. `verify-ssl` is an optional flag and its default value is true. CAPC skips verifying the host SSL certificates when the flag is set to false.
 
 ### Running the e2e tests

test/e2e/affinity_group.go

@@ -45,6 +45,14 @@ func AffinityGroupSpec(ctx context.Context, inputGetter func() CommonSpecInput)
 	BeforeEach(func() {
 		Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
 		input = inputGetter()
+
+		csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
+		zoneName := input.E2EConfig.GetVariable("CLOUDSTACK_ZONE_NAME")
+		numHosts := GetHostCount(csClient, zoneName)
+		if numHosts < 3 {
+			Skip("Too few ACS hosts to run conclusive affinity tests.  Please provision at least three for the zone.")
+		}
+
 		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
 		Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
 		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
@@ -91,8 +99,8 @@ func executeTest(ctx context.Context, input CommonSpecInput, namespace *corev1.N
 			Namespace:                namespace.Name,
 			ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
 			KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
-			ControlPlaneMachineCount: pointer.Int64Ptr(1),
-			WorkerMachineCount:       pointer.Int64Ptr(1),
+			ControlPlaneMachineCount: pointer.Int64Ptr(3),
+			WorkerMachineCount:       pointer.Int64Ptr(2),
 		},
 		WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
 		WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),

test/e2e/common.go

@@ -287,6 +287,24 @@ func CheckAffinityGroupsDeleted(client *cloudstack.CloudStackClient, affinityIds
 	return nil
 }
 
+func GetHostCount(client *cloudstack.CloudStackClient, zoneName string) int {
+	pz := client.Zone.NewListZonesParams()
+	pz.SetName(zoneName)
+	listZonesResponse, err := client.Zone.ListZones(pz)
+	Expect(err).To(BeNil(), "error listing zones")
+	Expect(listZonesResponse.Count).To(Equal(1), "no zones, or more than one zone resolve to zone name %s", zoneName)
+	zoneId := listZonesResponse.Zones[0].Id
+
+	ph := client.Host.NewListHostsParams()
+	ph.SetZoneid(zoneId)
+	ph.SetHypervisor("KVM")
+	ph.SetResourcestate("Enabled")
+	ph.SetState("Up")
+	listHostsResponse, err := client.Host.ListHosts(ph)
+	Expect(err).To(BeNil(), "error listing hosts")
+	return listHostsResponse.Count
+}
+
 func CheckAffinityGroup(client *cloudstack.CloudStackClient, clusterName string, affinityType string) []string {
 	By("Listing all machines")
 	p := client.VirtualMachine.NewListVirtualMachinesParams()