From 1c9547b83c0117ad8fb897f9b2210628559c2431 Mon Sep 17 00:00:00 2001 From: arshadda Date: Thu, 25 Sep 2025 09:23:27 +0530 Subject: [PATCH] debug Objects with Changed Resource Versions --- test/e2e/quick_start.go | 286 ++++++++++++---------- test/e2e/quick_start_test.go | 4 +- test/framework/resourceversion_helpers.go | 104 +++++++- 3 files changed, 263 insertions(+), 131 deletions(-) diff --git a/test/e2e/quick_start.go b/test/e2e/quick_start.go index 20a3beb63396..7e8c1163af1e 100644 --- a/test/e2e/quick_start.go +++ b/test/e2e/quick_start.go @@ -27,8 +27,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api/test/e2e/internal/log" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -111,156 +114,183 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput) clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult ) - BeforeEach(func() { - Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) - input = inputGetter() - Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) - Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) - Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) - Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + for i := 0; i < 20; i++ { - Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) - if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" { - if input.ExtensionConfigName == "" { - input.ExtensionConfigName = specName + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" { + if input.ExtensionConfigName == "" { + input.ExtensionConfigName = specName + } } - } - // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated) + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated) - if input.DeployClusterClassInSeparateNamespace { - clusterClassNamespace = framework.CreateNamespace(ctx, framework.CreateNamespaceInput{Creator: input.BootstrapClusterProxy.GetClient(), Name: fmt.Sprintf("%s-clusterclass", namespace.Name)}, "40s", "10s") - Expect(clusterClassNamespace).ToNot(BeNil(), "Failed to create namespace") - } + if input.DeployClusterClassInSeparateNamespace { + clusterClassNamespace = framework.CreateNamespace(ctx, framework.CreateNamespaceInput{Creator: input.BootstrapClusterProxy.GetClient(), Name: fmt.Sprintf("%s-clusterclass", namespace.Name)}, "40s", "10s") + Expect(clusterClassNamespace).ToNot(BeNil(), "Failed to create namespace") + } - clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) - }) + clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) + }) - It("Should create a workload cluster", func() { - By("Creating a workload cluster") + It("Should create a workload cluster", func() { + By("Creating a workload cluster") - infrastructureProvider := clusterctl.DefaultInfrastructureProvider - if input.InfrastructureProvider != nil { - infrastructureProvider = *input.InfrastructureProvider - } + infrastructureProvider := clusterctl.DefaultInfrastructureProvider + if input.InfrastructureProvider != nil { + infrastructureProvider = *input.InfrastructureProvider + } - flavor := clusterctl.DefaultFlavor - if input.Flavor != nil { - flavor = *input.Flavor - } + flavor := clusterctl.DefaultFlavor + if input.Flavor != nil { + flavor = *input.Flavor + } - controlPlaneMachineCount := ptr.To[int64](1) - if input.ControlPlaneMachineCount != nil { - controlPlaneMachineCount = input.ControlPlaneMachineCount - } + controlPlaneMachineCount := ptr.To[int64](1) + if input.ControlPlaneMachineCount != nil { + controlPlaneMachineCount = input.ControlPlaneMachineCount + } - workerMachineCount := ptr.To[int64](1) - if input.WorkerMachineCount != nil { - workerMachineCount = input.WorkerMachineCount - } + workerMachineCount := ptr.To[int64](1) + if input.WorkerMachineCount != nil { + workerMachineCount = input.WorkerMachineCount + } - clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6)) - if input.ClusterName != nil { - clusterName = *input.ClusterName - } + clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + if input.ClusterName != nil { + clusterName = *input.ClusterName + } + + if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" { + // NOTE: test extension is already deployed in the management cluster. If for any reason in future we want + // to make this test more self-contained this test should be modified in order to create an additional + // management cluster; also the E2E test configuration should be modified introducing something like + // optional:true allowing to define which providers should not be installed by default in + // a management cluster. + By("Deploy Test Extension ExtensionConfig") + + // In this test we are defaulting all handlers to non-blocking because we don't expect the handlers to block the + // cluster lifecycle by default. Setting defaultAllHandlersToBlocking to false enforces that the test-extension + // automatically creates the ConfigMap with non-blocking preloaded responses. + defaultAllHandlersToBlocking := false + // select on the current namespace + // This is necessary so in CI this test doesn't influence other tests by enabling lifecycle hooks + // in other test namespaces. + namespaces := []string{namespace.Name} + if input.DeployClusterClassInSeparateNamespace { + // Add the ClusterClass namespace, if the ClusterClass is deployed in a separate namespace. + namespaces = append(namespaces, clusterClassNamespace.Name) + } + extensionConfig := extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...) + Expect(input.BootstrapClusterProxy.GetClient().Create(ctx, + extensionConfig)). + To(Succeed(), "Failed to create the ExtensionConfig") + } + + variables := map[string]string{ + // This is used to template the name of the ExtensionConfig into the ClusterClass. + "EXTENSION_CONFIG_NAME": input.ExtensionConfigName, + } + maps.Copy(variables, input.ClusterctlVariables) - if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" { - // NOTE: test extension is already deployed in the management cluster. If for any reason in future we want - // to make this test more self-contained this test should be modified in order to create an additional - // management cluster; also the E2E test configuration should be modified introducing something like - // optional:true allowing to define which providers should not be installed by default in - // a management cluster. - By("Deploy Test Extension ExtensionConfig") - - // In this test we are defaulting all handlers to non-blocking because we don't expect the handlers to block the - // cluster lifecycle by default. Setting defaultAllHandlersToBlocking to false enforces that the test-extension - // automatically creates the ConfigMap with non-blocking preloaded responses. - defaultAllHandlersToBlocking := false - // select on the current namespace - // This is necessary so in CI this test doesn't influence other tests by enabling lifecycle hooks - // in other test namespaces. - namespaces := []string{namespace.Name} if input.DeployClusterClassInSeparateNamespace { - // Add the ClusterClass namespace, if the ClusterClass is deployed in a separate namespace. - namespaces = append(namespaces, clusterClassNamespace.Name) + variables["CLUSTER_CLASS_NAMESPACE"] = clusterClassNamespace.Name + By("Creating a cluster referencing a ClusterClass from another namespace") } - extensionConfig := extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, defaultAllHandlersToBlocking, namespaces...) - Expect(input.BootstrapClusterProxy.GetClient().Create(ctx, - extensionConfig)). - To(Succeed(), "Failed to create the ExtensionConfig") - } - variables := map[string]string{ - // This is used to template the name of the ExtensionConfig into the ClusterClass. - "EXTENSION_CONFIG_NAME": input.ExtensionConfigName, - } - maps.Copy(variables, input.ClusterctlVariables) + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + ClusterctlVariables: variables, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: infrastructureProvider, + Flavor: flavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.MustGetVariable(KubernetesVersion), + ControlPlaneMachineCount: controlPlaneMachineCount, + WorkerMachineCount: workerMachineCount, + }, + ControlPlaneWaiters: input.ControlPlaneWaiters, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + PostMachinesProvisioned: func() { + if input.PostMachinesProvisioned != nil { + input.PostMachinesProvisioned(input.BootstrapClusterProxy, namespace.Name, clusterName) + } + }, + }, clusterResources) + + Byf("Verify Cluster Available condition is true") + framework.VerifyClusterAvailable(ctx, framework.VerifyClusterAvailableInput{ + Getter: input.BootstrapClusterProxy.GetClient(), + Name: clusterResources.Cluster.Name, + Namespace: clusterResources.Cluster.Namespace, + }) + + Byf("Verify Machines Ready condition is true") + framework.VerifyMachinesReady(ctx, framework.VerifyMachinesReadyInput{ + Lister: input.BootstrapClusterProxy.GetClient(), + Name: clusterResources.Cluster.Name, + Namespace: clusterResources.Cluster.Namespace, + }) + + By("PASSED!") - if input.DeployClusterClassInSeparateNamespace { - variables["CLUSTER_CLASS_NAMESPACE"] = clusterClassNamespace.Name - By("Creating a cluster referencing a ClusterClass from another namespace") - } + }) - clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ - ClusterProxy: input.BootstrapClusterProxy, - ConfigCluster: clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), - ClusterctlConfigPath: input.ClusterctlConfigPath, - ClusterctlVariables: variables, - KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: infrastructureProvider, - Flavor: flavor, - Namespace: namespace.Name, - ClusterName: clusterName, - KubernetesVersion: input.E2EConfig.MustGetVariable(KubernetesVersion), - ControlPlaneMachineCount: controlPlaneMachineCount, - WorkerMachineCount: workerMachineCount, - }, - ControlPlaneWaiters: input.ControlPlaneWaiters, - WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), - WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), - WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), - PostMachinesProvisioned: func() { - if input.PostMachinesProvisioned != nil { - input.PostMachinesProvisioned(input.BootstrapClusterProxy, namespace.Name, clusterName) + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + if !input.SkipCleanup { + if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" { + Eventually(func() error { + return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true)) + }, 10*time.Second, 1*time.Second).Should(Succeed(), "Deleting ExtensionConfig failed") + } + if input.DeployClusterClassInSeparateNamespace { + DeleteNamespace11(ctx, framework.DeleteNamespaceInput{ + Deleter: input.BootstrapClusterProxy.GetClient(), + Name: clusterClassNamespace.Name, + }) } - }, - }, clusterResources) - - Byf("Verify Cluster Available condition is true") - framework.VerifyClusterAvailable(ctx, framework.VerifyClusterAvailableInput{ - Getter: input.BootstrapClusterProxy.GetClient(), - Name: clusterResources.Cluster.Name, - Namespace: clusterResources.Cluster.Namespace, + } }) - Byf("Verify Machines Ready condition is true") - framework.VerifyMachinesReady(ctx, framework.VerifyMachinesReadyInput{ - Lister: input.BootstrapClusterProxy.GetClient(), - Name: clusterResources.Cluster.Name, - Namespace: clusterResources.Cluster.Namespace, - }) + } +} - By("PASSED!") - }) +// DeleteNamespace is used to delete namespace object. +func DeleteNamespace11(ctx context.Context, input framework.DeleteNamespaceInput, intervals ...interface{}) { + Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteNamespace") + Expect(input.Deleter).NotTo(BeNil(), "input.Deleter is required for DeleteNamespace") + Expect(input.Name).NotTo(BeEmpty(), "input.Name is required for DeleteNamespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: input.Name, + }, + } + log.Logf("Deleting namespace %s", input.Name) + Eventually(func() error { + err := input.Deleter.Delete(ctx, ns) + if err != nil && !errors.IsNotFound(err) { + + return err - AfterEach(func() { - // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) - if !input.SkipCleanup { - if input.ExtensionServiceNamespace != "" && input.ExtensionServiceName != "" { - Eventually(func() error { - return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true)) - }, 10*time.Second, 1*time.Second).Should(Succeed(), "Deleting ExtensionConfig failed") - } - if input.DeployClusterClassInSeparateNamespace { - framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ - Deleter: input.BootstrapClusterProxy.GetClient(), - Name: clusterClassNamespace.Name, - }) - } } - }) + return nil + }, intervals...).Should(Succeed(), "Failed to delete namespace %s", input.Name) } diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index e9f0ef6ee5ad..3524e9ac46e0 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework/kubetest" ) -var _ = Describe("When following the Cluster API quick-start", func() { +var _ = Describe("When following the Cluster API quick-start11 [PR-Blocking] [ClusterClass]", Label("PR-Blocking", "ClusterClass"), func() { QuickStartSpec(ctx, func() QuickStartSpecInput { return QuickStartSpecInput{ E2EConfig: e2eConfig, @@ -75,7 +75,7 @@ var _ = Describe("When following the Cluster API quick-start", func() { }) }) -var _ = Describe("When following the Cluster API quick-start with ClusterClass [PR-Blocking] [ClusterClass]", Label("PR-Blocking", "ClusterClass"), func() { +var _ = Describe("When following the Cluster API quick-start with ClusterClass ", func() { QuickStartSpec(ctx, func() QuickStartSpecInput { return QuickStartSpecInput{ E2EConfig: e2eConfig, diff --git a/test/framework/resourceversion_helpers.go b/test/framework/resourceversion_helpers.go index ee79889a8e68..8699904497db 100644 --- a/test/framework/resourceversion_helpers.go +++ b/test/framework/resourceversion_helpers.go @@ -18,7 +18,9 @@ package framework import ( "context" + "encoding/json" "fmt" + "reflect" "strings" "time" @@ -38,6 +40,96 @@ func ValidateResourceVersionStable(ctx context.Context, proxy ClusterProxy, name byf("Check resourceVersions are stable") var previousResourceVersions map[string]string var previousObjects map[string]client.Object + var localPreviousResourceVersions map[string]string + + // for { + // // Exit if its more than 30 minutes. + // if time.Since(start) > time.Minute*5 { + // log.Fatal("timed out checking for resource version to be stable") + // } + + // if versionMatchCounter >= 5 { + // fmt.Println("Resource version remained same for more than 5 minutes, Total time", time.Since(start)) + // break + // } + + // // Fetch the objects + // objectsWithResourceVersion, objects, err := getObjectsWithResourceVersion(ctx, proxy, namespace, ownerGraphFilterFunction) + // if err != nil { + // log.Println("failed to get objects with the resource version", err) + // continue + // } + + // // Print the objects for debugging + // objectsWithResourceVersionJSON, err := json.Marshal(objectsWithResourceVersion) + // if err != nil { + // log.Println("failed to marshal objectsWithResourceVersion json", err) + // continue + // } + // fmt.Println("objectsWithResourceVersion", string(objectsWithResourceVersionJSON)) + + // objectsJSON, err := json.Marshal(objects) + // if err != nil { + // log.Println("failed to marshal objects json", err) + // continue + // } + // fmt.Println("objectsJSON", string(objectsJSON)) + + // if reflect.DeepEqual(objectsWithResourceVersion, localPreviousResourceVersions) { + // if versionMatchCounter == 0 { + // fmt.Println("Time taken for resource version to be same for the first time", time.Since(start)) + // } + // versionMatchCounter++ + // time.Sleep(time.Second * 30) + // } + // localPreviousResourceVersions = objectsWithResourceVersion + // previousObjects = objects + // } + // if time.Since(start) > time.Minute*1 { + // log.Fatal("failing the test as the time take is more than 1 minute") + // } + + start := time.Now() + Eventually(func(g Gomega) { + // Fetch the objects + objectsWithResourceVersion, objects, err := getObjectsWithResourceVersion(ctx, proxy, namespace, ownerGraphFilterFunction) + g.Expect(err).ToNot(HaveOccurred()) + + if localPreviousResourceVersions == nil { + // Initialize baseline but don't count this first sample toward consecutive successes. + localPreviousResourceVersions = objectsWithResourceVersion + previousObjects = objects + g.Expect(false).To(BeTrue(), "baseline initialized; start counting from next poll") + return + } + + // Print the objects for debugging + objectsWithResourceVersionJSON, err := json.Marshal(objectsWithResourceVersion) + g.Expect(err).ToNot(HaveOccurred()) + fmt.Println("objectsWithResourceVersion", string(objectsWithResourceVersionJSON)) + + objectsJSON, err := json.Marshal(objects) + g.Expect(err).ToNot(HaveOccurred()) + fmt.Println("objectsJSON", string(objectsJSON)) + if reflect.DeepEqual(objectsWithResourceVersion, localPreviousResourceVersions) { + fmt.Printf( + "resourceVersions first became equal at %s; elapsed: %s\n", + time.Now().Format(time.RFC3339), + time.Since(start), + ) + + } + + localPreviousResourceVersions = objectsWithResourceVersion + previousObjects = objects + + }). + WithContext(ctx). + WithTimeout(10*time.Minute). + WithPolling(30*time.Second). + MustPassRepeatedly(10). + Should(Succeed(), "checking resourceVersions") + Eventually(func(g Gomega) { objectsWithResourceVersion, objects, err := getObjectsWithResourceVersion(ctx, proxy, namespace, ownerGraphFilterFunction) g.Expect(err).ToNot(HaveOccurred()) @@ -56,8 +148,16 @@ func ValidateResourceVersionStable(ctx context.Context, proxy ClusterProxy, name Consistently(func(g Gomega) { objectsWithResourceVersion, objects, err := getObjectsWithResourceVersion(ctx, proxy, namespace, ownerGraphFilterFunction) g.Expect(err).ToNot(HaveOccurred()) + resource, er := yaml.Marshal(objectsWithResourceVersion) + g.Expect(er).ToNot(HaveOccurred()) + fmt.Printf("objectsWithResourceVersion:: %s\n", string(resource)) + resource1, er := yaml.Marshal(previousResourceVersions) + g.Expect(er).ToNot(HaveOccurred()) + fmt.Printf("previousResourceVersions:: %s\n", string(resource1)) + g.Expect(previousResourceVersions).To(BeComparableTo(objectsWithResourceVersion), printObjectDiff(previousObjects, objects)) }, 2*time.Minute, 15*time.Second).Should(Succeed(), "resourceVersions didn't stay stable") + } func printObjectDiff(previousObjects, newObjects map[string]client.Object) func() string { @@ -67,7 +167,9 @@ func printObjectDiff(previousObjects, newObjects map[string]client.Object) func( preservedObjects := objectIDs(previousObjects).Intersection(objectIDs(newObjects)) var output strings.Builder - + previousResource, _ := yaml.Marshal(previousObjects) + newResource, _ := yaml.Marshal(newObjects) + output.WriteString(fmt.Sprintf("\n printing Objects %s:\n%s\n", string(previousResource), string(newResource))) if len(createdObjects) > 0 { output.WriteString("\nDetected new objects\n") for objID := range createdObjects {