chore(tests): introduce testcontainers into integration tests as a partial replacement for chainsaw

Makefile

@@ -156,6 +156,11 @@ test: $(ENVTEST) manifests generate vet golangci-lint api-docs kustomize-lint he
 	$(info $(M) running $@)
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(BIN) -p path)" go test ./... -coverprofile cover.out
 
+.PHONY: test-short
+test-short: ## Skips slow integration tests
+	$(info $(M) running $@)
+	go test ./... -short -coverprofile cover.out
+
 .PHONY: vet
 vet: ## Run go vet against code.
 	$(info $(M) running $@)

api/v1beta1/suite_test.go

@@ -43,6 +43,10 @@ var (
 )
 
 func TestAPIs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("-short was passed, skipping CRDs")
+	}
+
 	RegisterFailHandler(Fail)
 
 	RunSpecs(t, "CRDs Suite")

controllers/config/operator_constants.go

@@ -18,6 +18,7 @@ const (
 
 	// Grafana env vars and admin user
 	DefaultAdminUser           = "admin"
+	DefaultAdminPassword       = "admin"
 	GrafanaAdminUserEnvVar     = "GF_SECURITY_ADMIN_USER"
 	GrafanaAdminPasswordEnvVar = "GF_SECURITY_ADMIN_PASSWORD" // #nosec G101
 	GrafanaPluginsEnvVar       = "GF_INSTALL_PLUGINS"

controllers/content/fetchers/suite_test.go

@@ -21,6 +21,10 @@ var (
 )
 
 func TestAPIs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("-short was passed, skipping Fetchers")
+	}
+
 	RegisterFailHandler(Fail)
 
 	RunSpecs(t, "Fetchers Suite")

controllers/content/suite_test.go

@@ -81,6 +81,10 @@ func (in *NopContentResource) DeepCopyInto(out *NopContentResource) {
 }
 
 func TestAPIs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("-short was passed, skipping Content")
+	}
+
 	RegisterFailHandler(Fail)
 
 	RunSpecs(t, "Content Suite")

controllers/controller_shared_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package controllers
 
 import (
-	"context"
 	"testing"
 
 	"github.com/grafana/grafana-operator/v5/api/v1beta1"
@@ -28,7 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // Reusable objectMetas and CommonSpecs to make test tables less verbose
@@ -73,6 +72,16 @@ var (
 			MatchLabels: map[string]string{"invalid-spec": "test"},
 		},
 	}
+
+	objectMetaSynchronized = metav1.ObjectMeta{
+		Namespace: "default",
+		Name:      "synchronized",
+	}
+	commonSpecSynchronized = v1beta1.GrafanaCommonSpec{
+		InstanceSelector: &metav1.LabelSelector{
+			MatchLabels: map[string]string{"synchronized": "test"},
+		},
+	}
 )
 
 func requestFromMeta(obj metav1.ObjectMeta) ctrl.Request {
@@ -345,24 +354,19 @@ func TestMergeReconcileErrors(t *testing.T) {
 }
 
 var _ = Describe("GetMatchingInstances functions", Ordered, func() {
-	ns1 := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{
-		Name: "get-matching-test",
-	}}
-	ns2 := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{
-		Name: "additional-grafana-namespace",
+	ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{
+		Name: "matching-instances",
 	}}
 	allowFolder := v1beta1.GrafanaFolder{
 		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns.Name,
 			Name:      "allow-cross-namespace",
-			Namespace: ns1.Name,
 		},
 		Spec: v1beta1.GrafanaFolderSpec{
 			GrafanaCommonSpec: v1beta1.GrafanaCommonSpec{
 				AllowCrossNamespaceImport: true,
 				InstanceSelector: &metav1.LabelSelector{
-					MatchLabels: map[string]string{
-						"test": "folder",
-					},
+					MatchLabels: map[string]string{"matching-instances": "test"},
 				},
 			},
 		},
@@ -373,69 +377,58 @@ var _ = Describe("GetMatchingInstances functions", Ordered, func() {
 	denyFolder.Spec.AllowCrossNamespaceImport = false
 
 	matchAllFolder := allowFolder.DeepCopy()
-	matchAllFolder.Name = "invalid-match-labels"
+	matchAllFolder.Name = "match-all-grafanas"
 	matchAllFolder.Spec.InstanceSelector = &metav1.LabelSelector{} // InstanceSelector is never nil
 
-	DefaultGrafana := v1beta1.Grafana{
+	BaseGrafana := v1beta1.Grafana{
 		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns.Name,
 			Name:      "instance",
-			Namespace: ns2.Name,
-			Labels: map[string]string{
-				"test": "folder",
-			},
+			Labels:    map[string]string{"matching-instances": "test"},
 		},
 		Spec: v1beta1.GrafanaSpec{},
 	}
-	matchesNothingGrafana := DefaultGrafana.DeepCopy()
-	matchesNothingGrafana.Name = "match-nothing-instance"
+	matchesNothingGrafana := BaseGrafana.DeepCopy()
+	matchesNothingGrafana.Name = "no-labels-instance"
 	matchesNothingGrafana.Labels = nil
 
-	secondNamespaceGrafana := DefaultGrafana.DeepCopy()
-	secondNamespaceGrafana.Name = "second-namespace-instance"
-	secondNamespaceGrafana.Namespace = ns1.Name
-
 	// Status update is skipped for this
-	unreadyGrafana := DefaultGrafana.DeepCopy()
+	unreadyGrafana := BaseGrafana.DeepCopy()
 	unreadyGrafana.Name = "unready-instance"
 
-	ctx := context.Background()
-	testLog := logf.FromContext(ctx).WithSink(logf.NullLogSink{})
-	ctx = logf.IntoContext(ctx, testLog)
+	createCRs := []client.Object{&ns, &allowFolder, denyFolder, matchAllFolder, unreadyGrafana}
 
 	// Pre-create all resources
 	BeforeAll(func() { // Necessary to use assertions
-		Expect(k8sClient.Create(ctx, &ns1)).NotTo(HaveOccurred())
-		Expect(k8sClient.Create(ctx, &ns2)).NotTo(HaveOccurred())
-		Expect(k8sClient.Create(ctx, &allowFolder)).NotTo(HaveOccurred())
-		Expect(k8sClient.Create(ctx, denyFolder)).NotTo(HaveOccurred())
-		Expect(k8sClient.Create(ctx, matchAllFolder)).NotTo(HaveOccurred())
-		Expect(k8sClient.Create(ctx, unreadyGrafana)).NotTo(HaveOccurred())
+		for _, cr := range createCRs {
+			Expect(k8sClient.Create(testCtx, cr)).Should(Succeed())
+		}
 
-		grafanas := []v1beta1.Grafana{DefaultGrafana, *matchesNothingGrafana, *secondNamespaceGrafana}
+		grafanas := []v1beta1.Grafana{BaseGrafana, *matchesNothingGrafana}
 		for _, instance := range grafanas {
-			Expect(k8sClient.Create(ctx, &instance)).NotTo(HaveOccurred())
+			Expect(k8sClient.Create(testCtx, &instance)).NotTo(HaveOccurred())
 
 			// Apply status to pass instance ready check
 			instance.Status.Stage = v1beta1.OperatorStageComplete
 			instance.Status.StageStatus = v1beta1.OperatorStageResultSuccess
-			Expect(k8sClient.Status().Update(ctx, &instance)).ToNot(HaveOccurred())
+			Expect(k8sClient.Status().Update(testCtx, &instance)).ToNot(HaveOccurred())
 		}
 	})
 
 	Context("Ensure AllowCrossNamespaceImport is upheld by GetScopedMatchingInstances", func() {
 		It("Finds all ready instances when instanceSelector is empty", func() {
-			instances, err := GetScopedMatchingInstances(ctx, k8sClient, matchAllFolder)
+			instances, err := GetScopedMatchingInstances(testCtx, k8sClient, matchAllFolder)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(instances).To(HaveLen(3 + 1)) // +1 To account for instance created in suite_test.go to provoke ApplyFailed conditions
+			Expect(instances).To(HaveLen(2 + 2)) // +2 To account for instances created in controllers/suite_test.go to provoke conditions
 		})
 		It("Finds all ready and Matching instances", func() {
-			instances, err := GetScopedMatchingInstances(ctx, k8sClient, &allowFolder)
+			instances, err := GetScopedMatchingInstances(testCtx, k8sClient, &allowFolder)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(instances).ToNot(BeEmpty())
 			Expect(instances).To(HaveLen(2))
 		})
 		It("Finds matching and ready and matching instance in namespace", func() {
-			instances, err := GetScopedMatchingInstances(ctx, k8sClient, denyFolder)
+			instances, err := GetScopedMatchingInstances(testCtx, k8sClient, denyFolder)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(instances).ToNot(BeEmpty())
 			Expect(instances).To(HaveLen(1))

controllers/datasource_controller_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	v1beta1 "github.com/grafana/grafana-operator/v5/api/v1beta1"
+	grafanaclient "github.com/grafana/grafana-operator/v5/controllers/client"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
@@ -42,64 +43,132 @@ func TestGetDatasourceContent(t *testing.T) {
 	})
 }
 
-var _ = Describe("Datasource: Reconciler", func() {
+var _ = Describe("Datasource: substitute reference values", func() {
 	It("Correctly substitutes valuesFrom", func() {
-		cm := corev1.ConfigMap{
+		cm := &corev1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      "test-valuesfrom-plain",
 				Namespace: "default",
+				Name:      "ds-valuesfrom-configmap",
 			},
 			Data: map[string]string{
-				"CUSTOM_URL":     "https://demo.promlabs.com",
-				"CUSTOM_TRACEID": "substituted",
+				"customTraceId": "substituted",
 			},
 		}
-		err := k8sClient.Create(testCtx, &cm)
-		Expect(err).ToNot(HaveOccurred())
-		cr := &v1beta1.GrafanaDatasource{
+		sc := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "default",
+				Name:      "ds-values-from-secret",
+			},
+			StringData: map[string]string{
+				"PROMETHEUS_TOKEN": "secret_token",
+				"URL":              "https://demo.promlabs.com",
+			},
+		}
+		ds := &v1beta1.GrafanaDatasource{
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace: "default",
+				Name:      "substitute-reference-values",
 			},
 			Spec: v1beta1.GrafanaDatasourceSpec{
+				GrafanaCommonSpec: v1beta1.GrafanaCommonSpec{
+					InstanceSelector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{
+							"dashboards": "grafana",
+						},
+					},
+				},
+				CustomUID: "substitute",
 				ValuesFrom: []v1beta1.ValueFrom{
+					{
+						TargetPath: "secureJsonData.httpHeaderValue1",
+						ValueFrom: v1beta1.ValueFromSource{
+							SecretKeyRef: &corev1.SecretKeySelector{
+								LocalObjectReference: corev1.LocalObjectReference{
+									Name: sc.Name,
+								},
+								Key: "PROMETHEUS_TOKEN",
+							},
+						},
+					},
 					{
 						TargetPath: "url",
 						ValueFrom: v1beta1.ValueFromSource{
-							ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
+							SecretKeyRef: &corev1.SecretKeySelector{
 								LocalObjectReference: corev1.LocalObjectReference{
-									Name: cm.Name,
+									Name: sc.Name,
 								},
-								Key: "CUSTOM_URL",
+								Key: "URL",
 							},
 						},
 					},
 					{
-						TargetPath: "jsonData.list[0].value",
+						TargetPath: "jsonData.exemplarTraceIdDestinations[1].name",
 						ValueFrom: v1beta1.ValueFromSource{
 							ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
 								LocalObjectReference: corev1.LocalObjectReference{
 									Name: cm.Name,
 								},
-								Key: "CUSTOM_TRACEID",
+								Key: "customTraceId",
 							},
 						},
 					},
 				},
 				Datasource: &v1beta1.GrafanaDatasourceInternal{
-					URL:      "${CUSTOM_URL}",
-					JSONData: json.RawMessage([]byte(`{"list":[{"value":"${CUSTOM_TRACEID}"}]}`)),
+					Name:   "substitute-prometheus",
+					Type:   "prometheus",
+					Access: "proxy",
+					URL:    "${URL}",
+					JSONData: json.RawMessage([]byte(`{
+						"tlsSkipVerify": true,
+						"timeInterval": "10s",
+						"httpHeaderName1": "Authorization",
+						"exemplarTraceIdDestinations": [
+							{"name": "traceID"},
+							{"name": "${customTraceId}"}
+						]
+					}`)),
+					SecureJSONData: json.RawMessage([]byte(`{
+						"httpHeaderValue1": "Bearer ${PROMETHEUS_TOKEN}"
+					}`)),
 				},
 			},
 		}
+		Expect(k8sClient.Create(testCtx, cm)).Should(Succeed())
+		Expect(k8sClient.Create(testCtx, sc)).Should(Succeed())
+		Expect(k8sClient.Create(testCtx, ds)).Should(Succeed())
 
-		r := GrafanaDatasourceReconciler{Client: k8sClient}
-		content, hash, err := r.buildDatasourceModel(testCtx, cr)
+		req := requestFromMeta(ds.ObjectMeta)
+		r := GrafanaDatasourceReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()}
+		_, err := r.Reconcile(testCtx, req)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(hash).ToNot(BeEmpty())
-		Expect(content.URL).To(Equal(cm.Data["CUSTOM_URL"]))
-		marshaled, err := json.Marshal(content.JSONData)
+
+		Expect(r.Get(testCtx, req.NamespacedName, ds)).Should(Succeed())
+		Expect(ds.Status.Conditions).Should(ContainElement(HaveField("Type", conditionDatasourceSynchronized)))
+		Expect(ds.Status.Conditions).Should(ContainElement(HaveField("Reason", conditionReasonApplySuccessful)))
+
+		cl, err := grafanaclient.NewGeneratedGrafanaClient(testCtx, k8sClient, externalGrafanaCr)
+		Expect(err).ToNot(HaveOccurred())
+
+		model, err := cl.Datasources.GetDataSourceByUID(ds.Spec.CustomUID)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(model.Payload.URL).To(Equal("https://demo.promlabs.com"))
+		Expect(model.Payload.SecureJSONFields["httpHeaderValue1"]).To(BeTrue())
+
+		// Serialize and Derserialize jsonData
+		b, err := json.Marshal(model.Payload.JSONData)
+		Expect(err).ToNot(HaveOccurred())
+
+		type ExemplarTraceIDDestination struct {
+			Name string `json:"name"`
+		}
+		type SubstitutedJSONData struct {
+			ExemplarTraceIDDestinations []ExemplarTraceIDDestination `json:"exemplarTraceIdDestinations"`
+		}
+		var jsonData SubstitutedJSONData // map with array of
+		err = json.Unmarshal(b, &jsonData)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(marshaled).To(ContainSubstring(cm.Data["CUSTOM_TRACEID"]))
+		Expect(jsonData.ExemplarTraceIDDestinations[1].Name).To(Equal("substituted"))
 	})
 })