Merge pull request #34130 from def-/pr-orchestratord-upgrade2

orchestratord test: Fix upgrade tests and reenable authenticator kinds

Author: def-

test/orchestratord/mzcompose.py

@@ -65,6 +65,10 @@ def get_tag(tag: str | None = None) -> str:
     return tag or f"v{ci_util.get_mz_version()}--pr.g{git.rev_parse('HEAD')}"
 
 
+def get_version(tag: str | None = None) -> MzVersion:
+    return MzVersion.parse_mz(get_tag(tag))
+
+
 def get_image(image: str, tag: str | None) -> str:
     return f'{image.rsplit(":", 1)[0]}:{get_tag(tag)}'
 
@@ -136,7 +140,9 @@ class Modification:
     pick_by_default: bool = True
 
     def __init__(self, value: Any):
-        assert value in self.values(), f"Expected {value} to be in {self.values()}"
+        assert value in self.values(
+            get_version()
+        ), f"Expected {value} to be in {self.values(get_version())}"
         self.value = value
 
     def to_dict(self) -> dict[str, Any]:
@@ -167,16 +173,16 @@ def from_dict(cls, data: dict[str, Any]) -> "Modification":
         return subclass(data["value"])
 
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         raise NotImplementedError
 
     @classmethod
     def bad_values(cls) -> list[Any]:
         return cls.failed_reconciliation_values()
 
     @classmethod
-    def good_values(cls) -> list[Any]:
-        return [value for value in cls.values() if value not in cls.bad_values()]
+    def good_values(cls, version: MzVersion) -> list[Any]:
+        return [value for value in cls.values(version) if value not in cls.bad_values()]
 
     @classmethod
     def failed_reconciliation_values(cls) -> list[Any]:
@@ -199,7 +205,7 @@ def all_modifications() -> list[type[Modification]]:
 
 class LicenseKey(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return ["valid", "invalid", "del"]
 
     @classmethod
@@ -220,7 +226,9 @@ def modify(self, definition: dict[str, Any]) -> None:
         elif self.value == "del":
             del definition["secret"]["stringData"]["license_key"]
         else:
-            raise ValueError(f"Unknown value {self.value}, only know {self.values()}")
+            raise ValueError(
+                f"Unknown value {self.value}, only know {self.values(get_version())}"
+            )
 
     def validate(self, mods: dict[type[Modification], Any]) -> None:
         if MzVersion.parse_mz(mods[EnvironmentdImageRef]) < MzVersion.parse_mz(
@@ -251,7 +259,7 @@ def check() -> None:
 
 class LicenseKeyCheck(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         # TODO: Reenable False when fixed
         return [None, True]
 
@@ -281,7 +289,7 @@ def validate(self, mods: dict[type[Modification], Any]) -> None:
 
 class BalancerdEnabled(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -322,7 +330,7 @@ def check() -> None:
 
 class BalancerdNodeSelector(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [
             {},
             # TODO: Reenable when we create nodes with these labels
@@ -362,7 +370,7 @@ def check() -> None:
 
 class ConsoleEnabled(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -404,7 +412,7 @@ def check() -> None:
 
 class EnableRBAC(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -433,7 +441,7 @@ class EnvironmentdImageRef(Modification):
     pick_by_default = False
 
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [str(version) for version in get_all_self_managed_versions()] + [
             get_tag()
         ]
@@ -472,7 +480,7 @@ class NumMaterializeEnvironments(Modification):
     pick_by_default = False
 
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [1, 2]
 
     @classmethod
@@ -539,7 +547,7 @@ def check() -> None:
 
 class TelemetryEnabled(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -565,7 +573,7 @@ def validate(self, mods: dict[type[Modification], Any]) -> None:
 
 class TelemetrySegmentClientSide(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -591,7 +599,7 @@ def validate(self, mods: dict[type[Modification], Any]) -> None:
 
 class ObservabilityEnabled(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -607,7 +615,7 @@ def validate(self, mods: dict[type[Modification], Any]) -> None:
 
 class ObservabilityPodMetricsEnabled(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -635,7 +643,7 @@ def validate(self, mods: dict[type[Modification], Any]) -> None:
 
 class ObservabilityPrometheusScrapeAnnotationsEnabled(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -664,7 +672,7 @@ def validate(self, mods: dict[type[Modification], Any]) -> None:
 # TODO: Fix in upgrade tests
 # class BalancerdReplicas(Modification):
 #     @classmethod
-#     def values(cls) -> list[Any]:
+#     def values(cls, version: MzVersion) -> list[Any]:
 #         return [None, 1, 2]
 #
 #     @classmethod
@@ -692,7 +700,7 @@ def validate(self, mods: dict[type[Modification], Any]) -> None:
 
 class ConsoleReplicas(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [None, 1, 2]
 
     @classmethod
@@ -764,7 +772,7 @@ def validate_container_resources(
 
 class SwapEnabledGlobal(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -817,7 +825,7 @@ def check_pods() -> None:
 
 class StorageClass(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [True, False]
 
     @classmethod
@@ -870,7 +878,7 @@ def check_pods() -> None:
 
 class EnvironmentdResources(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [
             None,
             {
@@ -920,7 +928,7 @@ def check_pods() -> None:
 
 class BalancerdResources(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [
             None,
             {
@@ -975,7 +983,7 @@ def check_pods() -> None:
 
 class ConsoleResources(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         return [
             None,
             {
@@ -1025,11 +1033,14 @@ def check_pods() -> None:
 
 class AuthenticatorKind(Modification):
     @classmethod
-    def values(cls) -> list[Any]:
+    def values(cls, version: MzVersion) -> list[Any]:
         # Test None, Password (v0.147.7+), and Sasl (v0.147.16+)
-        return ["None"]
-        # TODO: Reenable when database-issues#9899 is fixed
-        # return ["None", "Password", "Sasl"]
+        result = ["None"]
+        if version >= MzVersion.parse_mz("v0.147.7"):
+            result.append("Password")
+        if version >= MzVersion.parse_mz("v0.147.16"):
+            result.append("Sasl")
+        return result
 
     @classmethod
     def default(cls) -> Any:
@@ -1526,6 +1537,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
         definition["secret"] = materialize_setup[1]
         definition["materialize"] = materialize_setup[2]
 
+    current_version = get_version(args.tag)
     if args.orchestratord_override:
         definition["operator"]["operator"]["image"]["tag"] = get_tag(args.tag)
     # TODO: database-issues#9696, makes environmentd -> clusterd connections fail
@@ -1584,13 +1596,14 @@ def get_mods() -> Iterator[list[Modification]]:
             yield [NumMaterializeEnvironments(2)]
         elif properties == Properties.Individual:
             for mod_class in mod_classes:
-                for value in mod_class.values():
-                    yield [mod_class(value)]
+                for value in mod_class.values(current_version):
+                    if value in mod_class.values(current_version):
+                        yield [mod_class(value)]
         elif properties == Properties.Combine:
             assert args.runtime
             while time.time() < end_time:
                 yield [
-                    mod_class(rng.choice(mod_class.good_values()))
+                    mod_class(rng.choice(mod_class.good_values(current_version)))
                     for mod_class in mod_classes
                 ]
         else:
@@ -1615,6 +1628,7 @@ def get_mods() -> Iterator[list[Modification]]:
             versions = get_all_self_managed_versions()
             while time.time() < end_time:
                 selected_versions = sorted(list(rng.sample(versions, 2)))
+                current_version = selected_versions[0]
                 try:
                     mod = next(mods_it)
                 except StopIteration:
@@ -1637,6 +1651,7 @@ def get_mods() -> Iterator[list[Modification]]:
             while time.time() < end_time:
                 random.randint(2, len(versions))
                 selected_versions = sorted(list(rng.sample(versions, 2)))
+                current_version = selected_versions[0]
                 try:
                     mod = next(mods_it)
                 except StopIteration: