From 12c3e92891d78cbe910f00ee23dd6c745028184d Mon Sep 17 00:00:00 2001 From: mms-build-account Date: Fri, 1 Aug 2025 20:02:12 -0400 Subject: [PATCH 01/20] Updated --- release.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.json b/release.json index bd31a2d45..26a0964d4 100644 --- a/release.json +++ b/release.json @@ -105,7 +105,7 @@ ], "opsManagerMapping": { "Description": "These are the agents from which we start supporting static containers.", - "cloud_manager": "13.37.0.9590-1", + "cloud_manager": "13.38.0.9654-1", "cloud_manager_tools": "100.12.2", "ops_manager": { "6.0.25": { From 6dd26e5dafa2df4a8a38542431472e3241ad3a45 Mon Sep 17 00:00:00 2001 From: Evergreen Date: Sat, 2 Aug 2025 00:11:14 +0000 Subject: [PATCH 02/20] Run pre-commit hook --- config/manager/manager.yaml | 16 ++++++++-------- helm_chart/values-openshift.yaml | 8 ++++---- public/mongodb-kubernetes-openshift.yaml | 16 ++++++++-------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c2164c4f1..d9200cf0e 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -250,14 +250,14 @@ spec: value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.1.0" - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1_1_2_0 value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.2.0" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1_1_0_1 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1_1.0.1" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1_1_1_0 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1_1.1.0" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1_1_2_0 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1_1.2.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1_1_0_1 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1_1.0.1" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1_1_1_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1_1.1.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1_1_2_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1_1.2.0" - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_25 value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.25" - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_26 diff --git a/helm_chart/values-openshift.yaml b/helm_chart/values-openshift.yaml index 1177428c2..8da8baf59 100644 --- a/helm_chart/values-openshift.yaml +++ b/helm_chart/values-openshift.yaml @@ -158,10 +158,10 @@ relatedImages: - 12.0.35.7911-1_1.0.1 - 12.0.35.7911-1_1.1.0 - 12.0.35.7911-1_1.2.0 - - 13.37.0.9590-1 - - 13.37.0.9590-1_1.0.1 - - 13.37.0.9590-1_1.1.0 - - 13.37.0.9590-1_1.2.0 + - 13.38.0.9654-1 + - 13.38.0.9654-1_1.0.1 + - 13.38.0.9654-1_1.1.0 + - 13.38.0.9654-1_1.2.0 mongodbLegacyAppDb: - 4.2.11-ent - 4.2.2-ent diff --git a/public/mongodb-kubernetes-openshift.yaml b/public/mongodb-kubernetes-openshift.yaml index 52bec8aca..55a2ed65c 100644 --- a/public/mongodb-kubernetes-openshift.yaml +++ b/public/mongodb-kubernetes-openshift.yaml @@ -554,14 +554,14 @@ spec: value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.1.0" - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1_1_2_0 value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.2.0" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1_1_0_1 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1_1.0.1" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1_1_1_0 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1_1.1.0" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1_1_2_0 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1_1.2.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1_1_0_1 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1_1.0.1" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1_1_1_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1_1.1.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1_1_2_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1_1.2.0" - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_25 value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.25" - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_26 From d95b296b12fedc6d4c8baadb7fb06a6c1e1f0b21 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 10:58:19 +0200 Subject: [PATCH 03/20] removing pipeline and atomic matrix --- .evergreen.yml | 83 ++----- pipeline.py | 180 ++------------- scripts/detect_ops_manager_changes.py | 109 +++++++++ scripts/release/atomic_pipeline.py | 78 +------ scripts/test_detect_ops_manager_changes.py | 252 +++++++++++++++++++++ 5 files changed, 407 insertions(+), 295 deletions(-) create mode 100644 scripts/detect_ops_manager_changes.py create mode 100644 scripts/test_detect_ops_manager_changes.py diff --git a/.evergreen.yml b/.evergreen.yml index 4ce0a3fff..709d7c392 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -33,8 +33,7 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + - &base_no_om_image_dependency depends_on: @@ -52,8 +51,7 @@ variables: variant: init_test_run - name: build_init_appdb_images_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + - &community_dependency depends_on: @@ -61,8 +59,7 @@ variables: variant: init_test_run - name: build_test_image variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + - name: build_readiness_probe_image variant: init_test_run - name: build_upgrade_hook_image @@ -153,8 +150,7 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + - &base_om7_dependency_with_race depends_on: @@ -172,8 +168,7 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + - &base_om8_dependency depends_on: @@ -191,8 +186,7 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + parameters: - key: evergreen_retry @@ -347,19 +341,6 @@ tasks: image_name: init-ops-manager include_tags: release - - name: release_agent_operator_release - tags: [ "image_release" ] - allowed_requesters: [ "patch", "github_tag" ] - commands: - - func: clone - - func: setup_building_host - - func: quay_login - - func: setup_docker_sbom - - func: legacy_pipeline - vars: - image_name: agent - include_tags: release - # pct only triggers this variant once a new agent image is out - name: release_agent # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) @@ -392,21 +373,6 @@ tasks: working_dir: src/github.com/mongodb/mongodb-kubernetes binary: scripts/evergreen/precommit_bump.sh - # Pct only triggers this variant once a new agent image is out - # these releases the agent with the operator suffix (not patch id) on ecr to allow for digest pinning to pass. - # For this to work, we rely on skip_tags which is used to determine whether - # we want to release on quay or not, in this case - ecr instead. - # We rely on the init_database from ecr for the agent x operator images. - # This runs on agent releases that are not concurrent with operator releases. - - name: release_agents_on_ecr_conditional - commands: - - func: clone - - func: run_task_conditionally - vars: - condition_script: scripts/evergreen/should_release_agents_on_ecr.sh - variant: init_release_agents_on_ecr - task: release_agents_on_ecr - - name: release_agents_on_ecr # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) allowed_requesters: [ "patch", "github_pr" ] @@ -1334,8 +1300,7 @@ buildvariants: variant: init_test_run - name: build_init_database_image_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + tasks: - name: e2e_custom_domain_task_group @@ -1369,8 +1334,7 @@ buildvariants: variant: init_test_run - name: build_init_database_image_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + run_on: - ubuntu2204-small tasks: @@ -1617,8 +1581,7 @@ buildvariants: variant: init_tests_with_olm - name: build_init_database_image_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + tasks: - name: e2e_kind_olm_group @@ -1683,18 +1646,6 @@ buildvariants: - name: build_upgrade_hook_image - name: prepare_aws - - name: init_release_agents_on_ecr - display_name: init_release_agents_on_ecr - # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) - allowed_requesters: [ "patch", "github_pr" ] - tags: [ "release_agents_on_ecr" ] - # We want that to run first and finish asap. Digest pinning depends on this to succeed. - priority: 70 - run_on: - - ubuntu2204-large - tasks: - - name: release_agents_on_ecr_conditional - - name: run_pre_commit priority: 70 display_name: run_pre_commit @@ -1722,8 +1673,7 @@ buildvariants: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run - - name: build_agent_images_ubi - variant: init_test_run + run_on: - ubuntu2204-small tasks: @@ -1809,13 +1759,6 @@ buildvariants: - name: release_init_database - name: release_init_ops_manager - name: release_database - # Once we release the operator, we will also release the init databases, we require them to be out first - # such that we can reference them and retrieve those binaries. - # Since we immediately run daily rebuild after creating the image, we can ensure that the init_database is out - # such that the agent image build can use it. - - name: release_agent_operator_release - depends_on: - - name: release_init_database - name: preflight_release_images display_name: preflight_release_images @@ -1847,13 +1790,13 @@ buildvariants: # It will be called by pct while bumping the agent cloud manager image - name: release_agent - display_name: (Static Containers) Release Agent matrix + display_name: release_agent tags: [ "release_agent" ] run_on: - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 depends_on: - - variant: init_release_agents_on_ecr - name: '*' + - variant: init_test_run + name: build_agent_images_ubi # this ensures the agent gets released to ECR as well - variant: e2e_multi_cluster_kind name: '*' - variant: e2e_static_multi_cluster_2_clusters diff --git a/pipeline.py b/pipeline.py index 6ebb645bb..b0fe3c5ff 100755 --- a/pipeline.py +++ b/pipeline.py @@ -42,6 +42,7 @@ import docker from lib.base_logger import logger from lib.sonar.sonar import process_image +from scripts.detect_ops_manager_changes import detect_ops_manager_changes from scripts.evergreen.release.agent_matrix import ( get_supported_version_for_image, ) @@ -1245,18 +1246,11 @@ def build_agent_default_case(build_configuration: BuildConfiguration): """ Build the agent only for the latest operator for patches and operator releases. - See more information in the function: build_agent_on_agent_bump """ - release_json = get_release() - - is_release = build_configuration.is_release_step_executed() - - # We need to release [all agents x latest operator] on operator releases - if is_release: - agent_versions_to_build = gather_all_supported_agent_versions(release_json) - # We only need [latest agents (for each OM major version and for CM) x patch ID] for patches - else: - agent_versions_to_build = gather_latest_agent_versions(release_json, build_configuration.agent_to_build) + agent_versions_to_build = detect_ops_manager_changes() + if not agent_versions_to_build: + logger.info("No changes detected, skipping agent build") + return logger.info(f"Building Agent versions: {agent_versions_to_build}") @@ -1267,83 +1261,17 @@ def build_agent_default_case(build_configuration: BuildConfiguration): if build_configuration.parallel_factor > 0: max_workers = build_configuration.parallel_factor with ProcessPoolExecutor(max_workers=max_workers) as executor: - logger.info(f"running with factor of {max_workers}") - for agent_version in agent_versions_to_build: - # We don't need to keep create and push the same image on every build. - # It is enough to create and push the non-operator suffixed images only during releases to ecr and quay. - if build_configuration.is_release_step_executed() or build_configuration.all_agents: - tasks_queue.put( - executor.submit( - build_multi_arch_agent_in_sonar, - build_configuration, - agent_version[0], - agent_version[1], - ) - ) - _add_to_agent_queue(agent_version, build_configuration, executor, tasks_queue) - - queue_exception_handling(tasks_queue) - - -def build_agent_on_agent_bump(build_configuration: BuildConfiguration): - """ - Build the agent matrix (operator version x agent version), triggered by PCT. - - We have three cases where we need to build the agent: - - e2e test runs - - operator releases - - OM/CM bumps via PCT - - In OM/CM bumps, we release a new agent. - """ - release_json = get_release() - is_release = build_configuration.is_release_step_executed() - - if build_configuration.all_agents: - agent_versions_to_build = gather_all_supported_agent_versions(release_json) - else: - # we only need to release the latest images, we don't need to re-push old images, as we don't clean them up anymore. - agent_versions_to_build = gather_latest_agent_versions(release_json, build_configuration.agent_to_build) - - legacy_agent_versions_to_build = release_json["supportedImages"]["mongodb-agent"]["versions"] - - tasks_queue = Queue() - max_workers = 1 - if build_configuration.parallel: - max_workers = None - if build_configuration.parallel_factor > 0: - max_workers = build_configuration.parallel_factor - with ProcessPoolExecutor(max_workers=max_workers) as executor: - logger.info(f"running with factor of {max_workers}") - - # We need to regularly push legacy agents, otherwise ecr lifecycle policy will expire them. - # We only need to push them once in a while to ecr, so no quay required - if not is_release: - for legacy_agent in legacy_agent_versions_to_build: - tasks_queue.put( - executor.submit( - build_multi_arch_agent_in_sonar, - build_configuration, - legacy_agent, - # we assume that all legacy agents are build using that tools version - "100.9.4", - ) - ) - - for agent_version in agent_versions_to_build: + logger.info(f"Running with factor of {max_workers}") + for idx, agent_tools_version in enumerate(agent_versions_to_build): # We don't need to keep create and push the same image on every build. # It is enough to create and push the non-operator suffixed images only during releases to ecr and quay. - if build_configuration.is_release_step_executed() or build_configuration.all_agents: - tasks_queue.put( - executor.submit( - build_multi_arch_agent_in_sonar, - build_configuration, - agent_version[0], - agent_version[1], - ) - ) - logger.info(f"Building Agent versions: {agent_version}") - _add_to_agent_queue(agent_version, build_configuration, executor, tasks_queue) + logger.info(f"======= Building Agent {agent_tools_version} ({idx}/{len(agent_versions_to_build)})") + _build_agents( + agent_tools_version, + build_configuration, + executor, + tasks_queue, + ) queue_exception_handling(tasks_queue) @@ -1384,87 +1312,29 @@ def queue_exception_handling(tasks_queue): ) -def _add_to_agent_queue( - agent_version: Tuple[str, str], - build_configuration: BuildConfiguration, +def _build_agents( + agent_tools_version: Tuple[str, str], + build_configuration: ImageBuildConfiguration, executor: ProcessPoolExecutor, tasks_queue: Queue, ): - tools_version = agent_version[1] - image_version = f"{agent_version[0]}" + agent_version = agent_tools_version[0] + agent_distro = "rhel9_x86_64" + tools_version = agent_tools_version[1] + tools_distro = get_tools_distro(tools_version)["amd"] tasks_queue.put( executor.submit( build_multi_arch_agent_in_sonar, build_configuration, - image_version, + agent_version, + agent_distro, tools_version, + tools_distro, ) ) -def gather_all_supported_agent_versions(release: Dict) -> List[Tuple[str, str]]: - # This is a list of a tuples - agent version and corresponding tools version - agent_versions_to_build = list() - agent_versions_to_build.append( - ( - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"], - ) - ) - for _, om in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].items(): - agent_versions_to_build.append((om["agent_version"], om["tools_version"])) - - # lets not build the same image multiple times - return sorted(list(set(agent_versions_to_build))) - - -def gather_latest_agent_versions(release: Dict, agent_to_build: str = "") -> List[Tuple[str, str]]: - """ - This function is used when we release a new agent via OM bump. - That means we will need to release that agent with all supported operators. - Since we don't want to release all agents again, we only release the latest, which will contain the newly added one - :return: the latest agent for each major version - """ - agent_versions_to_build = list() - agent_versions_to_build.append( - ( - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"], - ) - ) - - latest_versions = {} - - for version in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].keys(): - parsed_version = semver.VersionInfo.parse(version) - major_version = parsed_version.major - if major_version in latest_versions: - latest_parsed_version = semver.VersionInfo.parse(str(latest_versions[major_version])) - latest_versions[major_version] = max(parsed_version, latest_parsed_version) - else: - latest_versions[major_version] = version - - for major_version, latest_version in latest_versions.items(): - agent_versions_to_build.append( - ( - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][str(latest_version)][ - "agent_version" - ], - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][str(latest_version)][ - "tools_version" - ], - ) - ) - - if agent_to_build != "": - for agent_tuple in agent_versions_to_build: - if agent_tuple[0] == agent_to_build: - return [agent_tuple] - - return sorted(list(set(agent_versions_to_build))) - - def get_builder_function_for_image_name() -> Dict[str, Callable]: """Returns a dictionary of image names that can be built.""" @@ -1478,7 +1348,7 @@ def get_builder_function_for_image_name() -> Dict[str, Callable]: "upgrade-hook": build_upgrade_hook_image, "operator-quick": build_operator_image_patch, "database": build_database_image, - "agent-pct": build_agent_on_agent_bump, + "agent-pct": build_agent_default_case, "agent": build_agent_default_case, # # Init images diff --git a/scripts/detect_ops_manager_changes.py b/scripts/detect_ops_manager_changes.py new file mode 100644 index 000000000..842955a92 --- /dev/null +++ b/scripts/detect_ops_manager_changes.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +Detects changes to opsManagerMapping in release.json for triggering agent releases. +Relies on git origin/master vs local release.json +""" +import json +import os +import subprocess +import sys +from typing import Dict, List, Optional, Tuple + + +def get_content_from_git(commit: str, file_path: str) -> Optional[str]: + try: + result = subprocess.run( + ["git", "show", f"{commit}:{file_path}"], capture_output=True, text=True, check=True, timeout=30 + ) + return result.stdout + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + return None + + +def load_base_release_json() -> Optional[Dict]: + base_revision = "origin/master" + + content = get_content_from_git(base_revision, "release.json") + if not content: + print(f"Could not retrieve release.json from {base_revision}") + return None + + try: + return json.loads(content) + except json.JSONDecodeError as e: + print(f"Invalid JSON in base release.json: {e}") + return None + + +def load_current_release_json() -> Optional[Dict]: + try: + with open("release.json", "r") as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError) as e: + print(f"Could not load current release.json: {e}") + return None + + +def extract_ops_manager_mapping(release_data: Dict) -> Dict: + if not release_data: + return {} + return release_data.get("supportedImages", {}).get("mongodb-agent", {}).get("opsManagerMapping", {}) + + +def get_changed_agents(current_mapping: Dict, base_mapping: Dict) -> List[Tuple[str, str]]: + """Returns list of (agent_version, tools_version) tuples for added/changed agents""" + added_agents = [] + + current_om_mapping = current_mapping.get("ops_manager", {}) + master_om_mapping = base_mapping.get("ops_manager", {}) + + for om_version, agent_tools_version in current_om_mapping.items(): + if om_version not in master_om_mapping or master_om_mapping[om_version] != agent_tools_version: + added_agents.append((agent_tools_version["agent_version"], agent_tools_version["tools_version"])) + + current_cm = current_mapping.get("cloud_manager") + master_cm = base_mapping.get("cloud_manager") + current_cm_tools = current_mapping.get("cloud_manager_tools") + master_cm_tools = base_mapping.get("cloud_manager_tools") + + if current_cm != master_cm or current_cm_tools != master_cm_tools: + added_agents.append((current_cm, current_cm_tools)) + + return list(set(added_agents)) + + +def detect_ops_manager_changes() -> List[Tuple[str, str]]: + """Returns (has_changes, changed_agents_list)""" + print("=== Detecting OM Mapping Changes (Local vs Base) ===") + + current_release = load_current_release_json() + if not current_release: + print("ERROR: Could not load current local release.json") + return [] + + base_release = load_base_release_json() + if not base_release: + print("WARNING: Could not load base release.json, assuming changes exist") + return [] + + current_mapping = extract_ops_manager_mapping(current_release) + base_mapping = extract_ops_manager_mapping(base_release) + + if current_mapping != base_mapping: + return get_changed_agents(current_mapping, base_mapping) + else: + return [] + + +if __name__ == "__main__": + if not os.path.exists("release.json"): + print("ERROR: release.json not found. Are you in the project root?") + sys.exit(2) + + agents = detect_ops_manager_changes() + if agents: + print(f"Changed agents: {agents}") + sys.exit(0) + else: + print("Skipping agent release") + sys.exit(1) diff --git a/scripts/release/atomic_pipeline.py b/scripts/release/atomic_pipeline.py index b24143dad..2cadf7de6 100755 --- a/scripts/release/atomic_pipeline.py +++ b/scripts/release/atomic_pipeline.py @@ -16,6 +16,7 @@ from packaging.version import Version from lib.base_logger import logger +from scripts.detect_ops_manager_changes import detect_ops_manager_changes from scripts.release.build.image_build_configuration import ImageBuildConfiguration from scripts.release.build.image_build_process import execute_docker_build from scripts.release.build.image_signing import ( @@ -278,18 +279,12 @@ def build_agent_default_case(build_configuration: ImageBuildConfiguration): Build the agent only for the latest operator for patches and operator releases. """ - release = load_release_file() + agent_versions_to_build = detect_ops_manager_changes() + if not agent_versions_to_build: + logger.info("No changes detected, skipping agent build") + return - # We need to release [all agents x latest operator] on operator releases - if build_configuration.is_release_scenario(): - agent_versions_to_build = gather_all_supported_agent_versions(release) - # We only need [latest agents (for each OM major version and for CM) x patch ID] for patches - else: - agent_versions_to_build = gather_latest_agent_versions(release) - - logger.info( - f"Building Agent versions: {agent_versions_to_build} for Operator versions: {build_configuration.version}" - ) + logger.info(f"Building Agent versions: {agent_versions_to_build}") tasks_queue = Queue() max_workers = 1 @@ -304,7 +299,7 @@ def build_agent_default_case(build_configuration: ImageBuildConfiguration): # We don't need to keep create and push the same image on every build. # It is enough to create and push the non-operator suffixed images only during releases to ecr and quay. logger.info(f"======= Building Agent {agent_tools_version} ({idx}/{len(agent_versions_to_build)})") - _build_agent_operator( + _build_agents( agent_tools_version, build_configuration, executor, @@ -314,64 +309,7 @@ def build_agent_default_case(build_configuration: ImageBuildConfiguration): queue_exception_handling(tasks_queue) -def gather_all_supported_agent_versions(release: Dict) -> List[Tuple[str, str]]: - # This is a list of a tuples - agent version and corresponding tools version - agent_versions_to_build = list() - agent_versions_to_build.append( - ( - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"], - ) - ) - for _, om in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].items(): - agent_versions_to_build.append((om["agent_version"], om["tools_version"])) - - # lets not build the same image multiple times - return sorted(list(set(agent_versions_to_build))) - - -def gather_latest_agent_versions(release: Dict) -> List[Tuple[str, str]]: - """ - This function is used when we release a new agent via OM bump. - That means we will need to release that agent with all supported operators. - Since we don’t want to release all agents again, we only release the latest, which will contain the newly added one - :return: the latest agent for each major version - """ - agent_versions_to_build = list() - agent_versions_to_build.append( - ( - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"], - ) - ) - - latest_versions = {} - - for version in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].keys(): - parsed_version = semver.VersionInfo.parse(version) - major_version = parsed_version.major - if major_version in latest_versions: - latest_parsed_version = semver.VersionInfo.parse(str(latest_versions[major_version])) - latest_versions[major_version] = max(parsed_version, latest_parsed_version) - else: - latest_versions[major_version] = version - - for major_version, latest_version in latest_versions.items(): - agent_versions_to_build.append( - ( - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][str(latest_version)][ - "agent_version" - ], - release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][str(latest_version)][ - "tools_version" - ], - ) - ) - - return sorted(list(set(agent_versions_to_build))) - - -def _build_agent_operator( +def _build_agents( agent_tools_version: Tuple[str, str], build_configuration: ImageBuildConfiguration, executor: ProcessPoolExecutor, diff --git a/scripts/test_detect_ops_manager_changes.py b/scripts/test_detect_ops_manager_changes.py new file mode 100644 index 000000000..67b0ddc41 --- /dev/null +++ b/scripts/test_detect_ops_manager_changes.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +""" +Tests for detect_ops_manager_changes.py +""" +import json +import os +import subprocess +import sys +import unittest +from unittest.mock import MagicMock, mock_open, patch + +# Add the scripts directory to the path so we can import the module +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from detect_ops_manager_changes import ( + detect_ops_manager_changes, + extract_ops_manager_mapping, + get_content_from_git, + load_current_release_json, +) + + +class TestDetectOpsManagerChanges(unittest.TestCase): + + def setUp(self): + """Set up test fixtures""" + self.master_release_data = { + "supportedImages": { + "mongodb-agent": { + "opsManagerMapping": { + "cloud_manager": "13.37.0.9590-1", + "cloud_manager_tools": "100.12.2", + "ops_manager": { + "6.0.26": {"agent_version": "12.0.34.7888-1", "tools_version": "100.10.0"}, + "7.0.11": {"agent_version": "107.0.11.8645-1", "tools_version": "100.10.0"}, + }, + } + } + } + } + + self.current_release_data = { + "supportedImages": { + "mongodb-agent": { + "opsManagerMapping": { + "cloud_manager": "13.37.0.9590-1", + "cloud_manager_tools": "100.12.2", + "ops_manager": { + "6.0.26": {"agent_version": "12.0.34.7888-1", "tools_version": "100.10.0"}, + "7.0.11": {"agent_version": "107.0.11.8645-1", "tools_version": "100.10.0"}, + }, + } + } + } + } + + def test_extract_ops_manager_mapping_valid(self): + """Test extracting opsManagerMapping from valid release data""" + mapping = extract_ops_manager_mapping(self.master_release_data) + expected = { + "cloud_manager": "13.37.0.9590-1", + "cloud_manager_tools": "100.12.2", + "ops_manager": { + "6.0.26": {"agent_version": "12.0.34.7888-1", "tools_version": "100.10.0"}, + "7.0.11": {"agent_version": "107.0.11.8645-1", "tools_version": "100.10.0"}, + }, + } + self.assertEqual(mapping, expected) + + def test_extract_ops_manager_mapping_empty(self): + """Test extracting from empty/invalid data""" + self.assertEqual(extract_ops_manager_mapping({}), {}) + self.assertEqual(extract_ops_manager_mapping(None), {}) + + def test_extract_ops_manager_mapping_missing_keys(self): + """Test extracting when keys are missing""" + incomplete_data = {"supportedImages": {}} + self.assertEqual(extract_ops_manager_mapping(incomplete_data), {}) + + @patch("builtins.open", new_callable=mock_open) + @patch("os.path.exists", return_value=True) + def test_load_current_release_json_success(self, mock_exists, mock_file): + """Test successfully loading current release.json""" + mock_file.return_value.read.return_value = json.dumps(self.current_release_data) + + result = load_current_release_json() + self.assertEqual(result, self.current_release_data) + + @patch("builtins.open", side_effect=FileNotFoundError) + def test_load_current_release_json_not_found(self, mock_file): + """Test handling missing release.json""" + result = load_current_release_json() + self.assertIsNone(result) + + @patch("builtins.open", new_callable=mock_open) + @patch("os.path.exists", return_value=True) + def test_load_current_release_json_invalid_json(self, mock_exists, mock_file): + """Test handling invalid JSON in release.json""" + mock_file.return_value.read.return_value = "invalid json" + + result = load_current_release_json() + self.assertIsNone(result) + + @patch("subprocess.run") + def test_safe_git_show_success(self, mock_run): + """Test successful git show operation""" + mock_result = MagicMock() + mock_result.stdout = json.dumps(self.master_release_data) + mock_run.return_value = mock_result + + result = get_content_from_git("abc123", "release.json") + self.assertEqual(result, json.dumps(self.master_release_data)) + + mock_run.assert_called_once_with( + ["git", "show", "abc123:release.json"], capture_output=True, text=True, check=True, timeout=30 + ) + + @patch("subprocess.run", side_effect=subprocess.CalledProcessError(1, "git")) + def test_safe_git_show_failure(self, mock_run): + """Test git show failure handling""" + result = get_content_from_git("abc123", "release.json") + self.assertIsNone(result) + + def test_no_changes_detected(self): + """Test when no changes are detected""" + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=self.current_release_data), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertFalse(has_changes) + self.assertEqual(changed_agents, []) + + def test_new_ops_manager_version_added(self): + """Test detection when new OM version is added""" + modified_current = json.loads(json.dumps(self.current_release_data)) + modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"]["8.0.0"] = { + "agent_version": "108.0.0.8694-1", + "tools_version": "100.10.0", + } + + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertTrue(has_changes) + self.assertIn(("108.0.0.8694-1", "100.10.0"), changed_agents) + + def test_ops_manager_version_modified(self): + """Test detection when OM version is modified""" + modified_current = json.loads(json.dumps(self.current_release_data)) + modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"]["6.0.26"][ + "agent_version" + ] = "12.0.35.7911-1" + + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertTrue(has_changes) + self.assertIn(("12.0.35.7911-1", "100.10.0"), changed_agents) + + def test_cloud_manager_changed(self): + """Test detection when cloud_manager is changed""" + modified_current = json.loads(json.dumps(self.current_release_data)) + modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"] = "13.38.0.9600-1" + + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertTrue(has_changes) + self.assertIn(("13.38.0.9600-1", "100.12.2"), changed_agents) + + def test_cloud_manager_tools_changed(self): + """Test detection when cloud_manager_tools is changed""" + modified_current = json.loads(json.dumps(self.current_release_data)) + modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"] = "100.13.0" + + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertTrue(has_changes) + self.assertIn(("13.37.0.9590-1", "100.13.0"), changed_agents) + + def test_ops_manager_version_removed(self): + """Test detection when OM version is removed""" + modified_current = json.loads(json.dumps(self.current_release_data)) + del modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"]["7.0.11"] + + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertTrue(has_changes) + + def test_both_om_and_cm_changed(self): + """Test detection when both OM version and cloud manager are changed""" + modified_current = json.loads(json.dumps(self.current_release_data)) + modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"]["8.0.0"] = { + "agent_version": "108.0.0.8694-1", + "tools_version": "100.10.0", + } + modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"] = "13.38.0.9600-1" + + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertTrue(has_changes) + self.assertIn(("108.0.0.8694-1", "100.10.0"), changed_agents) + self.assertIn(("13.38.0.9600-1", "100.12.2"), changed_agents) + self.assertEqual(len(changed_agents), 2) + + def test_current_release_load_failure(self): + """Test handling when current release.json cannot be loaded""" + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=None), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertFalse(has_changes) + self.assertEqual(changed_agents, []) + + def test_base_release_load_failure_fail_safe(self): + """Test fail-safe behavior when base release.json cannot be loaded""" + with ( + patch("detect_ops_manager_changes.load_current_release_json", return_value=self.current_release_data), + patch("detect_ops_manager_changes.load_release_json_from_master", return_value=None), + ): + + has_changes, changed_agents = detect_ops_manager_changes() + self.assertTrue(has_changes) + self.assertEqual(changed_agents, []) + + +if __name__ == "__main__": + unittest.main() From 3357e31a8814c2b8bcabda41c1eaf00eba9e5de9 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 11:36:14 +0200 Subject: [PATCH 04/20] removing pipeline and atomic matrix --- .evergreen.yml | 4 ---- pipeline.py | 10 +++----- scripts/detect_ops_manager_changes.py | 8 +++---- scripts/test_detect_ops_manager_changes.py | 28 ++++++++-------------- 4 files changed, 17 insertions(+), 33 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 709d7c392..66f9aa5dc 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -34,7 +34,6 @@ variables: - name: build_init_om_images_ubi variant: init_test_run - - &base_no_om_image_dependency depends_on: - name: build_om_images @@ -52,7 +51,6 @@ variables: - name: build_init_appdb_images_ubi variant: init_test_run - - &community_dependency depends_on: - name: build_operator_ubi @@ -151,7 +149,6 @@ variables: - name: build_init_om_images_ubi variant: init_test_run - - &base_om7_dependency_with_race depends_on: - name: build_om_images @@ -169,7 +166,6 @@ variables: - name: build_init_om_images_ubi variant: init_test_run - - &base_om8_dependency depends_on: - name: build_om_images diff --git a/pipeline.py b/pipeline.py index b0fe3c5ff..8d78eb45e 100755 --- a/pipeline.py +++ b/pipeline.py @@ -1242,11 +1242,7 @@ def build_multi_arch_agent_in_sonar( ) -def build_agent_default_case(build_configuration: BuildConfiguration): - """ - Build the agent only for the latest operator for patches and operator releases. - - """ +def build_agent(build_configuration: BuildConfiguration): agent_versions_to_build = detect_ops_manager_changes() if not agent_versions_to_build: logger.info("No changes detected, skipping agent build") @@ -1348,8 +1344,8 @@ def get_builder_function_for_image_name() -> Dict[str, Callable]: "upgrade-hook": build_upgrade_hook_image, "operator-quick": build_operator_image_patch, "database": build_database_image, - "agent-pct": build_agent_default_case, - "agent": build_agent_default_case, + "agent-pct": build_agent, + "agent": build_agent, # # Init images "init-appdb": build_init_appdb, diff --git a/scripts/detect_ops_manager_changes.py b/scripts/detect_ops_manager_changes.py index 842955a92..a6079582f 100644 --- a/scripts/detect_ops_manager_changes.py +++ b/scripts/detect_ops_manager_changes.py @@ -20,7 +20,7 @@ def get_content_from_git(commit: str, file_path: str) -> Optional[str]: return None -def load_base_release_json() -> Optional[Dict]: +def load_release_json_from_master() -> Optional[Dict]: base_revision = "origin/master" content = get_content_from_git(base_revision, "release.json") @@ -81,13 +81,13 @@ def detect_ops_manager_changes() -> List[Tuple[str, str]]: print("ERROR: Could not load current local release.json") return [] - base_release = load_base_release_json() - if not base_release: + master_release = load_release_json_from_master() + if not master_release: print("WARNING: Could not load base release.json, assuming changes exist") return [] current_mapping = extract_ops_manager_mapping(current_release) - base_mapping = extract_ops_manager_mapping(base_release) + base_mapping = extract_ops_manager_mapping(master_release) if current_mapping != base_mapping: return get_changed_agents(current_mapping, base_mapping) diff --git a/scripts/test_detect_ops_manager_changes.py b/scripts/test_detect_ops_manager_changes.py index 67b0ddc41..4c5c1d212 100644 --- a/scripts/test_detect_ops_manager_changes.py +++ b/scripts/test_detect_ops_manager_changes.py @@ -127,8 +127,7 @@ def test_no_changes_detected(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertFalse(has_changes) + changed_agents = detect_ops_manager_changes() self.assertEqual(changed_agents, []) def test_new_ops_manager_version_added(self): @@ -144,8 +143,7 @@ def test_new_ops_manager_version_added(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertTrue(has_changes) + changed_agents = detect_ops_manager_changes() self.assertIn(("108.0.0.8694-1", "100.10.0"), changed_agents) def test_ops_manager_version_modified(self): @@ -160,8 +158,7 @@ def test_ops_manager_version_modified(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertTrue(has_changes) + changed_agents = detect_ops_manager_changes() self.assertIn(("12.0.35.7911-1", "100.10.0"), changed_agents) def test_cloud_manager_changed(self): @@ -174,8 +171,7 @@ def test_cloud_manager_changed(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertTrue(has_changes) + changed_agents = detect_ops_manager_changes() self.assertIn(("13.38.0.9600-1", "100.12.2"), changed_agents) def test_cloud_manager_tools_changed(self): @@ -188,8 +184,7 @@ def test_cloud_manager_tools_changed(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertTrue(has_changes) + changed_agents = detect_ops_manager_changes() self.assertIn(("13.37.0.9590-1", "100.13.0"), changed_agents) def test_ops_manager_version_removed(self): @@ -202,8 +197,8 @@ def test_ops_manager_version_removed(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertTrue(has_changes) + changed_agents = detect_ops_manager_changes() + self.assertEqual(changed_agents, []) def test_both_om_and_cm_changed(self): """Test detection when both OM version and cloud manager are changed""" @@ -219,8 +214,7 @@ def test_both_om_and_cm_changed(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertTrue(has_changes) + changed_agents = detect_ops_manager_changes() self.assertIn(("108.0.0.8694-1", "100.10.0"), changed_agents) self.assertIn(("13.38.0.9600-1", "100.12.2"), changed_agents) self.assertEqual(len(changed_agents), 2) @@ -232,8 +226,7 @@ def test_current_release_load_failure(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertFalse(has_changes) + changed_agents = detect_ops_manager_changes() self.assertEqual(changed_agents, []) def test_base_release_load_failure_fail_safe(self): @@ -243,8 +236,7 @@ def test_base_release_load_failure_fail_safe(self): patch("detect_ops_manager_changes.load_release_json_from_master", return_value=None), ): - has_changes, changed_agents = detect_ops_manager_changes() - self.assertTrue(has_changes) + changed_agents = detect_ops_manager_changes() self.assertEqual(changed_agents, []) From 717abe1c99b6e883d6f680bcdb8d1bfe3c7bc4f0 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 12:00:36 +0200 Subject: [PATCH 05/20] removing pipeline and atomic matrix --- config/manager/manager.yaml | 4 ++-- public/mongodb-kubernetes-openshift.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 77997de55..c93f3acb5 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -162,8 +162,8 @@ spec: value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1" - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1 value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1" - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_26 value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.26" - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_27 diff --git a/public/mongodb-kubernetes-openshift.yaml b/public/mongodb-kubernetes-openshift.yaml index 1d25900e2..fda4f6729 100644 --- a/public/mongodb-kubernetes-openshift.yaml +++ b/public/mongodb-kubernetes-openshift.yaml @@ -466,8 +466,8 @@ spec: value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1" - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1 value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1" - - name: RELATED_IMAGE_AGENT_IMAGE_13_37_0_9590_1 - value: "quay.io/mongodb/mongodb-agent-ubi:13.37.0.9590-1" + - name: RELATED_IMAGE_AGENT_IMAGE_13_38_0_9654_1 + value: "quay.io/mongodb/mongodb-agent-ubi:13.38.0.9654-1" - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_26 value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.26" - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_27 From a2bb281408093caae9751d5cb6d6b082a530b01a Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 12:03:09 +0200 Subject: [PATCH 06/20] fix bc --- pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.py b/pipeline.py index 8d78eb45e..46b3fa54d 100755 --- a/pipeline.py +++ b/pipeline.py @@ -1310,7 +1310,7 @@ def queue_exception_handling(tasks_queue): def _build_agents( agent_tools_version: Tuple[str, str], - build_configuration: ImageBuildConfiguration, + build_configuration: BuildConfiguration, executor: ProcessPoolExecutor, tasks_queue: Queue, ): From 832739cfa0e0aaddab40366041becd601aa43f7e Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 13:38:47 +0200 Subject: [PATCH 07/20] fix python tests --- pipeline_test.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pipeline_test.py b/pipeline_test.py index cbaf2bd06..61bbed197 100644 --- a/pipeline_test.py +++ b/pipeline_test.py @@ -7,8 +7,6 @@ from pipeline import ( calculate_images_to_build, - gather_all_supported_agent_versions, - gather_latest_agent_versions, get_versions_to_rebuild, get_versions_to_rebuild_per_operator_version, is_version_in_range, @@ -159,23 +157,6 @@ def test_is_release_step_executed(description, case): assert result == case["expected"], f"Test failed: {description}. Expected {case['expected']}, got {result}." -def test_build_latest_agent_versions(): - latest_agents = gather_latest_agent_versions(release_json) - expected_agents = [ - ("107.0.11.8645-1", "100.10.0"), - ("12.0.31.7825-1", "100.9.4"), - ("13.19.0.8937-1", "100.9.4"), - ] - assert latest_agents == expected_agents - - -def test_get_versions_to_rebuild_same_version(): - supported_versions = gather_all_supported_agent_versions(release_json) - agents = get_versions_to_rebuild(supported_versions, "6.0.0_1.26.0", "6.0.0_1.26.0") - assert len(agents) == 1 - assert agents[0] == "6.0.0_1.26.0" - - def test_get_versions_to_rebuild_multiple_versions(): supported_versions = ["6.0.0", "6.0.1", "6.0.21", "6.11.0", "7.0.0"] expected_agents = ["6.0.0", "6.0.1", "6.0.21"] From a3e4f21a68110014f8dda7a7ec6d09523b5022cf Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 13:39:40 +0200 Subject: [PATCH 08/20] empty From 27732f037887596d7abb988c7d27f11722158c8f Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 13:58:07 +0200 Subject: [PATCH 09/20] push agent --- .evergreen.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index d447d91ed..b816b68cd 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -33,6 +33,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &base_no_om_image_dependency depends_on: @@ -50,6 +52,8 @@ variables: variant: init_test_run - name: build_init_appdb_images_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &community_dependency depends_on: @@ -57,7 +61,6 @@ variables: variant: init_test_run - name: build_test_image variant: init_test_run - - name: build_readiness_probe_image variant: init_test_run - name: build_upgrade_hook_image @@ -148,6 +151,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &base_om7_dependency_with_race depends_on: @@ -165,6 +170,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &base_om8_dependency depends_on: @@ -182,7 +189,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run - + - name: build_agent_images_ubi + variant: init_test_run parameters: - key: evergreen_retry From c581be01cbed4655e9dd22df28009fa3c057dc59 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 13:58:07 +0200 Subject: [PATCH 10/20] push agent --- .evergreen.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 66f9aa5dc..af499283d 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -33,6 +33,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &base_no_om_image_dependency depends_on: @@ -50,6 +52,8 @@ variables: variant: init_test_run - name: build_init_appdb_images_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &community_dependency depends_on: @@ -57,7 +61,6 @@ variables: variant: init_test_run - name: build_test_image variant: init_test_run - - name: build_readiness_probe_image variant: init_test_run - name: build_upgrade_hook_image @@ -148,6 +151,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &base_om7_dependency_with_race depends_on: @@ -165,6 +170,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &base_om8_dependency depends_on: @@ -182,7 +189,8 @@ variables: variant: init_test_run - name: build_init_om_images_ubi variant: init_test_run - + - name: build_agent_images_ubi + variant: init_test_run parameters: - key: evergreen_retry From 115b072d0903966647cf3dd752de42974782d5e7 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 14:00:41 +0200 Subject: [PATCH 11/20] push agent --- .evergreen.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen.yml b/.evergreen.yml index af499283d..a25775341 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -67,6 +67,8 @@ variables: variant: init_test_run - name: build_mco_test_image variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run - &setup_group setup_group_can_fail_task: true From 7d78b092679a1df7ffdbe50b079f938a56b6e8c1 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 15:43:09 +0200 Subject: [PATCH 12/20] add a way to push all agents --- .evergreen-functions.yml | 2 +- .evergreen.yml | 12 +- scripts/detect_ops_manager_changes.py | 148 ++++++++++++++++-- scripts/release/atomic_pipeline.py | 18 ++- .../build/image_build_configuration.py | 1 + scripts/release/pipeline_main.py | 10 +- scripts/test_detect_ops_manager_changes.py | 12 ++ 7 files changed, 173 insertions(+), 30 deletions(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 187e170db..97c5778fd 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -538,7 +538,7 @@ functions: shell: bash <<: *e2e_include_expansions_in_env working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} + binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents|--all-agents} # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function legacy_pipeline: diff --git a/.evergreen.yml b/.evergreen.yml index a25775341..bc09c5cf9 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -392,19 +392,15 @@ tasks: skip_tags: release - name: release_all_agents_on_ecr - # this enables us to run this manually (patch) and release all agent versions to ECR - # it's needed during operator new version release process - e2e tests (especially olm tests) - # will look for agent with new operator version suffix, but during PR checks we only build - # agent versions for most recent major OM versions and the tests will fail. Before running the PR - # we have to manually release all agents to ECR by triggering this patch + # this enables us to run this manually (patch) and release all agent versions to ECR to verify + # Dockerfile, script changes etc. allowed_requesters: [ "patch" ] commands: - func: clone - func: setup_building_host - - func: legacy_pipeline + - func: pipeline vars: - image_name: agent-pct - skip_tags: release + image_name: agent all_agents: true - name: build_test_image diff --git a/scripts/detect_ops_manager_changes.py b/scripts/detect_ops_manager_changes.py index a6079582f..dea823348 100644 --- a/scripts/detect_ops_manager_changes.py +++ b/scripts/detect_ops_manager_changes.py @@ -5,6 +5,7 @@ """ import json import os +import re import subprocess import sys from typing import Dict, List, Optional, Tuple @@ -72,6 +73,139 @@ def get_changed_agents(current_mapping: Dict, base_mapping: Dict) -> List[Tuple[ return list(set(added_agents)) +def get_om_version_from_evergreen(om_series: str) -> Optional[str]: + """Extract OM version from .evergreen.yml for a given series (60, 70, 80)""" + if not os.path.exists(".evergreen.yml"): + return None + + try: + with open(".evergreen.yml", "r") as f: + content = f.read() + + # Extract the OM version using the same pattern as the dev contexts + pattern = rf"^\s*-\s*&ops_manager_{om_series}_latest\s+(\S+)\s+#" + match = re.search(pattern, content, re.MULTILINE) + if match: + return match.group(1) + + except Exception as e: + pass + + return None + + +def scan_dev_contexts_for_agents() -> List[Tuple[str, str]]: + """Scan all dev context files and extract all agents used (both hardcoded and OM-mapped)""" + agents = set() + + release_data = load_current_release_json() + if not release_data: + print("ERROR: Could not load release.json") + return [] + + ops_manager_mapping = extract_ops_manager_mapping(release_data) + ops_manager_versions = ops_manager_mapping.get("ops_manager", {}) + + # Scan all files in scripts/dev/contexts directory + contexts_dir = "scripts/dev/contexts" + if not os.path.exists(contexts_dir): + print(f"ERROR: {contexts_dir} not found") + return [] + + for root, dirs, files in os.walk(contexts_dir): + for file in files: + file_path = os.path.join(root, file) + + try: + with open(file_path, "r") as f: + content = f.read() + + # Extract hardcoded AGENT_VERSION + agent_matches = re.findall(r"export AGENT_VERSION=([^\s]+)", content) + for agent_version in agent_matches: + tools_version = get_tools_version_for_agent(agent_version) + agents.add((agent_version, tools_version)) + print(f"Found hardcoded agent in {file}: {agent_version}, {tools_version}") + + # Extract OM series from grep patterns and resolve to agents + om_patterns = re.findall(r"ops_manager_(\d+)_latest", content) + for om_series in om_patterns: + om_version = get_om_version_from_evergreen(om_series) + if om_version and om_version in ops_manager_versions: + agent_tools = ops_manager_versions[om_version] + agent_version = agent_tools.get("agent_version") + tools_version = agent_tools.get("tools_version") + + if agent_version and tools_version: + agents.add((agent_version, tools_version)) + print( + f"Found OM-mapped agent in {file}: OM {om_version} -> {agent_version}, {tools_version}" + ) + + except Exception as e: + # Skip files that can't be read (binary files, etc.) + continue + + return list(agents) + + +def get_dev_context_agents() -> List[Tuple[str, str]]: + """Returns list of (agent_version, tools_version) tuples for agents used in dev contexts""" + return scan_dev_contexts_for_agents() + + +def get_tools_version_for_agent(agent_version: str) -> str: + """Get tools version for a given agent version from release.json""" + release_data = load_current_release_json() + if not release_data: + return "100.12.2" # Default fallback + + ops_manager_mapping = extract_ops_manager_mapping(release_data) + ops_manager_versions = ops_manager_mapping.get("ops_manager", {}) + + # Search through all OM versions to find matching agent version + for om_version, agent_tools in ops_manager_versions.items(): + if agent_tools.get("agent_version") == agent_version: + return agent_tools.get("tools_version", "100.12.2") + + # Check cloud_manager tools version as fallback + return ops_manager_mapping.get("cloud_manager_tools", "100.12.2") + + +def get_all_agents_for_rebuild() -> List[Tuple[str, str]]: + """Returns list of (agent_version, tools_version) tuples for all agents in release.json""" + agents = [] + + release_data = load_current_release_json() + if not release_data: + print("ERROR: Could not load release.json") + return [] + + ops_manager_mapping = extract_ops_manager_mapping(release_data) + + # Get all ops_manager agents + ops_manager_versions = ops_manager_mapping.get("ops_manager", {}) + for om_version, agent_tools in ops_manager_versions.items(): + agent_version = agent_tools.get("agent_version") + tools_version = agent_tools.get("tools_version") + if agent_version and tools_version: + agents.append((agent_version, tools_version)) + + # Get cloud_manager agent + cloud_manager_agent = ops_manager_mapping.get("cloud_manager") + cloud_manager_tools = ops_manager_mapping.get("cloud_manager_tools") + if cloud_manager_agent and cloud_manager_tools: + agents.append((cloud_manager_agent, cloud_manager_tools)) + + # Get the main agent version from release.json root + main_agent_version = release_data.get("agentVersion") + if main_agent_version: + tools_version = get_tools_version_for_agent(main_agent_version) + agents.append((main_agent_version, tools_version)) + + return list(set(agents)) # Remove duplicates + + def detect_ops_manager_changes() -> List[Tuple[str, str]]: """Returns (has_changes, changed_agents_list)""" print("=== Detecting OM Mapping Changes (Local vs Base) ===") @@ -93,17 +227,3 @@ def detect_ops_manager_changes() -> List[Tuple[str, str]]: return get_changed_agents(current_mapping, base_mapping) else: return [] - - -if __name__ == "__main__": - if not os.path.exists("release.json"): - print("ERROR: release.json not found. Are you in the project root?") - sys.exit(2) - - agents = detect_ops_manager_changes() - if agents: - print(f"Changed agents: {agents}") - sys.exit(0) - else: - print("Skipping agent release") - sys.exit(1) diff --git a/scripts/release/atomic_pipeline.py b/scripts/release/atomic_pipeline.py index 2cadf7de6..c706cb4b7 100755 --- a/scripts/release/atomic_pipeline.py +++ b/scripts/release/atomic_pipeline.py @@ -8,15 +8,17 @@ from concurrent.futures import ProcessPoolExecutor from copy import copy from queue import Queue -from typing import Dict, List, Optional, Tuple +from typing import Dict, Optional, Tuple import requests -import semver from opentelemetry import trace from packaging.version import Version from lib.base_logger import logger -from scripts.detect_ops_manager_changes import detect_ops_manager_changes +from scripts.detect_ops_manager_changes import ( + detect_ops_manager_changes, + get_all_agents_for_rebuild, +) from scripts.release.build.image_build_configuration import ImageBuildConfiguration from scripts.release.build.image_build_process import execute_docker_build from scripts.release.build.image_signing import ( @@ -274,12 +276,18 @@ def build_upgrade_hook_image(build_configuration: ImageBuildConfiguration): ) -def build_agent_default_case(build_configuration: ImageBuildConfiguration): +def build_agent(build_configuration: ImageBuildConfiguration): """ Build the agent only for the latest operator for patches and operator releases. """ - agent_versions_to_build = detect_ops_manager_changes() + if build_configuration.all_agents: + agent_versions_to_build = get_all_agents_for_rebuild() + logger.info("building all agents") + else: + agent_versions_to_build = detect_ops_manager_changes() + logger.info("building agents for changed OM versions") + if not agent_versions_to_build: logger.info("No changes detected, skipping agent build") return diff --git a/scripts/release/build/image_build_configuration.py b/scripts/release/build/image_build_configuration.py index 966b7d02e..72fe8db8d 100644 --- a/scripts/release/build/image_build_configuration.py +++ b/scripts/release/build/image_build_configuration.py @@ -17,6 +17,7 @@ class ImageBuildConfiguration: parallel_factor: int = 0 platforms: Optional[List[str]] = None sign: bool = False + all_agents: bool = False def is_release_scenario(self) -> bool: return self.scenario == BuildScenario.RELEASE diff --git a/scripts/release/pipeline_main.py b/scripts/release/pipeline_main.py index c7e2c0c97..b5fe22d9d 100644 --- a/scripts/release/pipeline_main.py +++ b/scripts/release/pipeline_main.py @@ -17,7 +17,7 @@ from lib.base_logger import logger from scripts.release.atomic_pipeline import ( - build_agent_default_case, + build_agent, build_database_image, build_init_appdb_image, build_init_database_image, @@ -74,7 +74,7 @@ def get_builder_function_for_image_name() -> Dict[str, Callable]: READINESS_PROBE_IMAGE: build_readiness_probe_image, UPGRADE_HOOK_IMAGE: build_upgrade_hook_image, DATABASE_IMAGE: build_database_image, - AGENT_IMAGE: build_agent_default_case, + AGENT_IMAGE: build_agent, # Init images INIT_APPDB_IMAGE: build_init_appdb_image, INIT_DATABASE_IMAGE: build_init_database_image, @@ -123,6 +123,7 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration: platforms=platforms, sign=sign, parallel_factor=args.parallel_factor, + all_agents=args.all_agents, ) @@ -251,6 +252,11 @@ def main(): type=int, help="Number of agent builds to run in parallel, defaults to number of cores", ) + parser.add_argument( + "--all-agents", + action="store_true", + help="Build all agent images.", + ) args = parser.parse_args() diff --git a/scripts/test_detect_ops_manager_changes.py b/scripts/test_detect_ops_manager_changes.py index 4c5c1d212..cfedf7bb5 100644 --- a/scripts/test_detect_ops_manager_changes.py +++ b/scripts/test_detect_ops_manager_changes.py @@ -14,8 +14,13 @@ from detect_ops_manager_changes import ( detect_ops_manager_changes, extract_ops_manager_mapping, + get_all_agents_for_rebuild, get_content_from_git, + get_dev_context_agents, + get_om_version_from_evergreen, + get_tools_version_for_agent, load_current_release_json, + scan_dev_contexts_for_agents, ) @@ -53,6 +58,13 @@ def setUp(self): } } + self.evergreen_content = """ +variables: + - &ops_manager_60_latest 6.0.27 # The order/index is important + - &ops_manager_70_latest 7.0.17 # The order/index is important + - &ops_manager_80_latest 8.0.12 # The order/index is important +""" + def test_extract_ops_manager_mapping_valid(self): """Test extracting opsManagerMapping from valid release data""" mapping = extract_ops_manager_mapping(self.master_release_data) From 18e2687d39d98b00cc8f63aa223db96f7114d93d Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 15:44:44 +0200 Subject: [PATCH 13/20] add a way to push all agents --- scripts/detect_ops_manager_changes.py | 81 ---------------------- scripts/test_detect_ops_manager_changes.py | 5 -- 2 files changed, 86 deletions(-) diff --git a/scripts/detect_ops_manager_changes.py b/scripts/detect_ops_manager_changes.py index dea823348..bca973685 100644 --- a/scripts/detect_ops_manager_changes.py +++ b/scripts/detect_ops_manager_changes.py @@ -73,87 +73,6 @@ def get_changed_agents(current_mapping: Dict, base_mapping: Dict) -> List[Tuple[ return list(set(added_agents)) -def get_om_version_from_evergreen(om_series: str) -> Optional[str]: - """Extract OM version from .evergreen.yml for a given series (60, 70, 80)""" - if not os.path.exists(".evergreen.yml"): - return None - - try: - with open(".evergreen.yml", "r") as f: - content = f.read() - - # Extract the OM version using the same pattern as the dev contexts - pattern = rf"^\s*-\s*&ops_manager_{om_series}_latest\s+(\S+)\s+#" - match = re.search(pattern, content, re.MULTILINE) - if match: - return match.group(1) - - except Exception as e: - pass - - return None - - -def scan_dev_contexts_for_agents() -> List[Tuple[str, str]]: - """Scan all dev context files and extract all agents used (both hardcoded and OM-mapped)""" - agents = set() - - release_data = load_current_release_json() - if not release_data: - print("ERROR: Could not load release.json") - return [] - - ops_manager_mapping = extract_ops_manager_mapping(release_data) - ops_manager_versions = ops_manager_mapping.get("ops_manager", {}) - - # Scan all files in scripts/dev/contexts directory - contexts_dir = "scripts/dev/contexts" - if not os.path.exists(contexts_dir): - print(f"ERROR: {contexts_dir} not found") - return [] - - for root, dirs, files in os.walk(contexts_dir): - for file in files: - file_path = os.path.join(root, file) - - try: - with open(file_path, "r") as f: - content = f.read() - - # Extract hardcoded AGENT_VERSION - agent_matches = re.findall(r"export AGENT_VERSION=([^\s]+)", content) - for agent_version in agent_matches: - tools_version = get_tools_version_for_agent(agent_version) - agents.add((agent_version, tools_version)) - print(f"Found hardcoded agent in {file}: {agent_version}, {tools_version}") - - # Extract OM series from grep patterns and resolve to agents - om_patterns = re.findall(r"ops_manager_(\d+)_latest", content) - for om_series in om_patterns: - om_version = get_om_version_from_evergreen(om_series) - if om_version and om_version in ops_manager_versions: - agent_tools = ops_manager_versions[om_version] - agent_version = agent_tools.get("agent_version") - tools_version = agent_tools.get("tools_version") - - if agent_version and tools_version: - agents.add((agent_version, tools_version)) - print( - f"Found OM-mapped agent in {file}: OM {om_version} -> {agent_version}, {tools_version}" - ) - - except Exception as e: - # Skip files that can't be read (binary files, etc.) - continue - - return list(agents) - - -def get_dev_context_agents() -> List[Tuple[str, str]]: - """Returns list of (agent_version, tools_version) tuples for agents used in dev contexts""" - return scan_dev_contexts_for_agents() - - def get_tools_version_for_agent(agent_version: str) -> str: """Get tools version for a given agent version from release.json""" release_data = load_current_release_json() diff --git a/scripts/test_detect_ops_manager_changes.py b/scripts/test_detect_ops_manager_changes.py index cfedf7bb5..317db7fed 100644 --- a/scripts/test_detect_ops_manager_changes.py +++ b/scripts/test_detect_ops_manager_changes.py @@ -14,13 +14,8 @@ from detect_ops_manager_changes import ( detect_ops_manager_changes, extract_ops_manager_mapping, - get_all_agents_for_rebuild, get_content_from_git, - get_dev_context_agents, - get_om_version_from_evergreen, - get_tools_version_for_agent, load_current_release_json, - scan_dev_contexts_for_agents, ) From 34775711359728ca8ebeb6881051365d628e9e32 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 15:58:15 +0200 Subject: [PATCH 14/20] push agent --- .evergreen-functions.yml | 2 +- .evergreen.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 97c5778fd..c6090ad63 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -538,7 +538,7 @@ functions: shell: bash <<: *e2e_include_expansions_in_env working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents|--all-agents} + binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents} # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function legacy_pipeline: diff --git a/.evergreen.yml b/.evergreen.yml index bc09c5cf9..7a329bc88 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -401,7 +401,7 @@ tasks: - func: pipeline vars: image_name: agent - all_agents: true + all_agents: "--all-agents" - name: build_test_image commands: From 931a774a9931702648b20c1fb3a51e3b21796695 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 17:02:29 +0200 Subject: [PATCH 15/20] change release folder --- pipeline.py | 2 +- scripts/release/atomic_pipeline.py | 8 ++-- .../detect_ops_manager_changes.py | 0 .../tests}/test_detect_ops_manager_changes.py | 42 +++++++++---------- 4 files changed, 25 insertions(+), 27 deletions(-) rename scripts/{ => release}/detect_ops_manager_changes.py (100%) rename scripts/{ => release/tests}/test_detect_ops_manager_changes.py (78%) diff --git a/pipeline.py b/pipeline.py index 46b3fa54d..07308a655 100755 --- a/pipeline.py +++ b/pipeline.py @@ -42,7 +42,6 @@ import docker from lib.base_logger import logger from lib.sonar.sonar import process_image -from scripts.detect_ops_manager_changes import detect_ops_manager_changes from scripts.evergreen.release.agent_matrix import ( get_supported_version_for_image, ) @@ -52,6 +51,7 @@ sign_image, verify_signature, ) +from scripts.release.detect_ops_manager_changes import detect_ops_manager_changes TRACER = trace.get_tracer("evergreen-agent") diff --git a/scripts/release/atomic_pipeline.py b/scripts/release/atomic_pipeline.py index c706cb4b7..8ea5831e1 100755 --- a/scripts/release/atomic_pipeline.py +++ b/scripts/release/atomic_pipeline.py @@ -15,10 +15,6 @@ from packaging.version import Version from lib.base_logger import logger -from scripts.detect_ops_manager_changes import ( - detect_ops_manager_changes, - get_all_agents_for_rebuild, -) from scripts.release.build.image_build_configuration import ImageBuildConfiguration from scripts.release.build.image_build_process import execute_docker_build from scripts.release.build.image_signing import ( @@ -26,6 +22,10 @@ sign_image, verify_signature, ) +from scripts.release.detect_ops_manager_changes import ( + detect_ops_manager_changes, + get_all_agents_for_rebuild, +) TRACER = trace.get_tracer("evergreen-agent") diff --git a/scripts/detect_ops_manager_changes.py b/scripts/release/detect_ops_manager_changes.py similarity index 100% rename from scripts/detect_ops_manager_changes.py rename to scripts/release/detect_ops_manager_changes.py diff --git a/scripts/test_detect_ops_manager_changes.py b/scripts/release/tests/test_detect_ops_manager_changes.py similarity index 78% rename from scripts/test_detect_ops_manager_changes.py rename to scripts/release/tests/test_detect_ops_manager_changes.py index 317db7fed..df4ec7f0b 100644 --- a/scripts/test_detect_ops_manager_changes.py +++ b/scripts/release/tests/test_detect_ops_manager_changes.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Tests for detect_ops_manager_changes.py +Tests for scripts.release.detect_ops_manager_changes.py """ import json import os @@ -9,9 +9,7 @@ import unittest from unittest.mock import MagicMock, mock_open, patch -# Add the scripts directory to the path so we can import the module -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from detect_ops_manager_changes import ( +from scripts.release.detect_ops_manager_changes import ( detect_ops_manager_changes, extract_ops_manager_mapping, get_content_from_git, @@ -130,8 +128,8 @@ def test_safe_git_show_failure(self, mock_run): def test_no_changes_detected(self): """Test when no changes are detected""" with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=self.current_release_data), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=self.current_release_data), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): changed_agents = detect_ops_manager_changes() @@ -146,8 +144,8 @@ def test_new_ops_manager_version_added(self): } with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): changed_agents = detect_ops_manager_changes() @@ -161,8 +159,8 @@ def test_ops_manager_version_modified(self): ] = "12.0.35.7911-1" with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): changed_agents = detect_ops_manager_changes() @@ -174,8 +172,8 @@ def test_cloud_manager_changed(self): modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"] = "13.38.0.9600-1" with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): changed_agents = detect_ops_manager_changes() @@ -187,8 +185,8 @@ def test_cloud_manager_tools_changed(self): modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"] = "100.13.0" with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): changed_agents = detect_ops_manager_changes() @@ -200,8 +198,8 @@ def test_ops_manager_version_removed(self): del modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"]["7.0.11"] with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): changed_agents = detect_ops_manager_changes() @@ -217,8 +215,8 @@ def test_both_om_and_cm_changed(self): modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"] = "13.38.0.9600-1" with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): changed_agents = detect_ops_manager_changes() @@ -229,8 +227,8 @@ def test_both_om_and_cm_changed(self): def test_current_release_load_failure(self): """Test handling when current release.json cannot be loaded""" with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=None), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=None), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), ): changed_agents = detect_ops_manager_changes() @@ -239,8 +237,8 @@ def test_current_release_load_failure(self): def test_base_release_load_failure_fail_safe(self): """Test fail-safe behavior when base release.json cannot be loaded""" with ( - patch("detect_ops_manager_changes.load_current_release_json", return_value=self.current_release_data), - patch("detect_ops_manager_changes.load_release_json_from_master", return_value=None), + patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=self.current_release_data), + patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=None), ): changed_agents = detect_ops_manager_changes() From 5ad2a77313edf04d41bb1c1c2f3dd552e5ad1cb0 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 17:24:48 +0200 Subject: [PATCH 16/20] linter --- .../tests/test_detect_ops_manager_changes.py | 74 +++++++++++++++---- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/scripts/release/tests/test_detect_ops_manager_changes.py b/scripts/release/tests/test_detect_ops_manager_changes.py index df4ec7f0b..8606c4424 100644 --- a/scripts/release/tests/test_detect_ops_manager_changes.py +++ b/scripts/release/tests/test_detect_ops_manager_changes.py @@ -128,8 +128,14 @@ def test_safe_git_show_failure(self, mock_run): def test_no_changes_detected(self): """Test when no changes are detected""" with ( - patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=self.current_release_data), - patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_current_release_json", + return_value=self.current_release_data, + ), + patch( + "scripts.release.detect_ops_manager_changes.load_release_json_from_master", + return_value=self.master_release_data, + ), ): changed_agents = detect_ops_manager_changes() @@ -144,8 +150,13 @@ def test_new_ops_manager_version_added(self): } with ( - patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current + ), + patch( + "scripts.release.detect_ops_manager_changes.load_release_json_from_master", + return_value=self.master_release_data, + ), ): changed_agents = detect_ops_manager_changes() @@ -159,8 +170,13 @@ def test_ops_manager_version_modified(self): ] = "12.0.35.7911-1" with ( - patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current + ), + patch( + "scripts.release.detect_ops_manager_changes.load_release_json_from_master", + return_value=self.master_release_data, + ), ): changed_agents = detect_ops_manager_changes() @@ -172,8 +188,13 @@ def test_cloud_manager_changed(self): modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"] = "13.38.0.9600-1" with ( - patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current + ), + patch( + "scripts.release.detect_ops_manager_changes.load_release_json_from_master", + return_value=self.master_release_data, + ), ): changed_agents = detect_ops_manager_changes() @@ -185,8 +206,13 @@ def test_cloud_manager_tools_changed(self): modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"] = "100.13.0" with ( - patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current + ), + patch( + "scripts.release.detect_ops_manager_changes.load_release_json_from_master", + return_value=self.master_release_data, + ), ): changed_agents = detect_ops_manager_changes() @@ -198,8 +224,13 @@ def test_ops_manager_version_removed(self): del modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"]["7.0.11"] with ( - patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current + ), + patch( + "scripts.release.detect_ops_manager_changes.load_release_json_from_master", + return_value=self.master_release_data, + ), ): changed_agents = detect_ops_manager_changes() @@ -215,8 +246,13 @@ def test_both_om_and_cm_changed(self): modified_current["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"] = "13.38.0.9600-1" with ( - patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current), - patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=modified_current + ), + patch( + "scripts.release.detect_ops_manager_changes.load_release_json_from_master", + return_value=self.master_release_data, + ), ): changed_agents = detect_ops_manager_changes() @@ -228,7 +264,10 @@ def test_current_release_load_failure(self): """Test handling when current release.json cannot be loaded""" with ( patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=None), - patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=self.master_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_release_json_from_master", + return_value=self.master_release_data, + ), ): changed_agents = detect_ops_manager_changes() @@ -237,7 +276,10 @@ def test_current_release_load_failure(self): def test_base_release_load_failure_fail_safe(self): """Test fail-safe behavior when base release.json cannot be loaded""" with ( - patch("scripts.release.detect_ops_manager_changes.load_current_release_json", return_value=self.current_release_data), + patch( + "scripts.release.detect_ops_manager_changes.load_current_release_json", + return_value=self.current_release_data, + ), patch("scripts.release.detect_ops_manager_changes.load_release_json_from_master", return_value=None), ): From 3ad71b706627659ddfc243a5d57cd16c6a63f646 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 18:05:30 +0200 Subject: [PATCH 17/20] linter --- .evergreen.yml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/.evergreen.yml b/.evergreen.yml index 7290d2768..ccc2ae35e 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -356,10 +356,9 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: legacy_pipeline + - func: pipeline vars: - image_name: agent-pct - include_tags: release + image_name: agent - name: run_precommit_and_push tags: ["patch-run"] @@ -379,18 +378,6 @@ tasks: working_dir: src/github.com/mongodb/mongodb-kubernetes binary: scripts/evergreen/precommit_bump.sh - - name: release_agents_on_ecr - # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) - allowed_requesters: [ "patch", "github_pr" ] - priority: 70 - commands: - - func: clone - - func: setup_building_host - - func: legacy_pipeline - vars: - image_name: agent-pct - skip_tags: release - - name: release_all_agents_on_ecr # this enables us to run this manually (patch) and release all agent versions to ECR to verify # Dockerfile, script changes etc. @@ -1560,6 +1547,8 @@ buildvariants: variant: init_test_run - name: prepare_and_upload_openshift_bundles_for_e2e variant: init_tests_with_olm + - name: build_agent_images_ubi + variant: init_test_run tasks: - name: e2e_kind_olm_group @@ -1583,6 +1572,8 @@ buildvariants: variant: init_tests_with_olm - name: build_init_database_image_ubi variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run tasks: - name: e2e_kind_olm_group From c3dd8648698c95501e2ebe8d135c56248655bee9 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 18:16:21 +0200 Subject: [PATCH 18/20] add release for agent --- .evergreen-functions.yml | 30 ++++++++++++++++++------------ .evergreen.yml | 2 +- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index c51b9dc45..8aab27584 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -517,22 +517,26 @@ functions: # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use docker buildx inspect --bootstrap - - command: ec2.assume_role - display_name: Assume IAM role with permissions to pull Kondukto API token + - command: subprocess.exec + retry_on_failure: true + type: setup params: - role_arn: ${kondukto_role_arn} + shell: bash + <<: *e2e_include_expansions_in_env + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents} + + release_pipeline: + - *switch_context - command: shell.exec - display_name: Pull Kondukto API token from AWS Secrets Manager and write it to file + type: setup params: - silent: true shell: bash - include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] script: | - set -e - # use AWS CLI to get the Kondukto API token from AWS Secrets Manager - kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text) - # write the KONDUKTO_TOKEN environment variable to Silkbomb environment file - echo "KONDUKTO_TOKEN=$kondukto_token" > ${workdir}/silkbomb.env + # Docker Hub workaround + # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror + docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use + docker buildx inspect --bootstrap - command: subprocess.exec retry_on_failure: true type: setup @@ -540,7 +544,9 @@ functions: shell: bash <<: *e2e_include_expansions_in_env working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents} + env: + git_tag: ${triggered_by_git_tag} + binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py ${image_name} --build-scenario release --version ${git_tag} # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function legacy_pipeline: diff --git a/.evergreen.yml b/.evergreen.yml index ccc2ae35e..2f93dc6d7 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -356,7 +356,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: release_pipeline vars: image_name: agent From 937e95374eb001be611c2efd9fca5044c3da61c8 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Thu, 14 Aug 2025 18:23:42 +0200 Subject: [PATCH 19/20] add release for agent --- .evergreen-functions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 8aab27584..32ea1ee6a 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -546,7 +546,7 @@ functions: working_dir: src/github.com/mongodb/mongodb-kubernetes env: git_tag: ${triggered_by_git_tag} - binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py ${image_name} --build-scenario release --version ${git_tag} + binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py ${image_name} --build-scenario release ${git_tag|--version ${git_tag}} # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function legacy_pipeline: From 03aba021eeb2fd1475c0ffa9ab15323b1d06147b Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 18 Aug 2025 11:51:19 +0200 Subject: [PATCH 20/20] add release for agent support back to pipeline.py --- .evergreen-functions.yml | 22 ---------------------- .evergreen.yml | 24 ++++++++++++------------ inventories/agent.yaml | 2 +- lib/sonar/template.py | 2 +- pipeline.py | 7 +------ 5 files changed, 15 insertions(+), 42 deletions(-) diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 32ea1ee6a..1abd6b451 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -526,28 +526,6 @@ functions: working_dir: src/github.com/mongodb/mongodb-kubernetes binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents} - release_pipeline: - - *switch_context - - command: shell.exec - type: setup - params: - shell: bash - script: | - # Docker Hub workaround - # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror - docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use - docker buildx inspect --bootstrap - - command: subprocess.exec - retry_on_failure: true - type: setup - params: - shell: bash - <<: *e2e_include_expansions_in_env - working_dir: src/github.com/mongodb/mongodb-kubernetes - env: - git_tag: ${triggered_by_git_tag} - binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py ${image_name} --build-scenario release ${git_tag|--version ${git_tag}} - # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function legacy_pipeline: - *switch_context diff --git a/.evergreen.yml b/.evergreen.yml index 2f93dc6d7..1895cc001 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -356,7 +356,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: release_pipeline + - func: legacy_pipeline vars: image_name: agent @@ -1787,17 +1787,17 @@ buildvariants: tags: [ "release_agent" ] run_on: - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780 - depends_on: - - variant: init_test_run - name: build_agent_images_ubi # this ensures the agent gets released to ECR as well - - variant: e2e_multi_cluster_kind - name: '*' - - variant: e2e_static_multi_cluster_2_clusters - name: '*' - - variant: e2e_mdb_kind_ubi_cloudqa - name: '*' - - variant: e2e_static_mdb_kind_ubi_cloudqa - name: '*' +# depends_on: +## - variant: init_test_run +## name: build_agent_images_ubi # this ensures the agent gets released to ECR as well +## - variant: e2e_multi_cluster_kind +## name: '*' +## - variant: e2e_static_multi_cluster_2_clusters +## name: '*' +## - variant: e2e_mdb_kind_ubi_cloudqa +## name: '*' +## - variant: e2e_static_mdb_kind_ubi_cloudqa +## name: '*' tasks: - name: release_agent diff --git a/inventories/agent.yaml b/inventories/agent.yaml index 876e4027b..bcb2d5889 100644 --- a/inventories/agent.yaml +++ b/inventories/agent.yaml @@ -3,7 +3,7 @@ vars: s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-agent images: - - name: mongodb-agent-ubi + - name: mongodb-agent vars: context: . template_context: docker/mongodb-agent diff --git a/lib/sonar/template.py b/lib/sonar/template.py index 98d8e1a11..e92fbb53a 100644 --- a/lib/sonar/template.py +++ b/lib/sonar/template.py @@ -13,7 +13,7 @@ def render(path: str, template_name: str, parameters: Dict[str, str]) -> str: """ env = jinja2.Environment(loader=jinja2.FileSystemLoader(path), undefined=jinja2.StrictUndefined) - template = "Dockerfile" + template = "Dockerfile.old" if template_name is not None: template = "Dockerfile.{}".format(template_name) diff --git a/pipeline.py b/pipeline.py index 07308a655..2935377f6 100755 --- a/pipeline.py +++ b/pipeline.py @@ -1203,7 +1203,6 @@ def build_multi_arch_agent_in_sonar( """ logger.info(f"building multi-arch base image for: {image_version}") - is_release = build_configuration.is_release_step_executed() args = { "version": image_version, "tools_version": tools_version, @@ -1233,7 +1232,7 @@ def build_multi_arch_agent_in_sonar( build_image_generic( config=build_configuration, - image_name="mongodb-agent-ubi", + image_name="mongodb-agent", inventory_file="inventories/agent.yaml", multi_arch_args_list=joined_args, with_image_base=False, @@ -1315,18 +1314,14 @@ def _build_agents( tasks_queue: Queue, ): agent_version = agent_tools_version[0] - agent_distro = "rhel9_x86_64" tools_version = agent_tools_version[1] - tools_distro = get_tools_distro(tools_version)["amd"] tasks_queue.put( executor.submit( build_multi_arch_agent_in_sonar, build_configuration, agent_version, - agent_distro, tools_version, - tools_distro, ) )