CLOUDP-338084 - removing and refactoring agent matrix from pipeline.py and atomic_pipeline.py

.evergreen-functions.yml

@@ -540,7 +540,7 @@ functions:
         shell: bash
         <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name}
+        binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents}
 
   # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function
   legacy_pipeline:

.evergreen.yml

@@ -61,14 +61,14 @@ variables:
         variant: init_test_run
       - name: build_test_image
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
       - name: build_readiness_probe_image
         variant: init_test_run
       - name: build_upgrade_hook_image
         variant: init_test_run
       - name: build_mco_test_image
         variant: init_test_run
+      - name: build_agent_images_ubi
+        variant: init_test_run
 
   - &setup_group
     setup_group_can_fail_task: true
@@ -347,19 +347,6 @@ tasks:
           image_name: init-ops-manager
           include_tags: release
 
-  - name: release_agent_operator_release
-    tags: [ "image_release" ]
-    allowed_requesters: [ "patch", "github_tag" ]
-    commands:
-      - func: clone
-      - func: setup_building_host
-      - func: quay_login
-      - func: setup_docker_sbom
-      - func: legacy_pipeline
-        vars:
-          image_name: agent
-          include_tags: release
-
   # pct only triggers this variant once a new agent image is out
   - name: release_agent
     # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr)
@@ -392,21 +379,6 @@ tasks:
           working_dir: src/github.com/mongodb/mongodb-kubernetes
           binary: scripts/evergreen/precommit_bump.sh
 
-  # Pct only triggers this variant once a new agent image is out
-  # these releases the agent with the operator suffix (not patch id) on ecr to allow for digest pinning to pass.
-  # For this to work, we rely on skip_tags which is used to determine whether
-  # we want to release on quay or not, in this case - ecr instead.
-  # We rely on the init_database from ecr for the agent x operator images.
-  # This runs on agent releases that are not concurrent with operator releases.
-  - name: release_agents_on_ecr_conditional
-    commands:
-      - func: clone
-      - func: run_task_conditionally
-        vars:
-          condition_script: scripts/evergreen/should_release_agents_on_ecr.sh
-          variant: init_release_agents_on_ecr
-          task: release_agents_on_ecr
-
   - name: release_agents_on_ecr
     # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr)
     allowed_requesters: [ "patch", "github_pr" ]
@@ -420,20 +392,16 @@ tasks:
           skip_tags: release
 
   - name: release_all_agents_on_ecr
-    # this enables us to run this manually (patch) and release all agent versions to ECR
-    # it's needed during operator new version release process - e2e tests (especially olm tests)
-    # will look for agent with new operator version suffix, but during PR checks we only build
-    # agent versions for most recent major OM versions and the tests will fail. Before running the PR
-    # we have to manually release all agents to ECR by triggering this patch
+    # this enables us to run this manually (patch) and release all agent versions to ECR to verify
+    # Dockerfile, script changes etc.
     allowed_requesters: [ "patch" ]
     commands:
       - func: clone
       - func: setup_building_host
-      - func: legacy_pipeline
+      - func: pipeline
         vars:
-          image_name: agent-pct
-          skip_tags: release
-          all_agents: true
+          image_name: agent
+          all_agents: "--all-agents"
 
   - name: build_test_image
     commands:
@@ -1334,8 +1302,7 @@ buildvariants:
         variant: init_test_run
       - name: build_init_database_image_ubi
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
+
     tasks:
       - name: e2e_custom_domain_task_group
 
@@ -1369,8 +1336,7 @@ buildvariants:
         variant: init_test_run
       - name: build_init_database_image_ubi
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
+
     run_on:
       - ubuntu2204-small
     tasks:
@@ -1617,8 +1583,7 @@ buildvariants:
         variant: init_tests_with_olm
       - name: build_init_database_image_ubi
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
+
     tasks:
       - name: e2e_kind_olm_group
 
@@ -1683,18 +1648,6 @@ buildvariants:
       - name: build_upgrade_hook_image
       - name: prepare_aws
 
-  - name: init_release_agents_on_ecr
-    display_name: init_release_agents_on_ecr
-    # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr)
-    allowed_requesters: [ "patch", "github_pr" ]
-    tags: [ "release_agents_on_ecr" ]
-    # We want that to run first and finish asap. Digest pinning depends on this to succeed.
-    priority: 70
-    run_on:
-      - ubuntu2204-large
-    tasks:
-      - name: release_agents_on_ecr_conditional
-
   - name: run_pre_commit
     priority: 70
     display_name: run_pre_commit
@@ -1722,8 +1675,7 @@ buildvariants:
         variant: init_test_run
       - name: build_init_om_images_ubi
         variant: init_test_run
-      - name: build_agent_images_ubi
-        variant: init_test_run
+
     run_on:
       - ubuntu2204-small
     tasks:
@@ -1809,13 +1761,6 @@ buildvariants:
       - name: release_init_database
       - name: release_init_ops_manager
       - name: release_database
-      # Once we release the operator, we will also release the init databases, we require them to be out first
-      # such that we can reference them and retrieve those binaries.
-      # Since we immediately run daily rebuild after creating the image, we can ensure that the init_database is out
-      # such that the agent image build can use it.
-      - name: release_agent_operator_release
-        depends_on:
-          - name: release_init_database
 
   - name: preflight_release_images
     display_name: preflight_release_images
@@ -1847,13 +1792,13 @@ buildvariants:
 
     # It will be called by pct while bumping the agent cloud manager image
   - name: release_agent
-    display_name: (Static Containers) Release Agent matrix
+    display_name: release_agent
     tags: [ "release_agent" ]
     run_on:
       - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
     depends_on:
-      - variant: init_release_agents_on_ecr
-        name: '*'
+      - variant: init_test_run
+        name: build_agent_images_ubi # this ensures the agent gets released to ECR as well
       - variant: e2e_multi_cluster_kind
         name: '*'
       - variant: e2e_static_multi_cluster_2_clusters