feat: support multiple SSM parameters for large runner matcher configs (#4790)

lambdas/functions/webhook/src/ConfigLoader.test.ts

@@ -168,6 +168,54 @@ describe('ConfigLoader Tests', () => {
         'Failed to load config: Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', // eslint-disable-line max-len
       );
     });
+
+    it('should load config successfully from multiple paths', async () => {
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2';
+      process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
+
+      const partialMatcher1 =
+        '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}';
+      const partialMatcher2 =
+        ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]';
+
+      const combinedMatcherConfig = [
+        { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } },
+        { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } },
+      ];
+
+      vi.mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config-1') return partialMatcher1;
+        if (paramPath === '/path/to/matcher/config-2') return partialMatcher2;
+        if (paramPath === '/path/to/webhook/secret') return 'secret';
+        return '';
+      });
+
+      const config: ConfigWebhook = await ConfigWebhook.load();
+
+      expect(config.matcherConfig).toEqual(combinedMatcherConfig);
+      expect(config.webhookSecret).toBe('secret');
+    });
+
+    it('should throw error if config loading fails from multiple paths', async () => {
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2';
+      process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret';
+
+      const partialMatcher1 =
+        '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}';
+      const partialMatcher2 =
+        ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}';
+
+      vi.mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config-1') return partialMatcher1;
+        if (paramPath === '/path/to/matcher/config-2') return partialMatcher2;
+        if (paramPath === '/path/to/webhook/secret') return 'secret';
+        return '';
+      });
+
+      await expect(ConfigWebhook.load()).rejects.toThrow(
+        "Failed to load config: Failed to parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196 (line 1 column 197)", // eslint-disable-line max-len
+      );
+    });
   });
 
   describe('ConfigWebhookEventBridge', () => {
@@ -229,6 +277,32 @@ describe('ConfigLoader Tests', () => {
       expect(config.matcherConfig).toEqual(matcherConfig);
     });
 
+    it('should load config successfully from multiple paths with repo allow list', async () => {
+      process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]';
+      process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2';
+
+      const partial1 =
+        '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}';
+      const partial2 =
+        ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]';
+
+      const combined: RunnerMatcherConfig[] = [
+        { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['x']], exactMatch: true } },
+        { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['y']], exactMatch: true } },
+      ];
+
+      vi.mocked(getParameter).mockImplementation(async (paramPath: string) => {
+        if (paramPath === '/path/to/matcher/config-1') return partial1;
+        if (paramPath === '/path/to/matcher/config-2') return partial2;
+        return '';
+      });
+
+      const config: ConfigDispatcher = await ConfigDispatcher.load();
+
+      expect(config.repositoryAllowList).toEqual(['repo1', 'repo2']);
+      expect(config.matcherConfig).toEqual(combined);
+    });
+
     it('should throw error if config loading fails', async () => {
       vi.mocked(getParameter).mockImplementation(async (paramPath: string) => {
         throw new Error(`Parameter ${paramPath} not found`);

lambdas/functions/webhook/src/ConfigLoader.ts

@@ -87,17 +87,44 @@ abstract class BaseConfig {
   }
 }
 
-export class ConfigWebhook extends BaseConfig {
-  repositoryAllowList: string[] = [];
+abstract class MatcherAwareConfig extends BaseConfig {
   matcherConfig: RunnerMatcherConfig[] = [];
+
+  protected async loadMatcherConfig(paramPathsEnv: string) {
+    if (!paramPathsEnv || paramPathsEnv === 'undefined' || paramPathsEnv === 'null' || !paramPathsEnv.includes(':')) {
+      await this.loadParameter(paramPathsEnv, 'matcherConfig');
+      return;
+    }
+
+    const paths = paramPathsEnv
+      .split(':')
+      .map((p) => p.trim())
+      .filter(Boolean);
+    let combinedString = '';
+
+    for (const path of paths) {
+      await this.loadParameter(path, 'matcherConfig');
+      combinedString += this.matcherConfig;
+    }
+
+    try {
+      this.matcherConfig = JSON.parse(combinedString);
+    } catch (error) {
+      this.configLoadingErrors.push(`Failed to parse combined matcher config: ${(error as Error).message}`);
+    }
+  }
+}
+
+export class ConfigWebhook extends MatcherAwareConfig {
+  repositoryAllowList: string[] = [];
   webhookSecret: string = '';
   workflowJobEventSecondaryQueue: string = '';
 
   async loadConfig(): Promise<void> {
     this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []);
 
     await Promise.all([
-      this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig'),
+      this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH),
       this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'),
     ]);
 
@@ -121,14 +148,13 @@ export class ConfigWebhookEventBridge extends BaseConfig {
   }
 }
 
-export class ConfigDispatcher extends BaseConfig {
+export class ConfigDispatcher extends MatcherAwareConfig {
   repositoryAllowList: string[] = [];
-  matcherConfig: RunnerMatcherConfig[] = [];
   workflowJobEventSecondaryQueue: string = ''; // Deprecated
 
   async loadConfig(): Promise<void> {
     this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []);
-    await this.loadParameter(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH, 'matcherConfig');
+    await this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH);
 
     validateRunnerMatcherConfig(this);
   }

modules/webhook/direct/README.md

@@ -40,7 +40,7 @@ No modules.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_config"></a> [config](#input\_config) | Configuration object for all variables. | <pre>object({<br/>    prefix = string<br/>    archive = optional(object({<br/>      enable         = optional(bool, true)<br/>      retention_days = optional(number, 7)<br/>    }), {})<br/>    tags = optional(map(string), {})<br/><br/>    lambda_subnet_ids         = optional(list(string), [])<br/>    lambda_security_group_ids = optional(list(string), [])<br/>    sqs_job_queues_arns       = list(string)<br/>    lambda_zip                = optional(string, null)<br/>    lambda_memory_size        = optional(number, 256)<br/>    lambda_timeout            = optional(number, 10)<br/>    role_permissions_boundary = optional(string, null)<br/>    role_path                 = optional(string, null)<br/>    logging_retention_in_days = optional(number, 180)<br/>    logging_kms_key_id        = optional(string, null)<br/>    lambda_s3_bucket          = optional(string, null)<br/>    lambda_s3_key             = optional(string, null)<br/>    lambda_s3_object_version  = optional(string, null)<br/>    lambda_apigateway_access_log_settings = optional(object({<br/>      destination_arn = string<br/>      format          = string<br/>    }), null)<br/>    repository_white_list = optional(list(string), [])<br/>    kms_key_arn           = optional(string, null)<br/>    log_level             = optional(string, "info")<br/>    lambda_runtime        = optional(string, "nodejs22.x")<br/>    aws_partition         = optional(string, "aws")<br/>    lambda_architecture   = optional(string, "arm64")<br/>    github_app_parameters = object({<br/>      webhook_secret = map(string)<br/>    })<br/>    tracing_config = optional(object({<br/>      mode                  = optional(string, null)<br/>      capture_http_requests = optional(bool, false)<br/>      capture_error         = optional(bool, false)<br/>    }), {})<br/>    lambda_tags       = optional(map(string), {})<br/>    api_gw_source_arn = string<br/>    ssm_parameter_runner_matcher_config = object({<br/>      name    = string<br/>      arn     = string<br/>      version = string<br/>    })<br/>  })</pre> | n/a | yes |
+| <a name="input_config"></a> [config](#input\_config) | Configuration object for all variables. | <pre>object({<br/>    prefix = string<br/>    archive = optional(object({<br/>      enable         = optional(bool, true)<br/>      retention_days = optional(number, 7)<br/>    }), {})<br/>    tags = optional(map(string), {})<br/><br/>    lambda_subnet_ids         = optional(list(string), [])<br/>    lambda_security_group_ids = optional(list(string), [])<br/>    sqs_job_queues_arns       = list(string)<br/>    lambda_zip                = optional(string, null)<br/>    lambda_memory_size        = optional(number, 256)<br/>    lambda_timeout            = optional(number, 10)<br/>    role_permissions_boundary = optional(string, null)<br/>    role_path                 = optional(string, null)<br/>    logging_retention_in_days = optional(number, 180)<br/>    logging_kms_key_id        = optional(string, null)<br/>    lambda_s3_bucket          = optional(string, null)<br/>    lambda_s3_key             = optional(string, null)<br/>    lambda_s3_object_version  = optional(string, null)<br/>    lambda_apigateway_access_log_settings = optional(object({<br/>      destination_arn = string<br/>      format          = string<br/>    }), null)<br/>    repository_white_list = optional(list(string), [])<br/>    kms_key_arn           = optional(string, null)<br/>    log_level             = optional(string, "info")<br/>    lambda_runtime        = optional(string, "nodejs22.x")<br/>    aws_partition         = optional(string, "aws")<br/>    lambda_architecture   = optional(string, "arm64")<br/>    github_app_parameters = object({<br/>      webhook_secret = map(string)<br/>    })<br/>    tracing_config = optional(object({<br/>      mode                  = optional(string, null)<br/>      capture_http_requests = optional(bool, false)<br/>      capture_error         = optional(bool, false)<br/>    }), {})<br/>    lambda_tags       = optional(map(string), {})<br/>    api_gw_source_arn = string<br/>    ssm_parameter_runner_matcher_config = list(object({<br/>      name    = string<br/>      arn     = string<br/>      version = string<br/>    }))<br/>  })</pre> | n/a | yes |
 
 ## Outputs
 

modules/webhook/direct/variables.tf

@@ -41,10 +41,10 @@ variable "config" {
     }), {})
     lambda_tags       = optional(map(string), {})
     api_gw_source_arn = string
-    ssm_parameter_runner_matcher_config = object({
+    ssm_parameter_runner_matcher_config = list(object({
       name    = string
       arn     = string
       version = string
-    })
+    }))
   })
 }

modules/webhook/direct/webhook.tf

@@ -26,8 +26,8 @@ resource "aws_lambda_function" "webhook" {
         POWERTOOLS_TRACER_CAPTURE_ERROR          = var.config.tracing_config.capture_error
         PARAMETER_GITHUB_APP_WEBHOOK_SECRET      = var.config.github_app_parameters.webhook_secret.name
         REPOSITORY_ALLOW_LIST                    = jsonencode(var.config.repository_white_list)
-        PARAMETER_RUNNER_MATCHER_CONFIG_PATH     = var.config.ssm_parameter_runner_matcher_config.name
-        PARAMETER_RUNNER_MATCHER_VERSION         = var.config.ssm_parameter_runner_matcher_config.version # enforce cold start after Changes in SSM parameter
+        PARAMETER_RUNNER_MATCHER_CONFIG_PATH     = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name])
+        PARAMETER_RUNNER_MATCHER_VERSION         = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter
       } : k => v if v != null
     }
   }
@@ -134,7 +134,12 @@ resource "aws_iam_role_policy" "webhook_ssm" {
   role = aws_iam_role.webhook_lambda.name
 
   policy = templatefile("${path.module}/../policies/lambda-ssm.json", {
-    resource_arns = jsonencode([var.config.github_app_parameters.webhook_secret.arn, var.config.ssm_parameter_runner_matcher_config.arn])
+    resource_arns = jsonencode(
+      concat(
+        [var.config.github_app_parameters.webhook_secret.arn],
+        [for p in var.config.ssm_parameter_runner_matcher_config : p.arn]
+      )
+    )
   })
 }
 

modules/webhook/eventbridge/README.md

@@ -54,7 +54,7 @@ No modules.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_config"></a> [config](#input\_config) | Configuration object for all variables. | <pre>object({<br/>    prefix = string<br/>    archive = optional(object({<br/>      enable         = optional(bool, true)<br/>      retention_days = optional(number, 7)<br/>    }), {})<br/>    tags = optional(map(string), {})<br/><br/>    lambda_subnet_ids         = optional(list(string), [])<br/>    lambda_security_group_ids = optional(list(string), [])<br/>    sqs_job_queues_arns       = list(string)<br/>    lambda_zip                = optional(string, null)<br/>    lambda_memory_size        = optional(number, 256)<br/>    lambda_timeout            = optional(number, 10)<br/>    role_permissions_boundary = optional(string, null)<br/>    role_path                 = optional(string, null)<br/>    logging_retention_in_days = optional(number, 180)<br/>    logging_kms_key_id        = optional(string, null)<br/>    lambda_s3_bucket          = optional(string, null)<br/>    lambda_s3_key             = optional(string, null)<br/>    lambda_s3_object_version  = optional(string, null)<br/>    lambda_apigateway_access_log_settings = optional(object({<br/>      destination_arn = string<br/>      format          = string<br/>    }), null)<br/>    repository_white_list = optional(list(string), [])<br/>    kms_key_arn           = optional(string, null)<br/>    log_level             = optional(string, "info")<br/>    lambda_runtime        = optional(string, "nodejs22.x")<br/>    aws_partition         = optional(string, "aws")<br/>    lambda_architecture   = optional(string, "arm64")<br/>    github_app_parameters = object({<br/>      webhook_secret = map(string)<br/>    })<br/>    tracing_config = optional(object({<br/>      mode                  = optional(string, null)<br/>      capture_http_requests = optional(bool, false)<br/>      capture_error         = optional(bool, false)<br/>    }), {})<br/>    lambda_tags       = optional(map(string), {})<br/>    api_gw_source_arn = string<br/>    ssm_parameter_runner_matcher_config = object({<br/>      name    = string<br/>      arn     = string<br/>      version = string<br/>    })<br/>    accept_events = optional(list(string), null)<br/>  })</pre> | n/a | yes |
+| <a name="input_config"></a> [config](#input\_config) | Configuration object for all variables. | <pre>object({<br/>    prefix = string<br/>    archive = optional(object({<br/>      enable         = optional(bool, true)<br/>      retention_days = optional(number, 7)<br/>    }), {})<br/>    tags = optional(map(string), {})<br/><br/>    lambda_subnet_ids         = optional(list(string), [])<br/>    lambda_security_group_ids = optional(list(string), [])<br/>    sqs_job_queues_arns       = list(string)<br/>    lambda_zip                = optional(string, null)<br/>    lambda_memory_size        = optional(number, 256)<br/>    lambda_timeout            = optional(number, 10)<br/>    role_permissions_boundary = optional(string, null)<br/>    role_path                 = optional(string, null)<br/>    logging_retention_in_days = optional(number, 180)<br/>    logging_kms_key_id        = optional(string, null)<br/>    lambda_s3_bucket          = optional(string, null)<br/>    lambda_s3_key             = optional(string, null)<br/>    lambda_s3_object_version  = optional(string, null)<br/>    lambda_apigateway_access_log_settings = optional(object({<br/>      destination_arn = string<br/>      format          = string<br/>    }), null)<br/>    repository_white_list = optional(list(string), [])<br/>    kms_key_arn           = optional(string, null)<br/>    log_level             = optional(string, "info")<br/>    lambda_runtime        = optional(string, "nodejs22.x")<br/>    aws_partition         = optional(string, "aws")<br/>    lambda_architecture   = optional(string, "arm64")<br/>    github_app_parameters = object({<br/>      webhook_secret = map(string)<br/>    })<br/>    tracing_config = optional(object({<br/>      mode                  = optional(string, null)<br/>      capture_http_requests = optional(bool, false)<br/>      capture_error         = optional(bool, false)<br/>    }), {})<br/>    lambda_tags       = optional(map(string), {})<br/>    api_gw_source_arn = string<br/>    ssm_parameter_runner_matcher_config = list(object({<br/>      name    = string<br/>      arn     = string<br/>      version = string<br/>    }))<br/>    accept_events = optional(list(string), null)<br/>  })</pre> | n/a | yes |
 
 ## Outputs
 

modules/webhook/eventbridge/dispatcher.tf

@@ -44,8 +44,8 @@ resource "aws_lambda_function" "dispatcher" {
         POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.tracing_config.capture_http_requests
         POWERTOOLS_TRACER_CAPTURE_ERROR          = var.config.tracing_config.capture_error
         # Parameters required for lambda configuration
-        PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name
-        PARAMETER_RUNNER_MATCHER_VERSION     = var.config.ssm_parameter_runner_matcher_config.version # enforce cold start after Changes in SSM parameter
+        PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name])
+        PARAMETER_RUNNER_MATCHER_VERSION     = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter
         REPOSITORY_ALLOW_LIST                = jsonencode(var.config.repository_white_list)
       } : k => v if v != null
     }
@@ -129,7 +129,12 @@ resource "aws_iam_role_policy" "dispatcher_ssm" {
   role = aws_iam_role.dispatcher_lambda.name
 
   policy = templatefile("${path.module}/../policies/lambda-ssm.json", {
-    resource_arns = jsonencode([var.config.ssm_parameter_runner_matcher_config.arn])
+    resource_arns = jsonencode(
+      concat(
+        [var.config.github_app_parameters.webhook_secret.arn],
+        [for p in var.config.ssm_parameter_runner_matcher_config : p.arn]
+      )
+    )
   })
 }
 

modules/webhook/eventbridge/variables.tf

@@ -41,11 +41,11 @@ variable "config" {
     }), {})
     lambda_tags       = optional(map(string), {})
     api_gw_source_arn = string
-    ssm_parameter_runner_matcher_config = object({
+    ssm_parameter_runner_matcher_config = list(object({
       name    = string
       arn     = string
       version = string
-    })
+    }))
     accept_events = optional(list(string), null)
   })
 }

modules/webhook/eventbridge/webhook.tf

@@ -31,7 +31,7 @@ resource "aws_lambda_function" "webhook" {
         ACCEPT_EVENTS                        = jsonencode(var.config.accept_events)
         EVENT_BUS_NAME                       = aws_cloudwatch_event_bus.main.name
         PARAMETER_GITHUB_APP_WEBHOOK_SECRET  = var.config.github_app_parameters.webhook_secret.name
-        PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.ssm_parameter_runner_matcher_config.name
+        PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name])
       } : k => v if v != null
     }
   }