feat(ec2_sd_configs): introduce optional pagination for AWS API calls

Author: pcnc

lib/promscrape/discovery/ec2/api.go

@@ -8,10 +8,11 @@ import (
 )
 
 type apiConfig struct {
-	awsConfig       *awsapi.Config
-	instanceFilters []awsapi.Filter
-	azFilters       []awsapi.Filter
-	port            int
+	awsConfig         *awsapi.Config
+	instanceFilters   []awsapi.Filter
+	azFilters         []awsapi.Filter
+	port              int
+	maxResultsPerPage int
 
 	// A map from AZ name to AZ id.
 	azMap     map[string]string
@@ -42,10 +43,11 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 		return nil, err
 	}
 	cfg := &apiConfig{
-		awsConfig:       awsCfg,
-		instanceFilters: sdc.InstanceFilters,
-		azFilters:       sdc.AZFilters,
-		port:            port,
+		awsConfig:         awsCfg,
+		instanceFilters:   sdc.InstanceFilters,
+		azFilters:         sdc.AZFilters,
+		port:              port,
+		maxResultsPerPage: sdc.MaxResultsPerPage,
 	}
 	return cfg, nil
 }

lib/promscrape/discovery/ec2/ec2.go

@@ -29,9 +29,10 @@ type SDConfig struct {
 	RoleARN string `yaml:"role_arn,omitempty"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option.
-	Port            *int            `yaml:"port,omitempty"`
-	InstanceFilters []awsapi.Filter `yaml:"filters,omitempty"`
-	AZFilters       []awsapi.Filter `yaml:"az_filters,omitempty"`
+	Port              *int            `yaml:"port,omitempty"`
+	InstanceFilters   []awsapi.Filter `yaml:"filters,omitempty"`
+	AZFilters         []awsapi.Filter `yaml:"az_filters,omitempty"`
+	MaxResultsPerPage int             `yaml:"max_results_per_page,omitempty"`
 }
 
 // GetLabels returns ec2 labels according to sdc.

lib/promscrape/discovery/ec2/instance.go

@@ -32,6 +32,15 @@ func getReservations(cfg *apiConfig) ([]Reservation, error) {
 	var rs []Reservation
 	pageToken := ""
 	instanceFilters := awsapi.GetFiltersQueryString(cfg.instanceFilters)
+
+	if cfg.maxResultsPerPage > 0 {
+		if len(instanceFilters) == 0 {
+			instanceFilters = fmt.Sprintf("MaxResults=%d", cfg.maxResultsPerPage)
+		} else {
+			instanceFilters += fmt.Sprintf("&MaxResults=%d", cfg.maxResultsPerPage)
+		}
+	}
+
 	for {
 		data, err := cfg.awsConfig.GetEC2APIResponse("DescribeInstances", instanceFilters, pageToken)
 		if err != nil {