feat: add support to send failed events to a COS bucket (#157)

Author: jor2

README.md

@@ -69,7 +69,7 @@ To create service credentials, access the Event Notifications service, and acces
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.0, <1.7.0 |
-| <a name="requirement_ibm"></a> [ibm](#requirement\_ibm) | >= 1.56.1, < 2.0.0 |
+| <a name="requirement_ibm"></a> [ibm](#requirement\_ibm) | >= 1.63.0, < 2.0.0 |
 | <a name="requirement_time"></a> [time](#requirement\_time) | >= 0.9.1 |
 
 ### Modules
@@ -82,18 +82,27 @@ To create service credentials, access the Event Notifications service, and acces
 
 | Name | Type |
 |------|------|
+| [ibm_en_destination_cos.cos_en_destination](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/en_destination_cos) | resource |
 | [ibm_en_integration.en_kms_integration](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/en_integration) | resource |
+| [ibm_iam_authorization_policy.cos_policy](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource |
 | [ibm_iam_authorization_policy.kms_policy](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/iam_authorization_policy) | resource |
 | [ibm_resource_instance.en_instance](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/resource_instance) | resource |
 | [ibm_resource_key.service_credentials](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/resource_key) | resource |
-| [time_sleep.wait_for_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
+| [time_sleep.wait_for_cos_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
+| [time_sleep.wait_for_kms_authorization_policy](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
 | [ibm_en_integrations.en_integrations](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/en_integrations) | data source |
+| [ibm_iam_account_settings.iam_account_settings](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/data-sources/iam_account_settings) | data source |
 
 ### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_cbr_rules"></a> [cbr\_rules](#input\_cbr\_rules) | The list of context-based restrictions rules to create. | <pre>list(object({<br>    description = string<br>    account_id  = string<br>    rule_contexts = list(object({<br>      attributes = optional(list(object({<br>        name  = string<br>        value = string<br>    }))) }))<br>    enforcement_mode = string<br>  }))</pre> | `[]` | no |
+| <a name="input_cos_bucket_name"></a> [cos\_bucket\_name](#input\_cos\_bucket\_name) | The name of an existing IBM Cloud Object Storage bucket which will be used for storage of failed delivery events. Required if `cos_integration_enabled` is set to true. | `string` | `null` | no |
+| <a name="input_cos_destination_name"></a> [cos\_destination\_name](#input\_cos\_destination\_name) | The name of the IBM Cloud Object Storage destination which will be created for the storage of failed delivery events. | `string` | `"COS Destination"` | no |
+| <a name="input_cos_endpoint"></a> [cos\_endpoint](#input\_cos\_endpoint) | The endpoint URL for your bucket region. For more information, see https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-endpoints. Required if `cos_integration_enabled` is set to true. | `string` | `null` | no |
+| <a name="input_cos_instance_id"></a> [cos\_instance\_id](#input\_cos\_instance\_id) | The ID of the IBM Cloud Object Storage instance in which the bucket that is defined in the `cos_bucket_name` variable exists. Required if `cos_integration_enabled` is set to true. | `string` | `null` | no |
+| <a name="input_cos_integration_enabled"></a> [cos\_integration\_enabled](#input\_cos\_integration\_enabled) | Set to `true` to connect a Cloud Object Storage service instance to your Event Notifications instance to collect events that failed delivery. If set to false, no failed events will be captured. | `bool` | `false` | no |
 | <a name="input_existing_kms_instance_crn"></a> [existing\_kms\_instance\_crn](#input\_existing\_kms\_instance\_crn) | The CRN of the Hyper Protect Crypto Services or Key Protect instance. Required only if `var.kms_encryption_enabled` is set to `true`. | `string` | `null` | no |
 | <a name="input_kms_encryption_enabled"></a> [kms\_encryption\_enabled](#input\_kms\_encryption\_enabled) | Set to `true` to control the encryption keys that are used to encrypt the data that you store in the Event Notifications instance. If set to `false`, the data is encrypted by using randomly generated keys. For more information, see [Managing encryption](https://cloud.ibm.com/docs/event-notifications?topic=event-notifications-en-managing-encryption). | `bool` | `false` | no |
 | <a name="input_kms_endpoint_url"></a> [kms\_endpoint\_url](#input\_kms\_endpoint\_url) | The URL of the KMS endpoint to use when configuring KMS encryption. The Hyper Protect Crypto Services endpoint URL format is `https://api.private.<REGION>.hs-crypto.cloud.ibm.com:<port>` and the Key Protect endpoint URL format is `https://<REGION>.kms.cloud.ibm.com`. | `string` | `null` | no |
@@ -104,7 +113,8 @@ To create service credentials, access the Event Notifications service, and acces
 | <a name="input_root_key_id"></a> [root\_key\_id](#input\_root\_key\_id) | The key ID of a root key, existing in the KMS instance passed in `var.existing_kms_instance_crn`, which will be used to encrypt the data encryption keys which are then used to encrypt the data. Required only if `var.kms_encryption_enabled` is set to `true`. | `string` | `null` | no |
 | <a name="input_service_credential_names"></a> [service\_credential\_names](#input\_service\_credential\_names) | The mapping of names and roles for service credentials that you want to create for the Event Notifications instance. | `map(string)` | `{}` | no |
 | <a name="input_service_endpoints"></a> [service\_endpoints](#input\_service\_endpoints) | Specify whether you want to enable public, or both public and private service endpoints. Possible values: `public`, `public-and-private` | `string` | `"public-and-private"` | no |
-| <a name="input_skip_iam_authorization_policy"></a> [skip\_iam\_authorization\_policy](#input\_skip\_iam\_authorization\_policy) | Set to `true` to skip the creation of an IAM authorization policy that permits all Event Notifications instances in the resource group to read the encryption key from the KMS instance. If set to `false`, specify a value for the KMS instance in the `existing_kms_instance_guid` variable. In addition, no policy is created if `kms_encryption_enabled` is set to `false`. | `bool` | `false` | no |
+| <a name="input_skip_en_cos_auth_policy"></a> [skip\_en\_cos\_auth\_policy](#input\_skip\_en\_cos\_auth\_policy) | Whether an IAM authorization policy is created for your Event Notifications instance to interact with your Object Storage bucket. Set to `true` to use an existing policy. Ignored if `cos_integration_enabled` is set to `false`. | `bool` | `false` | no |
+| <a name="input_skip_en_kms_auth_policy"></a> [skip\_en\_kms\_auth\_policy](#input\_skip\_en\_kms\_auth\_policy) | Set to `true` to skip the creation of an IAM authorization policy that permits all Event Notifications instances in the resource group to read the encryption key from the KMS instance. If set to `false`, specify a value for the KMS instance in the `existing_kms_instance_guid` variable. In addition, no policy is created if `kms_encryption_enabled` is set to `false`. | `bool` | `false` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | The list of tags to add to the Event Notifications instance. | `list(string)` | `[]` | no |
 
 ### Outputs

common-dev-assets

@@ -8,3 +8,4 @@ CRA_TARGETS:
       TF_VAR_resource_group_name: "test"
       TF_VAR_existing_kms_instance_crn: "crn:v1:bluemix:public:hs-crypto:us-south:a/abac0df06b644a9cabc6e44f55b3880e:e6dce284-e80f-46e1-a3c1-830f7adff7a9::"
       TF_VAR_kms_endpoint_url: "https://api.private.us-south.hs-crypto.cloud.ibm.com:8992"
+      TF_VAR_cross_region_location: "us"

cra-config.yaml

@@ -1,3 +1,10 @@
 {
-    "scc_rules": []
+    "scc_rules": [
+        {
+            "scc_rule_id": "rule-8cbd597c-7471-42bd-9c88-36b2696456e9",
+            "description": "Check whether Cloud Object Storage network access is restricted to a specific IP range",
+            "ignore_reason": "Tracked at https://github.com/terraform-ibm-modules/terraform-ibm-event-notifications/issues/161",
+            "is_valid": true
+        }
+    ]
 }

cra-tf-validate-ignore-rules.json

@@ -15,11 +15,12 @@ module "resource_group" {
 ##############################################################################
 
 module "event_notification" {
-  source            = "../../"
-  resource_group_id = module.resource_group.resource_group_id
-  name              = "${var.prefix}-en"
-  tags              = var.resource_tags
-  plan              = "lite"
-  service_endpoints = "public"
-  region            = var.region
+  source                  = "../../"
+  resource_group_id       = module.resource_group.resource_group_id
+  name                    = "${var.prefix}-en"
+  tags                    = var.resource_tags
+  plan                    = "lite"
+  service_endpoints       = "public"
+  region                  = var.region
+  cos_integration_enabled = false
 }

examples/basic/main.tf

@@ -6,7 +6,7 @@ terraform {
   required_providers {
     ibm = {
       source  = "IBM-Cloud/ibm"
-      version = "1.56.1"
+      version = "1.63.0"
     }
   }
 }

examples/basic/version.tf

@@ -1,10 +1,11 @@
 # Complete example with BYOK encryption and CBR rules
 
-An end-to-end example that does the following:
+An end-to-end example that creates the following infrastructure:
 
-- Create a new resource group if one is not passed in.
-- Create Key Protect instance with root key.
-- Create a new Event Notification instance with BYOK encryption.
-- Create a Virtual Private Cloud (VPC).
-- Create a context-based restriction (CBR) rule to only allow Event Notification to be accessible from within the VPC.
-- Create a service credentials for the Event Notification instance.
+- A resource group, if one is not passed in.
+- A Key Protect instance with a root key.
+- An Event Notifications instance with bring-your-own-key encryption.
+- An IBM Cloud Object Storage service instance and bucket to collect events that fail delivery.
+- A Virtual Private Cloud (VPC).
+- A context-based restriction (CBR) rule to allow Event Notifications to be accessible only from within the VPC.
+- Service credentials for the Event Notifications instance.

examples/complete/README.md

@@ -18,6 +18,7 @@ locals {
   key_ring_name = "en-key-ring"
   key_name      = "${var.prefix}-en"
 }
+
 module "key_protect_all_inclusive" {
   source                    = "terraform-ibm-modules/key-protect-all-inclusive/ibm"
   version                   = "4.8.5"
@@ -33,6 +34,26 @@ module "key_protect_all_inclusive" {
   }]
 }
 
+##############################################################################
+# Create Cloud Object Storage instance and a bucket
+##############################################################################
+
+locals {
+  bucket_name = "${var.prefix}-bucket"
+}
+
+module "cos" {
+  source                 = "terraform-ibm-modules/cos/ibm"
+  version                = "7.5.0"
+  resource_group_id      = module.resource_group.resource_group_id
+  region                 = var.region
+  cos_instance_name      = "${var.prefix}-cos"
+  cos_tags               = var.resource_tags
+  bucket_name            = local.bucket_name
+  retention_enabled      = false # disable retention for test environments - enable for stage/prod
+  kms_encryption_enabled = false
+}
+
 ##############################################################################
 # Get Cloud Account ID
 ##############################################################################
@@ -85,6 +106,12 @@ module "event_notification" {
   service_endpoints         = "public"
   service_credential_names  = var.service_credential_names
   region                    = var.region
+  # COS Related
+  cos_integration_enabled = true
+  cos_destination_name    = module.cos.cos_instance_name
+  cos_bucket_name         = module.cos.bucket_name
+  cos_instance_id         = module.cos.cos_instance_guid
+  cos_endpoint            = "https://${module.cos.s3_endpoint_public}"
   cbr_rules = [
     {
       description      = "${var.prefix}-event notification access only from vpc"

examples/complete/main.tf

@@ -6,7 +6,7 @@ terraform {
   required_providers {
     ibm = {
       source  = "IBM-Cloud/ibm"
-      version = ">= 1.49.0, < 2.0.0"
+      version = ">= 1.63.0, < 2.0.0"
     }
   }
 }

examples/complete/version.tf

@@ -50,12 +50,49 @@ module "cbr_zone" {
   }]
 }
 
+##############################################################################
+# Create COS Instance
+##############################################################################
+
+locals {
+  bucket_name       = "cos-bucket"
+  kms_instance_guid = element(split(":", var.existing_kms_instance_crn), length(split(":", var.existing_kms_instance_crn)) - 3)
+  root_key_id       = element(split(":", var.root_key_crn), length(split(":", var.root_key_crn)) - 1)
+}
+
+module "cos" {
+  source              = "terraform-ibm-modules/cos/ibm//modules/fscloud"
+  version             = "7.5.0"
+  resource_group_id   = module.resource_group.resource_group_id
+  create_cos_instance = true
+  create_resource_key = false
+  cos_instance_name   = "${var.prefix}-cos"
+  cos_plan            = "standard"
+  bucket_configs = [{
+    access_tags                   = []
+    add_bucket_name_suffix        = true
+    bucket_name                   = local.bucket_name
+    kms_encryption_enabled        = true
+    kms_guid                      = local.kms_instance_guid
+    kms_key_crn                   = var.root_key_crn
+    skip_iam_authorization_policy = false
+    management_endpoint_type      = "public"
+    storage_class                 = "smart"
+    region_location               = var.region
+    force_delete                  = true
+  }]
+}
+
+##############################################################################
+# Create Event Notifications Instance
+##############################################################################
+
 module "event_notification" {
   source                    = "../../modules/fscloud"
   resource_group_id         = module.resource_group.resource_group_id
   name                      = "${var.prefix}-en-fs"
   existing_kms_instance_crn = var.existing_kms_instance_crn
-  root_key_id               = var.root_key_id
+  root_key_id               = local.root_key_id
   kms_endpoint_url          = var.kms_endpoint_url
   tags                      = var.resource_tags
 
@@ -73,6 +110,11 @@ module "event_notification" {
     "en_custom_email_status_reporter" : "Custom Email Status Reporter",
   }
   region = var.region
+  # COS Related
+  cos_bucket_name         = module.cos.buckets[local.bucket_name].bucket_name
+  cos_instance_id         = module.cos.cos_instance_guid
+  skip_en_cos_auth_policy = false
+  cos_endpoint            = "https://${module.cos.buckets[local.bucket_name].s3_endpoint_private}"
   cbr_rules = [
     {
       description      = "${var.prefix}-event notification access only from vpc"