add tests for crr existing objects

Issue : ZENKO-5030

Author: SylvainSenechal

.github/scripts/end2end/configure-e2e.sh

@@ -96,6 +96,7 @@ kubectl run ${POD_NAME} \
   --env=AWS_SECRET_KEY=${AWS_SECRET_KEY} \
   --env=AWS_ENDPOINT=${AWS_ENDPOINT} \
   --env=AWS_FAIL_BUCKET_NAME=${AWS_FAIL_BUCKET_NAME} \
+  --env=AWS_REPLICATION_CTST_BUCKET_NAME=${AWS_REPLICATION_CTST_BUCKET_NAME} \
   --env=AZURE_BACKEND_DESTINATION_LOCATION=${AZURE_BACKEND_DESTINATION_LOCATION} \
   --env=AZURE_BACKEND_ENDPOINT=${AZURE_BACKEND_ENDPOINT} \
   --env=AZURE_BACKEND_QUEUE_ENDPOINT=${AZURE_BACKEND_QUEUE_ENDPOINT} \

.github/scripts/end2end/patch-coredns.sh

@@ -14,6 +14,7 @@ corefile="
     rewrite name exact ci-zenko-aws-crr-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact ci-zenko-aws-fail-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact ci-zenko-aws-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact ci-zenko-aws-replication-ctst-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact blob.azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local

.github/scripts/end2end/run-e2e-ctst.sh

@@ -78,6 +78,20 @@ SORBETD_RESTORE_TIMEOUT=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.so
 UTILIZATION_SERVICE_HOST=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.scuba.api.ingress.hostname}')
 UTILIZATION_SERVICE_PORT="80"
 
+MONGODB_REPLICASET=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.replicaSetHosts)
+MONGODB_AUTH_USERNAME=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.authCredentials.username)
+MONGODB_AUTH_PASSWORD=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.authCredentials.password)
+MONGODB_DATABASE=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.database)
+
+S3_UTILS_TAG=$(yq eval ".s3utils.tag" ../../../solution/deps.yaml)
+
+LOCATION_CONFIGS=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.locationConfig\.json}' | base64 -di | jq -r .)
+AWS_REPLICATION_ENDPOINT="http://$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.awsEndpoint')"
+AWS_REPLICATION_ACCESS_KEY=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.credentials.accessKey')
+AWS_REPLICATION_SECRET_KEY=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.credentials.secretKey')
+AWS_REPLICATION_LOCATION_TYPE=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].type')
+AWS_REPLICATION_REGION=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.region')
+
 # Setting CTST world params
 WORLD_PARAMETERS="$(jq -c <<EOF
 {
@@ -128,7 +142,19 @@ WORLD_PARAMETERS="$(jq -c <<EOF
   "DRAdminAccessKey":"${DR_ADMIN_ACCESS_KEY_ID}",
   "DRAdminSecretKey":"${DR_ADMIN_SECRET_ACCESS_KEY}",
   "UtilizationServiceHost":"${UTILIZATION_SERVICE_HOST}",
-  "UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}"
+  "UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}",
+  "AwsBackendDestinationReplicationLocation":"${AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION}",
+  "AwsReplicationBucketName":"${AWS_REPLICATION_CTST_BUCKET_NAME}",
+  "AwsReplicationEndpoint":"${AWS_REPLICATION_ENDPOINT}",
+  "AwsReplicationAccessKey":"${AWS_REPLICATION_ACCESS_KEY}",
+  "AwsReplicationSecretKey":"${AWS_REPLICATION_SECRET_KEY}",
+  "AwsReplicationLocationType":"${AWS_REPLICATION_LOCATION_TYPE}",
+  "AwsReplicationRegion":"${AWS_REPLICATION_REGION}",
+  "MongodbReplicaSet":"${MONGODB_REPLICASET}",
+  "MongodbAuthUsername":"${MONGODB_AUTH_USERNAME}",
+  "MongodbAuthPassword":"${MONGODB_AUTH_PASSWORD}",
+  "MongodbDatabase":"${MONGODB_DATABASE}",
+  "S3UtilsTag":"${S3_UTILS_TAG}"
 }
 EOF
 )"

.github/workflows/end2end.yaml

@@ -57,6 +57,7 @@ env:
   AWS_BACKEND_SOURCE_LOCATION: awsbackend
   AWS_BACKEND_DESTINATION_LOCATION: awsbackendmismatch
   AWS_BACKEND_DESTINATION_FAIL_LOCATION: awsbackendfail
+  AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION: awsbackendreplicationctst
   GCP_BACKEND_DESTINATION_LOCATION: gcpbackendmismatch
   AZURE_BACKEND_DESTINATION_LOCATION: azurebackendmismatch
   COLD_BACKEND_DESTINATION_LOCATION: e2e-cold
@@ -66,6 +67,7 @@ env:
   AWS_BUCKET_NAME: ci-zenko-aws-target-bucket
   AWS_CRR_BUCKET_NAME: ci-zenko-aws-crr-target-bucket
   AWS_FAIL_BUCKET_NAME: ci-zenko-aws-fail-target-bucket
+  AWS_REPLICATION_CTST_BUCKET_NAME: ci-zenko-aws-replication-ctst-bucket
   AZURE_CRR_BUCKET_NAME: ci-zenko-azure-crr-target-bucket
   AZURE_ARCHIVE_BUCKET_NAME: ci-zenko-azure-archive-target-bucket
   AZURE_ARCHIVE_BUCKET_NAME_2: ci-zenko-azure-archive-target-bucket-2

tests/ctst/features/replicatateExistingObjects.feature

@@ -0,0 +1,18 @@
+Feature: Replication
+    This feature tests replication using the script crrExistingObjects.js from S3utils,
+    that replicates objects created before the replication was enabled on source the bucket.
+
+    @2.7.0
+    @PreMerge
+    @ReplicationTest
+    Scenario Outline: Replicate objects created before creating the replication rule
+        Given an existing bucket "<sourceBucket>" "with" versioning, "without" ObjectLock "without" retention mode
+        And an object "<sourceObject>" that "exists"
+        And a replication configuration from "<sourceBucket>" bucket to "<replicationLocation>" location
+        When I run the job to replicate existing objects with status "<sourceObjStatus>" from bucket "<sourceBucket>" to location "<replicationLocation>"
+        Then the object "<sourceObject>" should be replicated within 60 seconds from "<sourceBucket>" bucket
+        And the replicated object "<sourceObject>" from "<sourceBucket>" bucket to "<replicationBucket>" bucket with location "<replicationLocation>" should be the same as its source object
+
+        Examples:
+            | sourceBucket  | sourceObjStatus |                    replicationBucket |    sourceObject |       replicationLocation |
+            | source-bucket |             NEW | ci-zenko-aws-replication-ctst-bucket | source-object-1 | awsbackendreplicationctst |

tests/ctst/steps/utils/utils.ts

@@ -13,7 +13,10 @@ import { extractPropertyFromResults, s3FunctionExtraParams, safeJsonParse } from
 import Zenko from 'world/Zenko';
 import assert from 'assert';
 import constants from 'common/constants';
+<<<<<<< HEAD
 import { getLocationConfigs } from './kubernetes';
+=======
+>>>>>>> c06fc489 (add tests for crr existing objects)
 import { S3Client } from '@aws-sdk/client-s3';
 
 enum AuthorizationType {
@@ -427,6 +430,11 @@ async function putBucketReplication(
     srcBucket: string,
     replicationLocation: string
 ) {
+<<<<<<< HEAD
+=======
+    const { storageClass, destinationBucket } = getReplicationLocationConfig(this, replicationLocation);
+
+>>>>>>> c06fc489 (add tests for crr existing objects)
     this.resetCommand();
     this.addCommandParameter({ bucket: srcBucket });
     this.addCommandParameter({
@@ -436,11 +444,16 @@ async function putBucketReplication(
                 {
                     Prefix: '',
                     Destination: {
+<<<<<<< HEAD
                         // Source bucket is not a typo :
                         // eslint-disable-next-line max-len
                         // https://documentation.scality.com/Artesca/4.0.1/data_management/bucket_operations/replication_workflow/create_a_replication_workflow.html
                         Bucket: `arn:aws:s3:::${srcBucket}`,
                         StorageClass: replicationLocation,
+=======
+                        Bucket: `arn:aws:s3:::${destinationBucket}`,
+                        StorageClass: storageClass,
+>>>>>>> c06fc489 (add tests for crr existing objects)
                     },
                     Status: 'Enabled',
                 },
@@ -453,7 +466,11 @@ async function putBucketReplication(
     if (res.err) {
         this.logger.error('Failed to put bucket replication', {
             srcBucket,
+<<<<<<< HEAD
             replicationLocation,
+=======
+            storageClass,
+>>>>>>> c06fc489 (add tests for crr existing objects)
             error: res.err,
         });
         throw new Error(`Failed to put bucket replication, err : ${res.err}`);

tests/ctst/world/Zenko.ts

@@ -89,6 +89,18 @@ export interface ZenkoWorldParameters extends ClientOptions {
     SorbetdRestoreTimeout: string;
     UtilizationServiceHost: string;
     UtilizationServicePort: string;
+    AwsBackendDestinationReplicationLocation: string;
+    AwsReplicationBucketName: string;
+    AwsReplicationEndpoint: string;
+    AwsReplicationAccessKey: string;
+    AwsReplicationSecretKey: string;
+    AwsReplicationLocationType: string;
+    AwsReplicationRegion: string;
+    MongodbReplicaSet: string;
+    MongodbAuthUsername: string;
+    MongodbAuthPassword: string;
+    MongodbDatabase: string;
+    S3UtilsTag: string;
     [key: string]: unknown;
 }
 

tests/zenko_tests/create_buckets.py

@@ -116,8 +116,9 @@ def create_aws_buckets():
     VERIFY_CERTIFICATES = get_env('VERIFY_CERTIFICATES', False)
     AWS_ACCESS_KEY = get_env('AWS_ACCESS_KEY')
     AWS_SECRET_KEY = get_env('AWS_SECRET_KEY')
-    AWS_FAIL_BUCKET_NAME = get_env('AWS_FAIL_BUCKET_NAME')
     AWS_ENDPOINT = get_env('AWS_ENDPOINT')
+    AWS_FAIL_BUCKET_NAME = get_env('AWS_FAIL_BUCKET_NAME')
+    AWS_REPLICATION_CTST_BUCKET_NAME = get_env('AWS_REPLICATION_CTST_BUCKET_NAME')
 
     s3c = Session(aws_access_key_id=AWS_ACCESS_KEY,
             aws_secret_access_key=AWS_SECRET_KEY)
@@ -127,7 +128,9 @@ def create_aws_buckets():
     ## Creating AWS buckets
     _log.info('Creating AWS buckets...')
     bucket_safe_create(aws_s3c_client.Bucket(AWS_FAIL_BUCKET_NAME))
+    bucket_safe_create(aws_s3c_client.Bucket(AWS_REPLICATION_CTST_BUCKET_NAME))
     aws_s3c_client.Bucket(AWS_FAIL_BUCKET_NAME).Versioning().enable()
+    aws_s3c_client.Bucket(AWS_REPLICATION_CTST_BUCKET_NAME).Versioning().enable()
 
 def create_azure_containers():
     AZURE_BACKEND_ENDPOINT = get_env("AZURE_BACKEND_ENDPOINT")

tests/zenko_tests/e2e-config.yaml.template

@@ -30,6 +30,15 @@ locations:
       secretKey: "${AWS_SECRET_KEY}"
       bucketMatch: yes
       repoId: []
+  - name: "${AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION}"
+    locationType: "location-aws-s3-v1"
+    details:
+      bucketName: "${AWS_REPLICATION_CTST_BUCKET_NAME}"
+      endpoint: "${AWS_ENDPOINT}"
+      accessKey: "${AWS_ACCESS_KEY}"
+      secretKey: "${AWS_SECRET_KEY}"
+      bucketMatch: no
+      repoId: []
   - name: ${AZURE_BACKEND_DESTINATION_LOCATION}
     locationType: location-azure-v1
     details: