Skip to content

Commit 24e59aa

Browse files
ctst test replicate object that failed to replicate
Issue: ZENKO-5050
1 parent 7325ac2 commit 24e59aa

File tree

12 files changed

+100
-97
lines changed

12 files changed

+100
-97
lines changed

.github/scripts/end2end/configure-e2e.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ kubectl run ${POD_NAME} \
9696
--env=AWS_SECRET_KEY=${AWS_SECRET_KEY} \
9797
--env=AWS_ENDPOINT=${AWS_ENDPOINT} \
9898
--env=AWS_FAIL_BUCKET_NAME=${AWS_FAIL_BUCKET_NAME} \
99-
--env=AWS_REPLICATION_CTST_BUCKET_NAME=${AWS_REPLICATION_CTST_BUCKET_NAME} \
99+
--env=AWS_REPLICATION_FAIL_CTST_BUCKET_NAME=${AWS_REPLICATION_FAIL_CTST_BUCKET_NAME} \
100100
--env=AZURE_BACKEND_DESTINATION_LOCATION=${AZURE_BACKEND_DESTINATION_LOCATION} \
101101
--env=AZURE_BACKEND_ENDPOINT=${AZURE_BACKEND_ENDPOINT} \
102102
--env=AZURE_BACKEND_QUEUE_ENDPOINT=${AZURE_BACKEND_QUEUE_ENDPOINT} \

.github/scripts/end2end/patch-coredns.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ corefile="
1414
rewrite name exact ci-zenko-aws-crr-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1515
rewrite name exact ci-zenko-aws-fail-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1616
rewrite name exact ci-zenko-aws-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
17-
rewrite name exact ci-zenko-aws-replication-ctst-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
17+
rewrite name exact ci-zenko-aws-replication-fail-ctst-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1818
rewrite name exact aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1919
rewrite name exact azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
2020
rewrite name exact blob.azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local

.github/scripts/end2end/run-e2e-ctst.sh

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -78,20 +78,6 @@ SORBETD_RESTORE_TIMEOUT=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.so
7878
UTILIZATION_SERVICE_HOST=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.scuba.api.ingress.hostname}')
7979
UTILIZATION_SERVICE_PORT="80"
8080

81-
MONGODB_REPLICASET=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.replicaSetHosts)
82-
MONGODB_AUTH_USERNAME=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.authCredentials.username)
83-
MONGODB_AUTH_PASSWORD=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.authCredentials.password)
84-
MONGODB_DATABASE=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.database)
85-
86-
S3_UTILS_TAG=$(yq eval ".s3utils.tag" ../../../solution/deps.yaml)
87-
88-
LOCATION_CONFIGS=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.locationConfig\.json}' | base64 -di | jq -r .)
89-
AWS_REPLICATION_ENDPOINT="http://$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.awsEndpoint')"
90-
AWS_REPLICATION_ACCESS_KEY=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.credentials.accessKey')
91-
AWS_REPLICATION_SECRET_KEY=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.credentials.secretKey')
92-
AWS_REPLICATION_LOCATION_TYPE=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].type')
93-
AWS_REPLICATION_REGION=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.region')
94-
9581
# Setting CTST world params
9682
WORLD_PARAMETERS="$(jq -c <<EOF
9783
{
@@ -142,19 +128,7 @@ WORLD_PARAMETERS="$(jq -c <<EOF
142128
"DRAdminAccessKey":"${DR_ADMIN_ACCESS_KEY_ID}",
143129
"DRAdminSecretKey":"${DR_ADMIN_SECRET_ACCESS_KEY}",
144130
"UtilizationServiceHost":"${UTILIZATION_SERVICE_HOST}",
145-
"UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}",
146-
"AwsBackendDestinationReplicationLocation":"${AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION}",
147-
"AwsReplicationBucketName":"${AWS_REPLICATION_CTST_BUCKET_NAME}",
148-
"AwsReplicationEndpoint":"${AWS_REPLICATION_ENDPOINT}",
149-
"AwsReplicationAccessKey":"${AWS_REPLICATION_ACCESS_KEY}",
150-
"AwsReplicationSecretKey":"${AWS_REPLICATION_SECRET_KEY}",
151-
"AwsReplicationLocationType":"${AWS_REPLICATION_LOCATION_TYPE}",
152-
"AwsReplicationRegion":"${AWS_REPLICATION_REGION}",
153-
"MongodbReplicaSet":"${MONGODB_REPLICASET}",
154-
"MongodbAuthUsername":"${MONGODB_AUTH_USERNAME}",
155-
"MongodbAuthPassword":"${MONGODB_AUTH_PASSWORD}",
156-
"MongodbDatabase":"${MONGODB_DATABASE}",
157-
"S3UtilsTag":"${S3_UTILS_TAG}"
131+
"UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}"
158132
}
159133
EOF
160134
)"

.github/workflows/end2end.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ env:
5757
AWS_BACKEND_SOURCE_LOCATION: awsbackend
5858
AWS_BACKEND_DESTINATION_LOCATION: awsbackendmismatch
5959
AWS_BACKEND_DESTINATION_FAIL_LOCATION: awsbackendfail
60-
AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION: awsbackendreplicationctst
60+
AWS_BACKEND_DESTINATION_REPLICATION_FAIL_CTST_LOCATION: awsbackendreplicationctstfail
6161
GCP_BACKEND_DESTINATION_LOCATION: gcpbackendmismatch
6262
AZURE_BACKEND_DESTINATION_LOCATION: azurebackendmismatch
6363
COLD_BACKEND_DESTINATION_LOCATION: e2e-cold
@@ -67,7 +67,7 @@ env:
6767
AWS_BUCKET_NAME: ci-zenko-aws-target-bucket
6868
AWS_CRR_BUCKET_NAME: ci-zenko-aws-crr-target-bucket
6969
AWS_FAIL_BUCKET_NAME: ci-zenko-aws-fail-target-bucket
70-
AWS_REPLICATION_CTST_BUCKET_NAME: ci-zenko-aws-replication-ctst-bucket
70+
AWS_REPLICATION_FAIL_CTST_BUCKET_NAME: ci-zenko-aws-replication-fail-ctst-bucket
7171
AZURE_CRR_BUCKET_NAME: ci-zenko-azure-crr-target-bucket
7272
AZURE_ARCHIVE_BUCKET_NAME: ci-zenko-azure-archive-target-bucket
7373
AZURE_ARCHIVE_BUCKET_NAME_2: ci-zenko-azure-archive-target-bucket-2

tests/ctst/common/hooks.ts

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
After,
44
setParallelCanAssign,
55
parallelCanAssignHelpers,
6+
ITestCaseHookParameter,
67
} from '@cucumber/cucumber';
78
import Zenko from '../world/Zenko';
89
import { CacheHelper, Identity } from 'cli-testing';
@@ -19,13 +20,21 @@ import {
1920
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
2021

2122
const { atMostOnePicklePerTag } = parallelCanAssignHelpers;
22-
const noParallelRun = atMostOnePicklePerTag(['@AfterAll', '@PRA', '@ColdStorage']);
23+
const noParallelRun = atMostOnePicklePerTag([
24+
'@AfterAll',
25+
'@PRA',
26+
'@ColdStorage',
27+
'@Lockawsbackendreplicationctstfail'
28+
]);
2329

2430
setParallelCanAssign(noParallelRun);
2531

26-
Before(async function (this: Zenko) {
32+
Before(async function (this: Zenko, scenario: ITestCaseHookParameter) {
2733
this.resetSaved();
2834
Identity.resetIdentity();
35+
// Store scenario tags for access in step definitions
36+
const scenarioTags = scenario.pickle.tags?.map(tag => tag.name) || [];
37+
this.addToSaved('scenarioTags', scenarioTags);
2938
await Zenko.init(this.parameters);
3039
});
3140

tests/ctst/features/crrReplicationS3utils.feature

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,21 @@ Feature: Replication
99
Given an existing bucket "source-bucket" "with" versioning, "without" ObjectLock "without" retention mode
1010
And an object "source-object-1" that "exists"
1111
And a replication configuration to "awsbackendmismatch" location
12-
When I run the job to replicate existing objects with status "NEW"
13-
Then the object should eventually be replicated
12+
When the job to replicate existing objects with status "NEW" is executed
13+
Then the object should eventually "be" replicated
14+
And the replicated object should be the same as the source object
15+
16+
@2.12.0
17+
@PreMerge
18+
@ReplicationTest
19+
@Lockawsbackendreplicationctstfail
20+
Scenario Outline: Re-replicate objects that failed to replicate
21+
Given an existing bucket "source-bucket2" "with" versioning, "without" ObjectLock "without" retention mode
22+
And a replication configuration to "awsbackendreplicationctstfail" location
23+
And a deleted destination bucket on that location
24+
And an object "source-object-2" that "exists"
25+
Then the object should eventually "fail to be" replicated
26+
When the destination bucket on the location is created again
27+
And the job to replicate existing objects with status "FAILED" is executed
28+
Then the object should eventually "be" replicated
1429
And the replicated object should be the same as the source object

tests/ctst/features/replicatateExistingObjects.feature

Lines changed: 0 additions & 18 deletions
This file was deleted.

tests/ctst/steps/replication.ts

Lines changed: 60 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
1-
import { When, Then } from '@cucumber/cucumber';
1+
import { Given, When, Then } from '@cucumber/cucumber';
22
import Zenko from '../world/Zenko';
33
import { createAndRunPod, getMongoDBConfig, getZenkoVersion } from 'steps/utils/kubernetes';
44
import assert from 'assert';
5-
import { GetObjectCommand } from '@aws-sdk/client-s3';
5+
import {
6+
GetObjectCommand,
7+
DeleteBucketCommand,
8+
CreateBucketCommand,
9+
PutBucketVersioningCommand
10+
} from '@aws-sdk/client-s3';
611
import { v4 as uuidv4 } from 'uuid';
712
import { getObject, headObject, getReplicationLocationConfig } from 'steps/utils/utils';
813
import { safeJsonParse } from 'common/utils';
914

10-
When('I run the job to replicate existing objects with status {string}',
15+
When('the job to replicate existing objects with status {string} is executed',
1116
{ timeout: 600000 },
1217
async function (
1318
this: Zenko,
@@ -83,12 +88,12 @@ When('I run the job to replicate existing objects with status {string}',
8388
await createAndRunPod(this, podManifest);
8489
});
8590

86-
Then('the object should eventually be replicated',
87-
async function (this: Zenko) {
91+
Then('the object should eventually {string} replicated', { timeout: 360_000 },
92+
async function (this: Zenko, replicate: 'be' | 'fail to be') {
8893
const objectName = this.getSaved<string>('objectName');
8994
const bucketSource = this.getSaved<string>('bucketName');
9095
const startTime = Date.now();
91-
const replicationTimeoutMs = 90_000;
96+
const replicationTimeoutMs = 300_000;
9297
while (Date.now() - startTime < replicationTimeoutMs) {
9398
await new Promise(resolve => setTimeout(resolve, 3000));
9499

@@ -105,15 +110,27 @@ Then('the object should eventually be replicated',
105110
}>(response.stdout || '{}');
106111
assert(parsed.ok);
107112
const replicationStatus = parsed.result?.ReplicationStatus;
108-
assert.notStrictEqual(replicationStatus, 'FAILED', `replication failed for object ${objectName}`);
109-
if (replicationStatus === 'COMPLETED') {
110-
return;
113+
114+
if (replicate === 'be') {
115+
assert.notStrictEqual(replicationStatus, 'FAILED', `replication failed for object ${objectName}`);
116+
if (replicationStatus === 'COMPLETED') {
117+
return;
118+
}
119+
} else if (replicate === 'fail to be') {
120+
assert.notStrictEqual(
121+
replicationStatus,
122+
'COMPLETED',
123+
`expected replication to fail for object ${objectName}`
124+
);
125+
if (replicationStatus === 'FAILED') {
126+
return;
127+
}
111128
}
112129
if (replicationStatus === 'PENDING' || replicationStatus === 'PROCESSING') {
113130
continue;
114131
}
115132
}
116-
assert.fail(`Timeout: Object '${objectName}' was not replicated successfully until timeout`);
133+
assert.fail(`Timeout: Object '${objectName}' is still pending/processing after timeout`);
117134
});
118135

119136
Then(
@@ -173,3 +190,36 @@ Then(
173190
'REPLICA'
174191
);
175192
});
193+
194+
Given('a deleted destination bucket on that location', async function (this: Zenko) {
195+
const replicationLocation = this.getSaved<string>('replicationLocation');
196+
const scenarioTags = this.getSaved<string[]>('scenarioTags') || [];
197+
const hasTestLock = scenarioTags.includes(`@Lock${replicationLocation}`);
198+
assert.strictEqual(
199+
hasTestLock, true,
200+
'This step can only be run when the tag @Lock$replicationLocation is configured'
201+
);
202+
203+
const { destinationBucket, awsS3Client } =
204+
await getReplicationLocationConfig(this, replicationLocation);
205+
const command = new DeleteBucketCommand({
206+
Bucket: destinationBucket,
207+
});
208+
await awsS3Client.send(command);
209+
});
210+
211+
When('the destination bucket on the location is created again', async function (this: Zenko) {
212+
const { destinationBucket, awsS3Client } =
213+
await getReplicationLocationConfig(this, this.getSaved<string>('replicationLocation'));
214+
const command = new CreateBucketCommand({
215+
Bucket: destinationBucket,
216+
});
217+
await awsS3Client.send(command);
218+
const versioningCommand = new PutBucketVersioningCommand({
219+
Bucket: destinationBucket,
220+
VersioningConfiguration: {
221+
Status: 'Enabled',
222+
},
223+
});
224+
await awsS3Client.send(versioningCommand);
225+
});

tests/ctst/steps/utils/utils.ts

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,7 @@ import { extractPropertyFromResults, s3FunctionExtraParams, safeJsonParse } from
1313
import Zenko from 'world/Zenko';
1414
import assert from 'assert';
1515
import constants from 'common/constants';
16-
<<<<<<< HEAD
1716
import { getLocationConfigs } from './kubernetes';
18-
=======
19-
>>>>>>> c06fc489 (add tests for crr existing objects)
2017
import { S3Client } from '@aws-sdk/client-s3';
2118

2219
enum AuthorizationType {
@@ -430,11 +427,6 @@ async function putBucketReplication(
430427
srcBucket: string,
431428
replicationLocation: string
432429
) {
433-
<<<<<<< HEAD
434-
=======
435-
const { storageClass, destinationBucket } = getReplicationLocationConfig(this, replicationLocation);
436-
437-
>>>>>>> c06fc489 (add tests for crr existing objects)
438430
this.resetCommand();
439431
this.addCommandParameter({ bucket: srcBucket });
440432
this.addCommandParameter({
@@ -444,16 +436,11 @@ async function putBucketReplication(
444436
{
445437
Prefix: '',
446438
Destination: {
447-
<<<<<<< HEAD
448439
// Source bucket is not a typo :
449440
// eslint-disable-next-line max-len
450441
// https://documentation.scality.com/Artesca/4.0.1/data_management/bucket_operations/replication_workflow/create_a_replication_workflow.html
451442
Bucket: `arn:aws:s3:::${srcBucket}`,
452443
StorageClass: replicationLocation,
453-
=======
454-
Bucket: `arn:aws:s3:::${destinationBucket}`,
455-
StorageClass: storageClass,
456-
>>>>>>> c06fc489 (add tests for crr existing objects)
457444
},
458445
Status: 'Enabled',
459446
},
@@ -466,11 +453,7 @@ async function putBucketReplication(
466453
if (res.err) {
467454
this.logger.error('Failed to put bucket replication', {
468455
srcBucket,
469-
<<<<<<< HEAD
470456
replicationLocation,
471-
=======
472-
storageClass,
473-
>>>>>>> c06fc489 (add tests for crr existing objects)
474457
error: res.err,
475458
});
476459
throw new Error(`Failed to put bucket replication, err : ${res.err}`);

tests/ctst/world/Zenko.ts

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -89,18 +89,6 @@ export interface ZenkoWorldParameters extends ClientOptions {
8989
SorbetdRestoreTimeout: string;
9090
UtilizationServiceHost: string;
9191
UtilizationServicePort: string;
92-
AwsBackendDestinationReplicationLocation: string;
93-
AwsReplicationBucketName: string;
94-
AwsReplicationEndpoint: string;
95-
AwsReplicationAccessKey: string;
96-
AwsReplicationSecretKey: string;
97-
AwsReplicationLocationType: string;
98-
AwsReplicationRegion: string;
99-
MongodbReplicaSet: string;
100-
MongodbAuthUsername: string;
101-
MongodbAuthPassword: string;
102-
MongodbDatabase: string;
103-
S3UtilsTag: string;
10492
[key: string]: unknown;
10593
}
10694

0 commit comments

Comments
 (0)