Skip to content

Commit 0017638

Browse files
add tests for crr existing objects
Issue : ZENKO-5030
1 parent b4586fe commit 0017638

File tree

15 files changed

+536
-3
lines changed

15 files changed

+536
-3
lines changed

.github/scripts/end2end/configure-e2e.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ kubectl run ${POD_NAME} \
9696
--env=AWS_SECRET_KEY=${AWS_SECRET_KEY} \
9797
--env=AWS_ENDPOINT=${AWS_ENDPOINT} \
9898
--env=AWS_FAIL_BUCKET_NAME=${AWS_FAIL_BUCKET_NAME} \
99+
--env=AWS_REPLICATION_CTST_BUCKET_NAME=${AWS_REPLICATION_CTST_BUCKET_NAME} \
99100
--env=AZURE_BACKEND_DESTINATION_LOCATION=${AZURE_BACKEND_DESTINATION_LOCATION} \
100101
--env=AZURE_BACKEND_ENDPOINT=${AZURE_BACKEND_ENDPOINT} \
101102
--env=AZURE_BACKEND_QUEUE_ENDPOINT=${AZURE_BACKEND_QUEUE_ENDPOINT} \

.github/scripts/end2end/patch-coredns.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ corefile="
1414
rewrite name exact ci-zenko-aws-crr-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1515
rewrite name exact ci-zenko-aws-fail-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1616
rewrite name exact ci-zenko-aws-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
17+
rewrite name exact ci-zenko-aws-replication-ctst-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1718
rewrite name exact aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1819
rewrite name exact azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
1920
rewrite name exact blob.azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local

.github/scripts/end2end/run-e2e-ctst.sh

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,20 @@ SORBETD_RESTORE_TIMEOUT=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.so
7878
UTILIZATION_SERVICE_HOST=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.scuba.api.ingress.hostname}')
7979
UTILIZATION_SERVICE_PORT="80"
8080

81+
MONGODB_REPLICASET=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.replicaSetHosts)
82+
MONGODB_AUTH_USERNAME=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.authCredentials.username)
83+
MONGODB_AUTH_PASSWORD=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.authCredentials.password)
84+
MONGODB_DATABASE=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .mongodb.database)
85+
86+
S3_UTILS_TAG=$(yq eval ".s3utils.tag" ../../../solution/deps.yaml)
87+
88+
LOCATION_CONFIGS=$(kubectl get secrets -l app.kubernetes.io/name=connector-cloudserver-config -o jsonpath='{.items[0].data.locationConfig\.json}' | base64 -di | jq -r .)
89+
AWS_REPLICATION_ENDPOINT="http://$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.awsEndpoint')"
90+
AWS_REPLICATION_ACCESS_KEY=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.credentials.accessKey')
91+
AWS_REPLICATION_SECRET_KEY=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.credentials.secretKey')
92+
AWS_REPLICATION_LOCATION_TYPE=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].type')
93+
AWS_REPLICATION_REGION=$(echo "$LOCATION_CONFIGS" | jq -r --arg loc "$AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION" '.[$loc].details.region')
94+
8195
# Setting CTST world params
8296
WORLD_PARAMETERS="$(jq -c <<EOF
8397
{
@@ -128,7 +142,19 @@ WORLD_PARAMETERS="$(jq -c <<EOF
128142
"DRAdminAccessKey":"${DR_ADMIN_ACCESS_KEY_ID}",
129143
"DRAdminSecretKey":"${DR_ADMIN_SECRET_ACCESS_KEY}",
130144
"UtilizationServiceHost":"${UTILIZATION_SERVICE_HOST}",
131-
"UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}"
145+
"UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}",
146+
"AwsBackendDestinationReplicationLocation":"${AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION}",
147+
"AwsReplicationBucketName":"${AWS_REPLICATION_CTST_BUCKET_NAME}",
148+
"AwsReplicationEndpoint":"${AWS_REPLICATION_ENDPOINT}",
149+
"AwsReplicationAccessKey":"${AWS_REPLICATION_ACCESS_KEY}",
150+
"AwsReplicationSecretKey":"${AWS_REPLICATION_SECRET_KEY}",
151+
"AwsReplicationLocationType":"${AWS_REPLICATION_LOCATION_TYPE}",
152+
"AwsReplicationRegion":"${AWS_REPLICATION_REGION}",
153+
"MongodbReplicaSet":"${MONGODB_REPLICASET}",
154+
"MongodbAuthUsername":"${MONGODB_AUTH_USERNAME}",
155+
"MongodbAuthPassword":"${MONGODB_AUTH_PASSWORD}",
156+
"MongodbDatabase":"${MONGODB_DATABASE}",
157+
"S3UtilsTag":"${S3_UTILS_TAG}"
132158
}
133159
EOF
134160
)"

.github/workflows/end2end.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ env:
5757
AWS_BACKEND_SOURCE_LOCATION: awsbackend
5858
AWS_BACKEND_DESTINATION_LOCATION: awsbackendmismatch
5959
AWS_BACKEND_DESTINATION_FAIL_LOCATION: awsbackendfail
60+
AWS_BACKEND_DESTINATION_REPLICATION_CTST_LOCATION: awsbackendreplicationctst
6061
GCP_BACKEND_DESTINATION_LOCATION: gcpbackendmismatch
6162
AZURE_BACKEND_DESTINATION_LOCATION: azurebackendmismatch
6263
COLD_BACKEND_DESTINATION_LOCATION: e2e-cold
@@ -66,6 +67,7 @@ env:
6667
AWS_BUCKET_NAME: ci-zenko-aws-target-bucket
6768
AWS_CRR_BUCKET_NAME: ci-zenko-aws-crr-target-bucket
6869
AWS_FAIL_BUCKET_NAME: ci-zenko-aws-fail-target-bucket
70+
AWS_REPLICATION_CTST_BUCKET_NAME: ci-zenko-aws-replication-ctst-bucket
6971
AZURE_CRR_BUCKET_NAME: ci-zenko-azure-crr-target-bucket
7072
AZURE_ARCHIVE_BUCKET_NAME: ci-zenko-azure-archive-target-bucket
7173
AZURE_ARCHIVE_BUCKET_NAME_2: ci-zenko-azure-archive-target-bucket-2

tests/ctst/common/common.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
verifyObjectLocation,
1616
restoreObject,
1717
addTransitionWorkflow,
18+
putBucketReplication,
1819
} from 'steps/utils/utils';
1920
import { ActionPermissionsType } from 'steps/bucket-policies/utils';
2021
import constants from './constants';
@@ -259,6 +260,12 @@ Given('a transition workflow to {string} location', async function (this: Zenko,
259260
await addTransitionWorkflow.call(this, location);
260261
});
261262

263+
Given('a replication configuration to {string} location',
264+
async function (this: Zenko, replicationLocation: string) {
265+
this.addToSaved('replicationLocation', replicationLocation);
266+
await putBucketReplication.call(this, this.getSaved<string>('bucketName'), replicationLocation);
267+
});
268+
262269
When('i restore object {string} for {int} days', async function (this: Zenko, objectName: string, days: number) {
263270
await restoreObject.call(this, objectName, days);
264271
});
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
Feature: Replication
2+
This feature tests replication using the script crrExistingObjects.js from S3utils,
3+
that replicates objects created before the replication was enabled on source the bucket.
4+
5+
@2.12.0
6+
@PreMerge
7+
@ReplicationTest
8+
Scenario Outline: Replicate objects created before creating the replication rule
9+
Given an existing bucket "source-bucket" "with" versioning, "without" ObjectLock "without" retention mode
10+
And an object "source-object-1" that "exists"
11+
And a replication configuration to "awsbackendmismatch" location
12+
When I run the job to replicate existing objects with status "NEW"
13+
Then the object should eventually be replicated
14+
And the replicated object should be the same as the source object
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
Feature: Replication
2+
This feature tests replication using the script crrExistingObjects.js from S3utils,
3+
that replicates objects created before the replication was enabled on source the bucket.
4+
5+
@2.7.0
6+
@PreMerge
7+
@ReplicationTest
8+
Scenario Outline: Replicate objects created before creating the replication rule
9+
Given an existing bucket "<sourceBucket>" "with" versioning, "without" ObjectLock "without" retention mode
10+
And an object "<sourceObject>" that "exists"
11+
And a replication configuration from "<sourceBucket>" bucket to "<replicationLocation>" location
12+
When I run the job to replicate existing objects with status "<sourceObjStatus>" from bucket "<sourceBucket>" to location "<replicationLocation>"
13+
Then the object "<sourceObject>" should be replicated within 60 seconds from "<sourceBucket>" bucket
14+
And the replicated object "<sourceObject>" from "<sourceBucket>" bucket to "<replicationBucket>" bucket with location "<replicationLocation>" should be the same as its source object
15+
16+
Examples:
17+
| sourceBucket | sourceObjStatus | replicationBucket | sourceObject | replicationLocation |
18+
| source-bucket | NEW | ci-zenko-aws-replication-ctst-bucket | source-object-1 | awsbackendreplicationctst |

tests/ctst/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
"prometheus-query": "^3.4.0",
1919
"proper-lockfile": "^4.1.2",
2020
"qs": "^6.13.0",
21-
"scubaclient": "git+https://github.com/scality/scubaclient#^1.1.2"
21+
"scubaclient": "git+https://github.com/scality/scubaclient#^1.1.2",
22+
"uuid": "^11.1.0"
2223
},
2324
"devDependencies": {
2425
"@aws-sdk/client-iam": "^3.582.0",

tests/ctst/steps/replication.ts

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import { When, Then } from '@cucumber/cucumber';
2+
import Zenko from '../world/Zenko';
3+
import { createAndRunPod, getMongoDBConfig, getZenkoVersion } from 'steps/utils/kubernetes';
4+
import assert from 'assert';
5+
import { GetObjectCommand } from '@aws-sdk/client-s3';
6+
import { Utils } from 'cli-testing';
7+
import { getObject, headObject, getReplicationLocationConfig } from 'steps/utils/utils';
8+
import { safeJsonParse } from 'common/utils';
9+
10+
When('I run the job to replicate existing objects with status {string}',
11+
{ timeout: 600000 },
12+
async function (
13+
this: Zenko,
14+
sourceObjectStatus: string,
15+
) {
16+
const sourceBucket = this.getSaved<string>('bucketName');
17+
const replicationLocation = this.getSaved<string>('replicationLocation');
18+
const { replicaSetHosts } = await getMongoDBConfig(this);
19+
const { locationType } = await getReplicationLocationConfig(this, replicationLocation);
20+
const zenkoVersion = await getZenkoVersion(this);
21+
const s3utilsVersion = zenkoVersion.spec.versions.s3utils;
22+
const podManifest = {
23+
apiVersion: 'v1',
24+
kind: 'Pod',
25+
metadata: {
26+
name: `s3utils-crr-existing-${Utils.randomString().toLowerCase()}`,
27+
namespace: 'default',
28+
labels: {
29+
app: 's3utils',
30+
script: 'crrExistingObjects.js'
31+
}
32+
},
33+
spec: {
34+
restartPolicy: 'Never',
35+
containers: [
36+
{
37+
name: 's3utils',
38+
image: `${s3utilsVersion.image}:${s3utilsVersion.tag}`,
39+
command: ['node'],
40+
args: ['crrExistingObjects.js', sourceBucket],
41+
env: [
42+
{
43+
name: 'MONGODB_REPLICASET',
44+
value: replicaSetHosts.join(',')
45+
},
46+
{
47+
name: 'MONGODB_AUTH_USERNAME',
48+
valueFrom: {
49+
secretKeyRef: {
50+
name: 'mongodb-db-creds',
51+
key: 'mongodb-username'
52+
}
53+
}
54+
},
55+
{
56+
name: 'MONGODB_AUTH_PASSWORD',
57+
valueFrom: {
58+
secretKeyRef: {
59+
name: 'mongodb-db-creds',
60+
key: 'mongodb-password'
61+
}
62+
}
63+
},
64+
{
65+
name: 'MONGODB_DATABASE',
66+
valueFrom: {
67+
secretKeyRef: {
68+
name: 'mongodb-db-creds',
69+
key: 'mongodb-database'
70+
}
71+
}
72+
},
73+
{ name: 'MONGODB_SHARD_COLLECTIONS', value: 'true' },
74+
{ name: 'STORAGE_TYPE', value: locationType },
75+
{ name: 'TARGET_REPLICATION_STATUS', value: sourceObjectStatus },
76+
{ name: 'SITE_NAME', value: replicationLocation },
77+
]
78+
}
79+
]
80+
}
81+
};
82+
83+
await createAndRunPod(this, podManifest);
84+
});
85+
86+
Then('the object should eventually be replicated',
87+
async function (this: Zenko) {
88+
const objectName = this.getSaved<string>('objectName');
89+
const bucketSource = this.getSaved<string>('bucketName');
90+
const startTime = Date.now();
91+
const replicationTimeoutMs = 90_000;
92+
while (Date.now() - startTime < replicationTimeoutMs) {
93+
await new Promise(resolve => setTimeout(resolve, 3000));
94+
95+
const response = await headObject(this, objectName, bucketSource);
96+
assert(response.stdout);
97+
assert.strictEqual(response.statusCode, 200, `failed to headobject, ${response.statusCode}`);
98+
const parsed = safeJsonParse<{
99+
ReplicationStatus?: string;
100+
LastModified?: string;
101+
ETag?: string;
102+
ContentLength?: number;
103+
VersionId?: string;
104+
Metadata?: Record<string, string>;
105+
}>(response.stdout || '{}');
106+
assert(parsed.ok);
107+
const replicationStatus = parsed.result?.ReplicationStatus;
108+
assert.notStrictEqual(replicationStatus, 'FAILED', `replication failed for object ${objectName}`);
109+
if (replicationStatus === 'COMPLETED') {
110+
return;
111+
}
112+
if (replicationStatus === 'PENDING' || replicationStatus === 'PROCESSING') {
113+
continue;
114+
}
115+
}
116+
assert.fail(`Timeout: Object '${objectName}' was not replicated successfully until timeout`);
117+
});
118+
119+
Then(
120+
'the replicated object should be the same as the source object',
121+
async function (
122+
this: Zenko,
123+
) {
124+
const objectName = this.getSaved<string>('objectName');
125+
const bucketSource = this.getSaved<string>('bucketName');
126+
const replicationLocation = this.getSaved<string>('replicationLocation');
127+
const { destinationBucket, bucketMatch, awsS3Client } =
128+
await getReplicationLocationConfig(this, replicationLocation);
129+
130+
// When bucketMatch is disabled on the destination bucket,
131+
// replicated objects are named sourceBucket/objectName
132+
let key = `${bucketSource}/${objectName}`;
133+
if (bucketMatch) {
134+
key = objectName;
135+
}
136+
137+
const command = new GetObjectCommand({
138+
Bucket: destinationBucket,
139+
Key: key,
140+
});
141+
const replicaObj = await awsS3Client.send(command);
142+
const sourceResponse = await getObject(this, objectName, bucketSource);
143+
assert.strictEqual(sourceResponse.statusCode, 200, `failed to getObject, ${sourceResponse.statusCode}`);
144+
const sourceObj = safeJsonParse<{
145+
ReplicationStatus?: string;
146+
LastModified?: string;
147+
ETag?: string;
148+
ContentLength?: number;
149+
VersionId?: string;
150+
Metadata?: Record<string, string>;
151+
}>(sourceResponse.stdout || '{}');
152+
assert(sourceObj.ok);
153+
154+
assert.strictEqual(sourceObj.result?.ReplicationStatus, 'COMPLETED');
155+
assert.strictEqual(
156+
sourceObj.result?.ContentLength,
157+
replicaObj.ContentLength
158+
);
159+
assert.strictEqual(
160+
sourceObj.result?.Metadata?.[`${replicationLocation}-version-id`],
161+
replicaObj.VersionId
162+
);
163+
assert.strictEqual(
164+
sourceObj.result?.Metadata?.[`${replicationLocation}-replication-status`],
165+
'COMPLETED'
166+
);
167+
assert.strictEqual(
168+
sourceObj.result?.VersionId,
169+
replicaObj.Metadata?.['scal-version-id']
170+
);
171+
assert.strictEqual(
172+
replicaObj.Metadata?.['scal-replication-status'],
173+
'REPLICA'
174+
);
175+
});

0 commit comments

Comments
 (0)