From a424363f7dd014a01796c64f9f60d747a8d65016 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Thu, 28 May 2026 19:32:59 +0200 Subject: [PATCH 1/3] Implement CRR Cascaded feature Issue: CLDSRV-897 --- constants.js | 7 + lib/api/apiUtils/object/getReplicationInfo.js | 14 +- lib/routes/routeBackbeat.js | 173 +++++++- package.json | 2 +- tests/functional/backbeat/putData.js | 145 +++++++ tests/functional/backbeat/putMetadata.js | 395 ++++++++++++++++++ tests/unit/api/apiUtils/getReplicationInfo.js | 43 ++ tests/unit/routes/routeBackbeat.js | 44 +- yarn.lock | 13 +- 9 files changed, 815 insertions(+), 21 deletions(-) create mode 100644 tests/functional/backbeat/putData.js create mode 100644 tests/functional/backbeat/putMetadata.js diff --git a/constants.js b/constants.js index 1337f593e8..c7e7b28155 100644 --- a/constants.js +++ b/constants.js @@ -279,6 +279,13 @@ const constants = { rateLimitDefaultConfigCacheTTL: 30000, // 30 seconds rateLimitDefaultBurstCapacity: 1, rateLimitCleanupInterval: 10000, // 10 seconds + // S3-compatible Scality locations excluded as CRR cascade targets: they use the + // MultiBackend S3 path which bypasses putData/putMetadata routes, so loop detection + // cannot fire on those destinations. + crrCascadeBlockedLocationTypes: [ + 'location-scality-ring-s3-v1', + 'location-scality-artesca-s3-v1', + ], // Supported attributes for the GetObjectAttributes 'x-amz-optional-attributes' header. supportedGetObjectAttributes: new Set([ 'StorageClass', diff --git a/lib/api/apiUtils/object/getReplicationInfo.js b/lib/api/apiUtils/object/getReplicationInfo.js index 2320c56e69..ab21143b90 100644 --- a/lib/api/apiUtils/object/getReplicationInfo.js +++ b/lib/api/apiUtils/object/getReplicationInfo.js @@ -66,9 +66,12 @@ function _canUserReplicate(authInfo) { * @param {string} operationType - The type of operation to replicate * @param {object} objectMD - The object metadata * @param {AuthInfo} [authInfo] - authentication info of object owner + * @param {string[]} [blockedSiteTypes=[]] - location types to exclude from the returned backends * @return {object|undefined} */ -function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) { +function getReplicationInfo( + s3config, objKey, bucketMD, isMD, objSize, + operationType, objectMD, authInfo, blockedSiteTypes = []) { const config = bucketMD.getReplicationConfiguration(); if (!config || !_canUserReplicate(authInfo)) { return undefined; @@ -76,13 +79,20 @@ function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operation const isCloud = site => !!replicationBackends[s3config.locationConstraints[site]?.type]; const rules = _withDefaultStorageClass(config.rules || [], s3config); - const backends = ReplicationConfiguration.resolveBackends( + const allBackends = ReplicationConfiguration.resolveBackends( { ...config, rules }, objKey, isCloud, objectMD?.replicationInfo?.backends, ); + const backends = blockedSiteTypes.length > 0 + ? allBackends.filter(b => { + const loc = s3config.locationConstraints[b.site]; + return !loc || !blockedSiteTypes.includes(loc.type); + }) + : allBackends; + if (backends.length === 0) { return undefined; } diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index cde9c63a50..c3bc64ac22 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -1,3 +1,4 @@ +const { constants: { HTTP_STATUS_CONFLICT } } = require('http2'); const url = require('url'); const async = require('async'); const httpProxy = require('http-proxy'); @@ -7,7 +8,13 @@ const joi = require('@hapi/joi'); const backbeatProxy = httpProxy.createProxyServer({ ignorePath: true, }); -const { auth, errors, errorInstances, s3middleware, s3routes, models, storage } = require('arsenal'); +const { auth, errors, errorInstances, s3middleware, s3routes, models, storage, versioning } = require('arsenal'); +const { decode, encode } = versioning.VersionID; +const { + VersionIdCollisionException, + StaleMicroVersionIdException, + MicroVersionIdAlreadyStoredException, +} = require('@scality/cloudserverclient'); const { responseJSONBody } = s3routes.routesUtils; const { getSubPartIds } = s3middleware.azureHelper.mpuUtils; @@ -21,6 +28,7 @@ const locationStorageCheck = require('../api/apiUtils/object/locationStorageChec const { dataStore } = require('../api/apiUtils/object/storeObject'); const prepareRequestContexts = require('../api/apiUtils/authorization/prepareRequestContexts'); const { decodeVersionId } = require('../api/apiUtils/object/versioning'); +const getReplicationInfo = require('../api/apiUtils/object/getReplicationInfo'); const locationKeysHaveChanged = require('../api/apiUtils/object/locationKeysHaveChanged'); const { standardMetadataValidateBucketAndObj, metadataGetObject } = require('../metadata/metadataUtils'); const { config } = require('../Config'); @@ -32,6 +40,7 @@ const { } = require('../api/apiUtils/integrity/validateChecksums'); const { BackendInfo } = models; const { pushReplicationMetric } = require('./utilities/pushReplicationMetric'); +const writeContinue = require('../utilities/writeContinue'); const kms = require('../kms/wrapper'); const { listLifecycleCurrents } = require('../api/backbeat/listLifecycleCurrents'); const { listLifecycleNonCurrents } = require('../api/backbeat/listLifecycleNonCurrents'); @@ -93,7 +102,7 @@ function _isObjectRequest(req) { return ['data', 'metadata', 'multiplebackenddata', 'multiplebackendmetadata'].includes(req.resourceType); } -function _respondWithHeaders(response, payload, extraHeaders, log, callback) { +function _respondWithHeaders(response, payload, extraHeaders, log, callback, statusCode = 200) { let body = ''; if (typeof payload === 'string') { body = payload; @@ -115,10 +124,10 @@ function _respondWithHeaders(response, payload, extraHeaders, log, callback) { // eslint-disable-next-line no-param-reassign response.serverAccessLog.endTurnAroundTime = process.hrtime.bigint(); } - response.writeHead(200, httpHeaders); + response.writeHead(statusCode, httpHeaders); response.end(body, 'utf8', () => { log.end().info('responded with payload', { - httpCode: 200, + httpCode: statusCode, contentLength: Buffer.byteLength(body), }); callback(); @@ -129,6 +138,15 @@ function _respond(response, payload, log, callback) { _respondWithHeaders(response, payload, {}, log, callback); } +function _respondWithHeaderCrrConflict(response, log, callback, code, message, mvId) { + return _respondWithHeaders( + response, + { code, message }, + { 'x-scal-micro-version-id': mvId ? encode(mvId) : '' }, + log, callback, HTTP_STATUS_CONFLICT, + ); +} + function _getRequestPayload(req, cb) { const payload = []; let payloadLen = 0; @@ -414,6 +432,40 @@ function putData(request, response, bucketInfo, objMd, log, callback) { log.error(errMessage); return callback(errorInstances.BadRequest.customizeDescription(errMessage)); } + + const incomingVersionIdEncoded = request.headers['x-scal-version-id']; + if (incomingVersionIdEncoded !== undefined) { + const incomingVersionIdDecoded = incomingVersionIdEncoded !== 'null' + ? decode(incomingVersionIdEncoded) + : 'null'; + if (incomingVersionIdDecoded instanceof Error) { + log.error('crr putData: failed to decode x-scal-version-id header', { + method: 'putData', + error: incomingVersionIdDecoded.message, + }); + return callback(errorInstances.BadRequest.customizeDescription( + 'bad request: invalid x-scal-version-id header')); + } + if (objMd && objMd.versionId === incomingVersionIdDecoded) { + // Data already at destination for this version; return 409 with the existing + // microVersionId so backbeat can decide if putMetadata is still needed. + log.debug('crr putData: version already at destination', { + method: 'putData', + bucketName: request.bucketName, + objectKey: request.objectKey, + hasMicroVersionId: !!objMd.microVersionId, + }); + request.resume(); + return _respondWithHeaderCrrConflict( + response, log, callback, + VersionIdCollisionException.name, + 'version id already at destination', + objMd.microVersionId, + ); + } + } + + writeContinue(request, response); const context = { bucketName: request.bucketName, owner: canonicalID, @@ -539,6 +591,66 @@ function getCanonicalIdsByAccountId(accountId, log, cb) { } function putMetadata(request, response, bucketInfo, objMd, log, callback) { + const { bucketName, objectKey } = request; + + const encodedMicroVersionId = request.headers['x-scal-micro-version-id']; + // When the header is present this is a conditional write: accept only if + // the incoming microVersionId is newer than the one already stored. + // '' is a valid value meaning the source has no microVersionId (null revision). + // Note: '' is falsy so the presence check must be !== undefined, not truthy. + const hasMicroVersionId = encodedMicroVersionId !== undefined; + let incomingMicroVersionId = null; + if (hasMicroVersionId && objMd) { + // '' means source has no microVersionId, treated as older revision + incomingMicroVersionId = encodedMicroVersionId === '' ? null : decode(encodedMicroVersionId); + if (incomingMicroVersionId instanceof Error) { + log.error('putMetadata: failed to decode x-scal-micro-version-id header', { + method: 'putMetadata', + error: incomingMicroVersionId.message, + }); + return callback(errorInstances.BadRequest.customizeDescription( + 'bad request: invalid x-scal-micro-version-id header')); + } + + const isIncomingOlderThanCurrent = (incoming, current) => { + if (current === null) { // nothing to be stale against + return false; + } + // null incoming : source has no cascade revision => oldest possible state + // larger string = older revision (reverse-chronological order) + return incoming === null || incoming > current; + }; + + const objectMicroVersionId = objMd.microVersionId || null; + let conflictErr = null; + if (incomingMicroVersionId === objectMicroVersionId) { + conflictErr = { + err: MicroVersionIdAlreadyStoredException.name, + message: 'incoming microVersionId already at destination', + }; + } else if (isIncomingOlderThanCurrent(incomingMicroVersionId, objectMicroVersionId)) { + conflictErr = { + err: StaleMicroVersionIdException.name, + message: 'incoming revision is older than destination', + mvId: objMd?.microVersionId, + }; + } + if (conflictErr) { + log.debug(`putMetadata: ${conflictErr.message}`, { + method: 'putMetadata', + bucketName, + objectKey, + }); + request.resume(); + return _respondWithHeaderCrrConflict( + response, log, callback, + conflictErr.err, + conflictErr.message, + conflictErr.mvId, + ); + } + } + return _getRequestPayload(request, (err, payload) => { if (err) { return callback(err); @@ -552,7 +664,16 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { return callback(errors.MalformedPOSTRequest); } - const { headers, bucketName, objectKey } = request; + if (omVal.replicationInfo?.status === 'REPLICA') { + omVal.replicationInfo.isReplica = true; + } + + if (incomingMicroVersionId !== null && incomingMicroVersionId !== (omVal.microVersionId ?? null)) { + return callback(errors.BadRequest.customizeDescription( + 'bad request: x-scal-micro-version-id header does not match body microVersionId')); + } + + const { headers } = request; // Destination-side delete-marker replication. // We need the REPLICA status to distinguish from @@ -560,7 +681,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { if ( omVal.isDeleteMarker && omVal.replicationInfo && - omVal.replicationInfo.status === 'REPLICA' && + omVal.replicationInfo.isReplica && request.serverAccessLog ) { // eslint-disable-next-line no-param-reassign @@ -576,7 +697,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // The REPLICA status excludes source-side replication-status updates. if ( omVal.replicationInfo && - omVal.replicationInfo.status === 'REPLICA' && + omVal.replicationInfo.isReplica && (omVal.originOp === 's3:ObjectTagging:Put' || omVal.originOp === 's3:ObjectTagging:Delete') && request.serverAccessLog ) { @@ -593,7 +714,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // The REPLICA status excludes source-side replication-status updates. if ( omVal.replicationInfo && - omVal.replicationInfo.status === 'REPLICA' && + omVal.replicationInfo.isReplica && omVal.originOp === 's3:ObjectAcl:Put' && request.serverAccessLog ) { @@ -672,9 +793,8 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // then we want to create a version for the replica object even though // none was provided in the object metadata value. if (omVal.replicationInfo.isNFS) { - const { isReplica } = omVal.replicationInfo; - versioning = isReplica; - omVal.replicationInfo.isNFS = !isReplica; + versioning = omVal.replicationInfo.isReplica; + omVal.replicationInfo.isNFS = !omVal.replicationInfo.isReplica; } const options = { @@ -724,6 +844,37 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { options.isNull = isNull; } + const isReplicationWrite = !!headers['x-scal-replication-content']; + if (isReplicationWrite) { + const isMDOnly = headers['x-scal-replication-content'] === 'METADATA'; + const objSize = omVal['content-length'] || 0; + + const nextReplInfo = getReplicationInfo( + config, objectKey, bucketInfo, isMDOnly, objSize, + null, null, null, constants.crrCascadeBlockedLocationTypes); + + const hasNextHop = nextReplInfo && nextReplInfo.backends.length > 0; + + // Replicating further requires x-scal-micro-version-id for loop detection. + if (hasNextHop && !hasMicroVersionId) { + return callback(errors.InternalError.customizeDescription( + 'x-scal-micro-version-id is required when replication would trigger a next hop')); + } + + omVal.replicationInfo = hasNextHop ? nextReplInfo : { + status: '', + backends: [], + content: [], + destination: undefined, + storageClass: undefined, + role: undefined, + storageType: undefined, + dataStoreVersionId: undefined, + isNFS: undefined, + }; + omVal.replicationInfo.isReplica = true; + } + return async.series( [ // Zenko's CRR delegates replacing the account diff --git a/package.json b/package.json index a7f3df8652..e5a5f6a88d 100644 --- a/package.json +++ b/package.json @@ -64,11 +64,11 @@ "vaultclient": "scality/vaultclient#8.5.3", "werelogs": "scality/werelogs#semver:^8.2.4", "ws": "^8.18.0", + "@scality/cloudserverclient": "1.0.9", "xml2js": "^0.6.2" }, "devDependencies": { "@eslint/compat": "^1.2.2", - "@scality/cloudserverclient": "1.0.7", "@scality/eslint-config-scality": "scality/Guidelines#8.3.1", "eslint": "^9.14.0", "eslint-plugin-import": "^2.31.0", diff --git a/tests/functional/backbeat/putData.js b/tests/functional/backbeat/putData.js new file mode 100644 index 0000000000..bac6f126aa --- /dev/null +++ b/tests/functional/backbeat/putData.js @@ -0,0 +1,145 @@ +'use strict'; + +const assert = require('assert'); +const { createHash } = require('crypto'); +const { v4: uuidv4 } = require('uuid'); +const { + CreateBucketCommand, + PutBucketVersioningCommand, + PutObjectCommand, +} = require('@aws-sdk/client-s3'); + +const { versioning } = require('arsenal'); +const { ExternalNullVersionId } = versioning.VersioningConstants; +const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); + +const { + BackbeatRoutesClient, + PutDataCommand, + VersionIdCollisionException, +} = require('@scality/cloudserverclient'); + +const { generateVersionId, encode: encodeVersionId } = versioning.VersionID; + +const TEST_BUCKET = `bucket-putdata-${uuidv4().split('-')[0]}`; +const OBJECT_BODY = 'imAboutToBeCascadedWitNoParachuteInMyBack'; +const OBJECT_MD5_HEX = createHash('md5').update(OBJECT_BODY).digest('hex'); +const CANONICAL_ID = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; +const bucketUtil = new BucketUtility('default', {}); +const s3 = bucketUtil.s3; + +let backbeatClient; + +async function putData(key, { versionId } = {}) { + return backbeatClient.send(new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: key, + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + VersionId: versionId || undefined, + Body: Buffer.from(OBJECT_BODY), + })); +} + +before(async () => { + const creds = await s3.config.credentials(); + backbeatClient = new BackbeatRoutesClient({ + endpoint: `http://${process.env.IP || '127.0.0.1'}:8000`, + region: 'us-east-1', + credentials: { + accessKeyId: creds.accessKeyId, + secretAccessKey: creds.secretAccessKey, + }, + forcePathStyle: true, + }); + + await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + })); +}); + +describe('putData : VersionId collision detection', () => { + it('should throw VersionIdCollisionException with microVersionId when versionId matches current master', + async () => { + const key = 'putdata-collision'; + const putResult = await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + })); + const encodedVersionId = putResult.VersionId; + assert.ok(encodedVersionId, 'PutObject should return a VersionId'); + + try { + await putData(key, { versionId: encodedVersionId }); + assert.fail('expected VersionIdCollisionException'); + } catch (err) { + assert.ok(err instanceof VersionIdCollisionException, + `expected VersionIdCollisionException, got ${err.constructor.name}`); + // microVersionId in the error lets backbeat decide whether to proceed + // with metadata-only replication or skip entirely (loop/stale detection). + // '' signals the existing object has no microVersionId (original write state). + assert.strictEqual(err.microVersionId, '', + 'microVersionId in exception should be empty when object has no microVersionId'); + } + }); + + it('should write data normally when VersionId does not match the current master', async () => { + const key = 'putdata-no-collision'; + + await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + })); + + const differentVersionId = encodeVersionId(generateVersionId('different-instance', 'RG001')); + const output = await putData(key, { versionId: differentVersionId }); + assert.ok(output.Location, 'should return a Location when data is written normally'); + }); +}); + +describe('putData : null-version objects (ExternalNullVersionId)', () => { + // Null-version objects created before versioning was enabled use Arsenal constant ExternalNullVersionId = 'null' + // getEncodedVersionId() returns 'null' as-is (no base62 encoding), and objMd.versionId is + // undefined in metadata : collision detection is not possible, so putData must write normally. + it('should write normally when VersionId is "null" (ExternalNullVersionId)', async () => { + const output = await backbeatClient.send(new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: 'putdata-null-version', + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + VersionId: ExternalNullVersionId, + Body: Buffer.from(OBJECT_BODY), + })); + assert.ok(output.Location, 'putData with null-version versionId should write normally'); + }); +}); + +describe('putData : baseline (no cascade headers)', () => { + it('should succeed normally when putData has no VersionId header', async () => { + const key = 'putdata-baseline-no-version-id'; + const output = await putData(key); + assert.ok(output.Location, 'putData without VersionId should return a Location'); + }); + + it('should succeed when putData is sent without Expect: 100-continue (old-client compat)', async () => { + const key = 'putdata-baseline-no-expect-continue'; + const output = await backbeatClient.send(new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: key, + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + Body: Buffer.from(OBJECT_BODY), + })); + assert.ok(output.Location, + 'putData without Expect: 100-continue should return a Location'); + }); +}); diff --git a/tests/functional/backbeat/putMetadata.js b/tests/functional/backbeat/putMetadata.js new file mode 100644 index 0000000000..4478eb659f --- /dev/null +++ b/tests/functional/backbeat/putMetadata.js @@ -0,0 +1,395 @@ +'use strict'; + +const assert = require('assert'); +const { createHash } = require('crypto'); +const { v4: uuidv4 } = require('uuid'); +const { + CreateBucketCommand, + PutBucketReplicationCommand, + PutBucketVersioningCommand, + PutObjectCommand, + PutObjectTaggingCommand, +} = require('@aws-sdk/client-s3'); + +const { versioning, models } = require('arsenal'); +const { ObjectMD } = models; +const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); + +const { + BackbeatRoutesClient, + GetMetadataCommand, + PutMetadataCommand, + MicroVersionIdAlreadyStoredException, + StaleMicroVersionIdException, +} = require('@scality/cloudserverclient'); + +const { generateVersionId, encode: encodeVersionId } = versioning.VersionID; + +const TEST_BUCKET = `bucket-putmetadata-${uuidv4().split('-')[0]}`; +const TEST_BUCKET_CRR = `bucket-putmetadata-crr-${uuidv4().split('-')[0]}`; +const DEST_BUCKET = `bucket-putmetadata-dest-${uuidv4().split('-')[0]}`; +const OBJECT_BODY = 'imAboutToBeCascadedWitNoParachuteInMyBack'; +const OBJECT_MD5_HEX = createHash('md5').update(OBJECT_BODY).digest('hex'); +const CANONICAL_ID = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; +const bucketUtil = new BucketUtility('default', {}); +const s3 = bucketUtil.s3; + +let backbeatClient; + +function makeMicroVersionId() { + const raw = generateVersionId('test-instance', 'RG001'); + return { raw, encoded: encodeVersionId(raw) }; +} + +function buildMetadataBody(overrides) { + const obj = Object.assign({ + 'content-length': Buffer.byteLength(OBJECT_BODY), + 'content-type': 'text/plain', + 'last-modified': new Date().toISOString(), + 'x-amz-version-id': 'null', + 'owner-id': CANONICAL_ID, + 'owner-display-name': 'test', + 'content-md5': OBJECT_MD5_HEX, + replicationInfo: { + status: 'REPLICA', + isReplica: true, + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }, + }, overrides || {}); + return Buffer.from(JSON.stringify(obj)); +} + +// putMetadata without x-scal-replication-content — tests pure conditional update semantics +async function putMetadata(key, mvId) { + const bodyOverrides = mvId ? { microVersionId: mvId.raw } : {}; + return backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: mvId ? mvId.encoded : undefined, + Body: buildMetadataBody(bodyOverrides), + })); +} + +before(async () => { + const creds = await s3.config.credentials(); + backbeatClient = new BackbeatRoutesClient({ + endpoint: `http://${process.env.IP || '127.0.0.1'}:8000`, + region: 'us-east-1', + credentials: { + accessKeyId: creds.accessKeyId, + secretAccessKey: creds.secretAccessKey, + }, + forcePathStyle: true, + }); + + await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + await s3.send(new CreateBucketCommand({ Bucket: DEST_BUCKET })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: DEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET_CRR })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET_CRR, + VersioningConfiguration: { Status: 'Enabled' }, + })); + await s3.send(new PutBucketReplicationCommand({ + Bucket: TEST_BUCKET_CRR, + ReplicationConfiguration: { + Role: 'arn:aws:iam::account-id:role/src-resource,' + + 'arn:aws:iam::account-id:role/dest-resource', + Rules: [{ + Status: 'Enabled', + Prefix: '', + Destination: { + Bucket: `arn:aws:s3:::${DEST_BUCKET}`, + StorageClass: 'zenko', + }, + }], + }, + })); +}); + +// These tests send no x-scal-replication-content header — microVersionId is used +// purely as a conditional update mechanism, independent of replication. +describe('putMetadata : microVersionId conditional updates (no replication content)', () => { + it('should succeed on first write when destination has no microVersionId', async () => { + const key = 'putmetadata-cond-first-write'; + const mvId = makeMicroVersionId(); + await putMetadata(key, mvId); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), mvId.raw); + }); + + it('should succeed on first write when source has no microVersionId (putObject replication)', async () => { + const key = 'putmetadata-cond-first-write-null-mvid'; + // '' = source has no microVersionId (putObject state): must not be treated as a loop + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: '', + Body: buildMetadataBody({}), + })); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), undefined, + 'stored object should have no microVersionId when source had none'); + }); + + it('should throw MicroVersionIdAlreadyStoredException on second write with the same microVersionId', + async () => { + const key = 'putmetadata-cond-loop'; + const mvId = makeMicroVersionId(); + await putMetadata(key, mvId); + await assert.rejects( + () => putMetadata(key, mvId), + err => { + assert.ok(err instanceof MicroVersionIdAlreadyStoredException, + 'second write with same id should throw MicroVersionIdAlreadyStoredException'); + return true; + }, + ); + }); + + it('should throw StaleMicroVersionIdException when writing with an older microVersionId', async () => { + const key = 'putmetadata-cond-stale'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + await putMetadata(key, newerMvId); + await assert.rejects( + () => putMetadata(key, olderMvId), + err => { + assert.ok(err instanceof StaleMicroVersionIdException, + `expected StaleMicroVersionIdException, got ${err.constructor.name}`); + return true; + }, + ); + }); + + it('should succeed and overwrite when writing with a newer microVersionId', async () => { + const key = 'putmetadata-cond-forward'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + await putMetadata(key, olderMvId); + await putMetadata(key, newerMvId); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), newerMvId.raw, + 'stored microVersionId should be the newer one'); + }); + + it('should return 400 when MicroVersionId header does not match body microVersionId', async () => { + const key = 'putmetadata-cond-header-body-mismatch'; + // The mismatch check only fires when the object already exists (objMd non-null), + // because incomingMicroVersionId is only decoded in that path. + const initialMvId = makeMicroVersionId(); + await putMetadata(key, initialMvId); + const headerMvId = makeMicroVersionId(); + const bodyMvId = makeMicroVersionId(); + await assert.rejects( + () => backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: headerMvId.encoded, + Body: buildMetadataBody({ microVersionId: bodyMvId.raw }), + })), + err => { + assert.strictEqual(err.$metadata.httpStatusCode, 400, + 'mismatched header and body microVersionId should return 400'); + return true; + }, + ); + }); + + it('should throw StaleMicroVersionIdException after objectPutTagging bumped microVersionId', + async () => { + const key = 'putmetadata-cond-stale-tagging'; + + await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + })); + + const olderMvId = makeMicroVersionId(); + + await s3.send(new PutObjectTaggingCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + Tagging: { TagSet: [{ Key: 'crr', Value: 'cascade' }] }, + })); + + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET_CRR, Key: key })); + assert.ok(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), + 'objectPutTagging should have set a microVersionId'); + + await assert.rejects( + () => backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: olderMvId.encoded, + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + })), + err => { + assert.ok(err instanceof StaleMicroVersionIdException, + `expected StaleMicroVersionIdException, got ${err.constructor.name}`); + return true; + }, + ); + }); +}); + +// These tests send x-scal-replication-content to simulate backbeat replication writes. +// microVersionId conditional update semantics apply on top of the replication behavior. +describe('putMetadata : cascade replication behavior (with replication content)', () => { + it('should succeed on first write of a zero-byte object without replication content', async () => { + const key = 'putmetadata-crr-zero-byte'; + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: '', + Body: buildMetadataBody({ 'content-length': 0, location: null }), + })); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getContentLength(), 0, + 'zero-byte object should be stored correctly'); + }); + + it('should throw MicroVersionIdAlreadyStoredException on loop even with replication content', + async () => { + // Verifies that microVersionId loop detection fires regardless of whether + // x-scal-replication-content is set + const key = 'putmetadata-crr-loop'; + const mvId = makeMicroVersionId(); + await putMetadata(key, mvId); + await assert.rejects( + () => backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: mvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: mvId.raw }), + })), + err => { + assert.ok(err instanceof MicroVersionIdAlreadyStoredException, + 'loop detection should fire even with x-scal-replication-content set'); + return true; + }, + ); + }); + + it('should throw StaleMicroVersionIdException on stale even with replication content', async () => { + const key = 'putmetadata-crr-stale'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + await putMetadata(key, newerMvId); + await assert.rejects( + () => backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: olderMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + })), + err => { + assert.ok(err instanceof StaleMicroVersionIdException, + 'stale detection should fire even with x-scal-replication-content set'); + return true; + }, + ); + }); + + it('should clear replicationInfo when no CRR rules match on destination bucket', async () => { + const key = 'putmetadata-crr-no-rules'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + // First write creates the object (required before METADATA-content writes). + await putMetadata(key, olderMvId); + // Second write with x-scal-replication-content triggers the cascade block. + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: newerMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: newerMvId.raw }), + })); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + const storedMd = new ObjectMD(JSON.parse(Body)); + assert.strictEqual(storedMd.getReplicationStatus(), '', + 'replication status should be cleared when no CRR rules match'); + assert.deepStrictEqual(storedMd.getReplicationBackends(), [], + 'replication backends should be empty when no CRR rules match'); + assert.strictEqual(storedMd.getReplicationIsReplica(), true, + 'isReplica should be preserved regardless of cascade triggering'); + }); + + it('should set replication status to PENDING and preserve isReplica when bucket has CRR rules', + async () => { + const key = 'putmetadata-crr-next-hop'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: olderMvId.encoded, + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + })); + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: newerMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: newerMvId.raw }), + })); + + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET_CRR, Key: key })); + const storedMd = new ObjectMD(JSON.parse(Body)); + assert.strictEqual(storedMd.getMicroVersionId(), newerMvId.raw, + 'stored microVersionId should be the newer one'); + assert.strictEqual(storedMd.getReplicationStatus(), 'PENDING', + 'replication status should be PENDING when a CRR rule matches'); + assert.ok(storedMd.getReplicationBackends().length > 0, + 'replication backends should be populated when a CRR rule matches'); + assert.strictEqual(storedMd.getReplicationIsReplica(), true, + 'isReplica should be preserved regardless of cascade triggering'); + }); +}); + +describe('putMetadata : baseline (no cascade headers)', () => { + it('should succeed normally when putMetadata has no MicroVersionId header', async () => { + const key = 'putmetadata-baseline-no-mvid'; + await putMetadata(key, null); + }); + + it('should not set a microVersionId on a regular S3 PutObject', async () => { + const key = 'putmetadata-baseline-s3put'; + await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + })); + const { Body } = await backbeatClient.send( + new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), undefined, + 'a regular S3 PutObject should not set a microVersionId'); + }); +}); diff --git a/tests/unit/api/apiUtils/getReplicationInfo.js b/tests/unit/api/apiUtils/getReplicationInfo.js index 2cb4f8ae67..56236661a7 100644 --- a/tests/unit/api/apiUtils/getReplicationInfo.js +++ b/tests/unit/api/apiUtils/getReplicationInfo.js @@ -609,4 +609,47 @@ describe('getReplicationInfo helper', () => { assert.strictEqual(cloudBackend.storageType, undefined); }); }); + + describe('blockedSiteTypes filtering', () => { + const RING_TYPE = 'location-scality-ring-s3-v1'; + const configWithRing = { + ...TEST_CONFIG, + locationConstraints: { + ...TEST_CONFIG.locationConstraints, + 'ring-site': { type: RING_TYPE }, + }, + }; + + it('should exclude backends whose location type is blocked', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { prefix: '', enabled: true, storageClass: 'awsbackend' }, + { prefix: '', enabled: true, storageClass: 'ring-site' }, + ], + destination: 'tosomewhere', + }; + const info = getReplicationInfo( + configWithRing, 'fookey', + new BucketInfo('b', 'id', 'name', new Date().toJSON(), + null, null, null, null, null, null, null, null, null, replicationConfig), + true, 123, null, null, null, [RING_TYPE]); + assert.strictEqual(info.backends.length, 1); + assert.strictEqual(info.backends[0].site, 'awsbackend'); + }); + + it('should return undefined when all backends are blocked', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [{ prefix: '', enabled: true, storageClass: 'ring-site' }], + destination: 'tosomewhere', + }; + const info = getReplicationInfo( + configWithRing, 'fookey', + new BucketInfo('b', 'id', 'name', new Date().toJSON(), + null, null, null, null, null, null, null, null, null, replicationConfig), + true, 123, null, null, null, [RING_TYPE]); + assert.strictEqual(info, undefined); + }); + }); }); diff --git a/tests/unit/routes/routeBackbeat.js b/tests/unit/routes/routeBackbeat.js index 6a581bfb82..aa0de287e1 100644 --- a/tests/unit/routes/routeBackbeat.js +++ b/tests/unit/routes/routeBackbeat.js @@ -10,7 +10,7 @@ const { routeBackbeat } = require('../../../lib/routes/routeBackbeat'); const { DummyRequestLogger, versioningTestUtils, makeAuthInfo } = require('../helpers'); const dataWrapper = require('../../../lib/data/wrapper'); const DummyRequest = require('../DummyRequest'); -const { auth, errors } = require('arsenal'); +const { auth, errors, versioning } = require('arsenal'); const AuthInfo = auth.AuthInfo; const { config } = require('../../../lib/Config'); const quotaUtils = require('../../../lib/api/apiUtils/quotas/quotaUtils'); @@ -180,10 +180,52 @@ describe('routeBackbeat', () => { assert.deepStrictEqual(mockResponse.body, [{}]); }); + it('should return 409 VersionIdCollisionException when versionId matches master, no microVersionId', async () => { + // Old objects without microVersionId: data is already at destination. + // Cloudserver returns 409 VersionIdCollisionException with empty x-scal-micro-version-id. + // Backbeat sees empty MicroVersionId and proceeds to putMetadata (partAlreadyAtDest). + const rawVersionId = versioning.VersionID.generateVersionId('test', 'RG001'); + const encodedVersionId = versioning.VersionID.encode(rawVersionId); + + mockRequest = prepareDummyRequest({ + 'x-scal-canonical-id': 'id', + 'content-md5': '1234', + 'content-length': '0', + 'x-scal-versioning-required': 'true', + 'x-scal-version-id': encodedVersionId, + }); + mockRequest.method = 'PUT'; + mockRequest.url = '/_/backbeat/data/bucket0/key0'; + mockRequest.destroy = () => {}; + + metadataUtils.standardMetadataValidateBucketAndObj.callsFake( + (params, denies, log, callback) => { + callback(null, { + getVersioningConfiguration: () => ({ Status: 'Enabled' }), + isVersioningEnabled: () => true, + getLocationConstraint: () => undefined, + }, { + versionId: rawVersionId, + // no microVersionId — old-format object + }); + }); + + routeBackbeat('127.0.0.1', mockRequest, mockResponse, log); + void await endPromise; + + sinon.assert.notCalled(storeObject.dataStore); + assert.strictEqual(mockResponse.statusCode, 409); + assert.strictEqual(mockResponse.body.code, 'VersionIdCollisionException'); + const [, responseHeaders] = mockResponse.writeHead.firstCall.args; + assert.strictEqual(responseHeaders['x-scal-micro-version-id'], '', + 'should have empty x-scal-micro-version-id header for old-format objects'); + }); + describe('putMetadata', () => { const bucketInfo = { getVersioningConfiguration: () => ({ Status: 'Enabled' }), isVersioningEnabled: () => true, + getReplicationConfiguration: () => null, }; let dataDeleteSpy; diff --git a/yarn.lock b/yarn.lock index e465d55da2..586c43f42b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3662,14 +3662,15 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@scality/cloudserverclient@1.0.7": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@scality/cloudserverclient/-/cloudserverclient-1.0.7.tgz#ee9eed09cc7da5e97d5ad8359f429218a0a30859" - integrity sha512-gMDtI/ufRDVqWJlYkvqxXRfZOChBCw9uXt2lsaIvMuiqx/pDTNZMVLfPyRN5vx0tb6HP+5ZAe2FyPdtKXG24ng== +"@scality/cloudserverclient@1.0.9": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@scality/cloudserverclient/-/cloudserverclient-1.0.9.tgz#0a333e39436e1f1e74279c3b5a11645be1fffbc1" + integrity sha512-ZGqH4G535opDAEP2PxdYDFG7kjZ/eBrzP3ZmO49goFVjRhyDCZ1gT0Pask3/wZcyysZ2Au1qylorZSLPiZmB2A== dependencies: "@aws-sdk/client-s3" "^3.1009.0" + "@aws-sdk/middleware-expect-continue" "^3.972.8" JSONStream "^1.3.5" - fast-xml-parser "^4.3.2" + fast-xml-parser "^5.5.7" "@scality/eslint-config-scality@scality/Guidelines#8.3.1": version "8.3.1" @@ -8312,7 +8313,7 @@ fast-xml-builder@^1.1.4: dependencies: path-expression-matcher "^1.1.3" -fast-xml-parser@5.2.5, fast-xml-parser@5.3.6, fast-xml-parser@5.5.6, fast-xml-parser@^4.3.2, fast-xml-parser@^5.0.7, fast-xml-parser@^5.5.6: +fast-xml-parser@5.2.5, fast-xml-parser@5.3.6, fast-xml-parser@5.5.6, fast-xml-parser@^5.0.7, fast-xml-parser@^5.5.6, fast-xml-parser@^5.5.7: version "5.5.7" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-5.5.7.tgz#e1ddc86662d808450a19cf2fb6ccc9c3c9933c5d" integrity sha512-LteOsISQ2GEiDHZch6L9hB0+MLoYVLToR7xotrzU0opCICBkxOPgHAy1HxAvtxfJNXDJpgAsQN30mkrfpO2Prg== From 77140ea8f32e14a5e0569fa0f8eaa2641de63244 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Thu, 16 Jul 2026 17:24:18 +0200 Subject: [PATCH 2/3] apply prettier Issue: CLDSRV-897 --- constants.js | 62 +-- lib/api/apiUtils/object/getReplicationInfo.js | 25 +- lib/routes/routeBackbeat.js | 86 ++-- tests/functional/backbeat/putData.js | 127 ++--- tests/functional/backbeat/putMetadata.js | 449 ++++++++++-------- tests/unit/api/apiUtils/getReplicationInfo.js | 58 ++- tests/unit/routes/routeBackbeat.js | 24 +- 7 files changed, 482 insertions(+), 349 deletions(-) diff --git a/constants.js b/constants.js index c7e7b28155..cdcd1c5d1a 100644 --- a/constants.js +++ b/constants.js @@ -46,8 +46,7 @@ const constants = { // only public resources publicId: 'http://acs.amazonaws.com/groups/global/AllUsers', // All Authenticated Users is an ACL group. - allAuthedUsersId: 'http://acs.amazonaws.com/groups/' + - 'global/AuthenticatedUsers', + allAuthedUsersId: 'http://acs.amazonaws.com/groups/' + 'global/AuthenticatedUsers', // LogId is used for the AWS logger to write the logs // to the destination bucket. This style of logging is // to be implemented later but the logId is used in the @@ -74,8 +73,7 @@ const constants = { // Max size on put part or copy part is 5GB. For functional // testing use 110 MB as max - maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 : - 5368709120, + maximumAllowedPartSize: process.env.MPU_TESTING === 'yes' ? 110100480 : 5368709120, // Max size allowed in a single put object request is 5GB // https://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html @@ -101,15 +99,14 @@ const constants = { defaultApiBodySizeLimits: { // Multi Objects Delete request can be large : up to 1000 keys of 1024 bytes is // already 1mb, with the other fields it could reach 2mb - 'multiObjectDelete': 2 * 1024 * 1024, + multiObjectDelete: 2 * 1024 * 1024, // AWS sets the maximum size for bucket policies to 20 KB // https://docs.aws.amazon.com/AmazonS3/latest/userguide/add-bucket-policy.html - 'bucketPutPolicy': 20 * 1024, + bucketPutPolicy: 20 * 1024, }, // hex digest of sha256 hash of empty string: - emptyStringHash: crypto.createHash('sha256') - .update('', 'binary').digest('hex'), + emptyStringHash: crypto.createHash('sha256').update('', 'binary').digest('hex'), // Queries supported by AWS that we do not currently support. // Non-bucket queries @@ -164,8 +161,7 @@ const constants = { /* eslint-enable camelcase */ mpuMDStoredOnS3Backend: { azure: true }, azureAccountNameRegex: /^[a-z0-9]{3,24}$/, - base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' + - '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'), + base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' + '(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'), productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko', // location constraint delimiter zenkoSeparator: ':', @@ -203,32 +199,15 @@ const constants = { invalidObjectUserMetadataHeader: 'x-amz-missing-meta', // Bucket specific queries supported by AWS that we do not currently support // these queries may or may not be supported at object level - unsupportedBucketQueries: [ - ], - suppressedUtapiEventFields: [ - 'object', - 'location', - 'versionId', - ], - allowedUtapiEventFilterFields: [ - 'operationId', - 'location', - 'account', - 'user', - 'bucket', - ], - arrayOfAllowed: [ - 'objectPutTagging', - 'objectPutLegalHold', - 'objectPutRetention', - ], + unsupportedBucketQueries: [], + suppressedUtapiEventFields: ['object', 'location', 'versionId'], + allowedUtapiEventFilterFields: ['operationId', 'location', 'account', 'user', 'bucket'], + arrayOfAllowed: ['objectPutTagging', 'objectPutLegalHold', 'objectPutRetention'], allowedUtapiEventFilterStates: ['allow', 'deny'], allowedRestoreObjectRequestTierValues: ['Standard'], // Only STANDARD class is supported, but keep the option to override supported values for now. // This should be removed in CLDSRV-639. - validStorageClasses: process.env.VALID_STORAGE_CLASSES?.split(',') || [ - 'STANDARD', - ], + validStorageClasses: process.env.VALID_STORAGE_CLASSES?.split(',') || ['STANDARD'], lifecycleListing: { CURRENT_TYPE: 'current', NON_CURRENT_TYPE: 'noncurrent', @@ -261,11 +240,7 @@ const constants = { assumedRoleArnResourceType: 'assumed-role', // Session name of the backbeat lifecycle assumed role session. backbeatLifecycleSessionName: 'backbeat-lifecycle', - actionsToConsiderAsObjectPut: [ - 'initiateMultipartUpload', - 'objectPutPart', - 'completeMultipartUpload', - ], + actionsToConsiderAsObjectPut: ['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'], // if requester is not bucket owner, bucket policy actions should be denied with // MethodNotAllowed error onlyOwnerAllowed: [ @@ -282,18 +257,9 @@ const constants = { // S3-compatible Scality locations excluded as CRR cascade targets: they use the // MultiBackend S3 path which bypasses putData/putMetadata routes, so loop detection // cannot fire on those destinations. - crrCascadeBlockedLocationTypes: [ - 'location-scality-ring-s3-v1', - 'location-scality-artesca-s3-v1', - ], + crrCascadeBlockedLocationTypes: ['location-scality-ring-s3-v1', 'location-scality-artesca-s3-v1'], // Supported attributes for the GetObjectAttributes 'x-amz-optional-attributes' header. - supportedGetObjectAttributes: new Set([ - 'StorageClass', - 'ObjectSize', - 'ObjectParts', - 'Checksum', - 'ETag', - ]), + supportedGetObjectAttributes: new Set(['StorageClass', 'ObjectSize', 'ObjectParts', 'Checksum', 'ETag']), }; module.exports = constants; diff --git a/lib/api/apiUtils/object/getReplicationInfo.js b/lib/api/apiUtils/object/getReplicationInfo.js index ab21143b90..b88bf2ac36 100644 --- a/lib/api/apiUtils/object/getReplicationInfo.js +++ b/lib/api/apiUtils/object/getReplicationInfo.js @@ -70,8 +70,16 @@ function _canUserReplicate(authInfo) { * @return {object|undefined} */ function getReplicationInfo( - s3config, objKey, bucketMD, isMD, objSize, - operationType, objectMD, authInfo, blockedSiteTypes = []) { + s3config, + objKey, + bucketMD, + isMD, + objSize, + operationType, + objectMD, + authInfo, + blockedSiteTypes = [], +) { const config = bucketMD.getReplicationConfiguration(); if (!config || !_canUserReplicate(authInfo)) { return undefined; @@ -86,12 +94,13 @@ function getReplicationInfo( objectMD?.replicationInfo?.backends, ); - const backends = blockedSiteTypes.length > 0 - ? allBackends.filter(b => { - const loc = s3config.locationConstraints[b.site]; - return !loc || !blockedSiteTypes.includes(loc.type); - }) - : allBackends; + const backends = + blockedSiteTypes.length > 0 + ? allBackends.filter(b => { + const loc = s3config.locationConstraints[b.site]; + return !loc || !blockedSiteTypes.includes(loc.type); + }) + : allBackends; if (backends.length === 0) { return undefined; diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index c3bc64ac22..719478df9f 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -1,4 +1,6 @@ -const { constants: { HTTP_STATUS_CONFLICT } } = require('http2'); +const { + constants: { HTTP_STATUS_CONFLICT }, +} = require('http2'); const url = require('url'); const async = require('async'); const httpProxy = require('http-proxy'); @@ -143,7 +145,9 @@ function _respondWithHeaderCrrConflict(response, log, callback, code, message, m response, { code, message }, { 'x-scal-micro-version-id': mvId ? encode(mvId) : '' }, - log, callback, HTTP_STATUS_CONFLICT, + log, + callback, + HTTP_STATUS_CONFLICT, ); } @@ -435,16 +439,16 @@ function putData(request, response, bucketInfo, objMd, log, callback) { const incomingVersionIdEncoded = request.headers['x-scal-version-id']; if (incomingVersionIdEncoded !== undefined) { - const incomingVersionIdDecoded = incomingVersionIdEncoded !== 'null' - ? decode(incomingVersionIdEncoded) - : 'null'; + const incomingVersionIdDecoded = + incomingVersionIdEncoded !== 'null' ? decode(incomingVersionIdEncoded) : 'null'; if (incomingVersionIdDecoded instanceof Error) { log.error('crr putData: failed to decode x-scal-version-id header', { method: 'putData', error: incomingVersionIdDecoded.message, }); - return callback(errorInstances.BadRequest.customizeDescription( - 'bad request: invalid x-scal-version-id header')); + return callback( + errorInstances.BadRequest.customizeDescription('bad request: invalid x-scal-version-id header'), + ); } if (objMd && objMd.versionId === incomingVersionIdDecoded) { // Data already at destination for this version; return 409 with the existing @@ -457,7 +461,9 @@ function putData(request, response, bucketInfo, objMd, log, callback) { }); request.resume(); return _respondWithHeaderCrrConflict( - response, log, callback, + response, + log, + callback, VersionIdCollisionException.name, 'version id already at destination', objMd.microVersionId, @@ -608,13 +614,15 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { method: 'putMetadata', error: incomingMicroVersionId.message, }); - return callback(errorInstances.BadRequest.customizeDescription( - 'bad request: invalid x-scal-micro-version-id header')); + return callback( + errorInstances.BadRequest.customizeDescription('bad request: invalid x-scal-micro-version-id header'), + ); } const isIncomingOlderThanCurrent = (incoming, current) => { - if (current === null) { // nothing to be stale against - return false; + if (current === null) { + // nothing to be stale against + return false; } // null incoming : source has no cascade revision => oldest possible state // larger string = older revision (reverse-chronological order) @@ -643,7 +651,9 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { }); request.resume(); return _respondWithHeaderCrrConflict( - response, log, callback, + response, + log, + callback, conflictErr.err, conflictErr.message, conflictErr.mvId, @@ -669,8 +679,11 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { } if (incomingMicroVersionId !== null && incomingMicroVersionId !== (omVal.microVersionId ?? null)) { - return callback(errors.BadRequest.customizeDescription( - 'bad request: x-scal-micro-version-id header does not match body microVersionId')); + return callback( + errors.BadRequest.customizeDescription( + 'bad request: x-scal-micro-version-id header does not match body microVersionId', + ), + ); } const { headers } = request; @@ -850,28 +863,41 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { const objSize = omVal['content-length'] || 0; const nextReplInfo = getReplicationInfo( - config, objectKey, bucketInfo, isMDOnly, objSize, - null, null, null, constants.crrCascadeBlockedLocationTypes); + config, + objectKey, + bucketInfo, + isMDOnly, + objSize, + null, + null, + null, + constants.crrCascadeBlockedLocationTypes, + ); const hasNextHop = nextReplInfo && nextReplInfo.backends.length > 0; // Replicating further requires x-scal-micro-version-id for loop detection. if (hasNextHop && !hasMicroVersionId) { - return callback(errors.InternalError.customizeDescription( - 'x-scal-micro-version-id is required when replication would trigger a next hop')); + return callback( + errors.InternalError.customizeDescription( + 'x-scal-micro-version-id is required when replication would trigger a next hop', + ), + ); } - omVal.replicationInfo = hasNextHop ? nextReplInfo : { - status: '', - backends: [], - content: [], - destination: undefined, - storageClass: undefined, - role: undefined, - storageType: undefined, - dataStoreVersionId: undefined, - isNFS: undefined, - }; + omVal.replicationInfo = hasNextHop + ? nextReplInfo + : { + status: '', + backends: [], + content: [], + destination: undefined, + storageClass: undefined, + role: undefined, + storageType: undefined, + dataStoreVersionId: undefined, + isNFS: undefined, + }; omVal.replicationInfo.isReplica = true; } diff --git a/tests/functional/backbeat/putData.js b/tests/functional/backbeat/putData.js index bac6f126aa..b4cb871860 100644 --- a/tests/functional/backbeat/putData.js +++ b/tests/functional/backbeat/putData.js @@ -3,21 +3,13 @@ const assert = require('assert'); const { createHash } = require('crypto'); const { v4: uuidv4 } = require('uuid'); -const { - CreateBucketCommand, - PutBucketVersioningCommand, - PutObjectCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutBucketVersioningCommand, PutObjectCommand } = require('@aws-sdk/client-s3'); const { versioning } = require('arsenal'); const { ExternalNullVersionId } = versioning.VersioningConstants; const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); -const { - BackbeatRoutesClient, - PutDataCommand, - VersionIdCollisionException, -} = require('@scality/cloudserverclient'); +const { BackbeatRoutesClient, PutDataCommand, VersionIdCollisionException } = require('@scality/cloudserverclient'); const { generateVersionId, encode: encodeVersionId } = versioning.VersionID; @@ -31,15 +23,17 @@ const s3 = bucketUtil.s3; let backbeatClient; async function putData(key, { versionId } = {}) { - return backbeatClient.send(new PutDataCommand({ - Bucket: TEST_BUCKET, - Key: key, - ContentMD5: OBJECT_MD5_HEX, - CanonicalID: CANONICAL_ID, - VersioningRequired: true, - VersionId: versionId || undefined, - Body: Buffer.from(OBJECT_BODY), - })); + return backbeatClient.send( + new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: key, + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + VersionId: versionId || undefined, + Body: Buffer.from(OBJECT_BODY), + }), + ); } before(async () => { @@ -55,22 +49,25 @@ before(async () => { }); await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: TEST_BUCKET, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); }); describe('putData : VersionId collision detection', () => { - it('should throw VersionIdCollisionException with microVersionId when versionId matches current master', - async () => { + it('should throw VersionIdCollisionException with microVersionId when versionId matches master', async () => { const key = 'putdata-collision'; - const putResult = await s3.send(new PutObjectCommand({ - Bucket: TEST_BUCKET, - Key: key, - Body: Buffer.from(OBJECT_BODY), - ContentType: 'text/plain', - })); + const putResult = await s3.send( + new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + }), + ); const encodedVersionId = putResult.VersionId; assert.ok(encodedVersionId, 'PutObject should return a VersionId'); @@ -78,25 +75,32 @@ describe('putData : VersionId collision detection', () => { await putData(key, { versionId: encodedVersionId }); assert.fail('expected VersionIdCollisionException'); } catch (err) { - assert.ok(err instanceof VersionIdCollisionException, - `expected VersionIdCollisionException, got ${err.constructor.name}`); + assert.ok( + err instanceof VersionIdCollisionException, + `expected VersionIdCollisionException, got ${err.constructor.name}`, + ); // microVersionId in the error lets backbeat decide whether to proceed // with metadata-only replication or skip entirely (loop/stale detection). // '' signals the existing object has no microVersionId (original write state). - assert.strictEqual(err.microVersionId, '', - 'microVersionId in exception should be empty when object has no microVersionId'); + assert.strictEqual( + err.microVersionId, + '', + 'microVersionId in exception should be empty when object has no microVersionId', + ); } }); it('should write data normally when VersionId does not match the current master', async () => { const key = 'putdata-no-collision'; - await s3.send(new PutObjectCommand({ - Bucket: TEST_BUCKET, - Key: key, - Body: Buffer.from(OBJECT_BODY), - ContentType: 'text/plain', - })); + await s3.send( + new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + }), + ); const differentVersionId = encodeVersionId(generateVersionId('different-instance', 'RG001')); const output = await putData(key, { versionId: differentVersionId }); @@ -109,15 +113,17 @@ describe('putData : null-version objects (ExternalNullVersionId)', () => { // getEncodedVersionId() returns 'null' as-is (no base62 encoding), and objMd.versionId is // undefined in metadata : collision detection is not possible, so putData must write normally. it('should write normally when VersionId is "null" (ExternalNullVersionId)', async () => { - const output = await backbeatClient.send(new PutDataCommand({ - Bucket: TEST_BUCKET, - Key: 'putdata-null-version', - ContentMD5: OBJECT_MD5_HEX, - CanonicalID: CANONICAL_ID, - VersioningRequired: true, - VersionId: ExternalNullVersionId, - Body: Buffer.from(OBJECT_BODY), - })); + const output = await backbeatClient.send( + new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: 'putdata-null-version', + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + VersionId: ExternalNullVersionId, + Body: Buffer.from(OBJECT_BODY), + }), + ); assert.ok(output.Location, 'putData with null-version versionId should write normally'); }); }); @@ -131,15 +137,16 @@ describe('putData : baseline (no cascade headers)', () => { it('should succeed when putData is sent without Expect: 100-continue (old-client compat)', async () => { const key = 'putdata-baseline-no-expect-continue'; - const output = await backbeatClient.send(new PutDataCommand({ - Bucket: TEST_BUCKET, - Key: key, - ContentMD5: OBJECT_MD5_HEX, - CanonicalID: CANONICAL_ID, - VersioningRequired: true, - Body: Buffer.from(OBJECT_BODY), - })); - assert.ok(output.Location, - 'putData without Expect: 100-continue should return a Location'); + const output = await backbeatClient.send( + new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: key, + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + Body: Buffer.from(OBJECT_BODY), + }), + ); + assert.ok(output.Location, 'putData without Expect: 100-continue should return a Location'); }); }); diff --git a/tests/functional/backbeat/putMetadata.js b/tests/functional/backbeat/putMetadata.js index 4478eb659f..3de0af9e6b 100644 --- a/tests/functional/backbeat/putMetadata.js +++ b/tests/functional/backbeat/putMetadata.js @@ -42,38 +42,43 @@ function makeMicroVersionId() { } function buildMetadataBody(overrides) { - const obj = Object.assign({ - 'content-length': Buffer.byteLength(OBJECT_BODY), - 'content-type': 'text/plain', - 'last-modified': new Date().toISOString(), - 'x-amz-version-id': 'null', - 'owner-id': CANONICAL_ID, - 'owner-display-name': 'test', - 'content-md5': OBJECT_MD5_HEX, - replicationInfo: { - status: 'REPLICA', - isReplica: true, - backends: [], - content: [], - destination: '', - storageClass: '', - role: '', - storageType: '', - dataStoreVersionId: '', + const obj = Object.assign( + { + 'content-length': Buffer.byteLength(OBJECT_BODY), + 'content-type': 'text/plain', + 'last-modified': new Date().toISOString(), + 'x-amz-version-id': 'null', + 'owner-id': CANONICAL_ID, + 'owner-display-name': 'test', + 'content-md5': OBJECT_MD5_HEX, + replicationInfo: { + status: 'REPLICA', + isReplica: true, + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }, }, - }, overrides || {}); + overrides || {}, + ); return Buffer.from(JSON.stringify(obj)); } // putMetadata without x-scal-replication-content — tests pure conditional update semantics async function putMetadata(key, mvId) { const bodyOverrides = mvId ? { microVersionId: mvId.raw } : {}; - return backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET, - Key: key, - MicroVersionId: mvId ? mvId.encoded : undefined, - Body: buildMetadataBody(bodyOverrides), - })); + return backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: mvId ? mvId.encoded : undefined, + Body: buildMetadataBody(bodyOverrides), + }), + ); } before(async () => { @@ -89,37 +94,46 @@ before(async () => { }); await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: TEST_BUCKET, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: DEST_BUCKET })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: DEST_BUCKET, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: DEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET_CRR })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: TEST_BUCKET_CRR, - VersioningConfiguration: { Status: 'Enabled' }, - })); - await s3.send(new PutBucketReplicationCommand({ - Bucket: TEST_BUCKET_CRR, - ReplicationConfiguration: { - Role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - Rules: [{ - Status: 'Enabled', - Prefix: '', - Destination: { - Bucket: `arn:aws:s3:::${DEST_BUCKET}`, - StorageClass: 'zenko', - }, - }], - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET_CRR, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + await s3.send( + new PutBucketReplicationCommand({ + Bucket: TEST_BUCKET_CRR, + ReplicationConfiguration: { + Role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + Rules: [ + { + Status: 'Enabled', + Prefix: '', + Destination: { + Bucket: `arn:aws:s3:::${DEST_BUCKET}`, + StorageClass: 'zenko', + }, + }, + ], + }, + }), + ); }); // These tests send no x-scal-replication-content header — microVersionId is used @@ -129,36 +143,40 @@ describe('putMetadata : microVersionId conditional updates (no replication conte const key = 'putmetadata-cond-first-write'; const mvId = makeMicroVersionId(); await putMetadata(key, mvId); - const { Body } = await backbeatClient.send( - new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + const { Body } = await backbeatClient.send(new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), mvId.raw); }); it('should succeed on first write when source has no microVersionId (putObject replication)', async () => { const key = 'putmetadata-cond-first-write-null-mvid'; // '' = source has no microVersionId (putObject state): must not be treated as a loop - await backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET, - Key: key, - MicroVersionId: '', - Body: buildMetadataBody({}), - })); - const { Body } = await backbeatClient.send( - new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); - assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), undefined, - 'stored object should have no microVersionId when source had none'); + await backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: '', + Body: buildMetadataBody({}), + }), + ); + const { Body } = await backbeatClient.send(new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual( + new ObjectMD(JSON.parse(Body)).getMicroVersionId(), + undefined, + 'stored object should have no microVersionId when source had none', + ); }); - it('should throw MicroVersionIdAlreadyStoredException on second write with the same microVersionId', - async () => { + it('should throw MicroVersionIdAlreadyStoredException on second write with the same microVersionId', async () => { const key = 'putmetadata-cond-loop'; const mvId = makeMicroVersionId(); await putMetadata(key, mvId); await assert.rejects( () => putMetadata(key, mvId), err => { - assert.ok(err instanceof MicroVersionIdAlreadyStoredException, - 'second write with same id should throw MicroVersionIdAlreadyStoredException'); + assert.ok( + err instanceof MicroVersionIdAlreadyStoredException, + 'second write with same id should throw MicroVersionIdAlreadyStoredException', + ); return true; }, ); @@ -172,8 +190,10 @@ describe('putMetadata : microVersionId conditional updates (no replication conte await assert.rejects( () => putMetadata(key, olderMvId), err => { - assert.ok(err instanceof StaleMicroVersionIdException, - `expected StaleMicroVersionIdException, got ${err.constructor.name}`); + assert.ok( + err instanceof StaleMicroVersionIdException, + `expected StaleMicroVersionIdException, got ${err.constructor.name}`, + ); return true; }, ); @@ -185,10 +205,12 @@ describe('putMetadata : microVersionId conditional updates (no replication conte const newerMvId = makeMicroVersionId(); await putMetadata(key, olderMvId); await putMetadata(key, newerMvId); - const { Body } = await backbeatClient.send( - new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); - assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), newerMvId.raw, - 'stored microVersionId should be the newer one'); + const { Body } = await backbeatClient.send(new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual( + new ObjectMD(JSON.parse(Body)).getMicroVersionId(), + newerMvId.raw, + 'stored microVersionId should be the newer one', + ); }); it('should return 400 when MicroVersionId header does not match body microVersionId', async () => { @@ -200,54 +222,69 @@ describe('putMetadata : microVersionId conditional updates (no replication conte const headerMvId = makeMicroVersionId(); const bodyMvId = makeMicroVersionId(); await assert.rejects( - () => backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET, - Key: key, - MicroVersionId: headerMvId.encoded, - Body: buildMetadataBody({ microVersionId: bodyMvId.raw }), - })), + () => + backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: headerMvId.encoded, + Body: buildMetadataBody({ microVersionId: bodyMvId.raw }), + }), + ), err => { - assert.strictEqual(err.$metadata.httpStatusCode, 400, - 'mismatched header and body microVersionId should return 400'); + assert.strictEqual( + err.$metadata.httpStatusCode, + 400, + 'mismatched header and body microVersionId should return 400', + ); return true; }, ); }); - it('should throw StaleMicroVersionIdException after objectPutTagging bumped microVersionId', - async () => { + it('should throw StaleMicroVersionIdException after objectPutTagging bumped microVersionId', async () => { const key = 'putmetadata-cond-stale-tagging'; - await s3.send(new PutObjectCommand({ - Bucket: TEST_BUCKET_CRR, - Key: key, - Body: Buffer.from(OBJECT_BODY), - ContentType: 'text/plain', - })); + await s3.send( + new PutObjectCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + }), + ); const olderMvId = makeMicroVersionId(); - await s3.send(new PutObjectTaggingCommand({ - Bucket: TEST_BUCKET_CRR, - Key: key, - Tagging: { TagSet: [{ Key: 'crr', Value: 'cascade' }] }, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + Tagging: { TagSet: [{ Key: 'crr', Value: 'cascade' }] }, + }), + ); - const { Body } = await backbeatClient.send( - new GetMetadataCommand({ Bucket: TEST_BUCKET_CRR, Key: key })); - assert.ok(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), - 'objectPutTagging should have set a microVersionId'); + const { Body } = await backbeatClient.send(new GetMetadataCommand({ Bucket: TEST_BUCKET_CRR, Key: key })); + assert.ok( + new ObjectMD(JSON.parse(Body)).getMicroVersionId(), + 'objectPutTagging should have set a microVersionId', + ); await assert.rejects( - () => backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET_CRR, - Key: key, - MicroVersionId: olderMvId.encoded, - Body: buildMetadataBody({ microVersionId: olderMvId.raw }), - })), + () => + backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: olderMvId.encoded, + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + }), + ), err => { - assert.ok(err instanceof StaleMicroVersionIdException, - `expected StaleMicroVersionIdException, got ${err.constructor.name}`); + assert.ok( + err instanceof StaleMicroVersionIdException, + `expected StaleMicroVersionIdException, got ${err.constructor.name}`, + ); return true; }, ); @@ -259,36 +296,44 @@ describe('putMetadata : microVersionId conditional updates (no replication conte describe('putMetadata : cascade replication behavior (with replication content)', () => { it('should succeed on first write of a zero-byte object without replication content', async () => { const key = 'putmetadata-crr-zero-byte'; - await backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET, - Key: key, - MicroVersionId: '', - Body: buildMetadataBody({ 'content-length': 0, location: null }), - })); - const { Body } = await backbeatClient.send( - new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); - assert.strictEqual(new ObjectMD(JSON.parse(Body)).getContentLength(), 0, - 'zero-byte object should be stored correctly'); + await backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: '', + Body: buildMetadataBody({ 'content-length': 0, location: null }), + }), + ); + const { Body } = await backbeatClient.send(new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual( + new ObjectMD(JSON.parse(Body)).getContentLength(), + 0, + 'zero-byte object should be stored correctly', + ); }); - it('should throw MicroVersionIdAlreadyStoredException on loop even with replication content', - async () => { + it('should throw MicroVersionIdAlreadyStoredException on loop even with replication content', async () => { // Verifies that microVersionId loop detection fires regardless of whether // x-scal-replication-content is set const key = 'putmetadata-crr-loop'; const mvId = makeMicroVersionId(); await putMetadata(key, mvId); await assert.rejects( - () => backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET, - Key: key, - MicroVersionId: mvId.encoded, - ReplicationContent: 'METADATA', - Body: buildMetadataBody({ microVersionId: mvId.raw }), - })), + () => + backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: mvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: mvId.raw }), + }), + ), err => { - assert.ok(err instanceof MicroVersionIdAlreadyStoredException, - 'loop detection should fire even with x-scal-replication-content set'); + assert.ok( + err instanceof MicroVersionIdAlreadyStoredException, + 'loop detection should fire even with x-scal-replication-content set', + ); return true; }, ); @@ -300,16 +345,21 @@ describe('putMetadata : cascade replication behavior (with replication content)' const newerMvId = makeMicroVersionId(); await putMetadata(key, newerMvId); await assert.rejects( - () => backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET, - Key: key, - MicroVersionId: olderMvId.encoded, - ReplicationContent: 'METADATA', - Body: buildMetadataBody({ microVersionId: olderMvId.raw }), - })), + () => + backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: olderMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + }), + ), err => { - assert.ok(err instanceof StaleMicroVersionIdException, - 'stale detection should fire even with x-scal-replication-content set'); + assert.ok( + err instanceof StaleMicroVersionIdException, + 'stale detection should fire even with x-scal-replication-content set', + ); return true; }, ); @@ -322,55 +372,78 @@ describe('putMetadata : cascade replication behavior (with replication content)' // First write creates the object (required before METADATA-content writes). await putMetadata(key, olderMvId); // Second write with x-scal-replication-content triggers the cascade block. - await backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET, - Key: key, - MicroVersionId: newerMvId.encoded, - ReplicationContent: 'METADATA', - Body: buildMetadataBody({ microVersionId: newerMvId.raw }), - })); - const { Body } = await backbeatClient.send( - new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + await backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: newerMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: newerMvId.raw }), + }), + ); + const { Body } = await backbeatClient.send(new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); const storedMd = new ObjectMD(JSON.parse(Body)); - assert.strictEqual(storedMd.getReplicationStatus(), '', - 'replication status should be cleared when no CRR rules match'); - assert.deepStrictEqual(storedMd.getReplicationBackends(), [], - 'replication backends should be empty when no CRR rules match'); - assert.strictEqual(storedMd.getReplicationIsReplica(), true, - 'isReplica should be preserved regardless of cascade triggering'); + assert.strictEqual( + storedMd.getReplicationStatus(), + '', + 'replication status should be cleared when no CRR rules match', + ); + assert.deepStrictEqual( + storedMd.getReplicationBackends(), + [], + 'replication backends should be empty when no CRR rules match', + ); + assert.strictEqual( + storedMd.getReplicationIsReplica(), + true, + 'isReplica should be preserved regardless of cascade triggering', + ); }); - it('should set replication status to PENDING and preserve isReplica when bucket has CRR rules', - async () => { + it('should set replication status to PENDING and preserve isReplica when bucket has CRR rules', async () => { const key = 'putmetadata-crr-next-hop'; const olderMvId = makeMicroVersionId(); const newerMvId = makeMicroVersionId(); - await backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET_CRR, - Key: key, - MicroVersionId: olderMvId.encoded, - Body: buildMetadataBody({ microVersionId: olderMvId.raw }), - })); - await backbeatClient.send(new PutMetadataCommand({ - Bucket: TEST_BUCKET_CRR, - Key: key, - MicroVersionId: newerMvId.encoded, - ReplicationContent: 'METADATA', - Body: buildMetadataBody({ microVersionId: newerMvId.raw }), - })); + await backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: olderMvId.encoded, + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + }), + ); + await backbeatClient.send( + new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: newerMvId.encoded, + ReplicationContent: 'METADATA', + Body: buildMetadataBody({ microVersionId: newerMvId.raw }), + }), + ); - const { Body } = await backbeatClient.send( - new GetMetadataCommand({ Bucket: TEST_BUCKET_CRR, Key: key })); + const { Body } = await backbeatClient.send(new GetMetadataCommand({ Bucket: TEST_BUCKET_CRR, Key: key })); const storedMd = new ObjectMD(JSON.parse(Body)); - assert.strictEqual(storedMd.getMicroVersionId(), newerMvId.raw, - 'stored microVersionId should be the newer one'); - assert.strictEqual(storedMd.getReplicationStatus(), 'PENDING', - 'replication status should be PENDING when a CRR rule matches'); - assert.ok(storedMd.getReplicationBackends().length > 0, - 'replication backends should be populated when a CRR rule matches'); - assert.strictEqual(storedMd.getReplicationIsReplica(), true, - 'isReplica should be preserved regardless of cascade triggering'); + assert.strictEqual( + storedMd.getMicroVersionId(), + newerMvId.raw, + 'stored microVersionId should be the newer one', + ); + assert.strictEqual( + storedMd.getReplicationStatus(), + 'PENDING', + 'replication status should be PENDING when a CRR rule matches', + ); + assert.ok( + storedMd.getReplicationBackends().length > 0, + 'replication backends should be populated when a CRR rule matches', + ); + assert.strictEqual( + storedMd.getReplicationIsReplica(), + true, + 'isReplica should be preserved regardless of cascade triggering', + ); }); }); @@ -382,14 +455,18 @@ describe('putMetadata : baseline (no cascade headers)', () => { it('should not set a microVersionId on a regular S3 PutObject', async () => { const key = 'putmetadata-baseline-s3put'; - await s3.send(new PutObjectCommand({ - Bucket: TEST_BUCKET, - Key: key, - Body: Buffer.from(OBJECT_BODY), - })); - const { Body } = await backbeatClient.send( - new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); - assert.strictEqual(new ObjectMD(JSON.parse(Body)).getMicroVersionId(), undefined, - 'a regular S3 PutObject should not set a microVersionId'); + await s3.send( + new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + }), + ); + const { Body } = await backbeatClient.send(new GetMetadataCommand({ Bucket: TEST_BUCKET, Key: key })); + assert.strictEqual( + new ObjectMD(JSON.parse(Body)).getMicroVersionId(), + undefined, + 'a regular S3 PutObject should not set a microVersionId', + ); }); }); diff --git a/tests/unit/api/apiUtils/getReplicationInfo.js b/tests/unit/api/apiUtils/getReplicationInfo.js index 56236661a7..e12b4b3bba 100644 --- a/tests/unit/api/apiUtils/getReplicationInfo.js +++ b/tests/unit/api/apiUtils/getReplicationInfo.js @@ -630,10 +630,31 @@ describe('getReplicationInfo helper', () => { destination: 'tosomewhere', }; const info = getReplicationInfo( - configWithRing, 'fookey', - new BucketInfo('b', 'id', 'name', new Date().toJSON(), - null, null, null, null, null, null, null, null, null, replicationConfig), - true, 123, null, null, null, [RING_TYPE]); + configWithRing, + 'fookey', + new BucketInfo( + 'b', + 'id', + 'name', + new Date().toJSON(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ), + true, + 123, + null, + null, + null, + [RING_TYPE], + ); assert.strictEqual(info.backends.length, 1); assert.strictEqual(info.backends[0].site, 'awsbackend'); }); @@ -645,10 +666,31 @@ describe('getReplicationInfo helper', () => { destination: 'tosomewhere', }; const info = getReplicationInfo( - configWithRing, 'fookey', - new BucketInfo('b', 'id', 'name', new Date().toJSON(), - null, null, null, null, null, null, null, null, null, replicationConfig), - true, 123, null, null, null, [RING_TYPE]); + configWithRing, + 'fookey', + new BucketInfo( + 'b', + 'id', + 'name', + new Date().toJSON(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ), + true, + 123, + null, + null, + null, + [RING_TYPE], + ); assert.strictEqual(info, undefined); }); }); diff --git a/tests/unit/routes/routeBackbeat.js b/tests/unit/routes/routeBackbeat.js index aa0de287e1..19b4cd1ef9 100644 --- a/tests/unit/routes/routeBackbeat.js +++ b/tests/unit/routes/routeBackbeat.js @@ -198,27 +198,33 @@ describe('routeBackbeat', () => { mockRequest.url = '/_/backbeat/data/bucket0/key0'; mockRequest.destroy = () => {}; - metadataUtils.standardMetadataValidateBucketAndObj.callsFake( - (params, denies, log, callback) => { - callback(null, { + metadataUtils.standardMetadataValidateBucketAndObj.callsFake((params, denies, log, callback) => { + callback( + null, + { getVersioningConfiguration: () => ({ Status: 'Enabled' }), isVersioningEnabled: () => true, getLocationConstraint: () => undefined, - }, { + }, + { versionId: rawVersionId, // no microVersionId — old-format object - }); - }); + }, + ); + }); routeBackbeat('127.0.0.1', mockRequest, mockResponse, log); - void await endPromise; + void (await endPromise); sinon.assert.notCalled(storeObject.dataStore); assert.strictEqual(mockResponse.statusCode, 409); assert.strictEqual(mockResponse.body.code, 'VersionIdCollisionException'); const [, responseHeaders] = mockResponse.writeHead.firstCall.args; - assert.strictEqual(responseHeaders['x-scal-micro-version-id'], '', - 'should have empty x-scal-micro-version-id header for old-format objects'); + assert.strictEqual( + responseHeaders['x-scal-micro-version-id'], + '', + 'should have empty x-scal-micro-version-id header for old-format objects', + ); }); describe('putMetadata', () => { From e65838c69060f78656794d010b3d852dc432c7f3 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Thu, 16 Jul 2026 17:24:59 +0200 Subject: [PATCH 3/3] bump to 9.4.0-preview.6 Issue: CLDSRV-897 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e5a5f6a88d..0db3eff54b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@zenko/cloudserver", - "version": "9.4.0-preview.5", + "version": "9.4.0-preview.6", "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol", "main": "index.js", "engines": {