CLDSRV-899: Log replication events in server access logs

.github/docker/docker-compose.yaml

@@ -125,7 +125,7 @@ services:
     depends_on:
       - redis
   metadata-standalone:
-    image: ghcr.io/scality/metadata:8.11.0-standalone
+    image: ghcr.io/scality/metadata:9.11.0-standalone
     profiles: ['metadata-standalone']
     network_mode: 'host'
     volumes:

lib/routes/routeBackbeat.js

@@ -408,6 +408,10 @@ function createCipherBundle(bucketInfo, isV2Request, log, cb) {
 }
 
 function putData(request, response, bucketInfo, objMd, log, callback) {
+    if (request.serverAccessLog) {
+        // eslint-disable-next-line no-param-reassign
+        request.serverAccessLog.replication = true;
+    }
     let errMessage;
     const canonicalID = request.headers['x-scal-canonical-id'];
     if (canonicalID === undefined) {
@@ -532,6 +536,52 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
 
         const { headers, bucketName, objectKey } = request;
 
+        // Destination-side delete-marker replication.
+        // We need the REPLICA status to distinguish from
+        // source-side replication status updates that also carry isDeleteMarker=true.
+        if (omVal.isDeleteMarker
+            && omVal.replicationInfo
+            && omVal.replicationInfo.status === 'REPLICA'
+            && request.serverAccessLog) {
+            // eslint-disable-next-line no-param-reassign
+            request.serverAccessLog.replication = true;
+            // eslint-disable-next-line no-param-reassign
+            request.serverAccessLog.deleteMarker = true;
+        }
+
+        // Destination-side tag-only replication.
+        // AWS uses REST.PUT.OBJECT_TAGGING for both - a tag-delete
+        // is replicated as a PUT of an empty tag set with the same
+        // URI shape.
+        // The REPLICA status excludes source-side replication-status updates.
+        if (omVal.replicationInfo
+            && omVal.replicationInfo.status === 'REPLICA'
+            && (omVal.originOp === 's3:ObjectTagging:Put'
+                || omVal.originOp === 's3:ObjectTagging:Delete')
+            && request.serverAccessLog) {
+            // eslint-disable-next-line no-param-reassign
+            request.serverAccessLog.replication = true;
+            // eslint-disable-next-line no-param-reassign
+            request.serverAccessLog.tagging = true;
+        }
+
+        // Destination-side ACL-only replication.
+        // AWS uses REST.PUT.ACL on the destination with URI
+        // PUT /<bucket>/<key>?acl&versionId=<srcVersionId> and
+        // populates the aclRequired field.
+        // The REPLICA status excludes source-side replication-status updates.
+        if (omVal.replicationInfo
+            && omVal.replicationInfo.status === 'REPLICA'
+            && omVal.originOp === 's3:ObjectAcl:Put'
+            && request.serverAccessLog) {
+            // eslint-disable-next-line no-param-reassign
+            request.serverAccessLog.replication = true;
+            // eslint-disable-next-line no-param-reassign
+            request.serverAccessLog.acl = true;
+            // eslint-disable-next-line no-param-reassign
+            request.serverAccessLog.aclRequired = 'Yes';
+        }
+
         if (headers['x-scal-replication-content'] === 'METADATA') {
             if (!objMd) {
                 return callback(errors.ObjNotFound);

lib/utilities/serverAccessLogger.js

@@ -291,6 +291,18 @@ function getOperation(req) {
         if (req.serverAccessLog.expiration) {
             return 'S3.EXPIRE.OBJECT';
         }
+        if (req.serverAccessLog.replication) {
+            if (req.serverAccessLog.deleteMarker) {
+                return 'REST.DELETE.OBJECT';
+            }
+            if (req.serverAccessLog.tagging) {
+                return 'REST.PUT.OBJECT_TAGGING';
+            }
+            if (req.serverAccessLog.acl) {
+                return 'REST.PUT.ACL';
+            }
+            return 'REST.PUT.OBJECT';
+        }
         return `REST.${req.method}.BACKBEAT`;
     }
 
@@ -519,9 +531,10 @@ function buildLogEntry(req, params, options) {
 
         // Scality server access logs extra fields
         logFormatVersion: SERVER_ACCESS_LOG_FORMAT_VERSION,
-        // For non-expiration backbeat requests, force loggingEnabled
-        // to false to prevent delivery to log courier.
-        loggingEnabled: (params.backbeat && !params.expiration) ? false : (params.enabled ?? undefined),
+        // For backbeat requests other than expiration and replication,
+        // force loggingEnabled to false to prevent delivery to log courier.
+        loggingEnabled: (params.backbeat && !params.expiration && !params.replication)
+            ? false : (params.enabled ?? undefined),
         loggingTargetBucket: params.loggingEnabled?.TargetBucket ?? undefined,
         loggingTargetPrefix: params.loggingEnabled?.TargetPrefix ?? undefined,
         awsAccessKeyID: authInfo?.getAccessKey() ?? undefined,
@@ -628,6 +641,38 @@ function logServerAccess(req, res) {
         logEntry.awsAccessKeyID = undefined;
     }
 
+    // Match AWS log shape for replication entries: blank clientIP, userAgent
+    // and referer, and rewrite requestURI from the internal /_/backbeat/{data,metadata}
+    // endpoint to a standard verb on the destination bucket.
+    // The synthesized requestURI matches the AWS requestURI for replication.
+    // It is not an actual cloudserver endpoint (replication is delivered via /_/backbeat/...);
+    // the operation field is the authoritative signal that the entry is a replication event.
+    //   - Delete-marker replication arrives as a putMetadata (PUT) but is logged
+    //     as a DELETE.
+    //   - Tag-only replication appends ?tagging&versionId=<destVid> to match the
+    //     AWS PutObjectTagging URI.
+    //   - ACL-only replication appends ?acl&versionId=<destVid> to match the
+    //     AWS PutObjectAcl URI.
+    if (params.replication) {
+        logEntry.clientIP = undefined;
+        logEntry.userAgent = undefined;
+        logEntry.referer = undefined;
+        let method = req.method;
+        let query = '';
+        if (params.deleteMarker) {
+            method = 'DELETE';
+        } else if (params.tagging) {
+            const versionId = req.query?.versionId;
+            query = versionId ? `?tagging&versionId=${versionId}` : '?tagging';
+        } else if (params.acl) {
+            const versionId = req.query?.versionId;
+            query = versionId ? `?acl&versionId=${versionId}` : '?acl';
+        }
+        const encodedKey = params.objectKey.split('/').map(encodeURIComponent).join('/');
+        logEntry.requestURI =
+            `${method} /${params.bucketName}/${encodedKey}${query} HTTP/${req.httpVersion ?? '1.1'}`;
+    }
+
     if (params.internalLogRequestQueue && params.internalLogRequestQueue.length > 0) {
         if (logEntry.operation === 'REST.POST.MULTI_OBJECT_DELETE') {
             for (const entry of params.internalLogRequestQueue) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenko/cloudserver",
-  "version": "9.2.39",
+  "version": "9.2.40",
   "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
   "main": "index.js",
   "engines": {

tests/multipleBackend/backendHealthcheckResponse.js

@@ -20,7 +20,9 @@ const log = new DummyRequestLogger();
 const locConstraints = Object.keys(config.locationConstraints);
 const azureClient = getAzureClient();
 
-describe('Healthcheck response', () => {
+describe('Healthcheck response', function describeHealthcheck() {
+    this.timeout(60000);
+
     it('should return result for every location constraint in ' +
     'locationConfig and every external locations with flightCheckOnStartUp ' +
     'set to true', done => {