CloudWatch Log Group to S3 Bucket Export Automation using Lambda

Prerequisite

CloudWatch log group is created from where the data would be exported.
S3 bucket is created where you want to import the data.

Steps for Configuration

First step is to create a Lambda instance that houses the source code for receiving CloudWatch events and storing them to our S3 instance.

Search for the Lambda service in your AWS account, navigate to functions, and select Create Function.
Under Basic Information, provide:
1. Function name
2. Runtime (Node.js 20x)
3. Instruction set Architecture (x86_64 default)

Make sure your newly created execution role should have following policy:

You can check the policy in Lambda > Configuration > Permission and click role name

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:PutLogEvents",
                "logs:GetLogEvents",
                "logs:DescribeLogStreams",
                "logs:CreateExportTask",
                "logs:FilterLogEvents"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:GetObject",
                "s3:PutObject",
                "s3:ListBucket"
            ],
            "Resource": [
                "arn:aws:s3:::<S3_BUCKET_NAME>",
                "arn:aws:s3:::<S3_BUCKET_NAME>/*"
            ]
        }
    ]
}

Replace S3_BUCKET_NAME with actual bucket name.

Once the lambda function is created. Navigate to Code, copy and paste the following code.

const AWS = require('aws-sdk');
const cloudwatchlogs = new AWS.CloudWatchLogs();
const s3 = new AWS.S3();
const logGroupName = 'LOGGROUP_NAME';
const s3BucketName = 'S3_BUCKET_NAME';
const s3KeyPrefix = 'PREFIX/'; 
exports.handler = (event, context, callback) => {
    getLastExportedTimestamp((lastTimestampErr, lastTimestamp) => {
        if (lastTimestampErr) {
            console.error('Error getting last exported timestamp:', lastTimestampErr);
            return callback(lastTimestampErr);
        }
            processLogs(lastTimestamp + 1, callback);
    });
};
function getLastExportedTimestamp(callback) {
    s3.getObject({ Bucket: s3BucketName, Key: 'lastTimestamp.txt' }, (err, data) => {
        if (err) {
            if (err.code === 'NoSuchKey') {
                callback(null, 0); // Start from the beginning if the file doesn't exist
            } else {
                callback(err);
            }
        } else {
            callback(null, parseInt(data.Body.toString('utf-8').trim()));
        }
    });
}
function processLogs(lastTimestamp, callback) {
    let latestTimestamp = lastTimestamp;
    function retrieveLogs(token) {
        getLogEvents(logGroupName, lastTimestamp, token, (err, data) => {
            if (err) {
                console.error(`Error getting log events for stream:`, err);
                return callback(err);
            }
            const logEvents = data.events;
            const nextToken = data.nextToken;
            if (logEvents.length > 0) {
                exportToS3(logEvents, (exportErr) => {
                    if (exportErr) {
                        console.error(`Error exporting log events to S3 for stream:`, exportErr);
                        return callback(exportErr);
                    }
                    const streamLatestTimestamp = logEvents[logEvents.length - 1].timestamp;
                    latestTimestamp = Math.max(latestTimestamp, streamLatestTimestamp);
                    if (nextToken) {
                        retrieveLogs(nextToken);
                    } else {
                        updateLastExportedTimestamp(latestTimestamp, callback);
                    }
                });
            } else {
                if (nextToken) {
                    retrieveLogs(nextToken);
                } else {
                    updateLastExportedTimestamp(latestTimestamp, callback);
                }
            }
        });
    }
    // Start the recursive retrieval
    retrieveLogs(null);
}
function getLogEvents(logGroupName, startTime, token, callback) {
    const params = {
        logGroupName: logGroupName,
        startTime: startTime,
        nextToken: token
    };
    cloudwatchlogs.filterLogEvents(params, (err, data) => {
        if (err) {
            callback(err);
        } else {
            callback(null, data);
        }
    });
}
function exportToS3(logEvents, callback) {
    const logData = logEvents.map(event => event.message).join('\n');
    const s3Key = `${s3KeyPrefix}${Date.now()}.txt`;
    s3.putObject({ Bucket: s3BucketName, Key: s3Key, Body: logData }, (err, data) => {
        if (err) {
            callback(err);
        } else {
            callback(null);
        }
    });
}
function updateLastExportedTimestamp(timestamp, callback) {
    s3.putObject({ Bucket: s3BucketName, Key: 'lastTimestamp.txt', Body: timestamp.toString() }, (err, data) => {
        if (err) {
            callback(err);
        } else {
            callback(null);
        }
    });
}

Replace s3BucketName, logGroupName, s3KeyPrefix with actual values.

image-20240209-114741 (1).png

Click Deploy
Click to + Add Trigger and choose EventBridge

Select Create new rule
Provide the Rule name
Enter the Rule description
Schedule expression will act as CRON which will automatically trigger the event on matching expression.

Set the 2-minutes rate which invokes the lambda function every 2 minutes. You can specify as per organization’s policy or the interval you want lambda to execute.

Syntax:

rate(value unit)

Make sure your S3 bucket has following policy:

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "AWS": "<LAMBDA_EXECUTION_IAM_ROLE_ARN>"
            },
            "Action": [
                "s3:GetObject",
                "s3:PutObject",
                "s3:ListBucket"
            ],
            "Resource": [
                "arn:aws:s3:::<S3_BUCKET_NAME>",
                "arn:aws:s3:::<S3_BUCKET_NAME>/*"
            ]
        }
    ]
}

Replace S3_BUCKET_NAME and <LAMBDA_EXECUTION_IAM_ROLE_ARN> with your actual values.

You can get <LAMBDA_EXECUTION_IAM_ROLE_ARN> from: lambda > configuration > permission > click on the role name → Copy the ARN