Update to AWS SDK 3 (#380)

* start migration - one test kind of working * one test working! 💥 * dead-letter tests passing 🎉 * tests passing locally * tests passing with tape wildcard locally * update jest tests * create pre-release * temporarily change codepipeline branch from master to aws-sdk-3 * temp change secret * comment out webhook * move codepipeline machine to v7 * reinstate webhook - building binaries in docker for now * refactor to move QueueUrl param location * update fixtures to add QueueUrl * remove unused packages * back to 9.0.1 * reset CF * add changelog and notes on v9.0.1 --------- Co-authored-by: GitHub Actions Bot <>
mapbox · Sep 21, 2023 · 1088763 · 1088763
1 parent 84be7bf
commit 1088763
Show file tree

Hide file tree

Showing 16 changed files with 9,686 additions and 6,113 deletions.
diff --git a/bin/dead-letter.js b/bin/dead-letter.js
@@ -3,7 +3,8 @@
 
 /* eslint-disable no-console */
 
-const AWS = require('aws-sdk');
+const { CloudFormation } = require('@aws-sdk/client-cloudformation')
+const { SQS } = require('@aws-sdk/client-sqs');
 const inquirer = require('inquirer');
 const stream = require('stream');
 const { default: Queue } = require('p-queue');
@@ -33,8 +34,8 @@ const main = async () => {
 
   if (!cli.flags.stackName) cli.showHelp();
 
-  const sqs = new AWS.SQS({ region: cli.flags.region });
-  const cfn = new AWS.CloudFormation({ region: cli.flags.region });
+  const sqs = new SQS({ region: cli.flags.region });
+  const cfn = new CloudFormation({ region: cli.flags.region });
 
   const actions = { purge, writeOut, replay, triage };
 
@@ -46,7 +47,7 @@ const main = async () => {
 };
 
 async function findQueues(cfn, options) {
-  const res = await cfn.describeStacks({ StackName: options.stackName }).promise();
+  const res = await cfn.describeStacks({ StackName: options.stackName });
   if (!res.Stacks[0]) {
     throw new Error(`Could not find ${options.stackName} in ${options.region}`);
   }
@@ -121,7 +122,7 @@ async function purge(sqs, queue) {
   });
 
   if (answers.purge)
-    return await sqs.purgeQueue({ QueueUrl: queue.deadLetter }).promise();
+    return await sqs.purgeQueue({ QueueUrl: queue.deadLetter });
 
   return Promise.resolve();
 }
@@ -269,7 +270,7 @@ async function receive(sqs, count, queueUrl) {
     WaitTimeSeconds: 1,
     MaxNumberOfMessages: count,
     VisibilityTimeout: 10 * 60
-  }).promise();
+  });
 
   return (data.Messages || []).map((message) => ({
     id: message.MessageId,
@@ -285,7 +286,7 @@ async function returnOne(sqs, queueUrl, message) {
     QueueUrl: queueUrl,
     ReceiptHandle: handle,
     VisibilityTimeout: 0
-  }).promise();
+  });
 }
 
 async function returnMany(sqs, queueUrl, handles) {
@@ -304,14 +305,14 @@ async function replayOne(sqs, queueUrl, message) {
   return await sqs.sendMessage({
     QueueUrl: queueUrl,
     MessageBody: message.body
-  }).promise();
+  });
 }
 
 async function deleteOne(sqs, queueUrl, message) {
   return await sqs.deleteMessage({
     QueueUrl: queueUrl,
     ReceiptHandle: message.handle
-  }).promise();
+  });
 }
 
 function receiveAll(sqs, queueUrl) {

diff --git a/bin/watchbot-binary-generator.js b/bin/watchbot-binary-generator.js
@@ -5,7 +5,7 @@
 const fs = require('fs');
 const cp = require('child_process');
 const util = require('util');
-const AWS = require('aws-sdk');
+const { S3 } = require('@aws-sdk/client-s3');
 const exec = util.promisify(cp.exec);
 const wbg = { exec };
 
@@ -37,7 +37,7 @@ wbg.getTagForSha = getTagForSha;
  * uploadBundle - uploads watchbot binaries to S3
  */
 const uploadBundle = async (buildTarget) => {
-  const s3 = new AWS.S3();
+  const s3 = new S3();
   const Bucket = 'watchbot-binaries';
 
   let targets = [
@@ -66,7 +66,7 @@ const uploadBundle = async (buildTarget) => {
         Key: `${target.prefix}/${tag}/watchbot`,
         Body: fs.createReadStream(target.pkg),
         ACL: 'public-read'
-      }).promise();
+      });
     });
 
     await Promise.all(uploads);

diff --git a/changelog.md b/changelog.md
@@ -1,3 +1,8 @@
+### 9.0.1
+
+-  Bug fix: TotalMessagesLambda now working as expected; watchbot stacks now scaling down when no tasks in queue
+-  Move from aws-sdk v2 to v3
+
 ### 9.0.0
 
 -  Breaking change: use Node 18 for lambdas and binaries going forward

diff --git a/cloudformation/ecs-watchbot-generate-binaries.template.js b/cloudformation/ecs-watchbot-generate-binaries.template.js
@@ -86,7 +86,7 @@ const Resources = {
       Environment: {
         Type: 'LINUX_CONTAINER',
         ComputeType: 'BUILD_GENERAL1_SMALL',
-        Image: 'aws/codebuild/standard:5.0'
+        Image: 'aws/codebuild/standard:7.0'
       },
       ServiceRole: cf.getAtt('BundlerRole', 'Arn'),
       Source: {

diff --git a/cloudformation/travis.template.js b/cloudformation/travis.template.js
diff --git a/lib/message.js b/lib/message.js
@@ -1,7 +1,7 @@
 'use strict';
 
 const url = require('url');
-const AWS = require('aws-sdk');
+const { SQS } = require('@aws-sdk/client-sqs');
 const Logger = require('./logger');
 
 class Message {
@@ -50,6 +50,7 @@ class Message {
 
     this.id = sqsMessage.MessageId;
     this.handle = sqsMessage.ReceiptHandle;
+    this.queueUrl = options.queueUrl;
 
     this.env = {
       MessageId: sqsMessage.MessageId,
@@ -65,9 +66,8 @@ class Message {
 
     if (envSubject) this.env.Subject = envSubject;
 
-    this.sqs = new AWS.SQS({
-      region: url.parse(options.queueUrl).host.split('.')[1],
-      params: { QueueUrl: options.queueUrl }
+    this.sqs = new SQS({
+      region: url.parse(options.queueUrl).host.split('.')[1]
     });
 
     this.logger = Logger.create({ type: 'watcher', structuredLogging: options.structuredLogging }, this);
@@ -82,33 +82,38 @@ class Message {
     if (receives > 14) return;
 
     const params = {
+      QueueUrl: this.queueUrl,
       ReceiptHandle: this.handle,
       VisibilityTimeout: Math.pow(2, receives)
     };
 
     try {
-      return await this.sqs.changeMessageVisibility(params).promise();
+      return await this.sqs.changeMessageVisibility(params);
     } catch (err) {
       this.logger.queueError(err);
     }
   }
 
   async heartbeat() {
     const params = {
+      QueueUrl: this.queueUrl,
       ReceiptHandle: this.handle,
       VisibilityTimeout: 180
     };
     try {
-      return await this.sqs.changeMessageVisibility(params).promise();
+      return await this.sqs.changeMessageVisibility(params);
     } catch (err) {
       this.logger.queueError(err);
     }
   }
 
   async complete() {
-    const params = { ReceiptHandle: this.handle };
+    const params = {
+      QueueUrl: this.queueUrl,
+      ReceiptHandle: this.handle
+    };
     try {
-      return await this.sqs.deleteMessage(params).promise();
+      return await this.sqs.deleteMessage(params);
     } catch (err) {
       this.logger.queueError(err);
     }

diff --git a/lib/messages.js b/lib/messages.js
@@ -1,7 +1,7 @@
 'use strict';
 
 const url = require('url');
-const AWS = require('aws-sdk');
+const { SQS } = require('@aws-sdk/client-sqs');
 const Logger = require('./logger');
 const Message = require('./message');
 
@@ -10,16 +10,16 @@ class Messages {
     if (!options.queueUrl) throw new Error('Missing options: queueUrl');
     this.options = options;
 
-    this.sqs = new AWS.SQS({
-      region: url.parse(options.queueUrl).host.split('.')[1],
-      params: { QueueUrl: options.queueUrl }
+    this.sqs = new SQS({
+      region: url.parse(options.queueUrl).host.split('.')[1]
     });
 
     this.logger = Logger.create({ type: 'watcher', structuredLogging: options.structuredLogging });
   }
 
   async waitFor(num = 1) {
     const params = {
+      QueueUrl: this.options.queueUrl,
       AttributeNames: [
         'SentTimestamp',
         'ApproximateFirstReceiveTimestamp',
@@ -35,7 +35,7 @@ class Messages {
 
         let data;
         try {
-          data = await this.sqs.receiveMessage(params).promise();
+          data = await this.sqs.receiveMessage(params);
         } catch (err) {
           this.logger.queueError(err);
           return setImmediate(poll);

diff --git a/lib/template.js b/lib/template.js
@@ -779,15 +779,16 @@ module.exports = (options = {}) => {
       Timeout: 60,
       Code: {
         ZipFile: cf.sub(`
-          const AWS = require('aws-sdk');
+          const { SQS } = require('@aws-sdk/client-sqs');
+          const { CloudWatch } = require('@aws-sdk/client-cloudwatch');
           exports.handler = function(event, context, callback) {
-            const sqs = new AWS.SQS({ region: process.env.AWS_DEFAULT_REGION });
-            const cw = new AWS.CloudWatch({ region: process.env.AWS_DEFAULT_REGION });
+            const sqs = new SQS({ region: process.env.AWS_DEFAULT_REGION });
+            const cw = new CloudWatch({ region: process.env.AWS_DEFAULT_REGION });
 
             return sqs.getQueueAttributes({
               QueueUrl: ${'\'${QueueUrl}\''},
               AttributeNames: ['ApproximateNumberOfMessagesNotVisible', 'ApproximateNumberOfMessages']
-            }).promise()
+            })
               .then((attrs) => {
                 return cw.putMetricData({
                   Namespace: 'Mapbox/ecs-watchbot',
@@ -797,7 +798,7 @@ module.exports = (options = {}) => {
                     Value: Number(attrs.Attributes.ApproximateNumberOfMessagesNotVisible) +
                             Number(attrs.Attributes.ApproximateNumberOfMessages)
                   }]
-                }).promise();
+                })
               })
               .then((metric) => callback(null, metric))
               .catch((err) => callback(err));