Skip to content

Instantly share code, notes, and snippets.

@dcortesnet
Last active April 3, 2023 23:25
Show Gist options
  • Save dcortesnet/c6c42c7664b691d4243f0781dd52d6b0 to your computer and use it in GitHub Desktop.
Save dcortesnet/c6c42c7664b691d4243f0781dd52d6b0 to your computer and use it in GitHub Desktop.
Nodejs aws SQS server producer and consumer
const express = require('express');
const fs = require('fs');
const app = express();
const port = 3000;
const dotenv = require('dotenv');
const { SQS, SendMessageCommand } = require('@aws-sdk/client-sqs');
const { Consumer } = require('sqs-consumer');
dotenv.config();
const clientSQS = new SQS({
credentials: {
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_KEY,
},
region: process.env.AWS_REGION,
apiVersion: '2012-11-05',
});
const consumerSQS = Consumer.create({
queueUrl: process.env.AWS_SQS_OUTPUT_QUEUE_URL,
handleMessage: async (message) => {
const body = message.Body;
const product = JSON.parse(body);
console.log(product);
},
sqs: clientSQS
});
consumerSQS.on("error", (err) => {
console.log("AWS SQS error", err);
});
consumerSQS.on('processing_error', (err) => {
console.log("AWS SQS processing error", err);
});
consumerSQS.on('timeout_error', (err) => {
console.log("AWS SQS timeout error", err);
});
consumerSQS.start();
app.post('/execute', async (req, res) => {
try {
const messagesIds = [];
const data = fs.readFileSync('./products.json');
const { products } = JSON.parse(data);
for (const product of products) {
const sqsMessage = {
DelaySeconds: 2,
MessageBody: JSON.stringify(product),
QueueUrl: process.env.AWS_SQS_INPUT_QUEUE_URL
};
const command = new SendMessageCommand(sqsMessage);
const response = await clientSQS.send(command);
messagesIds.push(response);
}
return res.json({ message: "The scraping are starting", messagesIds })
} catch (error) {
return res.status(500).json({ message: "Internal server error" })
}
});
app.listen(port, () => {
console.log(`Example app listening on port ${port}`);
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment