Scribd
Upload
3 views2 pages

TFD

This document is a Node.js application using Express and Multer for handling CSV file uploads. It processes the uploaded CSV files, uploads the data to an S3 bucket, and integrates with AWS QuickSight by retrieving the DataSetId. The application runs a server on port 3000 and provides an endpoint for file uploads, returning a success message upon completion.

Uploaded by

Hardik
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as TXT, PDF, TXT or read online on Scribd
3 views2 pages

TFD

This document is a Node.js application using Express and Multer for handling CSV file uploads. It processes the uploaded CSV files, uploads the data to an S3 bucket, and integrates with AWS QuickSight by retrieving the DataSetId. The application runs a server on port 3000 and provides an endpoint for file uploads, returning a success message upon completion.

Uploaded by

Hardik
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as TXT, PDF, TXT or read online on Scribd
You are on page 1/ 2

const express = require('express');

const multer = require('multer');


const csv = require('csv-parser');
const fs = require('fs');
const AWS = require('aws-sdk');

const app = express();


const port = 3000;

// Set up Multer for handling file uploads


const upload = multer({ dest: 'uploads/' });

// Initialize AWS SDK with your credentials and region


AWS.config.update({
accessKeyId: 'AKIA6GBMATKKV4BUOU6K',
secretAccessKey: '+I/XUqoUMfr4C4D/5jvCK3hTl5CMCBZftqWIdyZ6',
region: 'Asia Pacific (Mumbai)'
});

// Create an instance of the S3 service


const s3 = new AWS.S3();
// Create an instance of the QuickSight service
const quicksight = new AWS.QuickSight();

// Define a function to retrieve DataSetId


function getDataSetId(dataSetName, callback) {
// Define parameters for listDataSets API operation
const listDataSetsParams = {
AwsAccountId: '975049890453' // Replace with your AWS account ID
};

// Call listDataSets API operation


quicksight.listDataSets(listDataSetsParams, (err, data) => {
if (err) {
console.error('Error listing datasets:', err);
callback(err, null);
} else {
// Iterate through datasets and find the one you're interested in
const dataSet = data.DataSetSummaries.find(dataset => dataset.Name ===
dataSetName);
if (dataSet) {
callback(null, dataSet.DataSetId);
} else {
console.log('Dataset not found');
callback('Dataset not found', null);
}
}
});
}

// Endpoint to handle file uploads, data processing, and QuickSight integration


app.post('/upload', upload.single('csvFile'), async (req, res) => {
if (!req.file) {
return res.status(400).json({ error: 'No file uploaded' });
}

try {
// Process the uploaded CSV file
const filePath = req.file.path;
const results = [];

fs.createReadStream(filePath)
.pipe(csv())
.on('data', (data) => {
// Process each row of data as needed
results.push(data);
})
.on('end', async () => {
// Upload processed data to S3
const bucketName = 'cclp';
const objectKey = `processed-data/${Date.now()}.json`; // Example
object key
const s3Params = {
Bucket: bucketName,
Key: objectKey,
Body: JSON.stringify({ csvData: results }),
ContentType: 'application/json'
};

await s3.upload(s3Params).promise();

// Define parameters for QuickSight integration


const dataSetName = 'amazon-data-source'; // Name of your dataset
in QuickSight

// Retrieve DataSetId using getDataSetId function


getDataSetId(dataSetName, (err, dataSetId) => {
if (err) {
// Handle error
console.error('Error retrieving DataSetId:', err);
res.status(500).json({ error: 'Error retrieving
DataSetId' });
} else {
// Use dataSetId for further QuickSight integration
console.log('DataSetId:', dataSetId);
res.json({ message: 'Data processed, stored in S3, and
integrated with QuickSight. DataSetId: ' + dataSetId });
}
});
});
} catch (error) {
console.error('Error processing CSV file:', error);
res.status(500).json({ error: 'Error processing CSV file' });
}
});

// Start the server


app.listen(port, () => {
console.log(`Backend server is running at http://localhost:${port}`);
});

You might also like