From d1af4d5b309228e2ab45cee8f1304729b3b652a3 Mon Sep 17 00:00:00 2001 From: ArFe Date: Mon, 29 Mar 2021 14:43:17 -0400 Subject: [PATCH 1/4] Fix commander not parsing arguments Commander does not accept getting the options directly from program. Need to create options variable. Added this line (after .parse(process.argv) ): const options = program.opts(); And replace program by options thereafter. --- dynamoDBtoCSV.js | 68 +++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/dynamoDBtoCSV.js b/dynamoDBtoCSV.js index 38ef6d0..3f55ae3 100644 --- a/dynamoDBtoCSV.js +++ b/dynamoDBtoCSV.js @@ -3,6 +3,7 @@ const AWS = require("aws-sdk"); const unmarshal = require("dynamodb-marshaler").unmarshal; const Papa = require("papaparse"); const fs = require("fs"); +const { stringify } = require("querystring"); let headers = []; let unMarshalledArray = []; @@ -32,31 +33,33 @@ program .option("-s, --size [size]", "Number of lines to read before writing.", 5000) .parse(process.argv); -if (!program.table) { + const options = program.opts(); + +if (!options.table) { console.log("You must specify a table"); - program.outputHelp(); + options.outputHelp(); process.exit(1); } -if (program.region && AWS.config.credentials) { - AWS.config.update({ region: program.region }); +if (options.region && AWS.config.credentials) { + AWS.config.update({ region: options.region }); } else { AWS.config.loadFromPath(__dirname + "/config.json"); } -if (program.endpoint) { - AWS.config.update({ endpoint: program.endpoint }); +if (options.endpoint) { + AWS.config.update({ endpoint: options.endpoint }); } -if (program.profile) { - let newCreds = new AWS.SharedIniFileCredentials({ profile: program.profile }); - newCreds.profile = program.profile; +if (options.profile) { + let newCreds = new AWS.SharedIniFileCredentials({ profile: options.profile }); + newCreds.profile = options.profile; AWS.config.update({ credentials: newCreds }); } -if (program.envcreds) { +if (options.envcreds) { let newCreds = AWS.config.credentials; - newCreds.profile = program.profile; + newCreds.profile = options.profile; AWS.config.update({ credentials: { accessKeyId: process.env.AWS_ACCESS_KEY_ID, @@ -66,15 +69,15 @@ if (program.envcreds) { }); } -if (program.mfa && program.profile) { +if (options.mfa && options.profile) { const creds = new AWS.SharedIniFileCredentials({ - tokenCodeFn: (serial, cb) => {cb(null, program.mfa)}, - profile: program.profile + tokenCodeFn: (serial, cb) => {cb(null, options.mfa)}, + profile: options.profile }); // Update config to include MFA AWS.config.update({ credentials: creds }); -} else if (program.mfa && !program.profile) { +} else if (options.mfa && !options.profile) { console.log('error: MFA requires a profile(-p [profile]) to work'); process.exit(1); } @@ -82,33 +85,33 @@ if (program.mfa && program.profile) { const dynamoDB = new AWS.DynamoDB(); const query = { - TableName: program.table, - IndexName: program.index, - Select: program.count ? "COUNT" : (program.select ? "SPECIFIC_ATTRIBUTES" : (program.index ? "ALL_PROJECTED_ATTRIBUTES" : "ALL_ATTRIBUTES")), - KeyConditionExpression: program.keyExpression, - ExpressionAttributeValues: JSON.parse(program.keyExpressionValues), - ProjectionExpression: program.select, + TableName: options.table, + IndexName: options.index, + Select: options.count ? "COUNT" : (options.select ? "SPECIFIC_ATTRIBUTES" : (options.index ? "ALL_PROJECTED_ATTRIBUTES" : "ALL_ATTRIBUTES")), + KeyConditionExpression: options.keyExpression, + ExpressionAttributeValues: JSON.parse(options.keyExpressionValues), + ProjectionExpression: options.select, Limit: 1000 }; const scanQuery = { - TableName: program.table, - IndexName: program.index, + TableName: options.table, + IndexName: options.index, Limit: 1000 }; // if there is a target file, open a write stream -if (!program.describe && program.file) { - var stream = fs.createWriteStream(program.file, { flags: 'a' }); +if (!options.describe && options.file) { + var stream = fs.createWriteStream(options.file, { flags: 'a' }); } let rowCount = 0; let writeCount = 0; -let writeChunk = program.size; +let writeChunk = options.size; const describeTable = () => { dynamoDB.describeTable( { - TableName: program.table + TableName: options.table }, function (err, data) { if (!err) { @@ -142,7 +145,7 @@ const describeTable = () => { const appendStats = (params, items) => { for (let i = 0; i < items.length; i++) { let item = items[i]; - let key = item[program.stats].S; + let key = item[options.stats].S; if (params.stats[key]) { params.stats[key]++; @@ -199,7 +202,7 @@ const queryDynamoDB = (params) => { let query = params.query; dynamoDB.query(query, function (err, data) { if (!err) { - if (program.stats) { + if (options.stats) { processStats(params, data); } else { processRows(params, data); @@ -219,7 +222,7 @@ const unparseData = (lastEvaluatedKey) => { // remove column names after first write chunk. endData = endData.replace(/(.*\r\n)/, "");; } - if (program.file) { + if (options.file) { writeData(endData); } else { console.log(endData); @@ -265,6 +268,7 @@ const unMarshalIntoArray = (items) => { }); } -if (program.describe) describeTable(scanQuery); -if (program.keyExpression) queryDynamoDB({ "query": query, stats: {} }); +if (options.describe) describeTable(scanQuery); +if (options.keyExpression) queryDynamoDB({ "query": query, stats: {} }); else scanDynamoDB(scanQuery); + From 44de6878a517e828b0f69f5a8e3547c109fb8a67 Mon Sep 17 00:00:00 2001 From: ArFe Date: Mon, 29 Mar 2021 14:48:36 -0400 Subject: [PATCH 2/4] Fix commander not parsing arguments Commander does not accept getting the options directly from program. Need to create options variable. Added this line (after .parse(process.argv) ): const options = program.opts(); And replace program by options thereafter. --- dynamoDBtoCSV.js | 1 - 1 file changed, 1 deletion(-) diff --git a/dynamoDBtoCSV.js b/dynamoDBtoCSV.js index 3f55ae3..5269256 100644 --- a/dynamoDBtoCSV.js +++ b/dynamoDBtoCSV.js @@ -3,7 +3,6 @@ const AWS = require("aws-sdk"); const unmarshal = require("dynamodb-marshaler").unmarshal; const Papa = require("papaparse"); const fs = require("fs"); -const { stringify } = require("querystring"); let headers = []; let unMarshalledArray = []; From c051fee840df0c63ad190bc3a86bf491a54326a2 Mon Sep 17 00:00:00 2001 From: ArFe Date: Mon, 29 Mar 2021 15:40:19 -0400 Subject: [PATCH 3/4] Correct outputHelp(); --- dynamoDBtoCSV.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dynamoDBtoCSV.js b/dynamoDBtoCSV.js index 5269256..76a349c 100644 --- a/dynamoDBtoCSV.js +++ b/dynamoDBtoCSV.js @@ -36,7 +36,7 @@ program if (!options.table) { console.log("You must specify a table"); - options.outputHelp(); + program.outputHelp(); process.exit(1); } From 36a6a0a920cb6a85f8c16471e07894390228302d Mon Sep 17 00:00:00 2001 From: ArFe Date: Wed, 31 Mar 2021 08:52:36 -0400 Subject: [PATCH 4/4] Add ProjectionExpression: options.select to the ScanQuery Add ProjectionExpression: options.select to the ScanQuery to fix --select not selecting specific fields if no primary key is presented. --- dynamoDBtoCSV.js | 1 + 1 file changed, 1 insertion(+) diff --git a/dynamoDBtoCSV.js b/dynamoDBtoCSV.js index 76a349c..b54cfc2 100644 --- a/dynamoDBtoCSV.js +++ b/dynamoDBtoCSV.js @@ -96,6 +96,7 @@ const query = { const scanQuery = { TableName: options.table, IndexName: options.index, + ProjectionExpression: options.select, Limit: 1000 };