From 870b9404dc4d05b05c2d843301a3cc2a1f139902 Mon Sep 17 00:00:00 2001 From: diugalde Date: Mon, 26 Sep 2016 10:59:34 -0600 Subject: [PATCH] Add index-service features --- .editorconfig | 15 ++++++ .gitignore | 35 +++++++++++++ README.md | 28 ++++++++++ bin/index_service | 4 ++ lib/config.js | 18 +++++++ lib/index_service.js | 66 ++++++++++++++++++++++++ lib/lo_indexer.js | 119 +++++++++++++++++++++++++++++++++++++++++++ lib/log.js | 20 ++++++++ package.json | 36 +++++++++++++ 9 files changed, 341 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100755 bin/index_service create mode 100644 lib/config.js create mode 100644 lib/index_service.js create mode 100644 lib/lo_indexer.js create mode 100644 lib/log.js create mode 100644 package.json diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..80902cb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[**.{md,markdown}] +trim_trailing_whitespace = false + +[**.{js}] +indent_style = space +indent_size = 4 +indent_brace_style = K&R diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4aad341 --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +# Logs +*.log + +# Runtime data +pids +*.pid +*.seed + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directory +# Deployed apps should consider commenting this line out: +# see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git +node_modules +.tmp + +# bower +app/bower_components +test/bower_components + +# WebStorm idea +.idea +*.iws +.idea_modules/ +.envrc \ No newline at end of file diff --git a/README.md b/README.md index e69de29..33348db 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,28 @@ +# cl-index + +Common Library Index Service + +--- + +## Build and Run + +1. First, you need to install nodejs v6.4.0. You can follow the instructions here [NodeJS](https://nodejs.org). + +2. Install all the dependencies: +```bash +$ npm install +``` + +3. You must set some environment variables before running the application: +```bash +$ export CL_ES_URL=localhost:9200 +$ export CL_RMQ_URL=amqp://localhost +``` + +4. Before executing the main file, you need to make sure that your rabbitMQ and elasticSearch instances are running (check docker-compose file in the cl-lo project). + +5. Execute the main file to start the server: +```bash +$ chmod +x bin/index_service +$ ./bin/index_service +``` diff --git a/bin/index_service b/bin/index_service new file mode 100755 index 0000000..bf37b5f --- /dev/null +++ b/bin/index_service @@ -0,0 +1,4 @@ +#!/usr/bin/node + +const indexService = require('../lib/index_service'); +indexService.main(); diff --git a/lib/config.js b/lib/config.js new file mode 100644 index 0000000..aed080e --- /dev/null +++ b/lib/config.js @@ -0,0 +1,18 @@ +/** + * Created by diugalde on 23/09/16. + */ + +const config = { + elasticSearch: { + url: 'localhost:9200' || process.env.CL_ES_URL, + index: 'clc_learning_object_index', + docType: 'clc_learning_object' + }, + + rabbitMQ: { + url: 'amqp://localhost' || process.env.CL_RMQ_URL, + loQueue: 'cl_lo_queue' + } +}; + +module.exports = config; diff --git a/lib/index_service.js b/lib/index_service.js new file mode 100644 index 0000000..be941c9 --- /dev/null +++ b/lib/index_service.js @@ -0,0 +1,66 @@ +/** + * Created by diugalde on 23/09/16. + */ + +const amqp = require('amqplib'); + +const config = require('./config'); +const loIndexer = require('./lo_indexer'); +const log = require('./log'); + + +/** + * Processes queue messages. + * + * @param msg - JSON string with the following format: { + * action: add | remove | update, + * content: object + * } + * @returns Promise - It will true if the msg was successfully processed. Error otherwise. + */ +function process(msg) { + try { + let msgObj = JSON.parse(msg.content.toString()); + if (msgObj.action && loIndexer[msgObj.action]) { + log.info(`Executing ${msgObj.action} operation...`); + return loIndexer[msgObj.action](msgObj.content); + } else { + // If it is an unknown msg, just return true. + return Promise.resolve(true) + } + } catch(err) { + return Promise.reject(err) + } +} + +function main() { + + log.info('Starting cl-index...'); + + const queueName = config.rabbitMQ.loQueue; + + // Connects and listen RabbitMQ messages. + amqp.connect(config.rabbitMQ.url).then(function(conn) { + return conn.createChannel(); + }).then(function(ch) { + return ch.assertQueue(queueName).then(function() { + return ch.consume(queueName, function(msg) { + if (msg) { + process(msg).then(function(res) { + if (res === true) { + ch.ack(msg) + } + }).catch(function(err) { + log.info(err) + }) + } + }); + }); + }).catch(function(err) { + log.info(err) + }); +} + +module.exports = { + main +}; diff --git a/lib/lo_indexer.js b/lib/lo_indexer.js new file mode 100644 index 0000000..c0012d7 --- /dev/null +++ b/lib/lo_indexer.js @@ -0,0 +1,119 @@ +/** + * Created by diugalde on 23/09/16. + */ + + +const elasticSearch = require('elasticsearch'); +const rp = require('request-promise'); + +const config = require('./config'); +const log = require('./log'); + + +const docType = config.elasticSearch.docType; +const indexName = config.elasticSearch.index; + +const client = new elasticSearch.Client({ + host: config.elasticSearch.url +}); + +var loIndexer = { + + /** + * Creates a new document in the elasticSearch index. + * + * @param lo - object (LearningObject structure) + * @returns Promise or error. + */ + add(lo) { + return client.index({ + index: indexName, + type: docType, + id: lo.id, + body: lo + }).then(function(res) { + // May have to do file work here (get from url or ). + log.info(res); + return Promise.resolve(true) + }).catch(function(err) { + return Promise.reject(err) + }) + }, + + /** + * Removes document from the elasticSearch index. + * + * @param loId - string (LearningObject's id). + * @returns Promise or error. + */ + remove(loId) { + return client.delete({ + index: indexName, + type: docType, + id: loId + }).then(function(res) { + log.info(res); + return Promise.resolve(true) + }).catch(function(err) { + if (err.status === 404) { + log.info(`Lo with id '${loId}' not found.`); + return Promise.resolve(true) + } + return Promise.reject(err) + }) + }, + + /** + * Updates existing document in elasticSearch. + * + * @param updatedLO - object (This object should have the following structure: {doc: LearningObject, file: null | {base64: '', format: ''}} + * @returns Promise or error. + */ + update(updatedLO) { + let lo = updatedLO.doc; + + return _getAttachment(updatedLO.file).then(function(attachment) { + if (attachment !== null) { + lo.file = attachment + } + return client.update({ + index: indexName, + type: docType, + id: lo.id, + body: {doc: lo} + }) + }).then(function(res){ + log.info(res); + return Promise.resolve(true) + }).catch(function(err) { + return Promise.reject(err) + }) + } +}; + + +/** + * Retrieves the file's base64 string. If the fileObj has an url format, it will download the HTML page first. + * + * @param fileObj - object (Example structure: null | {base64: '', format: ''}) + * @returns Promise - (Base64 equivalent). + * @private + */ +function _getAttachment(fileObj) { + if (fileObj && fileObj !== null) { + if (fileObj.format === 'url') { + let url = new Buffer(fileObj.base64, 'base64').toString('ascii'); + return rp(url).then(function(res) { + return Promise.resolve(new Buffer(res).toString('base64')) + }).catch(function(err) { + log.info(err); + return Promise.resolve(new Buffer(`Error while getting ${url}`).toString('base64')) + }) + } + return Promise.resolve(fileObj.base64) + } + return Promise.resolve(null) +} + + +module.exports = Object.create(loIndexer); diff --git a/lib/log.js b/lib/log.js new file mode 100644 index 0000000..3dcb3c7 --- /dev/null +++ b/lib/log.js @@ -0,0 +1,20 @@ +/** + * Created by diugalde on 23/09/16. + */ + +const bunyan = require('bunyan'); +const bunyanFormat = require('bunyan-format'); + +const APP_NAME = 'cl-index'; + +// Init bunyan log with a nice output format. +var bunyanFormatStdOut = bunyanFormat({ outputMode: 'short' }); + +var log = bunyan.createLogger({ + name: APP_NAME, + streams: [ + { level: "info", stream: bunyanFormatStdOut } + ] +}); + +module.exports = log; diff --git a/package.json b/package.json new file mode 100644 index 0000000..99a6ef5 --- /dev/null +++ b/package.json @@ -0,0 +1,36 @@ +{ + "name": "cl-index", + "version": "0.0.1", + "description": "Common Library index service. Reads messages from a RabbitMQ queue and interacts with an ElasticSearch index.", + "main": "index_service.js", + "scripts": { + "test": "mocha --recursive --reporter spec" + }, + "repository": { + "type": "git", + "url": "git+ssh://git@gitlab.com:edify/cl-index.git" + }, + "keywords": [ + "cl", + "common", + "library", + "index", + "elasticSearch", + "rabbitMQ" + ], + "author": "diugalde", + "license": "ISC", + "bugs": { + "url": "https://gitlab.com/edify/cl-index/issues" + }, + "homepage": "https://gitlab.com/edify/cl-index.git#README", + "dependencies": { + "amqplib": "^0.4.2", + "bunyan": "^1.8.1", + "bunyan-format": "^0.2.1", + "elasticsearch": "^11.0.1", + "mocha": "^3.0.2", + "request": "^2.75.0", + "request-promise": "^4.1.1" + } +}