Add index-service features

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[**.{md,markdown}]
+trim_trailing_whitespace = false
+
+[**.{js}]
+indent_style = space
+indent_size = 4
+indent_brace_style = K&R

.gitignore

@@ -0,0 +1,35 @@
+# Logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# Deployed apps should consider commenting this line out:
+# see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git
+node_modules
+.tmp
+
+# bower
+app/bower_components
+test/bower_components
+
+# WebStorm idea
+.idea
+*.iws
+.idea_modules/
+.envrc

README.md

@@ -0,0 +1,28 @@
+# cl-index
+
+Common Library Index Service
+
+---
+
+## Build and Run
+
+1.  First, you need to install nodejs v6.4.0. You can follow the instructions here [NodeJS](https://nodejs.org).
+
+2.  Install all the dependencies:
+```bash
+$ npm install
+```
+
+3.  You must set some environment variables before running the application:
+```bash
+$ export CL_ES_URL=localhost:9200
+$ export CL_RMQ_URL=amqp://localhost
+```
+
+4.  Before executing the main file, you need to make sure that your rabbitMQ and elasticSearch instances are running (check docker-compose file in the cl-lo project).
+
+5.  Execute the main file to start the server:
+```bash
+$ chmod +x bin/index_service
+$ ./bin/index_service
+```

bin/index_service

@@ -0,0 +1,4 @@
+#!/usr/bin/node
+
+const indexService = require('../lib/index_service');
+indexService.main();

lib/config.js

@@ -0,0 +1,18 @@
+/**
+ * Created by diugalde on 23/09/16.
+ */
+
+const config = {
+    elasticSearch: {
+        url: 'localhost:9200' || process.env.CL_ES_URL,
+        index: 'clc_learning_object_index',
+        docType: 'clc_learning_object'
+    },
+
+    rabbitMQ: {
+        url: 'amqp://localhost' || process.env.CL_RMQ_URL,
+        loQueue:  'cl_lo_queue'
+    }
+};
+
+module.exports = config;

lib/index_service.js

@@ -0,0 +1,66 @@
+/**
+ * Created by diugalde on 23/09/16.
+ */
+
+const amqp = require('amqplib');
+
+const config = require('./config');
+const loIndexer = require('./lo_indexer');
+const log = require('./log');
+
+
+/**
+ * Processes queue messages.
+ *
+ * @param msg - JSON string with the following format: {
+ *                                                          action: add | remove | update,
+ *                                                          content: object
+ *                                                     }
+ * @returns Promise<boolean> - It will true if the msg was successfully processed. Error otherwise.
+ */
+function process(msg) {
+    try {
+        let msgObj = JSON.parse(msg.content.toString());
+        if (msgObj.action && loIndexer[msgObj.action]) {
+            log.info(`Executing ${msgObj.action} operation...`);
+            return loIndexer[msgObj.action](msgObj.content);
+        } else {
+            // If it is an unknown msg, just return true.
+            return Promise.resolve(true)
+        }
+    } catch(err) {
+        return Promise.reject(err)
+    }
+}
+
+function main() {
+
+    log.info('Starting cl-index...');
+
+    const queueName = config.rabbitMQ.loQueue;
+
+    // Connects and listen RabbitMQ messages.
+    amqp.connect(config.rabbitMQ.url).then(function(conn) {
+        return conn.createChannel();
+    }).then(function(ch) {
+        return ch.assertQueue(queueName).then(function() {
+            return ch.consume(queueName, function(msg) {
+                if (msg) {
+                    process(msg).then(function(res) {
+                        if (res === true) {
+                            ch.ack(msg)
+                        }
+                    }).catch(function(err) {
+                        log.info(err)
+                    })
+                }
+            });
+        });
+    }).catch(function(err) {
+        log.info(err)
+    });
+}
+
+module.exports = {
+    main
+};

lib/lo_indexer.js

@@ -0,0 +1,119 @@
+/**
+ * Created by diugalde on 23/09/16.
+ */
+
+
+const elasticSearch = require('elasticsearch');
+const rp = require('request-promise');
+
+const config = require('./config');
+const log = require('./log');
+
+
+const docType = config.elasticSearch.docType;
+const indexName = config.elasticSearch.index;
+
+const client = new elasticSearch.Client({
+    host: config.elasticSearch.url
+});
+
+var loIndexer = {
+
+    /**
+     * Creates a new document in the elasticSearch index.
+     *
+     * @param lo - object (LearningObject structure)
+     * @returns Promise<boolean> or error.
+     */
+    add(lo) {
+        return client.index({
+            index: indexName,
+            type: docType,
+            id: lo.id,
+            body: lo
+        }).then(function(res) {
+            // May have to do file work here (get from url or ).
+            log.info(res);
+            return Promise.resolve(true)
+        }).catch(function(err) {
+            return Promise.reject(err)
+        })
+    },
+
+    /**
+     * Removes document from the elasticSearch index.
+     *
+     * @param loId - string (LearningObject's id).
+     * @returns Promise<boolean> or error.
+     */
+    remove(loId) {
+        return client.delete({
+            index: indexName,
+            type: docType,
+            id: loId
+        }).then(function(res) {
+            log.info(res);
+            return Promise.resolve(true)
+        }).catch(function(err) {
+            if (err.status === 404) {
+                log.info(`Lo with id '${loId}' not found.`);
+                return Promise.resolve(true)
+            }
+            return Promise.reject(err)
+        })
+    },
+
+    /**
+     * Updates existing document in elasticSearch.
+     *
+     * @param updatedLO - object (This object should have the following structure: {doc: LearningObject, file: null | {base64: '', format: ''}}
+     * @returns Promise<boolean> or error.
+     */
+    update(updatedLO) {
+        let lo = updatedLO.doc;
+
+        return _getAttachment(updatedLO.file).then(function(attachment) {
+            if (attachment !== null) {
+                lo.file = attachment
+            }
+            return client.update({
+                index: indexName,
+                type: docType,
+                id: lo.id,
+                body: {doc: lo}
+            })
+        }).then(function(res){
+            log.info(res);
+            return Promise.resolve(true)
+        }).catch(function(err) {
+            return Promise.reject(err)
+        })
+    }
+};
+
+
+/**
+ * Retrieves the file's base64 string. If the fileObj has an url format, it will download the HTML page first.
+ *
+ * @param fileObj - object (Example structure: null | {base64: '', format: ''})
+ * @returns Promise<string> - (Base64 equivalent).
+ * @private
+ */
+function _getAttachment(fileObj) {
+    if (fileObj && fileObj !== null) {
+        if (fileObj.format === 'url') {
+            let url = new Buffer(fileObj.base64, 'base64').toString('ascii');
+            return rp(url).then(function(res) {
+                return Promise.resolve(new Buffer(res).toString('base64'))
+            }).catch(function(err) {
+                log.info(err);
+                return Promise.resolve(new Buffer(`Error while getting ${url}`).toString('base64'))
+            })
+        }
+        return Promise.resolve(fileObj.base64)
+    }
+    return Promise.resolve(null)
+}
+
+
+module.exports = Object.create(loIndexer);

lib/log.js

@@ -0,0 +1,20 @@
+/**
+ * Created by diugalde on 23/09/16.
+ */
+
+const bunyan = require('bunyan');
+const bunyanFormat = require('bunyan-format');
+
+const APP_NAME = 'cl-index';
+
+// Init bunyan log with a nice output format.
+var bunyanFormatStdOut = bunyanFormat({ outputMode: 'short' });
+
+var log = bunyan.createLogger({
+    name: APP_NAME,
+    streams: [
+        { level: "info", stream: bunyanFormatStdOut }
+    ]
+});
+
+module.exports = log;

package.json

@@ -0,0 +1,36 @@
+{
+  "name": "cl-index",
+  "version": "0.0.1",
+  "description": "Common Library index service. Reads messages from a RabbitMQ queue and interacts with an ElasticSearch index.",
+  "main": "index_service.js",
+  "scripts": {
+    "test": "mocha --recursive --reporter spec"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://[email protected]:edify/cl-index.git"
+  },
+  "keywords": [
+    "cl",
+    "common",
+    "library",
+    "index",
+    "elasticSearch",
+    "rabbitMQ"
+  ],
+  "author": "diugalde",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://gitlab.com/edify/cl-index/issues"
+  },
+  "homepage": "https://gitlab.com/edify/cl-index.git#README",
+  "dependencies": {
+    "amqplib": "^0.4.2",
+    "bunyan": "^1.8.1",
+    "bunyan-format": "^0.2.1",
+    "elasticsearch": "^11.0.1",
+    "mocha": "^3.0.2",
+    "request": "^2.75.0",
+    "request-promise": "^4.1.1"
+  }
+}