WIP

NEW_CONNECTION.md

@@ -33,20 +33,6 @@ The following are the instructions for updating the Tab React JS file
     ....
 ```
 
-## Updating the Settings.react.js
-Update the fetchData function to make the calls to appropriate sessions.js backend.  
-Add your new connection to the following list:
-
-```javascript
-    if (contains(connectionObject.dialect, [
-                    DIALECTS.APACHE_IMPALA,
-                    DIALECTS.APACHE_SPARK,
-                    DIALECTS.IBM_DB2,
-                    DIALECTS.MYSQL, DIALECTS.MARIADB, DIALECTS.POSTGRES,
-                    DIALECTS.REDSHIFT, DIALECTS.MSSQL, DIALECTS.SQLITE
-    ])) {
-```
-
 ## New Datastore file
 This file should be added in the following location
 backend/persistent/datastores

app/components/Settings/Settings.react.js

@@ -194,14 +194,7 @@ class Settings extends Component {
         }
 
         const connectionObject = connections[selectedTab] || {};
-        if (contains(connectionObject.dialect, [
-                    DIALECTS.APACHE_IMPALA,
-                    DIALECTS.APACHE_SPARK,
-                    DIALECTS.IBM_DB2,
-                    DIALECTS.MYSQL, DIALECTS.MARIADB, DIALECTS.POSTGRES,
-                    DIALECTS.REDSHIFT, DIALECTS.MSSQL, DIALECTS.SQLITE,
-                    DIALECTS.DATA_WORLD
-        ])) {
+        if (contains(connectionObject.dialect, SQL_DIALECTS_USING_EDITOR)) {
             if (connectRequest.status === 200 && !tablesRequest.status) {
                 this.setState({editMode: false});
                 getTables();

app/components/Settings/Tabs/Tab.react.js

@@ -32,6 +32,8 @@ export default class ConnectionTab extends Component {
               label = `Apache Impala (${connectionObject.host}:${connectionObject.port})`;
         } else if (dialect === DIALECTS.APACHE_SPARK) {
             label = `Apache Spark (${connectionObject.host}:${connectionObject.port})`;
+        } else if (connectionObject.dialect === DIALECTS.CSV) {
+            label = `CSV (${connectionObject.url})`;
         } else if (connectionObject.dialect === DIALECTS.ELASTICSEARCH) {
             label = `Elasticsearch (${connectionObject.host})`;
         } else if (connectionObject.dialect === DIALECTS.SQLITE) {

app/constants/constants.js

@@ -15,7 +15,8 @@ export const DIALECTS = {
     APACHE_SPARK: 'apache spark',
     APACHE_IMPALA: 'apache impala',
     APACHE_DRILL: 'apache drill',
-    DATA_WORLD: 'data.world'
+    DATA_WORLD: 'data.world',
+    CSV: 'csv'
 };
 
 export const SQL_DIALECTS_USING_EDITOR = [
@@ -28,7 +29,8 @@ export const SQL_DIALECTS_USING_EDITOR = [
     'ibm db2',
     'apache spark',
     'apache impala',
-    'data.world'
+    'data.world',
+    'csv'
 ];
 
 const commonSqlOptions = [
@@ -74,6 +76,9 @@ const hadoopQLOptions = [
 export const CONNECTION_CONFIG = {
     [DIALECTS.APACHE_IMPALA]: hadoopQLOptions,
     [DIALECTS.APACHE_SPARK]: hadoopQLOptions,
+    [DIALECTS.CSV]: [
+        {'label': 'URL to CSV File', 'value': 'database', 'type': 'text'}
+    ],
     [DIALECTS.IBM_DB2]: commonSqlOptions,
     [DIALECTS.MYSQL]: commonSqlOptions,
     [DIALECTS.MARIADB]: commonSqlOptions,
@@ -201,6 +206,7 @@ export const CONNECTION_CONFIG = {
 export const LOGOS = {
     [DIALECTS.APACHE_SPARK]: 'images/spark-logo.png',
     [DIALECTS.APACHE_IMPALA]: 'images/impala-logo.png',
+    [DIALECTS.CSV]: 'images/csv-logo.png',
     [DIALECTS.IBM_DB2]: 'images/ibmdb2-logo.png',
     [DIALECTS.REDSHIFT]: 'images/redshift-logo.png',
     [DIALECTS.POSTGRES]: 'images/postgres-logo.png',
@@ -216,6 +222,8 @@ export const LOGOS = {
 
 export function PREVIEW_QUERY(connection, table, elasticsearchIndex) {
     switch (connection.dialect) {
+        case DIALECTS.CSV:
+            return 'SELECT TOP 1000 * FROM ?';
         case DIALECTS.IBM_DB2:
             return `SELECT * FROM ${table} FETCH FIRST 1000 ROWS ONLY`;
         case DIALECTS.APACHE_IMPALA:
@@ -324,6 +332,10 @@ export const SAMPLE_DBS = {
         host: 'spark.test.plotly.host',
         dialect: DIALECTS.APACHE_SPARK
     },
+    [DIALECTS.CSV]: {
+        dialect: DIALECTS.CSV,
+        url: `file://${__dirname}/data/26k-consumer-complaints.csv`
+    },
     [DIALECTS.IBM_DB2]: {
         username: 'db2user1',
         password: 'w8wfy99DvEmgkBsE',

backend/persistent/datastores/Datastores.js

@@ -5,6 +5,7 @@ import * as ApacheDrill from './ApacheDrill';
 import * as IbmDb2 from './ibmdb2';
 import * as ApacheLivy from './livy';
 import * as ApacheImpala from './impala';
+import * as CSV from './csv';
 import * as DataWorld from './dataworld';
 import * as DatastoreMock from './datastoremock';
 
@@ -44,6 +45,8 @@ function getDatastoreClient(connection) {
         return ApacheLivy;
     } else if (dialect === 'apache impala') {
         return ApacheImpala;
+    } else if (dialect === 'csv') {
+        return CSV;
     } else if (dialect === 'ibm db2') {
         return IbmDb2;
     } else if (dialect === 'data.world') {

backend/persistent/datastores/csv.js

@@ -0,0 +1,92 @@
+const alasql = require('alasql');
+import fetch from 'node-fetch';
+import papa from 'papaparse';
+import {type} from 'ramda';
+
+import {parseSQL} from '../../parse';
+
+// connect() stores the data parsed from CSV files here
+const connectionData = {};
+
+// define type error thrown by this connector
+export function CSVError(url, errors) {
+      this.name = 'CSVError';
+      this.message = 'Failed to parse CSV file ' + url;
+
+      if (Error.captureStackTrace) {
+          Error.captureStackTrace(this, CSVError);
+      } else {
+          this.stack = new Error(this.message).stack;
+      }
+
+      this.url = url;
+      this.errors = errors;
+
+      if (errors && errors[0] && errors[0].message) {
+          this.message = errors[0].message;
+      }
+}
+CSVError.prototype = Object.create(Error.prototype);
+CSVError.prototype.constructor = CSVError;
+
+export function connect(connection) {
+    const url = connection.database;
+
+    return fetch(url)
+    .then(res => res.text())
+    .then(body => {
+        return new Promise(function(resolve) {
+            console.log(body);
+            papa.parse(body, {
+                download: false,
+                dynamicTyping: true,
+                header: true,
+                worker: true,
+
+                complete: function({data, errors, meta}) {
+                    if (errors.length) {
+                        throw new CSVError(url, errors);
+                    }
+
+                    connection.meta = meta;
+
+                    connectionData[connection.url] = data;
+
+                    resolve(connection);
+                }
+            });
+        });
+    });
+}
+
+export function tables() {
+    // To take advantage of alaSQL's parser, the table is named '?'
+    return Promise.resolve(['?']);
+}
+
+export function schemas(connection) {
+    const columnnames = ['TABNAME', 'COLNAME', 'TYPENAME'];
+    const rows = connection.meta.fields.map(columnName => {
+        return ['?', columnName, getType(columnName)];
+    });
+
+    return Promise.resolve({columnnames, rows});
+
+    function getType(columnName) {
+        const data = connectionData[connection.url];
+
+        for (let i = 0; i < data.length; i++) {
+            const cell = data[i][columnName];
+            if (cell) return type(cell);
+        }
+
+        return 'String';
+    }
+}
+
+export function query(queryString, connection) {
+    const data = connectionData[connection.url];
+
+    // In the query `SELECT * FROM ?`, alaSQL replaces ? with data
+    return alasql.promise(queryString, [data]).then(parseSQL);
+}

package.json

@@ -215,10 +215,12 @@
     "yamljs": "^0.3.0"
   },
   "dependencies": {
+    "alasql": "^0.4.5",
     "csv-parse": "^2.0.0",
     "font-awesome": "^4.6.1",
     "ibm_db": "git+https://[email protected]/n-riesco/node-ibm_db.git#patched-v2.2.1",
     "mysql": "^2.15.0",
+    "papaparse": "^4.3.7",
     "pg": "^4.5.5",
     "pg-hstore": "^2.3.2",
     "restify": "^4.3.2",

webpack.config.base.js

@@ -2,6 +2,7 @@ import path from 'path';
 
 export default {
     module: {
+        noParse: [/alasql/],
         rules: [{
             test: /\.jsx?$/,
             use: [{

webpack.config.electron.js

@@ -14,6 +14,8 @@ export default {
     },
 
     plugins: [
+        ...baseConfig.plugins,
+
         new webpack.optimize.UglifyJsPlugin({
             sourceMap: false,
             compressor: {

webpack.config.headless.js

@@ -14,6 +14,8 @@ export default {
     },
 
     plugins: [
+        ...baseConfig.plugins,
+
         new webpack.optimize.UglifyJsPlugin({
             compressor: {
                 warnings: false

webpack.config.production.js

@@ -16,6 +16,7 @@ const config = {
 
     plugins: [
         ...baseConfig.plugins,
+
         new webpack.DefinePlugin({
             __DEV__: false,
             'process.env.NODE_ENV': JSON.stringify('production')

webpack.config.web.js

@@ -27,6 +27,7 @@ const config = {
 
     plugins: [
         ...baseConfig.plugins,
+
         new webpack.DefinePlugin({
             __DEV__: false,
             'process.env.NODE_ENV': JSON.stringify('production')