From 876aa5ba0dc72c351ea19caec13e3b87ebc13cd1 Mon Sep 17 00:00:00 2001 From: YaphetKG Date: Wed, 10 Apr 2024 10:51:50 -0400 Subject: [PATCH 1/3] adding csv stream --- src/ldf-client-ui.js | 61 +++++++++++++++++++++++++++++++--------- src/ldf-client-worker.js | 49 +++++++++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 15 deletions(-) diff --git a/src/ldf-client-ui.js b/src/ldf-client-ui.js index c825a09e..6f17e63d 100644 --- a/src/ldf-client-ui.js +++ b/src/ldf-client-ui.js @@ -718,21 +718,54 @@ if (typeof global.process === 'undefined') }, _downloadCSV: function () { - if (!this._resultCount) return alert('Please execute a query to download'); - else { - let csvContent = 'data:text/csv;charset=utf-8,'; - let header = Object.keys(this.bindingResults[0]); - csvContent += [header].map(e => e.join(',')) + '\n'; - this.bindingResults.forEach(function (v, i, a) { - let line = []; - header.forEach(function (h, i) { - line.push(v[h]); - }); - csvContent += [line].map(e => e.join(',')) + '\n'; - }); - let encodedUri = encodeURI(csvContent); - window.open(encodedUri); + // Let the worker execute the query + var context = { + ...this._getQueryContext(), + sources: Object.keys(this.options.selectedDatasources).map(function (datasource) { + var type; + var posAt = datasource.indexOf('@'); + if (posAt > 0) { + type = datasource.substr(0, posAt); + datasource = datasource.substr(posAt + 1, datasource.length); + } + datasource = resolve(datasource, window.location.href); + return { type: type, value: datasource }; + }), + }; + var prefixesString = ''; + if (this.options.queryFormat === 'sparql') { + for (var prefix in this.options.prefixes) + prefixesString += 'PREFIX ' + prefix + ': <' + this.options.prefixes[prefix] + '>\n'; } + var query = prefixesString + this.$queryTextsIndexed[this.options.queryFormat].val(); + this._queryWorker.postMessage({ + type: 'querycsv', + query: query, + context: context, + resultsToTree: this.options.resultsToTree, + }); + // + // this._queryWorker.postMessage({ + // type: 'querycsv', + // query: query, + // context: context, + // resultsToTree: this.options.resultsToTree, + // }); + // if (!this._resultCount) return alert('Please execute a query to download'); + // else { + // let csvContent = 'data:text/csv;charset=utf-8,'; + // let header = Object.keys(this.bindingResults[0]); + // csvContent += [header].map(e => e.join(',')) + '\n'; + // this.bindingResults.forEach(function (v, i, a) { + // let line = []; + // header.forEach(function (h, i) { + // line.push(v[h]); + // }); + // csvContent += [line].map(e => e.join(',')) + '\n'; + // }); + // let encodedUri = encodeURI(csvContent); + // window.open(encodedUri); + // } }, _downloadLog: function () { let filename = 'execution.log'; diff --git a/src/ldf-client-worker.js b/src/ldf-client-worker.js index 606ba327..258912d2 100644 --- a/src/ldf-client-worker.js +++ b/src/ldf-client-worker.js @@ -36,8 +36,56 @@ function initEngine(config) { config.context.queryFormat = { language: config.context.queryFormat }; } + + + + + + + + // Handlers of incoming messages var handlers = { + + querycsv: function (config) { + initEngine(config); + config.context.log = logger; + engine.query(config.query, config.context).then(async function (result){ + const { data } = await engine.resultToString(result, "text/csv"); + if (result.resultType === 'bindings') { + + const decoder = new TextDecoder("utf-8"); + const queuingStrategy = new CountQueuingStrategy({ highWaterMark: 1 }); + let result = ""; + const writableStream = new WritableStream( + { + // Implement the sink + write(chunk) { + return new Promise((resolve, reject) => { + const buffer = new ArrayBuffer(1); + const view = new Uint8Array(buffer); + view[0] = chunk; + const decoded = decoder.decode(view, { stream: true }); + console.log(`Chunk decoded: ${decoded}`); + result += decoded; + resolve(); + }); + }, + close() { + console.log(result) + }, + abort(err) { + console.error("Sink error:", err); + }, + }, + queuingStrategy, + ); + data.pipe(writableStream) + } + + + }) + }, // Execute the given query with the given options query: function (config) { initEngine(config); @@ -48,7 +96,6 @@ var handlers = { .then(async function (result) { // Post query metadata postMessage({ type: 'queryInfo', queryType: result.resultType }); - var bindings = result.resultType === 'bindings'; var resultsToTree = config.resultsToTree; switch (result.resultType) { From f73c36b9446cceb4d12421d2f1fab458bd748abb Mon Sep 17 00:00:00 2001 From: YaphetKG Date: Thu, 11 Apr 2024 14:59:59 -0400 Subject: [PATCH 2/3] adding streaming of csv --- src/ldf-client-ui.js | 32 +++++++++------------------- src/ldf-client-worker.js | 45 +++++++++++++--------------------------- 2 files changed, 24 insertions(+), 53 deletions(-) diff --git a/src/ldf-client-ui.js b/src/ldf-client-ui.js index 6f17e63d..0d37fd2d 100644 --- a/src/ldf-client-ui.js +++ b/src/ldf-client-ui.js @@ -744,28 +744,6 @@ if (typeof global.process === 'undefined') context: context, resultsToTree: this.options.resultsToTree, }); - // - // this._queryWorker.postMessage({ - // type: 'querycsv', - // query: query, - // context: context, - // resultsToTree: this.options.resultsToTree, - // }); - // if (!this._resultCount) return alert('Please execute a query to download'); - // else { - // let csvContent = 'data:text/csv;charset=utf-8,'; - // let header = Object.keys(this.bindingResults[0]); - // csvContent += [header].map(e => e.join(',')) + '\n'; - // this.bindingResults.forEach(function (v, i, a) { - // let line = []; - // header.forEach(function (h, i) { - // line.push(v[h]); - // }); - // csvContent += [line].map(e => e.join(',')) + '\n'; - // }); - // let encodedUri = encodeURI(csvContent); - // window.open(encodedUri); - // } }, _downloadLog: function () { let filename = 'execution.log'; @@ -1080,6 +1058,16 @@ if (typeof global.process === 'undefined') return self._logAppender(data.log); case 'error': return this.onerror(data.error); case 'webIdName': return self._setWebIdName(data.name); + case 'resultCsv': + let filename = 'data.csv'; + let blob = new Blob([data.data], { type: 'text/json' }); + let e = document.createEvent('MouseEvents'); + let a = document.createElement('a'); + a.download = filename; + a.href = window.URL.createObjectURL(blob); + a.dataset.downloadurl = ['text/json', a.download, a.href].join(':'); + e.initMouseEvent('click', true, false, window, 0, 0, 0, 0, 0, false, false, false, false, 0, null); + a.dispatchEvent(e); } }; this._queryWorker.onerror = function (error) { diff --git a/src/ldf-client-worker.js b/src/ldf-client-worker.js index 258912d2..75d3a41a 100644 --- a/src/ldf-client-worker.js +++ b/src/ldf-client-worker.js @@ -46,41 +46,24 @@ function initEngine(config) { // Handlers of incoming messages var handlers = { - - querycsv: function (config) { + querycsv: async function (config) { initEngine(config); - config.context.log = logger; engine.query(config.query, config.context).then(async function (result){ const { data } = await engine.resultToString(result, "text/csv"); + function readableToString(stream) { + const chunks = []; + return new Promise((res, rej) => { + stream.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + stream.on('error', (e) => { rej(e); }); + stream.on('end', () => { res(Buffer.concat(chunks).toString('utf-8')); }); + }); + } if (result.resultType === 'bindings') { - - const decoder = new TextDecoder("utf-8"); - const queuingStrategy = new CountQueuingStrategy({ highWaterMark: 1 }); - let result = ""; - const writableStream = new WritableStream( - { - // Implement the sink - write(chunk) { - return new Promise((resolve, reject) => { - const buffer = new ArrayBuffer(1); - const view = new Uint8Array(buffer); - view[0] = chunk; - const decoded = decoder.decode(view, { stream: true }); - console.log(`Chunk decoded: ${decoded}`); - result += decoded; - resolve(); - }); - }, - close() { - console.log(result) - }, - abort(err) { - console.error("Sink error:", err); - }, - }, - queuingStrategy, - ); - data.pipe(writableStream) + resultsCsv = await readableToString(data); + postMessage({ + type: "resultCsv", + data: resultsCsv + }) } From 046747ccc32aae82568509bf9de2379f2cecb071 Mon Sep 17 00:00:00 2001 From: YaphetKG Date: Thu, 11 Apr 2024 15:01:22 -0400 Subject: [PATCH 3/3] formatting --- src/ldf-client-worker.js | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/ldf-client-worker.js b/src/ldf-client-worker.js index 75d3a41a..fd506542 100644 --- a/src/ldf-client-worker.js +++ b/src/ldf-client-worker.js @@ -36,20 +36,12 @@ function initEngine(config) { config.context.queryFormat = { language: config.context.queryFormat }; } - - - - - - - - // Handlers of incoming messages var handlers = { querycsv: async function (config) { initEngine(config); - engine.query(config.query, config.context).then(async function (result){ - const { data } = await engine.resultToString(result, "text/csv"); + engine.query(config.query, config.context).then(async function (result) { + const { data } = await engine.resultToString(result, 'text/csv'); function readableToString(stream) { const chunks = []; return new Promise((res, rej) => { @@ -59,15 +51,13 @@ var handlers = { }); } if (result.resultType === 'bindings') { - resultsCsv = await readableToString(data); + let resultsCsv = await readableToString(data); postMessage({ - type: "resultCsv", - data: resultsCsv - }) + type: 'resultCsv', + data: resultsCsv, + }); } - - - }) + }); }, // Execute the given query with the given options query: function (config) {