diff --git a/packages/cli/src/utils/registry/registry.js b/packages/cli/src/utils/registry/registry.js index b43ee643..d3242aa7 100644 --- a/packages/cli/src/utils/registry/registry.js +++ b/packages/cli/src/utils/registry/registry.js @@ -1,131 +1,44 @@ import fs from 'fs' import path from 'path' -import axios from 'axios' -import https from 'https' -import FormData from 'form-data' import logger from '../debug' import session from '../session' -const REGISTRY_TIMEOUT = 60000 - const { debug } = logger('utils-registry') class Registry { - constructor () { - debug('Registry.constructor') - - const registryInstance = process.env.SYNCANO_SOCKET_REGISTRY_INSTANCE || 'socket-registry' - this.registryHost = `${registryInstance}.${session.ENDPOINT_HOST}` - this.registryHostUrl = `https://${this.registryHost}` - - this.fileStorageHost = session.getHost() - this.fileStorageEndpoint = `/v2/instances/${registryInstance}/classes/storage/objects/` - - if (session.project) { - this.installEndpoint = `/v2/instances/${session.project.instance}/sockets/install/` - this.installUrl = `https://${session.getHost()}${this.installEndpoint}` - } - } - - getFullSocket (name, version) { - return this.searchSocketByName(name, version) - } - async searchSocketByName (name, version) { - debug(`searchSocketByName: ${name} (${version})`) - const response = await axios.request({ - url: `https://${this.registryHost}/registry/get/`, - method: 'POST', - timeout: REGISTRY_TIMEOUT, - data: { - name, - version - }, - headers: { - 'X-Syncano-Account-Key': session.settings.account.getAuthKey() - } - }) - return response.data + debug(`searchSocketByName: ${name} ${version}`) + return session.connection.registry.searchSocketByName(name, version) } static getSocket (socket) { debug('getSocket') const fileName = path.join(session.getBuildPath(), `${socket.name}.zip`) - const file = fs.createWriteStream(fileName) - return new Promise((resolve, reject) => { - https.get(socket.url, (response) => { - response.pipe(file) - file.on('finish', () => { - debug('Socket zip downloaded') - file.close(resolve) - }) - }) - }) + const fileDescriptor = fs.createWriteStream(fileName) + + return session.connection.registry.getSocket(socket.url, fileDescriptor) } async publishSocket (socketName, version) { - debug('publishSocket', socketName) - const response = await axios.request({ - url: `${this.registryHostUrl}/registry/publish/`, - method: 'POST', - timeout: REGISTRY_TIMEOUT, - data: { - name: socketName, - version - }, - headers: { - 'X-Syncano-Account-Key': session.settings.account.getAuthKey() - } - }) - return response.data + debug(`publishSocket: ${socketName}, ${version}`) + return session.connection.registry.publishSocket(socketName, version) } async searchSocketsByAll (keyword) { - const response = await axios.request({ - url: `${this.registryHostUrl}/registry/search/`, - method: 'POST', - timeout: REGISTRY_TIMEOUT, - data: { keyword }, - headers: { - 'X-Syncano-Account-Key': session.settings.account.getAuthKey() - } - }) - return response.data + debug(`searchSocketsByAll: ${keyword}`) + return session.connection.registry.searchSocketsByAll(keyword) } async submitSocket (socket) { + debug(`submitSocket: ${socket.name}`) await socket.createPackageZip() - const form = new FormData() - form.append('file', fs.createReadStream(socket.getSocketZip())) - form.append('name', socket.spec.name) - form.append('description', socket.spec.description) - form.append('version', socket.spec.version) - form.append('keywords', JSON.stringify(socket.spec.keywords || [])) - form.append('config', JSON.stringify(socket.getFullConfig())) - - return new Promise((resolve, reject) => { - form.submit({ - method: 'POST', - protocol: 'https:', - host: session.getHost(), - path: `${this.registryHostUrl}/registry/add/`, - headers: { - 'X-Syncano-Account-Key': session.settings.account.getAuthKey() - } - }, (err, res) => { - if (err) { - debug('Error while uploading file', err) - reject(err) - } - res.on('data', (data) => { - debug('Upload done', data.toString()) - resolve(data) - }) - }) - }) + return session + .connection + .registry + .submitSocket(socket.spec, socket.getFullConfig(), socket.getSocketZip()) } } diff --git a/packages/cli/src/utils/sockets/sockets.js b/packages/cli/src/utils/sockets/sockets.js index 0911e66b..f5cca078 100644 --- a/packages/cli/src/utils/sockets/sockets.js +++ b/packages/cli/src/utils/sockets/sockets.js @@ -448,7 +448,7 @@ class Socket { async loadFromRegistry () { debug(`loadFromRegistry: ${this.name}`) const registry = new Registry() - const registrySocket = await registry.getFullSocket(this.name) + const registrySocket = await registry.searchSocketByName(this.name) if (registrySocket.config) { this.spec = registrySocket.config diff --git a/packages/lib-js-core/package.json b/packages/lib-js-core/package.json index 15335e8c..6a41c586 100644 --- a/packages/lib-js-core/package.json +++ b/packages/lib-js-core/package.json @@ -37,6 +37,7 @@ "node-fetch": "2.0.0-alpha.9" }, "devDependencies": { + "@syncano/test-tools": "0.6.0", "babel-cli": "^6.26.0", "babel-eslint": "^8.0.1", "babel-plugin-add-module-exports": "^0.2.1", diff --git a/packages/lib-js-core/src/query-builder.js b/packages/lib-js-core/src/query-builder.js index b4e5c0f8..d27bfbba 100644 --- a/packages/lib-js-core/src/query-builder.js +++ b/packages/lib-js-core/src/query-builder.js @@ -17,6 +17,14 @@ export default class QueryBuilder { return `https://${host}/${apiVersion}` } + _getSyncanoRegistryURL () { + const {host} = this.instance + const endpointHost = host === 'api.syncano.io' ? 'syncano.space' : 'syncano.link' + const registryInstance = process.env.SYNCANO_SOCKET_REGISTRY_INSTANCE || 'socket-registry' + this.registryHost = `${registryInstance}.${endpointHost}` + return `https://${this.registryHost}` + } + _getInstanceURL (instanceName) { return `${this._getSyncanoURL()}/instances/${instanceName}` } diff --git a/packages/lib-js-core/src/registry.js b/packages/lib-js-core/src/registry.js new file mode 100644 index 00000000..27156af4 --- /dev/null +++ b/packages/lib-js-core/src/registry.js @@ -0,0 +1,99 @@ +import fs from 'fs' +import https from 'https' +import FormData from 'form-data' +import logger from 'debug' +import QueryBuilder from './query-builder' + +const debug = logger('core:registry') + +/** + * Connection with Syncano Registry. + * @property {Function} + * @example {@lang javascript} + * const socketList = await registry.searchSocketsByAll('facebook') + */ +export default class Socket extends QueryBuilder { + url (registryEndpoint) { + return `${this._getSyncanoRegistryURL()}/${registryEndpoint}/` + } + + async searchSocketsByAll (keyword) { + debug(`searchSocketsByAll: ${keyword}`) + const headers = { + 'X-Syncano-Account-Key': this.instance.accountKey + } + const options = { + method: 'POST', + body: JSON.stringify({ keyword }) + } + + return this.nonInstanceFetch(this.url('registry/search'), options, headers) + } + + async searchSocketByName (name, version) { + debug(`searchSocketByName: ${name} (${version})`) + + const headers = { + 'X-Syncano-Account-Key': this.instance.accountKey + } + const options = { + method: 'POST', + body: JSON.stringify({ + name, + version + }) + } + + return this.nonInstanceFetch(this.url('registry/get'), options, headers) + } + + async publishSocket (socketName, version) { + debug('publishSocket', socketName) + const headers = { + 'X-Syncano-Account-Key': this.instance.accountKey + } + const options = { + method: 'POST', + body: JSON.stringify({ + name: socketName, + version + }) + } + return this.nonInstanceFetch(this.url('registry/publish'), options, headers) + } + + async getSocket (url, fileDescriptor) { + debug('getSocket', url) + return new Promise((resolve, reject) => { + https.get(url, (response) => { + response.pipe(fileDescriptor) + fileDescriptor.on('finish', () => { + debug('Socket zip downloaded') + fileDescriptor.close(resolve) + }) + }) + }) + } + + async submitSocket (socketSpec, socketConfig, socketZipPath) { + debug('submitSocket', socketZipPath) + + const form = new FormData() + form.append('file', fs.createReadStream(socketZipPath)) + form.append('name', socketSpec.name) + form.append('description', socketSpec.description) + form.append('version', socketSpec.version) + form.append('keywords', JSON.stringify(socketSpec.keywords || [])) + form.append('config', JSON.stringify(socketConfig)) + + const headers = form.getHeaders() + headers['X-Syncano-Account-Key'] = this.instance.accountKey + + const options = { + method: 'POST', + body: form + } + + return this.nonInstanceFetch(this.url('registry/add'), options, headers) + } +} diff --git a/packages/lib-js-core/src/server.js b/packages/lib-js-core/src/server.js index 319048a2..14cf90cd 100644 --- a/packages/lib-js-core/src/server.js +++ b/packages/lib-js-core/src/server.js @@ -12,6 +12,7 @@ import Logger from './logger' import Channel from './channel' import Class from './class' import Settings from './settings' +import Registry from './registry' class Server { constructor (ctx = {}) { @@ -31,6 +32,7 @@ class Server { this.instance = new Instance(config) this.logger = Logger(config) this.users = new Users(config) + this.registry = new Registry(config) this.data = new Proxy(new Data(settings), { get (target, className) { return new Data(getConfig(className)) diff --git a/packages/lib-js-core/test/e2e/data.js b/packages/lib-js-core/test/e2e/data.js index 40394e83..61e5dd98 100644 --- a/packages/lib-js-core/test/e2e/data.js +++ b/packages/lib-js-core/test/e2e/data.js @@ -86,6 +86,134 @@ describe('Data', function () { }) }) + describe('#delete()', () => { + it('can delete single object', async () => { + const obj = await run().create({field_string: 'test1'}) + // TODO why undefined here? while in next tests we have ids? + run().delete(obj.id).should.become(undefined) + }) + + it('can delete multiple objects', async () => { + const obj1 = await run().create({field_string: 'test1'}) + const obj2 = await run().create({field_string: 'test1'}) + await run().delete([obj1.id, obj2.id]).should.become([obj1.id, obj2.id]) + await run().list().should.become([]) + }) + + it('can delete multiple objects by query', async () => { + const obj1 = await run().create({field_string: 'test1'}) + await run().create({field_string: 'test2'}) + await run().create({field_string: 'test3'}) + + run() + .where('id', 'lte', obj1.id) + .delete() + // TODO: should we really respond here with the deleted objects? + .should.become([obj1.id]) + }) + }) + + describe('#count()', () => { + it('should be able to get number of records', async () => { + const records = '.' + .repeat(120) + .split('.') + .map((a, i) => ({ + field_string: `item ${i}` + })) + await run().create(records) + + run().count().should.eventually.equal(121) + }) + }) + + describe('#fields()', () => { + it('should be able to whitelist fields', async () => { + await run().create({'field_integer': 182}) + + run() + .fields('field_integer') + .first() + .should.become({field_integer: 182}) + }) + + it('should be able to map field names', async () => { + const username = getRandomString() + const user = await users.create({username, password: 'test'}) + const obj = await run().create({author: user.id}) + + run() + .with('author') + .fields('id', 'author.username as name') + .find(obj.id) + .should.become({id: obj.id, name: username}) + }) + + it('should handle list of records', async () => { + await run().create({field_string: 'test1'}) + await run().create({field_string: 'test2'}) + await run().create({field_string: 'test3'}) + + await run() + .fields(['field_string']) + .take(2) + .list() + .should.become([ + {field_string: 'test1'}, + {field_string: 'test2'} + ]) + }) + + it('should work with create method', () => + run() + .fields('field_string') + .create({ + field_string: 'test create method' + }) + .should.become({field_string: 'test create method'})) + + it('should work with batch create', () => + run() + .fields('field_string') + .create([ + {field_string: 'test batch create method 1'}, + {field_string: 'test batch create method 2'} + ]) + .should.become([ + {field_string: 'test batch create method 1'}, + {field_string: 'test batch create method 2'} + ])) + + it('should work with update method', async () => { + await run().create({field_string: 'test3'}) + const id = await run().value('id') + + run() + .fields('field_string') + .update(id, { + field_string: 'test create method' + }) + .should.become({field_string: 'test create method'}) + }) + + it('should work with batch update', async () => { + await run().create({field_string: 'test1'}) + await run().create({field_string: 'test2'}) + const ids = await run().take(2).pluck('id') + + run() + .fields('field_string') + .update([ + [ids[0], {field_string: 'test batch update method 1'}], + [ids[1], {field_string: 'test batch update method 2'}] + ]) + .should.become([ + {field_string: 'test batch update method 1'}, + {field_string: 'test batch update method 2'} + ]) + }) + }) + describe('#take()', () => { it('should limit number of results', async () => { // Adding 3 objects to set limit of listing 2 @@ -442,93 +570,6 @@ describe('Data', function () { }) }) - describe('#fields()', () => { - it('should be able to whitelist fields', async () => { - await run().create({'field_integer': 182}) - - run() - .fields('field_integer') - .first() - .should.become({field_integer: 182}) - }) - - it('should be able to map field names', async () => { - const username = getRandomString() - const user = await users.create({username, password: 'test'}) - const obj = await run().create({author: user.id}) - - run() - .with('author') - .fields('id', 'author.username as name') - .find(obj.id) - .should.become({id: obj.id, name: username}) - }) - - it('should handle list of records', async () => { - await run().create({field_string: 'test1'}) - await run().create({field_string: 'test2'}) - await run().create({field_string: 'test3'}) - - await run() - .fields(['field_string']) - .take(2) - .list() - .should.become([ - {field_string: 'test1'}, - {field_string: 'test2'} - ]) - }) - - it('should work with create method', () => - run() - .fields('field_string') - .create({ - field_string: 'test create method' - }) - .should.become({field_string: 'test create method'})) - - it('should work with batch create', () => - run() - .fields('field_string') - .create([ - {field_string: 'test batch create method 1'}, - {field_string: 'test batch create method 2'} - ]) - .should.become([ - {field_string: 'test batch create method 1'}, - {field_string: 'test batch create method 2'} - ])) - - it('should work with update method', async () => { - await run().create({field_string: 'test3'}) - const id = await run().value('id') - - run() - .fields('field_string') - .update(id, { - field_string: 'test create method' - }) - .should.become({field_string: 'test create method'}) - }) - - it('should work with batch update', async () => { - await run().create({field_string: 'test1'}) - await run().create({field_string: 'test2'}) - const ids = await run().take(2).pluck('id') - - run() - .fields('field_string') - .update([ - [ids[0], {field_string: 'test batch update method 1'}], - [ids[1], {field_string: 'test batch update method 2'}] - ]) - .should.become([ - {field_string: 'test batch update method 1'}, - {field_string: 'test batch update method 2'} - ]) - }) - }) - describe('#update()', () => { it('can update single object', async () => { const obj = await run().create({field_string: 'test1'}) @@ -643,47 +684,6 @@ describe('Data', function () { }) }) - describe('#count()', () => { - it('should be able to get number of records', async () => { - const records = '.' - .repeat(120) - .split('.') - .map((a, i) => ({ - field_string: `item ${i}` - })) - await run().create(records) - - run().count().should.eventually.equal(121) - }) - }) - - describe('#delete()', () => { - it('can delete single object', async () => { - const obj = await run().create({field_string: 'test1'}) - // TODO why undefined here? while in next tests we have ids? - run().delete(obj.id).should.become(undefined) - }) - - it('can delete multiple objects', async () => { - const obj1 = await run().create({field_string: 'test1'}) - const obj2 = await run().create({field_string: 'test1'}) - await run().delete([obj1.id, obj2.id]).should.become([obj1.id, obj2.id]) - await run().list().should.become([]) - }) - - it('can delete multiple objects by query', async () => { - const obj1 = await run().create({field_string: 'test1'}) - await run().create({field_string: 'test2'}) - await run().create({field_string: 'test3'}) - - run() - .where('id', 'lte', obj1.id) - .delete() - // TODO: should we really respond here with the deleted objects? - .should.become([obj1.id]) - }) - }) - describe('#orderBy()', () => { it('can sort records ascending', async () => { await run().create({field_string: 'abcdef'}) diff --git a/packages/lib-js-core/test/e2e/instance.js b/packages/lib-js-core/test/e2e/instance.js index fb8bfdc6..51fbb2ce 100644 --- a/packages/lib-js-core/test/e2e/instance.js +++ b/packages/lib-js-core/test/e2e/instance.js @@ -1,3 +1,4 @@ +/* globals it describe before */ /* eslint-disable no-unused-expressions */ import {expect} from 'chai' diff --git a/packages/lib-js-core/test/e2e/registry.js b/packages/lib-js-core/test/e2e/registry.js new file mode 100644 index 00000000..194955bb --- /dev/null +++ b/packages/lib-js-core/test/e2e/registry.js @@ -0,0 +1,43 @@ +/* global it describe before after */ +// import Server from '../../src' + +import { + // deleteInstance, + // createProject, + // uniqueInstance +} from '@syncano/test-tools' + +describe('Registry', function () { + // let registry + // const testInstance = uniqueInstance() + + // before(async () => { + // await createProject(testInstance) + // registry = new Server({accountKey: process.env.E2E_ACCOUNT_KEY}).registry + // }) + // after(async () => deleteInstance(testInstance)) + // + // it.skip('search non-existing Socket by keyword', async () => { + // await registry.searchSocketsByAll('keyword') + // }) + // + // it.skip('search for existing Socket by keyword', async () => { + // await registry.searchSocketsByAll('keyword') + // }) + // + // it.skip('submit Socket', async (done) => { + // await registry.submitSocket('keyword') + // }) + // + // it.skip('publish Socket', async (done) => { + // await registry.publishSocket('keyword') + // }) + // + // it.skip('get by name', async (done) => { + // await registry.getSocket('keyword') + // }) + // + // it.skip('get by name and version', async (done) => { + // await registry.getSocket('keyword') + // }) +}) diff --git a/packages/registry/syncano/organization/yarn.lock b/packages/registry/syncano/organization/yarn.lock index 4c6ac45e..4997f1a6 100644 --- a/packages/registry/syncano/organization/yarn.lock +++ b/packages/registry/syncano/organization/yarn.lock @@ -2,16 +2,6 @@ # yarn lockfile v1 -"@syncano/core@0.6.0": - version "0.6.0" - resolved "https://registry.yarnpkg.com/@syncano/core/-/core-0.6.0.tgz#a37255c1647e0b48600e9e3cfea9719669e68c57" - dependencies: - debug "^3.1.0" - form-data "^2.3.1" - lodash.get "^4.4.2" - lodash.merge "^4.6.0" - lodash.set "^4.3.2" - node-fetch "2.0.0-alpha.9" abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" @@ -719,12 +709,6 @@ debug@^2.2.0, debug@^2.6.8: dependencies: ms "2.0.0" -debug@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - dependencies: - ms "2.0.0" - deep-extend@~0.4.0: version "0.4.2" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.2.tgz#48b699c27e334bf89f10892be432f6e4c7d34a7f" @@ -835,14 +819,6 @@ forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" -form-data@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.5" - mime-types "^2.1.12" - form-data@~2.1.1: version "2.1.4" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1" @@ -1155,18 +1131,6 @@ kind-of@^4.0.0: dependencies: is-buffer "^1.1.5" -lodash.get@^4.4.2: - version "4.4.2" - resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" - -lodash.merge@^4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.0.tgz#69884ba144ac33fe699737a6086deffadd0f89c5" - -lodash.set@^4.3.2: - version "4.3.2" - resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23" - lodash@^4.17.4, lodash@latest: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -1233,10 +1197,6 @@ nan@^2.3.0: version "2.8.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a" -node-fetch@2.0.0-alpha.9: - version "2.0.0-alpha.9" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.0.0-alpha.9.tgz#990c0634f510f76449a0d6f6eaec96b22f273628" - node-fetch@^1.6.3: version "1.6.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.6.3.tgz#dc234edd6489982d58e8f0db4f695029abcd8c04" diff --git a/packages/registry/syncano/registry/src/add.js b/packages/registry/syncano/registry/src/add.js index bce38a25..d559d338 100644 --- a/packages/registry/syncano/registry/src/add.js +++ b/packages/registry/syncano/registry/src/add.js @@ -77,7 +77,7 @@ export default async (ctx) => { ? socket.keywords.map(keyword => keyword.name) : [] }) } catch (err) { - error(err) + error(err.message) response.json({message: err.message}, 400) } }