From e566581264285c231d5e3cabab82d2016faf11ad Mon Sep 17 00:00:00 2001 From: debuggy Date: Tue, 2 Jul 2019 18:13:00 +0800 Subject: [PATCH 1/6] fix bug when pylonAddress not set in .env --- .../components/data/data-component.jsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/webportal/src/app/job-submission/components/data/data-component.jsx b/src/webportal/src/app/job-submission/components/data/data-component.jsx index caf88abd3a..4e3ec93a49 100644 --- a/src/webportal/src/app/job-submission/components/data/data-component.jsx +++ b/src/webportal/src/app/job-submission/components/data/data-component.jsx @@ -46,14 +46,20 @@ function reducer(state, action) { } export const DataComponent = React.memo((props) => { - // add pylonAddress to .env for local debug - const hdfsHost = config.hasOwnProperty('pylonAddress') ? getHostNameFromUrl(config.pylonAddress) : window.location.hostname; + const envsubRegex = /^\${.*}$/; + let hdfsHost; + if (!config.pylonAddress || envsubRegex.test(config.pylonAddress)) { + hdfsHost = window.location.hostname; + } else { + // add pylonAddress to .env for local debug + hdfsHost = getHostNameFromUrl(config.pylonAddress); + } // TODO: add a judgement whether pylon is ready const hdfsClient = new WebHDFSClient(hdfsHost); const {onChange} = props; const [teamConfigs, setTeamConfigs] = useState(); const [defaultTeamConfigs, setDefaultTeamConfigs] = useState(); - const [dataError, setDataError] = useState({ + const [dataError, setDataError] = useState({ customContainerPathError: false, customDataSourceError: false, }); From 94cbafc3874a12d66af19197c7f9a72ae51ebd66 Mon Sep 17 00:00:00 2001 From: debuggy Date: Tue, 2 Jul 2019 18:15:23 +0800 Subject: [PATCH 2/6] format --- .../src/app/job-submission/components/data/data-component.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webportal/src/app/job-submission/components/data/data-component.jsx b/src/webportal/src/app/job-submission/components/data/data-component.jsx index 4e3ec93a49..fd9f42728b 100644 --- a/src/webportal/src/app/job-submission/components/data/data-component.jsx +++ b/src/webportal/src/app/job-submission/components/data/data-component.jsx @@ -59,7 +59,7 @@ export const DataComponent = React.memo((props) => { const {onChange} = props; const [teamConfigs, setTeamConfigs] = useState(); const [defaultTeamConfigs, setDefaultTeamConfigs] = useState(); - const [dataError, setDataError] = useState({ + const [dataError, setDataError] = useState({ customContainerPathError: false, customDataSourceError: false, }); From f383b667bce0da97b8a1026423f0734254a9285a Mon Sep 17 00:00:00 2001 From: debuggy Date: Tue, 2 Jul 2019 19:49:18 +0800 Subject: [PATCH 3/6] remain parameter default --- .../src/app/job-submission/components/data/data-component.jsx | 2 +- src/webportal/src/app/job-submission/utils/webhdfs.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/webportal/src/app/job-submission/components/data/data-component.jsx b/src/webportal/src/app/job-submission/components/data/data-component.jsx index fd9f42728b..b9f9b0ce8e 100644 --- a/src/webportal/src/app/job-submission/components/data/data-component.jsx +++ b/src/webportal/src/app/job-submission/components/data/data-component.jsx @@ -55,7 +55,7 @@ export const DataComponent = React.memo((props) => { hdfsHost = getHostNameFromUrl(config.pylonAddress); } // TODO: add a judgement whether pylon is ready - const hdfsClient = new WebHDFSClient(hdfsHost); + const hdfsClient = new WebHDFSClient(hdfsHost, undefined, undefined, 80, '/webhdfs/api/v1'); const {onChange} = props; const [teamConfigs, setTeamConfigs] = useState(); const [defaultTeamConfigs, setDefaultTeamConfigs] = useState(); diff --git a/src/webportal/src/app/job-submission/utils/webhdfs.js b/src/webportal/src/app/job-submission/utils/webhdfs.js index 2fdc0823ea..3d4579bbdb 100644 --- a/src/webportal/src/app/job-submission/utils/webhdfs.js +++ b/src/webportal/src/app/job-submission/utils/webhdfs.js @@ -2,9 +2,10 @@ import * as webhdfs from 'webhdfs'; import {promisify} from 'util'; export class WebHDFSClient { - constructor(host, user, timeout, port = '80', path = `/webhdfs/api/v1`) { + constructor(host, user, timeout, port = '50070', path = `/webhdfs/v1`) { this.host = `http://${host}:${port}`; this.endpoint = `http://${host}:${port}${path}`; + console.log(this.endpoint) this.client = webhdfs.createClient({host, port, user, path}, {timeout}); this.client.readdir = promisify(this.client.readdir); this.client.mkdir = promisify(this.client.mkdir); From 2426f25c8946d6dec143bf4a1f4c5a742f4a5ae7 Mon Sep 17 00:00:00 2001 From: debuggy Date: Wed, 3 Jul 2019 11:03:23 +0800 Subject: [PATCH 4/6] resolve comments --- src/webportal/src/app/env.js.template | 1 + .../components/data/data-component.jsx | 15 ++++++++------- .../src/app/job-submission/utils/utils.js | 5 +++++ .../src/app/job-submission/utils/webhdfs.js | 1 - 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/webportal/src/app/env.js.template b/src/webportal/src/app/env.js.template index c0ff8b7d13..aa5803a0b3 100644 --- a/src/webportal/src/app/env.js.template +++ b/src/webportal/src/app/env.js.template @@ -8,6 +8,7 @@ window.ENV = { promScrapeTime: '${PROM_SCRAPE_TIME}', authnMethod: '${AUTHN_METHOD}', pylonAddress: '${PYLON_ADDRESS}', + webHDFS: '${WEBHDFS_URI}', }; window.PAI_PLUGINS = [${WEBPORTAL_PLUGINS}][0] || []; diff --git a/src/webportal/src/app/job-submission/components/data/data-component.jsx b/src/webportal/src/app/job-submission/components/data/data-component.jsx index b9f9b0ce8e..bc40468512 100644 --- a/src/webportal/src/app/job-submission/components/data/data-component.jsx +++ b/src/webportal/src/app/job-submission/components/data/data-component.jsx @@ -8,7 +8,7 @@ import {MountTreeView} from './mount-tree-view'; import {SidebarCard} from '../sidebar/sidebar-card'; import {WebHDFSClient} from '../../utils/webhdfs'; import {HdfsContext} from '../../models/data/hdfs-context'; -import {getHostNameFromUrl} from '../../utils/utils'; +import {getHostNameFromUrl, getPortFromUrl} from '../../utils/utils'; import {MountDirectories} from '../../models/data/mount-directories'; import { fetchUserGroup, @@ -46,16 +46,17 @@ function reducer(state, action) { } export const DataComponent = React.memo((props) => { - const envsubRegex = /^\${.*}$/; + const envsubRegex = /^\${.*}$/; // the template string ${xx} will be reserved in envsub if not provide value let hdfsHost; - if (!config.pylonAddress || envsubRegex.test(config.pylonAddress)) { + let port; + if (!config.webHDFS || envsubRegex.test(config.webHDFS)) { hdfsHost = window.location.hostname; } else { - // add pylonAddress to .env for local debug - hdfsHost = getHostNameFromUrl(config.pylonAddress); + // add webHDFS to .env for local debug + hdfsHost = getHostNameFromUrl(config.webHDFS); + port = getPortFromUrl(config.webHDFS); } - // TODO: add a judgement whether pylon is ready - const hdfsClient = new WebHDFSClient(hdfsHost, undefined, undefined, 80, '/webhdfs/api/v1'); + const hdfsClient = new WebHDFSClient(hdfsHost, undefined, undefined, port); const {onChange} = props; const [teamConfigs, setTeamConfigs] = useState(); const [defaultTeamConfigs, setDefaultTeamConfigs] = useState(); diff --git a/src/webportal/src/app/job-submission/utils/utils.js b/src/webportal/src/app/job-submission/utils/utils.js index e4a3a5a163..58bc399a9e 100644 --- a/src/webportal/src/app/job-submission/utils/utils.js +++ b/src/webportal/src/app/job-submission/utils/utils.js @@ -126,6 +126,11 @@ export function getHostNameFromUrl(url) { return parser.hostname; } +export function getPortFromUrl(url) { + const parser = new URL(url); + return parser.port; +} + function addPreCommandsToProtocolTaskRoles(protocol, preCommands) { Object.keys(protocol.taskRoles).forEach((taskRoleKey) => { const taskRole = protocol.taskRoles[taskRoleKey]; diff --git a/src/webportal/src/app/job-submission/utils/webhdfs.js b/src/webportal/src/app/job-submission/utils/webhdfs.js index 3d4579bbdb..c7121cdd7f 100644 --- a/src/webportal/src/app/job-submission/utils/webhdfs.js +++ b/src/webportal/src/app/job-submission/utils/webhdfs.js @@ -5,7 +5,6 @@ export class WebHDFSClient { constructor(host, user, timeout, port = '50070', path = `/webhdfs/v1`) { this.host = `http://${host}:${port}`; this.endpoint = `http://${host}:${port}${path}`; - console.log(this.endpoint) this.client = webhdfs.createClient({host, port, user, path}, {timeout}); this.client.readdir = promisify(this.client.readdir); this.client.mkdir = promisify(this.client.mkdir); From 1bbc7c35fb6953f8ba9221cf5dbbf68033341975 Mon Sep 17 00:00:00 2001 From: debuggy Date: Wed, 3 Jul 2019 11:09:26 +0800 Subject: [PATCH 5/6] remove duplicate dependency --- src/webportal/package.json | 1 - src/webportal/yarn.lock | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/webportal/package.json b/src/webportal/package.json index fdd3712373..966f822591 100644 --- a/src/webportal/package.json +++ b/src/webportal/package.json @@ -60,7 +60,6 @@ "handlebars-loader": "~1.6.0", "html-loader": "~0.5.1", "html-webpack-plugin": "^4.0.0-beta.5", - "@hapi/joi": "~15.1.0", "joi-browser": "^13.4.0", "jquery": "~3.4.0", "js-cookie": "~2.2.0", diff --git a/src/webportal/yarn.lock b/src/webportal/yarn.lock index d3382937a3..8c91631728 100644 --- a/src/webportal/yarn.lock +++ b/src/webportal/yarn.lock @@ -708,7 +708,7 @@ resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-8.0.1.tgz#9712fa2ad124ac64668ab06ba847b1eaf83a03fd" integrity sha512-cctMYH5RLbElaUpZn3IJaUj9QNQD8iXDnl7xNY6KB1aFD2ciJrwpo3kvZowIT75uA+silJFDnSR2kGakALUymg== -"@hapi/joi@~15.1.0": +"@hapi/joi@^15.1.0": version "15.1.0" resolved "https://registry.yarnpkg.com/@hapi/joi/-/joi-15.1.0.tgz#940cb749b5c55c26ab3b34ce362e82b6162c8e7a" integrity sha512-n6kaRQO8S+kepUTbXL9O/UOL788Odqs38/VOfoCrATDtTvyfiO3fgjlSRaNkHabpTLgM7qru9ifqXlXbXk8SeQ== From a0bb11925c9c41f1309169ed1433a3de9cc913b2 Mon Sep 17 00:00:00 2001 From: debuggy Date: Wed, 3 Jul 2019 11:48:32 +0800 Subject: [PATCH 6/6] resolve comments --- src/webportal/src/app/env.js.template | 2 +- .../components/data/data-component.jsx | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/webportal/src/app/env.js.template b/src/webportal/src/app/env.js.template index aa5803a0b3..11c5dcbe1c 100644 --- a/src/webportal/src/app/env.js.template +++ b/src/webportal/src/app/env.js.template @@ -8,7 +8,7 @@ window.ENV = { promScrapeTime: '${PROM_SCRAPE_TIME}', authnMethod: '${AUTHN_METHOD}', pylonAddress: '${PYLON_ADDRESS}', - webHDFS: '${WEBHDFS_URI}', + webHDFSUri: '${WEBHDFS_URI}', }; window.PAI_PLUGINS = [${WEBPORTAL_PLUGINS}][0] || []; diff --git a/src/webportal/src/app/job-submission/components/data/data-component.jsx b/src/webportal/src/app/job-submission/components/data/data-component.jsx index bc40468512..e6a59bbcf5 100644 --- a/src/webportal/src/app/job-submission/components/data/data-component.jsx +++ b/src/webportal/src/app/job-submission/components/data/data-component.jsx @@ -49,14 +49,17 @@ export const DataComponent = React.memo((props) => { const envsubRegex = /^\${.*}$/; // the template string ${xx} will be reserved in envsub if not provide value let hdfsHost; let port; - if (!config.webHDFS || envsubRegex.test(config.webHDFS)) { + let apiPath; + if (!config.webHDFSUri || envsubRegex.test(config.webHDFSUri)) { hdfsHost = window.location.hostname; + port = 80; + apiPath = '/webhdfs/api/v1'; } else { - // add webHDFS to .env for local debug - hdfsHost = getHostNameFromUrl(config.webHDFS); - port = getPortFromUrl(config.webHDFS); + // add WEBHDFS_URI to .env for local debug + hdfsHost = getHostNameFromUrl(config.webHDFSUri); + port = getPortFromUrl(config.webHDFSUri); } - const hdfsClient = new WebHDFSClient(hdfsHost, undefined, undefined, port); + const hdfsClient = new WebHDFSClient(hdfsHost, undefined, undefined, port, apiPath); const {onChange} = props; const [teamConfigs, setTeamConfigs] = useState(); const [defaultTeamConfigs, setDefaultTeamConfigs] = useState();