diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml new file mode 100644 index 00000000..86ac9bba --- /dev/null +++ b/.github/workflows/kb_sdk-actions.yml @@ -0,0 +1,136 @@ +name: Basic kb-sdk tests + +on: + [push, pull_request] + +jobs: + + test_kb-sdk_builds: + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + language: [perl, python, java] + test_type: [base, example] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Build with Ant + run: | + make + + - name: Add bin to $PATH + run: | + echo "::add-path::$GITHUB_WORKSPACE/bin" + + - name: checking kb-sdk functions + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + env + kb-sdk help + kb-sdk version + + - name: init base repo + if: matrix.test_type == 'base' + env: + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + kb-sdk init -l $LANGUAGE_TOKEN -u user SampleApp + + - name: init example repo + if: matrix.test_type == 'example' + env: + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + kb-sdk init -l $LANGUAGE_TOKEN -u user --example SampleApp + + - name: test ${{ matrix.test_type }} repo + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + cd SampleApp + kb-sdk test || true + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + kb-sdk test + + - name: make resulting app available as artefact in case of failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: kbaseapp-${{ matrix.language }}-${{ matrix.test_type }}-${{ matrix.os }} + path: SampleApp + + + test_existing_repos: + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + app: [WsLargeDataIO, KBaseReport] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Build with Ant + run: | + make + + - name: Add bin to $PATH + run: | + echo "::add-path::$GITHUB_WORKSPACE/bin" + + - name: checking basic kb-sdk functions + run: | + env + kb-sdk help + kb-sdk version + + - name: Checkout existing kbase module ${{ matrix.app }} + uses: actions/checkout@v2 + with: + repository: kbaseapps/${{ matrix.app }} + path: kbase_app + + - name: prep for repo tests + run: | + cd kbase_app + kb-sdk test || true + + - name: edit test.cfg + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd kbase_app + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + + - name: run test + run: | + cd kbase_app + kb-sdk test + + - name: make resulting app available as artefact in case of failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} + path: kbase_app diff --git a/.travis.yml b/.travis.yml index 4e18eb02..6b83fb97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,5 +44,3 @@ script: - cd MyContigFilter - make - kb-sdk validate - - diff --git a/Makefile b/Makefile index 2ecd55e3..b1e51fb5 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ bin: jars-submodule-init echo '#!/bin/bash' > bin/kb-sdk echo 'DIR=$(DIR)' >> bin/kb-sdk echo 'KBASE_JARS_DIR=$$DIR/submodules/jars/lib/jars' >> bin/kb-sdk - @# Next command processes links in JAR_DEPS_BIN file and has 5 parts (one on each line): + @# Next command processes links in JAR_DEPS_BIN file and has 5 parts (one on each line): @# (1) removing comments @# (2) trimming each line (picking first word actually) @# (3) skipping empty lines diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 40808007..d71d78db 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,11 +1,21 @@ OVERVIEW ----------------------------------------- -The KBase SDK is a set of tools for developing new modules -that can be dynamically registered and run on the KBase -platform. Modules include all code, specification files, and -documentation needed to define and run a set of methods +The KBase SDK is a set of tools for developing new modules +that can be dynamically registered and run on the KBase +platform. Modules include all code, specification files, and +documentation needed to define and run a set of methods in the KBase Narrative interface. +VERSION: 1.2.2 (Released 06/xx/2020) +------------------------------------ +NEW FEATURES: +- kb-sdk version reports the build time and date of the instance +- kb-sdk can use the environment variable KBASE_TEST_TOKEN if a test token is not present in the test.cfg file +- GitHub Actions now used to run basic tests on initialising and testing new base and sample repos. + +BUG FIXES: +- Corrected a class name in the generated tests for the Java SDK module. + VERSION: 1.2.0 (Released 11/14/2018) ------------------------------------ NEW FEATURES: @@ -74,7 +84,7 @@ VERSION: 1.0.0 (Released 11/19/2015) ------------------------------------------ NEW FEATURES: - R language is now supported for generation of clients and server stubs. -- Test sub-command is fully implemented for all 4 languages (perl, +- Test sub-command is fully implemented for all 4 languages (perl, python, java and R). - Version sub-command was added diff --git a/build.xml b/build.xml index abbaa391..6b82a7fa 100644 --- a/build.xml +++ b/build.xml @@ -49,7 +49,7 @@ - + @@ -102,7 +102,7 @@ - + @@ -116,7 +116,8 @@ ### PLEASE DO NOT CHANGE THIS FILE MANUALLY! ### giturl=${git.url} branch=${git.branch} -commit=${git.commit} +commit=${git.commit} +build_timestamp=${DSTAMP}-${TSTAMP} Compiling with jar directory: ${jardir} @@ -205,7 +206,7 @@ echo $pid > $DIR/pid.txt Successfully built: ${ee.dir}/${ee.start.script} - + #!/bin/bash DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) kill $(<"$DIR/pid.txt") diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 9d6a0e06..58b54125 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -31,7 +31,7 @@ import us.kbase.mobu.validator.ModuleValidator; public class ModuleBuilder { - + private static final String defaultParentPackage = "us.kbase"; private static final String MODULE_BUILDER_SH_NAME = "kb-sdk"; @@ -45,36 +45,34 @@ public class ModuleBuilder { private static final String INSTALL_COMMAND = "install"; private static final String RUN_COMMAND = "run"; //private static final String SUBMIT_COMMAND = "submit"; - + public static final String GLOBAL_SDK_HOME_ENV_VAR = "KB_SDK_HOME"; public static final String DEFAULT_METHOD_STORE_URL = "https://appdev.kbase.us/services/narrative_method_store/rpc"; - - public static final String VERSION = "1.2.0"; - - + + public static final String VERSION = "1.2.2"; + public static void main(String[] args) throws Exception { - - // setup the basic CLI argument parser with global -h and --help commands - GlobalArgs gArgs = new GlobalArgs(); - JCommander jc = new JCommander(gArgs); - jc.setProgramName(MODULE_BUILDER_SH_NAME); - - - // add the 'init' command - InitCommandArgs initArgs = new InitCommandArgs(); - jc.addCommand(INIT_COMMAND, initArgs); - - // add the 'validate' command - ValidateCommandArgs validateArgs = new ValidateCommandArgs(); - jc.addCommand(VALIDATE_COMMAND, validateArgs); - - // add the 'compile' command - CompileCommandArgs compileArgs = new CompileCommandArgs(); - jc.addCommand(COMPILE_COMMAND, compileArgs); - - // add the 'help' command - HelpCommandArgs help = new HelpCommandArgs(); - jc.addCommand(HELP_COMMAND, help); + + // set up the basic CLI argument parser with global -h and --help commands + GlobalArgs gArgs = new GlobalArgs(); + JCommander jc = new JCommander(gArgs); + jc.setProgramName(MODULE_BUILDER_SH_NAME); + + // add the 'init' command + InitCommandArgs initArgs = new InitCommandArgs(); + jc.addCommand(INIT_COMMAND, initArgs); + + // add the 'validate' command + ValidateCommandArgs validateArgs = new ValidateCommandArgs(); + jc.addCommand(VALIDATE_COMMAND, validateArgs); + + // add the 'compile' command + CompileCommandArgs compileArgs = new CompileCommandArgs(); + jc.addCommand(COMPILE_COMMAND, compileArgs); + + // add the 'help' command + HelpCommandArgs help = new HelpCommandArgs(); + jc.addCommand(HELP_COMMAND, help); // add the 'test' command TestCommandArgs testArgs = new TestCommandArgs(); @@ -91,84 +89,87 @@ public static void main(String[] args) throws Exception { // add the 'install' command InstallCommandArgs installArgs = new InstallCommandArgs(); jc.addCommand(INSTALL_COMMAND, installArgs); - + // add the 'run' command RunCommandArgs runArgs = new RunCommandArgs(); jc.addCommand(RUN_COMMAND, runArgs); - // parse the arguments and gracefully catch any errors - try { - jc.parse(args); - } catch (RuntimeException e) { - showError("Command Line Argument Error", e.getMessage()); - System.exit(1); - } - - // if the help flag is set, then show some help and exit - if(gArgs.help) { - showBriefHelp(jc, System.out); - return; - } - - // no command entered, just print brief info and exit - if(jc.getParsedCommand()==null) { - showBriefHelp(jc, System.out); - return; - } - - // if we get here, we have a valid command, so process it and do stuff - int returnCode = 0; - if(jc.getParsedCommand().equals(HELP_COMMAND)) { - showCommandUsage(jc,help,System.out); - - } else if(jc.getParsedCommand().equals(INIT_COMMAND)) { - returnCode = runInitCommand(initArgs,jc); - } else if(jc.getParsedCommand().equals(VALIDATE_COMMAND)) { - returnCode = runValidateCommand(validateArgs,jc); - } else if(jc.getParsedCommand().equals(COMPILE_COMMAND)) { - returnCode = runCompileCommand(compileArgs,jc); + // print name and version + printVersion(); + + // parse the arguments and gracefully catch any errors + try { + jc.parse(args); + } catch (RuntimeException e) { + showError("Command Line Argument Error", e.getMessage()); + System.exit(1); + } + + // if the help flag is set, then show some help and exit + if(gArgs.help) { + showBriefHelp(jc, System.out); + return; + } + + // no command entered, just print brief info and exit + if(jc.getParsedCommand()==null) { + showBriefHelp(jc, System.out); + return; + } + + // if we get here, we have a valid command, so process it and do stuff + int returnCode = 0; + if(jc.getParsedCommand().equals(HELP_COMMAND)) { + showCommandUsage(jc,help,System.out); + + } else if(jc.getParsedCommand().equals(INIT_COMMAND)) { + returnCode = runInitCommand(initArgs,jc); + } else if(jc.getParsedCommand().equals(VALIDATE_COMMAND)) { + returnCode = runValidateCommand(validateArgs,jc); + } else if(jc.getParsedCommand().equals(COMPILE_COMMAND)) { + returnCode = runCompileCommand(compileArgs,jc); } else if(jc.getParsedCommand().equals(TEST_COMMAND)) { returnCode = runTestCommand(testArgs,jc); - } else if (jc.getParsedCommand().equals(VERSION_COMMAND)) { - returnCode = runVersionCommand(versionArgs, jc); - } else if (jc.getParsedCommand().equals(RENAME_COMMAND)) { - returnCode = runRenameCommand(renameArgs, jc); - } else if (jc.getParsedCommand().equals(INSTALL_COMMAND)) { - returnCode = runInstallCommand(installArgs, jc); - } else if (jc.getParsedCommand().equals(RUN_COMMAND)) { - returnCode = runRunCommand(runArgs, jc); - } - - if(returnCode!=0) { - System.exit(returnCode); - } + } else if (jc.getParsedCommand().equals(VERSION_COMMAND)) { + returnCode = runVersionCommand(versionArgs, jc); + } else if (jc.getParsedCommand().equals(RENAME_COMMAND)) { + returnCode = runRenameCommand(renameArgs, jc); + } else if (jc.getParsedCommand().equals(INSTALL_COMMAND)) { + returnCode = runInstallCommand(installArgs, jc); + } else if (jc.getParsedCommand().equals(RUN_COMMAND)) { + returnCode = runRunCommand(runArgs, jc); + } + + if(returnCode!=0) { + System.exit(returnCode); + } } - + private static int runValidateCommand(ValidateCommandArgs validateArgs, JCommander jc) { - // initialize - if(validateArgs.modules==null) { - validateArgs.modules = new ArrayList(); - } - if(validateArgs.modules.size()==0) { - validateArgs.modules.add("."); - } - try { - ModuleValidator mv = new ModuleValidator(validateArgs.modules,validateArgs.verbose, - validateArgs.methodStoreUrl, validateArgs.allowSyncMethods); - return mv.validateAll(); - } catch (Exception e) { + // initialize + if(validateArgs.modules==null) { + validateArgs.modules = new ArrayList(); + } + if(validateArgs.modules.size()==0) { + validateArgs.modules.add("."); + } + try { + ModuleValidator mv = new ModuleValidator(validateArgs.modules,validateArgs.verbose, + validateArgs.methodStoreUrl, validateArgs.allowSyncMethods); + return mv.validateAll(); + } catch (Exception e) { if (validateArgs.verbose) e.printStackTrace(); showError("Error while validating module", e.getMessage()); return 1; } - } + } /** * Runs the module initialization command - this creates a new module in the relative directory name given. * There's only a couple of possible arguments here in the initArgs: * userName (required) - the user's Github user name, used to set up some optional fields - * moduleNames (required) - this catchall represents the module's name. Any whitespace (e.g. token breaks) + * moduleNames (required) - this catchall represents the module's name. Any whitespace (e.g. token breaks) * are replaced with underscores. So if a user runs: * kb-sdk init my new module * they get a module called "my_new_module" in a directory of the same name. @@ -176,80 +177,80 @@ private static int runValidateCommand(ValidateCommandArgs validateArgs, JCommand * @param jc * @return */ - private static int runInitCommand(InitCommandArgs initArgs, JCommander jc) { - // Figure out module name. - // Join together spaced out names with underscores if necessary. - if (initArgs.moduleNames == null || initArgs.moduleNames.size() == 0) { - ModuleBuilder.showError("Init Error", "A module name is required."); - return 1; - } - String moduleName = StringUtils.join(initArgs.moduleNames, "_"); - - // Get username if available - String userName = null; - if (initArgs.userName != null) - userName = initArgs.userName; - - // Get chosen language - String language = ModuleInitializer.DEFAULT_LANGUAGE; - if (initArgs.language != null) { - language = initArgs.language; - } - try { - ModuleInitializer initer = new ModuleInitializer(moduleName, userName, language, initArgs.verbose); - initer.initialize(initArgs.example); - } - catch (Exception e) { + private static int runInitCommand(InitCommandArgs initArgs, JCommander jc) { + // Figure out module name. + // Join together spaced out names with underscores if necessary. + if (initArgs.moduleNames == null || initArgs.moduleNames.size() == 0) { + ModuleBuilder.showError("Init Error", "A module name is required."); + return 1; + } + String moduleName = StringUtils.join(initArgs.moduleNames, "_"); + + // Get username if available + String userName = null; + if (initArgs.userName != null) + userName = initArgs.userName; + + // Get chosen language + String language = ModuleInitializer.DEFAULT_LANGUAGE; + if (initArgs.language != null) { + language = initArgs.language; + } + try { + ModuleInitializer initer = new ModuleInitializer(moduleName, userName, language, initArgs.verbose); + initer.initialize(initArgs.example); + } + catch (Exception e) { if (initArgs.verbose) e.printStackTrace(); - showError("Error while initializing module", e.getMessage()); - return 1; - } - return 0; - } - - public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { - printVersion(); - // Step 1: convert list of args to a Files (this must be defined because it is required) - File specFile = null; - try { - if(a.specFileNames.size()>1) { - // right now we only support compiling one file at a time + showError("Error while initializing module", e.getMessage()); + return 1; + } + return 0; + } + + public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { + printVersion(); + // Step 1: convert list of args to a Files (this must be defined because it is required) + File specFile = null; + try { + if(a.specFileNames.size()>1) { + // right now we only support compiling one file at a time ModuleBuilder.showError("Compile Error","Too many input KIDL spec files provided", - "Currently there is only support for compiling one module at a time, which \n"+ - "is required to register data types and KIDL specs with the Workspace. This \n"+ - "code may be extended in the future to allow multiple modules to be compiled \n"+ - "together from separate files into the same client/server."); + "Currently there is only support for compiling one module at a time, which \n"+ + "is required to register data types and KIDL specs with the Workspace. This \n"+ + "code may be extended in the future to allow multiple modules to be compiled \n"+ + "together from separate files into the same client/server."); return 1; - } - List specFiles = convertToFiles(a.specFileNames); - specFile = specFiles.get(0); - } catch (IOException | RuntimeException e) { + } + List specFiles = convertToFiles(a.specFileNames); + specFile = specFiles.get(0); + } catch (IOException | RuntimeException e) { showError("Error accessing input KIDL spec file", e.getMessage()); if (a.verbose) { e.printStackTrace(); } return 1; - } - + } + if (a.clAsyncVer != null && a.dynservVer != null) { showError("Bad arguments", "clasyncver and dynserver cannot both be specified"); return 1; } - - // Step 2: check or create the output directory - File outDir = a.outDir == null ? new File(".") : new File(a.outDir); + + // Step 2: check or create the output directory + File outDir = a.outDir == null ? new File(".") : new File(a.outDir); try { - outDir = outDir.getCanonicalFile(); - } catch (IOException e) { + outDir = outDir.getCanonicalFile(); + } catch (IOException e) { showError("Error accessing output directory", e.getMessage()); return 1; - } + } if (!outDir.exists()) outDir.mkdirs(); - - // Step 3: validate the URL option if it is defined + + // Step 3: validate the URL option if it is defined if (a.url != null) { try { new URL(a.url); @@ -258,7 +259,7 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { return 1; } } - + // Step 4: info collection for status method File moduleDir = specFile.getAbsoluteFile().getParentFile(); String semanticVersion = null; @@ -290,13 +291,13 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { } } try { - RunCompileCommand.generate(specFile, a.url, a.jsClientSide, a.jsClientName, a.perlClientSide, - a.perlClientName, a.perlServerSide, a.perlServerName, a.perlImplName, - a.perlPsgiName, a.perlEnableRetries, a.pyClientSide, a.pyClientName, - a.pyServerSide, a.pyServerName, a.pyImplName, a.javaClientSide, - a.javaServerSide, a.javaPackageParent, a.javaSrcDir, a.javaLibDir, - a.javaBuildXml, a.javaGwtPackage, a.rClientSide, a.rClientName, - a.rServerSide, a.rServerName, a.rImplName, outDir, a.jsonSchema, + RunCompileCommand.generate(specFile, a.url, a.jsClientSide, a.jsClientName, a.perlClientSide, + a.perlClientName, a.perlServerSide, a.perlServerName, a.perlImplName, + a.perlPsgiName, a.perlEnableRetries, a.pyClientSide, a.pyClientName, + a.pyServerSide, a.pyServerName, a.pyImplName, a.javaClientSide, + a.javaServerSide, a.javaPackageParent, a.javaSrcDir, a.javaLibDir, + a.javaBuildXml, a.javaGwtPackage, a.rClientSide, a.rClientName, + a.rServerSide, a.rServerName, a.rImplName, outDir, a.jsonSchema, a.makefile, a.clAsyncVer, a.dynservVer, a.html, semanticVersion, gitUrl, gitCommitHash); } catch (Throwable e) { @@ -310,30 +311,30 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { return 0; } - private static String getCmdOutput(File workDir, String... cmd) throws Exception { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - ProcessHelper.cmd(cmd).exec(workDir, null, pw); - return sw.toString().trim(); - } - + private static String getCmdOutput(File workDir, String... cmd) throws Exception { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + ProcessHelper.cmd(cmd).exec(workDir, null, pw); + return sw.toString().trim(); + } + static List convertToFiles(List filenames) throws IOException { - List files = new ArrayList(filenames.size()); - for(String filename: filenames) { - File f = new File(filename); - if(!f.exists()) { - throw new IOException("KIDL file \""+filename+"\" does not exist"); - } + List files = new ArrayList(filenames.size()); + for(String filename: filenames) { + File f = new File(filename); + if(!f.exists()) { + throw new IOException("KIDL file \""+filename+"\" does not exist"); + } files.add(f); - } - return files; + } + return files; } - + public static class GlobalArgs { - @Parameter(names = {"-h","--help"}, help = true, description="Display help and full usage information.") - boolean help; + @Parameter(names = {"-h","--help"}, help = true, description="Display help and full usage information.") + boolean help; } - + /** * Runs the module test command - this runs tests in local docker container. * @param testArgs @@ -360,24 +361,32 @@ private static int runTestCommand(TestCommandArgs testArgs, JCommander jc) { } private static void printVersion() { - String gitCommit = getGitCommit(); - System.out.println("KBase SDK version " + VERSION + (gitCommit == null ? "" : (" (commit " + gitCommit + ")"))); + String gitCommit = getGitProp("commit"); + String timestamp = getGitProp("build_timestamp"); + System.out.println("KBase SDK version " + VERSION + + (timestamp == null ? "" : ("_" + timestamp)) + + (gitCommit == null ? "" : (" (commit " + gitCommit + ")"))); } - public static String getGitCommit() { - String gitCommit = null; + + public static String getGitProp(String propertyToGet) { + String propertyValue = null; + Properties gitProps = new Properties(); try { - Properties gitProps = new Properties(); InputStream is = ModuleBuilder.class.getResourceAsStream("git.properties"); gitProps.load(is); is.close(); - gitCommit = gitProps.getProperty("commit"); - } catch (Exception ignore) {} - return gitCommit; + propertyValue = gitProps.getProperty(propertyToGet); + } catch (Exception e) { + showError("Error while retrieving version information from git.properties file: ", e.getMessage()); + System.exit(1); + } + return propertyValue; } - + + private static int runVersionCommand(VersionCommandArgs testArgs, JCommander jc) { - printVersion(); + // version was printed at start up, so just exit return 0; } @@ -398,14 +407,14 @@ private static int runRenameCommand(RenameCommandArgs renameArgs, JCommander jc) private static int runInstallCommand(InstallCommandArgs installArgs, JCommander jc) { if (installArgs.moduleName == null || installArgs.moduleName.size() != 1) { - ModuleBuilder.showError("Command Line Argument Error", + ModuleBuilder.showError("Command Line Argument Error", "One and only one module name should be provided"); return 1; } try { return new ClientInstaller().install(installArgs.lang, installArgs.async, - installArgs.core || installArgs.sync, installArgs.dynamic, - installArgs.tagVer, installArgs.verbose, installArgs.moduleName.get(0), + installArgs.core || installArgs.sync, installArgs.dynamic, + installArgs.tagVer, installArgs.verbose, installArgs.moduleName.get(0), null, installArgs.clientName); } catch (Exception e) { if (installArgs.verbose) @@ -417,17 +426,17 @@ private static int runInstallCommand(InstallCommandArgs installArgs, JCommander private static int runRunCommand(RunCommandArgs runArgs, JCommander jc) { if (runArgs.methodName == null || runArgs.methodName.size() != 1) { - ModuleBuilder.showError("Command Line Argument Error", + ModuleBuilder.showError("Command Line Argument Error", "One and only one method name should be provided"); return 1; } try { if (runArgs.inputFile == null && runArgs.inputJson == null && !runArgs.stdin) throw new IllegalStateException("No one input method is used " + - "(should be one of '-i', '-j' or '-s')"); - return new ModuleRunner(runArgs.sdkHome).run(runArgs.methodName.get(0), - runArgs.inputFile, runArgs.stdin, runArgs.inputJson, runArgs.output, - runArgs.tagVer, runArgs.verbose, runArgs.keepTempFiles, runArgs.provRefs, + "(should be one of '-i', '-j' or '-s')"); + return new ModuleRunner(runArgs.sdkHome).run(runArgs.methodName.get(0), + runArgs.inputFile, runArgs.stdin, runArgs.inputJson, runArgs.output, + runArgs.tagVer, runArgs.verbose, runArgs.keepTempFiles, runArgs.provRefs, runArgs.mountPoints); } catch (Exception e) { if (runArgs.verbose) @@ -439,150 +448,150 @@ private static int runRunCommand(RunCommandArgs runArgs, JCommander jc) { @Parameters(commandDescription = "Validate a module or modules.") private static class ValidateCommandArgs { - @Parameter(names={"-v","--verbose"}, description="Show verbose output") + @Parameter(names={"-v","--verbose"}, description="Show verbose output") boolean verbose = false; - - @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL " + - "(default is " + DEFAULT_METHOD_STORE_URL + ")") - String methodStoreUrl = DEFAULT_METHOD_STORE_URL; - - @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + - "(advanced option, default is false)") + + @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL " + + "(default is " + DEFAULT_METHOD_STORE_URL + ")") + String methodStoreUrl = DEFAULT_METHOD_STORE_URL; + + @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + + "(advanced option, default is false)") boolean allowSyncMethods = false; - - @Parameter(description="[path to the module directories]") + + @Parameter(description="[path to the module directories]") List modules; } - + @Parameters(commandDescription = "Initialize a module in the current directory.") private static class InitCommandArgs { - @Parameter(names={"-v","--verbose"}, description="Show verbose output") - boolean verbose = false; - - @Parameter(required=true, names={"-u","--user"}, - description="(Required) provide a username to serve as the owner of this module") - String userName; - - @Parameter(names={"-e","--example"}, - description="Include a fully featured example in your module. " + - "This generates an example set of code and configurations that can " + - "be used to demonstrate a very simple example.") - boolean example = false; - - @Parameter(names={"-l","--language"}, description="Choose a language for developing " + - " code in your module. You can currently choose from Python, Perl, and Java " + - "(default=Python)") - String language = ModuleInitializer.DEFAULT_LANGUAGE; - - @Parameter(required=true, description="") - List moduleNames; + @Parameter(names={"-v","--verbose"}, description="Show verbose output") + boolean verbose = false; + + @Parameter(required=true, names={"-u","--user"}, + description="(Required) provide a username to serve as the owner of this module") + String userName; + + @Parameter(names={"-e","--example"}, + description="Include a fully featured example in your module. " + + "This generates an example set of code and configurations that can " + + "be used to demonstrate a very simple example.") + boolean example = false; + + @Parameter(names={"-l","--language"}, description="Choose a language for developing " + + " code in your module. You can currently choose from Python, Perl, and Java " + + "(default=Python)") + String language = ModuleInitializer.DEFAULT_LANGUAGE; + + @Parameter(required=true, description="") + List moduleNames; } - - + + @Parameters(commandDescription = "Get help and usage information.") private static class HelpCommandArgs { - @Parameter(description="Show usage for this command") + @Parameter(description="Show usage for this command") List command; - @Parameter(names={"-a","--all"}, description="Show usage for all commands") + @Parameter(names={"-a","--all"}, description="Show usage for all commands") boolean showAll = false; } - + @Parameters(commandDescription = "Compile a KIDL file into client and server code") public static class CompileCommandArgs { - - @Parameter(names="--out",description="Set the output folder name (default is the current directory)") - //, metaVar="") + + @Parameter(names="--out",description="Set the output folder name (default is the current directory)") + //, metaVar="") String outDir = null; - - @Parameter(names="--js", description="Generate a JavaScript client with a standard default name") + + @Parameter(names="--js", description="Generate a JavaScript client with a standard default name") boolean jsClientSide = false; - @Parameter(names="--jsclname", description="Generate a JavaScript client with the name provided by this option" + - "(overrides --js option)")//, metaVar = "") + @Parameter(names="--jsclname", description="Generate a JavaScript client with the name provided by this option" + + "(overrides --js option)")//, metaVar = "") String jsClientName = null; - @Parameter(names="--pl", description="Generate a Perl client with the default name") + @Parameter(names="--pl", description="Generate a Perl client with the default name") boolean perlClientSide = false; - @Parameter(names="--plclname", description="Generate a Perl client with the name provided optionally " + - "prefixed by subdirectories separated by :: as in the standard perl module syntax (e.g. "+ - "Bio::KBase::MyModule::Client, overrides the --pl option)")//, metaVar = "") + @Parameter(names="--plclname", description="Generate a Perl client with the name provided optionally " + + "prefixed by subdirectories separated by :: as in the standard perl module syntax (e.g. "+ + "Bio::KBase::MyModule::Client, overrides the --pl option)")//, metaVar = "") String perlClientName = null; - @Parameter(names="--plsrv", description="Generate a Perl server with a " + - "standard default name. If set, Perl clients will automatically be generated too.") + @Parameter(names="--plsrv", description="Generate a Perl server with a " + + "standard default name. If set, Perl clients will automatically be generated too.") boolean perlServerSide = false; - @Parameter(names="--plsrvname", description="Generate a Perl server with with the " + - "name provided, optionally prefixed by subdirectories separated by :: as "+ - "in the standard perl module syntax (e.g. Bio::KBase::MyModule::Server, "+ - "overrides the --plserv option). If set, Perl clients will be generated too.") - //, metaVar = "") + @Parameter(names="--plsrvname", description="Generate a Perl server with with the " + + "name provided, optionally prefixed by subdirectories separated by :: as "+ + "in the standard perl module syntax (e.g. Bio::KBase::MyModule::Server, "+ + "overrides the --plserv option). If set, Perl clients will be generated too.") + //, metaVar = "") String perlServerName = null; - @Parameter(names="--plimplname", description="Generate a Perl server implementation with the " + - "name provided, optionally prefixed by subdirectories separated by :: as "+ - "in the standard Perl module syntax (e.g. Bio::KBase::MyModule::Impl). "+ - "If set, Perl server and client code will be generated too.")//, metaVar = "") + @Parameter(names="--plimplname", description="Generate a Perl server implementation with the " + + "name provided, optionally prefixed by subdirectories separated by :: as "+ + "in the standard Perl module syntax (e.g. Bio::KBase::MyModule::Impl). "+ + "If set, Perl server and client code will be generated too.")//, metaVar = "") String perlImplName = null; - @Parameter(names="--plpsginame", description="Generate a perl PSGI file with the name provided.")//, metaVar = "") + @Parameter(names="--plpsginame", description="Generate a perl PSGI file with the name provided.")//, metaVar = "") String perlPsgiName = null; - @Parameter(names="--plenableretries", description="When set, generated Perl client code will enable "+ - "reconnection retries with the server") + @Parameter(names="--plenableretries", description="When set, generated Perl client code will enable "+ + "reconnection retries with the server") boolean perlEnableRetries = false; - @Parameter(names="--py", description="Generate a Python client with a standard default name") + @Parameter(names="--py", description="Generate a Python client with a standard default name") boolean pyClientSide = false; - @Parameter(names="--pyclname", description="Generate a Python client with with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.client,"+ - "overrides the --py option).")//, metaVar = "") + @Parameter(names="--pyclname", description="Generate a Python client with with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.client,"+ + "overrides the --py option).")//, metaVar = "") String pyClientName = null; - @Parameter(names="--pysrv", description="Generate a Python server with a " + - "standard default name. If set, Python clients will automatically be generated too.") + @Parameter(names="--pysrv", description="Generate a Python server with a " + + "standard default name. If set, Python clients will automatically be generated too.") boolean pyServerSide = false; - @Parameter(names="--pysrvname", description="Generate a Python server with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.server,"+ - "overrides the --pysrv option).")//, metaVar = "") + @Parameter(names="--pysrvname", description="Generate a Python server with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.server,"+ + "overrides the --pysrv option).")//, metaVar = "") String pyServerName = null; - @Parameter(names="--pyimplname", description="Generate a Python server implementation with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.impl)." + - " If set, Python server and client code will be generated too.")//, metaVar = "") + @Parameter(names="--pyimplname", description="Generate a Python server implementation with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.impl)." + + " If set, Python server and client code will be generated too.")//, metaVar = "") String pyImplName = null; - @Parameter(names="--java", description="Generate Java client code in the directory set by --javasrc") + @Parameter(names="--java", description="Generate Java client code in the directory set by --javasrc") boolean javaClientSide = false; - @Parameter(names="--javasrc",description="Set the output folder for generated Java code")//, metaVar = - //"") + @Parameter(names="--javasrc",description="Set the output folder for generated Java code")//, metaVar = + //"") String javaSrcDir = "src"; - @Parameter(names="--javalib",description="Set the output folder for jar files (if defined then --java will be " + - "treated as true automatically)")//, metaVar = - //"") + @Parameter(names="--javalib",description="Set the output folder for jar files (if defined then --java will be " + + "treated as true automatically)")//, metaVar = + //"") String javaLibDir = null; - @Parameter(names="--url", description="Set the default url for the service in generated client code")//, metaVar = "") + @Parameter(names="--url", description="Set the default url for the service in generated client code")//, metaVar = "") String url = null; - @Parameter(names="--javapackage",description="Set the Java package for generated code (module subpackages are " + - "created in this package), default value is " + defaultParentPackage)//, - //metaVar = "") + @Parameter(names="--javapackage",description="Set the Java package for generated code (module subpackages are " + + "created in this package), default value is " + defaultParentPackage)//, + //metaVar = "") String javaPackageParent = defaultParentPackage; - @Parameter(names="--javasrv", description="Generate Java server code in the directory set by --javasrc") + @Parameter(names="--javasrv", description="Generate Java server code in the directory set by --javasrc") boolean javaServerSide = false; - @Parameter(names="--javagwt",description="Generate a GWT client Java package (useful if you need " + - "copies of generated classes for GWT clients)")//, metaVar="") + @Parameter(names="--javagwt",description="Generate a GWT client Java package (useful if you need " + + "copies of generated classes for GWT clients)")//, metaVar="") String javaGwtPackage = null; @Parameter(names="--r", description="DEPRECATED Generate a R client with a standard default name") @@ -610,20 +619,20 @@ public static class CompileCommandArgs { " If set, R server and client code will be generated too.") String rImplName = null; - @Parameter(names="--jsonschema",description="Generate JSON schema documents for the types in the output folder specified.")//, metaVar="") + @Parameter(names="--jsonschema",description="Generate JSON schema documents for the types in the output folder specified.")//, metaVar="") String jsonSchema = null; - - @Parameter(names="--javabuildxml",description="Will generate build.xml template for Ant") + + @Parameter(names="--javabuildxml",description="Will generate build.xml template for Ant") boolean javaBuildXml; - - @Parameter(names="--makefile",description="Will generate makefile templates for servers and/or java client") + + @Parameter(names="--makefile",description="Will generate makefile templates for servers and/or java client") boolean makefile = false; @Parameter(names="--clasyncver",description="Will set in client code version of service for asyncronous calls " + - "(it could be git commit hash of version registered in catalog or one of version tags: dev/beta/release)") + "(it could be git commit hash of version registered in catalog or one of version tags: dev/beta/release)") String clAsyncVer = null; - + @Parameter(names="--dynservver", description="Clients will be built " + "for use with KBase dynamic services (e.g. with URL lookup " + "via the Service Wizard) with the specified version " + @@ -631,11 +640,11 @@ public static class CompileCommandArgs { "clasyncver may not be specified if " + "dynservver is specified.") String dynservVer = null; - + @Parameter(names="--html", description="Generate HTML version(s) of " + "the input spec file(s)") boolean html = false; - + @Parameter(names={"-v", "--verbose"}, description="Print full stack " + "trace on a compile failure") boolean verbose = false; @@ -643,16 +652,16 @@ public static class CompileCommandArgs { @Parameter(required=true, description="") List specFileNames; } - + @Parameters(commandDescription = "Test a module with local Docker.") private static class TestCommandArgs { @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL used in validation " + "(default is " + DEFAULT_METHOD_STORE_URL + ")") String methodStoreUrl = DEFAULT_METHOD_STORE_URL; - + @Parameter(names={"-s", "--skip_validation"}, description="Will skip validation step (default is false)") boolean skipValidation = false; - + @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + "(advanced option, part of validation settings, default value is false)") boolean allowSyncMethods = false; @@ -660,11 +669,11 @@ private static class TestCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") boolean verbose = false; } - + @Parameters(commandDescription = "Print current version of kb-sdk.") private static class VersionCommandArgs { } - + @Parameters(commandDescription = "Rename a module name.") private static class RenameCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") @@ -679,16 +688,16 @@ private static class InstallCommandArgs { @Parameter(names={"-l", "--language"}, description="Language of generated client code " + "(default is one defined in kbase.yml)") String lang; - + @Parameter(names={"-a", "--async"}, description="Force generation of asynchronous calls " + - "(default is chosen based on information registered in catalog)") + "(default is chosen based on information registered in catalog)") boolean async = false; @Parameter(names={"-s", "--sync"}, description="Depricated flag, means the same as -c (--core)") boolean sync = false; @Parameter(names={"-c", "--core"}, description="Force generation of calls to core service" + - "(WARNING: please use it only for core services not registered in catalog)") + "(WARNING: please use it only for core services not registered in catalog)") boolean core = false; @Parameter(names={"-d", "--dynamic"}, description="Force generation of dynamic service calls" + @@ -701,7 +710,7 @@ private static class InstallCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") boolean verbose = false; - + @Parameter(names={"-n","--clientname"}, description="Optional parameter defining custom client name " + "(default is module name)") String clientName = null; @@ -713,19 +722,19 @@ private static class InstallCommandArgs { @Parameters(commandDescription = "Run a method of registered KBase module locally.") private static class RunCommandArgs { @Parameter(names={"-i", "--input"}, description="Input JSON file " + - "(optional, if not set -s or -j should be used)") + "(optional, if not set -s or -j should be used)") File inputFile = null; @Parameter(names={"-s","--stdin"}, description="Flag for reading input data from STDIN " + - "(default is false, used if neither -i or -j is set)") + "(default is false, used if neither -i or -j is set)") boolean stdin = false; @Parameter(names={"-j","--json"}, description="Input JSON string " + - "(optional, if not set -s or -i should be used)") + "(optional, if not set -s or -i should be used)") String inputJson = null; @Parameter(names={"-o", "--output"}, description="Output JSON file " + - "(optional, if not set output will be printed to STDOUT)") + "(optional, if not set output will be printed to STDOUT)") File output = null; @Parameter(names={"-t","--tag-or-ver"}, description="Tag or version " + @@ -733,95 +742,95 @@ private static class RunCommandArgs { String tagVer = null; @Parameter(names={"-v","--verbose"}, description="Print more details including error " + - "stack traces") + "stack traces") boolean verbose = false; @Parameter(names={"-k","--keep-tmp"}, description="Keep temporary files/folders at the " + - "end (default value is false)") + "end (default value is false)") boolean keepTempFiles = false; - + @Parameter(names={"-h","--sdk-home"}, description="Home folder of kb-sdk where sdk.cfg " + - "file and run_local folder are expected to be found or created if absent " + + "file and run_local folder are expected to be found or created if absent " + "(default path is loaded from 'KB_SDK_HOME' system environment variable)") String sdkHome = null; @Parameter(names={"-r","--prov-refs"}, description="Optional comma-separated list of " + - "references workspace objects that will be refered from provenance") + "references workspace objects that will be refered from provenance") String provRefs = null; @Parameter(names={"-m","--mount-points"}, description="Optional comma-separated list of " + - "mount point pairs for docker container (this parameter contains of local folder" + - " and inner folder separated by ':', local folder points to place in local file " + - "system, inner folder appears inside docker container, if inner path is not " + - "absolute it's treated as relative to /kb/module/work folder; if some mount " + - "point path doesn't have ':' and inner part then it appears as " + - "/kb/module/work/tmp inside docker)") + "mount point pairs for docker container (this parameter contains of local folder" + + " and inner folder separated by ':', local folder points to place in local file " + + "system, inner folder appears inside docker container, if inner path is not " + + "absolute it's treated as relative to /kb/module/work folder; if some mount " + + "point path doesn't have ':' and inner part then it appears as " + + "/kb/module/work/tmp inside docker)") String mountPoints = null; @Parameter(required=true, description="") + "(with SDK module prefix followed by '.')>") List methodName; } private static void showBriefHelp(JCommander jc, PrintStream out) { - Map commands = jc.getCommands(); - out.println(""); - out.println(MODULE_BUILDER_SH_NAME + " - a developer tool for building and validating KBase modules"); - out.println(""); - out.println("usage: "+MODULE_BUILDER_SH_NAME+" [options]"); - out.println(""); - out.println(" The available commands are:"); - for (Map.Entry command : commands.entrySet()) { - out.println(" " + command.getKey() +" - "+jc.getCommandDescription(command.getKey())); - } - out.println(""); - out.println(" For help on a specific command, see \"kb-sdk help \"."); - out.println(" For full usage information, see \"kb-sdk help -a\"."); - out.println(""); + Map commands = jc.getCommands(); + out.println(""); + out.println(MODULE_BUILDER_SH_NAME + " - a developer tool for building and validating KBase modules"); + out.println(""); + out.println("usage: "+MODULE_BUILDER_SH_NAME+" [options]"); + out.println(""); + out.println(" The available commands are:"); + for (Map.Entry command : commands.entrySet()) { + out.println(" " + command.getKey() +" - "+jc.getCommandDescription(command.getKey())); + } + out.println(""); + out.println(" For help on a specific command, see \"kb-sdk help \"."); + out.println(" For full usage information, see \"kb-sdk help -a\"."); + out.println(""); }; - + private static void showCommandUsage(JCommander jc, HelpCommandArgs helpArgs, PrintStream out) { - if(helpArgs.showAll) { - showFullUsage(jc, out); - } else if(helpArgs.command !=null && helpArgs.command.size()>0) { - String indent = ""; - StringBuilder usage = new StringBuilder(); - for(String cmd:helpArgs.command) { - if(jc.getCommands().containsKey(cmd)) { - usage.append(MODULE_BUILDER_SH_NAME + " "+cmd+"\n"); - jc.usage(cmd,usage,indent+" "); - usage.append("\n"); - } else { - out.println("Command \""+cmd+"\" is not a valid command. To view available commands, run:"); - out.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); - return; - } - } - out.print(usage.toString()); - } else { - showBriefHelp(jc, out); - } + if(helpArgs.showAll) { + showFullUsage(jc, out); + } else if(helpArgs.command !=null && helpArgs.command.size()>0) { + String indent = ""; + StringBuilder usage = new StringBuilder(); + for(String cmd:helpArgs.command) { + if(jc.getCommands().containsKey(cmd)) { + usage.append(MODULE_BUILDER_SH_NAME + " "+cmd+"\n"); + jc.usage(cmd,usage,indent+" "); + usage.append("\n"); + } else { + out.println("Command \""+cmd+"\" is not a valid command. To view available commands, run:"); + out.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); + return; + } + } + out.print(usage.toString()); + } else { + showBriefHelp(jc, out); + } }; - + private static void showFullUsage(JCommander jc, PrintStream out) { - String indent = ""; - StringBuilder usage = new StringBuilder(); - jc.usage(usage,indent); - out.print(usage.toString()); + String indent = ""; + StringBuilder usage = new StringBuilder(); + jc.usage(usage,indent); + out.print(usage.toString()); }; - - + + private static void showError(String error, String message, String extraHelp) { - System.err.println(error + ": " + message+"\n"); - if(!extraHelp.isEmpty()) - System.err.println(extraHelp+"\n"); - System.err.println("For more help and usage information, run:"); - System.err.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); - System.err.println(""); + System.err.println(error + ": " + message+"\n"); + if(!extraHelp.isEmpty()) + System.err.println(extraHelp+"\n"); + System.err.println("For more help and usage information, run:"); + System.err.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); + System.err.println(""); } - + private static void showError(String error, String message) { - showError(error,message,""); + showError(error,message,""); } } diff --git a/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java b/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java index afef7a0e..535031ed 100644 --- a/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java +++ b/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java @@ -34,12 +34,12 @@ public class CompilationReporter { public static void prepareCompileReport(File codeDir, List services, - boolean perlServerSide, String perlImplName, boolean pyServerSide, + boolean perlServerSide, String perlImplName, boolean pyServerSide, String pyImplName, boolean rServerSide, String rImplName, - boolean javaServerSide, String javaPackageParent, String javaSrcPath, + boolean javaServerSide, String javaPackageParent, String javaSrcPath, JavaData javaParsingData, List specFiles, File reportFile) throws Exception { String sdkVersion = ModuleBuilder.VERSION; - String sdkGitCommit = ModuleBuilder.getGitCommit(); + String sdkGitCommit = ModuleBuilder.getGitProp("commit"); String moduleName = null; KbModule module = null; for (KbService srv : services) @@ -74,11 +74,11 @@ public static void prepareCompileReport(File codeDir, List services, FileSaver javaSrcDir = new DiskFileSaver(javaSrc); for (JavaModule jmodule : javaParsingData.getModules()) { if (jmodule.getOriginal().getModuleName().equals(moduleName)) { - String moduleDir = JavaTypeGenerator.sub(javaPackageParent, + String moduleDir = JavaTypeGenerator.sub(javaPackageParent, jmodule.getModulePackage()).replace('.', '/'); - String serverClassName = TextUtils.capitalize(jmodule.getModuleName()) + + String serverClassName = TextUtils.capitalize(jmodule.getModuleName()) + "Server"; - implFile = javaSrcDir.getAsFileOrNull(moduleDir + "/" + + implFile = javaSrcDir.getAsFileOrNull(moduleDir + "/" + serverClassName + ".java"); } } @@ -102,7 +102,7 @@ public static void prepareCompileReport(File codeDir, List services, public static Report createReport(List specFiles, String sdkVersion, String sdkGitCommit, String moduleName, - KbModule module, String implFilePath, String implCommentPrefix, + KbModule module, String implFilePath, String implCommentPrefix, String implText) throws Exception, IOException { Map funcPositions = new LinkedHashMap(); String commentPrefix = Pattern.quote(implCommentPrefix); @@ -111,8 +111,8 @@ public static Report createReport(List specFiles, if (comp instanceof KbFuncdef) { KbFuncdef func = (KbFuncdef)comp; String funcName = func.getName(); - Pattern p = Pattern.compile(MessageFormat.format(".*" + commentPrefix + - "BEGIN {0}\n(.*\n)?[ \t]*" + commentPrefix + "END {0}\n.*", + Pattern p = Pattern.compile(MessageFormat.format(".*" + commentPrefix + + "BEGIN {0}\n(.*\n)?[ \t]*" + commentPrefix + "END {0}\n.*", funcName), Pattern.DOTALL); FunctionPlace place = checkMatch(funcPositions, p, implText); if (place != null) diff --git a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java index 43141980..f8bb566a 100644 --- a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java +++ b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java @@ -17,111 +17,111 @@ import us.kbase.templates.TemplateFormatter; public class ModuleInitializer { - public static final String DEFAULT_LANGUAGE = "python"; + public static final String DEFAULT_LANGUAGE = "python"; - private String moduleName; - private String userName; - private String language; - private boolean verbose; - private File dir; - - private static String[] subdirs = {"data", - "scripts", - "lib", - "test", - "ui", - "ui/narrative", - "ui/narrative/methods"}; - - public ModuleInitializer(String moduleName, String userName, boolean verbose) { - this(moduleName, userName, DEFAULT_LANGUAGE, verbose); - } - - public ModuleInitializer(String moduleName, String userName, String language, boolean verbose) { - this(moduleName, userName, language, verbose, null); - } - - public ModuleInitializer(String moduleName, String userName, String language, + private String moduleName; + private String userName; + private String language; + private boolean verbose; + private File dir; + + private static String[] subdirs = {"data", + "scripts", + "lib", + "test", + "ui", + "ui/narrative", + "ui/narrative/methods"}; + + public ModuleInitializer(String moduleName, String userName, boolean verbose) { + this(moduleName, userName, DEFAULT_LANGUAGE, verbose); + } + + public ModuleInitializer(String moduleName, String userName, String language, boolean verbose) { + this(moduleName, userName, language, verbose, null); + } + + public ModuleInitializer(String moduleName, String userName, String language, boolean verbose, File dir) { - this.moduleName = moduleName; - this.userName = userName; - this.language = language == null ? DEFAULT_LANGUAGE : language; - this.verbose = verbose; - this.dir = dir; - } - - /** - * - * @param example - * @throws IOException - */ - public void initialize(boolean example) throws Exception { - if (this.moduleName == null) { - throw new RuntimeException("Unable to create a null directory!"); - } - String moduleDir = dir == null ? this.moduleName : + this.moduleName = moduleName; + this.userName = userName; + this.language = language == null ? DEFAULT_LANGUAGE : language; + this.verbose = verbose; + this.dir = dir; + } + + /** + * + * @param example + * @throws IOException + */ + public void initialize(boolean example) throws Exception { + if (this.moduleName == null) { + throw new RuntimeException("Unable to create a null directory!"); + } + String moduleDir = dir == null ? this.moduleName : new File(dir, this.moduleName).getPath(); - this.language = qualifyLanguage(this.language); - - if (this.verbose) { - String msg = "Initializing module \"" + this.moduleName + "\""; - if (example) - msg += " with example methods"; - System.out.println(msg); - } + this.language = qualifyLanguage(this.language); + + if (this.verbose) { + String msg = "Initializing module \"" + this.moduleName + "\""; + if (example) + msg += " with example methods"; + System.out.println(msg); + } - List subdirList = new ArrayList(Arrays.asList(subdirs)); - if (example && this.language.equals("r")) { - subdirList.add("ui/narrative/methods/count_contigs_in_set"); - subdirList.add("ui/narrative/methods/count_contigs_in_set/img"); - } - else { - subdirList.add("ui/narrative/methods/run_" + this.moduleName); - subdirList.add("ui/narrative/methods/run_" + this.moduleName + "/img"); - } - - // 1. build dir with moduleName - initDirectory(Paths.get(moduleDir), true); - - // 2. build skeleton subdirs - for (String dir : subdirList) { - initDirectory(Paths.get(moduleDir, dir), true); - } + List subdirList = new ArrayList(Arrays.asList(subdirs)); + if (example && this.language.equals("r")) { + subdirList.add("ui/narrative/methods/count_contigs_in_set"); + subdirList.add("ui/narrative/methods/count_contigs_in_set/img"); + } + else { + subdirList.add("ui/narrative/methods/run_" + this.moduleName); + subdirList.add("ui/narrative/methods/run_" + this.moduleName + "/img"); + } + + // 1. build dir with moduleName + initDirectory(Paths.get(moduleDir), true); + + // 2. build skeleton subdirs + for (String dir : subdirList) { + initDirectory(Paths.get(moduleDir, dir), true); + } + + /* + * 3. Fill in templated files and write them + * + * Set up the context - the set of variables used to flesh out the templates */ + String specFile = Paths.get(this.moduleName + ".spec").toString(); - /* - * 3. Fill in templated files and write them - * - * Set up the context - the set of variables used to flesh out the templates */ - String specFile = Paths.get(this.moduleName + ".spec").toString(); - - Map moduleContext = new HashMap(); - moduleContext.put("module_name", this.moduleName); - moduleContext.put("user_name", this.userName); - moduleContext.put("spec_file", specFile); - moduleContext.put("language", this.language); - moduleContext.put("module_root_path", Paths.get(moduleDir).toAbsolutePath()); - moduleContext.put("example", example); - moduleContext.put("dollar_sign", "$"); + Map moduleContext = new HashMap(); + moduleContext.put("module_name", this.moduleName); + moduleContext.put("user_name", this.userName); + moduleContext.put("spec_file", specFile); + moduleContext.put("language", this.language); + moduleContext.put("module_root_path", Paths.get(moduleDir).toAbsolutePath()); + moduleContext.put("example", example); + moduleContext.put("dollar_sign", "$"); moduleContext.put("os_name", System.getProperty("os.name")); - Map templateFiles = new HashMap(); - templateFiles.put("module_typespec", Paths.get(moduleDir, specFile)); - templateFiles.put("module_travis", Paths.get(moduleDir, ".travis.yml")); - templateFiles.put("module_dockerfile", Paths.get(moduleDir, "Dockerfile")); - templateFiles.put("module_readme", Paths.get(moduleDir, "README.md")); - templateFiles.put("module_release_notes", Paths.get(moduleDir, "RELEASE_NOTES.md")); - templateFiles.put("module_makefile", Paths.get(moduleDir, "Makefile")); - templateFiles.put("module_deploy_cfg", Paths.get(moduleDir, "deploy.cfg")); - templateFiles.put("module_license", Paths.get(moduleDir, "LICENSE")); - templateFiles.put("module_docker_entrypoint", Paths.get(moduleDir, "scripts", "entrypoint.sh")); + Map templateFiles = new HashMap(); + templateFiles.put("module_typespec", Paths.get(moduleDir, specFile)); + templateFiles.put("module_travis", Paths.get(moduleDir, ".travis.yml")); + templateFiles.put("module_dockerfile", Paths.get(moduleDir, "Dockerfile")); + templateFiles.put("module_readme", Paths.get(moduleDir, "README.md")); + templateFiles.put("module_release_notes", Paths.get(moduleDir, "RELEASE_NOTES.md")); + templateFiles.put("module_makefile", Paths.get(moduleDir, "Makefile")); + templateFiles.put("module_deploy_cfg", Paths.get(moduleDir, "deploy.cfg")); + templateFiles.put("module_license", Paths.get(moduleDir, "LICENSE")); + templateFiles.put("module_docker_entrypoint", Paths.get(moduleDir, "scripts", "entrypoint.sh")); templateFiles.put("module_prepare_deploy_cfg", Paths.get(moduleDir, "scripts", "prepare_deploy_cfg.py")); templateFiles.put("module_run_async", Paths.get(moduleDir, "scripts", "run_async.sh")); - templateFiles.put("module_readme_lib", Paths.get(moduleDir, "lib", "README.md")); - templateFiles.put("module_readme_ui", Paths.get(moduleDir, "ui", "README.md")); - templateFiles.put("module_readme_test", Paths.get(moduleDir, "test", "README.md")); - templateFiles.put("module_readme_data", Paths.get(moduleDir, "data", "README.md")); - templateFiles.put("module_config_yaml", Paths.get(moduleDir, "kbase.yml")); + templateFiles.put("module_readme_lib", Paths.get(moduleDir, "lib", "README.md")); + templateFiles.put("module_readme_ui", Paths.get(moduleDir, "ui", "README.md")); + templateFiles.put("module_readme_test", Paths.get(moduleDir, "test", "README.md")); + templateFiles.put("module_readme_data", Paths.get(moduleDir, "data", "README.md")); + templateFiles.put("module_config_yaml", Paths.get(moduleDir, "kbase.yml")); templateFiles.put("module_gitignore", Paths.get(moduleDir, ".gitignore")); templateFiles.put("module_dockerignore", Paths.get(moduleDir, ".dockerignore")); templateFiles.put("module_readme_test_local", Paths.get(moduleDir, "test_local", "readme.txt")); @@ -129,13 +129,13 @@ public void initialize(boolean example) throws Exception { templateFiles.put("module_run_tests", Paths.get(moduleDir, "test_local", "run_tests.sh")); templateFiles.put("module_run_bash", Paths.get(moduleDir, "test_local", "run_bash.sh")); templateFiles.put("module_run_docker", Paths.get(moduleDir, "test_local", "run_docker.sh")); - - switch (language) { - case "java": + + switch (language) { + case "java": templateFiles.put("module_build_xml", Paths.get(moduleDir, "build.xml")); - templateFiles.put("module_web_xml", Paths.get(moduleDir, "scripts", "web.xml")); + templateFiles.put("module_web_xml", Paths.get(moduleDir, "scripts", "web.xml")); templateFiles.put("module_jetty_xml", Paths.get(moduleDir, "scripts", "jetty.xml")); - fillTemplate(moduleContext, "module_typespec", templateFiles.remove("module_typespec")); + fillTemplate(moduleContext, "module_typespec", templateFiles.remove("module_typespec")); String javaPackageParent = "."; // Special value meaning top level package. moduleContext.put("java_package_parent", javaPackageParent); JavaData data = JavaTypeGenerator.parseSpec(new File(moduleDir, specFile)); @@ -148,7 +148,7 @@ public void initialize(boolean example) throws Exception { File testJavaFile = new File(testSrcDir, modulePackage.replace('.', '/') + "/test/" + javaModuleName + "ServerTest.java"); fillTemplate(moduleContext, "module_test_java_client", testJavaFile.toPath()); break; - case "python": + case "python": templateFiles.put("module_test_python_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.py")); templateFiles.put("module_tox", Paths.get(moduleDir, "tox.ini")); break; @@ -158,30 +158,27 @@ public void initialize(boolean example) throws Exception { case "r": templateFiles.put("module_test_r_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.r")); break; - } - - if (example && this.language.equals("r")) { - templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "spec.json")); - templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "display.yaml")); - } else { + } + + if (example && this.language.equals("r")) { + templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "spec.json")); + templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "display.yaml")); + } else { templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "run_"+this.moduleName, "spec.json")); templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "run_"+this.moduleName, "display.yaml")); } switch(this.language) { - // Perl just needs an impl file and a start server script + // start_server scripts are generated by the Makefile case "perl": - //templateFiles.put("module_start_perl_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile + // Perl needs an impl file templateFiles.put("module_perl_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.pm")); break; - // Python needs some empty __init__.py files and the impl file (Done, see TemplateBasedGenerator.initPyhtonPackages) case "python": + // python requires impl and __init__.py files initDirectory(Paths.get(moduleDir, "lib", this.moduleName), false); initFile(Paths.get(moduleDir, "lib", this.moduleName, "__init__.py"), false); templateFiles.put("module_python_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.py")); - //templateFiles.put("module_start_python_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile break; case "r": templateFiles.put("module_r_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.r")); @@ -194,131 +191,129 @@ public void initialize(boolean example) throws Exception { File serverJavaFile = new File(srcDir, modulePackage.replace('.', '/') + "/" + javaModuleName + "Server.java"); fillTemplate(moduleContext, "module_java_impl", serverJavaFile.toPath()); JavaTypeGenerator.processSpec(new File(moduleDir, specFile), srcDir, javaPackageParent, true, null, null, null); - //templateFiles.put("module_start_java_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile break; default: break; } - for (String templateName : templateFiles.keySet()) { - fillTemplate(moduleContext, templateName, templateFiles.get(templateName)); - } - - if (example) { - // Generated examples require some other SDK dependencies + for (String templateName : templateFiles.keySet()) { + fillTemplate(moduleContext, templateName, templateFiles.get(templateName)); + } + + if (example) { + // Generated examples require some other SDK dependencies new ClientInstaller(new File(moduleDir), false).install( this.language, - true, // async clients - false, // core or sync clients - false, // dynamic client - null, //tagVer + true, // async clients + false, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "AssemblyUtil", null, - null // clientName + null // clientName ); - } - // Let's install fresh workspace client in any cases (we need it at least in tests): - new ClientInstaller(new File(moduleDir), false).install( + } + // Let's install fresh workspace client in any cases (we need it at least in tests): + new ClientInstaller(new File(moduleDir), false).install( this.language, - false, // async clients - true, // core or sync clients - false, // dynamic client - null, //tagVer + false, // async clients + true, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "https://raw.githubusercontent.com/kbase/workspace_deluxe/master/workspace.spec", null, - null // clientName + null // clientName ); new ClientInstaller(new File(moduleDir), false).install( this.language, - true, // async clients - false, // core or sync clients - false, // dynamic client - null, //tagVer + true, // async clients + false, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "KBaseReport", null, - null // clientName + null // clientName ); - System.out.println("Done! Your module is available in the " + moduleDir + " directory."); - System.out.println("Compile and run the example methods with the following inputs:"); - System.out.println(" cd " + moduleDir); - System.out.println(" make (required after making changes to " + new File(specFile).getName() + ")"); - System.out.println(" kb-sdk test (will require setting test user account credentials in test_local/test.cfg)"); - System.out.println(); - } - - /** - * - * @param dirPath - * @throws IOException - */ - private void initDirectory(Path dirPath, boolean failOnExist) throws IOException { - if (this.verbose) System.out.println("Making directory \"" + dirPath.toString() + "\""); - File newDir = dirPath.toFile(); - if (!newDir.exists()) { - newDir.mkdirs(); - } - else if (failOnExist) { - throw new IOException("Error while creating module - " + dirPath + " already exists!"); - } - } - - private void initFile(Path filePath, boolean failOnExist) throws IOException { - if (this.verbose) System.out.println("Building empty file \"" + filePath.toString() + "\""); - boolean done = filePath.toFile().createNewFile(); - if (!done && failOnExist) - throw new IOException("Unable to create file \"" + filePath.toString() + "\" - file already exists!"); - } - /** - * - * @param context - * @param templateName - * @param outfile - * @throws IOException - */ - private void fillTemplate(Map context, String templateName, Path outfilePath) throws IOException { - if (this.verbose) System.out.println("Building file \"" + outfilePath.toString() + "\""); - initDirectory(outfilePath.getParent(), false); - TemplateFormatter.formatTemplate(templateName, context, outfilePath.toFile()); - } - - /** - * Takes a language string and returns a "qualified" form. E.g. "perl", "Perl", "pl", ".pl", should all - * return "perl", "Python", "python", ".py", and "py" should all return Python, etc. - * - * Right now, we support Perl, Python and Java for implementation languages - * @param language - * @return - */ - public static String qualifyLanguage(String language) { - String lang = language.toLowerCase(); - - String[] perlNames = {"perl", ".pl", "pl"}; - if (Arrays.asList(perlNames).contains(lang)) - return "perl"; - - String[] pythonNames = {"python", ".py", "py"}; - if (Arrays.asList(pythonNames).contains(lang)) - return "python"; - - String[] javaNames = {"java", ".java"}; - if (Arrays.asList(javaNames).contains(lang)) - return "java"; - - String[] rNames = {"r", ".r"}; - if (Arrays.asList(rNames).contains(lang)) { - System.out.println( - "************************************************************************\n" + - "WARNING: R support is deprecated and will be removed in a future release\n" + - "************************************************************************"); - return "r"; - } - - // If we get here, then we don't recognize it! throw a runtime exception - throw new RuntimeException("Unrecognized language: " + language + "\n\tWe currently only support Python, Perl, and Java."); - } -} \ No newline at end of file + System.out.println("Done! Your module is available in the " + moduleDir + " directory."); + System.out.println("Compile and run the example methods with the following inputs:"); + System.out.println(" cd " + moduleDir); + System.out.println(" make (required after making changes to " + new File(specFile).getName() + ")"); + System.out.println(" kb-sdk test (will require setting test user account credentials in test_local/test.cfg)"); + System.out.println(); + } + + /** + * + * @param dirPath + * @throws IOException + */ + private void initDirectory(Path dirPath, boolean failOnExist) throws IOException { + if (this.verbose) System.out.println("Making directory \"" + dirPath.toString() + "\""); + File newDir = dirPath.toFile(); + if (!newDir.exists()) { + newDir.mkdirs(); + } + else if (failOnExist) { + throw new IOException("Error while creating module - " + dirPath + " already exists!"); + } + } + + private void initFile(Path filePath, boolean failOnExist) throws IOException { + if (this.verbose) System.out.println("Building empty file \"" + filePath.toString() + "\""); + boolean done = filePath.toFile().createNewFile(); + if (!done && failOnExist) + throw new IOException("Unable to create file \"" + filePath.toString() + "\" - file already exists!"); + } + /** + * + * @param context + * @param templateName + * @param outfile + * @throws IOException + */ + private void fillTemplate(Map context, String templateName, Path outfilePath) throws IOException { + if (this.verbose) System.out.println("Building file \"" + outfilePath.toString() + "\""); + initDirectory(outfilePath.getParent(), false); + TemplateFormatter.formatTemplate(templateName, context, outfilePath.toFile()); + } + + /** + * Takes a language string and returns a "qualified" form. E.g. "perl", "Perl", "pl", ".pl", should all + * return "perl", "Python", "python", ".py", and "py" should all return Python, etc. + * + * Right now, we support Perl, Python and Java for implementation languages + * @param language + * @return + */ + public static String qualifyLanguage(String language) { + String lang = language.toLowerCase(); + + String[] perlNames = {"perl", ".pl", "pl"}; + if (Arrays.asList(perlNames).contains(lang)) + return "perl"; + + String[] pythonNames = {"python", ".py", "py"}; + if (Arrays.asList(pythonNames).contains(lang)) + return "python"; + + String[] javaNames = {"java", ".java"}; + if (Arrays.asList(javaNames).contains(lang)) + return "java"; + + String[] rNames = {"r", ".r"}; + if (Arrays.asList(rNames).contains(lang)) { + System.out.println( + "************************************************************************\n" + + "WARNING: R support is deprecated and will be removed in a future release\n" + + "************************************************************************"); + return "r"; + } + + // If we get here, then we don't recognize it! throw a runtime exception + throw new RuntimeException("Unrecognized language: " + language + "\n\tWe currently only support Python, Perl, and Java."); + } +} diff --git a/src/java/us/kbase/mobu/installer/ClientInstaller.java b/src/java/us/kbase/mobu/installer/ClientInstaller.java index bdb30951..6533e223 100644 --- a/src/java/us/kbase/mobu/installer/ClientInstaller.java +++ b/src/java/us/kbase/mobu/installer/ClientInstaller.java @@ -47,7 +47,7 @@ public class ClientInstaller { private final String language; private final Properties sdkConfig; private final URL catalogUrl; - + public ClientInstaller() throws Exception { this(null, true); } @@ -84,13 +84,13 @@ public ClientInstaller(File dir, boolean showWarnings) throws Exception { catalogUrl = new URL(catalogUrlText); } - public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, + public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, boolean verbose, String moduleName, String libDirName) throws Exception { return install(lang, async, core, dynamic, tagVer, verbose, moduleName, libDirName, null); } - - public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, - boolean verbose, String moduleName, String libDirName, String clientName) + + public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, + boolean verbose, String moduleName, String libDirName, String clientName) throws Exception { if (core && (dynamic || async)) { throw new IllegalStateException("It's not allowed to set 'core' mode and one " + @@ -142,7 +142,7 @@ public String loadIncludedSpec(String specFileName) throws Exception { }; } else { throw new IllegalStateException("Path " + moduleName + " is not recognized as " + - "existing local file or URL"); + "existing local file or URL"); } moduleName = null; } else { @@ -159,11 +159,11 @@ public String loadIncludedSpec(String specFileName) throws Exception { } if (modVer.getCompilationReport() == null) throw new IllegalStateException("Compilation report is not found for this " + - "version of [" + moduleName + "] module."); + "version of [" + moduleName + "] module."); List specFiles = modVer.getCompilationReport().getSpecFiles(); if (specFiles == null) throw new IllegalStateException("Compilation report returned from catalog is " + - "out of date. [" + moduleName + "] module should be reregistered again."); + "out of date. [" + moduleName + "] module should be reregistered again."); final List mainSpec = new ArrayList(); final Map deps = new LinkedHashMap(); for (SpecFile spec : specFiles) { @@ -183,22 +183,22 @@ public String loadIncludedSpec(String specFileName) throws Exception { }; } if (async && dynamic) { - compile(lang, async, false, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, async, false, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName, filePath); if (clientName == null) { clientName = moduleName; } - compile(lang, false, dynamic, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, false, dynamic, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName + "Service", filePath); } else { - compile(lang, async, dynamic, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, async, dynamic, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName, filePath); } return 0; } - private void compile(String lang, boolean async, boolean dynamic, String tagVer, - boolean verbose, String moduleName, String libDirName, FileProvider fp, + private void compile(String lang, boolean async, boolean dynamic, String tagVer, + boolean verbose, String moduleName, String libDirName, FileProvider fp, String semanticVersion, String gitUrl, String gitCommitHash, String clientName, String filePath) throws Exception { String url = null; @@ -216,7 +216,7 @@ private void compile(String lang, boolean async, boolean dynamic, String tagVer, final FileProvider fp2 = fp; IncludeProvider ip = new IncludeProvider() { @Override - public Map parseInclude(String includeLine) + public Map parseInclude(String includeLine) throws KidlParseException { String specPath = includeLine.trim(); if (specPath.startsWith("#include")) @@ -285,8 +285,8 @@ public Map parseInclude(String includeLine) String javaPackageParent = "."; String customClientClassName = TextUtils.capitalize(clientName) + "Client"; URL urlEndpoint = url == null ? null : new URL(url); - JavaTypeGenerator.processSpec(services, javaSrcDir, javaPackageParent, false, null, - null, urlEndpoint, null, null, clientAsyncVer, dynservVer, semanticVersion, + JavaTypeGenerator.processSpec(services, javaSrcDir, javaPackageParent, false, null, + null, urlEndpoint, null, null, clientAsyncVer, dynservVer, semanticVersion, gitUrl, gitCommitHash, null, customClientClassName); } else { String perlClientName = null; @@ -302,9 +302,9 @@ public Map parseInclude(String includeLine) if (isJS) jsClientName = "installed_clients/" + clientName + "Client"; FileSaver output = new DiskFileSaver(libDir); - TemplateBasedGenerator.generate(services, url, isJS, jsClientName, isPerl, + TemplateBasedGenerator.generate(services, url, isJS, jsClientName, isPerl, perlClientName, false, null, null, null, isPython, pyClientName, false, null, - null, isR, rClientName, false, null, null, false, ip, output, null, null, + null, isR, rClientName, false, null, null, false, ip, output, null, null, async, clientAsyncVer, dynservVer, semanticVersion, gitUrl, gitCommitHash); } // Now let's add record about this client to dependencies.json file @@ -313,8 +313,8 @@ public Map parseInclude(String includeLine) FileSaver depsDir = new DiskFileSaver(moduleDir); addDependency(moduleName, isSdk, versionTag, filePath, depsDir); } - - public static void addDependency(String moduleName, boolean isSdk, String versionTag, + + public static void addDependency(String moduleName, boolean isSdk, String versionTag, String filePath, FileSaver depsDir) throws Exception { Map depMap = new TreeMap(); ObjectMapper mapper = new ObjectMapper(); @@ -322,13 +322,13 @@ public static void addDependency(String moduleName, boolean isSdk, String versio File depsFile = depsDir.getAsFileOrNull("dependencies.json"); if (depsFile != null && depsFile.exists()) { try { - List deps = mapper.readValue(depsFile, + List deps = mapper.readValue(depsFile, new TypeReference>() {}); for (Dependency dep : deps) { depMap.put(dep.moduleName.toLowerCase(), dep); } } catch (Exception ex) { - throw new IllegalStateException("Error parsing depedencies file [" + depsFile + + throw new IllegalStateException("Error parsing dependencies file [" + depsFile + "]: " + ex.getMessage(), ex); } } @@ -343,7 +343,7 @@ public static void addDependency(String moduleName, boolean isSdk, String versio mapper.writeValue(depsWr, deps); } } - + private static boolean isUrl(String url) { try { new URL(url); @@ -352,7 +352,7 @@ private static boolean isUrl(String url) { return false; } } - + private static boolean isLocalFile(String path) { try { File f = new File(path); @@ -362,7 +362,7 @@ private static boolean isLocalFile(String path) { return false; } } - + private static interface FileProvider { public String loadMainSpec() throws Exception; public String loadIncludedSpec(String specFileName) throws Exception; diff --git a/src/java/us/kbase/mobu/tester/ConfigLoader.java b/src/java/us/kbase/mobu/tester/ConfigLoader.java index 05933993..5832da96 100644 --- a/src/java/us/kbase/mobu/tester/ConfigLoader.java +++ b/src/java/us/kbase/mobu/tester/ConfigLoader.java @@ -38,6 +38,11 @@ public ConfigLoader(Properties props, boolean testMode, if (authUrl == null) throw new IllegalStateException("Error: 'auth_service_url' parameter is not set in " + configPathInfo); + endPoint = props.getProperty("kbase_endpoint"); + if (endPoint == null) + throw new IllegalStateException("Error: KBase services end-point is not set in " + + configPathInfo); + String testPrefix = testMode ? "test_" : ""; String user = props.getProperty(testPrefix + "user"); if (user != null && user.trim().isEmpty()) { @@ -47,9 +52,13 @@ public ConfigLoader(Properties props, boolean testMode, String tokenString = props.getProperty(testPrefix + "token"); if (tokenString != null && tokenString.trim().isEmpty()) { tokenString = null; + } + if (tokenString == null) { + System.out.println("No token found in test.cfg file; checking environment"); String test_token = System.getenv("KBASE_TEST_TOKEN"); if (test_token != null && !test_token.trim().isEmpty()) { tokenString = test_token; + System.out.println("Using token from KBASE_TEST_TOKEN env var"); } } @@ -71,10 +80,6 @@ public ConfigLoader(Properties props, boolean testMode, } token = auth.login(user.trim(), password.trim()).getToken(); } - endPoint = props.getProperty("kbase_endpoint"); - if (endPoint == null) - throw new IllegalStateException("Error: KBase services end-point is not set in " + - configPathInfo); jobSrvUrl = getConfigUrl(props, "job_service_url", endPoint, "userandjobstate"); wsUrl = getConfigUrl(props, "workspace_url", endPoint, "ws"); shockUrl = getConfigUrl(props, "shock_url", endPoint, "shock-api"); diff --git a/src/java/us/kbase/mobu/tester/ModuleTester.java b/src/java/us/kbase/mobu/tester/ModuleTester.java index 7600b8d0..67662776 100644 --- a/src/java/us/kbase/mobu/tester/ModuleTester.java +++ b/src/java/us/kbase/mobu/tester/ModuleTester.java @@ -53,7 +53,7 @@ public class ModuleTester { public ModuleTester() throws Exception { this(null); } - + public ModuleTester(File dir) throws Exception { moduleDir = dir == null ? DirUtils.findModuleDir() : DirUtils.findModuleDir(dir); String kbaseYml = TextUtils.readFileText(new File(moduleDir, "kbase.yml")); @@ -70,7 +70,7 @@ public ModuleTester(File dir) throws Exception { ModuleInitializer.qualifyLanguage((String) kbaseYmlConfig.get("service-language")); moduleContext.put("os_name", System.getProperty("os.name")); } - + private static void checkIgnoreLine(File f, String line) throws IOException { List lines = new ArrayList(); if (f.exists()) @@ -82,18 +82,18 @@ private static void checkIgnoreLine(File f, String line) throws IOException { FileUtils.writeLines(f, lines); } } - + public int runTests(String methodStoreUrl, boolean skipValidation, boolean allowSyncMethods) throws Exception { if (skipValidation) { System.out.println("Validation step is skipped"); } else { - ModuleValidator mv = new ModuleValidator(Arrays.asList(moduleDir.getCanonicalPath()), + ModuleValidator mv = new ModuleValidator(Arrays.asList(moduleDir.getCanonicalPath()), false, methodStoreUrl, allowSyncMethods); int returnCode = mv.validateAll(); if (returnCode!=0) { System.out.println("You can skip validation step using -s (or --skip_validation)" + - " flag"); + " flag"); System.exit(returnCode); } } @@ -114,7 +114,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow if (kbaseYmlConfig.get("data-version") != null) { File refDataDir = new File(tlDir, "refdata"); if (!refDataDir.exists()) { - TemplateFormatter.formatTemplate("module_run_tests", moduleContext, + TemplateFormatter.formatTemplate("module_run_tests", moduleContext, runTestsSh); refDataDir.mkdir(); } @@ -128,7 +128,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow if (!testCfg.exists()) { TemplateFormatter.formatTemplate("module_test_cfg", moduleContext, testCfg); System.out.println("Set KBase account credentials in test_local/test.cfg and then " + - "test again"); + "test again"); return 1; } Properties props = new Properties(); @@ -138,10 +138,10 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow } finally { is.close(); } - + ConfigLoader cfgLoader = new ConfigLoader(props, true, "test_local/test.cfg", true); - - + + File workDir = new File(tlDir, "workdir"); workDir.mkdir(); File tokenFile = new File(workDir, "token"); @@ -175,7 +175,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow String callbackNetworksText = props.getProperty("callback_networks"); if (callbackNetworksText != null) { callbackNetworks = callbackNetworksText.trim().split("\\s*,\\s*"); - System.out.println("Custom network instarface list is defined: " + + System.out.println("Custom network instarface list is defined: " + Arrays.asList(callbackNetworks)); } URL callbackUrl = CallbackServer.getCallbackUrl(callbackPort, callbackNetworks); @@ -185,7 +185,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow JsonServerSyslog.setStaticUseSyslog(false); JsonServerSyslog.setStaticMlogFile("callback.log"); } - CallbackServerConfig cfg = cfgLoader.buildCallbackServerConfig(callbackUrl, + CallbackServerConfig cfg = cfgLoader.buildCallbackServerConfig(callbackUrl, tlDir.toPath(), new LineLogger() { @Override public void logNextLine(String line, boolean isError) { @@ -217,7 +217,7 @@ public void logNextLine(String line, boolean isError) { } else { if (callbackNetworks != null && callbackNetworks.length > 0) { throw new IllegalStateException("No proper callback IP was found, " + - "please check callback_networks parameter in test.cfg"); + "please check callback_networks parameter in test.cfg"); } System.out.println("WARNING: No callback URL was received " + "by the job runner. Local callbacks are disabled."); @@ -226,9 +226,13 @@ public void logNextLine(String line, boolean isError) { try { System.out.println(); ProcessHelper.cmd("chmod", "+x", runTestsSh.getCanonicalPath()).exec(tlDir); - int exitCode = ProcessHelper.cmd("bash", DirUtils.getFilePath(runTestsSh), - callbackUrl == null ? "http://fakecallbackurl" : - callbackUrl.toExternalForm()).exec(tlDir).getExitCode(); + int exitCode = ProcessHelper.cmd( + "bash", + DirUtils.getFilePath(runTestsSh), + callbackUrl == null + ? "http://fakecallbackurl" + : callbackUrl.toExternalForm() + ).exec(tlDir).getExitCode(); return exitCode; } finally { if (jettyServer != null) { @@ -251,7 +255,7 @@ public static boolean buildNewDockerImageWithCleanup(File moduleDir, File tlDir, ProcessHelper.cmd("bash", runDockerPath, "rm", "-v", "-f", cntId).exec(tlDir); } } - String oldImageId = findImageIdByName(tlDir, imageName, runDockerSh); + String oldImageId = findImageIdByName(tlDir, imageName, runDockerSh); System.out.println(); System.out.println("Build Docker image"); boolean ok = buildImage(moduleDir, imageName, runDockerSh); @@ -267,7 +271,7 @@ public static boolean buildNewDockerImageWithCleanup(File moduleDir, File tlDir, } return true; } - + public static String findImageIdByName(File tlDir, String imageName, File runDockerSh) throws Exception { List lines; @@ -295,7 +299,7 @@ public static String[] splitByWhiteSpaces(String line) { String[] parts = line.split("\\s+"); return parts; } - + private static List exec(File workDir, String... cmd) throws Exception { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); @@ -312,13 +316,13 @@ private static List exec(File workDir, String... cmd) throws Exception { br.close(); return ret; } - - public static boolean buildImage(File repoDir, String targetImageName, + + public static boolean buildImage(File repoDir, String targetImageName, File runDockerSh) throws Exception { String scriptPath = DirUtils.getFilePath(runDockerSh); String repoPath = DirUtils.getFilePath(repoDir); - Process p = Runtime.getRuntime().exec(new String[] {"bash", - scriptPath, "build", "--rm", "-t", + Process p = Runtime.getRuntime().exec(new String[] {"bash", + scriptPath, "build", "--rm", "-t", targetImageName, repoPath}); List workers = new ArrayList(); InputStream[] inputStreams = new InputStream[] {p.getInputStream(), p.getErrorStream()}; @@ -358,7 +362,7 @@ public void run() { } catch (Exception e) { e.printStackTrace(); throw new IllegalStateException("Error reading data from executed " + - "container", e); + "container", e); } } }); @@ -374,7 +378,7 @@ public void run() { if (cntIdToDelete[0] != null) { System.out.println("Cleaning up building container: " + cntIdToDelete[0]); Thread.sleep(1000); - ProcessHelper.cmd("bash", scriptPath, + ProcessHelper.cmd("bash", scriptPath, "rm", "-v", "-f", cntIdToDelete[0]).exec(repoDir); } } catch (Exception ex) { diff --git a/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java b/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java index ae1f2488..3cbd4ee9 100644 --- a/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java +++ b/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java @@ -21,44 +21,44 @@ public class ModuleTesterTest { - private static final String SIMPLE_MODULE_NAME = "ASimpleModule_for_unit_testing"; - private static final boolean cleanupAfterTests = true; - - private static List createdModuleNames = new ArrayList(); - private static AuthToken token; - + private static final String SIMPLE_MODULE_NAME = "ASimpleModule_for_unit_testing"; + private static final boolean cleanupAfterTests = true; + + private static List createdModuleNames = new ArrayList(); + private static AuthToken token; + @BeforeClass public static void beforeClass() throws Exception { token = TestConfigHelper.getToken(); } - @AfterClass - public static void tearDownModule() throws Exception { - if (cleanupAfterTests) - for (String moduleName : createdModuleNames) - try { - deleteDir(moduleName); - } catch (Exception ex) { - System.err.println("Error cleaning up module [" + - moduleName + "]: " + ex.getMessage()); - } - } - - @After - public void afterText() { - System.out.println(); - } - - private static void deleteDir(String moduleName) throws Exception { - File module = new File(moduleName); - if (module.exists() && module.isDirectory()) - FileUtils.deleteDirectory(module); - } - + @AfterClass + public static void tearDownModule() throws Exception { + if (cleanupAfterTests) + for (String moduleName : createdModuleNames) + try { + deleteDir(moduleName); + } catch (Exception ex) { + System.err.println("Error cleaning up module [" + + moduleName + "]: " + ex.getMessage()); + } + } + + @After + public void afterText() { + System.out.println(); + } + + private static void deleteDir(String moduleName) throws Exception { + File module = new File(moduleName); + if (module.exists() && module.isDirectory()) + FileUtils.deleteDirectory(module); + } + private void init(String lang, String moduleName) throws Exception { deleteDir(moduleName); createdModuleNames.add(moduleName); - ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), + ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), lang, false); initer.initialize(true); } @@ -68,13 +68,13 @@ private int runTestsInDocker(String moduleName) throws Exception { return runTestsInDocker(moduleDir, token); } - public static int runTestsInDocker(File moduleDir, AuthToken token) throws Exception { - return runTestsInDocker(moduleDir, token, false); - } - + public static int runTestsInDocker(File moduleDir, AuthToken token) throws Exception { + return runTestsInDocker(moduleDir, token, false); + } + public static int runTestsInDocker( final File moduleDir, - final AuthToken token, + final AuthToken token, final boolean skipValidation) throws Exception { DockerClientServerTester.correctDockerfile(moduleDir); @@ -83,7 +83,7 @@ public static int runTestsInDocker( "test_token=" + token.getToken() + "\n" + "kbase_endpoint=" + TestConfigHelper.getKBaseEndpoint() + "\n" + "auth_service_url=" + TestConfigHelper.getAuthServiceUrl() + "\n" + - "auth_service_url_allow_insecure=" + + "auth_service_url_allow_insecure=" + TestConfigHelper.getAuthServiceUrlInsecure() + "\n"; FileUtils.writeStringToFile(testCfgFile, testCfgText); int exitCode = new ModuleTester(moduleDir).runTests(ModuleBuilder.DEFAULT_METHOD_STORE_URL, @@ -92,84 +92,84 @@ public static int runTestsInDocker( return exitCode; } - @Test - public void testPerlModuleExample() throws Exception { - System.out.println("Test [testPerlModuleExample]"); - String lang = "perl"; - String moduleName = SIMPLE_MODULE_NAME + "Perl"; - init(lang, moduleName); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(0, exitCode); - } - - @Test - public void testPerlModuleError() throws Exception { - System.out.println("Test [testPerlModuleError]"); - String lang = "perl"; - String moduleName = SIMPLE_MODULE_NAME + "PerlError"; - init(lang, moduleName); - File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.pm"); - String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN filter_contigs", - " #BEGIN filter_contigs\n" + - " die \"Special error\";"); - FileUtils.writeStringToFile(implFile, implText); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(2, exitCode); - } - - @Test - public void testPythonModuleExample() throws Exception { - System.out.println("Test [testPythonModuleExample]"); - String lang = "python"; - String moduleName = SIMPLE_MODULE_NAME + "Python"; - init(lang, moduleName); - int exitCode = runTestsInDocker(moduleName); + @Test + public void testPerlModuleExample() throws Exception { + System.out.println("Test [testPerlModuleExample]"); + String lang = "perl"; + String moduleName = SIMPLE_MODULE_NAME + "Perl"; + init(lang, moduleName); + int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(0, exitCode); - } + } - @Test - public void testPythonModuleError() throws Exception { + @Test + public void testPerlModuleError() throws Exception { + System.out.println("Test [testPerlModuleError]"); + String lang = "perl"; + String moduleName = SIMPLE_MODULE_NAME + "PerlError"; + init(lang, moduleName); + File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.pm"); + String implText = FileUtils.readFileToString(implFile); + implText = implText.replace(" #BEGIN filter_contigs", + " #BEGIN filter_contigs\n" + + " die \"Special error\";"); + FileUtils.writeStringToFile(implFile, implText); + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(2, exitCode); + } + + @Test + public void testPythonModuleExample() throws Exception { + System.out.println("Test [testPythonModuleExample]"); + String lang = "python"; + String moduleName = SIMPLE_MODULE_NAME + "Python"; + init(lang, moduleName); + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(0, exitCode); + } + + @Test + public void testPythonModuleError() throws Exception { System.out.println("Test [testPythonModuleError]"); - String lang = "python"; + String lang = "python"; String moduleName = SIMPLE_MODULE_NAME + "PythonError"; init(lang, moduleName); File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.py"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN filter_contigs", + implText = implText.replace(" #BEGIN filter_contigs", " #BEGIN filter_contigs\n" + " raise ValueError('Special error')"); FileUtils.writeStringToFile(implFile, implText); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(2, exitCode); - } + } - @Test - public void testJavaModuleExample() throws Exception { + @Test + public void testJavaModuleExample() throws Exception { System.out.println("Test [testJavaModuleExample]"); - String lang = "java"; + String lang = "java"; String moduleName = SIMPLE_MODULE_NAME + "Java"; init(lang, moduleName); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(0, exitCode); - } - - @Test - public void testJavaModuleError() throws Exception { - System.out.println("Test [testJavaModuleError]"); - String lang = "java"; - String moduleName = SIMPLE_MODULE_NAME + "JavaError"; - init(lang, moduleName); + } + + @Test + public void testJavaModuleError() throws Exception { + System.out.println("Test [testJavaModuleError]"); + String lang = "java"; + String moduleName = SIMPLE_MODULE_NAME + "JavaError"; + init(lang, moduleName); File implFile = new File(moduleName + "/lib/src/" + - "asimplemoduleforunittestingjavaerror/ASimpleModuleForUnitTestingJavaErrorServer.java"); + "asimplemoduleforunittestingjavaerror/ASimpleModuleForUnitTestingJavaErrorServer.java"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" //BEGIN filter_contigs", + implText = implText.replace(" //BEGIN filter_contigs", " //BEGIN filter_contigs\n" + " if (true) throw new IllegalStateException(\"Special error\");"); FileUtils.writeStringToFile(implFile, implText); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(2, exitCode); - } + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(2, exitCode); + } @Test public void testRModuleError() throws Exception { @@ -179,14 +179,14 @@ public void testRModuleError() throws Exception { init(lang, moduleName); File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.r"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN count_contigs", + implText = implText.replace(" #BEGIN count_contigs", " #BEGIN count_contigs\n" + " stop(\"Special error\")"); FileUtils.writeStringToFile(implFile, implText); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(2, exitCode); } - + @Test public void testSelfCalls() throws Exception { System.out.println("Test [testSelfCalls]"); @@ -214,17 +214,17 @@ public void testSelfCalls() throws Exception { " returnVal = input * input\n" + " #END calc_square\n"; File moduleDir = new File(moduleName); - File implFile = new File(moduleDir, "lib/" + moduleName + "/" + + File implFile = new File(moduleDir, "lib/" + moduleName + "/" + moduleName + "Impl.py"); ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), lang, false); initer.initialize(false); File specFile = new File(moduleDir, moduleName + ".spec"); - String specText = FileUtils.readFileToString(specFile).replace("};", + String specText = FileUtils.readFileToString(specFile).replace("};", "funcdef run_local(int input) returns (int) authentication required;\n" + "funcdef calc_square(int input) returns (int) authentication required;\n" + "};"); File testFile = new File(moduleDir, "test/" + moduleName + "_server_test.py"); - String testCode = FileUtils.readFileToString(testFile).replace(" def test_your_method(self):", + String testCode = FileUtils.readFileToString(testFile).replace(" def test_your_method(self):", " def test_your_method(self):\n" + " self.assertEqual(25, self.getImpl().run_local(self.getContext(), 5)[0])" ); @@ -234,4 +234,4 @@ public void testSelfCalls() throws Exception { int exitCode = runTestsInDocker(moduleDir, token, true); Assert.assertEquals(0, exitCode); } -} \ No newline at end of file +} diff --git a/src/java/us/kbase/mobu/util/ProcessHelper.java b/src/java/us/kbase/mobu/util/ProcessHelper.java index c28e75f5..0f102886 100644 --- a/src/java/us/kbase/mobu/util/ProcessHelper.java +++ b/src/java/us/kbase/mobu/util/ProcessHelper.java @@ -61,7 +61,7 @@ public static ProcessHelper exec(CommandHolder cmd, File workDir, BufferedReader } public static ProcessHelper exec(CommandHolder cmd, File workDir, BufferedReader input, boolean saveOutput, boolean saveErrors, boolean waitFor) throws IOException { - return new ProcessHelper(cmd, workDir, saveOutput ? OutType.StringOut : OutType.SystemOut, + return new ProcessHelper(cmd, workDir, saveOutput ? OutType.StringOut : OutType.SystemOut, saveErrors ? OutType.StringErr : OutType.SystemErr, input, null, null, waitFor); } @@ -75,8 +75,9 @@ public ProcessHelper(CommandHolder cmd, File workDir, OutType outType, OutType e outPw = output; if (error != null) errPw = error; - process = cmd.cmdLine != null ? Runtime.getRuntime().exec(cmd.cmdLine, null, workDir) : - Runtime.getRuntime().exec(cmd.cmdParts, null, workDir); + process = cmd.cmdLine != null + ? Runtime.getRuntime().exec(cmd.cmdLine, null, workDir) + : Runtime.getRuntime().exec(cmd.cmdParts, null, workDir); Thread outTh = readInNewThread(process.getInputStream(), outType); Thread errTh = readInNewThread(process.getErrorStream(), errType); if (input != null) { @@ -108,9 +109,9 @@ public ProcessHelper(CommandHolder cmd, File workDir, OutType outType, OutType e } public Process getProcess() { - return process; - } - + return process; + } + private Thread readInNewThread(final InputStream is, final OutType outType) { Thread ret = new Thread(new Runnable() { public void run() { @@ -189,10 +190,10 @@ public CommandHolder add(String... newCmdParts) { } public CommandHolder dontWaitFor() { - waitFor = false; - return this; + waitFor = false; + return this; } - + public ProcessHelper exec(File workDir) throws IOException { return ProcessHelper.exec(this, workDir, waitFor); } diff --git a/src/java/us/kbase/mobu/validator/ModuleValidator.java b/src/java/us/kbase/mobu/validator/ModuleValidator.java index 9363d4b0..2f1a3d75 100644 --- a/src/java/us/kbase/mobu/validator/ModuleValidator.java +++ b/src/java/us/kbase/mobu/validator/ModuleValidator.java @@ -39,44 +39,44 @@ public class ModuleValidator { - - - private static final String KBASE_YML_FILE = "kbase.yml"; - - - protected List modulePaths; - protected boolean verbose; - protected String methodStoreUrl; - protected boolean allowSyncMethods; - - public ModuleValidator(List modulePaths, boolean verbose, - String defaultMethodStoreUrl, boolean allowSyncMethods) throws Exception { - this.modulePaths = modulePaths; - this.verbose = verbose; - if (modulePaths.size() == 1) { - File module = new File(modulePaths.get(0)); - File testCfg = new File(new File(module, "test_local"), "test.cfg"); - if (testCfg.exists()) { - Properties props = new Properties(); - InputStream is = new FileInputStream(testCfg); - try { - props.load(is); - } finally { - is.close(); - } - String endPoint = props.getProperty("kbase_endpoint"); - if (endPoint != null) - this.methodStoreUrl = endPoint + "/narrative_method_store/rpc"; - } - } - if (this.methodStoreUrl == null) { - this.methodStoreUrl = defaultMethodStoreUrl; - System.out.println("WARNING! 'kbase_endpoint' property was not found in " + - "/test_local/test.cfg so validation is done against NMS in appdev"); - } - this.allowSyncMethods = allowSyncMethods; - } - + + + private static final String KBASE_YML_FILE = "kbase.yml"; + + + protected List modulePaths; + protected boolean verbose; + protected String methodStoreUrl; + protected boolean allowSyncMethods; + + public ModuleValidator(List modulePaths, boolean verbose, + String defaultMethodStoreUrl, boolean allowSyncMethods) throws Exception { + this.modulePaths = modulePaths; + this.verbose = verbose; + if (modulePaths.size() == 1) { + File module = new File(modulePaths.get(0)); + File testCfg = new File(new File(module, "test_local"), "test.cfg"); + if (testCfg.exists()) { + Properties props = new Properties(); + InputStream is = new FileInputStream(testCfg); + try { + props.load(is); + } finally { + is.close(); + } + String endPoint = props.getProperty("kbase_endpoint"); + if (endPoint != null) + this.methodStoreUrl = endPoint + "/narrative_method_store/rpc"; + } + } + if (this.methodStoreUrl == null) { + this.methodStoreUrl = defaultMethodStoreUrl; + System.out.println("WARNING! 'kbase_endpoint' property was not found in " + + "/test_local/test.cfg so validation is done against NMS in appdev"); + } + this.allowSyncMethods = allowSyncMethods; + } + private static boolean isModuleDir(File dir) { return new File(dir, "Dockerfile").exists() && new File(dir, "Makefile").exists() && @@ -86,52 +86,52 @@ private static boolean isModuleDir(File dir) { new File(dir, "test").exists() && new File(dir, "ui").exists(); } - - public int validateAll() { - - int errors = 0; - - for(String modulePathString : modulePaths) { - File module = new File(modulePathString); - System.out.println("\nValidating module in ("+module+")"); - - if(!module.exists()) { - System.err.println(" **ERROR** - the module does not exist"); - errors++; continue; - } - if(!module.isDirectory()) { - System.err.println(" **ERROR** - the module location is not a directory."); - errors++; continue; - } - - try { - if(verbose) System.out.println(" - canonical path = "+module.getCanonicalPath()+""); - File dir = module.getCanonicalFile(); - while (!isModuleDir(dir)) { - dir = dir.getParentFile(); - if (dir == null) - throw new IllegalStateException(" **ERROR** - cannot find folder with module structure"); - } - module = dir; - } catch (IOException e) { - System.err.println(" **ERROR** - unable to extract module canonical path:"); - System.err.println(" "+e.getMessage()); - } - - - // 1) Validate the configuration file - try { - int status = validateKBaseYmlConfig(module); - if(status!=0) { - errors++; continue; - } - } catch (Exception e) { - System.err.println(" **ERROR** - configuration file validation failed:"); - System.err.println(" "+e.getMessage()); - errors++; continue; - } - - KbModule parsedKidl = null; + + public int validateAll() { + + int errors = 0; + + for(String modulePathString : modulePaths) { + File module = new File(modulePathString); + System.out.println("\nValidating module in ("+module+")"); + + if(!module.exists()) { + System.err.println(" **ERROR** - the module does not exist"); + errors++; continue; + } + if(!module.isDirectory()) { + System.err.println(" **ERROR** - the module location is not a directory."); + errors++; continue; + } + + try { + if(verbose) System.out.println(" - canonical path = "+module.getCanonicalPath()+""); + File dir = module.getCanonicalFile(); + while (!isModuleDir(dir)) { + dir = dir.getParentFile(); + if (dir == null) + throw new IllegalStateException(" **ERROR** - cannot find folder with module structure"); + } + module = dir; + } catch (IOException e) { + System.err.println(" **ERROR** - unable to extract module canonical path:"); + System.err.println(" "+e.getMessage()); + } + + + // 1) Validate the configuration file + try { + int status = validateKBaseYmlConfig(module); + if(status!=0) { + errors++; continue; + } + } catch (Exception e) { + System.err.println(" **ERROR** - configuration file validation failed:"); + System.err.println(" "+e.getMessage()); + errors++; continue; + } + + KbModule parsedKidl = null; try { Map config = parseKBaseYaml(new File(module, KBASE_YML_FILE)); String moduleName = (String)config.get("module-name"); @@ -153,78 +153,78 @@ public int validateAll() { errors++; continue; } - // 2) Validate UI components - - // 2a) Validate Narrative Methods - - File uiNarrativeMethodsDir = new File(new File(new File(module, "ui"), "narrative"), "methods"); - if (uiNarrativeMethodsDir.exists()) { - for (File methodDir : uiNarrativeMethodsDir.listFiles()) { - if (methodDir.isDirectory()) { - System.out.println("\nValidating method in ("+methodDir+")"); - try { - int status = validateMethodSpec(methodDir, parsedKidl, this.allowSyncMethods); - if (status != 0) { - errors++; - continue; - } - } catch (Exception e) { - System.err.println(" **ERROR** - method-spec validation failed:"); - System.err.println(" "+e.getMessage()); - errors++; continue; - } - } - } - } - - } - - - - - if(errors>0) { - if(modulePaths.size()==1) { - System.out.println("\n\nThis module contains errors.\n"); - } else { - System.out.println("\n\nErrors detected in "+errors +" of "+modulePaths.size()+" modules.\n"); - } - return 1; - } - if(modulePaths.size()==1) { - System.out.println("\n\nCongrats- this module is valid.\n"); - } else { - System.out.println("\n\nCongrats- all "+modulePaths.size()+" modules are valid.\n"); - } - return 0; - } - - protected int validateKBaseYmlConfig(File module) throws IOException { - File kbaseYmlFile = new File(module.getCanonicalPath()+File.separator+KBASE_YML_FILE); - if(verbose) System.out.println(" - configuration file = "+kbaseYmlFile); - - if(!kbaseYmlFile.exists()) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file does not exist in module directory."); - return 1; - } - if(kbaseYmlFile.isDirectory()) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is a directory, not a file."); - return 1; - } - - try { - parseKBaseYaml(kbaseYmlFile); - if(verbose) System.out.println(" - configuration file is valid YAML"); - } catch(Exception e) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is invalid:"); - System.err.println(" "+e.getMessage()); - return 1; - - } - - - - return 0; - } + // 2) Validate UI components + + // 2a) Validate Narrative Methods + + File uiNarrativeMethodsDir = new File(new File(new File(module, "ui"), "narrative"), "methods"); + if (uiNarrativeMethodsDir.exists()) { + for (File methodDir : uiNarrativeMethodsDir.listFiles()) { + if (methodDir.isDirectory()) { + System.out.println("\nValidating method in ("+methodDir+")"); + try { + int status = validateMethodSpec(methodDir, parsedKidl, this.allowSyncMethods); + if (status != 0) { + errors++; + continue; + } + } catch (Exception e) { + System.err.println(" **ERROR** - method-spec validation failed:"); + System.err.println(" "+e.getMessage()); + errors++; continue; + } + } + } + } + + } + + + + + if(errors>0) { + if(modulePaths.size()==1) { + System.out.println("\n\nThis module contains errors.\n"); + } else { + System.out.println("\n\nErrors detected in "+errors +" of "+modulePaths.size()+" modules.\n"); + } + return 1; + } + if(modulePaths.size()==1) { + System.out.println("\n\nCongrats- this module is valid.\n"); + } else { + System.out.println("\n\nCongrats- all "+modulePaths.size()+" modules are valid.\n"); + } + return 0; + } + + protected int validateKBaseYmlConfig(File module) throws IOException { + File kbaseYmlFile = new File(module.getCanonicalPath()+File.separator+KBASE_YML_FILE); + if(verbose) System.out.println(" - configuration file = "+kbaseYmlFile); + + if(!kbaseYmlFile.exists()) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file does not exist in module directory."); + return 1; + } + if(kbaseYmlFile.isDirectory()) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is a directory, not a file."); + return 1; + } + + try { + parseKBaseYaml(kbaseYmlFile); + if(verbose) System.out.println(" - configuration file is valid YAML"); + } catch(Exception e) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is invalid:"); + System.err.println(" "+e.getMessage()); + return 1; + + } + + + + return 0; + } @SuppressWarnings("unchecked") public Map parseKBaseYaml(File kbaseYmlFile) throws IOException { @@ -232,14 +232,14 @@ public Map parseKBaseYaml(File kbaseYmlFile) throws IOException { String kbaseYml = TextUtils.readFileText(kbaseYmlFile); return (Map) yaml.load(kbaseYml); } - - - protected int validateMethodSpec(File methodDir, KbModule parsedKidl, - boolean allowSyncMethods) throws IOException { - NarrativeMethodStoreClient nms = new NarrativeMethodStoreClient(new URL(methodStoreUrl)); - nms.setAllSSLCertificatesTrusted(true); - nms.setIsInsecureHttpConnectionAllowed(true); - String spec = FileUtils.readFileToString(new File(methodDir, "spec.json")); + + + protected int validateMethodSpec(File methodDir, KbModule parsedKidl, + boolean allowSyncMethods) throws IOException { + NarrativeMethodStoreClient nms = new NarrativeMethodStoreClient(new URL(methodStoreUrl)); + nms.setAllSSLCertificatesTrusted(true); + nms.setIsInsecureHttpConnectionAllowed(true); + String spec = FileUtils.readFileToString(new File(methodDir, "spec.json")); String display = FileUtils.readFileToString(new File(methodDir, "display.yaml")); Map extraFiles = new LinkedHashMap(); for (File f : methodDir.listFiles()) { @@ -272,11 +272,11 @@ protected int validateMethodSpec(File methodDir, KbModule parsedKidl, System.err.println(" "+e.getMessage()); return 1; } - } + } - public static void validateMethodSpecMapping(String specText, KbModule parsedKidl, - boolean allowSyncMethods) throws IOException { - JsonNode spec = new ObjectMapper().readTree(specText); + public static void validateMethodSpecMapping(String specText, KbModule parsedKidl, + boolean allowSyncMethods) throws IOException { + JsonNode spec = new ObjectMapper().readTree(specText); JsonNode behaviorNode = get("/", spec, "behavior"); if (behaviorNode.get("none") != null) return; // Don't pay attention at viewer methods (since they don't use back-end) @@ -336,7 +336,7 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } } if (func == null) { - throw new IllegalStateException(" **ERROR** - unknown method \"" + + throw new IllegalStateException(" **ERROR** - unknown method \"" + methodName + "\" defined within path " + "[behavior/service-mapping/method] in spec.json"); } @@ -361,9 +361,9 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid targetArgPos = targetArgPosNode.asInt(); if (targetArgPos >= func.getParameters().size()) { throw new IllegalStateException(" **ERROR** - value " + targetArgPos + " within " + - "path [" + path + "/target_argument_position] in spec.json is out of " + - "bounds (number of arguments defined for function \"" + methodName + "\" " + - "is " + func.getParameters().size() + ")"); + "path [" + path + "/target_argument_position] in spec.json is out of " + + "bounds (number of arguments defined for function \"" + methodName + "\" " + + "is " + func.getParameters().size() + ")"); } argsUsed.add(targetArgPos); KbType argType = func.getParameters().get(targetArgPos).getType(); @@ -378,7 +378,7 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid argType instanceof KbTuple) { throw new IllegalStateException(" **ERROR** - value " + targetProp + " within " + "path [" + path + "/target_property] in spec.json can't be applied to " + - "type " + argType.getClass().getSimpleName() + " (defined for argument " + + "type " + argType.getClass().getSimpleName() + " (defined for argument " + targetArgPos + ")"); } if (argType instanceof KbStruct) { @@ -392,8 +392,8 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } if (!found) { System.err.println(" **WARNINGS** - value \"" + targetProp + "\" within " + - "path [" + path + "/target_property] in spec.json doesn't match " + - "any field of structure defined as argument type" + + "path [" + path + "/target_property] in spec.json doesn't match " + + "any field of structure defined as argument type" + (struct.getName() != null ? (" (" + struct.getName() + ")") : "")); } } @@ -411,22 +411,22 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } if (func.getParameters().size() != argsUsed.size()) { throw new IllegalStateException(" **ERROR** - not all arguments are set for function " + - "\"" + func.getName() + "\", list of defined arguments is: " + argsUsed); + "\"" + func.getName() + "\", list of defined arguments is: " + argsUsed); } if (inputParamIdToType.size() != paramsUsed.size()) { Set paramsNotUsed = new TreeSet(inputParamIdToType.keySet()); paramsNotUsed.removeAll(paramsUsed); - System.err.println(" **WARNINGS** - some of input parameters are not used: " + + System.err.println(" **WARNINGS** - some of input parameters are not used: " + paramsNotUsed); } - } - - private static JsonNode get(String nodePath, JsonNode node, String childName) { - JsonNode ret = node.get(childName); - if (ret == null) - throw new IllegalStateException(" **ERROR** - can't find sub-node [" + childName + - "] within path [" + nodePath + "] in spec.json"); - return ret; - } + } + + private static JsonNode get(String nodePath, JsonNode node, String childName) { + JsonNode ret = node.get(childName); + if (ret == null) + throw new IllegalStateException(" **ERROR** - can't find sub-node [" + childName + + "] within path [" + nodePath + "] in spec.json"); + return ret; + } } diff --git a/src/java/us/kbase/scripts/test/Test8.perl.properties b/src/java/us/kbase/scripts/test/Test8.perl.properties index 31565d25..68e90e6b 100644 --- a/src/java/us/kbase/scripts/test/Test8.perl.properties +++ b/src/java/us/kbase/scripts/test/Test8.perl.properties @@ -2,7 +2,7 @@ package StoringImpl; use strict; use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = "0.1.0"; =head1 NAME @@ -226,7 +226,7 @@ sub m2second -=head2 version +=head2 version $return = $obj->version() diff --git a/src/java/us/kbase/templates/module_java_impl.vm.properties b/src/java/us/kbase/templates/module_java_impl.vm.properties index 97ee23b5..ee13035f 100644 --- a/src/java/us/kbase/templates/module_java_impl.vm.properties +++ b/src/java/us/kbase/templates/module_java_impl.vm.properties @@ -22,7 +22,7 @@ import assemblyutil.GetAssemblyParams; import assemblyutil.SaveAssemblyParams; import kbasereport.CreateParams; import kbasereport.KBaseReportClient; -import kbasereport.Report; +import kbasereport.SimpleReport; import kbasereport.ReportInfo; import kbasereport.WorkspaceObject; import net.sf.jfasta.FASTAElement; @@ -73,11 +73,11 @@ public class ${java_module_name}Server extends JsonServerServlet { public FilterContigsResults filterContigs(FilterContigsParams params, AuthToken authPart, RpcContext jsonRpcContext) throws Exception { FilterContigsResults returnVal = null; //BEGIN run_${module_name} - + // Print statements to stdout/stderr are captured and available as the App log System.out.println("Starting filter contigs. Parameters:"); System.out.println(params); - + /* Step 1 - Parse/examine the parameters and catch any errors * It is important to check that parameters exist and are defined, and that nice error * messages are returned to users. Parameter values go through basic validation when @@ -103,7 +103,7 @@ public class ${java_module_name}Server extends JsonServerServlet { throw new IllegalArgumentException("min_length parameter cannot be negative (" + minLength + ")"); } - + /* Step 2 - Download the input data as a Fasta file * We can use the AssemblyUtils module to download a FASTA file from our Assembly data * object. The return object gives us the path to the file that was created. @@ -142,26 +142,26 @@ public class ${java_module_name}Server extends JsonServerServlet { final String resultText = String.format("Filtered assembly to %s contigs out of %s", remaining, total); System.out.println(resultText); - + // Step 4 - Save the new Assembly back to the system - + final String newAssyRef = assyUtil.saveAssemblyFromFasta(new SaveAssemblyParams() .withAssemblyName(fileobj.getAssemblyName()) .withWorkspaceName(workspaceName) .withFile(new FastaAssemblyFile().withPath(out.toString()))); - + // Step 5 - Build a Report and return - + final KBaseReportClient kbr = new KBaseReportClient(callbackURL, authPart); // see note above about bad practice kbr.setIsInsecureHttpConnectionAllowed(true); final ReportInfo report = kbr.create(new CreateParams().withWorkspaceName(workspaceName) - .withReport(new Report().withTextMessage(resultText) + .withReport(new SimpleReport().withTextMessage(resultText) .withObjectsCreated(Arrays.asList(new WorkspaceObject() .withDescription("Filtered contigs") .withRef(newAssyRef))))); // Step 6: contruct the output to send back - + returnVal = new ReportResults() .withReportName(report.getName()) .withReportRef(report.getRef()); @@ -195,7 +195,7 @@ import java.net.MalformedURLException; import kbasereport.CreateParams; import kbasereport.KBaseReportClient; -import kbasereport.Report; +import kbasereport.SimpleReport; import kbasereport.ReportInfo; import kbasereport.WorkspaceObject; import ${java_package}.ReportResults; @@ -221,7 +221,7 @@ import ${java_package}.ReportResults; final String parameter_1 = params.get("parameter_1").asInstance(); final ReportInfo report = kbr.create(new CreateParams() .withWorkspaceName(ws_name) - .withReport(new Report() + .withReport(new SimpleReport() .withTextMessage(parameter_1) .withObjectsCreated(new java.util.ArrayList()))); @@ -230,4 +230,4 @@ import ${java_package}.ReportResults; .withReportRef(report.getRef()); //END run_${module_name} -#end \ No newline at end of file +#end diff --git a/src/java/us/kbase/templates/module_test_java_client.vm.properties b/src/java/us/kbase/templates/module_test_java_client.vm.properties index 51bdce43..0dbc2fd8 100644 --- a/src/java/us/kbase/templates/module_test_java_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_java_client.vm.properties @@ -46,7 +46,7 @@ public class ${java_module_name}ServerTest { private static ${java_module_name}Server impl = null; private static Path scratch; private static URL callbackURL; - + @BeforeClass public static void init() throws Exception { // Config loading @@ -72,7 +72,7 @@ public class ${java_module_name}ServerTest { .getAbsolutePath()); impl = new ${java_module_name}Server(); } - + private static String getWsName() throws Exception { if (wsName == null) { long suffix = System.currentTimeMillis(); @@ -81,13 +81,13 @@ public class ${java_module_name}ServerTest { } return wsName; } - + private static RpcContext getContext() { return new RpcContext().withProvenance(Arrays.asList(new ProvenanceAction() .withService("${module_name}").withMethod("please_never_use_it_in_production") .withMethodParams(new ArrayList()))); } - + @AfterClass public static void cleanup() { if (wsName != null) { @@ -99,7 +99,7 @@ public class ${java_module_name}ServerTest { } } } - + #if ($example) private String loadFASTA( final Path filename, @@ -116,9 +116,9 @@ public class ${java_module_name}ServerTest { .withAssemblyName(objectName) .withWorkspaceName(getWsName()) .withFile(new FastaAssemblyFile().withPath(filename.toString()))); - + } - + @Test public void test_run${java_module_name}_ok() throws Exception { // First load a test FASTA file as an KBase Assembly @@ -128,9 +128,9 @@ public class ${java_module_name}ServerTest { "agctt\n" + ">seq3\n" + "agcttttcatgg"; - + final String ref = loadFASTA(scratch.resolve("test1.fasta"), "TestAssembly", fastaContent); - + // second, call the implementation Map params = new HashMap<>(); params.put("workspace_name", new UObject(getWsName())); @@ -139,7 +139,7 @@ public class ${java_module_name}ServerTest { impl.run${java_module_name}(params, token, getContext()); } - + @Test public void test_run${java_module_name}_err1() throws Exception { try { @@ -153,7 +153,7 @@ public class ${java_module_name}ServerTest { ex.getMessage()); } } - + @Test public void test_run${java_module_name}_err2() throws Exception { try { @@ -167,7 +167,7 @@ public class ${java_module_name}ServerTest { Assert.assertEquals("min_length parameter cannot be negative (-10)", ex.getMessage()); } } - + @Test public void test_run${java_module_name}_err3() throws Exception { try { @@ -178,8 +178,12 @@ public class ${java_module_name}ServerTest { impl.run${java_module_name}(params, token, getContext()); Assert.fail("Error is expected above"); } catch (ServerException ex) { - Assert.assertEquals("Error on ObjectSpecification #1: Illegal number of separators " + - "/ in object reference fake", ex.getMessage()); + // ex.getMessage() returns the error string quoted + // so just test that the message is the same + Assert.assertTrue( ex.getMessage().contains( + "Error on ObjectSpecification #1: Illegal number of separators " + + "/ in object reference fake" + ) ); } } #else @@ -200,4 +204,4 @@ public class ${java_module_name}ServerTest { final ReportResults ret = impl.run${java_module_name}(params, token, getContext()); } #end -} \ No newline at end of file +}