From de6501481f3fa3f1b21e57b938d6994c78eb9178 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Mon, 22 Oct 2018 22:05:30 -0700 Subject: [PATCH 1/4] [MXNET-1155] Add scala packageTest utility --- Makefile | 8 + scala-package/core/pom.xml | 8 + scala-package/examples/pom.xml | 8 + scala-package/infer/pom.xml | 8 + scala-package/macros/pom.xml | 8 + scala-package/packageTest/Makefile | 77 +++++++++ scala-package/packageTest/README.md | 50 ++++++ scala-package/packageTest/core/pom.xml | 30 ++++ scala-package/packageTest/core/scripts | 1 + scala-package/packageTest/examples/pom.xml | 39 +++++ scala-package/packageTest/examples/scripts | 1 + scala-package/packageTest/infer/pom.xml | 29 ++++ scala-package/packageTest/pom.xml | 178 +++++++++++++++++++++ 13 files changed, 445 insertions(+) create mode 100644 scala-package/packageTest/Makefile create mode 100644 scala-package/packageTest/README.md create mode 100644 scala-package/packageTest/core/pom.xml create mode 120000 scala-package/packageTest/core/scripts create mode 100644 scala-package/packageTest/examples/pom.xml create mode 120000 scala-package/packageTest/examples/scripts create mode 100644 scala-package/packageTest/infer/pom.xml create mode 100644 scala-package/packageTest/pom.xml diff --git a/Makefile b/Makefile index 16ea59f3d585..65a191b03e59 100644 --- a/Makefile +++ b/Makefile @@ -603,6 +603,14 @@ scalaclean: (cd $(ROOTDIR)/scala-package; \ mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE)) +scalatestcompile: + (cd $(ROOTDIR)/scala-package; \ + mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ + -Dcurrent_libdir="$(ROOTDIR)/lib" \ + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + scalapkg: (cd $(ROOTDIR)/scala-package; \ mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 3425bb15f62a..ba24334d2e4d 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -63,6 +63,14 @@ META-INF/*.RSA + + + + test-jar + + + + org.apache.maven.plugins diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 9e8e119c3c4f..e10d11e679d9 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -119,6 +119,14 @@ org.apache.maven.plugins maven-jar-plugin + + + + test-jar + + + + org.apache.maven.plugins diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 3e6980cb6f4b..8c5f267c17e9 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -63,6 +63,14 @@ META-INF/*.RSA + + + + test-jar + + + + org.apache.maven.plugins diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index d435e211ceeb..891652306457 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -90,6 +90,14 @@ META-INF/*.RSA + + + + test-jar + + + + org.apache.maven.plugins diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile new file mode 100644 index 000000000000..315cf6f8a131 --- /dev/null +++ b/scala-package/packageTest/Makefile @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCALA_VERSION_PROFILE := 2.11 +SCALA_VERSION := 2.11.8 +MXNET_VERSION := 1.3.1-SNAPSHOT + +MXNET_REPO = https://repository.apache.org/content/repositories/snapshots + +ifeq ($(OS),Windows_NT) + UNAME_S := Windows +else + UNAME_S := $(shell uname -s) +endif + +ifeq ($(UNAME_S), Windows) + # TODO: currently scala package does not support windows + SCALA_PKG_PROFILE := windows +else + ifeq ($(UNAME_S), Darwin) + SCALA_PKG_PROFILE := osx-x86_64-cpu + else + SCALA_PKG_PROFILE := linux-x86_64 + ifeq ($(USE_CUDA), 1) + SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu + else + SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu + endif + endif +endif + +scalapkg: + (mvn package -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +scalaclean: + (mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +scalaintegrationtest: + (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -PwithExamples \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +scalaintegrationtestsnapshot: + (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -PfromSnapshots \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.repo=$(MXNET_REPO) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) + +scalaintegrationtestwithoutexamples: + (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.version=$(MXNET_VERSION) \ + -Dscala.version=$(SCALA_VERSION)) diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md new file mode 100644 index 000000000000..0972ea35836b --- /dev/null +++ b/scala-package/packageTest/README.md @@ -0,0 +1,50 @@ +# MXNet Scala Package Test + +This is an project created to run the test suite on a fully packaged mxnet jar. + +## Setup + +### Install Package + +To run the test suite, first install the package. This can be done either by installing directly from a jar with `mvn install:install-file -Dfile=` or by running `make scalainstall` in the main mxnet folder. Note that if you use `mvn install:install-file`, you will be unable to run the example tests unless you also install the mxnetexamples jar. You can run all tests except for those examples with `make scalaintegrationtestwithoutexamples`. + +### Build + +Build the mxnet tests by running `make scalapkg` and then `make scalatestcompile` from the main mxnet directory. This is needed for test discovery. + +## Run + +To run, ensure the versions are correct in the `Makefile`. Then, just run `make scalaintegrationtest` to execute the test suite + +## Clean + +You can clean temporary files and target artifacts by running `make scalaclean`. + +## Troubleshooting + +### Missing Examples + +If you fail with the following error +``` +[ERROR] Failed to execute goal org.scalatest:scalatest-maven-plugin:1.0:test (test) on project mxnet-scala-packagetest-examples_2.11: There are test failures -> [Help 1] +[ERROR] +[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. +[ERROR] Re-run Maven using the -X switch to enable full debug logging. +[ERROR] +[ERROR] For more information about the errors and possible solutions, please read the following articles: +[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException +[ERROR] +[ERROR] After correcting the problems, you can resume the build with the command +[ERROR] mvn -rf :mxnet-scala-packagetest-examples_2.11 +Makefile:57: recipe for target 'scalaintegrationtest' failed +make: *** [scalaintegrationtest] Error 1 +``` + +and stacktrace begins with the following, + +``` +*** RUN ABORTED *** + java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$ +``` + +you are missing the mxnetexamples package. See the "Install Package" section for details. diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml new file mode 100644 index 000000000000..71ace9bf58d7 --- /dev/null +++ b/scala-package/packageTest/core/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-core_2.11 + MXNet Scala Package Test + pom + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../core/target/test-classes + + + + + + diff --git a/scala-package/packageTest/core/scripts b/scala-package/packageTest/core/scripts new file mode 120000 index 000000000000..f806668aa847 --- /dev/null +++ b/scala-package/packageTest/core/scripts @@ -0,0 +1 @@ +../../core/scripts \ No newline at end of file diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml new file mode 100644 index 000000000000..722eb50762b8 --- /dev/null +++ b/scala-package/packageTest/examples/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-examples_2.11 + MXNet Scala Package Test + pom + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../examples/target/test-classes + + + + + + + + org.apache.mxnet + mxnet-examples_${mxnet.scalaprofile} + ${mxnet.version} + test + + + + diff --git a/scala-package/packageTest/examples/scripts b/scala-package/packageTest/examples/scripts new file mode 120000 index 000000000000..2bba4eeece74 --- /dev/null +++ b/scala-package/packageTest/examples/scripts @@ -0,0 +1 @@ +../../examples/scripts \ No newline at end of file diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml new file mode 100644 index 000000000000..409145ea2899 --- /dev/null +++ b/scala-package/packageTest/infer/pom.xml @@ -0,0 +1,29 @@ + + + 4.0.0 + + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + ../pom.xml + + + mxnet-scala-packagetest-infer_2.11 + MXNet Scala Package Test + pom + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.outputDirectory},${project.build.testOutputDirectory},../../infer/target/test-classes + + + + + diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml new file mode 100644 index 000000000000..1c16448c0692 --- /dev/null +++ b/scala-package/packageTest/pom.xml @@ -0,0 +1,178 @@ + + + 4.0.0 + PackageTest + mxnet-scala-packagetest_2.11 + 1.0-SNAPSHOT + MXNet Scala Package Test + pom + + + core + infer + + + + + withExamples + + examples + + + + fromSnapshots + + + apache-snapshots + ${mxnet.repo} + default + + true + + + + + + + + + org.apache.mxnet + mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} + ${mxnet.version} + + + org.scala-lang + scala-library + ${scala.version} + + + commons-io + commons-io + 2.4 + + + org.scalatest + scalatest_${mxnet.scalaprofile} + 3.0.4 + test + + + org.scalacheck + scalacheck_${mxnet.scalaprofile} + 1.13.5 + test + + + org.mockito + mockito-all + 1.10.19 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.6 + 1.6 + UTF-8 + + + + maven-resources-plugin + 2.7 + + + org.apache.maven.plugins + maven-dependency-plugin + 2.9 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.outputDirectory}/lib + runtime + test,provided + false + false + true + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.5 + + + package + + jar + test-jar + + + + **/* + + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + compile + + compile + + compile + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.19 + + true + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${skipTests} + ${project.build.directory}/surefire-reports + . + F + WDF TestSuite.txt + + + + test + integration-test + + test + + + + + + + + From b57499233b52944532e6018136af3a9aed67574e Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Fri, 30 Nov 2018 15:12:13 -0800 Subject: [PATCH 2/4] Clean up utility --- Makefile | 9 +++++ scala-package/.gitignore | 1 + scala-package/core/pom.xml | 8 ----- scala-package/examples/pom.xml | 22 ++++++++----- scala-package/infer/pom.xml | 8 ----- scala-package/macros/pom.xml | 8 ----- scala-package/packageTest/Makefile | 36 ++++++++++++-------- scala-package/packageTest/README.md | 38 ++++++++++++++++------ scala-package/packageTest/core/pom.xml | 9 +++++ scala-package/packageTest/examples/pom.xml | 9 +++++ scala-package/packageTest/infer/pom.xml | 9 +++++ scala-package/packageTest/pom.xml | 22 +++++++++++-- 12 files changed, 122 insertions(+), 57 deletions(-) diff --git a/Makefile b/Makefile index 65a191b03e59..218bfa0cd607 100644 --- a/Makefile +++ b/Makefile @@ -663,6 +663,15 @@ scaladeploy: -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") +scaladeploylocal: + (cd $(ROOTDIR)/scala-package; \ + mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ + -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \ + -Dgpg.skip \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") + jnilint: 3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src diff --git a/scala-package/.gitignore b/scala-package/.gitignore index 6aa4da6b1cfc..8bc87f53e802 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -6,3 +6,4 @@ core/src/main/scala/org/apache/mxnet/SymbolAPIBase.scala core/src/main/scala/org/apache/mxnet/SymbolBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ +local-snapshot \ No newline at end of file diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index ba24334d2e4d..3425bb15f62a 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -63,14 +63,6 @@ META-INF/*.RSA - - - - test-jar - - - - org.apache.maven.plugins diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index e10d11e679d9..80d484aebb2e 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -42,6 +42,20 @@ linux-x86_64-gpu + + deployLocal + + + + org.apache.maven.plugins + maven-deploy-plugin + + false + + + + + release @@ -119,14 +133,6 @@ org.apache.maven.plugins maven-jar-plugin - - - - test-jar - - - - org.apache.maven.plugins diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 8c5f267c17e9..3e6980cb6f4b 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -63,14 +63,6 @@ META-INF/*.RSA - - - - test-jar - - - - org.apache.maven.plugins diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index 891652306457..d435e211ceeb 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -90,14 +90,6 @@ META-INF/*.RSA - - - - test-jar - - - - org.apache.maven.plugins diff --git a/scala-package/packageTest/Makefile b/scala-package/packageTest/Makefile index 315cf6f8a131..6073ff8a722f 100644 --- a/scala-package/packageTest/Makefile +++ b/scala-package/packageTest/Makefile @@ -17,7 +17,7 @@ SCALA_VERSION_PROFILE := 2.11 SCALA_VERSION := 2.11.8 -MXNET_VERSION := 1.3.1-SNAPSHOT +MXNET_VERSION := "[1.3.0-SNAPSHOT,)" MXNET_REPO = https://repository.apache.org/content/repositories/snapshots @@ -43,35 +43,45 @@ else endif endif -scalapkg: - (mvn package -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ - -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ - -Dmxnet.version=$(MXNET_VERSION) \ - -Dscala.version=$(SCALA_VERSION)) +PROFILES := -Ptest +ifeq ($(UNIT), 1) + PROFILES := "$(PROFILES),unittest" +endif +ifeq ($(INTEGRATION), 1) + PROFILES := "$(PROFILES),integrationtest" +endif -scalaclean: +ifneq ($(UNIT), 1) + ifneq ($(INTEGRATION), 1) + PROFILES := "$(PROFILES),unittest,integrationtest" + endif +endif + + +clean: (mvn clean -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) -scalaintegrationtest: +testinstall: (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ - -PwithExamples \ + $(PROFILES) \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) -scalaintegrationtestsnapshot: +testlocal: (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ - -PfromSnapshots \ + $(PROFILES),fromLocal \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ - -Dmxnet.repo=$(MXNET_REPO) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) -scalaintegrationtestwithoutexamples: +testsnapshot: (mvn integration-test -Dmxnet.profile=$(SCALA_PKG_PROFILE) \ + $(PROFILES),fromSnapshots \ -Dmxnet.scalaprofile=$(SCALA_VERSION_PROFILE) \ + -Dmxnet.repo=$(MXNET_REPO) \ -Dmxnet.version=$(MXNET_VERSION) \ -Dscala.version=$(SCALA_VERSION)) diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md index 0972ea35836b..6affb90d3602 100644 --- a/scala-package/packageTest/README.md +++ b/scala-package/packageTest/README.md @@ -1,22 +1,40 @@ # MXNet Scala Package Test -This is an project created to run the test suite on a fully packaged mxnet jar. +This is an project created to run the test suite on a fully packaged mxnet jar. The test suite is found locally but mxnet is from the target jarfile. -## Setup +## General Setup -### Install Package +To setup the packageTest, you must first build your tests. To build the tests, follow these steps from the mxnet main directory: -To run the test suite, first install the package. This can be done either by installing directly from a jar with `mvn install:install-file -Dfile=` or by running `make scalainstall` in the main mxnet folder. Note that if you use `mvn install:install-file`, you will be unable to run the example tests unless you also install the mxnetexamples jar. You can run all tests except for those examples with `make scalaintegrationtestwithoutexamples`. +1. Build MXNet and the scala package from source following the directions [here](https://mxnet.incubator.apache.org/install/scala_setup.html#source) +2. Build the tests by running `make scalatestcompile`. +3. Follow setup instructions below for your testing goal -### Build +## Running -Build the mxnet tests by running `make scalapkg` and then `make scalatestcompile` from the main mxnet directory. This is needed for test discovery. +There are three different modes of operation for testing based on the location of the jar and where it is coming from: -## Run +### Test Installed Jars -To run, ensure the versions are correct in the `Makefile`. Then, just run `make scalaintegrationtest` to execute the test suite +If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`) using `mvn install:install-file -Dfile=`. This might be useful if you acquire the .jar file from elsewhere. You can also run `make scalainstall` to install from a local build. Then, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests. -## Clean +### Test Local Deployment + +To test the jars that would be produced by a deployment, you can run `make scaladeploylocal` from the main mxnet directory. This produces a local snapshot located at `scala-package/local-snapshot`. To test this local snapshot, run `make testlocal`. + +### Remote Repository Snapshot + +This mode is to test a jar located in a remote repository. The default repository is the apache snapshot repisotory located at `https://repository.apache.org/content/repositories/snapshots`. Note that the actual jar in a repisotory should be located at `$repoUrl/org/apache/mxnet/mxnet-full_$scalaVersion-$osMode/$version/*.jar`. + +Test the snapshot repo using `make testsnapshot` or a different repo using `make testsnapshot MXNET_REPO=$NEW_REPO_URL`. + +### Options + +You are able to run unit tests, integration tests, or both using this utility. To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to run both the unit and integration tests. However, the integration tests require that the mxnet examples be installed in addition to the full mxnet package (see test mode instructions above). + +An additional option, you can specify the mxnet version with `MXNET_VERSION=1.3.1-SNAPSHOT`. + +## Cleaning Up You can clean temporary files and target artifacts by running `make scalaclean`. @@ -47,4 +65,4 @@ and stacktrace begins with the following, java.lang.NoClassDefFoundError: org/apache/mxnetexamples/Util$ ``` -you are missing the mxnetexamples package. See the "Install Package" section for details. +you are missing the mxnetexamples package. See your test mode installation section for details. diff --git a/scala-package/packageTest/core/pom.xml b/scala-package/packageTest/core/pom.xml index 71ace9bf58d7..bdcd7662f082 100644 --- a/scala-package/packageTest/core/pom.xml +++ b/scala-package/packageTest/core/pom.xml @@ -14,6 +14,15 @@ MXNet Scala Package Test pom + + + unittest + + false + + + + diff --git a/scala-package/packageTest/examples/pom.xml b/scala-package/packageTest/examples/pom.xml index 722eb50762b8..e11be657e225 100644 --- a/scala-package/packageTest/examples/pom.xml +++ b/scala-package/packageTest/examples/pom.xml @@ -14,6 +14,15 @@ MXNet Scala Package Test pom + + + integrationtest + + false + + + + diff --git a/scala-package/packageTest/infer/pom.xml b/scala-package/packageTest/infer/pom.xml index 409145ea2899..7c5a096d6e14 100644 --- a/scala-package/packageTest/infer/pom.xml +++ b/scala-package/packageTest/infer/pom.xml @@ -14,6 +14,15 @@ MXNet Scala Package Test pom + + + unittest + + false + + + + diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index 1c16448c0692..9c5c11cf2779 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -16,7 +16,10 @@ - withExamples + test + + + integrationtest examples @@ -34,8 +37,24 @@ + + fromLocal + + + local-snapshot + file://${basedir}/../local-snapshot + + true + + + + + + true + + org.apache.mxnet @@ -119,7 +138,6 @@ package jar - test-jar From ea131f003cb946457d194d0777853b06a6048f8a Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Fri, 7 Dec 2018 11:09:30 -0800 Subject: [PATCH 3/4] Safe change directory in Makefile for scala --- Makefile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 218bfa0cd607..822704e26752 100644 --- a/Makefile +++ b/Makefile @@ -600,11 +600,11 @@ rpkgtest: Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)' scalaclean: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn clean -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE)) scalatestcompile: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn test-compile -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ @@ -612,7 +612,7 @@ scalatestcompile: -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scalapkg: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ @@ -620,51 +620,51 @@ scalapkg: -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scalaunittest: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),unittest -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalaintegrationtest: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn integration-test -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE),integrationtest -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalainstall: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests=true -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scalarelease-dryrun: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-prepare: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn release:clean release:prepare -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-perform: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn release:perform -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests=true\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scaladeploy: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scaladeploylocal: - (cd $(ROOTDIR)/scala-package; \ + (cd $(ROOTDIR)/scala-package && \ mvn deploy -Papache-release,deployLocal,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests=true -Dcxx="$(CXX)" \ -DaltDeploymentRepository=snapshot-repo::default::file:local-snapshot \ -Dgpg.skip \ From a3c4d45e9143d15d283f319db99da381ab503b69 Mon Sep 17 00:00:00 2001 From: Zach Kimberg Date: Mon, 10 Dec 2018 12:36:32 -0800 Subject: [PATCH 4/4] mvn install file instructions with details --- scala-package/packageTest/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scala-package/packageTest/README.md b/scala-package/packageTest/README.md index 6affb90d3602..3f1eeb842f07 100644 --- a/scala-package/packageTest/README.md +++ b/scala-package/packageTest/README.md @@ -16,7 +16,11 @@ There are three different modes of operation for testing based on the location o ### Test Installed Jars -If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`) using `mvn install:install-file -Dfile=`. This might be useful if you acquire the .jar file from elsewhere. You can also run `make scalainstall` to install from a local build. Then, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests. +If you have a jar file, you can install it to your maven cache repository(`~/.m2/repository`). This might be useful if you acquire the .jar file from elsewhere. To install, it is easiest to use `mvn install:install-file -Dfile= -DpomFile=`. If the pom file is not available, you can also run `mvn install:install-file -Dfile= -DgroupId= -DartifactId= -Dversion= -Dpackaging=`. With the full mxnet jar, this might look like `mvn install:install-file -Dfile= -DgroupId=org.apache.mxnet -DartifactId=mxnet-full_2.11-linux-x86_64-cpu -Dversion=1.3.0 -Dpackaging=jar`. + +You can also run `make scalainstall` to install from a local build. + +After installing, run `make testinstall` in the package test directory to run the tests. Note that unless you also install an additional mxnetexamples jar, you can only run the unit tests. ### Test Local Deployment