Copyfromto (#30)

* Refactor copyfromto * reuse data/aux data memory for sparse ndarray when possible * randomized some python & cpp tests * bug fix for identity_attr_like_rhs * bug fix for _backward_dot * Enable auto-fallback for ForwardOp * attach op fallback bug fix * Fix lint * Compiles on GPU * use kernel for rsp->dns. TODO: use memcopy * Add wrapper for aux data * rewrite python sparsend creation interface * add doc for csr rsp constructor * fix lint * more docs * pass lint * sparse embedding draft sparse embedding unit test pass Conflicts: src/operator/tensor/elemwise_unary_op.h tests/cpp/ndarray_test.cc tests/python/unittest/test_sparse_ndarray.py tests/python/unittest/test_sparse_operator.py * bug fix for NDArray init in graph executor * Add zeros() for csr. Handle zeros input in some csr dot operators * _slice for csr * fix lint * change is_zeros_hint to storage_initialized
eric-haibin-lin · May 14, 2017 · 1103a9a · 1103a9a
1 parent 4e84972
commit 1103a9a
Show file tree

Hide file tree

Showing 40 changed files with 1,709 additions and 925 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -205,7 +205,7 @@ del /Q *.7z
 def python_ut(docker_type) {
   timeout(time: max_time, unit: 'MINUTES') {
     sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/unittest"
-    // sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest"
+    sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest"
     sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/train"
   }
 }
@@ -215,7 +215,7 @@ def python_ut(docker_type) {
 def python_gpu_ut(docker_type) {
   timeout(time: max_time, unit: 'MINUTES') {
     sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/gpu"
-    // sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/gpu"
+    sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/gpu"
   }
 }
 

diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h
@@ -222,21 +222,6 @@ MXNET_DLL int MXNDArrayCreate(const mx_uint *shape,
                               int delay_alloc,
                               NDArrayHandle *out);
 
-/*!
- * \brief create a NDArray with specified sparse type, shape and aux data(e.g. index)
- * aux data is copied during construction.
- */
-MXNET_DLL int MXNDArrayCreateSparse(NDArrayHandle data,
-                    mx_uint num_aux,
-                    NDArrayHandle *aux_data,
-                    const mx_uint *shape,
-                    mx_uint ndim,
-                    int storage_type,
-                    int dev_type,
-                    int dev_id,
-                    int delay_alloc,
-                    int dtype,
-                    NDArrayHandle *out);
 /*!
  * \brief create a NDArray with specified shape and data type
  * \param shape the pointer to the shape
@@ -260,6 +245,20 @@ MXNET_DLL int MXNDArrayCreateEx(const mx_uint *shape,
 
 /*!
  * \brief create an empty sparse NDArray with specified shape and data type
+ * \param storage_type the storage type of the ndarray
+ * \param shape the pointer to the shape
+ * \param ndim the dimension of the shape
+ * \param dev_type device type, specify device we want to take
+ * \param dev_id the device id of the specific device
+ * \param delay_alloc whether to delay allocation until
+ *        the narray is first mutated
+ * \param dtype data type of created array
+ * \param num_aux the number of aux data to support this ndarray
+ * \param aux_type data type of the aux data for the created array
+ * \param aux_ndims the dimension of the shapes of aux data
+ * \param aux_shape the shapes of aux data
+ * \param out the returning handle
+ * \return 0 when success, -1 when failure happens
  */
 MXNET_DLL int MXNDArrayCreateSparseEx(int storage_type,
                     const mx_uint *shape,
@@ -269,7 +268,9 @@ MXNET_DLL int MXNDArrayCreateSparseEx(int storage_type,
                     int delay_alloc,
                     int dtype,
                     mx_uint num_aux,
-                    int *aux_types,
+                    int *aux_type,
+                    mx_uint *aux_ndims,
+                    const mx_uint *aux_shape,
                     NDArrayHandle *out);
 
 /*!
@@ -439,13 +440,26 @@ MXNET_DLL int MXNDArrayGetData(NDArrayHandle handle,
  */
 MXNET_DLL int MXNDArrayGetDType(NDArrayHandle handle,
                                int *out_dtype);
-// Get the aux type for ith aux data
+
+/*!
+ * \brief get the type of the ith aux data in NDArray
+ * \param handle the handle to the narray
+ * \param i the index of the aux data
+ * \param out_type pointer holder to get type of aux data
+ * \return 0 when success, -1 when failure happens
+ */
 MXNET_DLL int MXNDArrayGetAuxType(NDArrayHandle handle,
                                   mx_uint i,
-                                  int *out_aux_type);
-// Get the num of aux data to help store sparse NDArray
-MXNET_DLL int MXNDArrayGetNumAux(NDArrayHandle handle,
-                                 mx_uint *out_num_aux);
+                                  int *out_type);
+
+// Get the ith aux data blob wrapped in an NDArray
+MXNET_DLL int MXNDArrayGetAuxNDArray(NDArrayHandle handle,
+                                     mx_uint i,
+                                     NDArrayHandle *out);
+
+// Get the data blob wrapped in an NDArray
+MXNET_DLL int MXNDArrayGetDataNDArray(NDArrayHandle handle,
+                                      NDArrayHandle *out);
 /*!
  * \brief get the context of the NDArray
  * \param handle the handle to the narray