Support Quantilized Fully Connected #5
Conversation
…loc/delete in each run
| namespace mxnet { | ||
| namespace op { | ||
|
|
||
| namespace qfc { |
There was a problem hiding this comment.
please use a more descriptive name instead of qfc.
|
|
||
| struct QuantizedSumInitKernelWithBias { | ||
| // init sum data with bias for matrix b (n) | ||
| MSHADOW_XINLINE static void Map(int i, int32_t *out, |
There was a problem hiding this comment.
Previously in the optimization for embedding, we thought that Map function will have a low efficiency for omp. Do you have any profiling data here?
There was a problem hiding this comment.
in ut testing, map way is faster(2-3X) than omp
There was a problem hiding this comment.
Interesting. I thought kernel launch is also using omp.
| // shift data from int8(from -128 to 127) to uint8 (from 0 to 255) | ||
| int shift = 128; | ||
| Tensor<cpu, 1, uint8_t> shiftdata = | ||
| ctx.requested[qfc::kTempSpace].get_space_typed<cpu, 1, uint8_t>( |
| Tensor<cpu, 1, uint8_t> shiftdata = | ||
| ctx.requested[qfc::kTempSpace].get_space_typed<cpu, 1, uint8_t>( | ||
| Shape1(m * k), s); | ||
| Kernel<QuantizedShiftKernel, cpu>::Launch(s, m * k, data.data().dptr<int8_t>(), |
|
|
||
| NNVM_REGISTER_OP(_contrib_quantized_fully_connected) | ||
| .set_attr<FComputeEx>("FComputeEx<cpu>", | ||
| MKLDNNQuantizedFullyConnectedForward<int8_t>) |
There was a problem hiding this comment.
only supports int8? How about the quantized FC for GPU?
There was a problem hiding this comment.
.set_attr("FCompute", QuantizedFullyConnectedForwardGPU<int8_t, int32_t, int32_t>); in quantized_fully_connected.cu
| @@ -79,6 +79,22 @@ bool QuantizedFullyConnectedType(const nnvm::NodeAttrs& attrs, | |||
| return true; | |||
| } | |||
|
|
|||
| bool QuantizedFullyConnectedStorageType(const nnvm::NodeAttrs& attrs, | |||
| const int dev_mask, | |||
| namespace mxnet { | ||
| namespace op { | ||
|
|
||
| namespace quantilizedfc { |
There was a problem hiding this comment.
To align with the definition in fp32 fully_connected-inl.h, suggest to use quantized_fullc here.
| namespace op { | ||
|
|
||
| namespace quantilizedfc { | ||
| enum QuantilizedfcOpResource {kTempSpace}; |
|
|
||
| struct QuantizedSumInitKernelWithBias { | ||
| // init sum data with bias for matrix b (n) | ||
| MSHADOW_XINLINE static void Map(int i, int32_t *out, |
There was a problem hiding this comment.
Interesting. I thought kernel launch is also using omp.
| out[i] = bias[i] * float_for_one_bias_quant / | ||
| float_for_one_out_quant; | ||
| } else { | ||
| LOG(INFO) << "WARNING: QuantizedBiasAddKernel float_for_one_out_quant is 0 !"; |
| @@ -139,6 +141,9 @@ void MKLDNNQuantizedFullyConnectedForward(const nnvm::NodeAttrs& attrs, | |||
| out.data().dptr<int32_t>(), | |||
| n, | |||
| &oc); | |||
| #else | |||
| LOG(FATAL) << "s8u8s32 is not supported by the BLAS library"; | |||
There was a problem hiding this comment.
Maybe need change to "s8u8s32 is only supported by MKL BLAS library"?
Description
In this PR, it created quantilized fully connected op by using int8 gemm
@pengzhao-intel, @TaoLv , @ciyongch
Feature changes
New features
Unit-test changes
Checklist