From 393cdc2d0d2de49ec3a5a765c17b19eaf273dfd9 Mon Sep 17 00:00:00 2001 From: pcpLiu Date: Fri, 1 Dec 2017 10:37:26 -0500 Subject: [PATCH 1/2] Rename codecov.yml. Trying to fix ignoring path not working. Ref: https://github.com/codecov/support/issues/426 --- .codecov.yml => codecov.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .codecov.yml => codecov.yml (100%) diff --git a/.codecov.yml b/codecov.yml similarity index 100% rename from .codecov.yml rename to codecov.yml From 8b841edb9640e3a0c3e312f69799ec1bad657684 Mon Sep 17 00:00:00 2001 From: pcpLiu Date: Sat, 2 Dec 2017 16:21:55 -0500 Subject: [PATCH 2/2] Fix bug in batchnorm op. --- .mkdocs.yml | 5 +++-- .../Serrano/operators/nn/common/batchnorm_op.metal | 3 ++- .../Serrano/operators/nn/common/batchnorm_op.swift | 12 +++++++++--- docs/doc_generate.sh | 8 ++++---- docs/operators/batchnorm.md | 14 ++++++++++++++ 5 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 docs/operators/batchnorm.md diff --git a/.mkdocs.yml b/.mkdocs.yml index 8e861a4..c272b41 100644 --- a/.mkdocs.yml +++ b/.mkdocs.yml @@ -13,7 +13,7 @@ site_author: pcpLiu site_description: Serrano Framework Guides copyright: Copyright (c) 2017, Zhonghao (Tim) Liu -site_dir: generated_docs/v0.1.2-alpha/guides +site_dir: generated_docs/v0.1.5-alpha/guides google_analytics: - 'UA-54280547-7' @@ -44,7 +44,7 @@ extra_javascript: - 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML' extra: - version: v0.1.2-alpha + version: v0.1.5-alpha pages: @@ -66,6 +66,7 @@ pages: - Neural network: - Common: - Fully Connected: 'operators/fully_connected.md' + - BatchNormalization: 'operators/batchnorm.md' - Convolution: 'operators/conv.md' - Pooling: 'operators/pooling.md' - Activation: 'operators/activation.md' diff --git a/Source/Serrano/operators/nn/common/batchnorm_op.metal b/Source/Serrano/operators/nn/common/batchnorm_op.metal index b0ded9d..b2e3abd 100644 --- a/Source/Serrano/operators/nn/common/batchnorm_op.metal +++ b/Source/Serrano/operators/nn/common/batchnorm_op.metal @@ -15,6 +15,7 @@ typedef struct { int channels; int inputWidth; int inputHeight; + float epsilon; } BatchNormInfo; void kernel batchNorm_inference(constant float* input [[ buffer(0) ]], @@ -38,5 +39,5 @@ void kernel batchNorm_inference(constant float* input [[ buffer(0) ]], inputOutputOffset = thread_id.z * info.inputHeight * info.inputWidth + thread_id.y * info.inputWidth + thread_id.x; } - output[inputOutputOffset] = scale[thread_id.z] * (input[inputOutputOffset] - mean[thread_id.z]) / var[thread_id.z] + offset[thread_id.z]; + output[inputOutputOffset] = scale[thread_id.z] * (input[inputOutputOffset] - mean[thread_id.z]) / sqrt(var[thread_id.z] + info.epsilon) + offset[thread_id.z]; } diff --git a/Source/Serrano/operators/nn/common/batchnorm_op.swift b/Source/Serrano/operators/nn/common/batchnorm_op.swift index fd024eb..8f5c738 100644 --- a/Source/Serrano/operators/nn/common/batchnorm_op.swift +++ b/Source/Serrano/operators/nn/common/batchnorm_op.swift @@ -25,13 +25,15 @@ public struct BatchNormInfo { var channels: MetalInt var inputWidth: MetalInt var inputHeight: MetalInt + var epsilon: MetalFloat - public static func makeBatchNormInfo(channelOrder: TensorChannelOrder, input: Tensor) -> BatchNormInfo { + public static func makeBatchNormInfo(channelOrder: TensorChannelOrder, epsilon: Float, input: Tensor) -> BatchNormInfo { let (channel, height, width) = parseImgChannelShapeInfo(channelOrder, shapeArray: input.shape.shapeArray) return BatchNormInfo(channelPosition: channelOrder.rawValue.metalShort, channels: channel.metalInt, inputWidth: width.metalInt, - inputHeight: height.metalInt) + inputHeight: height.metalInt, + epsilon: epsilon.metalFloat) } } @@ -313,6 +315,10 @@ public class BatchNormOperator: ComputableOperator { /// CPU in inference internal func cpu_inference() { + //TODO: FIX ERROR + // sqrt(var + epsilon) +// let + // get reciprocal of movingVar let movingVarienceReciprocal = Tensor(repeatingValue: 0.0, tensorShape: self.movingVar!.shape) var count = Int32(self.movingVar!.count) @@ -421,7 +427,7 @@ public class BatchNormOperator: ComputableOperator { let inputBufferResource = input.gpuBufferResource() let outputBufferResource = output.gpuBufferResource() - var info = BatchNormInfo.makeBatchNormInfo(channelOrder: self.channelOrder, input: input) + var info = BatchNormInfo.makeBatchNormInfo(channelOrder: self.channelOrder, epsilon: self.epsilon, input: input) let encoder = commandBuffer!.makeComputeCommandEncoder() encoder.setComputePipelineState(kernel!) diff --git a/docs/doc_generate.sh b/docs/doc_generate.sh index d1f474b..639db94 100755 --- a/docs/doc_generate.sh +++ b/docs/doc_generate.sh @@ -5,11 +5,11 @@ jazzy \ --author pcpLiu \ --author_url https://github.com/pcpLiu \ --github_url https://github.com/pcpLiu/Serrano \ - --github-file-prefix https://github.com/pcpLiu/Serrano/tree/v0.1.2-alpha \ - --module-version v0.1.2-alpha \ + --github-file-prefix https://github.com/pcpLiu/Serrano/tree/v0.1.5-alpha \ + --module-version v0.1.5-alpha \ --xcodebuild-arguments -scheme,SerranoFramework \ --module Serrano \ --min-acl internal \ - --root-url http://serrano-lib.org/docs/v0.1.2-alpha/api/ \ - --output generated_docs/v0.1.2-alpha/api/ \ + --root-url http://serrano-lib.org/docs/v0.1.5-alpha/api/ \ + --output generated_docs/v0.1.5-alpha/api/ \ --theme fullwidth \ No newline at end of file diff --git a/docs/operators/batchnorm.md b/docs/operators/batchnorm.md new file mode 100644 index 0000000..391b785 --- /dev/null +++ b/docs/operators/batchnorm.md @@ -0,0 +1,14 @@ +Currently, Serrano supports 2D BatchNormalization ([API](http://serrano-lib.org/docs/latest/api/Classes/BatchNormOperator.html)). + +**Notes** + +- The `inputTensors` and `outputTensors` should have same number of tensors. +- There can be multiple tensors in `inputTensors` and all tensors in `inputTensors` should have same shapes. Operator will do calculation for each input tensor independently. + +## Initialization + +```swift +let bn = BatchNormOperator(channelOrder = TensorChannelOrder.Last) +``` + +- `channelOrder`. The feature channel.