ceildiv in layernorm

czzhao-sjtu · Apr 11, 2024 · 00b9fda · 00b9fda
1 parent 089cd5f
commit 00b9fda
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/dev/cuda/layernorm_forward.cu b/dev/cuda/layernorm_forward.cu
@@ -272,7 +272,7 @@ void layernorm_forward3(float* out, float* mean, float* rstd,
                        const int block_size) {
     assert(block_size % 32 == 0);
     const int N = B * T;
-    const int grid_size = N * 32 / block_size;
+    const int grid_size = CEIL_DIV(N * 32, block_size);
     layernorm_forward_kernel3<<<grid_size, block_size>>>(out, mean, rstd, inp, weight, bias, N, C);
     cudaCheck(cudaGetLastError());
 }