From 907cd949c77bf6271baf75fb5bbdbba699ce80af Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Thu, 11 Apr 2024 11:56:21 -0700 Subject: [PATCH 1/3] shift principle points by .5 --- docs/quickstart/data_conventions.md | 4 ++++ nerfstudio/data/datamanagers/full_images_datamanager.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/docs/quickstart/data_conventions.md b/docs/quickstart/data_conventions.md index a929ff53ff..fe7dfe2cb9 100644 --- a/docs/quickstart/data_conventions.md +++ b/docs/quickstart/data_conventions.md @@ -12,6 +12,10 @@ We use the OpenGL/Blender (and original NeRF) coordinate convention for cameras. Our world space is oriented such that the up vector is +Z. The XY plane is parallel to the ground plane. In the viewer, you'll notice that red, green, and blue vectors correspond to X, Y, and Z respectively. +### Pixel coordinates + +We assume coordinates correspond to the centers of pixels (e.g. generating a ray for pixel (0,0) will shoot the ray through the center of that pixel). This aligns more with graphics conventions, and is distinct from OpenCV where the corners are aligned with the pixel coordinate. +
## Dataset format diff --git a/nerfstudio/data/datamanagers/full_images_datamanager.py b/nerfstudio/data/datamanagers/full_images_datamanager.py index 38207affb7..01ec4216f9 100644 --- a/nerfstudio/data/datamanagers/full_images_datamanager.py +++ b/nerfstudio/data/datamanagers/full_images_datamanager.py @@ -348,7 +348,13 @@ def _undistort_image( ] ) if np.any(distortion_params): + # because OpenCV expects the pixel coord to be top-left, we need to shift the principal point by 0.5 + # see https://github.com/nerfstudio-project/nerfstudio/issues/3048 + K[0, 2] = K[0, 2] - 0.5 + K[1, 2] = K[1, 2] - 0.5 newK, roi = cv2.getOptimalNewCameraMatrix(K, distortion_params, (image.shape[1], image.shape[0]), 0) + newK[0, 2] = newK[0, 2] + 0.5 + newK[1, 2] = newK[1, 2] + 0.5 image = cv2.undistort(image, K, distortion_params, None, newK) # type: ignore else: newK = K From f2e76117439f1d62824413e05e9b2a9d40305ab3 Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Thu, 11 Apr 2024 12:13:33 -0700 Subject: [PATCH 2/3] fix ordering --- nerfstudio/data/datamanagers/full_images_datamanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/data/datamanagers/full_images_datamanager.py b/nerfstudio/data/datamanagers/full_images_datamanager.py index 01ec4216f9..e513fbc732 100644 --- a/nerfstudio/data/datamanagers/full_images_datamanager.py +++ b/nerfstudio/data/datamanagers/full_images_datamanager.py @@ -353,9 +353,9 @@ def _undistort_image( K[0, 2] = K[0, 2] - 0.5 K[1, 2] = K[1, 2] - 0.5 newK, roi = cv2.getOptimalNewCameraMatrix(K, distortion_params, (image.shape[1], image.shape[0]), 0) + image = cv2.undistort(image, K, distortion_params, None, newK) # type: ignore newK[0, 2] = newK[0, 2] + 0.5 newK[1, 2] = newK[1, 2] + 0.5 - image = cv2.undistort(image, K, distortion_params, None, newK) # type: ignore else: newK = K roi = 0, 0, image.shape[1], image.shape[0] From 34f203a87b5113f20f0075dc3d949c937432429b Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Thu, 11 Apr 2024 12:17:50 -0700 Subject: [PATCH 3/3] add fisheye offset --- .../data/datamanagers/full_images_datamanager.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nerfstudio/data/datamanagers/full_images_datamanager.py b/nerfstudio/data/datamanagers/full_images_datamanager.py index e513fbc732..67ff126dbb 100644 --- a/nerfstudio/data/datamanagers/full_images_datamanager.py +++ b/nerfstudio/data/datamanagers/full_images_datamanager.py @@ -347,15 +347,13 @@ def _undistort_image( 0, ] ) + # because OpenCV expects the pixel coord to be top-left, we need to shift the principal point by 0.5 + # see https://github.com/nerfstudio-project/nerfstudio/issues/3048 + K[0, 2] = K[0, 2] - 0.5 + K[1, 2] = K[1, 2] - 0.5 if np.any(distortion_params): - # because OpenCV expects the pixel coord to be top-left, we need to shift the principal point by 0.5 - # see https://github.com/nerfstudio-project/nerfstudio/issues/3048 - K[0, 2] = K[0, 2] - 0.5 - K[1, 2] = K[1, 2] - 0.5 newK, roi = cv2.getOptimalNewCameraMatrix(K, distortion_params, (image.shape[1], image.shape[0]), 0) image = cv2.undistort(image, K, distortion_params, None, newK) # type: ignore - newK[0, 2] = newK[0, 2] + 0.5 - newK[1, 2] = newK[1, 2] + 0.5 else: newK = K roi = 0, 0, image.shape[1], image.shape[0] @@ -373,9 +371,13 @@ def _undistort_image( mask = torch.from_numpy(mask).bool() if len(mask.shape) == 2: mask = mask[:, :, None] + newK[0, 2] = newK[0, 2] + 0.5 + newK[1, 2] = newK[1, 2] + 0.5 K = newK elif camera.camera_type.item() == CameraType.FISHEYE.value: + K[0, 2] = K[0, 2] - 0.5 + K[1, 2] = K[1, 2] - 0.5 distortion_params = np.array( [distortion_params[0], distortion_params[1], distortion_params[2], distortion_params[3]] ) @@ -394,6 +396,8 @@ def _undistort_image( mask = torch.from_numpy(mask).bool() if len(mask.shape) == 2: mask = mask[:, :, None] + newK[0, 2] = newK[0, 2] + 0.5 + newK[1, 2] = newK[1, 2] + 0.5 K = newK elif camera.camera_type.item() == CameraType.FISHEYE624.value: fisheye624_params = torch.cat(