diff --git a/augly/assets/tests/image/dfdc_expected_output/test_distort_barrel.png b/augly/assets/tests/image/dfdc_expected_output/test_distort_barrel.png new file mode 100644 index 00000000..7b372bde Binary files /dev/null and b/augly/assets/tests/image/dfdc_expected_output/test_distort_barrel.png differ diff --git a/augly/assets/tests/image/dfdc_expected_output/test_distort_pincushion.png b/augly/assets/tests/image/dfdc_expected_output/test_distort_pincushion.png new file mode 100644 index 00000000..650cb0c9 Binary files /dev/null and b/augly/assets/tests/image/dfdc_expected_output/test_distort_pincushion.png differ diff --git a/augly/image/__init__.py b/augly/image/__init__.py index 78562070..e2353157 100644 --- a/augly/image/__init__.py +++ b/augly/image/__init__.py @@ -13,6 +13,8 @@ contrast, convert_color, crop, + distort_barrel, + distort_pincushion, encoding_quality, grayscale, hflip, @@ -50,6 +52,8 @@ contrast_intensity, convert_color_intensity, crop_intensity, + distort_barrel_intensity, + distort_pincushion_intensity, encoding_quality_intensity, grayscale_intensity, hflip_intensity, @@ -86,6 +90,8 @@ Contrast, ConvertColor, Crop, + DistortBarrel, + DistortPincushion, EncodingQuality, Grayscale, HFlip, @@ -131,6 +137,8 @@ "Contrast", "ConvertColor", "Crop", + "DistortBarrel", + "DistortPincushion", "EncodingQuality", "Grayscale", "HFlip", @@ -173,6 +181,8 @@ "contrast", "convert_color", "crop", + "distort_barrel", + "distort_pincushion", "encoding_quality", "grayscale", "hflip", @@ -207,6 +217,8 @@ "contrast_intensity", "convert_color_intensity", "crop_intensity", + "distort_barrel_intensity", + "distort_pincushion_intensity", "encoding_quality_intensity", "grayscale_intensity", "hflip_intensity", diff --git a/augly/image/functional.py b/augly/image/functional.py index 171b7206..37efe4eb 100644 --- a/augly/image/functional.py +++ b/augly/image/functional.py @@ -621,6 +621,172 @@ def crop( return imutils.ret_and_save_image(aug_image, output_path) +def distort_barrel( + image: Union[str, Image.Image], + output_path: Optional[str] = None, + a: float = 0.0, + b: float = 0.0, + c: float = 0.0, + d: float = 1.0, + metadata: Optional[List[Dict[str, Any]]] = None, + bboxes: Optional[List[Tuple]] = None, + bbox_format: Optional[str] = None, +) -> Image.Image: + """ + Applies barrel distortion to the image with the following equation. + + To see effects of the coefficients in detail refer to + https://legacy.imagemagick.org/Usage/distorts/#barrel. + Below is a direct quotation from the document describing how barrel distortion + parameter works. + + > The values basically form a distortion equation such that... + + Rsrc = r * ( A*r3 + B*r2 + C*r + D ) + + Where "r" is the destination radius and "Rsrc" is the source pixel to get the + pixel color from. the radii are normalized so that radius = '1.0' for the half + minimum width or height of the input image. This may seem reversed but that is + because the Reverse Pixel Mapping technique is used to ensure complete coverage + of the resulting image. + + Setting A = B = C = 0 and D = 1 results in no change in the input image. Negative + values of A, B and C will result in reverse barrel effect closer to pincushion + effect. + + @param image: the path to an image or a variable of type PIL.Image.Image + to be augmented + + @param output_path: the path in which the resulting image will be stored. + If None, the resulting PIL Image will still be returned + + @param a: Coefficient A in the equation Rsrc(r). Larger values results in more + barrel effect, has higher effect than b and c. + + @param b: Coefficient B in the equation Rsrc(r). Larger values results in more + barrel effect, has lower effect than a and higher effect than c. + + @param c: Coefficient C in the equation Rsrc(r). Larger values results in more + barrel effect, has lower effect than a and b. + + @param d: Coefficient D in the equation Rsrc(r). Controls the overall scaling of the + image. In a positive domain, values larger than 1 will shrink the image. Negative + values would result in both vertically and horizontally flipped image scaled in a + mirrored way of positive domain. + + @param metadata: if set to be a list, metadata about the function execution + including its name, the source & dest width, height, etc. will be appended + to the inputted list. If set to None, no metadata will be appended or returned + + @param bboxes: a list of bounding boxes can be passed in here if desired. If + provided, this list will be modified in place such that each bounding box is + transformed according to this function + + @param bbox_format: signifies what bounding box format was used in `bboxes`. Must + specify `bbox_format` if `bboxes` is provided. Supported bbox_format values are + "pascal_voc", "pascal_voc_norm", "coco", and "yolo" + + @returns: the augmented PIL Image + """ + image = imutils.validate_and_load_image(image).convert("RGB") + func_kwargs = imutils.get_func_kwargs(metadata, locals()) + + aug_image = imutils.distort( + image=image, + method="barrel", + distortion_args=(a, b, c, d), + ) + + imutils.get_metadata( + metadata=metadata, + function_name="distort_barrel", + aug_image=aug_image, + **func_kwargs, + ) + + return imutils.ret_and_save_image(aug_image, output_path) + + +def distort_pincushion( + image: Union[str, Image.Image], + output_path: Optional[str] = None, + a: float = 0.0, + b: float = 0.0, + c: float = 0.0, + d: float = 1.0, + metadata: Optional[List[Dict[str, Any]]] = None, + bboxes: Optional[List[Tuple]] = None, + bbox_format: Optional[str] = None, +) -> Image.Image: + """ + To see effects of the coefficients in detail refer to + https://legacy.imagemagick.org/Usage/distorts/#barrelinverse. Below is a direct + quotation from the document describing how pincushion (barrel inverse) + distortion parameter works. + + > The 'BarrelInverse' distortion method is very similar to the previous + Barrel Distortion distortion method, and in fact takes the same set of arguments. + However the formula that is applied is slightly different, with the main part of + the equation dividing the radius. that is the Equation has been inverted. + + Rsrc = r / ( A*r3 + B*r2 + C*r + D ) + + NOTE: This equation does NOT produce the 'reverse' the 'Barrel' distortion. + You can NOT use it to 'undo' the previous distortion. + + @param image: the path to an image or a variable of type PIL.Image.Image + to be augmented + + @param output_path: the path in which the resulting image will be stored. + If None, the resulting PIL Image will still be returned + + @param a: Coefficient A in the equation Rsrc(r). Larger values results in more + pincushion effect, has higher effect than b and c. + + @param b: Coefficient B in the equation Rsrc(r). Larger values results in more + pincushion effect, has lower effect than a and higher effect than c. + + @param c: Coefficient C in the equation Rsrc(r). Larger values results in more + pincushion effect, has lower effect than a and b. + + @param d: Coefficient D in the equation Rsrc(r). Controls the overall scaling of + the image. In a positive domain, values larger than 1 will enlarge the image + (zoomed in). Negative values would result in both vertically and horizontally + flipped image scaled in a mirrored way of positive domain. + + @param metadata: if set to be a list, metadata about the function execution + including its name, the source & dest width, height, etc. will be appended + to the inputted list. If set to None, no metadata will be appended or returned + + @param bboxes: a list of bounding boxes can be passed in here if desired. If + provided, this list will be modified in place such that each bounding box is + transformed according to this function + + @param bbox_format: signifies what bounding box format was used in `bboxes`. Must + specify `bbox_format` if `bboxes` is provided. Supported bbox_format values are + "pascal_voc", "pascal_voc_norm", "coco", and "yolo" + + @returns: the augmented PIL Image + """ + image = imutils.validate_and_load_image(image).convert("RGB") + func_kwargs = imutils.get_func_kwargs(metadata, locals()) + + aug_image = imutils.distort( + image=image, + method="barrel_inverse", + distortion_args=(a, b, c, d), + ) + + imutils.get_metadata( + metadata=metadata, + function_name="distort_pincushion", + aug_image=aug_image, + **func_kwargs, + ) + + return imutils.ret_and_save_image(aug_image, output_path) + + def encoding_quality( image: Union[str, Image.Image], output_path: Optional[str] = None, diff --git a/augly/image/intensity.py b/augly/image/intensity.py index 8377f1a5..bddd2e94 100644 --- a/augly/image/intensity.py +++ b/augly/image/intensity.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Optional, Union, Tuple import augly.image.utils as imutils import numpy as np @@ -100,6 +100,14 @@ def crop_intensity(metadata: Dict[str, Any], **kwargs) -> float: return resize_intensity_helper(metadata) +def distort_barrel_intensity(a: float, b: float, c: float, d: float, **kwargs) -> float: + return distort_intensity_helper(coefficients=(a, b, c), scale=d) + + +def distort_pincushion_intensity(a: float, b: float, c: float, d: float, **kwargs) -> float: + return distort_intensity_helper(coefficients=(a, b, c), scale=d) + + def encoding_quality_intensity(quality: int, **kwargs): assert ( isinstance(quality, int) and 0 <= quality <= 100 @@ -320,6 +328,13 @@ def normalize_mult_factor(factor: float) -> float: return factor if factor >= 1 else 1 / factor +def distort_intensity_helper(coefficients: Tuple[float, float, float], scale: float) -> float: + coefficients_magnitude = np.abs(coefficients).sum() + adjusted_scale = np.exp(-(scale-1)**2) + intensity = 100 * (coefficients_magnitude / adjusted_scale) + return float(np.clip(intensity, 0, 100)) + + def mult_factor_intensity_helper(factor: float) -> float: factor = normalize_mult_factor(factor) max_factor = 10 diff --git a/augly/image/transforms.py b/augly/image/transforms.py index 98d2ba2d..78af5c67 100644 --- a/augly/image/transforms.py +++ b/augly/image/transforms.py @@ -258,6 +258,136 @@ def apply_transform( ) +class DistortBarrel(BaseTransform): + def __init__( + self, a: float = 0.0, b: float = 0.0, c: float = 0.0, d: float = 1.0, p: float = 1.0 + ): + """ + @param a: Coefficient A in the equation Rsrc(r). Larger values results in more + barrel effect, has higher effect than b and c. + + @param b: Coefficient B in the equation Rsrc(r). Larger values results in more + barrel effect, has lower effect than a and higher effect than c. + + @param c: Coefficient C in the equation Rsrc(r). Larger values results in more + barrel effect, has lower effect than a and b. + + @param d: Coefficient D in the equation Rsrc(r). Controls the overall scaling of + the image. In a positive domain, values larger than 1 will shrink the image. + Negative values would result in both vertically and horizontally flipped + image scaled in a mirrored way of positive domain. + + @param p: the probability of the transform being applied; default value is 1.0 + """ + super().__init__(p) + self.a = a + self.b = b + self.c = c + self.d = d + + def apply_transform( + self, + image: Image.Image, + metadata: Optional[List[Dict[str, Any]]] = None, + bboxes: Optional[List[Tuple]] = None, + bbox_format: Optional[str] = None, + ) -> Image.Image: + """ + Applies barrel distortion to the image + + @param image: PIL Image to be augmented + + @param metadata: if set to be a list, metadata about the function execution + including its name, the source & dest width, height, etc. will be appended to + the inputted list. If set to None, no metadata will be appended or returned + + @param bboxes: a list of bounding boxes can be passed in here if desired. If + provided, this list will be modified in place such that each bounding box is + transformed according to this function + + @param bbox_format: signifies what bounding box format was used in `bboxes`. Must + specify `bbox_format` if `bboxes` is provided. Supported bbox_format values + are "pascal_voc", "pascal_voc_norm", "coco", and "yolo" + + @returns: Augmented PIL Image + """ + return F.distort_barrel( + image, + a=self.a, + b=self.b, + c=self.c, + d=self.d, + metadata=metadata, + bboxes=bboxes, + bbox_format=bbox_format + ) + + +class DistortPincushion(BaseTransform): + def __init__( + self, a: float = 0.0, b: float = 0.0, c: float = 0.0, d: float = 1.0, p: float = 1.0 + ): + """ + @param a: Coefficient A in the equation Rsrc(r). Larger values results in more + pincushion effect, has higher effect than b and c. + + @param b: Coefficient B in the equation Rsrc(r). Larger values results in more + pincushion effect, has higher effect than b and c. + + @param c: Coefficient C in the equation Rsrc(r). Larger values results in more + pincushion effect, has higher effect than b and c. + + @param d: Coefficient D in the equation Rsrc(r). Controls the overall scaling of + the image. In a positive domain, values larger than 1 will enlarge the image + (zoomed in). Negative values would result in both vertically and horizontally + flipped image scaled in a mirrored way of positive domain. + + @param p: the probability of the transform being applied; default value is 1.0 + """ + super().__init__(p) + self.a = a + self.b = b + self.c = c + self.d = d + + def apply_transform( + self, + image: Image.Image, + metadata: Optional[List[Dict[str, Any]]] = None, + bboxes: Optional[List[Tuple]] = None, + bbox_format: Optional[str] = None, + ) -> Image.Image: + """ + Applies pinchusion distortion to the image + + @param image: PIL Image to be augmented + + @param metadata: if set to be a list, metadata about the function execution + including its name, the source & dest width, height, etc. will be appended to + the inputted list. If set to None, no metadata will be appended or returned + + @param bboxes: a list of bounding boxes can be passed in here if desired. If + provided, this list will be modified in place such that each bounding box is + transformed according to this function + + @param bbox_format: signifies what bounding box format was used in `bboxes`. Must + specify `bbox_format` if `bboxes` is provided. Supported bbox_format values + are "pascal_voc", "pascal_voc_norm", "coco", and "yolo" + + @returns: Augmented PIL Image + """ + return F.distort_pincushion( + image, + a=self.a, + b=self.b, + c=self.c, + d=self.d, + metadata=metadata, + bboxes=bboxes, + bbox_format=bbox_format + ) + + class Blur(BaseTransform): def __init__(self, radius: float = 2.0, p: float = 1.0): """ diff --git a/augly/image/utils/__init__.py b/augly/image/utils/__init__.py index 5b568b01..502e1614 100644 --- a/augly/image/utils/__init__.py +++ b/augly/image/utils/__init__.py @@ -5,6 +5,7 @@ from augly.image.utils.utils import ( compute_stripe_mask, compute_transform_coeffs, + distort, get_template_and_bbox, pad_with_black, resize_and_pad_to_given_size, @@ -21,6 +22,7 @@ "get_metadata", "compute_stripe_mask", "compute_transform_coeffs", + "distort", "get_template_and_bbox", "pad_with_black", "resize_and_pad_to_given_size", diff --git a/augly/image/utils/bboxes.py b/augly/image/utils/bboxes.py index 971f4312..f146dc12 100644 --- a/augly/image/utils/bboxes.py +++ b/augly/image/utils/bboxes.py @@ -26,6 +26,18 @@ def crop_bboxes_helper( ) +def distort_barrel_bboxes_helper(bbox: Tuple, **kwargs) -> Tuple: + raise NotImplementedError( + "Bounding box support has not yet been added to this augmentation", + ) + + +def distort_pincushion_bboxes_helper(bbox: Tuple, **kwargs) -> Tuple: + raise NotImplementedError( + "Bounding box support has not yet been added to this augmentation", + ) + + def hflip_bboxes_helper(bbox: Tuple, **kwargs) -> Tuple: """ When the src image is horizontally flipped, the bounding box also gets horizontally diff --git a/augly/image/utils/utils.py b/augly/image/utils/utils.py index 3ca82b4d..1264d903 100644 --- a/augly/image/utils/utils.py +++ b/augly/image/utils/utils.py @@ -9,6 +9,7 @@ import augly.utils as utils import numpy as np from PIL import Image +from wand.image import Image as wImage JPEG_EXTENSIONS = [".jpg", ".JPG", ".jpeg", ".JPEG"] @@ -221,3 +222,21 @@ def compute_stripe_mask( binary_mask = softmax_mask > (math.cos(math.pi * line_width) + 1) / 2 return binary_mask + + +def distort( + image: Image.Image, + method: str, + distortion_args: Tuple[float, float, float, float], +) -> Image.Image: + """ + Distorts the image with a specified type of distortion. This function wraps `Wand` + package `wand.image.Image.distort()` method to apply distortions. This function is + a helper function to apply lens distortions on the image, and it is not meant to be + used for methods explicitly written in AugLy. To see full set of distortion methods, + see https://docs.wand-py.org/en/0.5.3/wand/image.html#wand.image.DISTORTION_METHODS + """ + np_image = np.array(image) + w_image = wImage.from_array(np_image) + w_image.distort(method, distortion_args) + return Image.fromarray(np.array(w_image)) diff --git a/augly/tests/image_tests/functional_unit_test.py b/augly/tests/image_tests/functional_unit_test.py index 75f8f602..3c243a22 100644 --- a/augly/tests/image_tests/functional_unit_test.py +++ b/augly/tests/image_tests/functional_unit_test.py @@ -21,6 +21,12 @@ def test_blur(self): def test_brightness(self): self.evaluate_function(imaugs.brightness) + def test_distort_barrel(self): + self.evaluate_function(imaugs.distort_barrel, a=0.1) + + def test_distort_pincushion(self): + self.evaluate_function(imaugs.distort_pincushion, a=0.1) + def test_change_aspect_ratio(self): self.evaluate_function(imaugs.change_aspect_ratio) diff --git a/augly/tests/image_tests/transforms_unit_test.py b/augly/tests/image_tests/transforms_unit_test.py index 5e11c342..a9306aea 100644 --- a/augly/tests/image_tests/transforms_unit_test.py +++ b/augly/tests/image_tests/transforms_unit_test.py @@ -72,6 +72,28 @@ def test_ConvertColor(self): def test_Crop(self): self.evaluate_class(imaugs.Crop(), fname="crop") + def test_DistortBarrel(self): + try: + self.evaluate_class(imaugs.DistortBarrel(a=0.1), fname="distort_barrel") + except NotImplementedError: + pass + else: + self.assertTrue( + False, + "This augmentation doesn't have bounding box support, so should fail with NotImplementedError" + ) + + def test_DistortPincushion(self): + try: + self.evaluate_class(imaugs.DistortPincushion(a=0.1), fname="distort_pincushion") + except NotImplementedError: + pass + else: + self.assertTrue( + False, + "This augmentation doesn't have bounding box support, so should fail with NotImplementedError" + ) + def test_EncodingQuality(self): self.evaluate_class( imaugs.EncodingQuality(quality=30), fname="encoding_quality" diff --git a/augly/utils/expected_output/image_tests/expected_metadata.json b/augly/utils/expected_output/image_tests/expected_metadata.json index c761364e..4dcaa217 100644 --- a/augly/utils/expected_output/image_tests/expected_metadata.json +++ b/augly/utils/expected_output/image_tests/expected_metadata.json @@ -30,6 +30,36 @@ "src_width": 1920 } ], + "distort_barrel": [ + { + "a": 0.1, + "b": 0.0, + "c": 0.0, + "d": 1.0, + "dst_height": 1080, + "dst_width": 1920, + "intensity": 10.0, + "name": "distort_barrel", + "output_path": null, + "src_height": 1080, + "src_width": 1920 + } + ], + "distort_pincushion": [ + { + "a": 0.1, + "b": 0.0, + "c": 0.0, + "d": 1.0, + "dst_height": 1080, + "dst_width": 1920, + "intensity": 10.0, + "name": "distort_pincushion", + "output_path": null, + "src_height": 1080, + "src_width": 1920 + } + ], "blur": [ { "bbox_format": "yolo", diff --git a/requirements.txt b/requirements.txt index 4ac2a974..f6f1de9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ numpy>=1.19.5 Pillow>=8.2.0 python-magic>=0.4.22 regex>=2021.4.4 +Wand>=0.6.7