From 0ad9d0edd740b664c8762c8fc635ec0bffb5b09b Mon Sep 17 00:00:00 2001 From: Rist115 Date: Thu, 24 Nov 2022 10:21:17 +0900 Subject: [PATCH 01/10] add ratio in ms_deform_attn_ --- mmcv/ops/multi_scale_deform_attn.py | 6 ++++-- tests/test_ops/test_ms_deformable_attn.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mmcv/ops/multi_scale_deform_attn.py b/mmcv/ops/multi_scale_deform_attn.py index 509ae5f983..18a6ce4a34 100644 --- a/mmcv/ops/multi_scale_deform_attn.py +++ b/mmcv/ops/multi_scale_deform_attn.py @@ -189,6 +189,7 @@ def __init__(self, num_heads: int = 8, num_levels: int = 4, num_points: int = 4, + value_proj_ratio: float = 1.0, im2col_step: int = 64, dropout: float = 0.1, batch_first: bool = False, @@ -228,8 +229,9 @@ def _is_power_of_2(n): embed_dims, num_heads * num_levels * num_points * 2) self.attention_weights = nn.Linear(embed_dims, num_heads * num_levels * num_points) - self.value_proj = nn.Linear(embed_dims, embed_dims) - self.output_proj = nn.Linear(embed_dims, embed_dims) + self.value_proj_size = int(embed_dims * value_proj_ratio) + self.value_proj = nn.Linear(embed_dims, self.value_proj_size) + self.output_proj = nn.Linear(self.value_proj_size, embed_dims) self.init_weights() def init_weights(self) -> None: diff --git a/tests/test_ops/test_ms_deformable_attn.py b/tests/test_ops/test_ms_deformable_attn.py index 94223a6420..f85e0c0594 100644 --- a/tests/test_ops/test_ms_deformable_attn.py +++ b/tests/test_ops/test_ms_deformable_attn.py @@ -54,6 +54,19 @@ def test_multiscale_deformable_attention(device): spatial_shapes=spatial_shapes, level_start_index=level_start_index) + # test with value_spatial_shapes + msda = MultiScaleDeformableAttention( + embed_dims=3, num_levels=2, num_heads=3, value_spatial_shapes=0.5) + msda.init_weights() + msda.to(device) + msda( + query, + key, + key, + reference_points=reference_points, + spatial_shapes=spatial_shapes, + level_start_index=level_start_index) + def test_forward_multi_scale_deformable_attn_pytorch(): N, M, D = 1, 2, 2 From a02275f603d312040bdbb596499ca5715d450cfd Mon Sep 17 00:00:00 2001 From: Rist115 Date: Thu, 24 Nov 2022 10:53:20 +0900 Subject: [PATCH 02/10] add ratio in ms_deform_attn --- tests/test_ops/test_ms_deformable_attn.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_ops/test_ms_deformable_attn.py b/tests/test_ops/test_ms_deformable_attn.py index f85e0c0594..663454ef29 100644 --- a/tests/test_ops/test_ms_deformable_attn.py +++ b/tests/test_ops/test_ms_deformable_attn.py @@ -55,8 +55,16 @@ def test_multiscale_deformable_attention(device): level_start_index=level_start_index) # test with value_spatial_shapes + embed_dims = 6 + value_proj_ratio = 0.5 + query = torch.rand(num_query, bs, embed_dims).to(device) + key = torch.rand(num_query, bs, embed_dims).to(device) msda = MultiScaleDeformableAttention( - embed_dims=3, num_levels=2, num_heads=3, value_spatial_shapes=0.5) + embed_dims=embed_dims, + num_levels=2, + num_heads=3, + value_proj_ratio=value_proj_ratio) + assert msda.value_proj_size == int(embed_dims * value_proj_ratio) msda.init_weights() msda.to(device) msda( From 3e993c8ed3316f88cf0c6c0eb6c194900e83f0e2 Mon Sep 17 00:00:00 2001 From: takuoko Date: Thu, 24 Nov 2022 17:59:05 +0900 Subject: [PATCH 03/10] Update mmcv/ops/multi_scale_deform_attn.py Co-authored-by: Zaida Zhou <58739961+zhouzaida@users.noreply.github.com> --- mmcv/ops/multi_scale_deform_attn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmcv/ops/multi_scale_deform_attn.py b/mmcv/ops/multi_scale_deform_attn.py index 18a6ce4a34..5ef1b1eca7 100644 --- a/mmcv/ops/multi_scale_deform_attn.py +++ b/mmcv/ops/multi_scale_deform_attn.py @@ -229,8 +229,8 @@ def _is_power_of_2(n): embed_dims, num_heads * num_levels * num_points * 2) self.attention_weights = nn.Linear(embed_dims, num_heads * num_levels * num_points) - self.value_proj_size = int(embed_dims * value_proj_ratio) - self.value_proj = nn.Linear(embed_dims, self.value_proj_size) + value_proj_size = int(embed_dims * value_proj_ratio) + self.value_proj = nn.Linear(embed_dims, value_proj_size) self.output_proj = nn.Linear(self.value_proj_size, embed_dims) self.init_weights() From e008aa77e33da8ed61b29ff035888695efdecdff Mon Sep 17 00:00:00 2001 From: takuoko Date: Thu, 24 Nov 2022 17:59:17 +0900 Subject: [PATCH 04/10] Update tests/test_ops/test_ms_deformable_attn.py Co-authored-by: Zaida Zhou <58739961+zhouzaida@users.noreply.github.com> --- tests/test_ops/test_ms_deformable_attn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ops/test_ms_deformable_attn.py b/tests/test_ops/test_ms_deformable_attn.py index 663454ef29..aa4d5dcc22 100644 --- a/tests/test_ops/test_ms_deformable_attn.py +++ b/tests/test_ops/test_ms_deformable_attn.py @@ -54,7 +54,7 @@ def test_multiscale_deformable_attention(device): spatial_shapes=spatial_shapes, level_start_index=level_start_index) - # test with value_spatial_shapes + # test with value_proj_ratio embed_dims = 6 value_proj_ratio = 0.5 query = torch.rand(num_query, bs, embed_dims).to(device) From 2aac6deff5bfa7890b8d1e80ae1778b589b8fb0a Mon Sep 17 00:00:00 2001 From: Rist115 Date: Thu, 24 Nov 2022 18:02:30 +0900 Subject: [PATCH 05/10] add ratio in ms_deform_attn --- mmcv/ops/multi_scale_deform_attn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmcv/ops/multi_scale_deform_attn.py b/mmcv/ops/multi_scale_deform_attn.py index 5ef1b1eca7..e33f70d55d 100644 --- a/mmcv/ops/multi_scale_deform_attn.py +++ b/mmcv/ops/multi_scale_deform_attn.py @@ -180,6 +180,8 @@ class MultiScaleDeformableAttention(BaseModule): or (n, batch, embed_dim). Default to False. norm_cfg (dict): Config dict for normalization layer. Default: None. + value_proj_ratio (float): The expansion ratio of value_proj. + Default: 4. init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. Default: None. """ @@ -189,11 +191,11 @@ def __init__(self, num_heads: int = 8, num_levels: int = 4, num_points: int = 4, - value_proj_ratio: float = 1.0, im2col_step: int = 64, dropout: float = 0.1, batch_first: bool = False, norm_cfg: Optional[dict] = None, + value_proj_ratio: float = 1.0, init_cfg: Optional[mmengine.ConfigDict] = None): super().__init__(init_cfg) if embed_dims % num_heads != 0: From 3f04242ad3779edc308d9f0d4297f9dccbc845c1 Mon Sep 17 00:00:00 2001 From: Rist115 Date: Fri, 25 Nov 2022 15:24:27 +0900 Subject: [PATCH 06/10] add ratio in ms_deform_attn --- mmcv/ops/multi_scale_deform_attn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmcv/ops/multi_scale_deform_attn.py b/mmcv/ops/multi_scale_deform_attn.py index e33f70d55d..adf5620c30 100644 --- a/mmcv/ops/multi_scale_deform_attn.py +++ b/mmcv/ops/multi_scale_deform_attn.py @@ -195,8 +195,8 @@ def __init__(self, dropout: float = 0.1, batch_first: bool = False, norm_cfg: Optional[dict] = None, - value_proj_ratio: float = 1.0, - init_cfg: Optional[mmengine.ConfigDict] = None): + init_cfg: Optional[mmengine.ConfigDict] = None, + value_proj_ratio: float = 1.0): super().__init__(init_cfg) if embed_dims % num_heads != 0: raise ValueError(f'embed_dims must be divisible by num_heads, ' From e9726b0615badb9fa57f7f7b11931a8bc41f861a Mon Sep 17 00:00:00 2001 From: Rist115 Date: Fri, 25 Nov 2022 16:09:05 +0900 Subject: [PATCH 07/10] add ratio in ms_deform_attn --- tests/test_ops/test_ms_deformable_attn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_ops/test_ms_deformable_attn.py b/tests/test_ops/test_ms_deformable_attn.py index aa4d5dcc22..a29380552d 100644 --- a/tests/test_ops/test_ms_deformable_attn.py +++ b/tests/test_ops/test_ms_deformable_attn.py @@ -64,7 +64,6 @@ def test_multiscale_deformable_attention(device): num_levels=2, num_heads=3, value_proj_ratio=value_proj_ratio) - assert msda.value_proj_size == int(embed_dims * value_proj_ratio) msda.init_weights() msda.to(device) msda( From 78ef4d5930a6907c08b6cd0df3fdd282249b375f Mon Sep 17 00:00:00 2001 From: Rist115 Date: Fri, 25 Nov 2022 16:14:53 +0900 Subject: [PATCH 08/10] add ratio in ms_deform_attn --- mmcv/ops/multi_scale_deform_attn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmcv/ops/multi_scale_deform_attn.py b/mmcv/ops/multi_scale_deform_attn.py index adf5620c30..2075e7a037 100644 --- a/mmcv/ops/multi_scale_deform_attn.py +++ b/mmcv/ops/multi_scale_deform_attn.py @@ -180,10 +180,10 @@ class MultiScaleDeformableAttention(BaseModule): or (n, batch, embed_dim). Default to False. norm_cfg (dict): Config dict for normalization layer. Default: None. - value_proj_ratio (float): The expansion ratio of value_proj. - Default: 4. init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. Default: None. + value_proj_ratio (float): The expansion ratio of value_proj. + Default: 4. """ def __init__(self, From ed9c8f03f3840a6932305cc9ac0c8e30b8981674 Mon Sep 17 00:00:00 2001 From: Rist115 Date: Fri, 2 Dec 2022 10:30:40 +0900 Subject: [PATCH 09/10] add ratio in ms_deform_attn --- mmcv/ops/multi_scale_deform_attn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmcv/ops/multi_scale_deform_attn.py b/mmcv/ops/multi_scale_deform_attn.py index 2075e7a037..f8c8cbc661 100644 --- a/mmcv/ops/multi_scale_deform_attn.py +++ b/mmcv/ops/multi_scale_deform_attn.py @@ -233,7 +233,7 @@ def _is_power_of_2(n): num_heads * num_levels * num_points) value_proj_size = int(embed_dims * value_proj_ratio) self.value_proj = nn.Linear(embed_dims, value_proj_size) - self.output_proj = nn.Linear(self.value_proj_size, embed_dims) + self.output_proj = nn.Linear(value_proj_size, embed_dims) self.init_weights() def init_weights(self) -> None: From 64df8385d90f3770db174f3953e889b83d5cf4d2 Mon Sep 17 00:00:00 2001 From: Rist115 Date: Fri, 2 Dec 2022 10:35:53 +0900 Subject: [PATCH 10/10] add ratio in ms_deform_attn --- mmcv/ops/multi_scale_deform_attn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmcv/ops/multi_scale_deform_attn.py b/mmcv/ops/multi_scale_deform_attn.py index f8c8cbc661..c1d415621a 100644 --- a/mmcv/ops/multi_scale_deform_attn.py +++ b/mmcv/ops/multi_scale_deform_attn.py @@ -183,7 +183,7 @@ class MultiScaleDeformableAttention(BaseModule): init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. Default: None. value_proj_ratio (float): The expansion ratio of value_proj. - Default: 4. + Default: 1.0. """ def __init__(self,