diff --git a/packages/block-library/src/site-logo/index.php b/packages/block-library/src/site-logo/index.php
index 1127f4790a3a7f..5e1a1a09eded10 100644
--- a/packages/block-library/src/site-logo/index.php
+++ b/packages/block-library/src/site-logo/index.php
@@ -22,8 +22,8 @@ function render_block_core_site_logo( $attributes ) {
 	};
 
 	add_filter( 'wp_get_attachment_image_src', $adjust_width_height_filter );
-
 	$custom_logo = get_custom_logo();
+	remove_filter( 'wp_get_attachment_image_src', $adjust_width_height_filter );
 
 	if ( empty( $custom_logo ) ) {
 		return ''; // Return early if no custom logo is set, avoiding extraneous wrapper div.
@@ -56,7 +56,6 @@ function render_block_core_site_logo( $attributes ) {
 
 	$wrapper_attributes = get_block_wrapper_attributes( array( 'class' => implode( ' ', $classnames ) ) );
 	$html               = sprintf( '<div %s>%s</div>', $wrapper_attributes, $custom_logo );
-	remove_filter( 'wp_get_attachment_image_src', $adjust_width_height_filter );
 	return $html;
 }