Conditionally check for LABEL_PREFIX

As it is only used for x86_64 and ARM64 in longlong_asm_gnu.h, only
check the prefix when host is either of these architectures.

configure.ac

@@ -1396,12 +1396,16 @@ then
 
     GMP_FINISH
 else
-    # NOTE: LSYM_PREFIX is needed for longlong_asm_gnu.h
-    GMP_INIT(tmpconf)
-    GMP_ASM_TEXT
-    GMP_ASM_LABEL_SUFFIX
-    GMP_ASM_LSYM_PREFIX
-    rm -f $gmp_tmpconfigm4 $gmp_tmpconfigm4i $gmp_tmpconfigm4p
+    # LSYM_PREFIX is still needed for x86-64 and ARM64 in longlong_asm_gnu.h
+    case $host in
+        ARM64_PATTERN | X86_64_PATTERN)
+            GMP_INIT(tmpconf)
+            GMP_ASM_TEXT
+            GMP_ASM_LABEL_SUFFIX
+            GMP_ASM_LSYM_PREFIX
+            rm -f $gmp_tmpconfigm4 $gmp_tmpconfigm4i $gmp_tmpconfigm4p
+            ;;
+    esac
 fi
 
 if test "$enable_assembly" = "yes";