diff --git a/implementation/src/main/java/io/smallrye/config/ConfigMappingInterface.java b/implementation/src/main/java/io/smallrye/config/ConfigMappingInterface.java
index e823c73de..1758a9752 100644
--- a/implementation/src/main/java/io/smallrye/config/ConfigMappingInterface.java
+++ b/implementation/src/main/java/io/smallrye/config/ConfigMappingInterface.java
@@ -824,8 +824,10 @@ private static Class<? extends Converter<?>> getConvertWith(final AnnotatedType
     private static void validateConverter(final Type type, final Class<? extends Converter<?>> convertWith) {
         if (type instanceof Class) {
             try {
+                Class<?> classType = (Class<?>) type;
+                Class<?> effectiveType = classType.isPrimitive() ? PrimitiveProperty.boxTypes.get(classType) : classType;
                 Method convertMethod = convertWith.getMethod("convert", String.class);
-                if (!((Class<?>) type).isAssignableFrom(convertMethod.getReturnType())) {
+                if (!effectiveType.isAssignableFrom(convertMethod.getReturnType())) {
                     throw new IllegalArgumentException();
                 }
             } catch (NoSuchMethodException e) {
diff --git a/implementation/src/test/java/io/smallrye/config/ConfigMappingInterfaceTest.java b/implementation/src/test/java/io/smallrye/config/ConfigMappingInterfaceTest.java
index ad40ad0f0..125ab0a9b 100644
--- a/implementation/src/test/java/io/smallrye/config/ConfigMappingInterfaceTest.java
+++ b/implementation/src/test/java/io/smallrye/config/ConfigMappingInterfaceTest.java
@@ -383,6 +383,13 @@ void converters() {
         final Converters converters = config.getConfigMapping(Converters.class);
 
         assertEquals("bar", converters.foo());
+        assertTrue(converters.bprim());
+        assertEquals('c', converters.cprim());
+        assertEquals(-1, converters.iprim());
+        assertEquals(-1, converters.sprim());
+        assertEquals(-1L, converters.lprim());
+        assertEquals(-1.0, converters.fprim());
+        assertEquals(-1.0, converters.dprim());
     }
 
     @Test
@@ -661,6 +668,27 @@ public interface ComplexSample {
     public interface Converters {
         @WithConverter(FooBarConverter.class)
         String foo();
+
+        @WithConverter(BooleanConverter.class)
+        boolean bprim();
+
+        @WithConverter(CharacterConverter.class)
+        char cprim();
+
+        @WithConverter(IntegerConverter.class)
+        int iprim();
+
+        @WithConverter(ShortConverter.class)
+        short sprim();
+
+        @WithConverter(LongConverter.class)
+        long lprim();
+
+        @WithConverter(FloatConverter.class)
+        float fprim();
+
+        @WithConverter(DoubleConverter.class)
+        double dprim();
     }
 
     public static class FooBarConverter implements Converter<String> {
@@ -670,6 +698,55 @@ public String convert(final String value) {
         }
     }
 
+    public static class BooleanConverter implements Converter<Boolean> {
+        @Override
+        public Boolean convert(String value) throws IllegalArgumentException, NullPointerException {
+            return true;
+        }
+    }
+
+    public static class CharacterConverter implements Converter<Character> {
+        @Override
+        public Character convert(String value) throws IllegalArgumentException, NullPointerException {
+            return 'c';
+        }
+    }
+
+    public static class IntegerConverter implements Converter<Integer> {
+        @Override
+        public Integer convert(String value) throws IllegalArgumentException, NullPointerException {
+            return -1;
+        }
+    }
+
+    public static class ShortConverter implements Converter<Short> {
+        @Override
+        public Short convert(String value) throws IllegalArgumentException, NullPointerException {
+            return -1;
+        }
+    }
+
+    public static class LongConverter implements Converter<Long> {
+        @Override
+        public Long convert(String value) throws IllegalArgumentException, NullPointerException {
+            return -1L;
+        }
+    }
+
+    public static class FloatConverter implements Converter<Float> {
+        @Override
+        public Float convert(String value) throws IllegalArgumentException, NullPointerException {
+            return -1.0F;
+        }
+    }
+
+    public static class DoubleConverter implements Converter<Double> {
+        @Override
+        public Double convert(String value) throws IllegalArgumentException, NullPointerException {
+            return -1.0;
+        }
+    }
+
     @ConfigMapping(prefix = "cloud")
     public interface ServerAnnotated {
         String host();
diff --git a/implementation/src/test/java/io/smallrye/config/ConfigMappingWithConverterTest.java b/implementation/src/test/java/io/smallrye/config/ConfigMappingWithConverterTest.java
index 95d37216e..f445ed6f1 100644
--- a/implementation/src/test/java/io/smallrye/config/ConfigMappingWithConverterTest.java
+++ b/implementation/src/test/java/io/smallrye/config/ConfigMappingWithConverterTest.java
@@ -52,11 +52,27 @@ public Integer convert(final String value) throws IllegalArgumentException, Null
         }
     }
 
+    @ConfigMapping
+    interface WrongPrimitiveConverterType {
+        @WithConverter(IntegerConverter.class)
+        double type();
+
+        class IntegerConverter implements Converter<Integer> {
+
+            @Override
+            public Integer convert(final String value) throws IllegalArgumentException, NullPointerException {
+                return 0;
+            }
+
+        }
+    }
+
     @Test
     void wrongConverter() {
         assertThrows(IllegalArgumentException.class, () -> config(WrongConverterType.class));
         assertThrows(IllegalArgumentException.class, () -> config(WrongAbstractConverterType.class));
         assertThrows(IllegalArgumentException.class, () -> config(WrongSuperConverterType.class));
+        assertThrows(IllegalArgumentException.class, () -> config(WrongPrimitiveConverterType.class));
     }
 
     @ConfigMapping