fix: Add in-memory parser option (gh-826)

README.md

@@ -60,7 +60,7 @@ For more information and all configuration properties, see [Quarkus HTTP Referen
 *   Kaoto camel components: **3ee2af43623923a5c5e09df6f3f70657e1ccd09f**
 *   Kaoto view definitions: **94aae37dee4356d51ac34bfb757eb43a85ad2c0a**
 *   Camel-connectors: **3.21.0**
-*   Camel-connectors: **3.21.0**
+*   Camel-kamelets: **3.21.0**
 
 #### Updating step resources
 The repository contains steps repositories zip files which are bundled with Kaoto-backend during building.

camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRestDSLParseCatalog.java

@@ -180,10 +180,16 @@ private static Step getConsumesStep() {
     }
 
     @Override
-    public ParseCatalog<Step> getParser(String url) {
+    public ParseCatalog<Step> getParser() {
         return new CamelRestDSLParser();
     }
 
+    @Override
+    public ParseCatalog<Step> getParser(String url) {
+        //We are not expecting to get anything from here
+        return new EmptyParseCatalog<>();
+    }
+
     @Override
     public ParseCatalog<Step> getParser(String url, String tag) {
         //We are not expecting to get anything from here

camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRouteParseCatalog.java

@@ -22,6 +22,12 @@ public ParseCatalog<Step> getParser(final String url, final String tag) {
     }
 
 
+    @Override
+    public ParseCatalog<Step> getParser() {
+        //We are not expecting to get Camel Operators from memory
+        return new EmptyParseCatalog<>();
+    }
+
     @Override
     public ParseCatalog<Step> getParser(final String url) {
         ParseCatalog<Step> parseCatalog = new JarParseCatalog<>(url);

camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalog.java

@@ -4,6 +4,7 @@
 import io.kaoto.backend.api.metadata.catalog.StepCatalogParser;
 import io.kaoto.backend.metadata.ParseCatalog;
 import io.kaoto.backend.metadata.parser.ClusterParseCatalog;
+import io.kaoto.backend.metadata.parser.EmptyParseCatalog;
 import io.kaoto.backend.metadata.parser.GitParseCatalog;
 import io.kaoto.backend.metadata.parser.JarParseCatalog;
 import io.kaoto.backend.metadata.parser.LocalFolderParseCatalog;
@@ -48,6 +49,12 @@ public ParseCatalog<Step> getParser(final String url, final String tag) {
         return parseCatalog;
     }
 
+    @Override
+    public ParseCatalog<Step> getParser() {
+        //We are not expecting to get Kamelets from memory
+        return new EmptyParseCatalog<>();
+    }
+
     @Override
     public ParseCatalog<Step> getParser(final String url) {
         ParseCatalog<Step> parseCatalog = new JarParseCatalog<>(url);

camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java

@@ -6,13 +6,16 @@
 import io.quarkus.test.junit.QuarkusTest;
 import jakarta.enterprise.inject.Instance;
 import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Stream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -25,52 +28,175 @@ public class StepCatalogTest {
     @Inject
     private StepCatalog mainStepCatalog;
 
+    //helping class with all information about version/repo/file for Kamelets
+    private record KameletInfo() {
+        static final String REPO = "https://github.com/apache/camel-kamelets";
+        //don't rename, update-resources.sh script uses/updates this
+        static final String KAMELET_VERSION = "3.21.0";
+        static final String TAG = "v" + KAMELET_VERSION;
+
+        static final String FILE = "camel-kamelets-" + KAMELET_VERSION + ".jar";
+        static final int EXPECT_NUMBER = 215;
+
+        static String getFileFromRepo() {
+            return String.format("%s/archive/refs/tags/%s.zip", REPO, TAG);
+        }
+
+        static String getFileFromCentral() {
+            return String.format(
+                    "https://repo1.maven.org/maven2/org/apache/camel/kamelets/camel-kamelets/%s/camel-kamelets-%s.jar",
+                    KAMELET_VERSION, KAMELET_VERSION);
+        }
+
+        static String getFileFromResources() {
+            return "resource://" + FILE;
+        }
+    }
+
+    //helping class with all information about version/repo/file for Kaoto component metadata
+    private record ComponentMetadataInfo() {
+        static final String REPO = "https://github.com/KaotoIO/camel-component-metadata";
+        static final String TAG = "test-202308";
+        static final String FILE = "camel-component-metadata.zip";
+        static final int EXPECT_REPO_NUMBER = 54;
+        static final int EXPECT_FILE_NUMBER = 54;
+
+        static String getFileFromRepo() {
+            return String.format("%s/archive/refs/tags/%s.zip", REPO, TAG);
+        }
+
+        static String getFileFromResources() {
+            return "resource://" + FILE;
+        }
+    }
+
+    //helping class with all information about version/repo/file for Camel connectors
+    private record CamelConnectorsInfo() {
+        //don't rename, update-resources.sh script uses/updates this
+        static final String CAMEL_CONNECTORS_VERSION = "3.21.0";
+        static final String FILE = "camel-connectors-" + CAMEL_CONNECTORS_VERSION + ".zip";
+
+        static final int EXPECT_NUMBER = 847;
+
+        static String getFileFromResources() {
+            return "resource://" + FILE;
+        }
+    }
+
+    //helping class with all information about in-memory steps
+    private record InMemoryStepsInfo() {
+        //can be more steps (also with different types) in the future
+        static final int CAMEL_REST_DSL_IN_MEMORY_STEPS = 11;
+
+        static int getAllInMemoryStepsNumber() {
+            return CAMEL_REST_DSL_IN_MEMORY_STEPS;
+        }
+
+    }
+
+    private static Stream<Arguments> provideTestParametersForGitRepoTest() {
+        return Stream.of(
+                Arguments.of(KameletInfo.REPO + ".git", KameletInfo.TAG, KameletInfo.EXPECT_NUMBER),
+                Arguments.of(ComponentMetadataInfo.REPO + ".git",
+                        ComponentMetadataInfo.TAG, ComponentMetadataInfo.EXPECT_REPO_NUMBER)
+        );
+    }
+
     @ParameterizedTest
-    @CsvSource({"https://github.com/KaotoIO/camel-component-metadata.git,test-202308",
-            "https://github.com/apache/camel-kamelets.git,v3.21.0"})
-    void testParseFromGit(String repo, String tag) {
+    @MethodSource("provideTestParametersForGitRepoTest")
+    void testParseFromGit(String repo, String tag, int expectNumbers) {
         StepRepository stepRepository = new StepRepository(
                 Optional.empty(),
                 Optional.of(List.of(
                         new GitRepository(repo, tag, true, "all"))),
                 Optional.empty());
-        tryLoadSteps(stepRepository, repo + ":" + tag);
+        tryLoadSteps(stepRepository, repo + ":" + tag, expectNumbers);
+    }
+
+    private static Stream<Arguments> provideTestParametersForJarRepoTest() {
+        return Stream.of(
+                //kamelets
+                Arguments.of(KameletInfo.getFileFromRepo(), KameletInfo.EXPECT_NUMBER),
+                Arguments.of(KameletInfo.getFileFromCentral(), KameletInfo.EXPECT_NUMBER),
+                Arguments.of(KameletInfo.getFileFromResources(), KameletInfo.EXPECT_NUMBER),
+                //component-metadata
+                Arguments.of(ComponentMetadataInfo.getFileFromRepo(), ComponentMetadataInfo.EXPECT_REPO_NUMBER),
+                Arguments.of(ComponentMetadataInfo.getFileFromResources(), ComponentMetadataInfo.EXPECT_FILE_NUMBER),
+                //connectors
+                Arguments.of(CamelConnectorsInfo.getFileFromResources(), CamelConnectorsInfo.EXPECT_NUMBER)
+        );
     }
 
     @ParameterizedTest
-    @ValueSource(strings = {
-            "https://github.com/KaotoIO/camel-component-metadata/archive/refs/tags/test-202308.zip",
-            "https://repo1.maven.org/maven2/org/apache/camel/kamelets/camel-kamelets/3.21.0/camel-kamelets-3.21.0.jar",
-            "resource://camel-component-metadata.zip",
-            "resource://camel-kamelets-3.21.0.jar",
-            "resource://camel-connectors-3.21.0.zip"
-    })
-    void testParseFromFile(String resource) {
+    @MethodSource("provideTestParametersForJarRepoTest")
+    void testParseFromJar(String resource, int expectNumbers) {
         StepRepository stepRepository = new StepRepository(
                 Optional.of(List.of(
                         new LocationRepository(resource, true, "all"))),
                 Optional.empty(),
                 Optional.empty());
-        tryLoadSteps(stepRepository, resource);
+        tryLoadSteps(stepRepository, resource, expectNumbers);
     }
 
-    private void tryLoadSteps(StepRepository stepRepository, String resource) {
+    @Test
+    void testSameResourceFromMoreRepositories() {
+        StepRepository stepRepository = new StepRepository(
+                Optional.of(List.of(
+                        new LocationRepository(KameletInfo.getFileFromCentral(), true, "all"),
+                        new LocationRepository(KameletInfo.getFileFromRepo(), true, "all"))),
+                Optional.of(List.of(
+                        new GitRepository(KameletInfo.REPO + ".git", KameletInfo.TAG, true, "all"))),
+                Optional.empty());
+        tryLoadSteps(stepRepository, KameletInfo.getFileFromCentral() + " + " + KameletInfo.getFileFromRepo(),
+                KameletInfo.EXPECT_NUMBER);
+    }
+
+    @Test
+    void testMultipleResource() {
+        StepRepository stepRepository = new StepRepository(
+                Optional.empty(),
+                Optional.of(List.of(
+                        new GitRepository(KameletInfo.REPO + ".git", KameletInfo.TAG, true, "all"),
+                        new GitRepository(ComponentMetadataInfo.REPO + ".git", ComponentMetadataInfo.TAG,
+                                true, "all"))),
+                Optional.empty());
+        tryLoadSteps(stepRepository, KameletInfo.REPO + " + " + ComponentMetadataInfo.REPO,
+                KameletInfo.EXPECT_NUMBER + ComponentMetadataInfo.EXPECT_REPO_NUMBER);
+    }
+
+    private void tryLoadSteps(StepRepository stepRepository, String resource, int expectNumbers) {
+        tryLoadSteps(stepRepository, resource, expectNumbers, false);
+    }
+
+    private void tryLoadSteps(StepRepository stepRepository, String resource, int expectNumbers,
+                              boolean includeInMemorySteps) {
         StepCatalog anotherStepCatalog = new StepCatalog();
+        anotherStepCatalog.includeInMemoryCatalogs = includeInMemorySteps;
         anotherStepCatalog.setStepCatalogParsers(stepCatalogParsers);
         anotherStepCatalog.setKclient(kclient);
         anotherStepCatalog.setRepository(stepRepository);
         anotherStepCatalog.loadParsers();
         anotherStepCatalog.warmUpCatalog();
         Collection<Step> steps = anotherStepCatalog.getReadOnlyCatalog().getAll();
         assertThat(steps)
-                .isNotEmpty()
-                .as("Test that more than 50 steps was loaded from resource: " + resource)
-                .hasSizeGreaterThan(50)
+                .as("Test that all expected steps '" + expectNumbers + "' was loaded from resource: " + resource)
+                .hasSize(expectNumbers)
                 .as("Test that catalog contains steps only from resource '" + resource +
                         "' and not from different resources.")
                 .hasSizeLessThan(mainStepCatalog.getReadOnlyCatalog().getAll().size());
     }
 
+    @ParameterizedTest
+    @ValueSource(booleans = {true, false})
+    void testInMemorySteps(boolean shouldContains) {
+        StepRepository stepRepository = new StepRepository(
+                Optional.empty(),
+                Optional.empty(),
+                Optional.empty());
+        int numberOfSteps = shouldContains ? InMemoryStepsInfo.getAllInMemoryStepsNumber() : 0;
+        tryLoadSteps(stepRepository, "In memory steps (loaded from no repo)", numberOfSteps, shouldContains);
+    }
+
 
     // GitRepository implementation used for testing
     private record GitRepository(String url, String tag, boolean ifNoCluster, String kind) implements Repository.Git {

camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalogTest.java

@@ -41,6 +41,7 @@ class KameletParseCatalogTest {
 
     private static final String FILE_NAME = "camel-kamelets-%s.jar";
 
+    //don't rename, update-resources.sh script uses/updates this
     private static final String VERSION = "3.21.0";
 
     private static final Logger LOG = Logger.getLogger(KameletParseCatalogTest.class);

camel-support/src/test/java/io/kaoto/backend/camel/service/step/parser/kamelet/KameletStepParserServiceTest.java

@@ -39,6 +39,7 @@
 @QuarkusTest
 class KameletStepParserServiceTest {
 
+    //don't rename, update-resources.sh script uses/updates this
     private static final String VERSION = "3.21.0";
 
     private static String kamelet;

catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalog.java

@@ -17,6 +17,7 @@
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.enterprise.inject.Instance;
 import jakarta.inject.Inject;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
 
 /**
  * 🐱class StepCatalog
@@ -29,6 +30,10 @@
 @ApplicationScoped
 public class StepCatalog extends AbstractCatalog<Step> {
 
+    @ConfigProperty(name = "kaoto.step.catalog.include.inmemory.catalogs",
+            defaultValue = "true")
+    boolean includeInMemoryCatalogs;
+
     public static final String ALL = "all";
     private StepRepository repository;
 
@@ -45,7 +50,7 @@ protected List<ParseCatalog<Step>> loadParsers() {
         addZipJar(catalogs, clusterAvailable);
         addLocalFolder(catalogs, clusterAvailable);
         addGit(catalogs, clusterAvailable);
-
+        if (includeInMemoryCatalogs) addInMemoryParsers(catalogs);
         return catalogs;
     }
 
@@ -95,10 +100,10 @@ private void addLocalFolder(final List<ParseCatalog<Step>> catalogs, final boole
                 //And call only the parsers that apply
                 .flatMap(folder -> stepCatalogParsers.stream().parallel()
                     .filter(parser -> ALL.equalsIgnoreCase(folder.kind()) || parser.generatesKind(folder.kind()))
-                        .map(parser -> {
-                            File dir = new File(folder.url());
-                            return parser.getLocalFolder(dir.toPath());
-                        })
+                    .map(parser -> {
+                        File dir = new File(folder.url());
+                        return parser.getLocalFolder(dir.toPath());
+                    })
                 ).toList();
 
         catalogs.addAll(items);
@@ -119,6 +124,12 @@ private void addZipJar(final List<ParseCatalog<Step>> catalogs, final boolean cl
 
     }
 
+    private void addInMemoryParsers(final List<ParseCatalog<Step>> catalogs) {
+        List<ParseCatalog<Step>> items = stepCatalogParsers.stream().parallel().map(StepCatalogParser::getParser)
+                .toList();
+        catalogs.addAll(items);
+    }
+
     @Inject
     public void setRepository(final StepRepository repo) {
         this.repository = repo;

catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalogParser.java

@@ -14,6 +14,15 @@
  */
 public interface StepCatalogParser {
 
+    /*
+     * 🐱method getParser : ParseCatalog
+     *
+     * Loads all the elements from memory(code).
+     *
+     */
+    @WithSpan
+    ParseCatalog<Step> getParser();
+
     /*
      * 🐱method getParser : ParseCatalog
      * 🐱param url : String