From 9a212e9f987a0992cccf8edc80ba3c0e4675d254 Mon Sep 17 00:00:00 2001 From: Jose Date: Fri, 23 Sep 2022 12:50:56 +0200 Subject: [PATCH] Fetch Helm dependencies before packaging Related to https://github.com/quarkiverse/quarkus-helm/issues/93 --- .../listener/HelmWriterSessionListener.java | 51 +++++++++---- docs/documentation/helm.md | 8 +-- .../src/main/resources/application.properties | 2 - .../example/HelmKubernetesExampleTest.java | 5 +- tests/issue-fetch-helm-dependencies/pom.xml | 72 +++++++++++++++++++ .../io/dekorate/example/DemoApplication.java | 28 ++++++++ .../io/dekorate/example/HelloController.java | 32 +++++++++ .../src/main/resources/application.properties | 7 ++ .../example/IssueHelmFetchDependencies.java | 46 ++++++++++++ tests/pom.xml | 1 + 10 files changed, 228 insertions(+), 24 deletions(-) create mode 100644 tests/issue-fetch-helm-dependencies/pom.xml create mode 100644 tests/issue-fetch-helm-dependencies/src/main/java/io/dekorate/example/DemoApplication.java create mode 100644 tests/issue-fetch-helm-dependencies/src/main/java/io/dekorate/example/HelloController.java create mode 100644 tests/issue-fetch-helm-dependencies/src/main/resources/application.properties create mode 100644 tests/issue-fetch-helm-dependencies/src/test/java/io/dekorate/example/IssueHelmFetchDependencies.java diff --git a/annotations/helm-annotations/src/main/java/io/dekorate/helm/listener/HelmWriterSessionListener.java b/annotations/helm-annotations/src/main/java/io/dekorate/helm/listener/HelmWriterSessionListener.java index defa47243..ad8863a9f 100644 --- a/annotations/helm-annotations/src/main/java/io/dekorate/helm/listener/HelmWriterSessionListener.java +++ b/annotations/helm-annotations/src/main/java/io/dekorate/helm/listener/HelmWriterSessionListener.java @@ -18,6 +18,7 @@ import static io.dekorate.helm.util.HelmTarArchiver.createTarBall; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -35,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -57,6 +57,7 @@ import io.dekorate.helm.model.Maintainer; import io.dekorate.helm.util.HelmExpressionParser; import io.dekorate.project.Project; +import io.dekorate.utils.Exec; import io.dekorate.utils.Maps; import io.dekorate.utils.Serialization; import io.dekorate.utils.Strings; @@ -129,9 +130,6 @@ public Map writeHelmFiles(Session session, Project project, valuesByProfile)); artifacts.putAll(createChartYaml(helmConfig, project, outputDir)); artifacts.putAll(createValuesYaml(helmConfig, valuesReferences, inputDir, outputDir, prodValues, valuesByProfile)); - if (helmConfig.isCreateTarFile()) { - artifacts.putAll(createTarball(helmConfig, project, outputDir, artifacts, valuesByProfile.keySet())); - } // To follow Helm file structure standards: artifacts.putAll(createEmptyChartFolder(helmConfig, outputDir)); @@ -139,6 +137,12 @@ public Map writeHelmFiles(Session session, Project project, artifacts.putAll(addResourceIfExists(helmConfig, LICENSE, inputDir, outputDir)); artifacts.putAll(addResourceIfExists(helmConfig, README, inputDir, outputDir)); + // Final step: packaging + if (helmConfig.isCreateTarFile()) { + fetchDependencies(helmConfig, outputDir); + artifacts.putAll(createTarball(helmConfig, project, outputDir, artifacts)); + } + } catch (IOException e) { throw new RuntimeException("Error writing resources", e); } @@ -147,6 +151,22 @@ public Map writeHelmFiles(Session session, Project project, return artifacts; } + private void fetchDependencies(HelmChartConfig helmConfig, Path outputDir) { + if (helmConfig.getDependencies() != null && helmConfig.getDependencies().length > 0) { + Path chartFolder = getChartOutputDir(helmConfig, outputDir); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + boolean success = Exec.inPath(chartFolder) + .redirectingOutput(out) + .commands("helm", "dependency", "build"); + + if (success) { + LOGGER.info("Dependencies successfully fetched"); + } else { + throw new RuntimeException("Error fetching Helm dependencies. Cause: " + new String(out.toByteArray())); + } + } + } + private void validateHelmConfig(HelmChartConfig helmConfig) { if (Strings.isNullOrEmpty(helmConfig.getName())) { throw new RuntimeException("Helm Chart name is required!"); @@ -309,7 +329,7 @@ private boolean startWithDependencyPrefix(String property, io.dekorate.helm.conf } private Map createTarball(HelmChartConfig helmConfig, Project project, Path outputDir, - Map artifacts, Set profiles) throws IOException { + Map artifacts) throws IOException { File tarballFile = outputDir.resolve(String.format("%s-%s-%s.%s", helmConfig.getName(), getVersion(helmConfig, project), getHelmClassifier(artifacts), helmConfig.getExtension())) @@ -319,16 +339,17 @@ private Map createTarball(HelmChartConfig helmConfig, Project pr Path helmSources = getChartOutputDir(helmConfig, outputDir); - List yamls = new ArrayList<>(); - yamls.add(helmSources.resolve(CHART_FILENAME).toFile()); - yamls.add(helmSources.resolve(VALUES + YAML).toFile()); - for (String profile : profiles) { - yamls.add(helmSources.resolve(VALUES + "." + profile + YAML).toFile()); + List files = new ArrayList<>(); + for (String filePath : artifacts.keySet()) { + File file = new File(filePath); + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + } else { + files.add(file); + } } - yamls.addAll(listYamls(helmSources.resolve(TEMPLATES))); - - createTarBall(tarballFile, helmSources.toFile(), yamls, helmConfig.getExtension(), + createTarBall(tarballFile, helmSources.toFile(), files, helmConfig.getExtension(), tae -> tae.setName(String.format("%s/%s", helmConfig.getName(), tae.getName()))); return Collections.singletonMap(tarballFile.toString(), null); @@ -346,6 +367,7 @@ private Map processSourceFiles(HelmChartConfig helmConfig, Path List valuesReferences, Map prodValues, Map> valuesByProfile) throws IOException { + Map templates = new HashMap<>(); Path templatesDir = getChartOutputDir(helmConfig, outputDir).resolve(TEMPLATES); Files.createDirectories(templatesDir); List> resources = replaceValuesInYamls(helmConfig, generatedFiles, valuesReferences, prodValues, @@ -363,9 +385,10 @@ private Map processSourceFiles(HelmChartConfig helmConfig, Path .replaceAll("\\\\\\n(\\s)*\\\\(\\s)*}}", " }}"); writeFile(adaptedString, targetFile); + templates.put(targetFile.toString(), adaptedString); } - return Collections.emptyMap(); + return templates; } private List> replaceValuesInYamls(HelmChartConfig helmConfig, diff --git a/docs/documentation/helm.md b/docs/documentation/helm.md index 62883abe6..7962db857 100644 --- a/docs/documentation/helm.md +++ b/docs/documentation/helm.md @@ -342,10 +342,10 @@ Let's now see how you can add this configuration using the Dekorate Helm extensi `application.properties`: ``` -dekorate.helm.dependencies.postgresql.alias=database -dekorate.helm.dependencies.postgresql.name=postgresql -dekorate.helm.dependencies.postgresql.version=11.6.22 -dekorate.helm.dependencies.postgresql.repository=https://charts.bitnami.com/bitnami +dekorate.helm.dependencies[0].postgresql.alias=database +dekorate.helm.dependencies[0].postgresql.name=postgresql +dekorate.helm.dependencies[0].postgresql.version=11.6.22 +dekorate.helm.dependencies[0].postgresql.repository=https://charts.bitnami.com/bitnami dekorate.helm.values[0].property=database.global.postgresql.auth.postgresPassword dekorate.helm.values[0].value=secret diff --git a/examples/helm-on-kubernetes-example/src/main/resources/application.properties b/examples/helm-on-kubernetes-example/src/main/resources/application.properties index 983cb0623..1ba02edec 100644 --- a/examples/helm-on-kubernetes-example/src/main/resources/application.properties +++ b/examples/helm-on-kubernetes-example/src/main/resources/application.properties @@ -1,6 +1,4 @@ dekorate.helm.name=myChart -# Produce tar file -dekorate.helm.createTarFile=true # Dependencies dekorate.helm.dependencies[0].name=dependencyNameA dekorate.helm.dependencies[0].version=0.0.1 diff --git a/examples/helm-on-kubernetes-example/src/test/java/io/dekorate/example/HelmKubernetesExampleTest.java b/examples/helm-on-kubernetes-example/src/test/java/io/dekorate/example/HelmKubernetesExampleTest.java index ed8b37bef..3d9adc480 100644 --- a/examples/helm-on-kubernetes-example/src/test/java/io/dekorate/example/HelmKubernetesExampleTest.java +++ b/examples/helm-on-kubernetes-example/src/test/java/io/dekorate/example/HelmKubernetesExampleTest.java @@ -68,13 +68,10 @@ public void shouldHelmManifestsBeGenerated() throws IOException { assertNotNull(Main.class.getClassLoader().getResourceAsStream(CHART_OUTPUT_LOCATION + "/templates/deployment.yaml")); assertNotNull(Main.class.getClassLoader().getResourceAsStream(CHART_OUTPUT_LOCATION + "/templates/ingress.yaml")); assertNotNull(Main.class.getClassLoader().getResourceAsStream(CHART_OUTPUT_LOCATION + "/templates/service.yaml")); - // empty charts folder + // charts folder assertNotNull(Main.class.getClassLoader().getResourceAsStream(CHART_OUTPUT_LOCATION + "/charts")); // notes assertNotNull(Main.class.getClassLoader().getResourceAsStream(CHART_OUTPUT_LOCATION + "/templates/NOTES.txt")); - // zip manifest - String zipName = String.format("META-INF/dekorate/helm/%s-%s-helm.tar.gz", chart.getName(), chart.getVersion()); - assertNotNull(Main.class.getClassLoader().getResourceAsStream(zipName), "File '" + zipName + "' not found!"); // optional resources assertNotNull(Main.class.getClassLoader().getResourceAsStream(CHART_OUTPUT_LOCATION + "/LICENSE")); assertNotNull(Main.class.getClassLoader().getResourceAsStream(CHART_OUTPUT_LOCATION + "/README.md")); diff --git a/tests/issue-fetch-helm-dependencies/pom.xml b/tests/issue-fetch-helm-dependencies/pom.xml new file mode 100644 index 000000000..1b39dad09 --- /dev/null +++ b/tests/issue-fetch-helm-dependencies/pom.xml @@ -0,0 +1,72 @@ + + + 4.0.0 + + + dekorate-tests + io.dekorate + 3.1-SNAPSHOT + ../ + + + io.dekorate + issue-fetch-helm-dependencies + Dekorate :: Tests :: Annotations :: Helm :: Fetch dependencies + + + + + io.dekorate + kubernetes-annotations + ${project.version} + + + io.dekorate + dekorate-spring-boot + ${project.version} + + + io.dekorate + helm-annotations + ${project.version} + + + org.springframework.boot + spring-boot-starter-web + ${version.spring-boot} + + + + + org.junit.jupiter + junit-jupiter-api + ${version.junit-jupiter} + test + + + org.junit.jupiter + junit-jupiter-engine + ${version.junit-jupiter} + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + true + + false + + + + org.springframework.boot + spring-boot-maven-plugin + ${version.spring-boot} + + + + diff --git a/tests/issue-fetch-helm-dependencies/src/main/java/io/dekorate/example/DemoApplication.java b/tests/issue-fetch-helm-dependencies/src/main/java/io/dekorate/example/DemoApplication.java new file mode 100644 index 000000000..90fbe9518 --- /dev/null +++ b/tests/issue-fetch-helm-dependencies/src/main/java/io/dekorate/example/DemoApplication.java @@ -0,0 +1,28 @@ +/** + * Copyright 2018 The original authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.dekorate.example; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class DemoApplication { + + public static void main(String[] args) { + SpringApplication.run(DemoApplication.class, args); + } + +} diff --git a/tests/issue-fetch-helm-dependencies/src/main/java/io/dekorate/example/HelloController.java b/tests/issue-fetch-helm-dependencies/src/main/java/io/dekorate/example/HelloController.java new file mode 100644 index 000000000..7e28b773f --- /dev/null +++ b/tests/issue-fetch-helm-dependencies/src/main/java/io/dekorate/example/HelloController.java @@ -0,0 +1,32 @@ +/** + * Copyright 2018 The original authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +**/ + +package io.dekorate.example; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class HelloController { + + private static final String HELLO = "hello world!"; + + @RequestMapping("/") + public String hello() { + return HELLO; + } +} diff --git a/tests/issue-fetch-helm-dependencies/src/main/resources/application.properties b/tests/issue-fetch-helm-dependencies/src/main/resources/application.properties new file mode 100644 index 000000000..b27bccc1b --- /dev/null +++ b/tests/issue-fetch-helm-dependencies/src/main/resources/application.properties @@ -0,0 +1,7 @@ +dekorate.helm.name=issue-fetch-deps +dekorate.helm.dependencies[0].alias=db +dekorate.helm.dependencies[0].name=postgresql +dekorate.helm.dependencies[0].version=11.9.1 +dekorate.helm.dependencies[0].repository=https://charts.bitnami.com/bitnami +# Produce tar file: It needs to be enabled to fetch dependencies +dekorate.helm.createTarFile=true diff --git a/tests/issue-fetch-helm-dependencies/src/test/java/io/dekorate/example/IssueHelmFetchDependencies.java b/tests/issue-fetch-helm-dependencies/src/test/java/io/dekorate/example/IssueHelmFetchDependencies.java new file mode 100644 index 000000000..fa0273a47 --- /dev/null +++ b/tests/issue-fetch-helm-dependencies/src/test/java/io/dekorate/example/IssueHelmFetchDependencies.java @@ -0,0 +1,46 @@ +/** + * Copyright 2018 The original authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +**/ + +package io.dekorate.example; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; + +public class IssueHelmFetchDependencies { + + private static final String CHART_NAME = "issue-fetch-deps"; + + @Test + public void shouldFetchDependencies() throws FileNotFoundException { + assertTrue(Stream.of(Paths.get("target", "helm", "kubernetes").toFile().listFiles()) + .anyMatch(f -> f.getName().startsWith(CHART_NAME) && f.getName().endsWith(".tar.gz"))); + + assertNotNull(getResourceAsStream("charts/postgresql-11.6.22.tgz")); + } + + private final InputStream getResourceAsStream(String file) throws FileNotFoundException { + return new FileInputStream(Paths.get("target", "helm", "kubernetes").resolve(CHART_NAME).resolve(file).toFile()); + } +} diff --git a/tests/pom.xml b/tests/pom.xml index 7b5884224..c2557a33a 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -84,6 +84,7 @@ issue-1009-probe-tcp-socket feat-kubernetes-emptydir-volumes issue-spring-boot-openshift-named-port + issue-fetch-helm-dependencies