Merge branch 'develop'

dedica-team · Nov 7, 2019 · afc932d · afc932d
2 parents c1d46e0 + 5756c5f
commit afc932d
Show file tree

Hide file tree

Showing 48 changed files with 266 additions and 618 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,3 +1,4 @@
 .git
 .idea
-src
+src/main
+src/test/java
diff --git a/Dockerfile b/Dockerfile
@@ -3,5 +3,6 @@ FROM openjdk:11-jre
 
 VOLUME /tmp
 
+ADD src/test/resources/example /src/test/resources/example
 ADD target/nivio.jar app.jar
 ENTRYPOINT ["java","-jar","/app.jar"]
diff --git a/docs/source/model.rst b/docs/source/model.rst
@@ -165,20 +165,3 @@ will not be overwritten by templates applied later.
         statuses
 
 For CQ queries, read https://github.com/npgall/cqengine#string-based-queries-sql-and-cqn-dialects.
-
-Service state (alpha)
----------------------
-
-You can also add state providers which are used to gather live data and thereby provide state for the items. Currently only prometheus is supported.
-
-.. code-block:: yaml
-   :linenos:
-
-    identifier: nivio:example
-    name: Landscape example
-
-    ...
-
-    stateProviders:
-      - type: prometheus-exporter
-        target: http://prometheus_exporter.url
diff --git a/pom.xml b/pom.xml
@@ -42,7 +42,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.9.10</version>
+            <version>2.9.10.1</version>
         </dependency>
         <dependency>
             <groupId>javax.xml.bind</groupId>

diff --git a/src/main/java/de/bonndan/nivio/api/ApiController.java b/src/main/java/de/bonndan/nivio/api/ApiController.java
@@ -124,7 +124,7 @@ public ProcessLog deleteService(
         if (from == null)
             return new ProcessLog(new ProcessingException(landscape, "Could use fully qualified identifier " + fqi));
 
-        Optional<LandscapeItem> item = ServiceItems.find(FullyQualifiedIdentifier.build(from.getLandscape(), from.getGroup(), from.getIdentifier()), landscape.getItems());
+        Optional<LandscapeItem> item = Items.find(FullyQualifiedIdentifier.build(from.getLandscape(), from.getGroup(), from.getIdentifier()), landscape.getItems());
         if (!item.isPresent()) {
             return new ProcessLog(new ProcessingException(landscape, "Could find item " + fqi));
         }

diff --git a/src/main/java/de/bonndan/nivio/api/LandscapeDTOFactory.java b/src/main/java/de/bonndan/nivio/api/LandscapeDTOFactory.java
@@ -23,7 +23,6 @@ public static LandscapeDTO from(Landscape item) {
         l.identifier = item.getIdentifier();
         l.name = item.getName();
         l.contact = item.getContact();
-        l.stateProviders = item.getStateProviders();
         l.source = item.getSource();
 
         return l;

diff --git a/src/main/java/de/bonndan/nivio/api/dto/LandscapeDTO.java b/src/main/java/de/bonndan/nivio/api/dto/LandscapeDTO.java
@@ -3,7 +3,6 @@
 import de.bonndan.nivio.model.GroupItem;
 import de.bonndan.nivio.model.LandscapeConfig;
 import de.bonndan.nivio.model.Landscape;
-import de.bonndan.nivio.model.StateProviderConfig;
 import org.springframework.hateoas.ResourceSupport;
 
 import java.util.List;
@@ -17,7 +16,6 @@ public class LandscapeDTO extends ResourceSupport implements Landscape  {
     public String identifier;
     public String name;
     public String contact;
-    public List<StateProviderConfig> stateProviders;
     public String source;
 
     @Override
@@ -40,11 +38,6 @@ public String getSource() {
         return source;
     }
 
-    @Override
-    public List<StateProviderConfig> getStateProviders() {
-        return stateProviders;
-    }
-
     @Override
     public LandscapeConfig getConfig() {
         return null;

diff --git a/src/main/java/de/bonndan/nivio/input/Indexer.java b/src/main/java/de/bonndan/nivio/input/Indexer.java
@@ -101,7 +101,7 @@ private void diff(final LandscapeDescription input, final LandscapeImpl landscap
         kept.forEach(
                 item -> {
 
-                    ItemDescription description = (ItemDescription) ServiceItems.find(item.getFullyQualifiedIdentifier(), input.getItemDescriptions()).orElse(null);
+                    ItemDescription description = (ItemDescription) Items.find(item.getFullyQualifiedIdentifier(), input.getItemDescriptions()).orElse(null);
                     if (description == null) {
                         if (input.isPartial()) {
                             inLandscape.add((Item) item);
@@ -133,7 +133,7 @@ private void fillGroups(LandscapeDescription input, LandscapeImpl landscape) {
             GroupDescription description = (GroupDescription) groupItem;
             Group group = (Group) landscape.getGroups().get(description.getIdentifier());
             description.getContains().forEach(condition -> {
-                group.getItems().addAll(ServiceItems.query(condition, List.copyOf(landscape.getItems())));
+                group.getItems().addAll(Items.query(condition, List.copyOf(landscape.getItems())));
             });
         });
     }
@@ -155,28 +155,30 @@ private List<LandscapeItem> deleteUnreferenced(
         return removed;
     }
 
+
     private void linkItems(final LandscapeDescription input, final LandscapeImpl landscape, ProcessLog logger) {
 
         input.getItemDescriptions().forEach(serviceDescription -> {
-            Item origin = (Item) ServiceItems.pick(serviceDescription, landscape.getItems());
+            Item origin = (Item) Items.pick(serviceDescription, landscape.getItems());
             if (!input.isPartial()) {
                 logger.info("Clearing relations of " + origin);
                 origin.getRelations().clear(); //delete all relations on full update
             }
         });
 
         input.getItemDescriptions().forEach(serviceDescription -> {
-            Item origin = (Item) ServiceItems.pick(serviceDescription, landscape.getItems());
+            Item origin = (Item) Items.pick(serviceDescription, landscape.getItems());
             serviceDescription.getRelations().forEach(relationDescription -> {
 
                 var fqiSource = FullyQualifiedIdentifier.from(relationDescription.getSource());
                 var fqiTarget = FullyQualifiedIdentifier.from(relationDescription.getTarget());
-                Item source = (Item) ServiceItems.find(fqiSource, landscape.getItems()).orElse(null);
+                Item source = (Item) Items.find(fqiSource, landscape.getItems()).orElse(null);
                 if (source == null) {
                     logger.warn("Relation source " + relationDescription.getSource() + " not found");
                     return;
                 }
-                Item target = (Item) ServiceItems.find(fqiTarget, landscape.getItems()).orElse(null);
+                Item target = (Item) Items.find(fqiTarget, landscape.getItems()).orElse(null);
+
                 if (target == null) {
                     logger.warn("Relation target " + relationDescription.getTarget() + " not found");
                     return;

diff --git a/src/main/java/de/bonndan/nivio/input/ItemDescriptionFactory.java b/src/main/java/de/bonndan/nivio/input/ItemDescriptionFactory.java
@@ -1,17 +1,11 @@
 package de.bonndan.nivio.input;
 
-import de.bonndan.nivio.ProcessingException;
 import de.bonndan.nivio.input.dto.ItemDescription;
-import de.bonndan.nivio.input.dto.LandscapeDescription;
 import de.bonndan.nivio.input.dto.SourceReference;
-import de.bonndan.nivio.model.Lifecycle;
-import de.bonndan.nivio.model.ServiceItems;
 
 import java.util.List;
-import java.util.function.*;
 
 import static de.bonndan.nivio.util.SafeAssign.assignSafe;
-import static de.bonndan.nivio.util.SafeAssign.assignSafeIfAbsent;
 
 public interface ItemDescriptionFactory {
 

diff --git a/src/main/java/de/bonndan/nivio/input/ItemDescriptionFormatFactory.java b/src/main/java/de/bonndan/nivio/input/ItemDescriptionFormatFactory.java
@@ -6,6 +6,7 @@
 import de.bonndan.nivio.input.http.HttpService;
 import de.bonndan.nivio.input.kubernetes.ItemDescriptionFactoryKubernetes;
 import de.bonndan.nivio.input.nivio.ItemDescriptionFactoryNivio;
+import de.bonndan.nivio.input.rancher1.ItemDescriptionFactoryRancher1;
 import de.bonndan.nivio.util.URLHelper;
 
 import java.net.URL;
@@ -28,11 +29,14 @@ public static ItemDescriptionFactory getFactory(SourceReference reference, Lands
         if (reference == null || reference.getFormat() == null)
             return new ItemDescriptionFactoryNivio(fetcher, baseUrl);
 
+        //TODO use SourceFormat.from or similar
         switch (reference.getFormat()) {
             case DOCKER_COMPOSE2:
                 return new ItemDescriptionFactoryCompose2(new FileFetcher(new HttpService()), baseUrl);
             case KUBERNETES:
                 return new ItemDescriptionFactoryKubernetes(reference);
+            case RANCHER1_PROMETHEUS:
+                return new ItemDescriptionFactoryRancher1(baseUrl);
         }
 
         return new ItemDescriptionFactoryNivio(fetcher, baseUrl);

diff --git a/src/main/java/de/bonndan/nivio/input/RelationResolver.java b/src/main/java/de/bonndan/nivio/input/RelationResolver.java
@@ -3,13 +3,11 @@
 import de.bonndan.nivio.input.dto.ItemDescription;
 import de.bonndan.nivio.input.dto.LandscapeDescription;
 import de.bonndan.nivio.input.dto.RelationDescription;
+import de.bonndan.nivio.model.Items;
 import de.bonndan.nivio.model.LandscapeItem;
 import de.bonndan.nivio.model.RelationBuilder;
-import de.bonndan.nivio.model.RelationItem;
-import de.bonndan.nivio.model.ServiceItems;
 import org.springframework.util.StringUtils;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
@@ -34,7 +32,7 @@ private void resolveRelations(ItemDescription description, List<ItemDescription>
 
         //providers
         description.getProvidedBy().forEach(term -> {
-            ServiceItems.query(term, allItems).stream().findFirst().ifPresentOrElse(o -> {
+            Items.query(term, allItems).stream().findFirst().ifPresentOrElse(o -> {
                 RelationDescription rel = RelationBuilder.createProviderDescription((ItemDescription) o, description.getIdentifier());
                 description.addRelation(rel);
             }, () -> log.warn(description.getIdentifier() + ": no provider target found for term " + term));
@@ -60,7 +58,7 @@ private Optional<ItemDescription> resolveOne(ItemDescription description, String
             return Optional.of(description);
         }
 
-        Collection<? extends LandscapeItem> result = ServiceItems.query(term, allItems);
+        Collection<? extends LandscapeItem> result = Items.query(term, allItems);
         if (result.size() > 1) {
             log.warn(description.getIdentifier() + ": Found ambiguous sources matching " + term);
             return Optional.empty();

diff --git a/src/main/java/de/bonndan/nivio/input/SourceReferencesResolver.java b/src/main/java/de/bonndan/nivio/input/SourceReferencesResolver.java
@@ -1,16 +1,12 @@
 package de.bonndan.nivio.input;
 
 import de.bonndan.nivio.ProcessingException;
-import de.bonndan.nivio.input.dto.RelationDescription;
 import de.bonndan.nivio.input.dto.LandscapeDescription;
 import de.bonndan.nivio.input.dto.ItemDescription;
-import de.bonndan.nivio.model.ServiceItems;
-import org.springframework.util.StringUtils;
 
 import java.util.List;
 import java.util.Map;
 
-import static de.bonndan.nivio.model.ServiceItems.find;
 
 /**
  * Resolves source references into collections of item descriptions.
@@ -37,7 +33,6 @@ public void resolve(final LandscapeDescription landscapeDescription, Map<ItemDes
                 landscapeDescription.setIsPartial(true);
             }
         });
-
     }
 
 }
diff --git a/src/main/java/de/bonndan/nivio/input/TemplateResolver.java b/src/main/java/de/bonndan/nivio/input/TemplateResolver.java
@@ -37,7 +37,7 @@ private void applyTemplateValues(
             LandscapeDescription landscape
     ) {
         templateTargets.forEach(term ->
-                ServiceItems.query(term, landscape.getItemDescriptions())
+                Items.query(term, landscape.getItemDescriptions())
                         .forEach(item -> assignTemplateValues((ItemDescription) item, template))
         );
     }

diff --git a/src/main/java/de/bonndan/nivio/input/dto/ItemDescription.java b/src/main/java/de/bonndan/nivio/input/dto/ItemDescription.java
@@ -74,6 +74,11 @@ public ItemDescription(String identifier) {
         this.identifier = identifier;
     }
 
+    public ItemDescription(FullyQualifiedIdentifier fqi) {
+        this.identifier = fqi.getIdentifier();
+        this.group = fqi.getGroup();
+    }
+
     public String getIdentifier() {
         return identifier;
     }

diff --git a/src/main/java/de/bonndan/nivio/input/dto/LandscapeDescription.java b/src/main/java/de/bonndan/nivio/input/dto/LandscapeDescription.java
@@ -55,8 +55,6 @@ public class LandscapeDescription implements Landscape {
      */
     private List<ItemDescription> itemDescriptions = new ArrayList<>();
 
-    private List<StateProviderConfig> stateProviders;
-
     private LandscapeConfig config;
 
     private boolean isPartial = false;
@@ -94,15 +92,6 @@ public String getContact() {
         return contact;
     }
 
-    @Override
-    public List<StateProviderConfig> getStateProviders() {
-        return stateProviders;
-    }
-
-    public void setStateProviders(List<StateProviderConfig> stateProviders) {
-        this.stateProviders = stateProviders;
-    }
-
     public List<SourceReference> getSourceReferences() {
         return sources;
     }
@@ -142,7 +131,6 @@ public LandscapeImpl toLandscape() {
         landscape.setName(name);
         landscape.setContact(contact);
         landscape.setSource(source);
-        landscape.setStateProviders(stateProviders);
         landscape.setConfig(config);
         return landscape;
     }
@@ -155,7 +143,7 @@ public void addItems(List<ItemDescription> incoming) {
             desc.setEnvironment(this.identifier);
 
             ItemDescription existing = (ItemDescription)
-                    ServiceItems.find(desc.getIdentifier(), desc.getGroup(), itemDescriptions).orElse(null);
+                    Items.find(desc.getIdentifier(), desc.getGroup(), itemDescriptions).orElse(null);
             if (existing != null) {
                 ItemDescriptionFactory.assignNotNull(existing, desc);
             } else {

diff --git a/src/main/java/de/bonndan/nivio/input/dto/SourceFormat.java b/src/main/java/de/bonndan/nivio/input/dto/SourceFormat.java
@@ -8,12 +8,20 @@
 
 public enum SourceFormat {
 
+    //TODO move to package, use annotations or service loader
     DOCKER_COMPOSE2("docker-compose-v2"),
     KUBERNETES("kubernetes", "k8s"),
-    NIVIO("nivio", "");
+    NIVIO("nivio", ""),
+    RANCHER1_PROMETHEUS("rancher1-prometheus");
 
     private final List<String> formats;
 
+    private static final List<SourceFormat> KNOWN_FORMATS;
+
+    static {
+        KNOWN_FORMATS = Arrays.asList(NIVIO, DOCKER_COMPOSE2, KUBERNETES, RANCHER1_PROMETHEUS);
+    }
+
     SourceFormat(String format, String... formats) {
 
         this.formats = new ArrayList<>();
@@ -30,18 +38,9 @@ public static SourceFormat from(String format) {
             return NIVIO;
         format = format.toLowerCase();
 
-        if (DOCKER_COMPOSE2.formats.contains(format)) {
-            return DOCKER_COMPOSE2;
-        }
-
-        if (NIVIO.formats.contains(format)) {
-            return NIVIO;
-        }
-
-        if (KUBERNETES.formats.contains(format)) {
-            return KUBERNETES;
-        }
-
-        return NIVIO;
+        String finalFormat = format;
+        return KNOWN_FORMATS.stream()
+                .filter(sourceFormat -> sourceFormat.formats.contains(finalFormat))
+                .findFirst().orElse(NIVIO);
     }
 }
diff --git a/src/main/java/de/bonndan/nivio/input/dto/SourceReference.java b/src/main/java/de/bonndan/nivio/input/dto/SourceReference.java
@@ -1,5 +1,7 @@
 package de.bonndan.nivio.input.dto;
 
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonAnySetter;
 import org.springframework.util.StringUtils;
 
 import java.util.HashMap;
@@ -23,6 +25,8 @@ public class SourceReference {
     private Map<String, List<String>> assignTemplates = new HashMap<>();
     private String content;
 
+    private Map<String, Object> props = new HashMap<>();
+
     public SourceReference() {
     }
 
@@ -120,4 +124,14 @@ public void setContent(String content) {
     public String getContent() {
         return content;
     }
+
+    @JsonAnyGetter
+    public Object getProperty(String key) {
+        return props.get(key);
+    }
+
+    @JsonAnySetter
+    public void setProperty(String key, Object value) {
+        props.put(key, value);
+    }
 }
diff --git a/src/main/java/de/bonndan/nivio/input/kubernetes/ItemDescriptionFactoryKubernetes.java b/src/main/java/de/bonndan/nivio/input/kubernetes/ItemDescriptionFactoryKubernetes.java
@@ -6,7 +6,7 @@
 import de.bonndan.nivio.input.dto.SourceReference;
 import de.bonndan.nivio.model.LandscapeItem;
 import de.bonndan.nivio.model.RelationBuilder;
-import de.bonndan.nivio.model.ServiceItems;
+import de.bonndan.nivio.model.Items;
 import de.bonndan.nivio.util.URLHelper;
 import io.fabric8.kubernetes.api.model.HasMetadata;
 import io.fabric8.kubernetes.api.model.Pod;
@@ -89,7 +89,7 @@ private Collection<? extends ItemDescription> createDescriptionFromService(Servi
         if (selector != null)
              targetId = selector.getOrDefault("app", null);
         if (!StringUtils.isEmpty(targetId)) {
-            ServiceItems.find(targetId, group, items).ifPresent(target -> {
+            Items.find(targetId, group, items).ifPresent(target -> {
                 RelationDescription provides = RelationBuilder.provides(service, target);
                 ((ItemDescription) target).addRelation(provides);
                 service.addRelation(provides);