Feature 182 bucket tags - Tile UI editor (#198)

Parsing of bucket tags from env vars, set via CF tile ui (#182)

README.md

@@ -243,6 +243,8 @@ Buckets created in this manner will have file system access enabled, and file sh
 are created, new bucket users will be created to correspond to those bindings, and uid mappings will ensure that traffic
 coming from the application operates as the correct user.
 
+Information provided in response of binding creation request includes s3 URL, secret credentials and user Unix ID.
+
 The application mount point defaults to `/var/vcap/data/{binding-id-guid}` so that is where the file system will appear
 within the application container.  You can find this path and corresponding uid from within your application programmatically 
 by parsing it from the VCAP_SERVICES environment variable.  If you prefer to have the volume mounted to a specific path 

src/main/java/com/emc/ecs/servicebroker/config/CatalogConfig.java

@@ -12,6 +12,7 @@
 import org.springframework.context.annotation.Configuration;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -91,7 +92,7 @@ private List<PlanProxy> parsePlanCollection(String planCollectionJson) throws IO
         return thesePlans.stream().map(p -> {
             // TODO include cost amounts, displayname, and properties
             PlanMetadataProxy planMetadata = new PlanMetadataProxy(p.getBullets(), p.getCosts(), null, null);
-            PlanProxy plan = new PlanProxy(p.getGuid(), p.getName(), p.getDescription(), planMetadata, p.getFree(), p.getServiceSettings());
+            PlanProxy plan = new PlanProxy(p.getGuid(), p.getName(), p.getDescription(), planMetadata, p.getFree(), parseBucketTags(p.getServiceSettings()));
             plan.setRepositoryPlan(p.getRepositoryPlan());
             return plan;
         }).collect(Collectors.toList());
@@ -123,6 +124,7 @@ private Map<String, Object> parseServiceSettings(String settingsJson, int servic
 
         if (selector.getValue().equals("Bucket")) {
             settings.put(SERVICE_TYPE, ServiceType.BUCKET.getAlias());
+            settings = parseBucketTags(settings);
         } else if (selector.getValue().equals("Namespace")) {
             settings.put(SERVICE_TYPE, ServiceType.NAMESPACE.getAlias());
         } else {
@@ -160,6 +162,49 @@ private Map<String, Object> parseServiceSettings(String settingsJson, int servic
         return settings;
     }
 
+    static Map<String, Object> parseBucketTags(Map<String, Object> settings) {
+        List<Map<String, String>> bucketTags = new ArrayList<>();
+
+        String bucketTagsString = (String) settings.get(BUCKET_TAGS);
+
+        if (bucketTagsString == null) {
+            return settings;
+        }
+
+        if (!bucketTagsString.matches(BUCKET_TAGS_STRING_REGEX1)) {
+            throw new ServiceBrokerException("Bucket tags should consist only of allowed characters: letters, numbers, and spaces representable in UTF-8, and the following characters: + - . _ : / @");
+        }
+
+        if (!bucketTagsString.matches(BUCKET_TAGS_STRING_REGEX2)) {
+            throw new ServiceBrokerException("Bucket tags have inappropriate length: A tag key can be up to 128 Unicode characters in length, and tag values can be up to 256 Unicode characters in length");
+        }
+
+        if (!bucketTagsString.matches(BUCKET_TAGS_STRING_REGEX3)) {
+            throw new ServiceBrokerException("Bucket tags do not match format 'key1:value1,key2:value2'");
+        }
+
+        String[] tagsPairs = bucketTagsString.split(BUCKET_TAG_PAIRS_DELIMITER);
+
+        for(String tagPair: tagsPairs) {
+            String[] tag = tagPair.split(BUCKET_TAG_PAIR_KEY_VALUE_DELIMITER);
+            if (tag.length != 2) {
+                throw new ServiceBrokerException("Invalid bucket tag passed. Unable to split '" + tagPair +
+                        "' on key and value. '" + BUCKET_TAG_PAIR_KEY_VALUE_DELIMITER + "' as key-value delimiter expected");
+            }
+
+            Map<String, String> tagMap = new HashMap<>();
+            tagMap.put(KEY, tag[0]);
+            tagMap.put(VALUE, tag[1]);
+
+            bucketTags.add(tagMap);
+        }
+
+        settings.remove(BUCKET_TAGS);
+        settings.put(TAGS, bucketTags);
+
+        return settings;
+    }
+
     public void setServices(List<ServiceDefinitionProxy> services) {
         this.services = services;
     }

src/main/java/com/emc/ecs/servicebroker/model/Constants.java

@@ -33,6 +33,14 @@ public class Constants {
     public static final String S3_URL = "s3Url";
 
     public static final String TAGS = "tags";
+    public static final String KEY = "key";
+    public static final String VALUE = "value";
+    public static final String BUCKET_TAGS = "bucket_tags";
+    public static final String BUCKET_TAG_PAIRS_DELIMITER = ",";
+    public static final String BUCKET_TAG_PAIR_KEY_VALUE_DELIMITER = "=";
+    public static final String BUCKET_TAGS_STRING_REGEX1 = "^[\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s\\=\\,]*$";
+    public static final String BUCKET_TAGS_STRING_REGEX2 = "^(.{0,128}\\s?=\\s?.{0,256}\\s?,\\s?)*.{0,128}\\s?=\\s?.{0,256}$|^$";
+    public static final String BUCKET_TAGS_STRING_REGEX3 = "^([\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s]{0,128}\\s?=\\s?[\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s]{0,256}\\s?,\\s?)*[\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s]{0,128}\\s?=\\s?[\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s]{0,256}$|^$";
 
     public static final String SEARCH_METADATA = "search-metadata";
     public static final String SEARCH_METADATA_TYPE = "type";

src/main/java/com/emc/ecs/servicebroker/model/PlanCollectionInstance.java

@@ -70,6 +70,9 @@ public class PlanCollectionInstance {
     @JsonProperty("base_url")
     private String baseUrl;
 
+    @JsonProperty("bucket_tags")
+    private String bucketTags;
+
     public void setBullet1(String bullet1) {
         this.bullet1 = bullet1;
     }
@@ -176,6 +179,14 @@ public void setBaseUrl(String baseUrl) {
         this.baseUrl = baseUrl;
     }
 
+    public String getBucketTags() {
+        return bucketTags;
+    }
+
+    public void setBucketTags(String bucketTags) {
+        this.bucketTags = bucketTags;
+    }
+
     public List<CostProxy> getCosts() {
         Map<String, Double> val = new HashMap<>();
         val.put(costCurrency != null ? costCurrency.toLowerCase() : Currency.getInstance("USD").getCurrencyCode().toLowerCase(), Double.valueOf(costValue));
@@ -224,6 +235,10 @@ public Map<String, Object> getServiceSettings() {
             serviceSettings.put(QUOTA, quota);
         }
 
+        if (bucketTags != null) {
+            serviceSettings.put(BUCKET_TAGS, bucketTags);
+        }
+
         return serviceSettings;
     }
 }

src/main/java/com/emc/ecs/servicebroker/service/EcsService.java

@@ -25,6 +25,7 @@
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static com.emc.ecs.servicebroker.model.Constants.*;
 
@@ -702,6 +703,55 @@ public DataServiceReplicationGroup lookupReplicationGroup(String replicationGrou
                 .orElseThrow(() -> new ServiceBrokerException("ECS replication group not found: " + replicationGroup));
     }
 
+    /**
+     * Merge request bucket tags with with plan and service provided tags
+     * <p>
+     * Request bucket tags are overwritten with plan and service ones,
+     * while bucket tags provided in plan description are overwritten by service tags
+     * since service settings are forced by administrator through the catalog
+     */
+    static List<Map<String, String>> mergeBucketTags(ServiceDefinitionProxy service, PlanProxy plan, Map<String, Object> requestParameters) {
+        List<Map<String, String>> serviceTags = (List<Map<String, String>>)service.getServiceSettings().get(TAGS);
+        List<Map<String, String>> planTags = (List<Map<String, String>>)plan.getServiceSettings().get(TAGS);
+        List<Map<String, String>> requestedTags = (List<Map<String, String>>)requestParameters.get(TAGS);
+        List<Map<String, String>> unmatchedTags;
+
+        if (planTags != null && serviceTags != null) {
+            unmatchedTags = new ArrayList<>(planTags);
+
+            for (Map<String, String> planTag: planTags) {
+                for (Map<String, String> serviceTag: serviceTags) {
+                    if (planTag.get(KEY).equals(serviceTag.get(KEY))) {
+                        unmatchedTags.remove(planTag);
+                    }
+                }
+            }
+
+            serviceTags = Stream.concat(serviceTags.stream(), unmatchedTags.stream()).collect(Collectors.toList());
+        } else if (serviceTags == null && planTags != null) {
+            serviceTags = new ArrayList<>(planTags);
+        }
+
+        if (requestedTags != null && serviceTags != null) {
+            unmatchedTags = new ArrayList<>(requestedTags);
+
+            for (Map<String, String> requestedTag: requestedTags) {
+                for (Map<String, String> serviceTag: serviceTags) {
+                    if (requestedTag.get(KEY).equals(serviceTag.get(KEY))) {
+                        unmatchedTags.remove(requestedTag);
+                        break;
+                    }
+                }
+            }
+
+            serviceTags = Stream.concat(serviceTags.stream(), unmatchedTags.stream()).collect(Collectors.toList());
+        } else if (serviceTags == null && requestedTags != null) {
+            serviceTags = new ArrayList<>(requestedTags);
+        }
+
+        return serviceTags;
+    }
+
     /**
      * Merge request additional parameters with with broker, plan and service settings
      * <p>
@@ -718,6 +768,12 @@ static Map<String, Object> mergeParameters(BrokerConfig brokerConfig, ServiceDef
 
         ret.putAll(service.getServiceSettings());
 
+        List<Map<String, String>> tags = mergeBucketTags(service, plan, requestParameters);
+
+        if (tags != null) {
+            ret.put(TAGS, tags);
+        }
+
         return ret;
     }
 

src/test/java/com/emc/ecs/common/Fixtures.java

@@ -1,5 +1,6 @@
 package com.emc.ecs.common;
 
+import com.emc.ecs.management.sdk.model.BucketTagSetRootElement;
 import com.emc.ecs.servicebroker.model.PlanProxy;
 import com.emc.ecs.servicebroker.model.ServiceDefinitionProxy;
 import com.emc.ecs.servicebroker.model.ServiceType;
@@ -63,12 +64,17 @@ public class Fixtures {
     public static final String VOLUME_MOUNT_VALUE = "/mount/dir";
     public static final String SECRET_KEY_VALUE = "testKEY@ключ:/-s#cr#T";
     public static final String SHOULD_RAISE_AN_EXCEPTION = "should raise an exception";
+    public static final String BUCKET_TAGS_INVALID_CHARS = "key?=value!";
+    public static final String BUCKET_TAGS_INVALID_FORMAT = "key1:value1;key2:value2";
+    public static final String BUCKET_TAGS_LONG_KEY = "very very very very very very very very very very very very very very very very very very very very long key of exactly 129 chars=value";
+    public static final String BUCKET_TAGS_LONG_VALUE = "key=very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very great value accurately 257 chars long";
     public static final String KEY1 = "tag1";
     public static final String KEY2 = "tag2";
     public static final String KEY3 = "tag3";
     public static final String VALUE1 = "value1";
     public static final String VALUE2 = "value2";
     public static final String VALUE3 = "value3";
+    public static final String BUCKET_TAGS_STRING = KEY1 + "=" + VALUE1 + "," + KEY2 + "=" + VALUE2 + "," + KEY3 + "=" + VALUE3;
     public static final String MARKER = "marker1";
     public static final int PAGE_SIZE = 32;
     public static final String SYSTEM_METADATA_NAME = "Size";
@@ -567,4 +573,23 @@ public static DeleteServiceInstanceBindingRequest bucketBindingRemoveFixture() {
                 .planId(BUCKET_PLAN_ID1)
                 .build();
     }
+
+    public static List<Map<String, String>> listOfBucketTagsFixture() {
+        List<Map<String, String>> tags = new ArrayList<>();
+        Map<String, String> tag1 = new HashMap<>();
+        Map<String, String> tag2 = new HashMap<>();
+        Map<String, String> tag3 = new HashMap<>();
+
+        tag1.put(BucketTagSetRootElement.KEY, KEY1);
+        tag1.put(BucketTagSetRootElement.VALUE, VALUE1);
+        tag2.put(BucketTagSetRootElement.KEY, KEY2);
+        tag2.put(BucketTagSetRootElement.VALUE, VALUE2);
+        tag3.put(BucketTagSetRootElement.KEY, KEY3);
+        tag3.put(BucketTagSetRootElement.VALUE, VALUE3);
+
+        tags.add(tag1);
+        tags.add(tag2);
+        tags.add(tag3);
+        return tags;
+    }
 }

src/test/java/com/emc/ecs/servicebroker/config/CatalogConfigTest.java

@@ -1,22 +1,25 @@
 package com.emc.ecs.servicebroker.config;
 
+import com.emc.ecs.common.Fixtures;
+
+import org.apache.commons.collections.CollectionUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.ConfigFileApplicationContextInitializer;
+import org.springframework.cloud.servicebroker.exception.ServiceBrokerException;
 import org.springframework.cloud.servicebroker.model.catalog.Catalog;
 import org.springframework.cloud.servicebroker.model.catalog.Plan;
 import org.springframework.cloud.servicebroker.model.catalog.ServiceDefinition;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
-import static org.junit.Assert.assertEquals;
+import static com.emc.ecs.common.Fixtures.*;
+import static com.emc.ecs.servicebroker.model.Constants.*;
+import static org.junit.Assert.*;
 
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration(classes = Application.class,
@@ -66,6 +69,38 @@ public void testEcsBucketPlans() {
                         "S3 protocol access"));
     }
 
+    @Test
+    public void testBucketTagsValidation() {
+        Map<String, Object> settingsWithInvalidCharacters = new HashMap<>();
+        settingsWithInvalidCharacters.put(BUCKET_TAGS, BUCKET_TAGS_INVALID_CHARS);
+
+        Map<String, Object> settingsWithInvalidFormat = new HashMap<>();
+        settingsWithInvalidFormat.put(BUCKET_TAGS, BUCKET_TAGS_INVALID_FORMAT);
+
+        Map<String, Object> settingsWithLongKey = new HashMap<>();
+        settingsWithLongKey.put(BUCKET_TAGS, BUCKET_TAGS_LONG_KEY);
+
+        Map<String, Object> settingsWithLongValue = new HashMap<>();
+        settingsWithLongValue.put(BUCKET_TAGS, BUCKET_TAGS_LONG_VALUE);
+
+        assertThrows(ServiceBrokerException.class, () -> CatalogConfig.parseBucketTags(settingsWithInvalidCharacters));
+        assertThrows(ServiceBrokerException.class, () -> CatalogConfig.parseBucketTags(settingsWithInvalidFormat));
+        assertThrows(ServiceBrokerException.class, () -> CatalogConfig.parseBucketTags(settingsWithLongKey));
+        assertThrows(ServiceBrokerException.class, () -> CatalogConfig.parseBucketTags(settingsWithLongValue));
+    }
+
+    @Test
+    public void parseBucketTagsTest() {
+        Map<String, Object> settings = new HashMap<>();
+        settings.put(BUCKET_TAGS, Fixtures.BUCKET_TAGS_STRING);
+        settings = CatalogConfig.parseBucketTags(settings);
+
+        List<Map<String, String>> resultTags = (List<Map<String, String>>)settings.get(TAGS);
+        List<Map<String, String>> expectedTags = Fixtures.listOfBucketTagsFixture();
+
+        assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags));
+    }
+
     private void testServiceDefinition(ServiceDefinition service, String id,
                                        String name, String description, boolean bindable,
                                        boolean updatable, List<String> requires, String dashboardUrl) {