diff --git a/README.md b/README.md index 161b2415..31cd327b 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,8 @@ Buckets created in this manner will have file system access enabled, and file sh are created, new bucket users will be created to correspond to those bindings, and uid mappings will ensure that traffic coming from the application operates as the correct user. +Information provided in response of binding creation request includes s3 URL, secret credentials and user Unix ID. + The application mount point defaults to `/var/vcap/data/{binding-id-guid}` so that is where the file system will appear within the application container. You can find this path and corresponding uid from within your application programmatically by parsing it from the VCAP_SERVICES environment variable. If you prefer to have the volume mounted to a specific path diff --git a/src/main/java/com/emc/ecs/servicebroker/config/CatalogConfig.java b/src/main/java/com/emc/ecs/servicebroker/config/CatalogConfig.java index 9e6b493f..596f331c 100644 --- a/src/main/java/com/emc/ecs/servicebroker/config/CatalogConfig.java +++ b/src/main/java/com/emc/ecs/servicebroker/config/CatalogConfig.java @@ -12,6 +12,7 @@ import org.springframework.context.annotation.Configuration; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -91,7 +92,7 @@ private List parsePlanCollection(String planCollectionJson) throws IO return thesePlans.stream().map(p -> { // TODO include cost amounts, displayname, and properties PlanMetadataProxy planMetadata = new PlanMetadataProxy(p.getBullets(), p.getCosts(), null, null); - PlanProxy plan = new PlanProxy(p.getGuid(), p.getName(), p.getDescription(), planMetadata, p.getFree(), p.getServiceSettings()); + PlanProxy plan = new PlanProxy(p.getGuid(), p.getName(), p.getDescription(), planMetadata, p.getFree(), parseBucketTags(p.getServiceSettings())); plan.setRepositoryPlan(p.getRepositoryPlan()); return plan; }).collect(Collectors.toList()); @@ -123,6 +124,7 @@ private Map parseServiceSettings(String settingsJson, int servic if (selector.getValue().equals("Bucket")) { settings.put(SERVICE_TYPE, ServiceType.BUCKET.getAlias()); + settings = parseBucketTags(settings); } else if (selector.getValue().equals("Namespace")) { settings.put(SERVICE_TYPE, ServiceType.NAMESPACE.getAlias()); } else { @@ -160,6 +162,49 @@ private Map parseServiceSettings(String settingsJson, int servic return settings; } + static Map parseBucketTags(Map settings) { + List> bucketTags = new ArrayList<>(); + + String bucketTagsString = (String) settings.get(BUCKET_TAGS); + + if (bucketTagsString == null) { + return settings; + } + + if (!bucketTagsString.matches(BUCKET_TAGS_STRING_REGEX1)) { + throw new ServiceBrokerException("Bucket tags should consist only of allowed characters: letters, numbers, and spaces representable in UTF-8, and the following characters: + - . _ : / @"); + } + + if (!bucketTagsString.matches(BUCKET_TAGS_STRING_REGEX2)) { + throw new ServiceBrokerException("Bucket tags have inappropriate length: A tag key can be up to 128 Unicode characters in length, and tag values can be up to 256 Unicode characters in length"); + } + + if (!bucketTagsString.matches(BUCKET_TAGS_STRING_REGEX3)) { + throw new ServiceBrokerException("Bucket tags do not match format 'key1:value1,key2:value2'"); + } + + String[] tagsPairs = bucketTagsString.split(BUCKET_TAG_PAIRS_DELIMITER); + + for(String tagPair: tagsPairs) { + String[] tag = tagPair.split(BUCKET_TAG_PAIR_KEY_VALUE_DELIMITER); + if (tag.length != 2) { + throw new ServiceBrokerException("Invalid bucket tag passed. Unable to split '" + tagPair + + "' on key and value. '" + BUCKET_TAG_PAIR_KEY_VALUE_DELIMITER + "' as key-value delimiter expected"); + } + + Map tagMap = new HashMap<>(); + tagMap.put(KEY, tag[0]); + tagMap.put(VALUE, tag[1]); + + bucketTags.add(tagMap); + } + + settings.remove(BUCKET_TAGS); + settings.put(TAGS, bucketTags); + + return settings; + } + public void setServices(List services) { this.services = services; } diff --git a/src/main/java/com/emc/ecs/servicebroker/model/Constants.java b/src/main/java/com/emc/ecs/servicebroker/model/Constants.java index c3fc4bfe..6b94efe1 100644 --- a/src/main/java/com/emc/ecs/servicebroker/model/Constants.java +++ b/src/main/java/com/emc/ecs/servicebroker/model/Constants.java @@ -33,6 +33,14 @@ public class Constants { public static final String S3_URL = "s3Url"; public static final String TAGS = "tags"; + public static final String KEY = "key"; + public static final String VALUE = "value"; + public static final String BUCKET_TAGS = "bucket_tags"; + public static final String BUCKET_TAG_PAIRS_DELIMITER = ","; + public static final String BUCKET_TAG_PAIR_KEY_VALUE_DELIMITER = "="; + public static final String BUCKET_TAGS_STRING_REGEX1 = "^[\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s\\=\\,]*$"; + public static final String BUCKET_TAGS_STRING_REGEX2 = "^(.{0,128}\\s?=\\s?.{0,256}\\s?,\\s?)*.{0,128}\\s?=\\s?.{0,256}$|^$"; + public static final String BUCKET_TAGS_STRING_REGEX3 = "^([\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s]{0,128}\\s?=\\s?[\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s]{0,256}\\s?,\\s?)*[\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s]{0,128}\\s?=\\s?[\\w\\d\\+\\-\\.\\_\\:\\/\\@\\s]{0,256}$|^$"; public static final String SEARCH_METADATA = "search-metadata"; public static final String SEARCH_METADATA_TYPE = "type"; diff --git a/src/main/java/com/emc/ecs/servicebroker/model/PlanCollectionInstance.java b/src/main/java/com/emc/ecs/servicebroker/model/PlanCollectionInstance.java index 0bb8ca48..cda7afb0 100644 --- a/src/main/java/com/emc/ecs/servicebroker/model/PlanCollectionInstance.java +++ b/src/main/java/com/emc/ecs/servicebroker/model/PlanCollectionInstance.java @@ -70,6 +70,9 @@ public class PlanCollectionInstance { @JsonProperty("base_url") private String baseUrl; + @JsonProperty("bucket_tags") + private String bucketTags; + public void setBullet1(String bullet1) { this.bullet1 = bullet1; } @@ -176,6 +179,14 @@ public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; } + public String getBucketTags() { + return bucketTags; + } + + public void setBucketTags(String bucketTags) { + this.bucketTags = bucketTags; + } + public List getCosts() { Map val = new HashMap<>(); val.put(costCurrency != null ? costCurrency.toLowerCase() : Currency.getInstance("USD").getCurrencyCode().toLowerCase(), Double.valueOf(costValue)); @@ -224,6 +235,10 @@ public Map getServiceSettings() { serviceSettings.put(QUOTA, quota); } + if (bucketTags != null) { + serviceSettings.put(BUCKET_TAGS, bucketTags); + } + return serviceSettings; } } diff --git a/src/main/java/com/emc/ecs/servicebroker/service/EcsService.java b/src/main/java/com/emc/ecs/servicebroker/service/EcsService.java index c9292cb1..88d34941 100644 --- a/src/main/java/com/emc/ecs/servicebroker/service/EcsService.java +++ b/src/main/java/com/emc/ecs/servicebroker/service/EcsService.java @@ -25,6 +25,7 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.emc.ecs.servicebroker.model.Constants.*; @@ -702,6 +703,55 @@ public DataServiceReplicationGroup lookupReplicationGroup(String replicationGrou .orElseThrow(() -> new ServiceBrokerException("ECS replication group not found: " + replicationGroup)); } + /** + * Merge request bucket tags with with plan and service provided tags + *

+ * Request bucket tags are overwritten with plan and service ones, + * while bucket tags provided in plan description are overwritten by service tags + * since service settings are forced by administrator through the catalog + */ + static List> mergeBucketTags(ServiceDefinitionProxy service, PlanProxy plan, Map requestParameters) { + List> serviceTags = (List>)service.getServiceSettings().get(TAGS); + List> planTags = (List>)plan.getServiceSettings().get(TAGS); + List> requestedTags = (List>)requestParameters.get(TAGS); + List> unmatchedTags; + + if (planTags != null && serviceTags != null) { + unmatchedTags = new ArrayList<>(planTags); + + for (Map planTag: planTags) { + for (Map serviceTag: serviceTags) { + if (planTag.get(KEY).equals(serviceTag.get(KEY))) { + unmatchedTags.remove(planTag); + } + } + } + + serviceTags = Stream.concat(serviceTags.stream(), unmatchedTags.stream()).collect(Collectors.toList()); + } else if (serviceTags == null && planTags != null) { + serviceTags = new ArrayList<>(planTags); + } + + if (requestedTags != null && serviceTags != null) { + unmatchedTags = new ArrayList<>(requestedTags); + + for (Map requestedTag: requestedTags) { + for (Map serviceTag: serviceTags) { + if (requestedTag.get(KEY).equals(serviceTag.get(KEY))) { + unmatchedTags.remove(requestedTag); + break; + } + } + } + + serviceTags = Stream.concat(serviceTags.stream(), unmatchedTags.stream()).collect(Collectors.toList()); + } else if (serviceTags == null && requestedTags != null) { + serviceTags = new ArrayList<>(requestedTags); + } + + return serviceTags; + } + /** * Merge request additional parameters with with broker, plan and service settings *

@@ -718,6 +768,12 @@ static Map mergeParameters(BrokerConfig brokerConfig, ServiceDef ret.putAll(service.getServiceSettings()); + List> tags = mergeBucketTags(service, plan, requestParameters); + + if (tags != null) { + ret.put(TAGS, tags); + } + return ret; } diff --git a/src/test/java/com/emc/ecs/common/Fixtures.java b/src/test/java/com/emc/ecs/common/Fixtures.java index d1f51f98..8ddee2a9 100644 --- a/src/test/java/com/emc/ecs/common/Fixtures.java +++ b/src/test/java/com/emc/ecs/common/Fixtures.java @@ -1,5 +1,6 @@ package com.emc.ecs.common; +import com.emc.ecs.management.sdk.model.BucketTagSetRootElement; import com.emc.ecs.servicebroker.model.PlanProxy; import com.emc.ecs.servicebroker.model.ServiceDefinitionProxy; import com.emc.ecs.servicebroker.model.ServiceType; @@ -63,12 +64,17 @@ public class Fixtures { public static final String VOLUME_MOUNT_VALUE = "/mount/dir"; public static final String SECRET_KEY_VALUE = "testKEY@ключ:/-s#cr#T"; public static final String SHOULD_RAISE_AN_EXCEPTION = "should raise an exception"; + public static final String BUCKET_TAGS_INVALID_CHARS = "key?=value!"; + public static final String BUCKET_TAGS_INVALID_FORMAT = "key1:value1;key2:value2"; + public static final String BUCKET_TAGS_LONG_KEY = "very very very very very very very very very very very very very very very very very very very very long key of exactly 129 chars=value"; + public static final String BUCKET_TAGS_LONG_VALUE = "key=very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very great value accurately 257 chars long"; public static final String KEY1 = "tag1"; public static final String KEY2 = "tag2"; public static final String KEY3 = "tag3"; public static final String VALUE1 = "value1"; public static final String VALUE2 = "value2"; public static final String VALUE3 = "value3"; + public static final String BUCKET_TAGS_STRING = KEY1 + "=" + VALUE1 + "," + KEY2 + "=" + VALUE2 + "," + KEY3 + "=" + VALUE3; public static final String MARKER = "marker1"; public static final int PAGE_SIZE = 32; public static final String SYSTEM_METADATA_NAME = "Size"; @@ -567,4 +573,23 @@ public static DeleteServiceInstanceBindingRequest bucketBindingRemoveFixture() { .planId(BUCKET_PLAN_ID1) .build(); } + + public static List> listOfBucketTagsFixture() { + List> tags = new ArrayList<>(); + Map tag1 = new HashMap<>(); + Map tag2 = new HashMap<>(); + Map tag3 = new HashMap<>(); + + tag1.put(BucketTagSetRootElement.KEY, KEY1); + tag1.put(BucketTagSetRootElement.VALUE, VALUE1); + tag2.put(BucketTagSetRootElement.KEY, KEY2); + tag2.put(BucketTagSetRootElement.VALUE, VALUE2); + tag3.put(BucketTagSetRootElement.KEY, KEY3); + tag3.put(BucketTagSetRootElement.VALUE, VALUE3); + + tags.add(tag1); + tags.add(tag2); + tags.add(tag3); + return tags; + } } diff --git a/src/test/java/com/emc/ecs/servicebroker/config/CatalogConfigTest.java b/src/test/java/com/emc/ecs/servicebroker/config/CatalogConfigTest.java index 5c8e5dbd..528c10eb 100644 --- a/src/test/java/com/emc/ecs/servicebroker/config/CatalogConfigTest.java +++ b/src/test/java/com/emc/ecs/servicebroker/config/CatalogConfigTest.java @@ -1,9 +1,13 @@ package com.emc.ecs.servicebroker.config; +import com.emc.ecs.common.Fixtures; + +import org.apache.commons.collections.CollectionUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.ConfigFileApplicationContextInitializer; +import org.springframework.cloud.servicebroker.exception.ServiceBrokerException; import org.springframework.cloud.servicebroker.model.catalog.Catalog; import org.springframework.cloud.servicebroker.model.catalog.Plan; import org.springframework.cloud.servicebroker.model.catalog.ServiceDefinition; @@ -11,12 +15,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; -import static org.junit.Assert.assertEquals; +import static com.emc.ecs.common.Fixtures.*; +import static com.emc.ecs.servicebroker.model.Constants.*; +import static org.junit.Assert.*; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes = Application.class, @@ -66,6 +69,38 @@ public void testEcsBucketPlans() { "S3 protocol access")); } + @Test + public void testBucketTagsValidation() { + Map settingsWithInvalidCharacters = new HashMap<>(); + settingsWithInvalidCharacters.put(BUCKET_TAGS, BUCKET_TAGS_INVALID_CHARS); + + Map settingsWithInvalidFormat = new HashMap<>(); + settingsWithInvalidFormat.put(BUCKET_TAGS, BUCKET_TAGS_INVALID_FORMAT); + + Map settingsWithLongKey = new HashMap<>(); + settingsWithLongKey.put(BUCKET_TAGS, BUCKET_TAGS_LONG_KEY); + + Map settingsWithLongValue = new HashMap<>(); + settingsWithLongValue.put(BUCKET_TAGS, BUCKET_TAGS_LONG_VALUE); + + assertThrows(ServiceBrokerException.class, () -> CatalogConfig.parseBucketTags(settingsWithInvalidCharacters)); + assertThrows(ServiceBrokerException.class, () -> CatalogConfig.parseBucketTags(settingsWithInvalidFormat)); + assertThrows(ServiceBrokerException.class, () -> CatalogConfig.parseBucketTags(settingsWithLongKey)); + assertThrows(ServiceBrokerException.class, () -> CatalogConfig.parseBucketTags(settingsWithLongValue)); + } + + @Test + public void parseBucketTagsTest() { + Map settings = new HashMap<>(); + settings.put(BUCKET_TAGS, Fixtures.BUCKET_TAGS_STRING); + settings = CatalogConfig.parseBucketTags(settings); + + List> resultTags = (List>)settings.get(TAGS); + List> expectedTags = Fixtures.listOfBucketTagsFixture(); + + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + } + private void testServiceDefinition(ServiceDefinition service, String id, String name, String description, boolean bindable, boolean updatable, List requires, String dashboardUrl) { diff --git a/src/test/java/com/emc/ecs/servicebroker/service/EcsServiceTest.java b/src/test/java/com/emc/ecs/servicebroker/service/EcsServiceTest.java index d2bfc354..f14c9eb4 100644 --- a/src/test/java/com/emc/ecs/servicebroker/service/EcsServiceTest.java +++ b/src/test/java/com/emc/ecs/servicebroker/service/EcsServiceTest.java @@ -1593,6 +1593,104 @@ public void changeBucketTagsCreateTest() throws Exception { assertEquals(NAMESPACE_NAME, tagsParam.getNamespace()); } + /** + * A service can merge lists of bucket tags presented in service and plan descriptions and provided + * in request on bucket creation. + *

+ * Bucket tags obey following priority: + *

    + *
  • Plan bucket tags has lower priority than service tags and therefore would be overwritten by last ones
    • + *
  • Requested bucket tags has lower priority than plan tags and therefore would be overwritten by last ones
    • + *
+ *

+ */ + @Test + public void mergeBucketTagsTest() { + Map serviceParams = new HashMap<>(); + Map planParams = new HashMap<>(); + Map requestedParams = new HashMap<>(); + + serviceParams.put(TAGS, createListOfTags(KEY1, VALUE1)); + planParams.put(TAGS, createListOfTags(KEY1, VALUE2, KEY2, VALUE2)); + requestedParams.put(TAGS, createListOfTags(KEY1, VALUE3, KEY2, VALUE3, KEY3, VALUE3)); + + ServiceDefinitionProxy service = new ServiceDefinitionProxy(); + service.setServiceSettings(serviceParams); + + PlanProxy plan = new PlanProxy(); + plan.setServiceSettings(planParams); + + List> resultTags = EcsService.mergeBucketTags(service, plan, requestedParams); + List> expectedTags = createListOfTags(KEY1, VALUE1, KEY2, VALUE2, KEY3, VALUE3); + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + } + + /** + * A service can merge lists of bucket tags presented in service and plan descriptions and provided + * in request on bucket creation when one or more lists are null. + */ + @Test + public void mergeBucketTagsWithNullValueTest() { + Map serviceParams = new HashMap<>(); + Map planParams = new HashMap<>(); + Map requestedParams = new HashMap<>(); + + serviceParams.put(TAGS, createListOfTags(KEY1, VALUE1)); + planParams.put(TAGS, createListOfTags(KEY2, VALUE2)); + requestedParams.put(TAGS, createListOfTags(KEY3, VALUE3)); + + ServiceDefinitionProxy service = new ServiceDefinitionProxy(); + service.setServiceSettings(serviceParams); + + PlanProxy plan = new PlanProxy(); + plan.setServiceSettings(planParams); + + Map serviceParamsNull = new HashMap<>(); + Map planParamsNull = new HashMap<>(); + Map requestedParamsNull = new HashMap<>(); + + serviceParamsNull.put(TAGS, null); + planParamsNull.put(TAGS, null); + requestedParamsNull.put(TAGS, null); + + ServiceDefinitionProxy serviceNull = new ServiceDefinitionProxy(); + serviceNull.setServiceSettings(serviceParamsNull); + + PlanProxy planNull = new PlanProxy(); + planNull.setServiceSettings(planParamsNull); + + List> resultTags = EcsService.mergeBucketTags(serviceNull, planNull, requestedParamsNull); + assertNull(resultTags); + + resultTags = EcsService.mergeBucketTags(serviceNull, planNull, requestedParams); + List> expectedTags = createListOfTags(KEY3, VALUE3); + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + + resultTags = EcsService.mergeBucketTags(serviceNull, plan, requestedParamsNull); + expectedTags = createListOfTags(KEY2, VALUE2); + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + + resultTags = EcsService.mergeBucketTags(service, planNull, requestedParamsNull); + expectedTags = createListOfTags(KEY1, VALUE1); + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + + resultTags = EcsService.mergeBucketTags(service, plan, requestedParamsNull); + expectedTags = createListOfTags(KEY1, VALUE1, KEY2, VALUE2); + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + + resultTags = EcsService.mergeBucketTags(service, planNull, requestedParams); + expectedTags = createListOfTags(KEY1, VALUE1, KEY3, VALUE3); + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + + resultTags = EcsService.mergeBucketTags(serviceNull, plan, requestedParams); + expectedTags = createListOfTags(KEY2, VALUE2, KEY3, VALUE3); + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + + resultTags = EcsService.mergeBucketTags(service, plan, requestedParams); + expectedTags = createListOfTags(KEY1, VALUE1, KEY2, VALUE2, KEY3, VALUE3); + assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags)); + } + private void setupInitTest() throws EcsManagementClientException { DataServiceReplicationGroup rg = new DataServiceReplicationGroup(); rg.setName(RG_NAME);