Migrate custom role providers to licensed feature (elastic#79349)

This changes the license checking for custom role providers to use the
new `LicensedFeature` model with feature tracking.

To support this, and reduce code complexity the logic for managing
role providers has been moved out of the `CompositeRolesStore` into a
new `RoleProviders` test.

Backport of: elastic#79127

x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java

@@ -42,7 +42,6 @@ public class XPackLicenseState {
      */
     public enum Feature {
         SECURITY_AUDITING(OperationMode.GOLD, false),
-        SECURITY_CUSTOM_ROLE_PROVIDERS(OperationMode.PLATINUM, true),
         SECURITY_TOKEN_SERVICE(OperationMode.STANDARD, false),
         SECURITY_AUTHORIZATION_REALM(OperationMode.PLATINUM, true),
         SECURITY_AUTHORIZATION_ENGINE(OperationMode.PLATINUM, true),

x-pack/plugin/core/src/test/java/org/elasticsearch/license/MockLicenseState.java

@@ -9,8 +9,13 @@
 
 import org.elasticsearch.common.settings.Settings;
 
+import org.mockito.Mockito;
+
+import java.util.function.Consumer;
 import java.util.function.LongSupplier;
 
+import static org.mockito.Mockito.doAnswer;
+
 /** A license state that may be mocked by testing because the internal methods are made public */
 public class MockLicenseState extends XPackLicenseState {
 
@@ -32,4 +37,18 @@ public void enableUsageTracking(LicensedFeature feature, String contextName) {
     public void disableUsageTracking(LicensedFeature feature, String contextName) {
         super.disableUsageTracking(feature, contextName);
     }
+
+    public static MockLicenseState createMock() {
+        MockLicenseState mock = Mockito.mock(MockLicenseState.class);
+        Mockito.when(mock.copyCurrentLicenseState()).thenReturn(mock);
+        return mock;
+    }
+
+    public static void acceptListeners(MockLicenseState licenseState, Consumer<LicenseStateListener> addListener) {
+        doAnswer(inv -> {
+            final LicenseStateListener listener = (LicenseStateListener) inv.getArguments()[0];
+            addListener.accept(listener);
+            return null;
+        }).when(licenseState).addListener(Mockito.any());
+    }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java

@@ -93,7 +93,6 @@ public void testSecurityDefaults() {
         XPackLicenseState licenseState = new XPackLicenseState(settings, () -> 0);
         assertThat(licenseState.isSecurityEnabled(), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(true));
 
         licenseState = TestUtils.newTestLicenseState();
         assertSecurityNotAllowed(licenseState);
@@ -111,7 +110,6 @@ public void testSecurityBasicWithoutExplicitSecurityEnabled() {
 
         assertThat(licenseState.isSecurityEnabled(), is(false));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(false));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(false));
         assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(false));
 
         assertThat(licenseState.isSecurityEnabled(), is(false));
@@ -123,7 +121,6 @@ public void testSecurityBasicWithExplicitSecurityEnabled() {
         licenseState.update(BASIC, true, null);
         assertThat(licenseState.isSecurityEnabled(), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(false));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(false));
         assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(false));
 
         assertThat(licenseState.isSecurityEnabled(), is(true));
@@ -137,7 +134,6 @@ public void testSecurityStandard() {
 
         assertThat(licenseState.isSecurityEnabled(), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(false));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(false));
         assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true));
     }
 
@@ -149,7 +145,6 @@ public void testSecurityStandardExpired() {
 
         assertThat(licenseState.isSecurityEnabled(), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(false));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(false));
         assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true));
     }
 
@@ -161,7 +156,6 @@ public void testSecurityGold() {
 
         assertThat(licenseState.isSecurityEnabled(), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(false));
         assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true));
     }
 
@@ -173,7 +167,6 @@ public void testSecurityGoldExpired() {
 
         assertThat(licenseState.isSecurityEnabled(), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(false));
         assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true));
     }
 
@@ -185,7 +178,6 @@ public void testSecurityPlatinum() {
 
         assertThat(licenseState.isSecurityEnabled(), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true));
     }
 
@@ -197,7 +189,6 @@ public void testSecurityPlatinumExpired() {
 
         assertThat(licenseState.isSecurityEnabled(), is(true));
         assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true));
-        assertThat(licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS), is(false));
         assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true));
     }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -240,6 +240,7 @@
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
+import org.elasticsearch.xpack.security.authz.store.RoleProviders;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
 import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore;
 import org.elasticsearch.xpack.security.operator.OperatorOnlyRegistry;
@@ -311,6 +312,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -371,6 +373,10 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin,
     public static final LicensedFeature.Persistent CUSTOM_REALMS_FEATURE =
         LicensedFeature.persistentLenient(REALMS_FEATURE_FAMILY, "custom", License.OperationMode.PLATINUM);
 
+    // Custom role providers are Platinum+
+    public static final LicensedFeature.Persistent CUSTOM_ROLE_PROVIDERS_FEATURE =
+        LicensedFeature.persistent(null, "security-roles-provider", License.OperationMode.PLATINUM);
+
     private static final Logger logger = LogManager.getLogger(Security.class);
 
     public static final SystemIndexDescriptor SECURITY_MAIN_INDEX_DESCRIPTOR = getSecurityMainIndexDescriptor();
@@ -420,7 +426,6 @@ public Security(Settings settings, final Path configPath) {
             this.bootstrapChecks.set(Collections.emptyList());
         }
         this.securityExtensions.addAll(extensions);
-
     }
 
     private static void runStartupChecks(Settings settings) {
@@ -580,9 +585,15 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
             xContentRegistry);
         final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client, getLicenseState(), securityIndex.get());
         final ReservedRolesStore reservedRolesStore = new ReservedRolesStore();
-        List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> rolesProviders = new ArrayList<>();
+
+        final Map<String, List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>>> customRoleProviders = new LinkedHashMap<>();
         for (SecurityExtension extension : securityExtensions) {
-            rolesProviders.addAll(extension.getRolesProviders(extensionComponents));
+            final List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> providers = extension.getRolesProviders(
+                extensionComponents
+            );
+            if (providers != null && providers.isEmpty() == false) {
+                customRoleProviders.put(extension.toString(), providers);
+            }
         }
 
         final ApiKeyService apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, getLicenseState(), securityIndex.get(),
@@ -604,9 +615,17 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         );
         components.add(serviceAccountService);
 
-        final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore,
-            privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache, apiKeyService,
-            serviceAccountService, dlsBitsetCache.get(), new DeprecationRoleDescriptorConsumer(clusterService, threadPool));
+        final RoleProviders roleProviders = new RoleProviders(
+            reservedRolesStore,
+            fileRolesStore,
+            nativeRolesStore,
+            customRoleProviders,
+            getLicenseState()
+        );
+        final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, roleProviders,
+            privilegeStore, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache, apiKeyService,
+            serviceAccountService, dlsBitsetCache.get(), 
+            new DeprecationRoleDescriptorConsumer(clusterService, threadPool));
         securityIndex.get().addStateListener(allRolesStore::onSecurityIndexStateChange);
 
         // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java

@@ -28,7 +28,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.license.XPackLicenseState.Feature;
 import org.elasticsearch.xpack.core.common.IteratingActionListener;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
@@ -81,8 +80,8 @@
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed;
 
 /**
- * A composite roles store that combines built in roles, file-based roles, and index-based roles. Checks the built in roles first, then the
- * file roles, and finally the index roles.
+ * A composite roles store that can retrieve roles from multiple sources.
+ * @see RoleProviders
  */
 public class CompositeRolesStore {
 
@@ -95,8 +94,7 @@ public class CompositeRolesStore {
 
     private final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(CompositeRolesStore.class);
 
-    private final FileRolesStore fileRolesStore;
-    private final NativeRolesStore nativeRolesStore;
+    private final RoleProviders roleProviders;
     private final NativePrivilegeStore privilegeStore;
     private final XPackLicenseState licenseState;
     private final Consumer<Collection<RoleDescriptor>> effectiveRoleDescriptorsConsumer;
@@ -111,21 +109,27 @@ public class CompositeRolesStore {
     private final ApiKeyService apiKeyService;
     private final ServiceAccountService serviceAccountService;
     private final boolean isAnonymousEnabled;
-    private final List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> builtInRoleProviders;
-    private final List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> allRoleProviders;
 
-    public CompositeRolesStore(Settings settings, FileRolesStore fileRolesStore, NativeRolesStore nativeRolesStore,
-                               ReservedRolesStore reservedRolesStore, NativePrivilegeStore privilegeStore,
-                               List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> rolesProviders,
+    public CompositeRolesStore(Settings settings, RoleProviders roleProviders, NativePrivilegeStore privilegeStore,
                                ThreadContext threadContext, XPackLicenseState licenseState, FieldPermissionsCache fieldPermissionsCache,
                                ApiKeyService apiKeyService, ServiceAccountService serviceAccountService,
                                DocumentSubsetBitsetCache dlsBitsetCache,
                                Consumer<Collection<RoleDescriptor>> effectiveRoleDescriptorsConsumer) {
-        this.fileRolesStore = Objects.requireNonNull(fileRolesStore);
-        this.dlsBitsetCache = Objects.requireNonNull(dlsBitsetCache);
-        fileRolesStore.addListener(this::invalidate);
-        this.nativeRolesStore = Objects.requireNonNull(nativeRolesStore);
+        this.roleProviders = roleProviders;
+        roleProviders.addChangeListener(new RoleProviders.ChangeListener() {
+            @Override
+            public void rolesChanged(Set<String> roles) {
+                CompositeRolesStore.this.invalidate(roles);
+            }
+
+            @Override
+            public void providersChanged() {
+                CompositeRolesStore.this.invalidateAll();
+            }
+        });
+
         this.privilegeStore = Objects.requireNonNull(privilegeStore);
+        this.dlsBitsetCache = Objects.requireNonNull(dlsBitsetCache);
         this.licenseState = Objects.requireNonNull(licenseState);
         this.fieldPermissionsCache = Objects.requireNonNull(fieldPermissionsCache);
         this.apiKeyService = Objects.requireNonNull(apiKeyService);
@@ -145,16 +149,6 @@ public CompositeRolesStore(Settings settings, FileRolesStore fileRolesStore, Nat
             nlcBuilder.setMaximumWeight(nlcCacheSize);
         }
         this.negativeLookupCache = nlcBuilder.build();
-        this.builtInRoleProviders = Collections.unmodifiableList(Arrays.asList(reservedRolesStore, fileRolesStore, nativeRolesStore));
-        if (rolesProviders.isEmpty()) {
-            this.allRoleProviders = this.builtInRoleProviders;
-        } else {
-            List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> allList =
-                new ArrayList<>(builtInRoleProviders.size() + rolesProviders.size());
-            allList.addAll(builtInRoleProviders);
-            allList.addAll(rolesProviders);
-            this.allRoleProviders = Collections.unmodifiableList(allList);
-        }
         this.anonymousUser = new AnonymousUser(settings);
         this.isAnonymousEnabled = AnonymousUser.isAnonymousEnabled(settings);
     }
@@ -386,9 +380,7 @@ private void roleDescriptors(Set<String> roleNames, ActionListener<RolesRetrieva
 
     private void loadRoleDescriptorsAsync(Set<String> roleNames, ActionListener<RolesRetrievalResult> listener) {
         final RolesRetrievalResult rolesResult = new RolesRetrievalResult();
-        final List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> asyncRoleProviders =
-            licenseState.checkFeature(Feature.SECURITY_CUSTOM_ROLE_PROVIDERS) ? allRoleProviders : builtInRoleProviders;
-
+        final List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> asyncRoleProviders = roleProviders.getProviders();
         final ActionListener<RoleRetrievalResult> descriptorsListener =
             ContextPreservingActionListener.wrapPreservingContext(ActionListener.wrap(ignore -> {
                     rolesResult.setMissingRoles(roleNames);
@@ -519,13 +511,12 @@ public void invalidate(Set<String> roles) {
     }
 
     public void usageStats(ActionListener<Map<String, Object>> listener) {
-        final Map<String, Object> usage = new HashMap<>(2);
-        usage.put("file", fileRolesStore.usageStats());
+        final Map<String, Object> usage = new HashMap<>();
         usage.put("dls", Collections.singletonMap("bit_set_cache", dlsBitsetCache.usageStats()));
-        nativeRolesStore.usageStats(ActionListener.wrap(map -> {
-            usage.put("native", map);
-            listener.onResponse(usage);
-        }, listener::onFailure));
+        roleProviders.usageStats(listener.map(roleUsage -> {
+            usage.putAll(roleUsage);
+            return usage;
+        }));
     }
 
     public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) {