Skip to content

Commit

Permalink
Refactor BWC handling to support more flexible testing scenarios (#95803
Browse files Browse the repository at this point in the history
)
  • Loading branch information
mark-vieira authored May 3, 2023
1 parent b5413e5 commit 6ace408
Show file tree
Hide file tree
Showing 18 changed files with 291 additions and 220 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,11 @@ public void apply(Project project) {
TaskProvider<LoggedExec> addRemoteTaskProvider = tasks.register("addRemote", LoggedExec.class, addRemote -> {
addRemote.dependsOn(findRemoteTaskProvider);
addRemote.onlyIf("remote exists", task -> ((boolean) extraProperties.get("remoteExists")) == false);
addRemote.getWorkingDir().set(gitExtension.getCheckoutDir().get());
addRemote.getWorkingDir().set(gitExtension.getCheckoutDir());
String remoteRepo = remote.get();
// for testing only we can override the base remote url
String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
.getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git");
.getOrElse("https://github.com/" + remoteRepo + "/" + project.getRootProject().getName());
addRemote.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl);
});

Expand All @@ -105,8 +105,9 @@ public void apply(Project project) {
});
fetchLatest.onlyIf("online and gitFetchLatest == true", t -> isOffline == false && gitFetchLatest.get());
fetchLatest.dependsOn(addRemoteTaskProvider);
fetchLatest.getWorkingDir().set(gitExtension.getCheckoutDir().get());
fetchLatest.commandLine("git", "fetch", "--all");
fetchLatest.getWorkingDir().set(gitExtension.getCheckoutDir());
// Fetch latest from remotes, including tags, overriding any existing local refs
fetchLatest.commandLine("git", "fetch", "--all", "--tags", "--force");
});

String projectPath = project.getPath();
Expand All @@ -119,13 +120,20 @@ public void execute(Task task) {
String bwcBranch = gitExtension.getBwcBranch().get();
final String refspec = providerFactory.systemProperty("bwc.refspec." + bwcBranch)
.orElse(providerFactory.systemProperty("tests.bwc.refspec." + bwcBranch))
.orElse(
providerFactory.provider(
() -> task.getExtensions().getExtraProperties().has("refspec")
? task.getExtensions().getExtraProperties().get("refspec").toString()
: null
)
)
.getOrElse(remote.get() + "/" + bwcBranch);

String effectiveRefSpec = maybeAlignedRefSpec(task.getLogger(), refspec);
task.getLogger().lifecycle("Performing checkout of {}...", refspec);
LoggedExec.exec(execOperations, spec -> {
spec.workingDir(checkoutDir);
spec.commandLine("git", "checkout", effectiveRefSpec);
spec.commandLine("git", "checkout", "--recurse-submodules", effectiveRefSpec);
});

String checkoutHash = GitInfo.gitInfo(checkoutDir).getRevision();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,11 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
return projectName + distribution.getType().getName();
}

private static class ProjectBasedDistributionDependency implements DistributionDependency {
public static class ProjectBasedDistributionDependency implements DistributionDependency {

private Function<String, Dependency> function;

ProjectBasedDistributionDependency(Function<String, Dependency> function) {
public ProjectBasedDistributionDependency(Function<String, Dependency> function) {
this.function = function;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,7 @@ public void apply(Project project) {
File runtimeJavaHome = findRuntimeJavaHome();
boolean isRuntimeJavaHomeSet = Jvm.current().getJavaHome().equals(runtimeJavaHome) == false;

File rootDir = project.getRootDir();
GitInfo gitInfo = GitInfo.gitInfo(rootDir);
GitInfo gitInfo = GitInfo.gitInfo(project.getRootDir());

BuildParams.init(params -> {
params.reset();
Expand Down Expand Up @@ -132,7 +131,13 @@ public void apply(Project project) {
params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));
AtomicReference<BwcVersions> cache = new AtomicReference<>();
params.setBwcVersions(providers.provider(() -> cache.updateAndGet(val -> val == null ? resolveBwcVersions(rootDir) : val)));
params.setBwcVersions(
providers.provider(
() -> cache.updateAndGet(
val -> val == null ? resolveBwcVersions(Util.locateElasticsearchWorkspace(project.getGradle())) : val
)
)
);
});

// Enforce the minimum compiler version
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ public void apply(Project project) {
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME);

project.getDependencies().add(javaTestSourceSet.getImplementationConfigurationName(), project.project(":test:test-clusters"));
if (project.findProject(":test:test-clusters") != null) {
project.getDependencies().add(javaTestSourceSet.getImplementationConfigurationName(), project.project(":test:test-clusters"));
}

// setup the javaRestTest task
// we use a StandloneRestIntegTestTask here so that the conventions of RestTestBasePlugin don't create a test cluster
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ public Void call(Object... args) {
task.getExtensions().getExtraProperties().set("usesBwcDistribution", new Closure<Void>(task) {
@Override
public Void call(Object... args) {
if (args.length != 1 && args[0] instanceof Version == false) {
if (args.length != 1 || args[0] instanceof Version == false) {
throw new IllegalArgumentException("Expected exactly one argument of type org.elasticsearch.gradle.Version");
}

Expand All @@ -178,7 +178,7 @@ public Void call(Object... args) {
providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
);

if (version.before(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion())) {
if (version.getMajor() > 0 && version.before(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion())) {
// If we are upgrade testing older versions we also need to upgrade to 7.last
this.call(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion());
}
Expand Down
6 changes: 2 additions & 4 deletions test/test-clusters/build.gradle
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
import org.elasticsearch.gradle.internal.conventions.util.Util

apply plugin: 'elasticsearch.java'
apply plugin: 'com.github.johnrengelman.shadow'

dependencies {
shadow "junit:junit:${versions.junit}"
shadow "org.apache.logging.log4j:log4j-api:${versions.log4j}"

api "junit:junit:${versions.junit}"
implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,4 +106,9 @@ public interface ClusterHandle extends Closeable {
* @param version version to upgrade to
*/
void upgradeToVersion(Version version);

/**
* Cleans up any resources created by this cluster.
*/
void close();
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,22 @@
* Side Public License, v 1.
*/

package org.elasticsearch.test.cluster.local;
package org.elasticsearch.test.cluster;

import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.local.distribution.LocalDistributionResolver;
import org.elasticsearch.test.cluster.local.distribution.ReleasedDistributionResolver;
import org.elasticsearch.test.cluster.local.distribution.SnapshotDistributionResolver;
import org.elasticsearch.test.cluster.util.Version;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;

public class LocalElasticsearchCluster implements ElasticsearchCluster {
private final DefaultLocalClusterSpecBuilder builder;
private LocalClusterSpec spec;
private LocalClusterHandle handle;
import java.util.function.Supplier;

public LocalElasticsearchCluster(DefaultLocalClusterSpecBuilder builder) {
this.builder = builder;
public class DefaultElasticsearchCluster<S extends ClusterSpec, H extends ClusterHandle> implements ElasticsearchCluster {
private final Supplier<S> specProvider;
private final ClusterFactory<S, H> clusterFactory;
private H handle;

public DefaultElasticsearchCluster(Supplier<S> specProvider, ClusterFactory<S, H> clusterFactory) {
this.specProvider = specProvider;
this.clusterFactory = clusterFactory;
}

@Override
Expand All @@ -31,10 +30,8 @@ public Statement apply(Statement base, Description description) {
@Override
public void evaluate() throws Throwable {
try {
spec = builder.buildClusterSpec();
handle = new LocalClusterFactory(
new LocalDistributionResolver(new SnapshotDistributionResolver(new ReleasedDistributionResolver()))
).create(spec);
S spec = specProvider.get();
handle = clusterFactory.create(spec);
handle.start();
base.evaluate();
} finally {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@
public interface ElasticsearchCluster extends TestRule, ClusterHandle {

/**
* Creates a new {@link DefaultLocalClusterSpecBuilder} for defining a locally orchestrated cluster. Local clusters use a locally built
* Creates a new {@link LocalClusterSpecBuilder} for defining a locally orchestrated cluster. Local clusters use a locally built
* Elasticsearch distribution.
*
* @return a builder for a local cluster
*/
static LocalClusterSpecBuilder local() {
static LocalClusterSpecBuilder<ElasticsearchCluster> local() {
return new DefaultLocalClusterSpecBuilder();
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.test.cluster.local;

import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
import org.elasticsearch.test.cluster.local.model.User;
import org.elasticsearch.test.cluster.util.Version;
import org.elasticsearch.test.cluster.util.resource.Resource;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.Supplier;

public abstract class AbstractLocalClusterSpecBuilder<T extends ElasticsearchCluster> extends AbstractLocalSpecBuilder<
LocalClusterSpecBuilder<T>> implements LocalClusterSpecBuilder<T> {

private String name = "test-cluster";
private final List<DefaultLocalNodeSpecBuilder> nodeBuilders = new ArrayList<>();
private final List<User> users = new ArrayList<>();
private final List<Resource> roleFiles = new ArrayList<>();
private final List<Supplier<LocalClusterConfigProvider>> lazyConfigProviders = new ArrayList<>();

public AbstractLocalClusterSpecBuilder() {
super(null);
}

@Override
public AbstractLocalClusterSpecBuilder<T> name(String name) {
this.name = name;
return this;
}

@Override
public AbstractLocalClusterSpecBuilder<T> apply(LocalClusterConfigProvider configProvider) {
configProvider.apply(this);
return this;
}

@Override
public LocalClusterSpecBuilder<T> apply(Supplier<LocalClusterConfigProvider> configProvider) {
lazyConfigProviders.add(configProvider);
return this;
}

@Override
public AbstractLocalClusterSpecBuilder<T> nodes(int nodes) {
if (nodes < nodeBuilders.size()) {
throw new IllegalArgumentException(
"Cannot shrink cluster to " + nodes + ". " + nodeBuilders.size() + " nodes already configured"
);
}

int newNodes = nodes - nodeBuilders.size();
for (int i = 0; i < newNodes; i++) {
nodeBuilders.add(new DefaultLocalNodeSpecBuilder(this));
}

return this;
}

@Override
public AbstractLocalClusterSpecBuilder<T> withNode(Consumer<? super LocalNodeSpecBuilder> config) {
DefaultLocalNodeSpecBuilder builder = new DefaultLocalNodeSpecBuilder(this);
config.accept(builder);
nodeBuilders.add(builder);
return this;
}

@Override
public AbstractLocalClusterSpecBuilder<T> node(int index, Consumer<? super LocalNodeSpecBuilder> config) {
try {
DefaultLocalNodeSpecBuilder builder = nodeBuilders.get(index);
config.accept(builder);
} catch (IndexOutOfBoundsException e) {
throw new IllegalArgumentException(
"No node at index + " + index + " exists. Only " + nodeBuilders.size() + " nodes have been configured"
);
}
return this;
}

@Override
public AbstractLocalClusterSpecBuilder<T> user(String username, String password) {
this.users.add(new User(username, password));
return this;
}

@Override
public AbstractLocalClusterSpecBuilder<T> user(String username, String password, String role) {
this.users.add(new User(username, password, role));
return this;
}

@Override
public AbstractLocalClusterSpecBuilder<T> rolesFile(Resource rolesFile) {
this.roleFiles.add(rolesFile);
return this;
}

protected LocalClusterSpec buildClusterSpec() {
// Apply lazily provided configuration
lazyConfigProviders.forEach(s -> s.get().apply(this));

List<User> clusterUsers = users.isEmpty() ? List.of(User.DEFAULT_USER) : users;
LocalClusterSpec clusterSpec = new LocalClusterSpec(name, clusterUsers, roleFiles);
List<LocalNodeSpec> nodeSpecs;

if (nodeBuilders.isEmpty()) {
// No node-specific configuration so assume a single-node cluster
nodeSpecs = List.of(new DefaultLocalNodeSpecBuilder(this).build(clusterSpec));
} else {
nodeSpecs = nodeBuilders.stream().map(node -> node.build(clusterSpec)).toList();
}

clusterSpec.setNodes(nodeSpecs);
clusterSpec.validate();

return clusterSpec;
}

public static class DefaultLocalNodeSpecBuilder extends AbstractLocalSpecBuilder<LocalNodeSpecBuilder> implements LocalNodeSpecBuilder {
private String name;

protected DefaultLocalNodeSpecBuilder(AbstractLocalSpecBuilder<?> parent) {
super(parent);
}

@Override
public DefaultLocalNodeSpecBuilder name(String name) {
this.name = name;
return this;
}

private LocalNodeSpec build(LocalClusterSpec cluster) {

return new LocalNodeSpec(
cluster,
name,
Optional.ofNullable(getVersion()).orElse(Version.CURRENT),
getSettingsProviders(),
getSettings(),
getEnvironmentProviders(),
getEnvironment(),
getModules(),
getPlugins(),
Optional.ofNullable(getDistributionType()).orElse(DistributionType.INTEG_TEST),
getFeatures(),
getKeystoreProviders(),
getKeystoreSettings(),
getKeystoreFiles(),
getKeystorePassword(),
getExtraConfigFiles(),
getSystemProperties(),
getSecrets()
);
}
}
}
Loading

0 comments on commit 6ace408

Please sign in to comment.