Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Exposing the ability to log deprecated settings at non-critical level #79107

Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,14 @@ public HttpEntity getEntity() {

/**
* Tests if a string matches the RFC 7234 specification for warning headers.
* This assumes that the warn code is always 299 and the warn agent is always
* Elasticsearch.
* This assumes that the warn code is always 299 or 300 and the warn agent is
* always Elasticsearch.
*
* @param s the value of a warning header formatted according to RFC 7234
* @return {@code true} if the input string matches the specification
*/
private static boolean matchWarningHeaderPatternByPrefix(final String s) {
return s.startsWith("299 Elasticsearch-");
return s.startsWith("299 Elasticsearch-") || s.startsWith("300 Elasticsearch-");
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ public void testCompatibleFieldDeclarations() throws IOException {
RestApiVersion.minimumSupported());
StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null);
assertEquals(1, o.intField);
assertWarnings(false, "[struct_with_compatible_fields][1:14] " +
assertWarnings(false, false, "[struct_with_compatible_fields][1:14] " +
"Deprecated field [old_name] used, expected [new_name] instead");
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ class TestStruct {
objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
objectParser.parse(parser, s, null);
assertEquals("foo", s.test);
assertWarnings(false, "[foo][1:15] Deprecated field [old_test] used, expected [test] instead");
assertWarnings(false, false, "[foo][1:15] Deprecated field [old_test] used, expected [test] instead");
}

public void testFailOnValueType() throws IOException {
Expand Down Expand Up @@ -1072,7 +1072,7 @@ public void testCompatibleFieldDeclarations() throws IOException {
RestApiVersion.minimumSupported());
StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null);
assertEquals(1, o.intField);
assertWarnings(false, "[struct_with_compatible_fields][1:14] " +
assertWarnings(false, false, "[struct_with_compatible_fields][1:14] " +
"Deprecated field [old_name] used, expected [new_name] instead");

}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -470,8 +470,8 @@ public Processor create(

boolean valid = metadata.isValid(currentState.metadata().settings());
if (valid && metadata.isCloseToExpiration()) {
HeaderWarning.addWarning("database [{}] was not updated for over 25 days, geoip processor will stop working if there " +
"is no update for 30 days", databaseFile);
HeaderWarning.addWarning(DeprecationLogger.CRITICAL, "database [{}] was not updated for over 25 days, geoip processor" +
" will stop working if there is no update for 30 days", databaseFile);
}

return valid;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.test.ESTestCase;
Expand Down Expand Up @@ -59,8 +60,8 @@ public void testDeprecationOfLoneAccessKey() {
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
assertThat(credentials.getAWSSecretKey(), is(""));
assertSettingDeprecationsAndWarnings(new String[]{},
"Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not, which will be unsupported in future");
assertSettingDeprecationsAndWarnings(new Setting<?>[]{},
false, "Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not, which will be unsupported in future");
}

public void testDeprecationOfLoneSecretKey() {
Expand All @@ -70,8 +71,8 @@ public void testDeprecationOfLoneSecretKey() {
Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials();
assertThat(credentials.getAWSAccessKeyId(), is(""));
assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
assertSettingDeprecationsAndWarnings(new String[]{},
"Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not, which will be unsupported in future");
assertSettingDeprecationsAndWarnings(new Setting<?>[]{},
false, "Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not, which will be unsupported in future");
}

public void testRejectionOfLoneSessionToken() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public void testLoad() throws Exception {
}

if (deprecationWarnings.isEmpty() == false) {
assertSettingDeprecationsAndWarnings(deprecationWarnings.toArray(new Setting<?>[0]));
assertSettingDeprecationsAndWarnings(deprecationWarnings.toArray(new Setting<?>[0]), false);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ public void testClientInitializer() throws Exception {
expectThrows(IllegalArgumentException.class, () -> service.client("another_client", "repo", statsCollector));
assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
assertSettingDeprecationsAndWarnings(
new Setting<?>[] { GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING.getConcreteSettingForNamespace(clientName) });
new Setting<?>[] { GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING.getConcreteSettingForNamespace(clientName) },
false);
final Storage storage = service.client(clientName, "repo", statsCollector);
assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ public void testDeprecatedSettings() throws IOException, UserException {
final int iterations = randomIntBetween(0, 128);
for (int i = 0; i < iterations; i++) {
setting.get(settings);
assertSettingDeprecationsAndWarnings(new Setting<?>[]{setting});
assertSettingDeprecationsAndWarnings(new Setting<?>[]{setting}, false);
}

final String deprecationPath =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateUpdateTask;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
Expand Down Expand Up @@ -498,7 +499,7 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo
.collect(Collectors.joining(",")),
name);
logger.warn(warning);
HeaderWarning.addWarning(warning);
HeaderWarning.addWarning(DeprecationLogger.CRITICAL, warning);
}

ComposableIndexTemplate finalIndexTemplate = template;
Expand Down Expand Up @@ -828,7 +829,7 @@ static ClusterState innerPutTemplate(final ClusterState currentState, PutRequest
.collect(Collectors.joining(",")),
request.name);
logger.warn(warning);
HeaderWarning.addWarning(warning);
HeaderWarning.addWarning(DeprecationLogger.CRITICAL, warning);
} else {
// Otherwise, this is a hard error, the user should use V2 index templates instead
String error = String.format(Locale.ROOT, "legacy template [%s] has index patterns %s matching patterns" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

package org.elasticsearch.common.logging;

import org.apache.logging.log4j.Level;
import org.elasticsearch.Build;
import org.elasticsearch.Version;
import org.elasticsearch.common.util.concurrent.ThreadContext;
Expand All @@ -32,10 +33,11 @@
public class HeaderWarning {
/**
* Regular expression to test if a string matches the RFC7234 specification for warning headers. This pattern assumes that the warn code
* is always 299. Further, this pattern assumes that the warn agent represents a version of Elasticsearch including the build hash.
* is always 299 or 300. Further, this pattern assumes that the warn agent represents a version of Elasticsearch including the build
* hash.
*/
public static final Pattern WARNING_HEADER_PATTERN = Pattern.compile(
"299 " + // warn code
"(?:299|300) " + // log level code
"Elasticsearch-" + // warn agent
"\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-" + // warn agent
"(?:[a-f0-9]{7}(?:[a-f0-9]{33})?|unknown) " + // warn agent
Expand All @@ -53,15 +55,16 @@ public class HeaderWarning {

/*
* RFC7234 specifies the warning format as warn-code <space> warn-agent <space> "warn-text" [<space> "warn-date"]. Here, warn-code is a
* three-digit number with various standard warn codes specified. The warn code 299 is apt for our purposes as it represents a
* miscellaneous persistent warning (can be presented to a human, or logged, and must not be removed by a cache). The warn-agent is an
* arbitrary token; here we use the Elasticsearch version and build hash. The warn text must be quoted. The warn-date is an optional
* quoted field that can be in a variety of specified date formats; here we use RFC 1123 format.
* three-digit number with various standard warn codes specified, and is left off of this static prefix so that it can be added based
* on the log level received. The warn code will be either 299 or 300 at runtime, which are apt for our purposes as
* they represent miscellaneous persistent warnings (can be presented to a human, or logged, and must not be removed by a cache).
* The warn-agent is an arbitrary token; here we use the Elasticsearch version and build hash. The warn text must be quoted. The
* warn-date is an optional quoted field that can be in a variety of specified date formats; here we use RFC 1123 format.
*/
private static final String WARNING_PREFIX =
String.format(
Locale.ROOT,
"299 Elasticsearch-%s%s-%s",
" Elasticsearch-%s%s-%s",
Version.CURRENT.toString(),
Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "",
Build.CURRENT.hash());
Expand Down Expand Up @@ -189,14 +192,15 @@ private static boolean assertWarningValue(final String s, final String warningVa
* Format a warning string in the proper warning format by prepending a warn code, warn agent, wrapping the warning string in quotes,
* and appending the RFC 7231 date.
*
* @param level the level of the warning - Level.WARN or DeprecationLogger.CRITICAL
* @param s the warning string to format
* @return a warning value formatted according to RFC 7234
*/
public static String formatWarning(final String s) {
public static String formatWarning(final Level level, final String s) {
// Assume that the common scenario won't have a string to escape and encode.
int length = WARNING_PREFIX.length() + s.length() + 3;
int length = WARNING_PREFIX.length() + s.length() + 6;
final StringBuilder sb = new StringBuilder(length);
sb.append(WARNING_PREFIX).append(" \"").append(escapeAndEncode(s)).append("\"");
sb.append(level.intLevel() + WARNING_PREFIX).append(" \"").append(escapeAndEncode(s)).append("\"");
return sb.toString();
}

Expand Down Expand Up @@ -310,16 +314,21 @@ public static String getXOpaqueId() {
.orElse("");
}

public static void addWarning(String message, Object... params) {
addWarning(THREAD_CONTEXT, message, params);
public static void addWarning(Level level, String message, Object... params) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rather than adding another public method, can you please update the existing uses? It looks like there are ~12 calls to the existing method, and most are in tests, it should be easy to update with eg IntelliJ refactoring tools to add a parameter.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I went back and forth on this. I actually did this and backed it out in an attempt to make this PR smaller. I'll consolidate it into one method.

addWarning(THREAD_CONTEXT, level, message, params);
}

// package scope for testing
static void addWarning(Set<ThreadContext> threadContexts, String message, Object... params) {
addWarning(threadContexts, DeprecationLogger.CRITICAL, message, params);
}

// package scope for testing
static void addWarning(Set<ThreadContext> threadContexts, Level level, String message, Object... params) {
final Iterator<ThreadContext> iterator = threadContexts.iterator();
if (iterator.hasNext()) {
final String formattedMessage = LoggerMessageFormat.format(message, params);
final String warningHeaderValue = formatWarning(formattedMessage);
final String warningHeaderValue = formatWarning(level, formattedMessage);
assert WARNING_HEADER_PATTERN.matcher(warningHeaderValue).matches();
assert extractWarningValueFromWarningHeader(warningHeaderValue, false)
.equals(escapeAndEncode(formattedMessage));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ public void append(LogEvent event) {
String messagePattern = esLogMessage.getMessagePattern();
Object[] arguments = esLogMessage.getArguments();

HeaderWarning.addWarning(messagePattern, arguments);
HeaderWarning.addWarning(event.getLevel(), messagePattern, arguments);
} else {
final String formattedMessage = event.getMessage().getFormattedMessage();
HeaderWarning.addWarning(formattedMessage);
HeaderWarning.addWarning(event.getLevel(), formattedMessage);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,15 @@ public enum Property {
Final,

/**
* mark this setting as deprecated
* mark this setting as deprecated (critical level)
*/
Deprecated,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be nice, as a followup (since it will touch so many files), to rename this.


/**
* mark this setting as deprecated (warning level)
*/
DeprecatedWarning,

/**
* Node scope
*/
Expand Down Expand Up @@ -169,6 +174,9 @@ private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings
if (propertiesAsSet.contains(Property.Dynamic) && propertiesAsSet.contains(Property.OperatorDynamic)) {
throw new IllegalArgumentException("setting [" + key + "] cannot be both dynamic and operator dynamic");
}
if (propertiesAsSet.contains(Property.Deprecated) && propertiesAsSet.contains(Property.DeprecatedWarning)) {
throw new IllegalArgumentException("setting [" + key + "] cannot be deprecated at both critical and warning levels");
}
checkPropertyRequiresIndexScope(propertiesAsSet, Property.NotCopyableOnResize);
checkPropertyRequiresIndexScope(propertiesAsSet, Property.InternalIndex);
checkPropertyRequiresIndexScope(propertiesAsSet, Property.PrivateIndex);
Expand Down Expand Up @@ -370,8 +378,12 @@ public boolean hasIndexScope() {
/**
* Returns <code>true</code> if this setting is deprecated, otherwise <code>false</code>
*/
public boolean isDeprecated() {
return properties.contains(Property.Deprecated);
private boolean isDeprecated() {
return properties.contains(Property.Deprecated) || properties.contains(Property.DeprecatedWarning);
}

private boolean isDeprecatedWarningOnly() {
return properties.contains(Property.DeprecatedWarning);
}

/**
Expand Down Expand Up @@ -538,6 +550,8 @@ String innerGetRaw(final Settings settings) {
if (secureSettings != null && secureSettings.getSettingNames().contains(key)) {
throw new IllegalArgumentException("Setting [" + key + "] is a non-secure setting" +
" and must be stored inside elasticsearch.yml, but was found inside the Elasticsearch keystore");


}
final String found = settings.get(key);
if (found != null) {
Expand All @@ -552,13 +566,15 @@ void checkDeprecation(Settings settings) {
if (this.isDeprecated() && this.exists(settings)) {
// It would be convenient to show its replacement key, but replacement is often not so simple
final String key = getKey();

List<String> skipTheseDeprecations = settings.getAsList("deprecation.skip_deprecated_settings");
if (Regex.simpleMatch(skipTheseDeprecations, key) == false) {
Settings.DeprecationLoggerHolder.deprecationLogger
.critical(DeprecationCategory.SETTINGS, key,
"[{}] setting was deprecated in Elasticsearch and will be removed in a future release! "
+ "See the breaking changes documentation for the next major version.", key);
String message = "[{}] setting was deprecated in Elasticsearch and will be removed in a future release! "
+ "See the breaking changes documentation for the next major version.";
if (this.isDeprecatedWarningOnly()) {
Settings.DeprecationLoggerHolder.deprecationLogger.warn(DeprecationCategory.SETTINGS, key, message, key);
} else {
Settings.DeprecationLoggerHolder.deprecationLogger.critical(DeprecationCategory.SETTINGS, key, message, key);
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,15 +193,15 @@ public void testDeprecationLogging() {

final Settings toApplyDebug = Settings.builder().put("logger.org.elasticsearch", "debug").build();
final ClusterState afterDebug = settingsUpdater.updateSettings(clusterState, toApplyDebug, Settings.EMPTY, logger);
assertSettingDeprecationsAndWarnings(new Setting<?>[] { deprecatedSetting });
assertSettingDeprecationsAndWarnings(new Setting<?>[] { deprecatedSetting }, false);

final Settings toApplyUnset = Settings.builder().putNull("logger.org.elasticsearch").build();
final ClusterState afterUnset = settingsUpdater.updateSettings(afterDebug, toApplyUnset, Settings.EMPTY, logger);
assertSettingDeprecationsAndWarnings(new Setting<?>[] { deprecatedSetting });
assertSettingDeprecationsAndWarnings(new Setting<?>[] { deprecatedSetting }, false);

// we also check that if no settings are changed, deprecation logging still occurs
settingsUpdater.updateSettings(afterUnset, toApplyUnset, Settings.EMPTY, logger);
assertSettingDeprecationsAndWarnings(new Setting<?>[] { deprecatedSetting });
assertSettingDeprecationsAndWarnings(new Setting<?>[] { deprecatedSetting }, false);
}

public void testUpdateWithUnknownAndSettings() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1009,7 +1009,7 @@ Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLU
" actual free: [20.0%]"));

if (DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE.exists(diskSettings)) {
assertSettingDeprecationsAndWarnings(new Setting<?>[] { DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE });
assertSettingDeprecationsAndWarnings(new Setting<?>[] { DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE }, false);
}
}

Expand All @@ -1025,7 +1025,7 @@ public void testSingleDataNodeDeprecationWarning() {
equalTo("setting [cluster.routing.allocation.disk.watermark.enable_for_single_data_node=false] is not allowed," +
" only true is valid"));

assertSettingDeprecationsAndWarnings(new Setting<?>[]{ DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE });
assertSettingDeprecationsAndWarnings(new Setting<?>[]{ DiskThresholdDecider.ENABLE_FOR_SINGLE_DATA_NODE }, false);
}

public void testDiskThresholdWithSnapshotShardSizes() {
Expand Down
Loading