[Transform] improve update API (elastic#57648)

rewrite config on update if either version is outdated, credentials change, the update changes the config or deprecated settings are found. Deprecated settings get migrated to the new format. The upgrade can be easily extended to do any necessary re-writes. fixes elastic#56499
hendrikmuhs · Jun 4, 2020 · f1ff38e · f1ff38e
1 parent 2104b05
commit f1ff38e
Show file tree

Hide file tree

Showing 6 changed files with 385 additions and 8 deletions.
diff --git a/...core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/...core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java
@@ -423,6 +423,41 @@ public static TransformConfig fromXContent(final XContentParser parser, @Nullabl
         return lenient ? LENIENT_PARSER.apply(parser, optionalTransformId) : STRICT_PARSER.apply(parser, optionalTransformId);
     }
 
+    /**
+     * Rewrites the transform config according to the latest format, for example moving deprecated
+     * settings to its new place.
+     *
+     * @param transformConfig original config
+     * @return a rewritten transform config if a rewrite was necessary, otherwise the given transformConfig
+     */
+    public static TransformConfig rewriteForUpdate(final TransformConfig transformConfig) {
+
+        // quick checks for deprecated features, if none found just return the original
+        if (transformConfig.getPivotConfig() == null || transformConfig.getPivotConfig().getMaxPageSearchSize() == null) {
+            return transformConfig;
+        }
+
+        Builder builder = new Builder(transformConfig);
+
+        if (transformConfig.getPivotConfig() != null && transformConfig.getPivotConfig().getMaxPageSearchSize() != null) {
+            // create a new pivot config but set maxPageSearchSize to null
+            PivotConfig newPivotConfig = new PivotConfig(
+                transformConfig.getPivotConfig().getGroupConfig(),
+                transformConfig.getPivotConfig().getAggregationConfig(),
+                null
+            );
+            builder.setPivotConfig(newPivotConfig);
+
+            Integer maxPageSearchSizeDeprecated = transformConfig.getPivotConfig().getMaxPageSearchSize();
+            Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null
+                ? transformConfig.getSettings().getMaxPageSearchSize()
+                : maxPageSearchSizeDeprecated;
+
+            builder.setSettings(new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond()));
+        }
+        return builder.setVersion(Version.CURRENT).build();
+    }
+
     public static class Builder {
         private String id;
         private SourceConfig source;

diff --git a/...icsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java b/...icsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java
@@ -6,6 +6,8 @@
 
 package org.elasticsearch.xpack.core.transform.transforms.persistence;
 
+import org.elasticsearch.Version;
+
 public final class TransformInternalIndexConstants {
 
     /* Constants for internal indexes of the transform plugin
@@ -24,6 +26,7 @@ public final class TransformInternalIndexConstants {
     // internal index
 
     // version is not a rollover pattern, however padded because sort is string based
+    public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_7_0;
     public static final String INDEX_VERSION = "005";
     public static final String INDEX_PATTERN = ".transform-internal-";
     public static final String LATEST_INDEX_VERSIONED_NAME = INDEX_PATTERN + INDEX_VERSION;
@@ -42,7 +45,6 @@ public final class TransformInternalIndexConstants {
     public static final String AUDIT_INDEX_READ_ALIAS = ".transform-notifications-read";
     public static final String AUDIT_INDEX = AUDIT_INDEX_PREFIX + AUDIT_TEMPLATE_VERSION;
 
-    private TransformInternalIndexConstants() {
-    }
+    private TransformInternalIndexConstants() {}
 
 }
diff --git a/...src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/...src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java
@@ -313,9 +313,81 @@ public void testSetIdInBody() throws IOException {
         );
     }
 
+    public void testRewriteForUpdate() throws IOException {
+        String pivotTransform = "{"
+            + " \"id\" : \"body_id\","
+            + " \"source\" : {\"index\":\"src\"},"
+            + " \"dest\" : {\"index\": \"dest\"},"
+            + " \"pivot\" : {"
+            + " \"group_by\": {"
+            + "   \"id\": {"
+            + "     \"terms\": {"
+            + "       \"field\": \"id\""
+            + "} } },"
+            + " \"aggs\": {"
+            + "   \"avg\": {"
+            + "     \"avg\": {"
+            + "       \"field\": \"points\""
+            + "} } },"
+            + " \"max_page_search_size\" : 111"
+            + "},"
+            + " \"version\" : \""
+            + Version.V_7_6_0.toString()
+            + "\""
+            + "}";
+
+        TransformConfig transformConfig = createTransformConfigFromString(pivotTransform, "body_id", true);
+        TransformConfig transformConfigRewritten = TransformConfig.rewriteForUpdate(transformConfig);
+
+        assertNull(transformConfigRewritten.getPivotConfig().getMaxPageSearchSize());
+        assertNotNull(transformConfigRewritten.getSettings().getMaxPageSearchSize());
+        assertEquals(111L, transformConfigRewritten.getSettings().getMaxPageSearchSize().longValue());
+        assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
+        assertEquals(Version.CURRENT, transformConfigRewritten.getVersion());
+    }
+
+    public void testRewriteForUpdateConflicting() throws IOException {
+        String pivotTransform = "{"
+            + " \"id\" : \"body_id\","
+            + " \"source\" : {\"index\":\"src\"},"
+            + " \"dest\" : {\"index\": \"dest\"},"
+            + " \"pivot\" : {"
+            + " \"group_by\": {"
+            + "   \"id\": {"
+            + "     \"terms\": {"
+            + "       \"field\": \"id\""
+            + "} } },"
+            + " \"aggs\": {"
+            + "   \"avg\": {"
+            + "     \"avg\": {"
+            + "       \"field\": \"points\""
+            + "} } },"
+            + " \"max_page_search_size\": 111"
+            + "},"
+            + " \"settings\" : { \"max_page_search_size\": 555"
+            + "},"
+            + " \"version\" : \""
+            + Version.V_7_5_0.toString()
+            + "\""
+            + "}";
+
+        TransformConfig transformConfig = createTransformConfigFromString(pivotTransform, "body_id", true);
+        TransformConfig transformConfigRewritten = TransformConfig.rewriteForUpdate(transformConfig);
+
+        assertNull(transformConfigRewritten.getPivotConfig().getMaxPageSearchSize());
+        assertNotNull(transformConfigRewritten.getSettings().getMaxPageSearchSize());
+        assertEquals(555L, transformConfigRewritten.getSettings().getMaxPageSearchSize().longValue());
+        assertEquals(Version.CURRENT, transformConfigRewritten.getVersion());
+        assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
+    }
+
     private TransformConfig createTransformConfigFromString(String json, String id) throws IOException {
+        return createTransformConfigFromString(json, id, false);
+    }
+
+    private TransformConfig createTransformConfigFromString(String json, String id, boolean lenient) throws IOException {
         final XContentParser parser = XContentType.JSON.xContent()
             .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);
-        return TransformConfig.fromXContent(parser, id, false);
+        return TransformConfig.fromXContent(parser, id, lenient);
     }
 }
diff --git a/...ts/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/...ts/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
@@ -279,7 +279,11 @@ protected void stopTransform(String transformId, boolean force) throws Exception
     }
 
     protected void stopTransform(String transformId, boolean force, boolean waitForCheckpoint) throws Exception {
-        final Request stopTransformRequest = createRequestWithAuth("POST", getTransformEndpoint() + transformId + "/_stop", null);
+        stopTransform(transformId, null, force, false);
+    }
+
+    protected void stopTransform(String transformId, String authHeader, boolean force, boolean waitForCheckpoint) throws Exception {
+        final Request stopTransformRequest = createRequestWithAuth("POST", getTransformEndpoint() + transformId + "/_stop", authHeader);
         stopTransformRequest.addParameter(TransformField.FORCE.getPreferredName(), Boolean.toString(force));
         stopTransformRequest.addParameter(TransformField.WAIT_FOR_COMPLETION.getPreferredName(), Boolean.toString(true));
         stopTransformRequest.addParameter(TransformField.WAIT_FOR_CHECKPOINT.getPreferredName(), Boolean.toString(waitForCheckpoint));