From f8f9ee3dba05721fbbcb23ef9121c147cfb4fed4 Mon Sep 17 00:00:00 2001 From: Philipp Zehnder Date: Tue, 21 May 2024 15:11:52 +0200 Subject: [PATCH] fix(#2855): Issue with deletetransformationrule in adapter (#2887) * fix(#2855): Add e2e test to reproduce the bug * fix(#2855): Fix transformation rules for properties with same prefix --- .../schema/DeleteTransformationRule.java | 2 +- .../schema/DeleteTransformationRuleTest.java | 24 +++--- .../deleteTransformationRule/nestedInput.json | 6 ++ .../deleteTransformationRule/prefixInput.csv | 2 + .../support/utils/ConnectEventSchemaUtils.ts | 2 +- .../support/utils/connect/ConnectUtils.ts | 34 +++++---- .../rules/deleteTransformationRule.spec.ts | 76 +++++++++++++++++++ .../event-schema-preview.component.html | 1 + .../transformation-rule.service.spec.ts | 68 ----------------- .../services/transformation-rule.service.ts | 26 +------ 10 files changed, 119 insertions(+), 122 deletions(-) create mode 100644 ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json create mode 100644 ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv create mode 100644 ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java index 11fb3abe4c..ae2d1c70d9 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java @@ -38,6 +38,6 @@ protected List getEventKeys() { @Override protected void applyTransformation(Map event, List eventKeys) { - event.remove(eventKeys.get(0)); + event.remove(eventKeys.get(0)); } } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java index 0579efb627..7253b33bd5 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java @@ -18,41 +18,39 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.HashMap; import java.util.List; -import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; public class DeleteTransformationRuleTest { @Test public void transformSimple() { - Map event = new HashMap<>(); + var event = new HashMap(); event.put("key", "value"); - DeleteTransformationRule deleteRule = new DeleteTransformationRule(List.of("key")); + var deleteRule = new DeleteTransformationRule(List.of("key")); - Map result = deleteRule.apply(event); + var result = deleteRule.apply(event); - Assertions.assertEquals(0, - result.keySet().size()); + assertEquals(0, result.keySet().size()); } @Test public void transformNested() { - Map child = new HashMap<>(); + var child = new HashMap(); child.put("child", "value"); - Map event = new HashMap<>(); + var event = new HashMap(); event.put("parent", child); - DeleteTransformationRule deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child")); + var deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child")); - Map result = deleteRule.apply(event); + var result = deleteRule.apply(event); - Assertions.assertEquals(1, - result.keySet().size()); + assertEquals(1, result.keySet().size()); } } diff --git a/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json b/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json new file mode 100644 index 0000000000..92177c23ed --- /dev/null +++ b/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json @@ -0,0 +1,6 @@ +{ + "timestamp": 1667904471000, + "parent": { + "child": "text" + } +} diff --git a/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv b/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv new file mode 100644 index 0000000000..5280fe3838 --- /dev/null +++ b/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv @@ -0,0 +1,2 @@ +timestamp;reserved bit;reserved bit_1;reserved bit_2 +1715356080000;true;true;true diff --git a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts b/ui/cypress/support/utils/ConnectEventSchemaUtils.ts index 1d933399ff..495a47c738 100644 --- a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts +++ b/ui/cypress/support/utils/ConnectEventSchemaUtils.ts @@ -178,7 +178,7 @@ export class ConnectEventSchemaUtils { } public static deleteProperty(propertyName: string) { - cy.dataCy('delete-property-' + propertyName, { timeout: 10000 }) + cy.dataCy('"delete-property-' + propertyName + '"', { timeout: 10000 }) .children() .click({ force: true }); cy.dataCy('connect-schema-delete-properties-btn', { diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts index 407475c331..6fe6cfd708 100644 --- a/ui/cypress/support/utils/connect/ConnectUtils.ts +++ b/ui/cypress/support/utils/connect/ConnectUtils.ts @@ -278,29 +278,33 @@ export class ConnectUtils { public static setUpPreprocessingRuleTest( overwriteTimestamp: boolean, + adapterConfigurationBuilder?: AdapterBuilder, ): AdapterInput { - const adapterConfiguration = AdapterBuilder.create('File_Stream') - .setStoreInDataLake() - .setTimestampProperty('timestamp') - .addProtocolInput( - 'radio', - 'speed', - 'fastest_\\(ignore_original_time\\)', - ) - .addProtocolInput('radio', 'replayonce', 'yes') - .setName('Adapter to test rules') - .setFormat('csv') - .addFormatInput('input', ConnectBtns.csvDelimiter(), ';') - .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check'); + if (!adapterConfigurationBuilder) { + adapterConfigurationBuilder = AdapterBuilder.create('File_Stream') + .setStoreInDataLake() + .setTimestampProperty('timestamp') + .addProtocolInput( + 'radio', + 'speed', + 'fastest_\\(ignore_original_time\\)', + ) + .addProtocolInput('radio', 'replayonce', 'yes') + .setName('Adapter to test rules') + .setFormat('csv') + .addFormatInput('input', ConnectBtns.csvDelimiter(), ';') + .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check'); + } if (overwriteTimestamp) { - adapterConfiguration.addProtocolInput( + adapterConfigurationBuilder.addProtocolInput( 'checkbox', 'replaceTimestamp', 'check', ); } - adapterConfiguration = adapterConfiguration.build(); + + const adapterConfiguration = adapterConfigurationBuilder.build(); ConnectUtils.goToConnect(); ConnectUtils.goToNewAdapterPage(); diff --git a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts new file mode 100644 index 0000000000..debbc5473f --- /dev/null +++ b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils'; +import { FileManagementUtils } from '../../../support/utils/FileManagementUtils'; +import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils'; +import { AdapterBuilder } from '../../../support/builder/AdapterBuilder'; + +describe('Connect delete rule transformation', () => { + beforeEach('Setup Test', () => { + cy.initStreamPipesTest(); + }); + + it('Test delete with same prefix', () => { + FileManagementUtils.addFile( + 'connect/deleteTransformationRule/prefixInput.csv', + ); + + ConnectUtils.setUpPreprocessingRuleTest(false); + + ConnectEventSchemaUtils.deleteProperty('reserved bit'); + ConnectEventSchemaUtils.deleteProperty('reserved bit_1'); + ConnectEventSchemaUtils.deleteProperty('reserved bit_2'); + + cy.dataCy('schema-preview-result-event').should( + 'have.text', + '{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01715356080000}', + ); + }); + + it('Test delete nested properties', () => { + FileManagementUtils.addFile( + 'connect/deleteTransformationRule/nestedInput.json', + ); + + const adapterConfigurationBuilder = AdapterBuilder.create('File_Stream') + .setStoreInDataLake() + .setTimestampProperty('timestamp') + .addProtocolInput( + 'radio', + 'speed', + 'fastest_\\(ignore_original_time\\)', + ) + .addProtocolInput('radio', 'replayonce', 'yes') + .setName('Adapter to test rules') + .setFormat('json'); + + ConnectUtils.setUpPreprocessingRuleTest( + false, + adapterConfigurationBuilder, + ); + + // Test to delete the child property + ConnectEventSchemaUtils.deleteProperty('child'); + + // The resulting string contains non-breaking spaces character (\u00A0) + cy.dataCy('schema-preview-result-event').should( + 'have.text', + '{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}', + ); + }); +}); diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html index 41d8e5446c..41ae59c67b 100644 --- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html +++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html @@ -49,6 +49,7 @@ 

         
     
diff --git a/ui/src/app/connect/services/transformation-rule.service.spec.ts b/ui/src/app/connect/services/transformation-rule.service.spec.ts
index 39c938b913..a0030bf34f 100644
--- a/ui/src/app/connect/services/transformation-rule.service.spec.ts
+++ b/ui/src/app/connect/services/transformation-rule.service.spec.ts
@@ -19,7 +19,6 @@
 import { TransformationRuleService } from './transformation-rule.service';
 import {
     CreateNestedRuleDescription,
-    DeleteRuleDescription,
     EventPropertyNested,
     EventPropertyPrimitive,
     EventPropertyUnion,
@@ -196,73 +195,6 @@ describe('TransformationRuleService', () => {
 
         expect(result.length).toBe(1);
         expect(result[0].oldRuntimeKey).toBe('a');
-        // expect(result[0].newRuntimeKey).toBe('b.a');
-    });
-
-    it('Delete simple', () => {
-        const oldEventSchema: EventSchema = new EventSchema();
-        const eventProperty: EventPropertyPrimitive =
-            new EventPropertyPrimitive();
-        eventProperty.runtimeName = 'a';
-        oldEventSchema.eventProperties = [];
-        oldEventSchema.eventProperties.push(eventProperty);
-
-        const newEventSchema: EventSchema = new EventSchema();
-        newEventSchema.eventProperties = [];
-
-        const result: DeleteRuleDescription[] = service.getDeleteRules(
-            newEventSchema.eventProperties,
-            oldEventSchema,
-            newEventSchema,
-        );
-
-        expect(result.length).toBe(1);
-        expect(result[0].runtimeKey).toBe('a');
-    });
-
-    it('Delete nested', () => {
-        const oldEventSchema: EventSchema = new EventSchema();
-        const eventProperty: EventPropertyPrimitive =
-            new EventPropertyPrimitive();
-        eventProperty.elementId = 'id_2';
-        eventProperty.runtimeName = 'a';
-        const eventPropertyNested: EventPropertyNested =
-            new EventPropertyNested();
-        eventPropertyNested.elementId = 'id_1';
-        eventPropertyNested.eventProperties = [];
-        eventPropertyNested.eventProperties.push(eventProperty);
-        eventPropertyNested.runtimeName = 'b';
-        oldEventSchema.eventProperties = [];
-        oldEventSchema.eventProperties.push(eventPropertyNested);
-
-        let newEventSchema: EventSchema = new EventSchema();
-        const newEventPropertyNested: EventPropertyNested =
-            new EventPropertyNested();
-        newEventPropertyNested.elementId = 'id_1';
-        newEventPropertyNested.runtimeName = 'b';
-        newEventPropertyNested.eventProperties = [];
-        newEventSchema.eventProperties = [];
-        newEventSchema.eventProperties.push(newEventPropertyNested);
-
-        let result: DeleteRuleDescription[] = service.getDeleteRules(
-            newEventSchema.eventProperties,
-            oldEventSchema,
-            newEventSchema,
-        );
-
-        expect(result.length).toBe(1);
-        expect(result[0].runtimeKey).toBe('b.a');
-
-        newEventSchema = new EventSchema();
-        newEventSchema.eventProperties = [];
-        result = service.getDeleteRules(
-            newEventSchema.eventProperties,
-            oldEventSchema,
-            newEventSchema,
-        );
-
-        expect(result.length).toBe(1);
-        expect(result[0].runtimeKey).toBe('b');
     });
 
     it('Rename simple', () => {
diff --git a/ui/src/app/connect/services/transformation-rule.service.ts b/ui/src/app/connect/services/transformation-rule.service.ts
index f1cca8565c..b0fd79fa3d 100644
--- a/ui/src/app/connect/services/transformation-rule.service.ts
+++ b/ui/src/app/connect/services/transformation-rule.service.ts
@@ -125,13 +125,7 @@ export class TransformationRuleService {
                         targetSchema,
                     ),
                 )
-                .concat(
-                    this.getDeleteRules(
-                        targetSchema.eventProperties,
-                        originalSchema,
-                        targetSchema,
-                    ),
-                )
+                .concat(this.getDeleteRules(originalSchema, targetSchema))
                 .concat(
                     this.getUnitTransformRules(
                         targetSchema.eventProperties,
@@ -299,11 +293,10 @@ export class TransformationRuleService {
     }
 
     public getDeleteRules(
-        newEventProperties: EventProperty[],
         oldEventSchema: EventSchema,
         newEventSchema: EventSchema,
     ): DeleteRuleDescription[] {
-        let resultKeys: string[] = [];
+        const resultKeys: string[] = [];
 
         const allNewIds: string[] = this.getAllIds(
             newEventSchema.eventProperties,
@@ -323,21 +316,6 @@ export class TransformationRuleService {
             }
         }
 
-        const uniqEs6 = arrArg => {
-            return arrArg.filter((elem, pos, arr) => {
-                let r = true;
-                for (const a of arr) {
-                    if (elem.startsWith(a) && a !== elem) {
-                        r = false;
-                    }
-                }
-
-                return r;
-            });
-        };
-
-        resultKeys = uniqEs6(resultKeys);
-
         const resultRules: DeleteRuleDescription[] = [];
         for (const key of resultKeys) {
             const rule: DeleteRuleDescription = new DeleteRuleDescription();