fix(#2855): Issue with deletetransformationrule in adapter (#2887)

* fix(#2855): Add e2e test to reproduce the bug

* fix(#2855): Fix transformation rules for properties with same prefix

streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java

@@ -38,6 +38,6 @@ protected List<String> getEventKeys() {
 
   @Override
   protected void applyTransformation(Map<String, Object> event, List<String> eventKeys) {
-    event.remove(eventKeys.get(0));
+      event.remove(eventKeys.get(0));
   }
 }

streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java

@@ -18,41 +18,39 @@
 
 package org.apache.streampipes.connect.shared.preprocessing.transform.schema;
 
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class DeleteTransformationRuleTest {
 
   @Test
   public void transformSimple() {
-    Map<String, Object> event = new HashMap<>();
+    var event = new HashMap<String, Object>();
     event.put("key", "value");
 
-    DeleteTransformationRule deleteRule = new DeleteTransformationRule(List.of("key"));
+    var deleteRule = new DeleteTransformationRule(List.of("key"));
 
-    Map<String, Object> result = deleteRule.apply(event);
+    var result = deleteRule.apply(event);
 
-    Assertions.assertEquals(0,
-                            result.keySet().size());
+    assertEquals(0, result.keySet().size());
   }
 
   @Test
   public void transformNested() {
-    Map<String, Object> child = new HashMap<>();
+    var child = new HashMap<String, Object>();
     child.put("child", "value");
-    Map<String, Object> event = new HashMap<>();
+    var event = new HashMap<String, Object>();
     event.put("parent", child);
 
-    DeleteTransformationRule deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child"));
+    var deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child"));
 
-    Map<String, Object> result = deleteRule.apply(event);
+    var result = deleteRule.apply(event);
 
-    Assertions.assertEquals(1,
-                            result.keySet().size());
+    assertEquals(1, result.keySet().size());
   }
 }

ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json

@@ -0,0 +1,6 @@
+{
+  "timestamp": 1667904471000,
+  "parent": {
+    "child": "text"
+  }
+}

ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv

@@ -0,0 +1,2 @@
+timestamp;reserved bit;reserved bit_1;reserved bit_2
+1715356080000;true;true;true

ui/cypress/support/utils/ConnectEventSchemaUtils.ts

@@ -178,7 +178,7 @@ export class ConnectEventSchemaUtils {
     }
 
     public static deleteProperty(propertyName: string) {
-        cy.dataCy('delete-property-' + propertyName, { timeout: 10000 })
+        cy.dataCy('"delete-property-' + propertyName + '"', { timeout: 10000 })
             .children()
             .click({ force: true });
         cy.dataCy('connect-schema-delete-properties-btn', {

ui/cypress/support/utils/connect/ConnectUtils.ts

@@ -278,29 +278,33 @@ export class ConnectUtils {
 
     public static setUpPreprocessingRuleTest(
         overwriteTimestamp: boolean,
+        adapterConfigurationBuilder?: AdapterBuilder,
     ): AdapterInput {
-        const adapterConfiguration = AdapterBuilder.create('File_Stream')
-            .setStoreInDataLake()
-            .setTimestampProperty('timestamp')
-            .addProtocolInput(
-                'radio',
-                'speed',
-                'fastest_\\(ignore_original_time\\)',
-            )
-            .addProtocolInput('radio', 'replayonce', 'yes')
-            .setName('Adapter to test rules')
-            .setFormat('csv')
-            .addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
-            .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check');
+        if (!adapterConfigurationBuilder) {
+            adapterConfigurationBuilder = AdapterBuilder.create('File_Stream')
+                .setStoreInDataLake()
+                .setTimestampProperty('timestamp')
+                .addProtocolInput(
+                    'radio',
+                    'speed',
+                    'fastest_\\(ignore_original_time\\)',
+                )
+                .addProtocolInput('radio', 'replayonce', 'yes')
+                .setName('Adapter to test rules')
+                .setFormat('csv')
+                .addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
+                .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check');
+        }
 
         if (overwriteTimestamp) {
-            adapterConfiguration.addProtocolInput(
+            adapterConfigurationBuilder.addProtocolInput(
                 'checkbox',
                 'replaceTimestamp',
                 'check',
             );
         }
-        adapterConfiguration = adapterConfiguration.build();
+
+        const adapterConfiguration = adapterConfigurationBuilder.build();
 
         ConnectUtils.goToConnect();
         ConnectUtils.goToNewAdapterPage();

ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts

@@ -0,0 +1,76 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
+import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';
+import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils';
+import { AdapterBuilder } from '../../../support/builder/AdapterBuilder';
+
+describe('Connect delete rule transformation', () => {
+    beforeEach('Setup Test', () => {
+        cy.initStreamPipesTest();
+    });
+
+    it('Test delete with same prefix', () => {
+        FileManagementUtils.addFile(
+            'connect/deleteTransformationRule/prefixInput.csv',
+        );
+
+        ConnectUtils.setUpPreprocessingRuleTest(false);
+
+        ConnectEventSchemaUtils.deleteProperty('reserved bit');
+        ConnectEventSchemaUtils.deleteProperty('reserved bit_1');
+        ConnectEventSchemaUtils.deleteProperty('reserved bit_2');
+
+        cy.dataCy('schema-preview-result-event').should(
+            'have.text',
+            '{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01715356080000}',
+        );
+    });
+
+    it('Test delete nested properties', () => {
+        FileManagementUtils.addFile(
+            'connect/deleteTransformationRule/nestedInput.json',
+        );
+
+        const adapterConfigurationBuilder = AdapterBuilder.create('File_Stream')
+            .setStoreInDataLake()
+            .setTimestampProperty('timestamp')
+            .addProtocolInput(
+                'radio',
+                'speed',
+                'fastest_\\(ignore_original_time\\)',
+            )
+            .addProtocolInput('radio', 'replayonce', 'yes')
+            .setName('Adapter to test rules')
+            .setFormat('json');
+
+        ConnectUtils.setUpPreprocessingRuleTest(
+            false,
+            adapterConfigurationBuilder,
+        );
+
+        // Test to delete the child property
+        ConnectEventSchemaUtils.deleteProperty('child');
+
+        // The resulting string contains non-breaking spaces character (\u00A0)
+        cy.dataCy('schema-preview-result-event').should(
+            'have.text',
+            '{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
+        );
+    });
+});

ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html

@@ -49,6 +49,7 @@
             <pre
                 [innerHTML]="desiredField | jsonpretty"
                 class="preview-text"
+                data-cy="schema-preview-result-event"
             ></pre>
         </sp-basic-inner-panel>
     </div>

ui/src/app/connect/services/transformation-rule.service.spec.ts

@@ -19,7 +19,6 @@
 import { TransformationRuleService } from './transformation-rule.service';
 import {
     CreateNestedRuleDescription,
-    DeleteRuleDescription,
     EventPropertyNested,
     EventPropertyPrimitive,
     EventPropertyUnion,
@@ -196,73 +195,6 @@ describe('TransformationRuleService', () => {
 
         expect(result.length).toBe(1);
         expect(result[0].oldRuntimeKey).toBe('a');
-        // expect(result[0].newRuntimeKey).toBe('b.a');
-    });
-
-    it('Delete simple', () => {
-        const oldEventSchema: EventSchema = new EventSchema();
-        const eventProperty: EventPropertyPrimitive =
-            new EventPropertyPrimitive();
-        eventProperty.runtimeName = 'a';
-        oldEventSchema.eventProperties = [];
-        oldEventSchema.eventProperties.push(eventProperty);
-
-        const newEventSchema: EventSchema = new EventSchema();
-        newEventSchema.eventProperties = [];
-
-        const result: DeleteRuleDescription[] = service.getDeleteRules(
-            newEventSchema.eventProperties,
-            oldEventSchema,
-            newEventSchema,
-        );
-
-        expect(result.length).toBe(1);
-        expect(result[0].runtimeKey).toBe('a');
-    });
-
-    it('Delete nested', () => {
-        const oldEventSchema: EventSchema = new EventSchema();
-        const eventProperty: EventPropertyPrimitive =
-            new EventPropertyPrimitive();
-        eventProperty.elementId = 'id_2';
-        eventProperty.runtimeName = 'a';
-        const eventPropertyNested: EventPropertyNested =
-            new EventPropertyNested();
-        eventPropertyNested.elementId = 'id_1';
-        eventPropertyNested.eventProperties = [];
-        eventPropertyNested.eventProperties.push(eventProperty);
-        eventPropertyNested.runtimeName = 'b';
-        oldEventSchema.eventProperties = [];
-        oldEventSchema.eventProperties.push(eventPropertyNested);
-
-        let newEventSchema: EventSchema = new EventSchema();
-        const newEventPropertyNested: EventPropertyNested =
-            new EventPropertyNested();
-        newEventPropertyNested.elementId = 'id_1';
-        newEventPropertyNested.runtimeName = 'b';
-        newEventPropertyNested.eventProperties = [];
-        newEventSchema.eventProperties = [];
-        newEventSchema.eventProperties.push(newEventPropertyNested);
-
-        let result: DeleteRuleDescription[] = service.getDeleteRules(
-            newEventSchema.eventProperties,
-            oldEventSchema,
-            newEventSchema,
-        );
-
-        expect(result.length).toBe(1);
-        expect(result[0].runtimeKey).toBe('b.a');
-
-        newEventSchema = new EventSchema();
-        newEventSchema.eventProperties = [];
-        result = service.getDeleteRules(
-            newEventSchema.eventProperties,
-            oldEventSchema,
-            newEventSchema,
-        );
-
-        expect(result.length).toBe(1);
-        expect(result[0].runtimeKey).toBe('b');
     });
 
     it('Rename simple', () => {

ui/src/app/connect/services/transformation-rule.service.ts

@@ -125,13 +125,7 @@ export class TransformationRuleService {
                         targetSchema,
                     ),
                 )
-                .concat(
-                    this.getDeleteRules(
-                        targetSchema.eventProperties,
-                        originalSchema,
-                        targetSchema,
-                    ),
-                )
+                .concat(this.getDeleteRules(originalSchema, targetSchema))
                 .concat(
                     this.getUnitTransformRules(
                         targetSchema.eventProperties,
@@ -299,11 +293,10 @@ export class TransformationRuleService {
     }
 
     public getDeleteRules(
-        newEventProperties: EventProperty[],
         oldEventSchema: EventSchema,
         newEventSchema: EventSchema,
     ): DeleteRuleDescription[] {
-        let resultKeys: string[] = [];
+        const resultKeys: string[] = [];
 
         const allNewIds: string[] = this.getAllIds(
             newEventSchema.eventProperties,
@@ -323,21 +316,6 @@ export class TransformationRuleService {
             }
         }
 
-        const uniqEs6 = arrArg => {
-            return arrArg.filter((elem, pos, arr) => {
-                let r = true;
-                for (const a of arr) {
-                    if (elem.startsWith(a) && a !== elem) {
-                        r = false;
-                    }
-                }
-
-                return r;
-            });
-        };
-
-        resultKeys = uniqEs6(resultKeys);
-
         const resultRules: DeleteRuleDescription[] = [];
         for (const key of resultKeys) {
             const rule: DeleteRuleDescription = new DeleteRuleDescription();