NEOS-1618: Fixes IP Address Form, FK Col Rendering, JM Table Perf

frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataGenConnectionCard.tsx

@@ -5,6 +5,8 @@ import {
   getAllFormErrors,
 } from '@/components/jobs/SchemaTable/SchemaTable';
 import { getSchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler';
+import { TransformerResult } from '@/components/jobs/SchemaTable/transformer-handler';
+import { getTransformerFilter } from '@/components/jobs/SchemaTable/util';
 import { useAccount } from '@/components/providers/account-provider';
 import { Alert, AlertTitle } from '@/components/ui/alert';
 import { Button } from '@/components/ui/button';
@@ -25,8 +27,10 @@ import {
   SelectTrigger,
   SelectValue,
 } from '@/components/ui/select';
-import { getErrorMessage } from '@/util/util';
+import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler';
+import { getErrorMessage, getTransformerFromField } from '@/util/util';
 import {
+  JobMappingTransformerForm,
   convertJobMappingTransformerFormToJobMappingTransformer,
   convertJobMappingTransformerToForm,
 } from '@/yup-validations/jobs';
@@ -64,8 +68,12 @@ import {
   validateJobMapping,
 } from '../../../util';
 import SchemaPageSkeleton from './SchemaPageSkeleton';
+import { useOnApplyDefaultClick } from './useOnApplyDefaultClick';
 import { useOnImportMappings } from './useOnImportMappings';
-import { getOnSelectedTableToggle } from './util';
+import {
+  getFilteredTransformersForBulkSet,
+  getOnSelectedTableToggle,
+} from './util';
 
 interface Props {
   jobId: string;
@@ -138,7 +146,7 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
     [isSchemaMapValidating, isTableConstraintsValidating]
   );
   const [selectedTables, setSelectedTables] = useState<Set<string>>(new Set());
-  const { append, remove, fields } =
+  const { append, remove, fields, update } =
     useFieldArray<SingleTableEditSourceFormValues>({
       control: form.control,
       name: 'mappings',
@@ -220,6 +228,22 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
   const { mutateAsync: validateJobMappingsAsync } =
     useMutation(validateJobMappings);
 
+  const { handler, isLoading: isGetTransformersLoading } =
+    useGetTransformersHandler(account?.id ?? '');
+
+  function onTransformerUpdate(
+    index: number,
+    transformer: JobMappingTransformerForm
+  ): void {
+    const val = form.getValues(`mappings.${index}`);
+    update(index, {
+      schema: val.schema,
+      table: val.table,
+      column: val.column,
+      transformer,
+    });
+  }
+
   const { onClick: onImportMappingsClick } = useOnImportMappings({
     setMappings(mappings) {
       form.setValue('mappings', mappings, {
@@ -247,7 +271,45 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
     setSelectedTables: setSelectedTables,
   });
 
-  if (isJobLoading || isSchemaDataMapLoading) {
+  function getAvailableTransformers(idx: number): TransformerResult {
+    const row = fields[idx];
+    return handler.getFilteredTransformers(
+      getTransformerFilter(
+        schemaConstraintHandler,
+        {
+          schema: row.schema,
+          table: row.table,
+          column: row.column,
+        },
+        'sync'
+      )
+    );
+  }
+
+  const { onClick: onApplyDefaultClick } = useOnApplyDefaultClick({
+    getMappings() {
+      return form.getValues('mappings');
+    },
+    setTransformer: onTransformerUpdate,
+    constraintHandler: schemaConstraintHandler,
+    triggerUpdate() {
+      form.trigger('mappings');
+    },
+  });
+
+  function onTransformerBulkUpdate(
+    indices: number[],
+    config: JobMappingTransformerForm
+  ): void {
+    indices.forEach((idx) => {
+      onTransformerUpdate(idx, config);
+    });
+    setTimeout(() => {
+      form.trigger('mappings');
+    }, 0);
+  }
+
+  if (isJobLoading || isSchemaDataMapLoading || isGetTransformersLoading) {
     return <SchemaPageSkeleton />;
   }
 
@@ -466,6 +528,28 @@ export default function DataGenConnectionCard({ jobId }: Props): ReactElement {
           isJobMappingsValidating={isValidatingMappings}
           onValidate={validateMappings}
           onImportMappingsClick={onImportMappingsClick}
+          onTransformerUpdate={(idx, cfg) => {
+            onTransformerUpdate(idx, cfg);
+          }}
+          getAvailableTransformers={getAvailableTransformers}
+          getTransformerFromField={(idx) => {
+            const row = fields[idx];
+            return getTransformerFromField(handler, row.transformer);
+          }}
+          getAvailableTransformersForBulk={(rows) => {
+            return getFilteredTransformersForBulkSet(
+              rows,
+              handler,
+              schemaConstraintHandler,
+              'generate',
+              'relational'
+            );
+          }}
+          getTransformerFromFieldValue={(fvalue) => {
+            return getTransformerFromField(handler, fvalue);
+          }}
+          onApplyDefaultClick={onApplyDefaultClick}
+          onTransformerBulkUpdate={onTransformerBulkUpdate}
         />
 
         {form.formState.errors.mappings && (

frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/DataSyncConnectionCard.tsx

@@ -8,6 +8,11 @@ import {
   getAllFormErrors,
 } from '@/components/jobs/SchemaTable/SchemaTable';
 import { getSchemaConstraintHandler } from '@/components/jobs/SchemaTable/schema-constraint-handler';
+import { TransformerResult } from '@/components/jobs/SchemaTable/transformer-handler';
+import {
+  getTransformerFilter,
+  splitCollection,
+} from '@/components/jobs/SchemaTable/util';
 import { useAccount } from '@/components/providers/account-provider';
 import { Button } from '@/components/ui/button';
 import {
@@ -25,12 +30,14 @@ import {
   SelectTrigger,
   SelectValue,
 } from '@/components/ui/select';
+import { useGetTransformersHandler } from '@/libs/hooks/useGetTransformersHandler';
 import { splitConnections } from '@/libs/utils';
-import { getErrorMessage } from '@/util/util';
+import { getErrorMessage, getTransformerFromField } from '@/util/util';
 import {
   DataSyncSourceFormValues,
   EditDestinationOptionsFormValues,
   JobMappingFormValues,
+  JobMappingTransformerForm,
   VirtualForeignConstraintFormValues,
   convertJobMappingTransformerFormToJobMappingTransformer,
   convertJobMappingTransformerToForm,
@@ -87,11 +94,13 @@ import {
   validateJobMapping,
 } from '../../../util';
 import SchemaPageSkeleton from './SchemaPageSkeleton';
+import { useOnApplyDefaultClick } from './useOnApplyDefaultClick';
 import { useOnImportMappings } from './useOnImportMappings';
 import {
   getConnectionIdFromSource,
   getDestinationDetailsRecord,
   getDynamoDbDestinations,
+  getFilteredTransformersForBulkSet,
   getOnSelectedTableToggle,
   isDynamoDBConnection,
   isNosqlSource,
@@ -466,6 +475,22 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
     []
   );
 
+  const { handler, isLoading: isGetTransformersLoading } =
+    useGetTransformersHandler(account?.id ?? '');
+
+  function onTransformerUpdate(
+    index: number,
+    transformer: JobMappingTransformerForm
+  ): void {
+    const val = form.getValues(`mappings.${index}`);
+    update(index, {
+      schema: val.schema,
+      table: val.table,
+      column: val.column,
+      transformer,
+    });
+  }
+
   const { onClick: onImportMappingsClick } = useOnImportMappings({
     setMappings(mappings) {
       form.setValue('mappings', mappings, {
@@ -480,13 +505,7 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
     appendNewMappings(mappings) {
       append(mappings);
     },
-    setTransformer(idx, transformer) {
-      form.setValue(`mappings.${idx}.transformer`, transformer, {
-        shouldDirty: true,
-        shouldTouch: true,
-        shouldValidate: false,
-      });
-    },
+    setTransformer: onTransformerUpdate,
     async triggerUpdate() {
       await form.trigger('mappings');
       setTimeout(() => {
@@ -497,7 +516,23 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
     setSelectedTables: setSelectedTables,
   });
 
-  if (isConnectionsLoading || isSchemaDataMapLoading || isJobDataLoading) {
+  const { onClick: onApplyDefaultClick } = useOnApplyDefaultClick({
+    getMappings() {
+      return form.getValues('mappings');
+    },
+    setTransformer: onTransformerUpdate,
+    constraintHandler: schemaConstraintHandler,
+    triggerUpdate() {
+      form.trigger('mappings');
+    },
+  });
+
+  if (
+    isConnectionsLoading ||
+    isSchemaDataMapLoading ||
+    isJobDataLoading ||
+    isGetTransformersLoading
+  ) {
     return <SchemaPageSkeleton />;
   }
 
@@ -518,6 +553,34 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
         c.connectionConfig?.config.case !== 'gcpCloudstorageConfig'
     )
   );
+
+  function getAvailableTransformers(idx: number): TransformerResult {
+    const row = formMappings[idx];
+    return handler.getFilteredTransformers(
+      getTransformerFilter(
+        schemaConstraintHandler,
+        {
+          schema: row.schema,
+          table: row.table,
+          column: row.column,
+        },
+        'sync'
+      )
+    );
+  }
+
+  function onTransformerBulkUpdate(
+    indices: number[],
+    config: JobMappingTransformerForm
+  ): void {
+    indices.forEach((idx) => {
+      onTransformerUpdate(idx, config);
+    });
+    setTimeout(() => {
+      form.trigger('mappings');
+    }, 0);
+  }
+
   return (
     <Form {...form}>
       <form onSubmit={form.handleSubmit(onSubmit)}>
@@ -588,34 +651,20 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
                 validateMappingsResponse
               )}
               onValidate={validateMappings}
-              constraintHandler={schemaConstraintHandler}
-              onRemoveMappings={(values) => {
-                const valueSet = new Set(
-                  values.map((v) => `${v.schema}.${v.table}.${v.column}`)
-                );
-                const toRemove: number[] = [];
-                formMappings.forEach((mapping, idx) => {
-                  if (
-                    valueSet.has(
-                      `${mapping.schema}.${mapping.table}.${mapping.column}`
-                    )
-                  ) {
-                    toRemove.push(idx);
-                  }
-                });
-                if (toRemove.length > 0) {
-                  remove(toRemove);
+              onRemoveMappings={(indices) => {
+                const indexSet = new Set(indices);
+                const remainingTables = formMappings
+                  .filter((_, idx) => !indexSet.has(idx))
+                  .map((fm) => fm.table);
+
+                if (indices.length > 0) {
+                  remove(indices);
                 }
 
                 if (!source || isDynamoDBConnection(source)) {
                   return;
                 }
 
-                const toRemoveSet = new Set(toRemove);
-                const remainingTables = formMappings
-                  .filter((_, idx) => !toRemoveSet.has(idx))
-                  .map((fm) => fm.table);
-
                 // Check and update destinationOptions if needed
                 const destOpts = form.getValues('destinationOptions');
                 const updatedDestOpts = destOpts
@@ -653,7 +702,7 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
               onAddMappings={(values) => {
                 append(
                   values.map((v) => {
-                    const [schema, table] = v.collection.split('.');
+                    const [schema, table] = splitCollection(v.collection);
                     return {
                       schema,
                       table,
@@ -734,6 +783,25 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
                 dynamoDBDestinations.length > 0
               )}
               onImportMappingsClick={onImportMappingsClick}
+              getAvailableTransformers={getAvailableTransformers}
+              getTransformerFromField={(idx) => {
+                const row = formMappings[idx];
+                return getTransformerFromField(handler, row.transformer);
+              }}
+              getAvailableTransformersForBulk={(rows) => {
+                return getFilteredTransformersForBulkSet(
+                  rows,
+                  handler,
+                  schemaConstraintHandler,
+                  'sync',
+                  'nosql'
+                );
+              }}
+              getTransformerFromFieldValue={(fvalue) => {
+                return getTransformerFromField(handler, fvalue);
+              }}
+              onApplyDefaultClick={onApplyDefaultClick}
+              onTransformerBulkUpdate={onTransformerBulkUpdate}
             />
           )}
 
@@ -757,6 +825,28 @@ export default function DataSyncConnectionCard({ jobId }: Props): ReactElement {
               addVirtualForeignKey={addVirtualForeignKey}
               removeVirtualForeignKey={removeVirtualForeignKey}
               onImportMappingsClick={onImportMappingsClick}
+              onTransformerUpdate={(idx, cfg) => {
+                onTransformerUpdate(idx, cfg);
+              }}
+              getAvailableTransformers={getAvailableTransformers}
+              getTransformerFromField={(idx) => {
+                const row = formMappings[idx];
+                return getTransformerFromField(handler, row.transformer);
+              }}
+              getAvailableTransformersForBulk={(rows) => {
+                return getFilteredTransformersForBulkSet(
+                  rows,
+                  handler,
+                  schemaConstraintHandler,
+                  'sync',
+                  'relational'
+                );
+              }}
+              getTransformerFromFieldValue={(fvalue) => {
+                return getTransformerFromField(handler, fvalue);
+              }}
+              onApplyDefaultClick={onApplyDefaultClick}
+              onTransformerBulkUpdate={onTransformerBulkUpdate}
             />
           )}
           <div className="flex flex-row items-center justify-end w-full mt-4">