Speed up BQ by having 2 queries, and not an OR (#27981)

airbytehq · Jul 5, 2023 · 151e675 · 151e675
1 parent 308422e
commit 151e675
Showing 1 changed file with 5 additions and 4 deletions.
diff --git a/...va/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java b/...va/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java
@@ -375,8 +375,10 @@ String dedupFinalTable(final StreamId id,
                   ) as row_number FROM ${final_table_id}
                 )
                 WHERE row_number != 1
-              )
-              OR (
+              );
+            
+            DELETE FROM ${final_table_id}
+            WHERE
                 ${pk_list} IN (
                   SELECT (
             ${pk_cast_list}
@@ -386,8 +388,7 @@ String dedupFinalTable(final StreamId id,
                     JSON_QUERY(`_airbyte_data`, '$._ab_cdc_deleted_at') IS NOT NULL
                     AND JSON_TYPE(JSON_QUERY(`_airbyte_data`, '$._ab_cdc_deleted_at')) != 'null'
                     AND ${cursor_name} < ${cursor_cast}
-                )
-              );""");
+               );""");
   }
 
   @VisibleForTesting