Merge pull request #4 from meshtrade/parquet-optim

Add parquet serialisation empty data optimisation
meshtrade · Oct 30, 2024 · 5236c18 · 5236c18
2 parents 2587dc9 + f281f6f
commit 5236c18
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/etl/parquet/stage.go b/etl/parquet/stage.go
@@ -105,6 +105,14 @@ func (s *ParquetSerialiser[T]) Serialise(ctx context.Context, p *pipeline.Pipeli
 		inputStruct = append(inputStruct, inValue)
 	}
 
+	// check if there is data to serialise
+	if len(inputStruct) == 0 {
+		outChannel := make(chan []byte, 1)
+		outChannel <- []byte{}
+		close(outChannel)
+		return outChannel, nil
+	}
+
 	// get the reflection value of the input slice
 	timeType := reflect.TypeOf(time.Time{})