Reenable batch for release (georgia-tech-db#1302)

alexxx-db · Nov 22, 2023 · 3438293 · 3438293
1 parent 90019b8
commit 3438293
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 2 deletions.
diff --git a/evadb/storage/native_storage_engine.py b/evadb/storage/native_storage_engine.py
@@ -29,10 +29,14 @@
 from evadb.storage.abstract_storage_engine import AbstractStorageEngine
 from evadb.third_party.databases.interface import get_database_handler
 <<<<<<< HEAD
+<<<<<<< HEAD
 from evadb.utils.generic_utils import PickleSerializer, rebatch
 =======
 from evadb.utils.generic_utils import PickleSerializer
 >>>>>>> 40a10ce1 (Bump v0.3.4+ dev)
+=======
+from evadb.utils.generic_utils import PickleSerializer, rebatch
+>>>>>>> 3d009af6 (Reenable batch for release (#1302))
 from evadb.utils.logging_manager import logger
 
 
@@ -281,8 +285,8 @@ def read(
                         _deserialize_sql_row(row, ordered_columns) for row in result
                     )
 
-                for data_batch in result:
-                    yield Batch(pd.DataFrame([data_batch]))
+                for df in rebatch(result, batch_mem_size):
+                    yield Batch(pd.DataFrame(df))
 
 >>>>>>> 40a10ce1 (Bump v0.3.4+ dev)
 <<<<<<< HEAD

diff --git a/evadb/storage/sqlite_storage_engine.py b/evadb/storage/sqlite_storage_engine.py
@@ -31,13 +31,17 @@
 from evadb.storage.abstract_storage_engine import AbstractStorageEngine
 <<<<<<< HEAD
 <<<<<<< HEAD
+<<<<<<< HEAD
 from evadb.utils.generic_utils import PickleSerializer, rebatch
 =======
 from evadb.utils.generic_utils import PickleSerializer
 >>>>>>> 40a10ce1 (Bump v0.3.4+ dev)
 =======
 from evadb.utils.generic_utils import PickleSerializer
 >>>>>>> 6d6a14c8 (Bump v0.3.4+ dev)
+=======
+from evadb.utils.generic_utils import PickleSerializer, rebatch
+>>>>>>> 3d009af6 (Reenable batch for release (#1302))
 from evadb.utils.logging_manager import logger
 
 # Leveraging Dynamic schema in SQLAlchemy
@@ -199,12 +203,16 @@ def read(
             result = self._sql_session.execute(table_to_read.select()).fetchall()
 <<<<<<< HEAD
 <<<<<<< HEAD
+<<<<<<< HEAD
+=======
+>>>>>>> 3d009af6 (Reenable batch for release (#1302))
             result_iter = (
                 self._deserialize_sql_row(row._asdict(), table.columns)
                 for row in result
             )
             for df in rebatch(result_iter, batch_mem_size):
                 yield Batch(pd.DataFrame(df))
+<<<<<<< HEAD
 =======
 =======
 >>>>>>> 6d6a14c8 (Bump v0.3.4+ dev)
@@ -218,6 +226,8 @@ def read(
 >>>>>>> 40a10ce1 (Bump v0.3.4+ dev)
 =======
 >>>>>>> 6d6a14c8 (Bump v0.3.4+ dev)
+=======
+>>>>>>> 3d009af6 (Reenable batch for release (#1302))
         except Exception as e:
             err_msg = f"Failed to read the table {table.name} with exception {str(e)}"
             logger.exception(err_msg)

diff --git a/test/integration_tests/long/test_github_datasource.py b/test/integration_tests/long/test_github_datasource.py
@@ -31,12 +31,18 @@ def setUp(self):
     def tearDown(self):
         execute_query_fetch_all(self.evadb, "DROP DATABASE IF EXISTS github_data;")
 
+<<<<<<< HEAD
 <<<<<<< HEAD
     @pytest.mark.skip(
         reason="Need https://github.com/georgia-tech-db/evadb/pull/1280 for a cost-based rebatch optimization"
     )
 =======
 >>>>>>> 858b8c1c (Collection of fixes for the staging branch (#1253))
+=======
+    @pytest.mark.skip(
+        reason="Need https://github.com/georgia-tech-db/evadb/pull/1280 for a cost-based rebatch optimization"
+    )
+>>>>>>> 3d009af6 (Reenable batch for release (#1302))
     @pytest.mark.xfail(reason="Flaky testcase due to `bad request` error message")
     def test_should_run_select_query_in_github(self):
         # Create database.