chore(weave): table row response includes absolute row indices

wandb · Feb 4, 2025 · 3ee5fbd · 3ee5fbd
1 parent 2a63919
commit 3ee5fbd
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 11 deletions.
diff --git a/weave/trace_server/clickhouse_trace_server_batched.py b/weave/trace_server/clickhouse_trace_server_batched.py
@@ -922,7 +922,9 @@ def _table_query_stream(
         res = self._query_stream(query, parameters=pb.get_params())
 
         for row in res:
-            yield tsi.TableRowSchema(digest=row[0], val=json.loads(row[1]))
+            yield tsi.TableRowSchema(
+                digest=row[0], val=json.loads(row[1]), original_index=row[2]
+            )
 
     def table_query_stats(self, req: tsi.TableQueryStatsReq) -> tsi.TableQueryStatsRes:
         parameters: dict[str, Any] = {

diff --git a/weave/trace_server/table_query_builder.py b/weave/trace_server/table_query_builder.py
@@ -36,21 +36,22 @@ def make_natural_sort_table_query(
             row_digests_selection = f"arraySlice({row_digests_selection}, 1 + {{{pb.add_param(offset)}: Int64}}, {{{pb.add_param(limit)}: Int64}})"
 
     query = f"""
-    SELECT DISTINCT tr.digest, tr.val_dump, t.row_order
+    SELECT DISTINCT tr.digest, tr.val_dump, t.original_index + {{{pb.add_param(offset or 0)}: Int64}} - 1 as original_index
     FROM table_rows tr
     INNER JOIN (
-        SELECT row_digest, row_number() OVER () AS row_order
+        SELECT row_digest, original_index
         FROM (
-            SELECT {row_digests_selection} as row_digests
+            SELECT {row_digests_selection} as row_digests,
+                   arrayEnumerate(row_digests) as original_indices
             FROM tables
             WHERE project_id = {{{project_id_name}: String}}
             AND digest = {{{digest_name}: String}}
             LIMIT 1
         )
-        ARRAY JOIN row_digests AS row_digest
+        ARRAY JOIN row_digests AS row_digest, original_indices AS original_index
     ) AS t ON tr.digest = t.row_digest
     WHERE tr.project_id = {{{project_id_name}: String}}
-    ORDER BY row_order ASC
+    ORDER BY original_index ASC
     """
 
     return query
@@ -88,20 +89,21 @@ def make_standard_table_query(
     )
 
     query = f"""
-    SELECT tr.digest, tr.val_dump, tr.row_order FROM
+    SELECT tr.digest, tr.val_dump, tr.original_index FROM
     (
-        SELECT DISTINCT tr.digest, tr.val_dump, t.row_order
+        SELECT DISTINCT tr.digest, tr.val_dump, t.original_index
         FROM table_rows tr
         INNER JOIN (
-            SELECT row_digest, row_number() OVER () AS row_order
+            SELECT row_digest, original_index - 1 as original_index
             FROM (
-                SELECT row_digests
+                SELECT row_digests,
+                       arrayEnumerate(row_digests) as original_indices
                 FROM tables
                 WHERE project_id = {{{project_id_name}: String}}
                 AND digest = {{{digest_name}: String}}
                 LIMIT 1
             )
-            ARRAY JOIN row_digests AS row_digest
+            ARRAY JOIN row_digests AS row_digest, original_indices AS original_index
         ) AS t ON tr.digest = t.row_digest
         WHERE tr.project_id = {{{project_id_name}: String}}
         {sql_safe_filter_clause}

diff --git a/weave/trace_server/trace_server_interface.py b/weave/trace_server/trace_server_interface.py
@@ -611,6 +611,7 @@ class TableUpdateRes(BaseModel):
 class TableRowSchema(BaseModel):
     digest: str
     val: Any
+    original_index: Optional[int] = None
 
 
 class TableCreateRes(BaseModel):