test(clickhouse): test all functions

ibis-server/resources/function_list/clickhouse.csv

@@ -1,110 +1,26 @@
-function_type,name,return_type,description
-aggregate,count,UInt64,"Counts the number of rows or non-NULL values."
-aggregate,countIf,UInt64,"Counts the number of rows that match a condition."
-aggregate,min,Same as input,"Returns the minimum value."
-aggregate,max,Same as input,"Returns the maximum value."
-aggregate,sum,Numeric,"Returns the sum of values."
-aggregate,avg,Float64,"Returns the average of values."
-aggregate,uniq,UInt64,"Approximate number of different values using HyperLogLog."
-aggregate,uniqExact,UInt64,"Exact number of different values."
-aggregate,groupArray,Array,"Creates an array of values."
-aggregate,quantile,Float64,"Calculates quantile using reservoir sampling."
-aggregate,quantileDeterministic,Float64,"Deterministic calculation of quantile."
-aggregate,quantileExact,Float64,"Exact calculation of quantile."
-aggregate,quantileExactWeighted,Float64,"Exact calculation of weighted quantile."
-aggregate,varPop,Float64,"Calculate population variance."
-aggregate,varSamp,Float64,"Calculate sample variance."
-aggregate,stddevPop,Float64,"Calculate population standard deviation."
-aggregate,stddevSamp,Float64,"Calculate sample standard deviation."
-aggregate,covarPop,Float64,"Calculate population covariance."
-aggregate,covarSamp,Float64,"Calculate sample covariance."
-aggregate,corrStable,Float64,"Calculate correlation coefficient."
-aggregate,arrayUniq,UInt64,"Returns number of unique elements across arrays."
-aggregate,sumArray,Array,"Sum of arrays element-wise."
-aggregate,groupArrayArray,Array,"Combines multiple arrays into array of arrays."
-scalar,toInt8,Int8,"Converts input to Int8 data type."
-scalar,toInt16,Int16,"Converts input to Int16 data type."
-scalar,toInt32,Int32,"Converts input to Int32 data type."
-scalar,toInt64,Int64,"Converts input to Int64 data type."
-scalar,toUInt8,UInt8,"Converts input to UInt8 data type."
-scalar,toUInt16,UInt16,"Converts input to UInt16 data type."
-scalar,toUInt32,UInt32,"Converts input to UInt32 data type."
-scalar,toUInt64,UInt64,"Converts input to UInt64 data type."
-scalar,toFloat32,Float32,"Converts input to Float32 data type."
-scalar,toFloat64,Float64,"Converts input to Float64 data type."
-scalar,toDate,Date,"Converts input to Date type."
-scalar,toDateTime,DateTime,"Converts input to DateTime type."
-scalar,toString,String,"Converts input to String type."
-scalar,toDecimal32,Decimal32,"Converts input to Decimal32 type."
-scalar,toDecimal64,Decimal64,"Converts input to Decimal64 type."
-scalar,toDecimal128,Decimal128,"Converts input to Decimal128 type."
-scalar,abs,Numeric,"Returns absolute value."
-scalar,round,Same as input,"Rounds number to specified decimal places."
-scalar,floor,Same as input,"Rounds down to nearest integer."
-scalar,ceil,Same as input,"Rounds up to nearest integer."
-scalar,exp,Float64,"Returns e raised to the power of x."
-scalar,log,Float64,"Returns natural logarithm."
-scalar,log2,Float64,"Returns base-2 logarithm."
-scalar,log10,Float64,"Returns base-10 logarithm."
-scalar,power,Float64,"Raises a number to the specified power."
-scalar,sqrt,Float64,"Returns square root."
-scalar,cbrt,Float64,"Returns cube root."
-scalar,rand,UInt32,"Returns random number."
-scalar,rand64,UInt64,"Returns random 64-bit number."
-scalar,pi,Float64,"Returns value of π."
-scalar,e,Float64,"Returns value of e."
-scalar,match,UInt8,"Returns whether string matches regex pattern."
-scalar,like,UInt8,"Pattern matching using LIKE operator."
-scalar,notLike,UInt8,"Inverse of LIKE operator."
-scalar,concat,String,"Concatenates strings."
-scalar,substring,String,"Extracts substring."
-scalar,substringUTF8,String,"Extracts substring considering UTF-8 encoding."
-scalar,length,UInt64,"Returns string length in bytes."
-scalar,lengthUTF8,UInt64,"Returns string length in Unicode characters."
-scalar,lower,String,"Converts string to lowercase."
-scalar,upper,String,"Converts string to uppercase."
-scalar,trim,String,"Removes leading and trailing whitespace."
-scalar,ltrim,String,"Removes leading whitespace."
-scalar,rtrim,String,"Removes trailing whitespace."
-scalar,reverse,String,"Reverses string."
-scalar,replaceAll,String,"Replaces all occurrences of substring."
-scalar,replaceRegexpAll,String,"Replaces all regex matches."
-scalar,empty,UInt8,"Checks if array is empty."
-scalar,notEmpty,UInt8,"Checks if array is not empty."
-scalar,arrayLength,UInt64,"Returns length of array."
-scalar,arrayConcat,Array,"Concatenates arrays."
-scalar,arrayElement,Any,"Returns array element at specified index."
-scalar,has,UInt8,"Checks if array contains value."
-scalar,indexOf,UInt64,"Returns index of first occurrence of element."
-scalar,toYear,UInt16,"Extracts year from date/datetime."
-scalar,toQuarter,UInt8,"Extracts quarter from date/datetime."
-scalar,toMonth,UInt8,"Extracts month from date/datetime."
-scalar,toDayOfMonth,UInt8,"Extracts day of month from date/datetime."
-scalar,toDayOfWeek,UInt8,"Extracts day of week from date/datetime."
-scalar,toHour,UInt8,"Extracts hour from datetime."
-scalar,toMinute,UInt8,"Extracts minute from datetime."
-scalar,toSecond,UInt8,"Extracts second from datetime."
-scalar,now,DateTime,"Returns current date and time."
-scalar,today,Date,"Returns current date."
-scalar,yesterday,Date,"Returns yesterday's date."
-scalar,formatDateTime,String,"Formats date/time according to given format."
-scalar,JSONHas,UInt8,"Checks if JSON field exists."
-scalar,JSONLength,UInt64,"Returns length of JSON array or object."
-scalar,JSONType,String,"Returns type of JSON value."
-scalar,JSONExtractString,String,"Extracts string from JSON."
-scalar,JSONExtractFloat,Float64,"Extracts float from JSON."
-scalar,JSONExtractInt,Int64,"Extracts integer from JSON."
-scalar,IPv4NumToString,String,"Converts IPv4 to string."
-scalar,IPv4StringToNum,UInt32,"Converts string to IPv4."
-scalar,IPv6NumToString,String,"Converts IPv6 to string."
-scalar,IPv6StringToNum,FixedString(16),"Converts string to IPv6."
-window,rowNumberInAllBlocks,UInt64,"Returns row number across all blocks."
-window,rowNumberInBlock,UInt64,"Returns row number within current block."
-window,rank,UInt64,"Returns rank of current row with gaps."
-window,denseRank,UInt64,"Returns rank of current row without gaps."
-window,runningDifference,Same as input,"Returns difference between successive values."
-window,lagInFrame,Same as input,"Returns value from previous row in frame."
-window,leadInFrame,Same as input,"Returns value from next row in frame."
-window,firstValue,Same as input,"Returns first value in window frame."
-window,lastValue,Same as input,"Returns last value in window frame."
-window,nthValue,Same as input,"Returns nth value in window frame."
+function_type,name,return_type,param_names,param_types,description
+aggregate,uniq,UInt64,,Any,"Approximate number of different values using HyperLogLog."
+scalar,abs,Numeric,,Numeric,"Returns absolute value."
+scalar,round,Float,,Float, "Rounds number to specified decimal places."
+scalar,floor,Float,,Float,"Rounds down to nearest integer."
+scalar,ceil,Float,,Float,"Rounds up to nearest integer."
+scalar,exp,Float,,Float,"Returns e raised to the power of x."
+scalar,log,Float,,Float,"Returns natural logarithm."
+scalar,log2,Float,,Float,"Returns base-2 logarithm."
+scalar,log10,Float,,Float,"Returns base-10 logarithm."
+scalar,sqrt,Float,,Float,"Returns square root."
+scalar,rand,UInt32,,,"Returns random number."
+scalar,rand64,UInt64,,,"Returns random 64-bit number."
+scalar,pi,Float,,,"Returns value of π."
+scalar,e,Float,,,"Returns value of e."
+scalar,concat,String,,Array<String>, "Concatenates strings."
+scalar,length,UInt64,,String,"Returns string length in bytes."
+scalar,lower,String,,String,"Converts string to lowercase."
+scalar,upper,String,,String,"Converts string to uppercase."
+scalar,ltrim,String,,String,"Removes leading whitespace."
+scalar,rtrim,String,,String,"Removes trailing whitespace."
+scalar,reverse,String,,String,"Reverses string."
+scalar,empty,UInt8,,Array, "Checks if array is empty."
+scalar,now,DateTime,,,"Returns current date and time."
+scalar,today,Date,,,"Returns current date."
+scalar,yesterday,Date,,,"Returns yesterday's date."

ibis-server/tests/routers/v3/connector/clickhouse/test_functions.py

@@ -1,4 +1,5 @@
 import base64
+import os
 
 import orjson
 import pytest
@@ -8,6 +9,7 @@
 from app.main import app
 from tests.conftest import DATAFUSION_FUNCTION_COUNT, file_path
 from tests.routers.v3.connector.clickhouse.conftest import base_url
+from tests.util import FunctionCsvParser, SqlTestGenerator
 
 manifest = {
     "catalog": "my_catalog",
@@ -57,14 +59,14 @@ def test_function_list():
         response = client.get(url=f"{base_url}/functions")
         assert response.status_code == 200
         result = response.json()
-        assert len(result) == DATAFUSION_FUNCTION_COUNT + 79
+        assert len(result) == DATAFUSION_FUNCTION_COUNT + 5
         the_func = next(filter(lambda x: x["name"] == "abs", result))
         assert the_func == {
             "name": "abs",
             "description": "Returns absolute value.",
             "function_type": "scalar",
             "param_names": None,
-            "param_types": None,
+            "param_types": "Numeric",
             "return_type": "Numeric",
         }
 
@@ -107,3 +109,19 @@ def test_aggregate_function(manifest_str: str, connection_info):
             "data": [[1]],
             "dtypes": {"col": "uint64"},
         }
+
+    def test_functions(manifest_str: str, connection_info):
+        csv_path = os.path.join(function_list_path, "clickhouse.csv")
+        csv_parser = FunctionCsvParser(csv_path)
+        sql_generator = SqlTestGenerator("clickhouse")
+        for function in csv_parser.parse():
+            sql = sql_generator.generate_sql(function)
+            response = client.post(
+                url=f"{base_url}/query",
+                json={
+                    "connectionInfo": connection_info,
+                    "manifestStr": manifest_str,
+                    "sql": sql,
+                },
+            )
+            assert response.status_code == 200

ibis-server/tests/util/sql_generator.py

@@ -30,6 +30,8 @@ def _get_generator(self):
             return MySqlGenerator()
         if self.dialect == "mssql":
             return MSSqlGenerator()
+        if self.dialect == "clickhouse":
+            return ClickhouseSqlGenerator()
         raise NotImplementedError(f"Unsupported dialect: {self.dialect}")
 
     @staticmethod
@@ -191,3 +193,28 @@ def generate_window_sql(self, function: Function) -> str:
                 SELECT 5 AS id, 'A' AS category
             ) AS t
         """
+
+
+class ClickhouseSqlGenerator(SqlGenerator):
+    def generate_aggregate_sql(self, function: Function) -> str:
+        sample_args = self.generate_sample_args(function.param_types)
+        formatted_args = ", ".join(sample_args)
+        return f"SELECT {function.name}({formatted_args})"
+
+    def generate_scalar_sql(self, function: Function) -> str:
+        args = self.generate_sample_args(function.param_types)
+        formatted_args = ", ".join(args)
+        return f"SELECT {function.name}({formatted_args})"
+
+    def generate_window_sql(self, function: Function) -> str:
+        return f"""
+            SELECT
+                {function.name}() OVER (ORDER BY id) AS {function.name.lower()} 
+            FROM (
+                SELECT 1 AS id, 'A' AS category UNION ALL
+                SELECT 2 AS id, 'B' AS category UNION ALL
+                SELECT 3 AS id, 'A' AS category UNION ALL
+                SELECT 4 AS id, 'B' AS category UNION ALL
+                SELECT 5 AS id, 'A' AS category
+            ) AS t
+        """