Merge pull request #4 from david-engelmann/master

update setup cleanup to most recent version

.github/workflows/ci.yml

@@ -42,6 +42,7 @@ jobs:
           - +test-python3.8-arrow7.x.x
           - +test-python3.8-arrow8.x.x
           - +test-python3.8-arrow9.x.x
+          - +test-python3.8-arrow10.x.x
           - +test-python3.9-arrow1.x.x
           - +test-python3.9-arrow2.x.x
           - +test-python3.9-arrow3.x.x
@@ -51,6 +52,7 @@ jobs:
           - +test-python3.9-arrow7.x.x
           - +test-python3.9-arrow8.x.x
           - +test-python3.9-arrow9.x.x
+          - +test-python3.9-arrow10.x.x
           - +test-python3.10-arrow1.x.x
           - +test-python3.10-arrow2.x.x
           - +test-python3.10-arrow3.x.x
@@ -60,16 +62,17 @@ jobs:
           - +test-python3.10-arrow7.x.x
           - +test-python3.10-arrow8.x.x
           - +test-python3.10-arrow9.x.x
+          - +test-python3.10-arrow10.x.x
         experimental: [ false ]
         upload: [ false ]
         include:
-          - target: +test-python3.8-arrow9.x.x
+          - target: +test-python3.8-arrow10.x.x
             experimental: false
             upload: true
-          - target: +test-python3.9-arrow9.x.x
+          - target: +test-python3.9-arrow10.x.x
             experimental: false
             upload: true
-          - target: +test-python3.10-arrow9.x.x
+          - target: +test-python3.10-arrow10.x.x
             experimental: false
             upload: true
           - target: +test-python3.8-arrow-nightly

CHANGELOG.rst

@@ -3,6 +3,21 @@ Version history / changelog
 
 From version 2.0.0, turbodbc adapts semantic versioning.
 
+Version 4.5.6
+-------------
+
+* add in support and test for cursor.nextset() when dealing with multiple result sets
+* modified the pyarrow logic in the setup.py to utilize the following helper
+  functions
+    * pyarrow.get_include()
+    * pyarrow.get_libraries()
+    * pyarrow.get_library_dirs()
+* Bump pyarrow support to <11
+    * Requires an additional pyarrow specific include directory
+* Update the Microsoft ODBC Driver 17 for SQL Server version
+* Bump c++11 to c++17 to enable pyarrow10 support
+* Remove static lib logic for FindArrow
+
 Version 4.5.5
 -------------
 

Earthfile

@@ -22,7 +22,7 @@ os-base:
         wget -q https://packages.microsoft.com/config/debian/9/prod.list -O- > /etc/apt/sources.list.d/mssql-release.list && \
         apt-get update && \
         ACCEPT_EULA=Y apt-get install msodbcsql17 mssql-tools && \
-	ls -l  /opt/microsoft/msodbcsql17/lib64/ && \
+        ls -l  /opt/microsoft/msodbcsql17/lib64/ && \
         odbcinst -i -d -f /opt/microsoft/msodbcsql17/etc/odbcinst.ini
 
     # not used for the moment as Exasol is not tested here
@@ -212,6 +212,15 @@ test-python3.8-arrow9.x.x:
 
     SAVE ARTIFACT /result AS LOCAL result
 
+test-python3.8-arrow10.x.x:
+    ARG PYTHON_VERSION="3.8.12"
+    COPY --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
+        --build-arg ARROW_VERSION_RULE=">=10,<11" \
+        --build-arg NUMPY_VERSION_RULE=">=1.20.0" \
+        +test/result /result
+
+    SAVE ARTIFACT /result AS LOCAL result
+
 test-python3.8-arrow-nightly:
     ARG PYTHON_VERSION="3.8.12"
     COPY --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
@@ -302,6 +311,15 @@ test-python3.9-arrow9.x.x:
 
     SAVE ARTIFACT /result AS LOCAL result
 
+test-python3.9-arrow10.x.x:
+    ARG PYTHON_VERSION="3.9.10"
+    COPY --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
+        --build-arg ARROW_VERSION_RULE=">=10,<11" \
+        --build-arg NUMPY_VERSION_RULE=">=1.20.0" \
+        +test/result /result
+
+    SAVE ARTIFACT /result AS LOCAL result
+
 test-python3.9-arrow-nightly:
     ARG PYTHON_VERSION="3.9.10"
     COPY --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
@@ -392,6 +410,15 @@ test-python3.10-arrow9.x.x:
 
     SAVE ARTIFACT /result AS LOCAL result
 
+test-python3.10-arrow10.x.x:
+    ARG PYTHON_VERSION="3.10.2"
+    COPY --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
+        --build-arg ARROW_VERSION_RULE=">=10,<11" \
+        --build-arg NUMPY_VERSION_RULE=">=1.21.2" \
+        +test/result /result
+
+    SAVE ARTIFACT /result AS LOCAL result
+
 test-python3.10-arrow-nightly:
     ARG PYTHON_VERSION="3.10.2"
     COPY --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
@@ -400,7 +427,7 @@ test-python3.10-arrow-nightly:
         +test/result /result
 
     SAVE ARTIFACT /result AS LOCAL result/$EARTHLY_TARGET_NAME
-    
+
 test-python3.8-all:
     BUILD test-python3.8-arrow1.x.x
     BUILD test-python3.8-arrow2.x.x
@@ -411,6 +438,7 @@ test-python3.8-all:
     BUILD test-python3.8-arrow7.x.x
     BUILD test-python3.8-arrow8.x.x
     BUILD test-python3.8-arrow9.x.x
+    BUILD test-python3.8-arrow10.x.x
     BUILD test-python3.8-arrow-nightly
 
 test-python3.9-all:
@@ -423,8 +451,9 @@ test-python3.9-all:
     BUILD test-python3.9-arrow7.x.x
     BUILD test-python3.9-arrow8.x.x
     BUILD test-python3.9-arrow9.x.x
+    BUILD test-python3.9-arrow10.x.x
     BUILD test-python3.9-arrow-nightly
-    
+
 test-python3.10-all:
     BUILD test-python3.10-arrow1.x.x
     BUILD test-python3.10-arrow2.x.x
@@ -435,6 +464,7 @@ test-python3.10-all:
     BUILD test-python3.10-arrow7.x.x
     BUILD test-python3.10-arrow8.x.x
     BUILD test-python3.10-arrow9.x.x
+    BUILD test-python3.10-arrow10.x.x
     BUILD test-python3.10-arrow-nightly
 
 test-all:

cmake_scripts/FindArrow.cmake

@@ -15,11 +15,11 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# - Find ARROW (arrow/api.h, libarrow.a, libarrow.so)
+# - Find ARROW (arrow/api.h, libarrow.so)
 # This module defines
 #  ARROW_INCLUDE_DIR, directory containing headers
+#  PYARROW_INCLUDE_DIR, directory containing headers
 #  ARROW_LIBS, directory containing arrow libraries
-#  ARROW_STATIC_LIB, path to libarrow.a
 #  ARROW_SHARED_LIB, path to libarrow's shared library
 #  ARROW_FOUND, whether arrow has been found
 
@@ -41,19 +41,48 @@ if (NOT ARROW_HOME)
     pkg_get_variable(ARROW_SO_VERSION arrow so_version)
     message(STATUS "Arrow SO version: ${ARROW_SO_VERSION}")
     set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
+    execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import pyarrow as pa; print(pa.get_include());"
+                RESULT_VARIABLE _PYARROW_SEARCH_SUCCESS
+                OUTPUT_VARIABLE PYARROW_INCLUDE_DIR
+                ERROR_VARIABLE _PYARROW_ERROR_VALUE
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
     set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
     set(ARROW_SEARCH_LIB_PATH ${ARROW_LIBRARY_DIRS})
   elseif(DEFINED ENV{VIRTUAL_ENV})
-    find_path(ARROW_INCLUDE_DIR arrow/api.h HINTS
-      $ENV{VIRTUAL_ENV}/lib/*/site-packages/pyarrow/include)
-    get_filename_component(ARROW_SEARCH_LIB_PATH ${ARROW_INCLUDE_DIR} DIRECTORY)
+    execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import pyarrow as pa; print(pa.get_include());"
+                RESULT_VARIABLE _PYARROW_SEARCH_SUCCESS
+                OUTPUT_VARIABLE PYARROW_INCLUDE_DIR
+                ERROR_VARIABLE _PYARROW_ERROR_VALUE
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+    set(ARROW_INCLUDE_DIR ${PYARROW_INCLUDE_DIR})
+    execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import pyarrow as pa; print(pa.get_library_dirs());"
+            RESULT_VARIABLE _PYARROW_SEARCH_SUCCESS
+            OUTPUT_VARIABLE _PYARROW_VALUES_OUTPUT
+            ERROR_VARIABLE _PYARROW_ERROR_VALUE
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+    # convert to the path needed
+    string(REGEX REPLACE "," ";" _PYARROW_VALUES ${_PYARROW_VALUES_OUTPUT})
+    string(REGEX REPLACE "\\]" "" _PYARROW_VALUES ${_PYARROW_VALUES})
+    string(REGEX REPLACE "\\[" "" _PYARROW_VALUES ${_PYARROW_VALUES})
+    list(GET _PYARROW_VALUES 0 ARROW_SEARCH_LIB_PATH)
   else()
     if (MSVC)
       find_path(ARROW_INCLUDE_DIR arrow/api.h HINTS
         $ENV{PYTHON}/lib/site-packages/pyarrow/include)
+      execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import pyarrow as pa; print(pa.get_include());"
+                RESULT_VARIABLE _PYARROW_SEARCH_SUCCESS
+                OUTPUT_VARIABLE PYARROW_INCLUDE_DIR
+                ERROR_VARIABLE _PYARROW_ERROR_VALUE
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
     else()
       find_path(ARROW_INCLUDE_DIR arrow/api.h HINTS
         /usr/local/lib/*/dist-packages/pyarrow/include)
+      execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import pyarrow as pa; print(pa.get_include());"
+                RESULT_VARIABLE _PYARROW_SEARCH_SUCCESS
+                OUTPUT_VARIABLE PYARROW_INCLUDE_DIR
+                ERROR_VARIABLE _PYARROW_ERROR_VALUE
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
     endif()
     get_filename_component(ARROW_SEARCH_LIB_PATH ${ARROW_INCLUDE_DIR} DIRECTORY)
     set(ARROW_SEARCH_HEADER_PATHS ${ARROW_INCLUDE_DIR})
@@ -68,7 +97,13 @@ else()
     ${ARROW_HOME}/lib
     )
 
-  find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
+  execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "import pyarrow as pa; print(pa.get_include());"
+                RESULT_VARIABLE _PYARROW_SEARCH_SUCCESS
+                OUTPUT_VARIABLE PYARROW_INCLUDE_DIR
+                ERROR_VARIABLE _PYARROW_ERROR_VALUE
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  find_path(ARROW_INCLUDE_DIR arrow/api.h PATHS
     ${ARROW_SEARCH_HEADER_PATHS}
     # make sure we don't accidentally pick up a different version
     NO_DEFAULT_PATH
@@ -87,8 +122,10 @@ message(STATUS "Found ${ARROW_LIB_PATH} in ${ARROW_SEARCH_LIB_PATH}")
 get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
 
 find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
-  PATHS
-  ${ARROW_SEARCH_LIB_PATH}
+    PATHS
+    ${ARROW_SEARCH_LIB_PATH}/*/site-packages/pyarrow
+    ${ARROW_SEARCH_LIB_PATH}
+  DOC "Path to the libarrow_python headers"
   NO_DEFAULT_PATH)
 
 if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
@@ -98,18 +135,13 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
   set(ARROW_PYTHON_LIB_NAME arrow_python)
 
   if (MSVC)
-    set(ARROW_STATIC_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
-    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
     set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
     set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
     set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib)
     set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib)
   else()
-    set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
-    set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
-
     set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
-    set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_LIB_PATH})
   endif()
 endif()
 
@@ -134,10 +166,8 @@ endif ()
 
 mark_as_advanced(
   ARROW_INCLUDE_DIR
-  ARROW_STATIC_LIB
+  PYARROW_INCLUDE_DIR
   ARROW_SHARED_LIB
-  ARROW_PYTHON_STATIC_LIB
   ARROW_PYTHON_SHARED_LIB
-  ARROW_JEMALLOC_STATIC_LIB
   ARROW_JEMALLOC_SHARED_LIB
 )

cmake_scripts/default_build_setup.cmake

@@ -14,7 +14,7 @@ endif()
 message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
 
 # The language standard we use
-add_definitions("-std=c++11")
+add_definitions("-std=c++17")
 
 # build shared instead of static libraries
 set(BUILD_SHARED_LIBS TRUE)

cpp/turbodbc/Library/src/command.cpp

@@ -43,6 +43,11 @@ std::shared_ptr<turbodbc::result_sets::result_set> command::get_results()
     return results_;
 }
 
+bool command::more_results() const
+{
+    return statement_->more_results();
+}
+
 bound_parameter_set & command::get_parameters()
 {
     return params_;

cpp/turbodbc/Library/src/cursor.cpp

@@ -66,6 +66,11 @@ std::shared_ptr<result_sets::result_set> cursor::get_result_set() const
     return command_->get_results();
 }
 
+bool cursor::more_results() const
+{
+    return command_->more_results();
+}
+
 int64_t cursor::get_row_count()
 {
     return command_->get_row_count();

cpp/turbodbc/Library/turbodbc/command.h

@@ -38,6 +38,8 @@ class command {
 
     int64_t get_row_count();
 
+    bool more_results() const;
+
     /**
      * @brief discard any ressources associated with this instance
      * 

cpp/turbodbc/Library/turbodbc/cursor.h

@@ -30,6 +30,8 @@ class cursor {
 
     int64_t get_row_count();
 
+    bool more_results() const;
+
     std::shared_ptr <result_sets::result_set> get_result_set() const;
 
     std::shared_ptr<cpp_odbc::connection const> get_connection() const;

cpp/turbodbc_arrow/CMakeLists.txt

@@ -10,7 +10,7 @@ find_package(numpy REQUIRED)
 include_directories(SYSTEM ${Numpy_INCLUDE_DIR})
 
 find_package(Arrow REQUIRED)
-include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
+include_directories(SYSTEM ${ARROW_INCLUDE_DIR} ${PYARROW_INCLUDE_DIR})
 add_library(arrow SHARED IMPORTED)
 set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${ARROW_SHARED_LIB})
 add_library(arrow_python SHARED IMPORTED)

cpp/turbodbc_python/Library/src/python_bindings/cursor.cpp

@@ -16,6 +16,7 @@ void for_cursor(pybind11::module & module)
             .def("_reset",  &turbodbc::cursor::reset)
             .def("get_row_count", &turbodbc::cursor::get_row_count)
             .def("get_result_set", &turbodbc::cursor::get_result_set)
+            .def("more_results", &turbodbc::cursor::more_results)
         ;
 }
 

docs/pages/advanced_usage.rst

@@ -545,3 +545,19 @@ tables as query parameters with ``executemanycolumns()``:
 +------------------------------------------------------------------------------+--------------------------------+
 | ``STRING``                                                                   | ``VARCHAR`` (automatic sizing) |
 +------------------------------------------------------------------------------+--------------------------------+
+
+Using multiple results sets
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Some commands will return multi result sets, you can cycle through them with the
+`nextset` function. It will discard the current result set and load up the next
+result set, if another was loaded it return True. If no additional set is found
+it returns False:
+::
+
+    >>> cursor.execute("SELECT 4;SELECT 2;")
+    >>> first_set = cursor.fetchall()
+    >>> while(cursor.nextset()):
+    ...     remaining_set = cursor.fetchall();
+    >>> assert first_set = [[4]]
+    >>> assert remaining_set = [[2]]
+::

earthly/odbc/odbcinst.ini

@@ -10,4 +10,4 @@ Threading   = 2
 
 [ODBC Driver 17 for SQL Server]
 Description = Microsoft ODBC Driver 17 for SQL Server
-Driver      = /opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.1.1
+Driver      = /opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1

python/turbodbc/cursor.py

@@ -394,6 +394,16 @@ def fetchallarrow(self, strings_as_dictionary=False, adaptive_integers=False):
         else:
             raise Error(_NO_ARROW_SUPPORT_MSG)
 
+    def nextset(self) -> bool:
+        """
+        makes cursor skip to the next available result set, discarding any remaining rows from the
+        current result set. If there are no more result sets, the method returns False.
+        Otherwise, it returns a True and subsequent calls to the fetch methods will
+        return rows from the next result set.
+        """
+        self._assert_valid_result_set()
+        return self.impl.more_results()
+
     def close(self):
         """
         Close the cursor.