From 13d33e0f51c08932f0ed719ed75aa486238ab882 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 31 Aug 2020 09:50:10 -0400 Subject: [PATCH 1/7] Define dataset directory for Python package --- CMakeLists.txt | 8 ++++++++ gtsam/config.h.in | 1 + 2 files changed, 9 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9dce69903c..8a8b6ce017 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -428,6 +428,14 @@ add_subdirectory(CppUnitLite) if(GTSAM_BUILD_PYTHON) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/wrap/cmake") add_subdirectory(python) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # Common directory which always has write access + # This is because the exact location of the Python installation may vary + # depending on various factors such as python version and use of virtual envs. + set(GTSAM_PYTHON_DATASET_DIR "/tmp/gtsam_example_data") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(GTSAM_PYTHON_DATASET_DIR "$ENV{LOCALAPPDATA}/gtsam_example_data") + endif() endif() # Build GTSAM library diff --git a/gtsam/config.h.in b/gtsam/config.h.in index 9d1bd4ebdf..7f5be65b0c 100644 --- a/gtsam/config.h.in +++ b/gtsam/config.h.in @@ -27,6 +27,7 @@ // Paths to example datasets distributed with GTSAM #define GTSAM_SOURCE_TREE_DATASET_DIR "@GTSAM_SOURCE_DIR@/examples/Data" #define GTSAM_INSTALLED_DATASET_DIR "@GTSAM_TOOLBOX_INSTALL_PATH@/gtsam_examples/Data" +#define GTSAM_PYTHON_DATASET_DIR "@GTSAM_PYTHON_DATASET_DIR" // Whether GTSAM is compiled to use quaternions for Rot3 (otherwise uses rotation matrices) #cmakedefine GTSAM_USE_QUATERNIONS From a7f8ba7ac4a0430e06492699bba97b965a151213 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 31 Aug 2020 09:50:47 -0400 Subject: [PATCH 2/7] Add python dataset directory to search options --- gtsam/slam/dataset.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/gtsam/slam/dataset.cpp b/gtsam/slam/dataset.cpp index 4bd7bc7e24..de8150324d 100644 --- a/gtsam/slam/dataset.cpp +++ b/gtsam/slam/dataset.cpp @@ -69,6 +69,7 @@ string findExampleDataFile(const string &name) { // Constants below are defined by CMake, see gtsam/gtsam/CMakeLists.txt rootsToSearch.push_back(GTSAM_SOURCE_TREE_DATASET_DIR); rootsToSearch.push_back(GTSAM_INSTALLED_DATASET_DIR); + rootsToSearch.push_back(GTSAM_PYTHON_DATASET_DIR); // Search for filename as given, and with .graph and .txt extensions vector namesToSearch; From 3674b9c65f77638eaf46ec7241b5c8da6b1ada78 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 31 Aug 2020 09:51:14 -0400 Subject: [PATCH 3/7] Add data files to python package --- python/setup.py.in | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/setup.py.in b/python/setup.py.in index a7acdc4baf..1ffe978f3f 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,3 +1,4 @@ +import glob import os import sys @@ -8,6 +9,11 @@ except ImportError: packages = find_packages(where=".") print("PACKAGES: ", packages) + +data_path = '${GTSAM_SOURCE_DIR}/examples/Data/' +data_files_and_directories = glob.glob(data_path + '**', recursive=True) +data_files = [x for x in data_files_and_directories if not os.path.isdir(x)] + package_data = { '': [ './*.so', @@ -44,6 +50,7 @@ setup( ], packages=packages, package_data=package_data, + data_files=[('${GTSAM_PYTHON_DATASET_DIR}', data_files),], test_suite="gtsam.tests", install_requires=["numpy"], zip_safe=False, From ac3e564293c23aca557ecb7537dd4928b99721c7 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 31 Aug 2020 10:10:54 -0400 Subject: [PATCH 4/7] Better location for setting Python dataset directory in CMake --- CMakeLists.txt | 8 -------- python/CMakeLists.txt | 11 +++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a8b6ce017..9dce69903c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -428,14 +428,6 @@ add_subdirectory(CppUnitLite) if(GTSAM_BUILD_PYTHON) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/wrap/cmake") add_subdirectory(python) - if(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Darwin") - # Common directory which always has write access - # This is because the exact location of the Python installation may vary - # depending on various factors such as python version and use of virtual envs. - set(GTSAM_PYTHON_DATASET_DIR "/tmp/gtsam_example_data") - elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(GTSAM_PYTHON_DATASET_DIR "$ENV{LOCALAPPDATA}/gtsam_example_data") - endif() endif() # Build GTSAM library diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 23898f61d5..6fad7ba455 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,6 +4,17 @@ if (NOT GTSAM_BUILD_PYTHON) return() endif() +# Common directory for storing data stored with the package. +# This is because the exact location of the Python installation may vary +# depending on various factors such as python version and use of virtual envs. +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # Easy directory with write access. + set(GTSAM_PYTHON_DATASET_DIR "/tmp/gtsam_example_data") +elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + # %LocalAppData% directory in Windows. + set(GTSAM_PYTHON_DATASET_DIR "$ENV{LOCALAPPDATA}/gtsam_example_data") +endif() + # Generate setup.py. file(READ "${PROJECT_SOURCE_DIR}/README.md" README_CONTENTS) configure_file(${PROJECT_SOURCE_DIR}/python/setup.py.in From b23cdca66a4693008687251eba2e3a846ca10c9c Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 31 Aug 2020 13:55:09 -0400 Subject: [PATCH 5/7] Undo previous changes for common directory --- gtsam/config.h.in | 1 - gtsam/slam/dataset.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/gtsam/config.h.in b/gtsam/config.h.in index 7f5be65b0c..9d1bd4ebdf 100644 --- a/gtsam/config.h.in +++ b/gtsam/config.h.in @@ -27,7 +27,6 @@ // Paths to example datasets distributed with GTSAM #define GTSAM_SOURCE_TREE_DATASET_DIR "@GTSAM_SOURCE_DIR@/examples/Data" #define GTSAM_INSTALLED_DATASET_DIR "@GTSAM_TOOLBOX_INSTALL_PATH@/gtsam_examples/Data" -#define GTSAM_PYTHON_DATASET_DIR "@GTSAM_PYTHON_DATASET_DIR" // Whether GTSAM is compiled to use quaternions for Rot3 (otherwise uses rotation matrices) #cmakedefine GTSAM_USE_QUATERNIONS diff --git a/gtsam/slam/dataset.cpp b/gtsam/slam/dataset.cpp index de8150324d..4bd7bc7e24 100644 --- a/gtsam/slam/dataset.cpp +++ b/gtsam/slam/dataset.cpp @@ -69,7 +69,6 @@ string findExampleDataFile(const string &name) { // Constants below are defined by CMake, see gtsam/gtsam/CMakeLists.txt rootsToSearch.push_back(GTSAM_SOURCE_TREE_DATASET_DIR); rootsToSearch.push_back(GTSAM_INSTALLED_DATASET_DIR); - rootsToSearch.push_back(GTSAM_PYTHON_DATASET_DIR); // Search for filename as given, and with .graph and .txt extensions vector namesToSearch; From 47800f31127c23a22dc55734b7bc0c221007141e Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 31 Aug 2020 13:55:21 -0400 Subject: [PATCH 6/7] don't wrap findExampleDataFile --- gtsam/gtsam.i | 1 - 1 file changed, 1 deletion(-) diff --git a/gtsam/gtsam.i b/gtsam/gtsam.i index 79d489128f..33d8c11fee 100644 --- a/gtsam/gtsam.i +++ b/gtsam/gtsam.i @@ -2800,7 +2800,6 @@ class SfmData { gtsam::SfmTrack track(size_t idx) const; }; -string findExampleDataFile(string name); pair load2D(string filename, gtsam::noiseModel::Diagonal* model, int maxIndex, bool addNoise, bool smart); pair load2D(string filename, From 43dfe0f47d8e001572b7cb2471f5ca90515e2dfd Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 31 Aug 2020 13:56:56 -0400 Subject: [PATCH 7/7] Add python version of findExampleDataFile --- python/CMakeLists.txt | 13 +++---------- python/gtsam/__init__.py | 2 ++ python/gtsam/utils/__init__.py | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6fad7ba455..e15e571f5f 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,16 +4,9 @@ if (NOT GTSAM_BUILD_PYTHON) return() endif() -# Common directory for storing data stored with the package. -# This is because the exact location of the Python installation may vary -# depending on various factors such as python version and use of virtual envs. -if(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Darwin") - # Easy directory with write access. - set(GTSAM_PYTHON_DATASET_DIR "/tmp/gtsam_example_data") -elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") - # %LocalAppData% directory in Windows. - set(GTSAM_PYTHON_DATASET_DIR "$ENV{LOCALAPPDATA}/gtsam_example_data") -endif() +# Common directory for storing data/datasets stored with the package. +# This will store the data in the Python site package directly. +set(GTSAM_PYTHON_DATASET_DIR "./gtsam/Data") # Generate setup.py. file(READ "${PROJECT_SOURCE_DIR}/README.md" README_CONTENTS) diff --git a/python/gtsam/__init__.py b/python/gtsam/__init__.py index b2d1fb7e79..70a00c3dcd 100644 --- a/python/gtsam/__init__.py +++ b/python/gtsam/__init__.py @@ -1,4 +1,6 @@ +from . import utils from .gtsam import * +from .utils import findExampleDataFile def _init(): diff --git a/python/gtsam/utils/__init__.py b/python/gtsam/utils/__init__.py index e69de29bb2..5ee733ba48 100644 --- a/python/gtsam/utils/__init__.py +++ b/python/gtsam/utils/__init__.py @@ -0,0 +1,22 @@ +import glob +import os.path as osp + + +def findExampleDataFile(name): + """ + Find the example data file specified by `name`. + """ + # This is okay since gtsam will already be loaded + # before this function is accessed. Thus no effect. + import gtsam + + site_package_path = gtsam.__path__[0] + # add the * at the end to glob the entire directory + data_path = osp.join(site_package_path, "Data", "*") + + extensions = ("", ".graph", ".txt", ".out", ".xml", ".g2o") + + for data_file_path in glob.glob(data_path, recursive=True): + for ext in extensions: + if (name + ext) == osp.basename(data_file_path): + return data_file_path