diff --git a/autotest/ogr/ogr_index_test.py b/autotest/ogr/ogr_index_test.py index 942b0b81605f..786b2441d6d8 100755 --- a/autotest/ogr/ogr_index_test.py +++ b/autotest/ogr/ogr_index_test.py @@ -27,27 +27,19 @@ ############################################################################### import os +import contextlib - -import gdaltest from osgeo import ogr import ogrtest import pytest ############################################################################### + + @pytest.fixture(autouse=True, scope='module') def startup_and_cleanup(): yield - gdaltest.p_ds = None - gdaltest.s_ds = None - - gdaltest.p_lyr = None - gdaltest.s_lyr = None - - ogr.GetDriverByName('MapInfo File').DeleteDataSource('index_p.mif') - ogr.GetDriverByName('ESRI Shapefile').DeleteDataSource('join_t.dbf') - for filename in ['join_t.idm', 'join_t.ind']: assert not os.path.exists(filename) @@ -57,93 +49,70 @@ def startup_and_cleanup(): 'tmp/ogr_index_11.dbf') -############################################################################### -# Create a MIF file to be our primary table. - - -def test_ogr_index_1(): - - from osgeo import gdal - gdal.PushErrorHandler('CPLQuietErrorHandler') - try: - ogr.GetDriverByName('MapInfo File').DeleteDataSource('index_p.mif') - except AttributeError: - pass - try: - ogr.GetDriverByName('ESRI Shapefile').DeleteDataSource('join_t.dbf') - except AttributeError: - pass +@contextlib.contextmanager +def create_index_p_test_file(): + drv = ogr.GetDriverByName('MapInfo File') + p_ds = drv.CreateDataSource('index_p.mif') + p_lyr = p_ds.CreateLayer('index_p') - gdal.PopErrorHandler() + ogrtest.quick_create_layer_def(p_lyr, [('PKEY', ogr.OFTInteger)]) + ogrtest.quick_create_feature(p_lyr, [5], None) + ogrtest.quick_create_feature(p_lyr, [10], None) + ogrtest.quick_create_feature(p_lyr, [9], None) + ogrtest.quick_create_feature(p_lyr, [4], None) + ogrtest.quick_create_feature(p_lyr, [3], None) + ogrtest.quick_create_feature(p_lyr, [1], None) - drv = ogr.GetDriverByName('MapInfo File') - gdaltest.p_ds = drv.CreateDataSource('index_p.mif') - gdaltest.p_lyr = gdaltest.p_ds.CreateLayer('index_p') - - ogrtest.quick_create_layer_def(gdaltest.p_lyr, [('PKEY', ogr.OFTInteger)]) - ogrtest.quick_create_feature(gdaltest.p_lyr, [5], None) - ogrtest.quick_create_feature(gdaltest.p_lyr, [10], None) - ogrtest.quick_create_feature(gdaltest.p_lyr, [9], None) - ogrtest.quick_create_feature(gdaltest.p_lyr, [4], None) - ogrtest.quick_create_feature(gdaltest.p_lyr, [3], None) - ogrtest.quick_create_feature(gdaltest.p_lyr, [1], None) - -# It turns out mapinfo format doesn't allow GetFeatureCount() calls while -# writing ... it just blows an assert! -# if gdaltest.p_lyr.GetFeatureCount() != 7: -# gdaltest.post_reason( 'FeatureCount wrong' ) -# return 'fail' - - # Close and reopen, since it seems the .mif driver does not allow reading - # from a newly created (updatable) file. - - gdaltest.p_ds = None - gdaltest.p_ds = ogr.OpenShared('index_p.mif', update=0) - gdaltest.p_lyr = gdaltest.p_ds.GetLayerByName('index_p') + p_ds.Release() -############################################################################### -# Create a dbf file to be our secondary table. Close it, and reopen shared. + yield + ogr.GetDriverByName('MapInfo File').DeleteDataSource('index_p.mif') -def test_ogr_index_2(): +@contextlib.contextmanager +def create_join_t_test_file(create_index=False): drv = ogr.GetDriverByName('ESRI Shapefile') - gdaltest.s_ds = drv.CreateDataSource('join_t.dbf') - gdaltest.s_lyr = gdaltest.s_ds.CreateLayer('join_t', - geom_type=ogr.wkbNone) + s_ds = drv.CreateDataSource('join_t.dbf') + s_lyr = s_ds.CreateLayer('join_t', + geom_type=ogr.wkbNone) - ogrtest.quick_create_layer_def(gdaltest.s_lyr, + ogrtest.quick_create_layer_def(s_lyr, [('SKEY', ogr.OFTInteger), ('VALUE', ogr.OFTString, 16)]) for i in range(20): - ogrtest.quick_create_feature(gdaltest.s_lyr, [i, 'Value ' + str(i)], None) + ogrtest.quick_create_feature(s_lyr, [i, 'Value ' + str(i)], None) - assert gdaltest.s_lyr.GetFeatureCount() == 20, 'FeatureCount wrong' + if create_index: + s_ds.ExecuteSQL('CREATE INDEX ON join_t USING value') + s_ds.ExecuteSQL('CREATE INDEX ON join_t USING skey') - gdaltest.s_ds.Release() - gdaltest.s_lyr = None - gdaltest.s_ds = None + s_ds.Release() + + yield + + ogr.GetDriverByName('ESRI Shapefile').DeleteDataSource('join_t.dbf') - gdaltest.s_ds = ogr.OpenShared('join_t.dbf', update=1) - gdaltest.s_lyr = gdaltest.s_ds.GetLayerByName('join_t') ############################################################################### # Verify a simple join without indexing. -def test_ogr_index_3(): - +def test_ogr_index_can_join_without_index(): expect = ['Value 5', 'Value 10', 'Value 9', 'Value 4', 'Value 3', 'Value 1'] - sql_lyr = gdaltest.p_ds.ExecuteSQL( - 'SELECT * FROM index_p p ' + - 'LEFT JOIN "join_t.dbf".join_t j ON p.PKEY = j.SKEY ') + with create_index_p_test_file(), create_join_t_test_file(): + p_ds = ogr.OpenShared('index_p.mif', update=0) + + sql_lyr = p_ds.ExecuteSQL( + 'SELECT * FROM index_p p ' + + 'LEFT JOIN "join_t.dbf".join_t j ON p.PKEY = j.SKEY ') - tr = ogrtest.check_features_against_list(sql_lyr, 'VALUE', expect) + tr = ogrtest.check_features_against_list(sql_lyr, 'VALUE', expect) - gdaltest.p_ds.ReleaseResultSet(sql_lyr) + p_ds.ReleaseResultSet(sql_lyr) assert tr @@ -151,42 +120,41 @@ def test_ogr_index_3(): # Create an INDEX on the SKEY and VALUE field in the join table. -def test_ogr_index_4(): - - gdaltest.s_ds.ExecuteSQL('CREATE INDEX ON join_t USING value') - gdaltest.s_ds.ExecuteSQL('CREATE INDEX ON join_t USING skey') +def test_ogr_index_creating_index_causes_index_files_to_be_created(): + with create_join_t_test_file(create_index=True): + for filename in ['join_t.idm', 'join_t.ind']: + assert os.path.exists(filename) ############################################################################### # Check that indexable single int lookup works. -def test_ogr_index_5(): +def test_ogr_index_indexed_single_integer_lookup_works(): + expect = ['Value 5'] - gdaltest.s_lyr.SetAttributeFilter('SKEY = 5') + with create_join_t_test_file(create_index=True): + s_ds = ogr.OpenShared('join_t.dbf', update=1) - expect = ['Value 5'] + s_lyr = s_ds.GetLayerByName('join_t') + s_lyr.SetAttributeFilter('SKEY = 5') - tr = ogrtest.check_features_against_list(gdaltest.s_lyr, 'VALUE', expect) + tr = ogrtest.check_features_against_list(s_lyr, 'VALUE', expect) assert tr ############################################################################### # Check that indexable single string lookup works. -# -# We also close the datasource and reopen to ensure that reloaded indexes -# work OK too. -def test_ogr_index_6(): - - gdaltest.s_ds.Release() - gdaltest.s_ds = ogr.OpenShared('join_t.dbf', update=1) - gdaltest.s_lyr = gdaltest.s_ds.GetLayerByName('join_t') +def test_ogr_index_indexed_single_string_works(): + expect = [5] - gdaltest.s_lyr.SetAttributeFilter("VALUE='Value 5'") + with create_join_t_test_file(create_index=True): + s_ds = ogr.OpenShared('join_t.dbf', update=1) + s_lyr = s_ds.GetLayerByName('join_t') - expect = [5] + s_lyr.SetAttributeFilter("VALUE='Value 5'") - tr = ogrtest.check_features_against_list(gdaltest.s_lyr, 'SKEY', expect) + tr = ogrtest.check_features_against_list(s_lyr, 'SKEY', expect) assert tr @@ -194,13 +162,15 @@ def test_ogr_index_6(): # Check that range query that isn't currently implemented using index works. -def test_ogr_index_7(): - - gdaltest.s_lyr.SetAttributeFilter('SKEY < 3') - +def test_ogr_index_unimplemented_range_query_works(): expect = [0, 1, 2] - tr = ogrtest.check_features_against_list(gdaltest.s_lyr, 'SKEY', expect) + with create_join_t_test_file(create_index=True): + s_ds = ogr.OpenShared('join_t.dbf', update=1) + s_lyr = s_ds.GetLayerByName('join_t') + s_lyr.SetAttributeFilter('SKEY < 3') + + tr = ogrtest.check_features_against_list(s_lyr, 'SKEY', expect) assert tr @@ -208,74 +178,142 @@ def test_ogr_index_7(): # Try join again. -def test_ogr_index_8(): - +def test_ogr_index_indexed_join_works(): expect = ['Value 5', 'Value 10', 'Value 9', 'Value 4', 'Value 3', 'Value 1'] - sql_lyr = gdaltest.p_ds.ExecuteSQL( - 'SELECT * FROM index_p p ' + - 'LEFT JOIN "join_t.dbf".join_t j ON p.PKEY = j.SKEY ') + with create_index_p_test_file(), \ + create_join_t_test_file(create_index=True): + p_ds = ogr.OpenShared('index_p.mif', update=0) + sql_lyr = p_ds.ExecuteSQL( + 'SELECT * FROM index_p p ' + + 'LEFT JOIN "join_t.dbf".join_t j ON p.PKEY = j.SKEY ') - tr = ogrtest.check_features_against_list(sql_lyr, 'VALUE', expect) + tr = ogrtest.check_features_against_list(sql_lyr, 'VALUE', expect) - gdaltest.p_ds.ReleaseResultSet(sql_lyr) + p_ds.ReleaseResultSet(sql_lyr) assert tr ############################################################################### -# Verify that dropping both indexes gets rid of them, and that results still -# work. +# Test that dropping the index causes index files to be removed + +def test_ogr_index_drop_index_removes_files(): + with create_join_t_test_file(create_index=True): + s_ds = ogr.OpenShared('join_t.dbf', update=1) -def test_ogr_index_9(): + s_ds.ExecuteSQL('DROP INDEX ON join_t USING value') + s_ds.ExecuteSQL('DROP INDEX ON join_t USING skey') - gdaltest.s_ds.ExecuteSQL('DROP INDEX ON join_t USING value') - gdaltest.s_ds.ExecuteSQL('DROP INDEX ON join_t USING skey') + s_ds.Release() - gdaltest.s_lyr.SetAttributeFilter('SKEY = 5') + # After dataset closing, check that the index files do not exist after + # dropping the index + for filename in ['join_t.idm', 'join_t.ind']: + assert not os.path.exists(filename) +############################################################################### +# Test that attribute filters work after an index on that attribute is dropped + + +def test_ogr_index_attribute_filter_works_after_drop_index(): expect = ['Value 5'] - tr = ogrtest.check_features_against_list(gdaltest.s_lyr, 'VALUE', expect) - assert tr + with create_join_t_test_file(create_index=True): + s_ds = ogr.OpenShared('join_t.dbf', update=1) + s_lyr = s_ds.GetLayerByName('join_t') - gdaltest.s_ds.Release() + s_ds.ExecuteSQL('DROP INDEX ON join_t USING value') + s_ds.ExecuteSQL('DROP INDEX ON join_t USING skey') - # After dataset closing, check that the index files do not exist after - # dropping the index - for filename in ['join_t.idm', 'join_t.ind']: - assert not os.path.exists(filename) + s_lyr.SetAttributeFilter('SKEY = 5') - # Re-create an index - gdaltest.s_ds = ogr.OpenShared('join_t.dbf', update=1) - gdaltest.s_ds.ExecuteSQL('CREATE INDEX ON join_t USING value') - gdaltest.s_ds.Release() + tr = ogrtest.check_features_against_list(s_lyr, 'VALUE', expect) + assert tr - for filename in ['join_t.idm', 'join_t.ind']: - try: - os.stat(filename) - except OSError: - pytest.fail("%s should exist" % filename) - - f = open('join_t.idm', 'rt') - xml = f.read() - f.close() - assert xml.find('VALUE') != -1, 'VALUE column is not indexed (1)' - - # Close the dataset and re-open - gdaltest.s_ds = ogr.OpenShared('join_t.dbf', update=1) - # At this point the .ind was opened in read-only. Now it - # will be re-opened in read-write mode - gdaltest.s_ds.ExecuteSQL('CREATE INDEX ON join_t USING skey') - - gdaltest.s_ds.Release() - - f = open('join_t.idm', 'rt') - xml = f.read() - f.close() - assert xml.find('VALUE') != -1, 'VALUE column is not indexed (2)' - assert xml.find('SKEY') != -1, 'SKEY column is not indexed (2)' +############################################################################### +# Test that dropping, then re-creating the index causes index files to be +# opened + + +def test_ogr_index_recreating_index_causes_index_files_to_be_created(): + + with create_join_t_test_file(create_index=True): + s_ds = ogr.OpenShared('join_t.dbf', update=1) + + s_ds.ExecuteSQL('DROP INDEX ON join_t USING value') + s_ds.ExecuteSQL('DROP INDEX ON join_t USING skey') + + s_ds.Release() + + # Re-create an index + s_ds = ogr.OpenShared('join_t.dbf', update=1) + s_ds.ExecuteSQL('CREATE INDEX ON join_t USING value') + s_ds.Release() + + for filename in ['join_t.idm', 'join_t.ind']: + try: + os.stat(filename) + except (OSError, FileNotFoundError): + pytest.fail("%s should exist" % filename) + +############################################################################### +# Test that dropping, then re-creating the index causes the correct index data +# to be placed in the re-created index file + + +def test_ogr_index_recreating_index_causes_index_to_be_populated(): + + with create_join_t_test_file(create_index=True): + s_ds = ogr.OpenShared('join_t.dbf', update=1) + + s_ds.ExecuteSQL('DROP INDEX ON join_t USING value') + s_ds.ExecuteSQL('DROP INDEX ON join_t USING skey') + + s_ds.Release() + + # Re-create an index + s_ds = ogr.OpenShared('join_t.dbf', update=1) + s_ds.ExecuteSQL('CREATE INDEX ON join_t USING value') + s_ds.Release() + + with open('join_t.idm', 'rt') as f: + xml = f.read() + assert xml.find('VALUE') != -1, 'VALUE column is not indexed (1)' + +############################################################################### +# Text that adding an index to different columns in separate calls causes both +# columns to be indexed. + + +def test_ogr_index_creating_index_in_separate_steps_works(): + + with create_join_t_test_file(create_index=True): + s_ds = ogr.OpenShared('join_t.dbf', update=1) + + s_ds.ExecuteSQL('DROP INDEX ON join_t USING value') + s_ds.ExecuteSQL('DROP INDEX ON join_t USING skey') + + s_ds.Release() + + # Re-create an index + s_ds = ogr.OpenShared('join_t.dbf', update=1) + s_ds.ExecuteSQL('CREATE INDEX ON join_t USING value') + s_ds.Release() + + # Close the dataset and re-open + s_ds = ogr.OpenShared('join_t.dbf', update=1) + # At this point the .ind was opened in read-only. Now it + # will be re-opened in read-write mode + s_ds.ExecuteSQL('CREATE INDEX ON join_t USING skey') + + s_ds.Release() + + with open('join_t.idm', 'rt') as f: + xml = f.read() + assert xml.find('VALUE') != -1, 'VALUE column is not indexed (2)' + assert xml.find('SKEY') != -1, 'SKEY column is not indexed (2)' ############################################################################### # Test fix for #4326 @@ -283,7 +321,9 @@ def test_ogr_index_9(): def test_ogr_index_10(): - ds = ogr.GetDriverByName('ESRI Shapefile').CreateDataSource('tmp/ogr_index_10.shp') + ds = ogr.GetDriverByName('ESRI Shapefile').CreateDataSource( + 'tmp/ogr_index_10.shp' + ) lyr = ds.CreateLayer('ogr_index_10') lyr.CreateField(ogr.FieldDefn('intfield', ogr.OFTInteger)) lyr.CreateField(ogr.FieldDefn('realfield', ogr.OFTReal)) @@ -396,11 +436,12 @@ def ogr_index_11_check(lyr, expected_fids): assert feat is not None assert feat.GetFID() == expected_fid - def test_ogr_index_11(): - ds = ogr.GetDriverByName('ESRI Shapefile').CreateDataSource('tmp/ogr_index_11.dbf') + ds = ogr.GetDriverByName('ESRI Shapefile').CreateDataSource( + 'tmp/ogr_index_11.dbf' + ) lyr = ds.CreateLayer('ogr_index_11', geom_type=ogr.wkbNone) lyr.CreateField(ogr.FieldDefn('intfield', ogr.OFTInteger)) lyr.CreateField(ogr.FieldDefn('strfield', ogr.OFTString))