Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Import - TOPO: formatage des données MAJIC pour commune et voie #475

Merged
merged 2 commits into from
Feb 4, 2025
Merged

Import - TOPO: formatage des données MAJIC pour commune et voie #475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a7b512
Import - TOPO: formatage des données MAJIC pour commune et voie
mdouchin Jan 28, 2025
32ae806
Tests - Adaptation pour 2024 et fichiers TOPO
mdouchin Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions cadastre/cadastre_common_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,8 +262,9 @@ def postgisToSpatialite(sql: str, targetSrid: str = '2154') -> str:
{'in': r'alter table [^;]+drop constraint[^;]+;', 'out': ''},
# ~ {'in': r'^analyse [^;]+;', 'out': ''},
# replace
{'in': r'truncate (bati|fanr|lloc|nbat|pdll|prop)',
'out': r'drop table \1;create table \1 (tmp text)'},
{'in': r'truncate (bati|lloc|nbat|pdll|prop)',
'out': r'drop table if exists \1;create table \1 (tmp text)'},
{'in': r'truncate topo', 'out': r'delete from topo'},
{'in': r'truncate ', 'out': 'delete from '},
{'in': r'distinct on *\([a-z, ]+\)', 'out': 'distinct'},
{'in': r'serial', 'out': 'INTEGER PRIMARY KEY AUTOINCREMENT'},
Expand All @@ -276,6 +277,10 @@ def postgisToSpatialite(sql: str, targetSrid: str = '2154') -> str:
'out': r"date(substr(\2, 5, 4) || '-' || substr(\2, 3, 2) || '-' || substr(\2, 1, 2))"},
{'in': r"(to_date\()([^']+) *, *'DD/MM/YYYY' *\)",
'out': r"date(substr(\2, 7, 4) || '-' || substr(\2, 4, 2) || '-' || substr(\2, 1, 2))"},

{'in': r"(to_char\()(.+), 'DDD'\)",
'out': r"strftime('%j', \2)"},

{'in': r"(to_date\()([^']+) *, *'YYYYMMDD' *\)",
'out': r"date(substr(\2, 1, 4) || '-' || substr(\2, 5, 2) || '-' || substr(\2, 7, 2))"},
{'in': r"(to_char\()([^']+) *, *'dd/mm/YYYY' *\)",
Expand Down
236 changes: 139 additions & 97 deletions cadastre/cadastre_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,12 @@
from cadastre.definitions import (
IMPORT_MEMORY_ERROR_MESSAGE,
REGEX_BATI,
REGEX_FANTOIR,
REGEX_LOTLOCAL,
REGEX_NBATI,
REGEX_PDL,
REGEX_PROP,
URL_FANTOIR,
REGEX_TOPO,
URL_TOPO,
)
from cadastre.dialogs.dialog_common import CadastreCommon

Expand Down Expand Up @@ -118,9 +118,9 @@ def __init__(self, dialog):
'required': True
},
{
'key': '[FICHIER_FANTOIR]',
'regex': s.value("cadastre/regexFantoir", REGEX_FANTOIR, type=str),
'table': 'fanr',
'key': '[FICHIER_TOPO]',
'regex': s.value("cadastre/regexTopo", REGEX_TOPO, type=str),
'table': 'topo',
'required': True
},
{
Expand Down Expand Up @@ -165,7 +165,7 @@ def __init__(self, dialog):
if self.dialog.hasStructure:
self.hasConstraints = True

# Remove MAJIC from tables bati|fanr|lloc|nbat|pdll|prop
# Remove MAJIC from tables bati|topo|lloc|nbat|pdll|prop
self.removeMajicRawData = True

self.beginImport()
Expand Down Expand Up @@ -305,7 +305,7 @@ def importMajic(self):

# dict for parameters replacement
replaceDict = self.replaceDict.copy()
# mandatoryFilesKeys = ['[FICHIER_BATI]', '[FICHIER_FANTOIR]', '[FICHIER_NBATI]', '[FICHIER_PROP]']
# mandatoryFilesKeys = ['[FICHIER_BATI]', '[FICHIER_TOPO]', '[FICHIER_NBATI]', '[FICHIER_PROP]']
# missingMajicFiles = False

scriptList = []
Expand Down Expand Up @@ -350,6 +350,7 @@ def importMajic(self):
scriptList.append(importScript)

# Format data
replaceDict['DEPDIR'] = f'{self.dialog.edigeoDepartement}{self.dialog.edigeoDirection}'
scriptList.append(
{
'title': 'Mise en forme des données',
Expand Down Expand Up @@ -460,9 +461,8 @@ def get_available_majic_files(self) -> tuple[dict, dict]:
file_path = os.path.join(root, file_sub_path)
maj_list.append(file_path)

# Store dep_dir for this file
# avoid fantoir, as now it is given for the whole country
if table == 'fanr':
# avoid topo, since direction is not used in TOPO
if table == 'topo':
continue

# Get dep_dir : first line with content
Expand Down Expand Up @@ -490,17 +490,21 @@ def check_missing_majic_files(self, majic_files_found: dict) -> bool:
"<b>Des fichiers MAJIC importants sont manquants</b> :<br/>"
" <b>{}</b> <br/><br/>"
"Vérifier le chemin des fichiers MAJIC :<br/>"
"<b>{}</b> <br/>"
"<b>{}</b> <br/><br/>"
"ainsi que les mots recherchés pour chaque type de fichier configurés dans les options du plugin Cadastre :<br/>"
"<b>{}</b><br/><br/>"
"<b>NB:</b> Vous pouvez télécharger les fichiers FANTOIR à cette adresse :<br/>"
"<b>{}</b><br/><br/><br/>"
"<b>NB:</b> Vous pouvez télécharger les fichiers TOPO à cette adresse :<br/>"
"<a href='{}'>{}</a><br/>"
).format(
', '.join(missing_files),
', <br/>'.join(missing_files),
self.dialog.majicSourceDir,
', <br/>'.join([f"* {a['key'].strip('[]')}: {a['regex'].upper()}" for a in self.majicSourceFileNames]),
URL_FANTOIR,
URL_FANTOIR,
', <br/>'.join([
f"* {a['key'].strip('[]')}: {a['regex'].upper()}"
for a in self.majicSourceFileNames
if a['table'] in missing_files
]),
URL_TOPO,
URL_TOPO,
)
missing_majic_ignore = QMessageBox.question(
self.dialog,
Expand Down Expand Up @@ -691,9 +695,12 @@ def import_majic_into_database(self) -> bool:
self.totalSteps += len(majic_files_found[table])
processed_files_count += len(majic_files_found[table])
for file_path in majic_files_found[table]:
self.qc.updateLog(f'<b>{table}</b>')
self.qc.updateLog(file_path)

import_file = self.import_majic_file_into_database(table, file_path, dep_dir)
if table == 'topo':
import_file = self.import_file_with_ogr(file_path, 'topo')
else:
import_file = self.import_majic_file_into_database(table, file_path, dep_dir)
if not import_file:
continue

Expand Down Expand Up @@ -1187,6 +1194,12 @@ def replaceParametersInScript(self, scriptPath, replaceDict):
self.qc.updateLog(msg)
return msg

except KeyError as e:
msg = "<b>Erreur lors du paramétrage des scripts d'import: %s</b>" % e
self.go = False
self.qc.updateLog(msg)
return msg

finally:
QApplication.restoreOverrideCursor()

Expand Down Expand Up @@ -1408,7 +1421,7 @@ def importAllEdigeoToDatabase(self):
self.step = 0
self.totalSteps = len(thfList)
for thf in thfList:
self.importEdigeoThfToDatabase(thf)
self.import_file_with_ogr(thf, 'thf')
self.updateProgressBar()
if not self.go:
break
Expand Down Expand Up @@ -1442,110 +1455,139 @@ def importAllEdigeoToDatabase(self):
self.totalSteps = initialTotalSteps
QApplication.restoreOverrideCursor()

def importEdigeoThfToDatabase(self, filename):
def import_file_with_ogr(self, file_path: str, file_type: str):
"""
Import one edigeo THF files into database
Import file into the database.

It can either be an EDIGEO THF file or a TOPO file.
source : db_manager/dlg_import_vector.py
"""
if self.go:
# Get options
if not self.go:
return None

# SRID configurations
if file_type == 'thf':
targetSridOption = '-t_srs'
if self.sourceSridFull == self.targetSridFull:
targetSridOption = '-a_srs'

# Build ogr2ogr command
conn_name = self.dialog.connectionName
settings = QSettings()
settings.beginGroup(f"/{self.db.dbplugin().connectionSettingsKey()}/{conn_name}")
# Build ogr2ogr command
conn_name = self.dialog.connectionName
settings = QSettings()
settings.beginGroup(f"/{self.db.dbplugin().connectionSettingsKey()}/{conn_name}")

# normalising file path
filename = os.path.normpath(filename)
if self.dialog.dbType == 'postgis':
if not settings.contains("database"): # non-existent entry?
raise Exception(self.tr('There is no defined database connection "%s".') % conn_name)
settingsList = ["service", "host", "port", "database", "username", "password"]
service, host, port, database, username, password = (settings.value(x) for x in settingsList)

if service:
pg_access = 'PG:service={} active_schema={}'.format(
service,
self.dialog.schema
)
else:
# qgis can connect to postgis DB without a specified host param connection, but ogr2ogr cannot
if not host:
host = "localhost"

pg_access = 'PG:host={} port={} dbname={} active_schema={} user={} password={}'.format(
host,
port,
database,
self.dialog.schema,
username,
password
)
cmdArgs = [
'',
# normalising file path
file_path = os.path.normpath(file_path)
if self.dialog.dbType == 'postgis':
if not settings.contains("database"): # non-existent entry?
raise Exception(self.tr('There is no defined database connection "%s".') % conn_name)
settingsList = ["service", "host", "port", "database", "username", "password"]
service, host, port, database, username, password = (settings.value(x) for x in settingsList)

if service:
pg_access = 'PG:service={} active_schema={}'.format(
service,
self.dialog.schema
)
else:
# qgis can connect to postgis DB without a specified host param connection, but ogr2ogr cannot
if not host:
host = "localhost"

pg_access = 'PG:host={} port={} dbname={} active_schema={} user={} password={}'.format(
host,
port,
database,
self.dialog.schema,
username,
password
)
cmdArgs = [
'',
]
if file_type == 'thf':
cmdArgs += [
'-s_srs', self.sourceSridFull,
targetSridOption, self.targetSridFull,
'-append',
'-f', 'PostgreSQL',
pg_access,
filename,
]
cmdArgs += [
'-append',
'-f', 'PostgreSQL',
pg_access,
file_path,
'-lco', 'PG_USE_COPY=YES',
'-gt', '50000',
'--config', 'PG_USE_COPY', 'YES',
]
if file_type == 'thf':
cmdArgs += [
'-lco', 'GEOMETRY_NAME=geom',
'-lco', 'PG_USE_COPY=YES',
'-nlt', 'GEOMETRY',
'-gt', '50000',
'--config', 'OGR_EDIGEO_CREATE_LABEL_LAYERS', 'NO',
'--config', 'PG_USE_COPY', 'YES',
]
# -c client_encoding=latin1
if file_type == 'topo':
cmdArgs += [
'-nln', 'topo',
]
# -c client_encoding=latin1

if self.dialog.dbType == 'spatialite':
if not settings.contains("sqlitepath"): # non-existent entry?
self.go = False
raise Exception('there is no defined database connection "%s".' % conn_name)
if self.dialog.dbType == 'spatialite':
if not settings.contains("sqlitepath"): # non-existent entry?
self.go = False
raise Exception('there is no defined database connection "%s".' % conn_name)

database = settings.value("sqlitepath")
database = settings.value("sqlitepath")

cmdArgs = [
'',
cmdArgs = [
'',
]
if file_type == 'thf':
cmdArgs += [
'-s_srs', self.sourceSridFull,
targetSridOption, self.targetSridFull,
'-append',
'-f', 'SQLite',
database,
filename,
]
cmdArgs += [
'-append',
'-f', 'SQLite',
database,
file_path,
'-gt', '50000',
'--config', 'OGR_SQLITE_SYNCHRONOUS', 'OFF',
'--config', 'OGR_SQLITE_CACHE', '512'
]
if file_type == 'thf':
cmdArgs += [
'-lco', 'GEOMETRY_NAME=geom',
'-nlt', 'GEOMETRY',
'-dsco', 'SPATIALITE=YES',
'-gt', '50000',
'--config', 'OGR_EDIGEO_CREATE_LABEL_LAYERS', 'NO',
'--config', 'OGR_SQLITE_SYNCHRONOUS', 'OFF',
'--config', 'OGR_SQLITE_CACHE', '512'
]
if file_type == 'topo':
cmdArgs += [
'-nln', 'topo',
]

# self.qc.updateLog( ' '.join(cmdArgs))
# Run only if ogr2ogr found
if self.go:
# Workaround to get ogr2ogr error messages via stdout
# as ogr2ogr.py does not return exceptions nor error messages
# but only prints the error before returning False
stdout = sys.stdout
try:
sys.stdout = file = io.StringIO()
self.go = ogr2ogr(cmdArgs)
printedString = file.getvalue()
finally:
sys.stdout = stdout
# self.qc.updateLog( ' '.join(cmdArgs))
# Run only if ogr2ogr found
if self.go:
# Workaround to get ogr2ogr error messages via stdout
# as ogr2ogr.py does not return exceptions nor error messages
# but only prints the error before returning False
stdout = sys.stdout
try:
sys.stdout = file = io.StringIO()
self.go = ogr2ogr(cmdArgs)
printedString = file.getvalue()
finally:
sys.stdout = stdout

if not self.go:
self.qc.updateLog(
"<b>Erreur - L'import des données via OGR2OGR a échoué:</b>\n\n{}\n\n{}".format(
printedString,
cmdArgs
)
if not self.go:
self.qc.updateLog(
"<b>Erreur - L'import des données via OGR2OGR a échoué:</b>\n\n{}\n\n{}".format(
printedString,
cmdArgs
)
)

return None

Expand Down
6 changes: 3 additions & 3 deletions cadastre/definitions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
URL_FANTOIR = (
"https://drive.opendata.craig.fr/s/opendata?path=%2Fadresse%2Ffantoir"
URL_TOPO = (
"https://drive.opendata.craig.fr/s/opendata?path=%2Fadresse%2Ftopo"
)

URL_DOCUMENTATION = "https://docs.3liz.org/QgisCadastrePlugin/"

REGEX_BATI = "BATI"
REGEX_FANTOIR = "FANTOIR|FANR"
REGEX_LOTLOCAL = "LLOC|D166"
REGEX_NBATI = "NBAT"
REGEX_PDL = "PDL"
REGEX_PROP = "PROP"
REGEX_TOPO = "TOPO"

IMPORT_MEMORY_ERROR_MESSAGE = "<b>ERREUR : Mémoire</b></br>"
"Veuillez recommencer l'import en baissant la valeur du "
Expand Down
Loading