From 398b223bbeb6a06af145a2d2f047c7ff9ea0b317 Mon Sep 17 00:00:00 2001 From: Ez-and Date: Mon, 30 Dec 2024 05:00:18 +0300 Subject: [PATCH 1/3] Add delimiter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавлен параметр разделитель для включения в файл конфигурации --- parser_2gis/writer/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parser_2gis/writer/options.py b/parser_2gis/writer/options.py index bcdcd96..d5b1c8d 100644 --- a/parser_2gis/writer/options.py +++ b/parser_2gis/writer/options.py @@ -16,6 +16,7 @@ class CSVOptions(BaseModel): remove_empty_columns: Remove empty columns after parsing process finished. remove_duplicates: Remove duplicates after parsing process finished. join_char: Char for joining complex values. + delimiter: CSV char-delimiter """ add_rubrics: bool = True add_comments: bool = True @@ -23,6 +24,7 @@ class CSVOptions(BaseModel): remove_empty_columns: bool = True remove_duplicates: bool = True join_char: str = '; ' + delimiter: str = ',' class WriterOptions(BaseModel): From 209a6323cca5e4ca84b9b0350775196ad53c981d Mon Sep 17 00:00:00 2001 From: Ez-and Date: Mon, 30 Dec 2024 14:57:10 +0300 Subject: [PATCH 2/3] Add CustomDialect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавлено создание собственного диалекта для записи CSV с подтягиванием значения разделителя из конфига --- parser_2gis/writer/writers/csv_writer.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/parser_2gis/writer/writers/csv_writer.py b/parser_2gis/writer/writers/csv_writer.py index 5b90161..977d71f 100644 --- a/parser_2gis/writer/writers/csv_writer.py +++ b/parser_2gis/writer/writers/csv_writer.py @@ -16,6 +16,15 @@ class CSVWriter(FileWriter): """Writer to CSV table.""" + @property + def _dialect(self) -> csv.Dialect: + class CustomDialect(csv.Dialect): + delimiter = self._options.csv.delimiter + quoting = csv.QUOTE_MINIMAL + lineterminator = "\n" + quotechar= '"' + return CustomDialect + @property def _type_names(self) -> dict[str, str]: return { @@ -76,7 +85,7 @@ def _writerow(self, row: dict[str, Any]) -> None: def __enter__(self) -> CSVWriter: super().__enter__() - self._writer = csv.DictWriter(self._file, self._data_mapping.keys()) + self._writer = csv.DictWriter(self._file, self._data_mapping.keys(), dialect=self._dialect) self._writer.writerow(self._data_mapping) # Write header self._wrote_count = 0 return self @@ -98,9 +107,9 @@ def _remove_empty_columns(self) -> None: # Looking for empty columns with self._open_file(self._file_path, 'r') as f_csv: - csv_reader = csv.DictReader(f_csv, self._data_mapping.keys()) # type: ignore + csv_reader = csv.DictReader(f_csv, self._data_mapping.keys(), dialect=self._dialect) # type: ignore next(csv_reader, None) # Skip header - for row in csv.DictReader(f_csv, self._data_mapping.keys()): # type: ignore + for row in csv.DictReader(f_csv, self._data_mapping.keys(), dialect=self._dialect): # type: ignore for column_name in complex_columns_count.keys(): if row[column_name] != '': complex_columns_count[column_name] += 1 @@ -124,8 +133,8 @@ def _remove_empty_columns(self) -> None: with self._open_file(tmp_csv_name, 'w') as f_tmp_csv, \ self._open_file(self._file_path, 'r') as f_csv: - csv_writer = csv.DictWriter(f_tmp_csv, new_data_mapping.keys()) # type: ignore - csv_reader = csv.DictReader(f_csv, self._data_mapping.keys()) # type: ignore + csv_writer = csv.DictWriter(f_tmp_csv, new_data_mapping.keys(), dialect=self._dialect) # type: ignore + csv_reader = csv.DictReader(f_csv, self._data_mapping.keys(), dialect=self._dialect) # type: ignore csv_writer.writerow(new_data_mapping) # Write new header next(csv_reader, None) # Skip header From 2902bad34ed62a245c8af998bfc021fd9a7733a7 Mon Sep 17 00:00:00 2001 From: Ez-and Date: Mon, 30 Dec 2024 15:14:30 +0300 Subject: [PATCH 3/3] Add delimiter CSV GUI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавлена возможность менять разделитель через GUI --- parser_2gis/gui/settings.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/parser_2gis/gui/settings.py b/parser_2gis/gui/settings.py index c268212..9a86926 100644 --- a/parser_2gis/gui/settings.py +++ b/parser_2gis/gui/settings.py @@ -146,6 +146,20 @@ def gui_settings(config: Configuration) -> None: default=config.writer.csv.remove_duplicates, checkbox_color=sg.theme_input_background_color(), enable_events=True), ], + [ + sg.Column([ + [ + sg.Text('Разделитель CSV'), + ] + ], expand_x=True, pad=0), + sg.Column([ + [ + sg.Input(size=(3, 1), justification='center', key='-WRITER.CSV.DELIMITER-', + tooltip='Символ-разделитель колонок в CSV.', + default_text = config.writer.csv.delimiter), + ], + ], element_justification='right', pad=0), + ], [ sg.Column([ [