Fix: Resolve Typo Issue In EPA Historical Air Quality Pipeline.yaml (#…

…519)

* fix: Resolving issues as yet unresolved.

* fix: black issue

* fix: resolved datatype date not known

* fix: submit new changes following issues in production after deployment.

* fix: miscellaneous fixes.

* fix: Annual summaries now works.

* fix: co_hourly_summary now successful.

* fix: black hook issues

* fix: data type corrections for some fields.

* fix: Resolved file naming breaking issue in ozone_hourly pipeline. Not production ready.

* fix: Now fixed annual summaries.  Ready for AF test in dev.

* fix: Tested full load all pipelines.

* fix: Replicate full load fixes into incremental load dag.

* fix: Resolved black hooks

* fix: Change to schedule

* feat: Added PM25_FRM_DAILY_SUMMARY pipeline.

* fix: Resolve JSON issue in pipeline.yaml.

* fix: Resolve typo issue in pipeline.yaml.

datasets/epa_historical_air_quality/pipelines/epa_historical_air_quality/epa_historical_air_quality_dag.py

@@ -571,7 +571,7 @@
             "PIPELINE_NAME": "epa_historical_air_quality - pm25_frm_daily_summaries",
             "INPUT_CSV_HEADERS": '[\n  "State Code", "County Code", "Site Num", "Parameter Code", "POC",\n  "Latitude", "Longitude", "Datum", "Parameter Name", "Sample Duration",\n  "Pollutant Standard", "Date Local", "Units of Measure", "Event Type", "Observation Count",\n  "Observation Percent", "Arithmetic Mean", "1st Max Value", "1st Max Hour", "AQI",\n  "Method Code", "Method Name", "Local Site Name", "Address", "State Name",\n  "County Name", "City Name", "CBSA Name", "Date of Last Change"\n]',
             "DATA_DTYPES": '{\n  "State Code": "str", "County Code": "str", "Site Num": "str", "Parameter Code": "int32", "POC": "int32",\n  "Latitude": "float64", "Longitude": "float64", "Datum": "str", "Parameter Name": "str", "Sample Duration": "str",\n  "Pollutant Standard": "str", "Date Local": "str", "Units of Measure": "str", "Event Type": "str", "Observation Count": "int32",\n  "Observation Percent": "float64", "Arithmetic Mean": "float64", "1st Max Value": "float64", "1st Max Hour": "int32", "AQI": "str",\n  "Method Code": "str", "Method Name": "str", "Local Site Name": "str", "Address": "str", "State Name": "str",\n  "County Name": "str", "City Name": "str", "CBSA Name": "str", "Date of Last Change": "str"\n}',
-            "RENAME_HEADERS": '{ "State Code": "state_code",\n  "County Code": "county_code",\n  "Site Num": "site_num",\n  "Parameter Code": "parameter_code",\n  "POC": "poc",\n  "Latitude": "latitude",\n  "Longitude": "longitude",\n  "Datum": "datum",\n  "Parameter Name": "parameter_name",\n  "Sample Duration": "sample_duration",\n  "Pollutant Standard": "pollutant_standard",\n  "Date Local": "date_local",\n  "Units of Measure": "units_of_measure",\n  "Event Type": "event_type",\n  "Observation Count": "observation_count",\n  "Observation Percent": "observation_percent",\n  "Arithmetic Mean": "arithmetic_mean",\n  "1st Max Value": "first_max_value",\n  "1st Max Hour": "first_max_hour",\n  "AQI": "aqi",\n  "Method Code": "method_code",\n  "Method Name": "method_name",\n  "Local Site Name": "local_site_name",\n  "Address": "address",\n  "State Name": "state_name",\n  "County Name": "county_name",\n  "City Name": "city_name",\n  "CBSA Name": "cbsa_name",\n  "Date of Last Change": "date_of_last_change"\n}',
+            "RENAME_HEADERS_LIST": '{ "State Code": "state_code",\n  "County Code": "county_code",\n  "Site Num": "site_num",\n  "Parameter Code": "parameter_code",\n  "POC": "poc",\n  "Latitude": "latitude",\n  "Longitude": "longitude",\n  "Datum": "datum",\n  "Parameter Name": "parameter_name",\n  "Sample Duration": "sample_duration",\n  "Pollutant Standard": "pollutant_standard",\n  "Date Local": "date_local",\n  "Units of Measure": "units_of_measure",\n  "Event Type": "event_type",\n  "Observation Count": "observation_count",\n  "Observation Percent": "observation_percent",\n  "Arithmetic Mean": "arithmetic_mean",\n  "1st Max Value": "first_max_value",\n  "1st Max Hour": "first_max_hour",\n  "AQI": "aqi",\n  "Method Code": "method_code",\n  "Method Name": "method_name",\n  "Local Site Name": "local_site_name",\n  "Address": "address",\n  "State Name": "state_name",\n  "County Name": "county_name",\n  "City Name": "city_name",\n  "CBSA Name": "cbsa_name",\n  "Date of Last Change": "date_of_last_change"\n}',
             "OUTPUT_CSV_HEADERS": '[\n  "state_code",\n  "county_code",\n  "site_num",\n  "parameter_code",\n  "poc",\n  "latitude",\n  "longitude",\n  "datum",\n  "parameter_name",\n  "sample_duration",\n  "pollutant_standard",\n  "date_local",\n  "units_of_measure",\n  "event_type",\n  "observation_count",\n  "observation_percent",\n  "arithmetic_mean",\n  "first_max_value",\n  "first_max_hour",\n  "aqi",\n  "method_code",\n  "method_name",\n  "local_site_name",\n  "address",\n  "state_name",\n  "county_name",\n  "city_name",\n  "cbsa_name",\n  "date_of_last_change"\n]',
             "DROP_DEST_TABLE": "N",
         },

datasets/epa_historical_air_quality/pipelines/epa_historical_air_quality/pipeline.yaml

@@ -955,7 +955,7 @@ dag:
               "Method Code": "str", "Method Name": "str", "Local Site Name": "str", "Address": "str", "State Name": "str",
               "County Name": "str", "City Name": "str", "CBSA Name": "str", "Date of Last Change": "str"
             }
-          RENAME_HEADERS: >-
+          RENAME_HEADERS_LIST: >-
             { "State Code": "state_code",
               "County Code": "county_code",
               "Site Num": "site_num",

datasets/epa_historical_air_quality/pipelines/epa_historical_air_quality_full_load/epa_historical_air_quality_full_load_dag.py

@@ -571,7 +571,7 @@
             "PIPELINE_NAME": "epa_historical_air_quality - pm25_frm_daily_summaries",
             "INPUT_CSV_HEADERS": '[\n  "State Code", "County Code", "Site Num", "Parameter Code", "POC",\n  "Latitude", "Longitude", "Datum", "Parameter Name", "Sample Duration",\n  "Pollutant Standard", "Date Local", "Units of Measure", "Event Type", "Observation Count",\n  "Observation Percent", "Arithmetic Mean", "1st Max Value", "1st Max Hour", "AQI",\n  "Method Code", "Method Name", "Local Site Name", "Address", "State Name",\n  "County Name", "City Name", "CBSA Name", "Date of Last Change"\n]',
             "DATA_DTYPES": '{\n  "State Code": "str", "County Code": "str", "Site Num": "str", "Parameter Code": "int32", "POC": "int32",\n  "Latitude": "float64", "Longitude": "float64", "Datum": "str", "Parameter Name": "str", "Sample Duration": "str",\n  "Pollutant Standard": "str", "Date Local": "str", "Units of Measure": "str", "Event Type": "str", "Observation Count": "int32",\n  "Observation Percent": "float64", "Arithmetic Mean": "float64", "1st Max Value": "float64", "1st Max Hour": "int32", "AQI": "str",\n  "Method Code": "str", "Method Name": "str", "Local Site Name": "str", "Address": "str", "State Name": "str",\n  "County Name": "str", "City Name": "str", "CBSA Name": "str", "Date of Last Change": "str"\n}',
-            "RENAME_HEADERS": '{ "State Code": "state_code",\n  "County Code": "county_code",\n  "Site Num": "site_num",\n  "Parameter Code": "parameter_code",\n  "POC": "poc",\n  "Latitude": "latitude",\n  "Longitude": "longitude",\n  "Datum": "datum",\n  "Parameter Name": "parameter_name",\n  "Sample Duration": "sample_duration",\n  "Pollutant Standard": "pollutant_standard",\n  "Date Local": "date_local",\n  "Units of Measure": "units_of_measure",\n  "Event Type": "event_type",\n  "Observation Count": "observation_count",\n  "Observation Percent": "observation_percent",\n  "Arithmetic Mean": "arithmetic_mean",\n  "1st Max Value": "first_max_value",\n  "1st Max Hour": "first_max_hour",\n  "AQI": "aqi",\n  "Method Code": "method_code",\n  "Method Name": "method_name",\n  "Local Site Name": "local_site_name",\n  "Address": "address",\n  "State Name": "state_name",\n  "County Name": "county_name",\n  "City Name": "city_name",\n  "CBSA Name": "cbsa_name",\n  "Date of Last Change": "date_of_last_change"\n}',
+            "RENAME_HEADERS_LIST": '{ "State Code": "state_code",\n  "County Code": "county_code",\n  "Site Num": "site_num",\n  "Parameter Code": "parameter_code",\n  "POC": "poc",\n  "Latitude": "latitude",\n  "Longitude": "longitude",\n  "Datum": "datum",\n  "Parameter Name": "parameter_name",\n  "Sample Duration": "sample_duration",\n  "Pollutant Standard": "pollutant_standard",\n  "Date Local": "date_local",\n  "Units of Measure": "units_of_measure",\n  "Event Type": "event_type",\n  "Observation Count": "observation_count",\n  "Observation Percent": "observation_percent",\n  "Arithmetic Mean": "arithmetic_mean",\n  "1st Max Value": "first_max_value",\n  "1st Max Hour": "first_max_hour",\n  "AQI": "aqi",\n  "Method Code": "method_code",\n  "Method Name": "method_name",\n  "Local Site Name": "local_site_name",\n  "Address": "address",\n  "State Name": "state_name",\n  "County Name": "county_name",\n  "City Name": "city_name",\n  "CBSA Name": "cbsa_name",\n  "Date of Last Change": "date_of_last_change"\n}',
             "OUTPUT_CSV_HEADERS": '[\n  "state_code",\n  "county_code",\n  "site_num",\n  "parameter_code",\n  "poc",\n  "latitude",\n  "longitude",\n  "datum",\n  "parameter_name",\n  "sample_duration",\n  "pollutant_standard",\n  "date_local",\n  "units_of_measure",\n  "event_type",\n  "observation_count",\n  "observation_percent",\n  "arithmetic_mean",\n  "first_max_value",\n  "first_max_hour",\n  "aqi",\n  "method_code",\n  "method_name",\n  "local_site_name",\n  "address",\n  "state_name",\n  "county_name",\n  "city_name",\n  "cbsa_name",\n  "date_of_last_change"\n]',
             "DROP_DEST_TABLE": "N",
         },

datasets/epa_historical_air_quality/pipelines/epa_historical_air_quality_full_load/pipeline.yaml

@@ -958,7 +958,7 @@ dag:
               "Method Code": "str", "Method Name": "str", "Local Site Name": "str", "Address": "str", "State Name": "str",
               "County Name": "str", "City Name": "str", "CBSA Name": "str", "Date of Last Change": "str"
             }
-          RENAME_HEADERS: >-
+          RENAME_HEADERS_LIST: >-
             { "State Code": "state_code",
               "County Code": "county_code",
               "Site Num": "site_num",