diff --git a/datasets/noaa/pipelines/_images/ghcnd_stations_schema.json b/datasets/noaa/pipelines/_images/ghcnd_stations_schema.json index f8068f0f4..a72e10898 100644 --- a/datasets/noaa/pipelines/_images/ghcnd_stations_schema.json +++ b/datasets/noaa/pipelines/_images/ghcnd_stations_schema.json @@ -35,7 +35,7 @@ "mode": "NULLABLE" }, { - "name": "hcn_cm_flag", + "name": "hcn_crn_flag", "type": "STRING", "mode": "NULLABLE" }, diff --git a/datasets/noaa/pipelines/noaa/noaa_dag.py b/datasets/noaa/pipelines/noaa/noaa_dag.py index 3ee92891f..9b22fa0d5 100644 --- a/datasets/noaa/pipelines/noaa/noaa_dag.py +++ b/datasets/noaa/pipelines/noaa/noaa_dag.py @@ -27,7 +27,7 @@ dag_id="noaa.noaa", default_args=default_args, max_active_runs=1, - schedule_interval="0 6 * * 1", + schedule_interval="* 1 * * 6", catchup=False, default_view="graph", ) as dag: @@ -227,8 +227,8 @@ "DELETE_TARGET_FILE": "Y", "INPUT_CSV_HEADERS": '[\n "textdata"\n]', "DATA_DTYPES": '{\n "textdata": "str"\n}', - "REORDER_HEADERS_LIST": '[\n "id",\n "latitude",\n "longitude",\n "elevation",\n "state",\n "name",\n "gsn_flag",\n "hcn_cm_flag",\n "wmoid",\n "source_url",\n "etl_timestamp"\n]', - "SLICE_COLUMN_LIST": '{\n "id": ["textdata", "0", "11"],\n "latitude": ["textdata", "12", "20"],\n "longitude": ["textdata", "21", "30"],\n "elevation": ["textdata", "31", "37"],\n "state": ["textdata", "38", "40"],\n "name": ["textdata", "41", "71"],\n "gsn_flag": ["textdata", "72", "75"],\n "hcn_cm_flag": ["textdata", "76", "79"],\n "wmoid": ["textdata", "80", "85"]\n}', + "REORDER_HEADERS_LIST": '[\n "id",\n "latitude",\n "longitude",\n "elevation",\n "state",\n "name",\n "gsn_flag",\n "hcn_crn_flag",\n "wmoid",\n "source_url",\n "etl_timestamp"\n]', + "SLICE_COLUMN_LIST": '{\n "id": ["textdata", "0", "11"],\n "latitude": ["textdata", "12", "20"],\n "longitude": ["textdata", "21", "30"],\n "elevation": ["textdata", "31", "37"],\n "state": ["textdata", "38", "40"],\n "name": ["textdata", "41", "71"],\n "gsn_flag": ["textdata", "72", "75"],\n "hcn_crn_flag": ["textdata", "76", "79"],\n "wmoid": ["textdata", "80", "85"]\n}', }, resources={"request_ephemeral_storage": "4G", "limit_cpu": "3"}, ) diff --git a/datasets/noaa/pipelines/noaa/pipeline.yaml b/datasets/noaa/pipelines/noaa/pipeline.yaml index 66e06661c..54da56632 100644 --- a/datasets/noaa/pipelines/noaa/pipeline.yaml +++ b/datasets/noaa/pipelines/noaa/pipeline.yaml @@ -48,7 +48,7 @@ dag: depends_on_past: False start_date: '2021-03-01' max_active_runs: 1 - schedule_interval: "0 6 * * 1" # 06:00 on Monday + schedule_interval: "* 1 * * 6" catchup: False default_view: graph @@ -355,7 +355,7 @@ dag: "state", "name", "gsn_flag", - "hcn_cm_flag", + "hcn_crn_flag", "wmoid", "source_url", "etl_timestamp" @@ -369,7 +369,7 @@ dag: "state": ["textdata", "38", "40"], "name": ["textdata", "41", "71"], "gsn_flag": ["textdata", "72", "75"], - "hcn_cm_flag": ["textdata", "76", "79"], + "hcn_crn_flag": ["textdata", "76", "79"], "wmoid": ["textdata", "80", "85"] } resources: