✨ Add HML Download Portal

https://mesonet.agron.iastate.edu/request/hml.php
refs #277

cgi-bin/request/hml.py

@@ -0,0 +1,3 @@
+"""implemented in /pylib/iemweb/request/hml.py"""
+
+from iemweb.request.hml import application  # noqa: F401

htdocs/api/index.php

@@ -208,6 +208,7 @@
 <li><a href="/cgi-bin/request/gis/cwas.py?help">Center Weather Advisories (/cgi-bin/request/gis/cwas.py)</a></li>
 <li><a href="/cgi-bin/request/hads.py?help">HADS/DCP/SHEF Data (/cgi-bin/request/hads.py)</a></li>
 <li><a href="/cgi-bin/request/grx_rings.py?help">Gibson Ridge Range Ring Placefile (/cgi-bin/request/grx_rings.py)</a></li>
+<li><a href="/cgi-bin/request/hml.py?help">HML Processed Data (/cgi-bin/request/hml.py)</a></li>
 <li><a href="/cgi-bin/request/hourlyprecip.py?help">Hourly Precip (/cgi-bin/request/hourlyprecip.py)</a></li>
 <li><a href="/cgi-bin/request/nass_iowa.py?help">Iowa NASS (/cgi-bin/request/nass_iowa.py)</a></li>
 <li><a href="/cgi-bin/request/isusm.py?help">Iowa State Soil Moisture Network (/cgi-bin/request/isusm.py)</a></li>

htdocs/nws/index.php

@@ -92,6 +92,8 @@
   <li><a href="/plotting/auto/?q=160">HML based forecasts and observations</a>
     <br />An archive of HML processed products is used to drive an interactive
     plot of forecasts and observations.</li>
+  <li><a href="/request/hml.php">HML Processed Data Download</a>
+    <br />Download HML data for a site of your choice.</li>
   <li><a href="/cow/">IEM Cow Polygon Verification</a><br />
     Application does LSR based verification of Flash Flood Warnings.</li>
   <li><a href="/river/">River Forecast Point Monitor</a><br />

htdocs/request/hml.php

@@ -0,0 +1,137 @@
+<?php
+require_once "../../config/settings.inc.php";
+require_once "../../include/myview.php";
+require_once "../../include/imagemaps.php";
+require_once "../../include/forms.php";
+require_once "../../include/iemprop.php";
+require_once "../../include/database.inc.php";
+
+define("IEM_APPID", 164);
+$t = new MyView();
+$t->iemss = True;
+$t->title = "Hydrological Markup Language (HML) Processed Data Download";
+
+$bogus = 0;
+$y1select = yearSelect2(2012, date("Y"), "year1");
+$m1select = monthSelect(1, "month1");
+$d1select = daySelect2(1, "day1");
+
+$y2select = yearSelect2(2012, date("Y"), "year2");
+$m2select = monthSelect(date("m"), "month2");
+$d2select = daySelect2(date("d"), "day2");
+
+$t->content = <<<EOF
+<ol class="breadcrumb">
+ <li><a href="/nws/">NWS Mainpage</a></li>
+ <li class="active">Download HML Processed data</li>
+</ol>
+
+<p>The IEM attempts a high fidelity processing and archival of river gauge
+observations and forecasts found within the NWS HML Products.</p>
+
+<p><a href="/cgi-bin/request/hml.py?help" class="btn btn-default"><i class="fa fa-file"></i> Backend documentation</a>
+exists for those wishing to script against this service. The HML archive dates back to 2012.</p>
+
+<form method="GET" action="/cgi-bin/request/hml.py" name="dl" target="_blank">
+
+<div class="row">
+<div class="col-md-6">
+
+<p>
+<h3>1. Enter NWSLI Station Identifier:</h3>
+At this time, the IEM website does not have a map selection tool to pick from
+HML sites.  So you are stuck having to know the 5 character NWSLI identifier,
+sorry.
+<br /><input type="text" name="station" size="7" maxlength="5">
+</p>
+
+<p>
+<h3>2. Select Data Type:</h3>
+<input type="radio" name="kind" value="obs" checked id="obs">
+<label for="obs">Observations</label>
+<br />
+<input type="radio" name="kind" value="forecasts" id="forecasts">
+<label for="forecasts">Forecasts</label>: Note that you can only request
+forecast data for a single UTC year at a time.
+</p>
+
+</div>
+<div class="col-md-6">
+
+<h3>3. Timezone of Observations:</h3>
+<i>The timestamps used in the downloaded file will be set in the
+timezone you specify.</i>
+<SELECT name="tz">
+    <option value="UTC">UTC Time</option>
+    <option value="America/New_York">Eastern Time</option>
+    <option value="America/Chicago">Central Time</option>
+    <option value="America/Denver">Mountain Time</option>
+    <option value="America/Los_Angeles">Western Time</option>
+</SELECT>
+
+<h3>4. Select Start/End Time:</h3><br>
+<i>The end date is not inclusive.</i>
+<table class="table table-condensed">
+<thead>
+  <tr>
+    <td></td>
+    <th>Year</th><th>Month</th><th>Day</th>
+  </tr>
+</thead>
+<tbody>
+  <tr>
+    <th>Start:</th>
+    <td>{$y1select}</td><td>{$m1select}</td><td>{$d1select}</td>
+  </tr>
+
+  <tr>
+    <th>End:</th>
+    <td>{$y2select}</td><td>{$m2select}</td><td>{$d2select}</td>
+  </tr>
+</tbody>
+</table>
+
+<h3>5. Data Format:</h3>
+<select name="fmt">
+    <option value="csv">Comma</option>
+    <option value="excel">Excel</option>
+</select>
+
+<h3>Submit Form:</h3><br>
+<input type="submit" value="Process Data Request">
+<input type="reset">
+
+<p><strong>Observation Data Columns</strong>
+<table class="table table-striped">
+<thead><tr><th>Name</th><th>Description</th></tr></thead>
+<tbody>
+<tr><th>station</th><td>5 character station identifier</td></tr>
+<tr><th>valid[timezone]</th><td>Timestamp of observation</td></tr>
+<tr><th><code>Label for Values</code></th><td>Primary</td></tr>
+<tr><th><code>Label for Values</code></th><td>Secondary</td></tr>
+</tbody>
+</table></p>
+
+<p><strong>Observation Data Columns</strong>
+<table class="table table-striped">
+<thead><tr><th>Name</th><th>Description</th></tr></thead>
+<tbody>
+<tr><th>station</th><td>5 character station identifier</td></tr>
+<tr><th>issued[timezone]</th><td>Timestamp of forecast issuance</td></tr>
+<tr><th>primaryname</th><td>Label for the primary forecast value</td></tr>
+<tr><th>primaryunits</th><td>Units for the primary forecast value</td></tr>
+<tr><th>secondaryname</th><td>Label for the secondary forecast value</td></tr>
+<tr><th>secondaryunits</th><td>Units for the secondary forecast value</td></tr>
+<tr><th>forecast_valid[timezone]</th><td>Timestamp of forecast valid</td></tr>
+<tr><th>primary_value</th><td>Primary forecast value</td></tr>
+<tr><th>secondary_value</th><td>Secondary forecast value</td></tr>
+</tbody>
+</table></p>
+
+
+</div></div>
+
+</form>
+
+EOF;
+$t->render('full.phtml');

pylib/iemweb/autoplot/scripts100/p160.py

@@ -10,7 +10,10 @@
 <p>For the image format output options, you can optionally control if
 forecasts, observations, or both are plotted.  For the Interactive Chart
 version, this is controlled by clicking on the legend items which will
-hide and show the various lines.
+hide and show the various lines.</p>
+
+<p>A <a href="/request/hml.php">Download Portal</a> is available to more
+easily download the raw data in bulk.</p>
 """
 
 from datetime import timedelta

pylib/iemweb/request/hml.py

@@ -0,0 +1,213 @@
+""".. title:: Hydrological Markup Language (HML) Data
+
+Return to `API Services </api/#cgi>`_ or `HML Request </request/hml.php>`_.
+
+Documentation for /cgi-bin/request/hml.py
+-----------------------------------------
+
+This service provides the processed data from HML products.  This service
+does not emit the HML product itself, but rather the processed data. Due to
+lame reasons, you can only request forecast data within a single UTC year.
+
+Changelog
+---------
+
+- 2024-11-05: Initial implementation
+
+Example Usage
+~~~~~~~~~~~~~
+
+Provide all Guttenberg, IA GTTI4 observation data for 2024 in CSV format:
+
+https://mesonet.agron.iastate.edu/cgi-bin/request/hml.py\
+?station=GTTI4&sts=2024-01-01T00:00Z&ets=2025-01-01T00:00Z&fmt=csv\
+&kind=obs
+
+And then Excel
+
+https://mesonet.agron.iastate.edu/cgi-bin/request/hml.py\
+?station=GTTI4&sts=2024-01-01T00:00Z&ets=2025-01-01T00:00Z&fmt=excel\
+&kind=obs
+
+Provide all Guttenberg, IA GTTI4 forecast data for 2024 in CSV
+
+https://mesonet.agron.iastate.edu/cgi-bin/request/hml.py\
+?station=GTTI4&sts=2024-01-01T00:00Z&ets=2025-01-01T00:00Z&fmt=csv\
+&kind=forecast
+
+And then Excel
+
+https://mesonet.agron.iastate.edu/cgi-bin/request/hml.py\
+?station=GTTI4&sts=2024-01-01T00:00Z&ets=2025-01-01T00:00Z&fmt=excel\
+&kind=forecast
+
+"""
+
+from io import BytesIO
+from zoneinfo import ZoneInfo
+
+import pandas as pd
+from pydantic import AwareDatetime, Field, field_validator
+from pyiem.database import get_sqlalchemy_conn
+from pyiem.webutil import CGIModel, ListOrCSVType, iemapp
+from sqlalchemy import text
+
+EXL = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
+
+class MyModel(CGIModel):
+    """Our model"""
+
+    kind: str = Field(
+        "obs",
+        description="The type of data to request, either 'obs' or 'forecasts'",
+        pattern="^(obs|forecasts)$",
+    )
+    fmt: str = Field(
+        "csv",
+        description="The format of the output file, either 'csv' or 'excel'",
+        pattern="^(csv|excel)$",
+    )
+    tz: str = Field("UTC", description="The timezone to use for timestamps")
+    sts: AwareDatetime = Field(
+        None, description="The start timestamp for the data"
+    )
+    ets: AwareDatetime = Field(
+        None, description="The end timestamp for the data"
+    )
+    station: ListOrCSVType = Field(
+        ...,
+        description=(
+            "The station(s) to request data for, "
+            "either multi params or comma separated"
+        ),
+    )
+    year1: int = Field(None, description="The start year, if not using sts")
+    month1: int = Field(None, description="The start month, if not using sts")
+    day1: int = Field(None, description="The start day, if not using sts")
+    hour1: int = Field(0, description="The start hour, if not using sts")
+    minute1: int = Field(0, description="The start minute, if not using sts")
+    year2: int = Field(None, description="The end year, if not using ets")
+    month2: int = Field(None, description="The end month, if not using ets")
+    day2: int = Field(None, description="The end day, if not using ets")
+    hour2: int = Field(0, description="The end hour, if not using ets")
+    minute2: int = Field(0, description="The end minute, if not using ets")
+
+    @field_validator("tz", mode="before")
+    def check_tz(cls, value):
+        """Ensure the timezone is valid."""
+        try:
+            ZoneInfo(value)
+        except Exception as exp:
+            raise ValueError("Invalid timezone provided") from exp
+        return value
+
+
+def get_obs(dbconn, environ: dict) -> pd.DataFrame:
+    """Get data!"""
+    df = pd.read_sql(
+        text(
+            """
+        select distinct d.station, d.valid, k.label, d.value
+        from hml_observed_data d JOIN
+        hml_observed_keys k on (d.key = k.id) WHERE
+        station = ANY(:stations) and valid >= :sts and valid < :ets
+        ORDER by valid ASC
+        """
+        ),
+        dbconn,
+        params={
+            "stations": environ["station"],
+            "sts": environ["sts"],
+            "ets": environ["ets"],
+        },
+    )
+    # muck the timezones
+    if not df.empty:
+        tzinfo = ZoneInfo(environ["tz"])
+        df["valid"] = (
+            df["valid"].dt.tz_convert(tzinfo).dt.strftime("%Y-%m-%d %H:%M")
+        )
+        df = df.pivot_table(
+            index=["station", "valid"],
+            columns="label",
+            values="value",
+            aggfunc="first",
+        ).reset_index()
+        df = df.rename(
+            columns={
+                "valid": f"valid[{environ['tz']}]",
+            }
+        )
+    return df
+
+
+def get_forecasts(dbconn, environ: dict) -> pd.DataFrame:
+    """Get data!"""
+    year = environ["sts"].year
+    df = pd.read_sql(
+        text(
+            f"""
+        select station, issued, primaryname, primaryunits,
+        secondaryname, secondaryunits, valid as forecast_valid,
+        primary_value, secondary_value from hml_forecast f,
+        hml_forecast_data_{year} d WHERE
+        f.station = ANY(:stations) and f.issued >= :sts and f.issued < :ets
+        and f.id = d.hml_forecast_id ORDER by issued ASC, forecast_valid ASC
+        """
+        ),
+        dbconn,
+        params={
+            "stations": environ["station"],
+            "sts": environ["sts"],
+            "ets": environ["ets"],
+        },
+    )
+    # muck the timezones
+    if not df.empty:
+        tzinfo = ZoneInfo(environ["tz"])
+        for col in ["forecast_valid", "issued"]:
+            df[col] = (
+                df[col].dt.tz_convert(tzinfo).dt.strftime("%Y-%m-%d %H:%M")
+            )
+        df = df.rename(
+            columns={
+                "forecast_valid": f"forecast_valid[{environ['tz']}]",
+                "issued": f"issued[{environ['tz']}]",
+            }
+        )
+    return df
+
+
+def rect(station):
+    """Cleanup."""
+    station = station.upper()
+    return station[:5]
+
+
+@iemapp(help=__doc__, schema=MyModel)
+def application(environ, start_response):
+    """Get stuff"""
+    environ["station"] = [rect(x) for x in environ["station"]]
+    with get_sqlalchemy_conn("hml") as dbconn:
+        if environ["kind"] == "obs":
+            df = get_obs(dbconn, environ)
+        else:
+            df = get_forecasts(dbconn, environ)
+
+    bio = BytesIO()
+    if environ["fmt"] == "excel":
+        with pd.ExcelWriter(bio, engine="openpyxl") as writer:
+            df.to_excel(writer, sheet_name="HML Data", index=False)
+        headers = [
+            ("Content-type", EXL),
+            ("Content-disposition", "attachment;Filename=hml.xlsx"),
+        ]
+    else:
+        df.to_csv(bio, index=False)
+        headers = [
+            ("Content-type", "application/octet-stream"),
+            ("Content-disposition", "attachment;Filename=hml.csv"),
+        ]
+    start_response("200 OK", headers)
+    return [bio.getvalue()]