Further working on the SQL to load raster data (#12)

toolkit/cli/load/coordinates/__init__.py

@@ -0,0 +1,23 @@
+from postgis import connect
+
+"""
+Either create the view, or if the view exists, either
+refresh the view if it's a new model value, or do nothing.
+I couldn't figure out how to parametize the model string value
+using the psycog2 module. So instead I'm just checking that the
+value exists as a string in the models table. This should be safe
+"""
+def create_view(model):
+    cursor = connect().cursor()
+    cursor.execute("""
+      select 1 where exists ( select * from public.models where name = %s )
+    """, (model, ))
+    exists = not len(cursor.fetchall()) == 0
+
+    if not exists:
+      raise Exception('Cannot refresh coordinates for a model that doesn\'t exist')
+
+    with open('cli/load/coordinates/coordinates.sql', 'r') as file:
+      sql = file.read().replace('%s', "'" + model + "'") # TODO!!! escape properly
+      cursor = connect().cursor()
+      cursor.execute(sql)

toolkit/cli/load/coordinates/coordinates.sql

@@ -0,0 +1,65 @@
+do $$
+begin
+  if exists (
+    select
+      1
+    from
+      pg_matviews
+    where
+      matviewname = 'coordinates') then
+  if not exists (
+    select
+      1
+    from
+      coordinates c
+      join models m on m.id = c.modelid
+    where
+      m.name = %s) then
+  refresh materialized view coordinates;
+end if;
+else
+  create materialized view coordinates as
+  with lon as (
+    select
+      modelid,
+      geom pixel,
+      val longitude
+    from ( select distinct
+        m.id modelid,
+        (ST_PixelAsCentroids (rast, 1)).*
+      from
+        rasters r
+        join raster_xref_model rxm on rxm.rasterid = r.rid
+        join models m on m.id = rxm.modelid
+      where
+        filename like '%lon_rho') lon
+),
+lat as (
+  select
+    modelid,
+    geom pixel,
+    val latitude
+  from ( select distinct
+      m.id modelid,
+      (ST_PixelAsCentroids (rast, 1)).*
+    from
+      rasters r
+    left join raster_xref_model rxm on rxm.rasterid = r.rid
+    left join models m on m.id = rxm.modelid
+  where
+    filename like '%lat_rho') lat
+)
+select
+  lon.modelid,
+  lon.pixel,
+  lon.longitude,
+  lat.latitude,
+  st_point (lon.longitude, lat.latitude, 4326) coord
+from
+  lon
+  join lat on lat.pixel = lon.pixel
+    and lat.modelid = lon.modelid;
+end if;
+end
+$$;
+

toolkit/cli/load/metadata/__init__.py

@@ -0,0 +1,9 @@
+from postgis import connect
+
+
+# Setup view that summarizes models
+def create_view(model):
+    with open('cli/load/metadata/metadata.sql', 'r') as file:
+        sql = file.read()
+        cursor = connect().cursor()
+        cursor.execute(sql)

toolkit/cli/load/raster2pgsql.py → toolkit/cli/load/raster2pgsql/__init__.py

@@ -57,7 +57,7 @@ def register(now, nc_input_path, raster, model):
     else:
       cursor = connectPg().cursor()
       cursor.execute("""
-        select 1 where exists (select * from public.rasters where filename = %s)
+        select 1 where exists ( select * from public.rasters where filename = %s )
       """, (filename, ))
       exists = not len(cursor.fetchall()) == 0
       if exists:
@@ -108,5 +108,12 @@ def register(now, nc_input_path, raster, model):
   print("""Command:""".format(str(raster)), sub(' +', ' ', cmd))
   if os.system(cmd) != 0: raise Exception('raster2pgsql cmd failed: ' + sub(' +', ' ', cmd))
 
-  # TODO associate filename with the mode
+  """
+  Explicitly associate individual rasters with a model
+  """
+  with open('cli/load/raster2pgsql/update-raster_xref_model.sql') as file:
+    sql = file.read()
+    cursor = connectPg().cursor()
+    cursor.execute(sql, (filename, model,))
+    print(cursor.statusmessage)
 

toolkit/cli/load/raster2pgsql/update-raster_xref_model.sql

@@ -0,0 +1,24 @@
+;
+
+with ref as (
+  select
+    %s filename,
+    %s model
+)
+
+insert into raster_xref_model (rasterid, modelid)
+select
+  r.rid rasterid,
+  m.id modelid
+from
+  ref
+  left join rasters r on r.filename = ref.filename
+  left join models m on m.name = ref.model
+
+where not exists (
+  select 1
+  from raster_xref_model x
+  where x.rasterid = r.rid and x.modelid = m.id
+);
+
+

toolkit/postgis/schema.sql

@@ -1,4 +1,5 @@
-/**************** ENABLE spatial extensions ******************/
+;
+
 create extension if not exists postgis;
 
 create extension if not exists postgis_raster;
@@ -15,14 +16,6 @@ create extension if not exists address_standardizer_data_us;
 
 create extension if not exists postgis_tiger_geocoder;
 
-set postgis.gdal_enabled_drivers = 'ENABLE_ALL';
-
-set postgis.enable_outdb_rasters = 1;
-
--- Allow access to external data and formats permnently
--- alter database somisana_local set postgis.enable_outdb_rasters = true;
--- alter database somisana_local SET postgis.gdal_enabled_drivers TO 'ENABLE_ALL';
-/*********************************************************/
 create table if not exists public.rasters (
   rid serial not null primary key,
   rast public.raster null,
@@ -42,8 +35,8 @@ on conflict on constraint models_unique_name
 
 create table if not exists public.raster_xref_model (
   id serial primary key,
-  rasterId int references rasters (rid),
-  modelId int references models (id),
-  constraint raster_xref_filename_unique unique (rasterId, modelId)
+  rasterid int references rasters (rid) on delete cascade,
+  modelid int references models (id) on delete cascade,
+  constraint raster_xref_filename_unique_rasterid unique (rasterid)
 );