wip updating cli sql #12

.vscode/settings.json

@@ -44,6 +44,7 @@
     "pipenv",
     "pipfile",
     "POSTGIS",
+    "psycog",
     "psycopg",
     "pyenv",
     "pypi",

toolkit/cli/load/coordinates/__init__.py

@@ -1,11 +1,11 @@
 from postgis import connect
 
 """
-Either create the view, or if the view exists, either
-refresh the view if it's a new model value, or do nothing.
-I couldn't figure out how to parametize the model string value
-using the psycog2 module. So instead I'm just checking that the
-value exists as a string in the models table. This should be safe
+Update the coordinates table whenever a
+new model is used. The assumption is that
+once a model is run once, the grid is fixed
+in terms of XY and doesn't need further
+updates
 """
 def create_view(model):
     cursor = connect().cursor()
@@ -18,6 +18,6 @@ def create_view(model):
       raise Exception('Cannot refresh coordinates for a model that doesn\'t exist')
 
     with open('cli/load/coordinates/coordinates.sql', 'r') as file:
-      sql = file.read().replace('%s', "'" + model + "'") # TODO!!! escape properly
+      sql = file.read()
       cursor = connect().cursor()
-      cursor.execute(sql)
+      cursor.execute(sql, (model, model))

toolkit/cli/load/coordinates/coordinates.sql

@@ -1,38 +1,20 @@
-do $$
-begin
-  if exists (
-    select
-      1
-    from
-      pg_matviews
-    where
-      matviewname = 'coordinates') then
-  if not exists (
-    select
-      1
+;
+
+with lon as (
+  select
+    modelid,
+    geom pixel,
+    val longitude
+  from ( select distinct
+      m.id modelid,
+      (ST_PixelAsCentroids (rast, 1)).*
     from
-      coordinates c
-      join models m on m.id = c.modelid
+      rasters r
+      join raster_xref_model rxm on rxm.rasterid = r.rid
+      join models m on m.id = rxm.modelid
+        and m.name = %s
     where
-      m.name = %s) then
-  refresh materialized view coordinates;
-end if;
-else
-  create materialized view coordinates as
-  with lon as (
-    select
-      modelid,
-      geom pixel,
-      val longitude
-    from ( select distinct
-        m.id modelid,
-        (ST_PixelAsCentroids (rast, 1)).*
-      from
-        rasters r
-        join raster_xref_model rxm on rxm.rasterid = r.rid
-        join models m on m.id = rxm.modelid
-      where
-        filename like '%lon_rho') lon
+      filename like '%%lon_rho') lon
 ),
 lat as (
   select
@@ -46,20 +28,15 @@ lat as (
       rasters r
     left join raster_xref_model rxm on rxm.rasterid = r.rid
     left join models m on m.id = rxm.modelid
+      and m.name = %s
   where
-    filename like '%lat_rho') lat
-)
-select
-  lon.modelid,
-  lon.pixel,
-  lon.longitude,
-  lat.latitude,
-  st_point (lon.longitude, lat.latitude, 4326) coord
-from
-  lon
+    filename like '%%lat_rho') lat) merge into public.coordinates t using (
+  select
+    lon.modelid, lon.pixel, st_point (lon.longitude, lat.latitude, 4326) coord, lon.longitude, lat.latitude from lon
   join lat on lat.pixel = lon.pixel
-    and lat.modelid = lon.modelid;
-end if;
-end
-$$;
+    and lat.modelid = lon.modelid) s on s.modelid = t.modelid
+  and s.pixel = t.pixel
+  when not matched then
+    insert (modelid, pixel, coord, longitude, latitude)
+      values (s.modelid, s.pixel, s.coord, s.longitude, s.latitude);
 

toolkit/cli/load/metadata/metadata.sql

@@ -1,21 +1,34 @@
-drop view if exists metadata;
-
-create view metadata as
-select
-  modelId,
-  min_x,
-  max_x,
-  min_y,
-  max_y,
-  st_makeenvelope (min_x, min_y, max_x, max_y, 4326) extent
-from (
+do $$
+begin
+  if exists (
+    select
+      1
+    from
+      pg_matviews
+    where
+      matviewname = 'metadata') then
+  refresh materialized view metadata;
+else
+  create materialized view metadata as
   select
-    c.modelId,
-    min(c.longitude) min_x,
-    max(c.longitude) max_x,
-    max(c.latitude) max_y,
-    min(c.latitude) min_y
-  from
-    coordinates c
-  group by
-    modelId) t
+    modelId,
+    min_x,
+    max_x,
+    min_y,
+    max_y,
+    st_makeenvelope (min_x, min_y, max_x, max_y, 4326) extent
+  from (
+    select
+      c.modelId,
+      min(c.longitude) min_x,
+      max(c.longitude) max_x,
+      max(c.latitude) max_y,
+      min(c.latitude) min_y
+    from
+      coordinates c
+    group by
+      modelId) t;
+end if;
+end
+$$;
+

toolkit/postgis/drop-schema.sql

@@ -1,7 +1,12 @@
 -- MATERIALIZED VIEWS
-drop materialized view if exists public.coordinates cascade;
+drop materialized view if exists public.metadata cascade;
 
 -- TABLES
+drop table if exists public.coordinates cascade;
+
 drop table if exists public.raster_xref_model cascade;
+
 drop table if exists public.models cascade;
-drop table if exists public.rasters cascade;
+
+drop table if exists public.rasters cascade;
+

toolkit/postgis/schema.sql

@@ -23,20 +23,34 @@ create table if not exists public.rasters (
 );
 
 create table if not exists public.models (
-  id serial primary key,
-  name varchar(255),
-  constraint models_unique_name unique (name)
+  id serial not null primary key,
+  name varchar(255) not null unique
 );
 
-insert into public.models (name)
-  values ('algoa-bay-forecast'), ('false-bay-forecast')
-on conflict on constraint models_unique_name
-  do nothing;
+merge into public.models t
+using (
+  select
+    'algoa-bay-forecast' name
+  union
+    select
+      'false-bay-forecast' name) s on s.name = t.name
+when not matched then
+  insert (name)
+    values (s.name);
 
 create table if not exists public.raster_xref_model (
-  id serial primary key,
-  rasterid int references rasters (rid) on delete cascade,
-  modelid int references models (id) on delete cascade,
-  constraint raster_xref_filename_unique_rasterid unique (rasterid)
+  id serial not null primary key,
+  rasterid int not null unique references rasters (rid) on delete cascade,
+  modelid int not null references models (id) on delete cascade
+);
+
+create table if not exists public.coordinates (
+  id serial not null primary key,
+  modelid int not null references models (id) on delete cascade,
+  pixel geometry not null,
+  coord geometry not null,
+  longitude float not null,
+  latitude float not null,
+  constraint unique_coordinates unique (modelid, pixel)
 );