README.Rmd

---
output: github_document
---

<!-- README.md is generated from README.Rmd. Please edit that file -->

```{r, echo = FALSE}
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  fig.path = "README-"
)
```

## Overall focus

https://confluence.csiro.au/display/ACCESS/Home

simulations "piControl" "historical", "rcp45", "rcp85"

experiment "r1i1p1", 

See: "standard_output.pdf" and http://cmip-pcmdi.llnl.gov/cmip5/data_description.html 


# CMIP5files

browseables for CMIP5

Start here: https://github.com/mdsumner/nectar/wiki/Raijin-CMIP5

# CMIP5 files

These seem to cause a lot of grief, but NetCDF always does. 

SQLite DB built by Python code on /g/data, lists tokenized file paths. This is useful but needs to be run routinely so it's up to date, and really make a full path to all files. 

* list actual files, find best way to list (i.e. multiple find jobs?)
* scan each file and record metadata for each as per rancid::NetCDF or equivalent


# Dev Plan

* first pass only scan a sample of files, develop a scoping estimate of how long / how much etc. 
* Use try/catch defenses to be reliable, and feed back lessons into rancid package
* Build searchable table of files with DT/Shiny (can we publish this off Raijin?)
* Allow select file to deliver metadata DB as constellation of tables (or ncdump, whatevs)
* Identify common patterns across all files in variables, dimensions
* Use to develope use-case read code

```bash
module load   proj/4.9.0  netcdf/4.3.3.1p 
```

```{r}
#.libPaths("R/x86_64-pc-linux-gnu-library/3.2")
library(dplyr)
library(rancid)
db <- src_sqlite("/g/data1/ua6/unofficial-ESG-replica/tmp/tree/cmip5_raijin_latest.db")
cmip5 <- tbl(db, "cmip5")

access3 <- collect(filter(cmip5, model == "ACCESS1-3"))
id <- sample(access3$id, 1)
print(id)

nc <- NetCDF(sample(list.files(id, full.names = TRUE), 1))

nc$variable



ss <- sample(nrow(access3), min(c(100, nrow(access3))))
vv <- vector("list", length(ss))
system.time({
for (i in seq_along(vv)) {
fs <- sample(list.files(access3$id[i], full.names = TRUE, pattern = "nc$"), 1)
a <- try(NetCDF(fs))
if (!inherits(a, "try-error")) vv[[i]] <- a
}
})

object.size(vv)
# 4.88 Mb

any(unlist(lapply(vv, is.null)))

#save(vv, file = "~/cmip5_metadatasample.rda")
```
Etc. 

# Example

## 1. Use DT to get a user-useable path to a file. 

```{r}
library(DT)
d <- access3[sample(nrow(access3), 100), ]
datatable(d[, c(setdiff(names(d), "id"), "id")], class = 'cell-border stripe', filter = 'top', options = list(
  pageLength = 35, autoWidth = TRUE
))
```

## 2. Convert mapped data to model space and make a map

```R

print(nc$variable)
lonlat <- romscoords(nc$file$filename, c("lon", "lat"))
library(maptools)
data(wrld_simpl)
plot(raster(f, varname = "tauuo"))
plot(romsmap(wrld_simpl, lonlat), add = TRUE)
#install.packages(c("rworldmap", "rworldxtra"))


tt <- spbabel::sptable(wrld_simpl)
tt$x_[tt$x_ < 0] <- tt$x_[tt$x_ < 0] + 360
w <- spbabel::sp(tt)
plot(raster(f, varname = "tauuo"))
plot(romsmap(w, lonlat), add = TRUE)
```

![](Clipboard02.png)


module load         proj/4.9.0  netcdf/4.3.3.1p  gdal/2.0.0 
```R
dp <- "/g/data/ua6/unofficial-ESG-replica/tmp/tree/tds.ucar.edu/thredds/fileServer/datazone/cmip5_data/cmip5/output1/NCAR/CCSM4/rcp85/mon/ocean/Omon/r6i1p1/v20140820/tauuo"
library(raster)
f <- list.files(dp, full.names = TRUE)
library(rancid)
library(angstroms)
nc <- NetCDF(f)
print(nc$variable)

Source: local data frame [6 x 16]

          name ndims natts   prec         units                  longname
         (chr) (int) (int)  (chr)         (chr)                     (chr)
1          lat     2     4  float degrees_north       latitude coordinate
2 lat_vertices     3     1  float degrees_north              lat_vertices
3          lon     2     4  float  degrees_east      longitude coordinate
4 lon_vertices     3     1  float  degrees_east              lon_vertices
5        tauuo     3    12  float         N m-2 Surface Downward X Stress
6    time_bnds     2     0 double                               time_bnds
Variables not shown: group_index (int), storage (dbl), shuffle (lgl),
  compression (lgl), unlim (lgl), make_missing_value (lgl), missval (dbl),
  hasAddOffset (lgl), hasScaleFact (lgl), id (dbl)


lonlat <- romscoords(f, c("lon", "lat"))
#devtools::install_github("AustralianAntarcticDivision/rbgm")
library(rbgm)
fname <- system.file("extdata", "Antarctica_28.bgm", package = "rbgm")
bgm <- bgmfile(fname)
plot(boxSpatial(bgm), col = grey(seq(0, 1, length = nrow(bgm$boxes))))
#devtools::install_github("mdsumner/angstroms")
ar <- romsmap(boxSpatial(bgm), lonlat)


## now extract!!
 cn <- extract(brick(romsdata(f, "tauuo")), ar, cellnumbers = TRUE)

```