Skip to content

Commit

Permalink
support for vcr write to disk path (#345)
Browse files Browse the repository at this point in the history
* #290 - working on integrating vcr handling writing to disk, not done yet
related: ropensci/vcr#81 ropensci/webmockr#57

* playing with buoy tests, check back in #FIXME

* cpc_prcp test tweaks

* man file updates

* arc2 fixes, use new write to disk setup

* #290 writing to disk work, not done yet

* fixes for lcd and isd tests that also cache files

* fixed some tests #290

* fix storms test

* use relative paths with new vcr version

* move write to disk files to inst/ - test check on travis | ropensci/vcr#164

* try commenting out deleting  user cached files for ersst tests

* comment out one ersst test, does it work now?

* try ersst tests each with different file on disk - was previously using the same file
perhaps that was leading to failure if e.g. file was opened but not closed before next test ran amybe

* remove .Rinstignore, and try inst/test_files in .Rbuildignore if it works on travis

* remove inst/test_files from rbuildignore

* move test files back to tests/files/

* add .Rinstignore for ignoring test on disk files

* woops, needed to fix write_disk_path
  • Loading branch information
sckott authored Mar 23, 2020
1 parent 428381d commit 2a7f09f
Show file tree
Hide file tree
Showing 52 changed files with 17,603 additions and 40 deletions.
4 changes: 4 additions & 0 deletions .Rinstignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
tests/files/.*[.]csv$
tests/files/.*[.]nc$
tests/files/.*[.]gz$
tests/files/.*[.]RT$
3 changes: 2 additions & 1 deletion DESCRIPTION
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,9 @@ Suggests:
purrr,
ggmap,
ropenaq,
vcr (>= 0.2.2),
vcr (>= 0.5.0.92),
webmockr
Remotes: ropensci/vcr
RoxygenNote: 7.1.0
X-schema.org-applicationCategory: Climate
X-schema.org-keywords: earth, science, climate, precipitation, temperature, storm, buoy, NCDC, NOAA, tornadoe, sea ice, ISD
Expand Down
4 changes: 3 additions & 1 deletion R/arc2.R
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ arc2_get <- function(year, month, day, cache = TRUE, overwrite = FALSE, ...) {
key <- file.path(arc2_base_ftp(arc2_base(), date))
file <- file.path(arc2_cache$cache_path_get(), basename(key))
if (!file.exists(file)) {
suppressMessages(arc2_GET_write(sub("/$", "", key), file, overwrite, ...))
res <- suppressMessages(
arc2_GET_write(sub("/$", "", key), file, overwrite, ...))
file <- res$content
}
return(file)
}
Expand Down
5 changes: 4 additions & 1 deletion R/buoy.R
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
the_env <- new.env()

#' Get NOAA buoy data from the National Buoy Data Center
#'
#' @export
Expand Down Expand Up @@ -153,7 +155,8 @@ buoy_single_file_url <- function(dataset, buoyid, file){
get_ncdf_file <- function(path, buoyid, file, output, ...){
outpath <- sprintf("%s/%s%s", output, buoyid, file)
res <- crul::HttpClient$new(path, opts = list(...))$get(disk = outpath)
return(outpath)
the_env$res <- res
return(res$content)
}

# Download a single ncdf file
Expand Down
3 changes: 2 additions & 1 deletion R/cpc.R
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ cpc_get <- function(year, month, day, us, cache = TRUE, overwrite = FALSE, ...)
key <- cpc_key(year, month, day, us)
file <- file.path(cpc_cache$cache_path_get(), basename(key))
if (!file.exists(file)) {
suppressMessages(cpc_GET_write(sub("/$", "", key), file, overwrite, ...))
res <- suppressMessages(cpc_GET_write(sub("/$", "", key), file, overwrite, ...))
file <- res$content
}
return(file)
}
Expand Down
7 changes: 7 additions & 0 deletions tests/files/1945_02413099999.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>404 Not Found</title>
</head><body>
<h1>Not Found</h1>
<p>The requested URL /data/local-climatological-data/access/1945/02413099999.csv was not found on this server.</p>
</body></html>
Binary file added tests/files/201606.nc
Binary file not shown.
Binary file added tests/files/201609.nc
Binary file not shown.
Binary file added tests/files/20166.nc
Binary file not shown.
Binary file added tests/files/20167.nc
Binary file not shown.
Binary file added tests/files/20168.nc
Binary file not shown.
Binary file added tests/files/41001c1990.nc
Binary file not shown.
Binary file added tests/files/46081a2010.nc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
8 changes: 8 additions & 0 deletions tests/files/Storm.1970143N19091.ibtracs_all.v03r10.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
IBTrACS -- Version: v03r10
Serial_Num,Season,Num,Basin,Sub_basin,Name,ISO_time,Nature,Latitude,Longitude,Wind(WMO),Pres(WMO),Center,Wind(WMO) Percentile,Pres(WMO) Percentile,Track_type,Latitude_for_mapping,Longitude_for_mapping,Current Basin,hurdat_atl_lat,hurdat_atl_lon,hurdat_atl_grade,hurdat_atl_wind,hurdat_atl_pres,td9636_lat,td9636_lon,td9636_grade,td9636_wind,td9636_pres,reunion_lat,reunion_lon,reunion_grade,reunion_wind,reunion_pres,atcf_lat,atcf_lon,atcf_grade,atcf_wind,atcf_pres,mlc_natl_lat,mlc_natl_lon,mlc_natl_grade,mlc_natl_wind,mlc_natl_pres,ds824_sh_lat,ds824_sh_lon,ds824_sh_grade,ds824_sh_wind,ds824_sh_pres,ds824_ni_lat,ds824_ni_lon,ds824_ni_grade,ds824_ni_wind,ds824_ni_pres,bom_lat,bom_lon,bom_grade,bom_wind,bom_pres,ds824_au_lat,ds824_au_lon,ds824_au_grade,ds824_au_wind,ds824_au_pres,jtwc_sh_lat,jtwc_sh_lon,jtwc_sh_grade,jtwc_sh_wind,jtwc_sh_pres,jtwc_wp_lat,jtwc_wp_lon,jtwc_wp_grade,jtwc_wp_wind,jtwc_wp_pres,td9635_lat,td9635_lon,td9635_grade,td9635_wind,td9635_pres,ds824_wp_lat,ds824_wp_lon,ds824_wp_grade,ds824_wp_wind,ds824_wp_pres,jtwc_io_lat,jtwc_io_lon,jtwc_io_grade,jtwc_io_wind,jtwc_io_pres,cma_lat,cma_lon,cma_grade,cma_wind,cma_pres,hurdat_epa_lat,hurdat_epa_lon,hurdat_epa_grade,hurdat_epa_wind,hurdat_epa_pres,jtwc_ep_lat,jtwc_ep_lon,jtwc_ep_grade,jtwc_ep_wind,jtwc_ep_pres,ds824_ep_lat,ds824_ep_lon,ds824_ep_grade,ds824_ep_wind,ds824_ep_pres,jtwc_cp_lat,jtwc_cp_lon,jtwc_cp_grade,jtwc_cp_wind,jtwc_cp_pres,tokyo_lat,tokyo_lon,tokyo_grade,tokyo_wind,tokyo_pres,neumann_lat,neumann_lon,neumann_grade,neumann_wind,neumann_pres,hko_lat,hko_lon,hko_grade,hko_wind,hko_pres,cphc_lat,cphc_lon,cphc_grade,cphc_wind,cphc_pres,wellington_lat,wellington_lon,wellington_grade,wellington_wind,wellington_pres,newdelhi_lat,newdelhi_lon,newdelhi_grade,newdelhi_wind,newdelhi_pres,nadi_lat,nadi_lon,nadi_grade,nadi_wind,nadi_pres,reunion_rmw,reunion_wind_radii_1_ne,reunion_wind_radii_1_se,reunion_wind_radii_1_sw,reunion_wind_radii_1_nw,reunion_wind_radii_2_ne,reunion_wind_radii_2_se,reunion_wind_radii_2_sw,reunion_wind_radii_2_nw,bom_mn_hurr_xtnt,bom_mn_gale_xtnt,bom_mn_eye_diam,bom_roci,atcf_rmw,atcf_poci,atcf_roci,atcf_eye,atcf_wrad34_rad1,atcf_wrad34_rad2,atcf_wrad34_rad3,atcf_wrad34_rad4,atcf_wrad50_rad1,atcf_wrad50_rad2,atcf_wrad50_rad3,atcf_wrad50_rad4,atcf_wrad64_rad1,atcf_wrad64_rad2,atcf_wrad64_rad3,atcf_wrad64_rad4,tokyo_dir50,tokyo_long50,tokyo_short50,tokyo_dir30,tokyo_long30,tokyo_short30,jtwc_??_rmw,jtwc_??_poci,jtwc_??_roci,jtwc_??_eye,jtwc_??_wrad34_rad1,jtwc_??_wrad34_rad2,jtwc_??_wrad34_rad3,jtwc_??_wrad34_rad4,jtwc_??_wrad50_rad1,jtwc_??_wrad50_rad2,jtwc_??_wrad50_rad3,jtwc_??_wrad50_rad4,jtwc_??_wrad64_rad1,jtwc_??_wrad64_rad2,jtwc_??_wrad64_rad3,jtwc_??_wrad64_rad4
N/A,Year,#,BB,BB,N/A,YYYY-MM-DD HH:MM:SS,N/A,deg_north,deg_east,kt,mb,N/A,%,%,N/A,degrees_north,degrees_east,N/A,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,deg_north,deg_east, ,kt,mb,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,mb,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,Quad,nmile,nmile,Quad,nmile,nmile,nmile,mb,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile,nmile
1970143N19091,1970,02, NI, BB,02B,1970-05-22 18:00:00, TS,-999.,-999.,-999.,-999.,N/A,-999.,-999.,main, 19.30, 90.70,NI,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0, 19.3, 90.7,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0, 19.3, 90.7,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,0.00000,0.00000,0.00000,0.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000
1970143N19091,1970,02, NI, BB,02B,1970-05-23 00:00:00, TS,-999.,-999.,-999.,-999.,N/A,-999.,-999.,main, 19.50, 90.90,NI,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0, 19.5, 90.9,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0, 19.5, 90.9,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,0.00000,0.00000,0.00000,0.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000
1970143N19091,1970,02, NI, BB,02B,1970-05-23 06:00:00, TS,-999.,-999.,-999.,-999.,N/A,-999.,-999.,main, 19.81, 91.32,NI,-999.0,-999.0,-999.0,-999.0,-999.0, 19.9, 91.4,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0, 19.8, 91.3,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0, 19.8, 91.3,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,0.00000,0.00000,0.00000,0.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000
1970143N19091,1970,02, NI, BB,02B,1970-05-23 12:00:00, TS,-999.,-999.,-999.,-999.,N/A,-999.,-999.,main, 20.33, 91.95,NI,-999.0,-999.0,-999.0,-999.0,-999.0, 20.4, 92.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0, 20.3, 91.9,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0, 20.3, 91.9,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,0.00000,0.00000,0.00000,0.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000,-1.00000
1970143N19091,1970,02, NI, BB,02B,1970-05-23 18:00:00, NR,-999.,-999.,-999.,-999.,N/A,-999.,-999.,main, 20.85, 92.60,NI,-999.0,-999.0,-999.0,-999.0,-999.0, 20.9, 92.6,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.0,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000,-999.000
Loading

0 comments on commit 2a7f09f

Please sign in to comment.