Update tests

.gitignore

@@ -54,3 +54,58 @@ tests/testthat/testInputs/v_7.0/gcamv7.0_test_specify_queries_standardized.csv
 tests/testthat/testInputs/v_7.0/gcamv7.0_test_specify_queries_standardized.RData
 tests/testthat/testInputs/v_7.0/gcamv7.0_test_specify_queries_standardized.xlsx
 debug.R
+tests/testthat/testInputs/v_7.0/database_basexdb_CO2price_test_standardized.csv
+tests/testthat/testInputs/v_7.0/database_basexdb_CO2price_test_standardized.RData
+tests/testthat/testInputs/v_7.0/database_basexdb_CO2price_test_standardized.xlsx
+tests/testthat/testInputs/v_7.0/database_basexdb_policy_standardized.csv
+tests/testthat/testInputs/v_7.0/database_basexdb_policy_standardized.RData
+tests/testthat/testInputs/v_7.0/database_basexdb_policy_standardized.xlsx
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.10.1_test.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.10.2_test.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.10.3_test.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.11.3_test_scenarios.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.11.6_test_scenarios.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.11.7_test_scenarios.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.1_test.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.2_test.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.3_test.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_ref_gcamv7.4_test.dat
+tests/testthat/testInputs/v_7.0/database_basexdb_test_scenarios7_standardized.csv
+tests/testthat/testInputs/v_7.0/database_basexdb_test_scenarios7_standardized.RData
+tests/testthat/testInputs/v_7.0/database_basexdb_test_scenarios7_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.10.1_test_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.10.1_test_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.10.1_test_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.10.2_test_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.10.2_test_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.10.2_test_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.10.3_test_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.10.3_test_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.10.3_test_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.11.3_test_scenarios_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.11.3_test_scenarios_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.11.3_test_scenarios_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.11.6_test_scenarios_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.11.6_test_scenarios_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.11.6_test_scenarios_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.11.7_test_scenarios_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.11.7_test_scenarios_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.11.7_test_scenarios_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.1_test_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.1_test_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.1_test_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.2_test_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.2_test_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.2_test_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.3_test_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.3_test_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.3_test_standardized.xlsx
+tests/testthat/testInputs/v_7.0/gcamv7.4_test_standardized.csv
+tests/testthat/testInputs/v_7.0/gcamv7.4_test_standardized.RData
+tests/testthat/testInputs/v_7.0/gcamv7.4_test_standardized.xlsx
+tests/testthat/testInputs/v_7.0/test7_standardized.csv
+tests/testthat/testInputs/v_7.0/test7_standardized.RData
+tests/testthat/testInputs/v_7.0/test7_standardized.xlsx
+tests/testthat/testOutputs/v_7.0/test7_output.csv
+tests/testthat/testOutputs/v_7.0/test7_output.RData
+tests/testthat/testOutputs/v_7.0/test7_output.xlsx

R/functions.R

@@ -1296,7 +1296,7 @@ get_ag_prices <- function() {
     ) %>%
     filter(!is.na(var)) %>%
     group_by(scenario, region, sector) %>%
-    mutate(value = value * unit_conv / value[year == 2005]) %>%
+    mutate(value = value * unit_conv / value[year == 2015]) %>%
     ungroup() %>%
     # do the mean by variable
     group_by(scenario, region, var, year, ) %>%
@@ -1822,17 +1822,10 @@ get_energy_price <- function() {
       grepl("Primary Energy\\|Biomass", var) |
       grepl("Primary Energy\\|Gas", var) |
       grepl("Primary Energy\\|Oil", var) |
-      grepl("Secondary Energy\\|Electricity", var) |
-      grepl("Secondary Energy\\|Gas", var) |
-      grepl("Secondary Energy\\|Liquids", var) |
-      grepl("Secondary Energy\\|Liquids\\|Biomass", var) |
-      grepl("Secondary Energy\\|Liquids\\|Oil", var) |
-      grepl("Secondary Energy\\|Solids", var) |
-      grepl("Secondary Energy\\|Solids\\|Biomass", var) |
-      grepl("Secondary Energy\\|Solids\\|Coal", var)) %>%
+      grepl("Secondary Energy\\|Electricity", var)) %>%
     mutate(var = paste(var, "Index", sep = "|")) %>%
     group_by(scenario, region, var) %>%
-    mutate(value = value / value[year == 2020]) %>%
+    mutate(value = value / value[year == 2015]) %>%
     ungroup() %>%
     select(all_of(gcamreport::long_columns)) %>%
     bind_rows(energy_price)
@@ -2448,7 +2441,7 @@ do_bind_results <- function() {
   GCAM_DATA_wGLOBAL <- GCAM_DATA_wGLOBAL %>% filter(year <= final_year.global)
 
 
-  report <<-
+  report_pre <-
     gcamreport::template %>%
     inner_join(
       GCAM_DATA_wGLOBAL %>%
@@ -2460,7 +2453,15 @@ do_bind_results <- function() {
     #            by = c("scenario" = "GCAM_scenario")) %>%
     rename(Region = region) %>%
     #  rename(Model = ?..Model) %>%
-    rename(Scenario = scenario) %>%
+    rename(Scenario = scenario)
+
+  # Add year columns if not present
+  missing_cols <- setdiff(reporting_columns.global, colnames(report_pre))
+  for (col in missing_cols) {
+    report_pre <- report_pre %>% mutate(!!col := NA)
+  }
+
+  report <<- report_pre %>%
     select(all_of(reporting_columns.global)) %>%
     filter(!is.na(Region)) # Drop variables we don't report
 

R/main.R

@@ -95,16 +95,7 @@ load_project <- function(project_path, desired_regions = "All", scenarios = NULL
   # load the project
   prj <- loadProject(project_path)
 
-  # filter the regions if not all of them are considered (desired_regions != 'All')
-  if (!(identical(desired_regions, "All"))) {
-    # for all scenarios in prj
-    for (s in names(prj)) {
-      # for all variables in prj
-      for (v in names(prj[[s]])) {
-        prj[[s]][[v]] <- filter_loading_regions(prj[[s]][[v]], desired_regions, v)
-      }
-    }
-  }
+  # check the scnarios are present in the prj
   if (is.null(scenarios)) {
     scenarios.global <<- listScenarios(prj)
   } else {
@@ -122,6 +113,17 @@ load_project <- function(project_path, desired_regions = "All", scenarios = NULL
     }
   }
 
+  # filter the regions if not all of them are considered (desired_regions != 'All')
+  if (!(identical(desired_regions, "All"))) {
+    # for all scenarios in prj
+    for (s in names(prj)) {
+      # for all variables in prj
+      for (v in names(prj[[s]])) {
+        prj[[s]][[v]] <- filter_loading_regions(prj[[s]][[v]], desired_regions, v)
+      }
+    }
+  }
+
   prj <<- prj
 }
 

inst/extdata/mappings/CO2market_frag_map.csv

@@ -1,5 +1,6 @@
 # mapping file for CO2 markets fragmentation,
 market,var,sector
+CO2,Price|Carbon,Energy and Industrial Processes
 CO2,Price|Carbon|Energy,Energy and Industrial Processes
 CO2,Price|Carbon|Energy|Demand|Industry,Industry
 CO2,Price|Carbon|Energy|Demand|Residential and Commercial,Residential and Commercial

inst/extdata/mappings/energy_prices_map.csv

@@ -19,10 +19,4 @@ regional biomass,,Price|Primary Energy,Price|Primary Energy|Biomass,,,,,,,1
 regional coal,,Price|Primary Energy,Price|Primary Energy|Coal,,,,,,,1
 regional natural gas,,Price|Primary Energy,Price|Primary Energy|Gas,,,,,,,1
 regional oil,,Price|Primary Energy,Price|Primary Energy|Oil,,,,,,,1
-electricity,,Price|Secondary Energy,Price|Secondary Energy|Electricity,,,,,,,1
-gas processing,natural gas,Price|Secondary Energy,Price|Secondary Energy|Gases,Price|Secondary Energy|Gases|Gas,,,,,,1
-H2 central production,,Price|Secondary Energy,Price|Secondary Energy|Hydrogen,,,,,,,1
-refining,biomass liquids,Price|Secondary Energy,Price|Secondary Energy|Liquids,Price|Secondary Energy|Liquids|Biomass,,,,,,1
-refining,oil refining,Price|Secondary Energy,Price|Secondary Energy|Liquids,Price|Secondary Energy|Liquids|Oil,,,,,,1
-delivered biomass,,Price|Secondary Energy,Price|Secondary Energy|Solids,Price|Secondary Energy|Solids|Biomass,,,,,,1
-delivered coal,,Price|Secondary Energy,Price|Secondary Energy|Solids,Price|Secondary Energy|Solids|Coal,,,,,,1
+elect_td_ind,,Price|Secondary Energy,Price|Secondary Energy|Electricity,,,,,,,1

inst/extdata/template/reporting_template.csv

@@ -854,6 +854,7 @@
 "GCAM 7.0","Price|Agriculture|Livestock|Food|SheepGoat|Index","Index (2005 = 1)","ag_prices_clean"
 "GCAM 7.0","Price|Agriculture|Livestock|Other|Index","Index (2005 = 1)","ag_prices_clean"
 "GCAM 7.0","Price|Carbon","US$2010/t","co2_price_clean"
+"GCAM 7.0","Price|Carbon|Energy","US$2010/t","co2_price_clean"
 "GCAM 7.0","Price|Carbon|Energy|Demand|Industry","US$2010/t CO2","co2_price_clean"
 "GCAM 7.0","Price|Carbon|Energy|Demand|Residential and Commercial","US$2010/t CO2","co2_price_clean"
 "GCAM 7.0","Price|Carbon|Energy|Demand|Transportation","US$2010/t CO2","co2_price_clean"
@@ -905,22 +906,22 @@
 "GCAM 7.0","Price|Secondary Energy","US$2010/GJ","energy_price_clean"
 "GCAM 7.0","Price|Secondary Energy|Electricity","US$2010/GJ","energy_price_clean"
 "GCAM 7.0","Price|Secondary Energy|Electricity|Index","Index (2020 = 1)","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Gases|Gas","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Gases|Index","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Gases|Gas|Index","Index (2020 = 1)","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Hydrogen","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Liquids","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Liquids|Index","Index (2020 = 1)","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Liquids|Biomass","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Liquids|Biomass|Index","Index (2020 = 1)","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Liquids|Oil","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Liquids|Oil|Index","Index (2020 = 1)","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Solids","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Solids|Index","Index (2020 = 1)","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Solids|Biomass","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Solids|Biomass|Index","Index (2020 = 1)","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Solids|Coal","US$2010/GJ","energy_price_clean"
-"GCAM 7.0","Price|Secondary Energy|Solids|Coal|Index","Index (2020 = 1)","energy_price_clean"
+"GCAM 7.0","Price|Secondary Energy|Gases|Gas","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Gases|Index","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Gases|Gas|Index","Index (2020 = 1)",""
+"GCAM 7.0","Price|Secondary Energy|Hydrogen","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Liquids","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Liquids|Index","Index (2020 = 1)",""
+"GCAM 7.0","Price|Secondary Energy|Liquids|Biomass","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Liquids|Biomass|Index","Index (2020 = 1)",""
+"GCAM 7.0","Price|Secondary Energy|Liquids|Oil","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Liquids|Oil|Index","Index (2020 = 1)",""
+"GCAM 7.0","Price|Secondary Energy|Solids","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Solids|Index","Index (2020 = 1)",""
+"GCAM 7.0","Price|Secondary Energy|Solids|Biomass","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Solids|Biomass|Index","Index (2020 = 1)",""
+"GCAM 7.0","Price|Secondary Energy|Solids|Coal","US$2010/GJ",""
+"GCAM 7.0","Price|Secondary Energy|Solids|Coal|Index","Index (2020 = 1)",""
 "GCAM 7.0","Primary Energy","EJ/yr","primary_energy_clean"
 "GCAM 7.0","Primary Energy|Biomass","EJ/yr","primary_energy_clean"
 "GCAM 7.0","Primary Energy|Biomass|1st Generation","EJ/yr","primary_energy_clean"