update zenodo archive data to latest FAOSTAT prepare for the PR and r…

…elase

R/constants.R

@@ -5,7 +5,7 @@
 # Key parameters
 # If TRUE, process raw FAO data
 # If FALSE, use prebuilt data to load FAO data
-Process_Raw_FAO_Data <- FALSE
+Process_Raw_FAO_Data <- TRUE
 
 # OUTPUT_Export_CSV option moved to driver_drake as write_csv_model
 # The default is NULL
@@ -31,15 +31,15 @@ DIR_RAW_DATA_FAOSTAT <- "FAOSTAT"
 
 # Historical years of focus ----
 #*******************************************
-FAOSTAT_Hist_Year <- seq(1970, 2021)
+FAOSTAT_Hist_Year <- seq(1973, 2022)
 #Bilateral trade year starts from 1986 but higher quality after 1992
 #FAOSTAT_Hist_Year_Bilateral <- seq(1992, 2020)
-FAOSTAT_Hist_Year_TMBilateral <- seq(2010, 2021)
-FAOSTAT_Hist_Year_TCL <- seq(1973, 2021)
+FAOSTAT_Hist_Year_TMBilateral <- seq(2010, 2022)
+FAOSTAT_Hist_Year_TCL <- seq(1973, 2022)
 FAOSTAT_Hist_Year_FBSH <- seq(1973, 2013)
-FAOSTAT_Hist_Year_FBS <- seq(2010, 2021) # New FBS years
+FAOSTAT_Hist_Year_FBS <- seq(2010, 2022) # New FBS years
 MIN_HIST_PP_YEAR = 2010 # first producer price year
-Hist_MEAN_Year_NUTRIENT_MASS_CONV <- 2010:2021 # average cal per g
+Hist_MEAN_Year_NUTRIENT_MASS_CONV <- 2010:2022 # average cal per g
 
 
 

R/xfaostat_L101_RawDataPreProc1_QCL.R

@@ -82,7 +82,11 @@ module_xfaostat_L101_RawDataPreProc1_QCL <- function(command, ...) {
       # All Coir (coconut fiber; item_code == 813) was previously filtered out due to NA, but now available for a few regions
       filter(!is.na(value)) %>%
       # remove accent
-      rm_accent("item", "area") -> QCL1
+      rm_accent("item", "area") %>%
+      # need to ensure all area names came from the same source! 107 and 223
+      mutate(area = replace(area, area == "CA?te dIvoire", "Cote dIvoire"),
+             area = replace(area, area == "TA?rkiye", "Turkiye"))-> QCL1
+
 
     QCL1 %>% spread(year, value) ->
       QCL_wide

R/xfaostat_L101_RawDataPreProc7_FO.R

@@ -53,7 +53,7 @@ module_xfaostat_L101_RawDataPreProc7_FO <- function(command, ...) {
       FO %>% filter(year >= min(FAOSTAT_Hist_Year),
                     area_code < 350,     # Rm aggregated area
                     item_code %in% c(1861, 1864, 1865, 2038, 1868, 1871,
-                                     1634, 1873, 1872, 1875)) %>%
+                                     1634, 1873, 1872, 1875, 1876)) %>%
       # see meta data in https://www.fao.org/faostat/en/#data/FO
       # 1861	Roundwood
       # 1864	Wood fuel
@@ -65,6 +65,7 @@ module_xfaostat_L101_RawDataPreProc7_FO <- function(command, ...) {
       # 1873	Wood-based panels
       # 1872	Sawnwood
       # 1875	Wood pulp
+      # 1876  Paper and paperboard
       select(area_code,
              area,
              item_code,

R/xfaostat_L105_DataConnectionToSUA.R

@@ -85,7 +85,8 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) {
 
     SCL %>% distinct(element)
     # Update SCL element name for convenience
-    SCL %>% mutate(element = gsub(" Quantity| supply quantity \\(tonnes\\)| \\(non-food\\)", "", element)) ->
+    # Will need to update element with element_code for better improvemence
+    SCL %>% mutate(element = gsub(" supply quantity \\(tonnes\\)| \\(non-food\\)| quantity| Quantity", "", element)) ->
       SCL
     SCL_element_new <-
       c("Opening stocks", "Production", "Export", "Import", "Stock Variation",
@@ -138,7 +139,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) {
       # keep only balance items
       filter(!element_code %in% c(645, 664, 674, 684)) %>%
       # simplify elements and make them consistent with SUA
-      mutate(element = gsub(" Quantity| supply quantity \\(tonnes\\)| \\(non-food\\)", "", element),
+      mutate(element = gsub("supply quantity \\(tonnes\\)| \\(non-food\\)| quantity| Quantity", "", element),
              element = replace(element, element == "Losses", "Loss"),
              element = replace(element, element == "Processing", "Processed")) %>%
       # convert units back to tonnes first since FBS originally used 1000 tons

R/xfaostat_L201_Forestry.R

@@ -42,14 +42,15 @@ module_xfaostat_L201_Forestry <- function(command, ...) {
 
     ## Proprocess and quick clean ----
 
-    c(1865, 1634, 1873, 872, 1875) -> Key_FO_Items
+    c(1865, 1634, 1873, 1872, 1875, 1876) -> Key_FO_Items
     # 1865	Industrial roundwood
     # 1634	Veneer sheets
     # 1873	Wood-based panels
     # 1872	Sawnwood
     # 1875	Wood pulp
+    # 1876  Paper and paperboard
 
-    FO_RoundwoodProducts %>% filter(item_code %in% c(1865, 1634, 1873, 872, 1875)) ->
+    FO_RoundwoodProducts %>% filter(item_code %in% c(1865, 1634, 1873, 1872, 1875, 1876)) ->
       L201.FO_RoundwoodProducts
 
 

R/xfaostat_helper_funcs.R

@@ -145,14 +145,15 @@ FF_download_FAOSTAT <- function(DATASETCODE,
 
 FF_download_RemoteArchive <-
   function(DATASETCODE = NULL,
-           RemoteArchiveURL = "https://zenodo.org/record/11602356/files/",
+           RemoteArchiveURL = "https://zenodo.org/record/13941470/files/",
            DATA_FOLDER = file.path("inst/extdata", DIR_RAW_DATA_FAOSTAT),
            OverWrite = FALSE){
 
 
 
     # version v1.0.0 (August, 2022): "https://zenodo.org/record/8260225/files/"
-    # version v1.0.1 (June, 2024):"https://zenodo.org/record/11602356/files/"
+    # version v1.0.1_temp (June, 2024):"https://zenodo.org/record/11602356/files/"
+    # version v1.0.1 (October, 2024):"https://zenodo.org/record/13941470/files/"
 
     warnings("The current archive is for gcamfaostat v1.0.1")
 

R/yfaostat_GCAM_CSVExport.R

@@ -190,7 +190,7 @@ module_yfaostat_GCAM_CSVExport <- function(command, ...) {
         FBSH_CB
 
       FBSH_CB %>%
-        mutate(unit = "1000 tonnes", value = value / 1000) %>%
+        mutate(unit = "1000 tonnes", value = value) %>%
         filter(year <= min(FAOSTAT_Hist_Year_FBS) - 1)  %>%
         filter(!is.na(year)) %>%
         spread(year, value) %>%