From 7e8eaf64aed3f1f785a6baacdf39a9bf32823288 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Tue, 17 Sep 2024 13:22:39 -0700 Subject: [PATCH 1/9] Update 04_Installing-PEcAn-OSX.Rmd - primary goal was to add DuckDB - also added brew install instructions Q: is there a reason to keep all of the instructions for manually downloading and compiling? Some of these provide specific versions; 1) it isn't clear if this is desired and 2) brew makes it much easier! --- .../03_install_OS/04_Installing-PEcAn-OSX.Rmd | 110 ++++++++++++++++-- 1 file changed, 100 insertions(+), 10 deletions(-) diff --git a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd index d057ea41246..a18bff06231 100755 --- a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd +++ b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd @@ -7,6 +7,26 @@ This document also contains information on how to install the Rstudio server edi #### Install build environment +##### Option 1: Homebrew + +```bash +# R +brew install r +# gfortran +brew install gcc +# OpenMPI +brew install open-mpi +# szip +brew install szip +# HDF5 +brew install hdf5 +## homebrew should configure hdf5 with fortran and cxx, otherwise: +## brew install hdf5 --with-fortran --with-cxx +``` + + +##### Option 2: Download and install + ```bash # install R # download from http://cran.r-project.org/bin/macosx/ @@ -42,14 +62,42 @@ sudo make install cd .. ``` -#### Install Postgres +#### Install Postgres and PostGIS + +##### Option 1: + +For MacOS, the Postgres.app provides Postgres with PostGIS +already installed (http://postgresapp.com/). + +To run Postgres: +* Open Postgres.app. +* In the menu bar, click the elephant icon and select “Open psql”. + +##### Option 2: install using homebrew: + +```bash +# optional: remove existing postgres installations with: +# brew uninstall --force postgresql + +# install postgres and postgis: +brew install postgres +brew install postgis + +# to run Postgres: +brew services start postgresql +``` -For those on a Mac I use the following app for postgresql which has -postgis already installed (http://postgresapp.com/) +#### Enable PostGIS -To get postgis run the following commands in psql: +To enable PostGIS, you should start Postgres: ```bash +psql postgres +``` + +And then run the following commands: + +```sql ##### Enable PostGIS (includes raster) CREATE EXTENSION postgis; ##### Enable Topology @@ -60,23 +108,42 @@ CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION postgis_tiger_geocoder; ``` -To check your postgis run the following command again in psql: `SELECT PostGIS_full_version();` +To check your postgis run the following command again in psql: + +```sql +SELECT PostGIS_full_version();` +``` #### Additional installs ##### Install JAGS -Download JAGS from http://sourceforge.net/projects/mcmc-jags/files/JAGS/3.x/Mac%20OS%20X/JAGS-Mavericks-3.4.0.dmg/download + +##### Option 1: using homebrew + +```bash +brew install jags +``` + +##### Option 2: Download + +Download JAGS from http://sourceforge.net/projects/mcmc-jags/files/JAGS/3.x/Mac%20OS%20X/JAGS-Mavericks-3.4.0.dmg/download. + ##### Install udunits -Installing udunits-2 on MacOSX is done from source. +##### Option 1: Install using homebrew: + +```bash +brew install udunits +``` + +##### Option 2: Install udunits-2 on MacOSX is done from source. * download most recent [version of Udunits here](http://www.unidata.ucar.edu/downloads/udunits/index.jsp) * instructions for [compiling from source](http://www.unidata.ucar.edu/software/udunits/udunits-2/udunits2.html#Obtain) - ```bash curl -o udunits-2.1.24.tar.gz ftp://ftp.unidata.ucar.edu/pub/udunits/udunits-2.1.24.tar.gz tar zxf udunits-2.1.24.tar.gz @@ -86,6 +153,19 @@ make sudo make install ``` +#### Install DuckDB + +##### Option 1: Install using homebrew + +```bash +brew install duckdb +``` + +##### Option 2: Download and install: + +For manual installation instructions, see [DuckDB installation instructions](https://duckdb.org/docs/installation/?version=stable&environment=cli&platform=macos&download_method=direct). + + #### Apache Configuration Mac does not support pdo/postgresql by default. The easiest way to install is use: http://php-osx.liip.ch/ @@ -104,8 +184,18 @@ EOF #### Ruby -The default version of ruby should work. Or use [JewelryBox](https://jewelrybox.unfiniti.com/). +The default version of Ruby should work. Or use [JewelryBox](https://jewelrybox.unfiniti.com/). + +```bash +brew install ruby +``` #### Rstudio Server -For the mac you can download [Rstudio Desktop](http://www.rstudio.com/). +For MacOS, you can download [Rstudio Desktop](http://www.rstudio.com/). + +Or using homebrew: + +```bash +brew install --cask rstudio +``` From 25bbb5c58731e7f533d1be7c82b311c9d095812c Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Tue, 17 Sep 2024 13:34:50 -0700 Subject: [PATCH 2/9] Update 04_Installing-PEcAn-OSX.Rmd Additional edits, Jewelry box is deprecated https://github.com/remear/jewelrybox/issues/215 --- .../03_install_OS/04_Installing-PEcAn-OSX.Rmd | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd index a18bff06231..44f482ac877 100755 --- a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd +++ b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd @@ -1,9 +1,10 @@ ### Mac OSX {#macosx} -These are specific notes for installing PEcAn on Mac OSX and will be referenced from the main [installing PEcAn](Installing-PEcAn) page. You will at least need to install the build environment and Postgres sections. If you want to access the database/PEcAn using a web browser you will need to install Apache. To access the database using the BETY interface, you will need to have Ruby installed. +These are specific notes for installing PEcAn on Mac OSX and referenced from the [installing PEcAn](Installing-PEcAn) page. -This document also contains information on how to install the Rstudio server edition as well as any other packages that can be helpful. +Build environment and Postgres sections are required to install and use PEcAn. +Optional software includes Apache, Rails, and Rstudio. Apache is required to run the BETYdb UI and the web-based version of PEcAn - both are optional. Rails is requred to use the BETYdb web interface. Rstudio is a commonly used IDE for R. #### Install build environment @@ -166,7 +167,7 @@ brew install duckdb For manual installation instructions, see [DuckDB installation instructions](https://duckdb.org/docs/installation/?version=stable&environment=cli&platform=macos&download_method=direct). -#### Apache Configuration +#### Apache Configuration (Optional) Mac does not support pdo/postgresql by default. The easiest way to install is use: http://php-osx.liip.ch/ @@ -182,15 +183,19 @@ Alias /pecan ${PWD}/pecan/web EOF ``` -#### Ruby +#### Ruby (Optional) -The default version of Ruby should work. Or use [JewelryBox](https://jewelrybox.unfiniti.com/). +_Note: it is recommended that BETYdb be run using Docker because the application uses unsupported versions of Ruby and Rails._ + +The BETYdb application requires Ruby version 2.7.7, as specified in [PecanProject/bety/.ruby-version](https://github.com/PecanProject/bety/blob/develop/.ruby-version). ```bash -brew install ruby +brew install rbenv +rbenv init +rbenv install 2.7.7 ``` -#### Rstudio Server +#### Rstudio (Optional) For MacOS, you can download [Rstudio Desktop](http://www.rstudio.com/). From 74f1d4149481b191bf548fbf7790d954df1292cc Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Tue, 17 Sep 2024 14:03:39 -0700 Subject: [PATCH 3/9] =?UTF-8?q?manually=20updated=20docs=20that=20should?= =?UTF-8?q?=20have=20gone=20with=20e31bc0aa6bd0d461b776=E2=80=A6=20(#3384)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * manually updated docs that should have gone with e31bc0aa6bd0d461b776c22f47cbd9e6e4f370bb * typo --- modules/uncertainty/man/read.sa.output.Rd | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/uncertainty/man/read.sa.output.Rd b/modules/uncertainty/man/read.sa.output.Rd index 06a0f07196f..a6c252ce23a 100644 --- a/modules/uncertainty/man/read.sa.output.Rd +++ b/modules/uncertainty/man/read.sa.output.Rd @@ -49,6 +49,4 @@ Reads output of sensitivity analysis runs } \author{ Ryan Kelly, David LeBauer, Rob Kooper, Mike Dietze, Istem Fer - -Ryan Kelly, David LeBauer, Rob Kooper, Mike Dietze } From d7f15209d1b247150c5eaded92084afc36c027df Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Tue, 17 Sep 2024 17:17:04 -0700 Subject: [PATCH 4/9] remove install duckdb as dependency --- .../03_install_OS/04_Installing-PEcAn-OSX.Rmd | 8 -------- 1 file changed, 8 deletions(-) diff --git a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd index 44f482ac877..fa47a0b352f 100755 --- a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd +++ b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd @@ -154,14 +154,6 @@ make sudo make install ``` -#### Install DuckDB - -##### Option 1: Install using homebrew - -```bash -brew install duckdb -``` - ##### Option 2: Download and install: For manual installation instructions, see [DuckDB installation instructions](https://duckdb.org/docs/installation/?version=stable&environment=cli&platform=macos&download_method=direct). From 92881ba6a504ab5dfcc6d2326692e7ed1c8ae55d Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Tue, 17 Sep 2024 17:17:31 -0700 Subject: [PATCH 5/9] Update 04_Installing-PEcAn-OSX.Rmd --- .../03_install_OS/04_Installing-PEcAn-OSX.Rmd | 5 ----- 1 file changed, 5 deletions(-) diff --git a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd index fa47a0b352f..ef99efa21a6 100755 --- a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd +++ b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd @@ -154,11 +154,6 @@ make sudo make install ``` -##### Option 2: Download and install: - -For manual installation instructions, see [DuckDB installation instructions](https://duckdb.org/docs/installation/?version=stable&environment=cli&platform=macos&download_method=direct). - - #### Apache Configuration (Optional) Mac does not support pdo/postgresql by default. The easiest way to install is use: http://php-osx.liip.ch/ From f22a7c4bbc532e4551f7bc9624cef649da317ac1 Mon Sep 17 00:00:00 2001 From: Chris Black Date: Tue, 17 Sep 2024 17:19:50 -0700 Subject: [PATCH 6/9] double-equal is a bashism and Actions steps can secretly run as sh (#3380) Co-authored-by: David LeBauer --- .github/workflows/book.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 240ebd88774..14e14c75d06 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -65,7 +65,7 @@ jobs: git config --global user.email "pecanproj@gmail.com" git config --global user.name "GitHub Documentation Robot" export VERSION=$(echo $GITHUB_REF | sed 's,.*/,,' ) - if [ "$VERSION" == "main" ]; then + if [ "$VERSION" = "main" ]; then export VERSION=latest fi cd pecan-documentation From 043f236ab7692ee04086e48760429b1c77870203 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Tue, 17 Sep 2024 17:32:26 -0700 Subject: [PATCH 7/9] Update 04_Installing-PEcAn-OSX.Rmd --- .../03_install_OS/04_Installing-PEcAn-OSX.Rmd | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd index ef99efa21a6..a37f1228cfd 100755 --- a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd +++ b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd @@ -8,33 +8,17 @@ Optional software includes Apache, Rails, and Rstudio. Apache is required to run #### Install build environment -##### Option 1: Homebrew +##### Step 1: Install R -```bash -# R -brew install r -# gfortran -brew install gcc -# OpenMPI -brew install open-mpi -# szip -brew install szip -# HDF5 -brew install hdf5 -## homebrew should configure hdf5 with fortran and cxx, otherwise: -## brew install hdf5 --with-fortran --with-cxx -``` +Download from +##### Option 1: Download and install -##### Option 2: Download and install +R: download from http://cran.r-project.org/bin/macosx/ -```bash -# install R -# download from http://cran.r-project.org/bin/macosx/ - -# install gfortran -# download from http://cran.r-project.org/bin/macosx/tools/ +gfortran: download from http://cran.r-project.org/bin/macosx/tools/ +```bash # install OpenMPI curl -o openmpi-1.6.3.tar.gz http://www.open-mpi.org/software/ompi/v1.6/downloads/openmpi-1.6.3.tar.gz tar zxf openmpi-1.6.3.tar.gz @@ -63,6 +47,24 @@ sudo make install cd .. ``` +##### Option 2: Homebrew + +```bash +# R +brew install --cask r +# gfortran +brew install gcc +# OpenMPI +brew install open-mpi +# szip +brew install szip +# HDF5 +brew install hdf5 +## homebrew should configure hdf5 with fortran and cxx, otherwise: +## brew install hdf5 --with-fortran --with-cxx +``` + + #### Install Postgres and PostGIS ##### Option 1: @@ -80,8 +82,11 @@ To run Postgres: # optional: remove existing postgres installations with: # brew uninstall --force postgresql -# install postgres and postgis: -brew install postgres +# install Postgres, fixed at v12 (officially supported by BETYdb): +brew install postgres@12 +brew pin postgres@12 + +# PostGIS brew install postgis # to run Postgres: From ad630cc27e02a3bfeb9dae02ded360db273aae38 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Tue, 17 Sep 2024 17:34:42 -0700 Subject: [PATCH 8/9] Update 04_Installing-PEcAn-OSX.Rmd --- .../93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd | 4 ---- 1 file changed, 4 deletions(-) diff --git a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd index a37f1228cfd..d48107a6cab 100755 --- a/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd +++ b/book_source/03_topical_pages/93_installation/03_install_OS/04_Installing-PEcAn-OSX.Rmd @@ -8,10 +8,6 @@ Optional software includes Apache, Rails, and Rstudio. Apache is required to run #### Install build environment -##### Step 1: Install R - -Download from - ##### Option 1: Download and install R: download from http://cran.r-project.org/bin/macosx/ From 09a58d24a81cfad45521207682b0e71bf4d193c9 Mon Sep 17 00:00:00 2001 From: Abhinav Pandey Date: Sat, 21 Sep 2024 13:19:44 +0530 Subject: [PATCH 9/9] Refactor `met.process` and `dbfiles` (#3319) * Refactor met.process.R to correctly handle standerdized_result Signed-off-by: Abhinav Pandey * Remove ucommented sections and replace NULL declarations with c() Signed-off-by: Abhinav Pandey * Revert to default NULL instead of c() Signed-off-by: Abhinav Pandey * Add small modification to dbfiles.R Signed-off-by: Abhinav Pandey * Fix 'return' mistake in dbfiles.R Signed-off-by: Abhinav Pandey * Omit return Type Signed-off-by: Abhinav Pandey * Update return statement in dbfiles.R Signed-off-by: Abhinav Pandey * Update invisible statements to `return` after execution Signed-off-by: Abhinav Pandey * refactor return statements Signed-off-by: Abhinav Pandey * Update base/db/R/dbfiles.R Co-authored-by: Chris Black * Apply suggestions from code review Co-authored-by: Chris Black * Apply standardization changes Signed-off-by: Abhinav Pandey --------- Signed-off-by: Abhinav Pandey Co-authored-by: Chris Black --- base/db/R/dbfiles.R | 28 ++++++++++++------------- modules/data.atmosphere/R/met.process.R | 21 ++++++++++--------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/base/db/R/dbfiles.R b/base/db/R/dbfiles.R index 4214c06837b..1cab9e069e7 100644 --- a/base/db/R/dbfiles.R +++ b/base/db/R/dbfiles.R @@ -50,9 +50,8 @@ dbfile.input.insert <- function(in.path, in.prefix, siteid, startdate, enddate, # setup parent part of query if specified - if (is.na(parentid)) { - parent <- "" - } else { + parent <- "" + if (!is.na(parentid)) { parent <- paste0(" AND parent_id=", parentid) } @@ -242,13 +241,13 @@ dbfile.input.check <- function(siteid, startdate = NULL, enddate = NULL, mimetyp formatid <- get.id(table = "formats", colnames = c("mimetype_id", "name"), values = c(mimetypeid, formatname), con = con) if (is.null(formatid)) { - invisible(data.frame()) + return (invisible(data.frame())) } # setup parent part of query if specified - if (is.na(parentid)) { - parent <- "" - } else { + parent <- "" + + if (!is.na(parentid)) { parent <- paste0(" AND parent_id=", parentid) } @@ -450,7 +449,7 @@ dbfile.posterior.check <- function(pft, mimetype, formatname, con, hostname = PE # find appropriate pft pftid <- get.id(table = "pfts", values = "name", colnames = pft, con = con) if (is.null(pftid)) { - invisible(data.frame()) + return (invisible(data.frame())) } # find appropriate format @@ -461,7 +460,7 @@ dbfile.posterior.check <- function(pft, mimetype, formatname, con, hostname = PE formatid <- get.id(table = "formats", colnames = c("mimetype_id", "name"), values = c(mimetypeid, formatname), con = con) if (is.null(formatid)) { - invisible(data.frame()) + return (invisible(data.frame())) } # find appropriate posterior @@ -473,7 +472,7 @@ dbfile.posterior.check <- function(pft, mimetype, formatname, con, hostname = PE con = con )[["id"]] if (is.null(posteriorid)) { - invisible(data.frame()) + return (invisible(data.frame())) } invisible(dbfile.check(type = "Posterior", container.id = posteriorid, con = con, hostname = hostname)) @@ -639,12 +638,12 @@ dbfile.file <- function(type, id, con, hostname = PEcAn.remote::fqdn()) { if (nrow(files) > 1) { PEcAn.logger::logger.warn("multiple files found for", id, "returned; using the first one found") - invisible(file.path(files[1, "file_path"], files[1, "file_name"])) + return(invisible(file.path(files[1, "file_path"], files[1, "file_name"]))) } else if (nrow(files) == 1) { - invisible(file.path(files[1, "file_path"], files[1, "file_name"])) + return(invisible(file.path(files[1, "file_path"], files[1, "file_name"]))) } else { PEcAn.logger::logger.warn("no files found for ", id, "in database") - invisible(NA) + return(invisible(NA)) } } @@ -662,7 +661,8 @@ dbfile.id <- function(type, file, con, hostname = PEcAn.remote::fqdn()) { # find appropriate host hostid <- db.query(query = paste0("SELECT id FROM machines WHERE hostname='", hostname, "'"), con = con)[["id"]] if (is.null(hostid)) { - invisible(NA) + PEcAn.logger::logger.warn("hostid not found in database") + return (invisible(NA)) } # find file diff --git a/modules/data.atmosphere/R/met.process.R b/modules/data.atmosphere/R/met.process.R index 1380e324744..047defde304 100644 --- a/modules/data.atmosphere/R/met.process.R +++ b/modules/data.atmosphere/R/met.process.R @@ -255,13 +255,14 @@ met.process <- function(site, input_met, start_date, end_date, model, #--------------------------------------------------------------------------------------------------# # Change to Site Level - Standardized Met (i.e. ready for conversion to model specific format) if (stage$standardize) { - standardize_result <- list() - + id_stdized <- list() + ready.id <- list(input.id = NULL, dbfile.id = NULL) + for (i in seq_along(cf.id[[1]])) { if (register$scale == "regional") { #### Site extraction - standardize_result[[i]] <- .extract.nc.module(cf.id = list(input.id = cf.id$container_id[i], + id_stdized <- .extract.nc.module(cf.id = list(input.id = cf.id$container_id[i], dbfile.id = cf.id$id[i]), register = register, dir = dir, @@ -277,7 +278,7 @@ met.process <- function(site, input_met, start_date, end_date, model, # Expand to support ensemble names in the future } else if (register$scale == "site") { ##### Site Level Processing - standardize_result[[i]] <- .metgapfill.module(cf.id = list(input.id = cf.id$input.id[i], dbfile.id = cf.id$dbfile.id[i]), + id_stdized <- .metgapfill.module(cf.id = list(input.id = cf.id$input.id[i], dbfile.id = cf.id$dbfile.id[i]), register = register, dir = dir, met = met, @@ -288,15 +289,15 @@ met.process <- function(site, input_met, start_date, end_date, model, host = host, overwrite = overwrite$standardize, ensemble_name = i) + } else { + # No action taken. These ids will be dropped from ready.id + id_stdized <- NULL } + + ready.id$input.id <- c(ready.id$input.id, id_stdized$input.id) + ready.id$dbfile.id <- c(ready.id$dbfile.id, id_stdized$dbfile.id) } # End for loop - ready.id <- list(input.id = NULL, dbfile.id = NULL) - - for (i in seq_along(standardize_result)) { - ready.id$input.id <- c(ready.id$input.id, standardize_result[[i]]$input.id) - ready.id$dbfile.id <- c(ready.id$dbfile.id, standardize_result[[i]]$dbfile.id) - } } else { ready.id <- input_met$id