diff --git a/R/Database.R b/R/Database.R index 69a137a..71eafed 100644 --- a/R/Database.R +++ b/R/Database.R @@ -303,12 +303,37 @@ exportDatabaseToCsv <- function( ) } + # max number of rows extracted at a time + maxRowCount <- 100000 + # get the table names using the function in uploadToDatabase.R tables <- getResultTables() # extract result per table for(table in tables){ - sql <- "select * from @resultSchema.@appendtotable@tablename" + + ParallelLogger::logInfo(paste0('Exporting rows from ', table, ' to csv file')) + # get row count and figure out number of loops + sql <- "select count(*) as N from @resultSchema.@appendtotable@tablename;" + sql <- SqlRender::render( + sql = sql, + resultSchema = resultSchema, + appendtotable = tablePrefix, + tablename = table + ) + sql <- SqlRender::translate( + sql = sql, + targetDialect = targetDialect, + tempEmulationSchema = tempEmulationSchema + ) + countN <- DatabaseConnector::querySql( + connection = connection, + sql = sql, + snakeCaseToCamelCase = F + )$N + + # get column names + sql <- "select * from @resultSchema.@appendtotable@tablename where 1=0;" sql <- SqlRender::render( sql = sql, resultSchema = resultSchema, @@ -320,6 +345,43 @@ exportDatabaseToCsv <- function( targetDialect = targetDialect, tempEmulationSchema = tempEmulationSchema ) + cnames <- colnames(DatabaseConnector::querySql( + connection = connection, + sql = sql, + snakeCaseToCamelCase = F + )) + + inds <- floor(countN/maxRowCount) + tableAppend = F + pb = utils::txtProgressBar(min = 0, max = countN, initial = 0) + + for(i in 1:length(inds)){ + + startRow <- (i-1)*maxRowCount + 1 + endRow <- min(i*maxRowCount, countN) + + sql <- "select @cnames from + (select *, + ROW_NUMBER() OVER(ORDER BY @cnames) AS row + from @resultSchema.@appendtotable@tablename + ) temp + where + temp.row >= @start_row and + temp.row <= @end_row;" + sql <- SqlRender::render( + sql = sql, + resultSchema = resultSchema, + appendtotable = tablePrefix, + tablename = table, + cnames = paste(cnames, collapse = ','), + start_row = startRow, + end_row = endRow + ) + sql <- SqlRender::translate( + sql = sql, + targetDialect = targetDialect, + tempEmulationSchema = tempEmulationSchema + ) result <- DatabaseConnector::querySql( connection = connection, sql = sql, @@ -330,8 +392,14 @@ exportDatabaseToCsv <- function( # save the results as a csv readr::write_csv( x = result, - file = file.path(saveDirectory, paste0(tolower(filePrefix), table,'.csv')) + file = file.path(saveDirectory, paste0(tolower(filePrefix), table,'.csv')), + append = tableAppend ) + tableAppend = T + utils::setTxtProgressBar(pb,endRow) + + } + close(pb) } invisible(saveDirectory)