Update Database.R

- adding code to export in batches

R/Database.R

@@ -303,12 +303,37 @@ exportDatabaseToCsv <- function(
     )
   }
 
+  # max number of rows extracted at a time
+  maxRowCount <- 100000
+
   # get the table names using the function in uploadToDatabase.R
   tables <- getResultTables()
 
   # extract result per table
   for(table in tables){
-    sql <- "select * from @resultSchema.@appendtotable@tablename"
+
+    ParallelLogger::logInfo(paste0('Exporting rows from ', table, ' to csv file'))
+    # get row count and figure out number of loops
+    sql <- "select count(*) as N from @resultSchema.@appendtotable@tablename;"
+    sql <- SqlRender::render(
+      sql = sql,
+      resultSchema = resultSchema,
+      appendtotable = tablePrefix,
+      tablename = table
+    )
+    sql <- SqlRender::translate(
+      sql = sql,
+      targetDialect = targetDialect,
+      tempEmulationSchema = tempEmulationSchema
+    )
+    countN <- DatabaseConnector::querySql(
+      connection = connection,
+      sql = sql,
+      snakeCaseToCamelCase = F
+    )$N
+
+    # get column names
+    sql <- "select * from @resultSchema.@appendtotable@tablename where 1=0;"
     sql <- SqlRender::render(
       sql = sql,
       resultSchema = resultSchema,
@@ -320,6 +345,43 @@ exportDatabaseToCsv <- function(
       targetDialect = targetDialect,
       tempEmulationSchema = tempEmulationSchema
     )
+    cnames <- colnames(DatabaseConnector::querySql(
+      connection = connection,
+      sql = sql,
+      snakeCaseToCamelCase = F
+    ))
+
+    inds <- floor(countN/maxRowCount)
+    tableAppend = F
+    pb = utils::txtProgressBar(min = 0, max = countN, initial = 0)
+
+    for(i in 1:length(inds)){
+
+      startRow <- (i-1)*maxRowCount + 1
+      endRow <- min(i*maxRowCount, countN)
+
+    sql <- "select @cnames from
+    (select *,
+    ROW_NUMBER() OVER(ORDER BY @cnames) AS row
+    from @resultSchema.@appendtotable@tablename
+    ) temp
+    where
+    temp.row >= @start_row and
+    temp.row <= @end_row;"
+    sql <- SqlRender::render(
+      sql = sql,
+      resultSchema = resultSchema,
+      appendtotable = tablePrefix,
+      tablename = table,
+      cnames = paste(cnames, collapse = ','),
+      start_row = startRow,
+      end_row = endRow
+    )
+    sql <- SqlRender::translate(
+      sql = sql,
+      targetDialect = targetDialect,
+      tempEmulationSchema = tempEmulationSchema
+    )
     result <- DatabaseConnector::querySql(
       connection = connection,
       sql = sql,
@@ -330,8 +392,14 @@ exportDatabaseToCsv <- function(
     # save the results as a csv
     readr::write_csv(
       x = result,
-      file = file.path(saveDirectory, paste0(tolower(filePrefix), table,'.csv'))
+      file = file.path(saveDirectory, paste0(tolower(filePrefix), table,'.csv')),
+      append = tableAppend
     )
+    tableAppend = T
+    utils::setTxtProgressBar(pb,endRow)
+
+    }
+    close(pb)
   }
 
   invisible(saveDirectory)