Parallelize downloads and uploads (#174)

* Parallelize downloads and uploads * Add Concurrently flag * Use Concurrently flag and add it to tests
tmspzz · Mar 10, 2019 · 95ae3f4 · 95ae3f4
1 parent 6f94ffe
commit 95ae3f4
Show file tree

Hide file tree

Showing 10 changed files with 153 additions and 85 deletions.
diff --git a/integration-tests/current-framework-named-yaml.bats b/integration-tests/current-framework-named-yaml.bats
@@ -86,7 +86,7 @@ teardown() {
   ls
   echo `pwd`
 
-  run rome upload --cache-prefix travis ${FRAMEWORK_REPO_NAME}
+  run rome upload --concurrently --cache-prefix travis ${FRAMEWORK_REPO_NAME}
 
   [ "$status" -eq 0 ]
 
@@ -150,7 +150,7 @@ teardown() {
   sleep 4 
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME} 
+  run rome download --concurrently --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME} 
 
   [ "$status" -eq 0 ]
 
@@ -188,7 +188,7 @@ teardown() {
   fi
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis ${FRAMEWORK_REPO_NAME} 
+  run rome download --concurrently --cache-prefix travis ${FRAMEWORK_REPO_NAME} 
 
   [ "$status" -eq 0 ]
 
@@ -229,7 +229,7 @@ teardown() {
   sleep 4 
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME}
+  run rome download --concurrently --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME}
 
   [ "$status" -eq 0 ]
 

diff --git a/integration-tests/current-framework-yaml.bats b/integration-tests/current-framework-yaml.bats
@@ -82,7 +82,7 @@ teardown() {
   ls
   echo `pwd`
 
-  run rome upload --cache-prefix travis --no-skip-current
+  run rome upload --concurrently --cache-prefix travis --no-skip-current
 
   [ "$status" -eq 0 ]
 
@@ -146,7 +146,7 @@ teardown() {
   sleep 4 
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --skip-local-cache --no-skip-current
+  run rome download --concurrently --cache-prefix travis --skip-local-cache --no-skip-current
 
   [ "$status" -eq 0 ]
 
@@ -184,7 +184,7 @@ teardown() {
   fi
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --no-skip-current
+  run rome download --concurrently --cache-prefix travis --no-skip-current
 
   [ "$status" -eq 0 ]
 
@@ -222,7 +222,7 @@ teardown() {
   ls
   echo `pwd`
 
-  run rome upload --cache-prefix travis --no-skip-current Alamofire
+  run rome upload --concurrently --cache-prefix travis --no-skip-current Alamofire
 
   [ "$status" -eq 0 ]
 
@@ -286,7 +286,7 @@ teardown() {
   sleep 4 
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --skip-local-cache --no-skip-current Alamofire
+  run rome download --concurrently --cache-prefix travis --skip-local-cache --no-skip-current Alamofire
 
   [ "$status" -eq 0 ]
 

diff --git a/integration-tests/dynamic-frameworks-ini.bats b/integration-tests/dynamic-frameworks-ini.bats
@@ -68,7 +68,7 @@ teardown() {
   MINIO_HTTP_TRACE=output.log minio server minio-buckets &
   sleep 4 
 
-  run rome upload --cache-prefix travis
+  run rome upload --concurrently --cache-prefix travis
 
   if [ -d "minio-buckets/rome" ]; then
     cp -R minio-buckets/rome/ ../_rome_bkp
@@ -130,7 +130,7 @@ teardown() {
   sleep 4 
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --skip-local-cache
+  run rome download --concurrently --cache-prefix travis --skip-local-cache
 
   [ "$status" -eq 0 ]
 
@@ -166,7 +166,7 @@ teardown() {
   fi
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis
+  run rome download --concurrently --cache-prefix travis
 
   [ "$status" -eq 0 ]
 

diff --git a/integration-tests/dynamic-frameworks-yml.bats b/integration-tests/dynamic-frameworks-yml.bats
@@ -90,7 +90,7 @@ teardown() {
   MINIO_HTTP_TRACE=output.log minio server minio-buckets &
   sleep 4 
 
-  run rome upload --cache-prefix travis
+  run rome upload --concurrently --cache-prefix travis
 
   if [ -d "minio-buckets/rome" ]; then
     cp -R minio-buckets/rome/ ../_rome_bkp
@@ -187,7 +187,7 @@ teardown() {
   sleep 4 
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --skip-local-cache
+  run rome download --concurrently --cache-prefix travis --skip-local-cache
 
   [ "$status" -eq 0 ]
 
@@ -245,7 +245,7 @@ teardown() {
   fi
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis
+  run rome download --concurrently --cache-prefix travis
 
   [ "$status" -eq 0 ]
 

diff --git a/integration-tests/static-frameworks-ini.bats b/integration-tests/static-frameworks-ini.bats
@@ -63,7 +63,7 @@ teardown() {
   MINIO_HTTP_TRACE=output.log minio server minio-buckets &
   sleep 4 
 
-  run rome upload --cache-prefix travis
+  run rome upload --concurrently --cache-prefix travis
 
   if [ -d "minio-buckets/rome" ]; then
     cp -R minio-buckets/rome/ ../_rome_bkp
@@ -111,7 +111,7 @@ teardown() {
   sleep 4 
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --skip-local-cache
+  run rome download --concurrently --cache-prefix travis --skip-local-cache
 
   [ "$status" -eq 0 ]
 
@@ -141,7 +141,7 @@ teardown() {
   fi
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis
+  run rome download --concurrently --cache-prefix travis
 
   [ "$status" -eq 0 ]
 

diff --git a/integration-tests/static-frameworks-yml.bats b/integration-tests/static-frameworks-yml.bats
@@ -82,7 +82,7 @@ teardown() {
   MINIO_HTTP_TRACE=output.log minio server minio-buckets &
   sleep 4 
 
-  run rome upload --cache-prefix travis --romefile ../Romefile.yaml
+  run rome upload --concurrently --cache-prefix travis --romefile ../Romefile.yaml
 
   if [ -d "minio-buckets/rome" ]; then
     cp -R minio-buckets/rome/ ../_rome_bkp
@@ -144,7 +144,7 @@ teardown() {
   sleep 4 
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --skip-local-cache --romefile ../Romefile.yaml
+  run rome download --concurrently --cache-prefix travis --skip-local-cache --romefile ../Romefile.yaml
 
   [ "$status" -eq 0 ]
 
@@ -185,7 +185,7 @@ teardown() {
   fi
 
   rm -rf Carthage/Build
-  run rome download --cache-prefix travis --romefile ../Romefile.yaml
+  run rome download --concurrently --cache-prefix travis --romefile ../Romefile.yaml
 
   [ "$status" -eq 0 ]
 

diff --git a/src/CommandParsers.hs b/src/CommandParsers.hs
@@ -49,6 +49,13 @@ noSkipCurrentParser = NoSkipCurrentFlag <$> Opts.switch
        "Do not skip the `currentMap` section in the Romefile when performing the operation."
   )
 
+concurrentlyParser :: Opts.Parser ConcurrentlyFlag
+concurrentlyParser = ConcurrentlyFlag <$> Opts.switch
+  (  Opts.long "concurrently"
+  <> Opts.help
+       "Maximise concurrency while performing the operation. Might make verbose output hard to follow."
+  )
+
 reposParser :: Opts.Parser [ProjectName]
 reposParser = Opts.many
   (Opts.argument
@@ -88,6 +95,7 @@ udcPayloadParser =
     <*> skipLocalCacheParser
     <*> noIgnoreParser
     <*> noSkipCurrentParser
+    <*> concurrentlyParser
 
 uploadParser :: Opts.Parser RomeCommand
 uploadParser = pure Upload <*> udcPayloadParser