From 9cfc6d6a0bc5f81f5bba9cd0707d6945c53e72ca Mon Sep 17 00:00:00 2001 From: Seth Van Buren Date: Thu, 23 May 2019 20:24:58 +1000 Subject: [PATCH] Fix issue with logger library and windows. Tidy README.md --- .gitignore | 2 ++ Makefile | 8 ++++++-- README.md | 2 +- s3kor.go | 22 ++++++++++++++++++++-- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a6aae9d..9a84bab 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ s3kor.linux s3kor.mac s3kor.linux.gz s3kor +s3kor.windows +s3kor.windows.gz diff --git a/Makefile b/Makefile index 7069543..16bfab3 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,15 @@ linux: env GOOS=linux GOARCH=amd64 go build -o s3kor.linux gzip -fk9 s3kor.linux +windows: + env GOOS=windows GOARCH=amd64 go build -o s3kor.windows + gzip -fk9 s3kor.windows + clean: - rm s3kor.linux s3kor.linux.gz + rm s3kor.linux s3kor.linux.gz s3kor.windows s3kor.windows.gz s3kor publish-test: goreleaser --snapshot --rm-dist publish: - goreleaser --rm-dist --skip-validate \ No newline at end of file + goreleaser --rm-dist --skip-validate diff --git a/README.md b/README.md index d8dadc9..e7f837d 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Args: file or s3 location ``` -Tha maximum. concurrent uploads (`--concurrent` or `-c`) is dependent not only on your upload bandwidth but also the maximum open file limits per process on your system and the performance of the soucre drive. +Tha maximum concurrent uploads (`--concurrent` or `-c`) is dependent not only on your upload bandwidth but also the maximum open file limits per process on your system and the performance of the soucre drive. You can check your file limits in linux, macos and other flavour of OS with `ulimit -n`. Changing this limit in the os is possible and not always dangerous. Instructions on how to change it vary between OS so they are not described here. `s3kor` impacts these limits both in walking the file system and uploading the file so there is not a 1 to 1 correlation between the max limit ond the value you pass to `--concurrent`. Try to pass `s3kor` a max value that is about 20% less than the systems max limit value. diff --git a/s3kor.go b/s3kor.go index 3e8782c..1c90a83 100755 --- a/s3kor.go +++ b/s3kor.go @@ -3,7 +3,9 @@ package main import ( "fmt" "io/ioutil" + "net/url" "os" + "runtime" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -63,6 +65,12 @@ var ( date = "unknown" ) +///Needed to workaround abug with zap logger and daft windows file paths/names +func newWinFileSink(u *url.URL) (zap.Sink, error) { + // Remove leading slash left by url.Parse() + return os.OpenFile(u.Path[1:], os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) +} + func main() { //Lets keep a track on how long things are taking us //startTime := time.Now() @@ -86,9 +94,19 @@ func main() { } - config.OutputPaths = []string{ - logFile.Name(), + //workaround for windows file paths and names + + if runtime.GOOS == "windows" { + zap.RegisterSink("winfile", newWinFileSink) + config.OutputPaths = []string{ + "winfile:///" + logFile.Name(), + } + } else { + config.OutputPaths = []string{ + logFile.Name(), + } } + logger, _ := config.Build() zap.ReplaceGlobals(logger) zap.RedirectStdLog(logger)