From b36d6d2b71b554029561dda434fc76e13205b1e9 Mon Sep 17 00:00:00 2001 From: Halim Samy Date: Wed, 1 Mar 2023 22:12:38 +0200 Subject: [PATCH 1/9] Fix s3 mock missing endpoint --- index.js | 1 + test/basic.js | 32 ++++++++++++++++---------------- test/util/mock-s3.js | 2 +- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/index.js b/index.js index f64fa00..021a088 100644 --- a/index.js +++ b/index.js @@ -117,6 +117,7 @@ function S3Storage (opts) { switch (typeof opts.key) { case 'function': this.getKey = opts.key; break + case 'string': this.getKey = staticValue(opts.key); break case 'undefined': this.getKey = defaultKey; break default: throw new TypeError('Expected opts.key to be undefined or function') } diff --git a/test/basic.js b/test/basic.js index 9b7dc2f..da4a6a4 100644 --- a/test/basic.js +++ b/test/basic.js @@ -64,7 +64,7 @@ describe('Multer S3', function () { it('upload files', function (done) { var s3 = mockS3() var form = new FormData() - var storage = multerS3({ s3: s3, bucket: 'test' }) + var storage = multerS3({ s3: s3, bucket: 'test', key: 'mock-location' }) var upload = multer({ storage: storage }) var parser = upload.single('image') var image = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) @@ -82,7 +82,7 @@ describe('Multer S3', function () { assert.equal(req.file.size, 68) assert.equal(req.file.bucket, 'test') assert.equal(req.file.etag, 'mock-etag') - assert.equal(req.file.location, 'mock-location') + assert.equal(req.file.location, 'https//test.hostname/mock-location') done() }) @@ -91,7 +91,7 @@ describe('Multer S3', function () { it('uploads file with AES256 server-side encryption', function (done) { var s3 = mockS3() var form = new FormData() - var storage = multerS3({ s3: s3, bucket: 'test', serverSideEncryption: 'AES256' }) + var storage = multerS3({ s3: s3, bucket: 'test', key: 'mock-location', serverSideEncryption: 'AES256' }) var upload = multer({ storage: storage }) var parser = upload.single('image') var image = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) @@ -109,7 +109,7 @@ describe('Multer S3', function () { assert.equal(req.file.size, 68) assert.equal(req.file.bucket, 'test') assert.equal(req.file.etag, 'mock-etag') - assert.equal(req.file.location, 'mock-location') + assert.equal(req.file.location, 'https//test.hostname/mock-location') assert.equal(req.file.serverSideEncryption, 'AES256') done() @@ -119,7 +119,7 @@ describe('Multer S3', function () { it('uploads file with AWS KMS-managed server-side encryption', function (done) { var s3 = mockS3() var form = new FormData() - var storage = multerS3({ s3: s3, bucket: 'test', serverSideEncryption: 'aws:kms' }) + var storage = multerS3({ s3: s3, bucket: 'test', key: 'mock-location', serverSideEncryption: 'aws:kms' }) var upload = multer({ storage: storage }) var parser = upload.single('image') var image = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) @@ -137,7 +137,7 @@ describe('Multer S3', function () { assert.equal(req.file.size, 68) assert.equal(req.file.bucket, 'test') assert.equal(req.file.etag, 'mock-etag') - assert.equal(req.file.location, 'mock-location') + assert.equal(req.file.location, 'https//test.hostname/mock-location') assert.equal(req.file.serverSideEncryption, 'aws:kms') done() @@ -147,7 +147,7 @@ describe('Multer S3', function () { it('uploads PNG file with correct content-type', function (done) { var s3 = mockS3() var form = new FormData() - var storage = multerS3({ s3: s3, bucket: 'test', serverSideEncryption: 'aws:kms', contentType: multerS3.AUTO_CONTENT_TYPE }) + var storage = multerS3({ s3: s3, bucket: 'test', key: 'mock-location', serverSideEncryption: 'aws:kms', contentType: multerS3.AUTO_CONTENT_TYPE }) var upload = multer({ storage: storage }) var parser = upload.single('image') var image = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) @@ -166,7 +166,7 @@ describe('Multer S3', function () { assert.equal(req.file.size, 68) assert.equal(req.file.bucket, 'test') assert.equal(req.file.etag, 'mock-etag') - assert.equal(req.file.location, 'mock-location') + assert.equal(req.file.location, 'https//test.hostname/mock-location') assert.equal(req.file.serverSideEncryption, 'aws:kms') done() @@ -176,7 +176,7 @@ describe('Multer S3', function () { it('uploads pure SVG file with correct content-type', function (done) { var s3 = mockS3() var form = new FormData() - var storage = multerS3({ s3: s3, bucket: 'test', serverSideEncryption: 'aws:kms', contentType: multerS3.AUTO_CONTENT_TYPE }) + var storage = multerS3({ s3: s3, bucket: 'test', key: 'mock-location', serverSideEncryption: 'aws:kms', contentType: multerS3.AUTO_CONTENT_TYPE }) var upload = multer({ storage: storage }) var parser = upload.single('image') var image = fs.createReadStream(path.join(__dirname, 'files', 'test.svg')) @@ -195,7 +195,7 @@ describe('Multer S3', function () { assert.equal(req.file.size, 100) assert.equal(req.file.bucket, 'test') assert.equal(req.file.etag, 'mock-etag') - assert.equal(req.file.location, 'mock-location') + assert.equal(req.file.location, 'https//test.hostname/mock-location') assert.equal(req.file.serverSideEncryption, 'aws:kms') done() @@ -205,7 +205,7 @@ describe('Multer S3', function () { it('uploads common SVG file with correct content-type', function (done) { var s3 = mockS3() var form = new FormData() - var storage = multerS3({ s3: s3, bucket: 'test', serverSideEncryption: 'aws:kms', contentType: multerS3.AUTO_CONTENT_TYPE }) + var storage = multerS3({ s3: s3, bucket: 'test', key: 'mock-location', serverSideEncryption: 'aws:kms', contentType: multerS3.AUTO_CONTENT_TYPE }) var upload = multer({ storage: storage }) var parser = upload.single('image') var image = fs.createReadStream(path.join(__dirname, 'files', 'test2.svg')) @@ -224,7 +224,7 @@ describe('Multer S3', function () { assert.equal(req.file.size, 285) assert.equal(req.file.bucket, 'test') assert.equal(req.file.etag, 'mock-etag') - assert.equal(req.file.location, 'mock-location') + assert.equal(req.file.location, 'https//test.hostname/mock-location') assert.equal(req.file.serverSideEncryption, 'aws:kms') done() @@ -236,7 +236,7 @@ describe('Multer S3', function () { var s3 = mockS3() var form = new FormData() - var storage = multerS3({ s3: s3, bucket: 'test', serverSideEncryption: 'aws:kms', contentType: multerS3.AUTO_CONTENT_TYPE }) + var storage = multerS3({ s3: s3, bucket: 'test', key: 'mock-location', serverSideEncryption: 'aws:kms', contentType: multerS3.AUTO_CONTENT_TYPE }) var upload = multer({ storage: storage }) var parser = upload.single('image') fs.writeFileSync(path.join(__dirname, 'files', 'test_generated.svg'), ' ({ protocol: 'https', hostname: 'hostname' }) } } } module.exports = createMockS3 From 7119d585656c063fbc8803b944ce2f9585863d79 Mon Sep 17 00:00:00 2001 From: Halim Samy Date: Thu, 2 Mar 2023 00:18:20 +0200 Subject: [PATCH 2/9] Update to a newer version of file-type --- index.js | 27 ++++---- package-lock.json | 170 +++++++++++++++++++++++++++++++++++++++++++--- package.json | 4 +- 3 files changed, 178 insertions(+), 23 deletions(-) diff --git a/index.js b/index.js index 021a088..8dcfab5 100644 --- a/index.js +++ b/index.js @@ -47,22 +47,23 @@ function defaultKey (req, file, cb) { function autoContentType (req, file, cb) { file.stream.once('data', function (firstChunk) { - var type = fileType(firstChunk) - var mime = 'application/octet-stream' // default type - - // Make sure to check xml-extension for svg files. - if ((!type || type.ext === 'xml') && isSvg(firstChunk.toString())) { - mime = 'image/svg+xml' - } else if (type) { - mime = type.mime - } + fileType.fromBuffer(firstChunk).then(function (type) { + var mime = 'application/octet-stream' // default type + + // Make sure to check xml-extension for svg files. + if ((!type || type.ext === 'xml') && isSvg(firstChunk.toString())) { + mime = 'image/svg+xml' + } else if (type) { + mime = type.mime + } - var outStream = new stream.PassThrough() + var outStream = new stream.PassThrough() - outStream.write(firstChunk) - file.stream.pipe(outStream) + outStream.write(firstChunk) + file.stream.pipe(outStream) - cb(null, mime, outStream) + cb(null, mime, outStream) + }) }) } diff --git a/package-lock.json b/package-lock.json index 6d8f601..e66958a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "@aws-sdk/lib-storage": "^3.46.0", - "file-type": "^3.3.0", + "file-type": "^16.5.4", "html-comment-regex": "^1.1.2", "run-parallel": "^1.1.6" }, @@ -1326,6 +1326,11 @@ "node": ">= 12.0.0" } }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -3255,11 +3260,19 @@ } }, "node_modules/file-type": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", - "integrity": "sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==", + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "dependencies": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + }, "engines": { - "node": ">=0.10.0" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" } }, "node_modules/finalhandler": { @@ -4730,6 +4743,18 @@ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=", "dev": true }, + "node_modules/peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/pkg-config": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/pkg-config/-/pkg-config-1.1.1.tgz", @@ -4885,6 +4910,42 @@ "string_decoder": "~0.10.x" } }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "dependencies": { + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/readable-web-to-node-stream/node_modules/readable-stream": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", + "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readable-web-to-node-stream/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/readline2": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/readline2/-/readline2-1.0.1.tgz", @@ -5637,6 +5698,22 @@ "node": ">=0.8.0" } }, + "node_modules/strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/supports-color": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-1.2.0.tgz", @@ -5722,6 +5799,22 @@ "node": ">=0.6" } }, + "node_modules/token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/trim-right": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", @@ -7056,6 +7149,11 @@ "tslib": "^2.3.1" } }, + "@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -8691,9 +8789,14 @@ } }, "file-type": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", - "integrity": "sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==" + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "requires": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + } }, "finalhandler": { "version": "1.2.0", @@ -9913,6 +10016,11 @@ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=", "dev": true }, + "peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==" + }, "pkg-config": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/pkg-config/-/pkg-config-1.1.1.tgz", @@ -10028,6 +10136,34 @@ "string_decoder": "~0.10.x" } }, + "readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "requires": { + "readable-stream": "^3.6.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", + "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, "readline2": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/readline2/-/readline2-1.0.1.tgz", @@ -10641,6 +10777,15 @@ "integrity": "sha1-HhX7ysl9Pumb8tc7TGVrCCu6+5E=", "dev": true }, + "strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "requires": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + } + }, "supports-color": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-1.2.0.tgz", @@ -10701,6 +10846,15 @@ "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", "dev": true }, + "token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "requires": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + } + }, "trim-right": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", diff --git a/package.json b/package.json index e1a31a1..fd96f82 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "homepage": "https://github.com/badunk/multer-s3#readme", "dependencies": { "@aws-sdk/lib-storage": "^3.46.0", - "file-type": "^3.3.0", + "file-type": "^16.5.4", "html-comment-regex": "^1.1.2", "run-parallel": "^1.1.6" }, @@ -43,4 +43,4 @@ "standard": "^5.4.1", "xtend": "^4.0.1" } -} +} \ No newline at end of file From aa772e3624077516f439a6137e4748455c695c91 Mon Sep 17 00:00:00 2001 From: Halim Samy Date: Thu, 2 Mar 2023 01:12:39 +0200 Subject: [PATCH 3/9] Impl transforms --- index.js | 167 +++++++++++++++++++++++++++++++++++++------------- test/basic.js | 155 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 278 insertions(+), 44 deletions(-) diff --git a/index.js b/index.js index 8dcfab5..6f80632 100644 --- a/index.js +++ b/index.js @@ -23,6 +23,8 @@ var defaultContentEncoding = staticValue(null) var defaultStorageClass = staticValue('STANDARD') var defaultSSE = staticValue(null) var defaultSSEKMS = staticValue(null) +var defaultShouldTransform = staticValue(false) +var defaultTransforms = [] // Regular expression to detect svg file content, inspired by: https://github.com/sindresorhus/is-svg/blob/master/index.js // It is not always possible to check for an end tag if a file is very big. The firstChunk, see below, might not be the entire file. @@ -78,7 +80,8 @@ function collect (storage, req, file, cb) { storage.getStorageClass.bind(storage, req, file), storage.getSSE.bind(storage, req, file), storage.getSSEKMS.bind(storage, req, file), - storage.getContentEncoding.bind(storage, req, file) + storage.getContentEncoding.bind(storage, req, file), + storage.getShouldTransform.bind(storage, req, file) ], function (err, values) { if (err) return cb(err) @@ -97,7 +100,8 @@ function collect (storage, req, file, cb) { replacementStream: replacementStream, serverSideEncryption: values[7], sseKmsKeyId: values[8], - contentEncoding: values[9] + contentEncoding: values[9], + shouldTransform: values[10] }) }) }) @@ -183,64 +187,139 @@ function S3Storage (opts) { case 'undefined': this.getSSEKMS = defaultSSEKMS; break default: throw new TypeError('Expected opts.sseKmsKeyId to be undefined, string, or function') } + + switch (typeof opts.shouldTransform) { + case 'function': this.getShouldTransform = opts.shouldTransform; break + case 'boolean': this.getShouldTransform = staticValue(opts.shouldTransform); break + case 'undefined': this.getShouldTransform = defaultShouldTransform; break + default: throw new TypeError('Expected opts.shouldTransform to be undefined, boolean or function') + } + + switch (typeof opts.transforms) { + case 'object': this.transforms = opts.transforms; break + case 'undefined': this.transforms = defaultTransforms; break + default: throw new TypeError('Expected opts.transforms to be undefined or object') + } + + this.transforms.map(function (transform) { + switch (typeof transform.id) { + case 'string': break + default: throw new TypeError('Expected opts.transform[].id to be string') + } + + switch (typeof transform.key) { + case 'function': break + case 'string': transform.key = staticValue(transform.key); break + case 'undefined': transform.key = defaultKey(); break + default: throw new TypeError('Expected opts.transform[].key to be unedefined, string or function') + } + + switch (typeof transform.transform) { + case 'function': break + default: throw new TypeError('Expected opts.transform[].transform to be function') + } + + return transform + }) } S3Storage.prototype._handleFile = function (req, file, cb) { collect(this, req, file, function (err, opts) { if (err) return cb(err) - var currentSize = 0 - - var params = { - Bucket: opts.bucket, - Key: opts.key, - ACL: opts.acl, - CacheControl: opts.cacheControl, - ContentType: opts.contentType, - Metadata: opts.metadata, - StorageClass: opts.storageClass, - ServerSideEncryption: opts.serverSideEncryption, - SSEKMSKeyId: opts.sseKmsKeyId, - Body: (opts.replacementStream || file.stream) + if (opts.shouldTransform) { + this.transformUpload(opts, req, file, cb) + } else { + this.directUpload(opts, file, cb) } + }) +} - if (opts.contentDisposition) { - params.ContentDisposition = opts.contentDisposition - } +S3Storage.prototype.directUpload = function (opts, file, cb, piper, key, id) { + var currentSize = 0 + + var params = { + Bucket: opts.bucket, + Key: key || opts.key, + ACL: opts.acl, + CacheControl: opts.cacheControl, + ContentType: opts.contentType, + Metadata: opts.metadata, + StorageClass: opts.storageClass, + ServerSideEncryption: opts.serverSideEncryption, + SSEKMSKeyId: opts.sseKmsKeyId, + Body: piper ? (opts.replacementStream || file.stream).pipe(piper) : (opts.replacementStream || file.stream) + } - if (opts.contentEncoding) { - params.ContentEncoding = opts.contentEncoding - } + if (opts.contentDisposition) { + params.ContentDisposition = opts.contentDisposition + } - var upload = new Upload({ - client: this.s3, - params: params - }) + if (opts.contentEncoding) { + params.ContentEncoding = opts.contentEncoding + } + + var upload = new Upload({ + client: this.s3, + params: params + }) + + upload.on('httpUploadProgress', function (ev) { + if (ev.total) currentSize = ev.total + }) + + util.callbackify(upload.done.bind(upload))(function (err, result) { + if (err) return cb(err) - upload.on('httpUploadProgress', function (ev) { - if (ev.total) currentSize = ev.total + cb(null, { + id: id, + size: currentSize, + bucket: opts.bucket, + key: key || opts.key, + acl: opts.acl, + contentType: opts.contentType, + contentDisposition: opts.contentDisposition, + contentEncoding: opts.contentEncoding, + storageClass: opts.storageClass, + serverSideEncryption: opts.serverSideEncryption, + metadata: opts.metadata, + location: result.Location, + etag: result.ETag, + versionId: result.VersionId }) + }) +} - util.callbackify(upload.done.bind(upload))(function (err, result) { +S3Storage.prototype.transformUpload = function (opts, req, file, cb) { + var storage = this + var transforms = {} + var transformsCount = 0 + + parallel( + storage.transforms.map(function (transform) { + return transform.key.bind(storage, req, file) + }), + function (err, keys) { if (err) return cb(err) - cb(null, { - size: currentSize, - bucket: opts.bucket, - key: opts.key, - acl: opts.acl, - contentType: opts.contentType, - contentDisposition: opts.contentDisposition, - contentEncoding: opts.contentEncoding, - storageClass: opts.storageClass, - serverSideEncryption: opts.serverSideEncryption, - metadata: opts.metadata, - location: result.Location, - etag: result.ETag, - versionId: result.VersionId + keys.forEach(function (key, i) { + storage.transforms[i].transform(req, file, function (err, piper) { + if (err) return cb(err) + var id = storage.transforms[i].id || i + + storage.directUpload(opts, file, function (err, result) { + if (err) return cb(err) + + transforms[id] = result + transformsCount++ + if (transformsCount === keys.length) { + cb(null, { transforms }) + } + }, piper, key, id) + }) }) - }) - }) + } + ) } S3Storage.prototype._removeFile = function (req, file, cb) { diff --git a/test/basic.js b/test/basic.js index da4a6a4..39dbec9 100644 --- a/test/basic.js +++ b/test/basic.js @@ -290,4 +290,159 @@ describe('Multer S3', function () { done() }) }) + + it('uploads PNG file with transforms', function (done) { + var s3 = mockS3() + var form = new FormData() + var storage = multerS3({ + s3: s3, + bucket: 'test', + contentType: multerS3.AUTO_CONTENT_TYPE, + shouldTransform: true, + transforms: [ + { + id: 'original', + key: 'original', + transform: function (req, file, cb) { + cb(null, new stream.PassThrough()) + } + }, + { + id: 'thumbnail', + key: function (req, file, cb) { + cb(null, 'thumbnail') + }, + transform: function (req, file, cb) { + cb(null, new stream.PassThrough()) + } + } + ] + }) + var upload = multer({ storage: storage }) + var parser = upload.single('image') + var image = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) + + form.append('name', 'Multer') + form.append('image', image) + + submitForm(parser, form, function (err, req) { + assert.ifError(err) + + assert.equal(req.body.name, 'Multer') + + assert.equal(req.file.fieldname, 'image') + assert.equal(req.file.mimetype, 'image/png') + assert.equal(req.file.originalname, 'ffffff.png') + assert.equal(req.file.transforms.original.size, 68) + assert.equal(req.file.transforms.original.bucket, 'test') + assert.equal(req.file.transforms.original.etag, 'mock-etag') + assert.equal(req.file.transforms.original.location, 'https//test.hostname/original') + assert.equal(req.file.transforms.thumbnail.size, 68) + assert.equal(req.file.transforms.thumbnail.bucket, 'test') + assert.equal(req.file.transforms.thumbnail.etag, 'mock-etag') + assert.equal(req.file.transforms.thumbnail.location, 'https//test.hostname/thumbnail') + + done() + }) + }) + + // it('uploads a PNG and SVG file with transforms', function (done) { + // var s3 = mockS3() + // var form = new FormData() + // var storage = multerS3({ + // s3: s3, + // bucket: 'test', + // contentType: multerS3.AUTO_CONTENT_TYPE, + // shouldTransform: true, + // transforms: [ + // { + // id: 'original', + // key: 'original', + // transform: function (req, file, cb) { + // cb(null, new stream.PassThrough()) + // } + // }, + // { + // id: 'thumbnail', + // key: function (req, file, cb) { + // cb(null, 'thumbnail') + // }, + // transform: function (req, file, cb) { + // cb(null, new stream.PassThrough()) + // } + // } + // ] + // }) + // var upload = multer({ storage: storage }) + // var parser = upload.array('images', 2) + // var image1 = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) + // var image2 = fs.createReadStream(path.join(__dirname, 'files', 'test2.svg')) + + // form.append('name', 'Multer') + // form.append('images', image1) + // form.append('images', image2) + + // submitForm(parser, form, function (err, req) { + // assert.ifError(err) + + // assert.equal(req.body.name, 'Multer') + + // assert.equal(req.files[0].fieldname, 'images') + // assert.equal(req.files[0].mimetype, 'image/png') + // assert.equal(req.files[0].originalname, 'ffffff.png') + // assert.equal(req.files[0].transforms.original.size, 68) + // assert.equal(req.files[0].transforms.original.bucket, 'test') + // assert.equal(req.files[0].transforms.original.etag, 'mock-etag') + // assert.equal(req.files[0].transforms.original.location, 'https//test.hostname/original') + // assert.equal(req.files[0].transforms.thumbnail.size, 68) + // assert.equal(req.files[0].transforms.thumbnail.bucket, 'test') + // assert.equal(req.files[0].transforms.thumbnail.etag, 'mock-etag') + // assert.equal(req.files[0].transforms.thumbnail.location, 'https//test.hostname/thumbnail') + + // assert.equal(req.files[1].fieldname, 'images') + // assert.equal(req.files[1].mimetype, 'image/svg+xml') + // assert.equal(req.files[1].originalname, 'test2.svg') + // }) + // }) + + // it('uploads a PNG and SVG file', function (done) { + // var s3 = mockS3() + // var form = new FormData() + // var storage = multerS3({ + // s3: s3, + // bucket: 'test', + // key: 'mock-location', + // contentType: multerS3.AUTO_CONTENT_TYPE + // }) + // var upload = multer({ storage: storage }) + // var parser = upload.array('images', 2) + // var image1 = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) + // var image2 = fs.createReadStream(path.join(__dirname, 'files', 'test2.svg')) + + // form.append('name', 'Multer') + // form.append('images', image1) + // form.append('images', image2) + + // submitForm(parser, form, function (err, req) { + // assert.ifError(err) + + // assert.equal(req.body.name, 'Multer') + + // assert.equal(req.files[0].fieldname, 'images') + // assert.equal(req.files[0].mimetype, 'image/png') + // assert.equal(req.files[0].originalname, 'ffffff.png') + // assert.equal(req.files[0].size, 68) + // assert.equal(req.files[0].bucket, 'test') + // assert.equal(req.files[0].etag, 'mock-etag') + // assert.equal(req.files[0].location, 'https//test.hostname/mock-location') + + // assert.equal(req.files[1].fieldname, 'images') + // assert.equal(req.files[1].mimetype, 'image/svg+xml') + // assert.equal(req.files[1].originalname, 'test2.svg') + // assert.equal(req.files[1].size, 285) + // assert.equal(req.files[1].bucket, 'test') + // assert.equal(req.files[1].etag, 'mock-etag') + // assert.equal(req.files[1].location, 'https//test.hostname/mock-location') + // }) + // }) }) From a0ba940ae286c6457b4e3e259d6df0a08b9351aa Mon Sep 17 00:00:00 2001 From: Halim Samy Date: Thu, 2 Mar 2023 01:59:46 +0200 Subject: [PATCH 4/9] Update README.md --- README.md | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/README.md b/README.md index c9e6dd2..05fc2f4 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,74 @@ var upload = multer({ ``` You may also use a function as the `contentEncoding`, which should be of the form `function(req, file, cb)`. +## Transforming Files Before Upload + +The optional `shouldTransform` option tells multer whether it should transform the file before it is uploaded. By default, it is set to `false`. If set to `true`, `transforms` option must be added, which tells how to transform the file. `transforms` option should be an `Array`, containing objects with can have properties `id`, `key` and `transform`. + +```javascript +var upload = multer({ + storage: multerS3({ + s3: s3, + bucket: 'some-bucket', + shouldTransform: function (req, file, cb) { + cb(null, /^image/i.test(file.mimetype)) + }, + transforms: [{ + id: 'original', + key: function (req, file, cb) { + cb(null, 'image-original.jpg') + }, + transform: function (req, file, cb) { + cb(null, sharp().jpeg()) + } + }, { + id: 'thumbnail', + key: function (req, file, cb) { + cb(null, 'image-thumbnail.jpg') + }, + transform: function (req, file, cb) { + cb(null, sharp().resize(100, 100).jpeg()) + } + }] + }) +}) +``` +If this option is used, each file passed to your router request will have a `transforms` object, with every transform you defined. +```json +{ + "data": { + "fieldname": "image", + "originalname": "image.jpg", + "encoding": "7bit", + "mimetype": "image/jpg", + "transforms": { + "original": { + "id": "original", + "size": 18006, + "bucket": "some-bucket", + "key": "image-original.jpg", + "acl": "public-read", + "contentType": "image/jpg", + "metadata": null, + "location": "https://some-bucket.s3.us-east-1.amazonaws.com/image-original.jpg", + "etag": "\"76c09df7bdd752a749f91b9663838fb2\"" + }, + "thumbnail": { + "id": "thumbnail", + "size": 18006, + "bucket": "some-bucket", + "key": "image-thumbnail.jpg", + "acl": "public-read", + "contentType": "image/jpg", + "metadata": null, + "location": "https://some-bucket.s3.us-east-1.amazonaws.com/image-thumbnail.jpg", + "etag": "\"9d554e03e37c79bff7ce31d375900db6\"" + } + } + } +} +``` + ## Testing The tests mock all access to S3 and can be run completely offline. From 98ddc2380058c40beb5b55848e7756461eb96361 Mon Sep 17 00:00:00 2001 From: Halim Samy Date: Thu, 2 Mar 2023 03:48:00 +0200 Subject: [PATCH 5/9] Update README.md --- README.md | 54 ++++++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 05fc2f4..f320e5e 100644 --- a/README.md +++ b/README.md @@ -264,34 +264,32 @@ var upload = multer({ If this option is used, each file passed to your router request will have a `transforms` object, with every transform you defined. ```json { - "data": { - "fieldname": "image", - "originalname": "image.jpg", - "encoding": "7bit", - "mimetype": "image/jpg", - "transforms": { - "original": { - "id": "original", - "size": 18006, - "bucket": "some-bucket", - "key": "image-original.jpg", - "acl": "public-read", - "contentType": "image/jpg", - "metadata": null, - "location": "https://some-bucket.s3.us-east-1.amazonaws.com/image-original.jpg", - "etag": "\"76c09df7bdd752a749f91b9663838fb2\"" - }, - "thumbnail": { - "id": "thumbnail", - "size": 18006, - "bucket": "some-bucket", - "key": "image-thumbnail.jpg", - "acl": "public-read", - "contentType": "image/jpg", - "metadata": null, - "location": "https://some-bucket.s3.us-east-1.amazonaws.com/image-thumbnail.jpg", - "etag": "\"9d554e03e37c79bff7ce31d375900db6\"" - } + "fieldname":"image", + "originalname":"image.jpg", + "encoding":"7bit", + "mimetype":"image/jpg", + "transforms":{ + "original":{ + "id":"original", + "size":18006, + "bucket":"some-bucket", + "key":"image-original.jpg", + "acl":"public-read", + "contentType":"image/jpg", + "metadata":null, + "location":"https://some-bucket.s3.us-east-1.amazonaws.com/image-original.jpg", + "etag":"76c09df7bdd752a749f91b9663838fb2" + }, + "thumbnail":{ + "id":"thumbnail", + "size":18006, + "bucket":"some-bucket", + "key":"image-thumbnail.jpg", + "acl":"public-read", + "contentType":"image/jpg", + "metadata":null, + "location":"https://some-bucket.s3.us-east-1.amazonaws.com/image-thumbnail.jpg", + "etag":"9d554e03e37c79bff7ce31d375900db6" } } } From 410d3a4e94c4b99a132499a59147f7bdd6ccacc8 Mon Sep 17 00:00:00 2001 From: Halim Date: Thu, 30 Nov 2023 15:01:20 +0200 Subject: [PATCH 6/9] Remove deprecated dependencies and cleanup --- README.md | 8 +-- index.js | 131 +++++++++++++++++++++++++--------------------- package-lock.json | 20 +++---- package.json | 6 +-- test/basic.js | 38 +++++++------- 5 files changed, 101 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index f320e5e..51170db 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ You may also use a function as the `contentEncoding`, which should be of the for ## Transforming Files Before Upload -The optional `shouldTransform` option tells multer whether it should transform the file before it is uploaded. By default, it is set to `false`. If set to `true`, `transforms` option must be added, which tells how to transform the file. `transforms` option should be an `Array`, containing objects with can have properties `id`, `key` and `transform`. +The optional `shouldTransform` option tells multer whether it should transform the file before it is uploaded. By default, it is set to `false`. If set to `true`, `transformers` option must be added, which tells how to transform the file. `transformers` option should be an `Array`, containing objects with can have properties `id`, `key` and `transform`. ```javascript var upload = multer({ @@ -241,7 +241,7 @@ var upload = multer({ shouldTransform: function (req, file, cb) { cb(null, /^image/i.test(file.mimetype)) }, - transforms: [{ + transformers: [{ id: 'original', key: function (req, file, cb) { cb(null, 'image-original.jpg') @@ -261,14 +261,14 @@ var upload = multer({ }) }) ``` -If this option is used, each file passed to your router request will have a `transforms` object, with every transform you defined. +If this option is used, each file passed to your router request will have a `transformations` object, with every transform you defined. ```json { "fieldname":"image", "originalname":"image.jpg", "encoding":"7bit", "mimetype":"image/jpg", - "transforms":{ + "transformations":{ "original":{ "id":"original", "size":18006, diff --git a/index.js b/index.js index 6f80632..8db327e 100644 --- a/index.js +++ b/index.js @@ -1,21 +1,12 @@ var crypto = require('crypto') var stream = require('stream') var fileType = require('file-type') -var htmlCommentRegex = require('html-comment-regex') -var parallel = require('run-parallel') var Upload = require('@aws-sdk/lib-storage').Upload var DeleteObjectCommand = require('@aws-sdk/client-s3').DeleteObjectCommand var util = require('util') -function staticValue (value) { - return function (req, file, cb) { - cb(null, value) - } -} - var defaultAcl = staticValue('private') var defaultContentType = staticValue('application/octet-stream') - var defaultMetadata = staticValue(undefined) var defaultCacheControl = staticValue(null) var defaultContentDisposition = staticValue(null) @@ -24,22 +15,7 @@ var defaultStorageClass = staticValue('STANDARD') var defaultSSE = staticValue(null) var defaultSSEKMS = staticValue(null) var defaultShouldTransform = staticValue(false) -var defaultTransforms = [] - -// Regular expression to detect svg file content, inspired by: https://github.com/sindresorhus/is-svg/blob/master/index.js -// It is not always possible to check for an end tag if a file is very big. The firstChunk, see below, might not be the entire file. -var svgRegex = /^\s*(?:<\?xml[^>]*>\s*)?(?:]*>\s*)?]*>/i - -function isSvg (svg) { - // Remove DTD entities - svg = svg.replace(/\s*/img, '') - // Remove DTD markup declarations - svg = svg.replace(/\[?(?:\s*]*>\s*)*\]?/g, '') - // Remove HTML comments - svg = svg.replace(htmlCommentRegex, '') - - return svgRegex.test(svg) -} +var defaultTransformers = [] function defaultKey (req, file, cb) { crypto.randomBytes(16, function (err, raw) { @@ -47,7 +23,42 @@ function defaultKey (req, file, cb) { }) } +function staticValue (value) { + return function (req, file, cb) { + cb(null, value) + } +} + +function waterfall (funcs, callback) { + var index = 0 + var values = [] + + function next (err, value) { + if (err) return callback(err) + values.push(value) + if (index >= funcs.length) return callback(null, values) + funcs[index++](next) + } + + funcs[index++](next) +} + function autoContentType (req, file, cb) { + // Regular expression to detect svg file content, inspired by: https://github.com/sindresorhus/is-svg/blob/master/index.js + // It is not always possible to check for an end tag if a file is very big. The firstChunk, see below, might not be the entire file. + var svgRegex = /^\s*(?:<\?xml[^>]*>\s*)?(?:]*>\s*)?]*>/i + + function isSvg (svg) { + // Remove DTD entities + svg = svg.replace(/\s*/img, '') + // Remove DTD markup declarations + svg = svg.replace(/\[?(?:\s*]*>\s*)*\]?/g, '') + // Remove HTML comments + svg = svg.replace(//g, '') + + return svgRegex.test(svg) + } + file.stream.once('data', function (firstChunk) { fileType.fromBuffer(firstChunk).then(function (type) { var mime = 'application/octet-stream' // default type @@ -70,7 +81,7 @@ function autoContentType (req, file, cb) { } function collect (storage, req, file, cb) { - parallel([ + waterfall([ storage.getBucket.bind(storage, req, file), storage.getKey.bind(storage, req, file), storage.getAcl.bind(storage, req, file), @@ -195,43 +206,31 @@ function S3Storage (opts) { default: throw new TypeError('Expected opts.shouldTransform to be undefined, boolean or function') } - switch (typeof opts.transforms) { - case 'object': this.transforms = opts.transforms; break - case 'undefined': this.transforms = defaultTransforms; break + switch (typeof opts.transformers) { + case 'object': this.transformers = opts.transformers; break + case 'undefined': this.transformers = defaultTransformers; break default: throw new TypeError('Expected opts.transforms to be undefined or object') } - this.transforms.map(function (transform) { - switch (typeof transform.id) { + this.transformers.map(function (transformer) { + switch (typeof transformer.id) { case 'string': break - default: throw new TypeError('Expected opts.transform[].id to be string') + default: throw new TypeError('Expected opts.transformer[].id to be string') } - switch (typeof transform.key) { + switch (typeof transformer.key) { case 'function': break - case 'string': transform.key = staticValue(transform.key); break - case 'undefined': transform.key = defaultKey(); break - default: throw new TypeError('Expected opts.transform[].key to be unedefined, string or function') + case 'string': transformer.key = staticValue(transformer.key); break + case 'undefined': transformer.key = defaultKey(); break + default: throw new TypeError('Expected opts.transformer[].key to be unedefined, string or function') } - switch (typeof transform.transform) { + switch (typeof transformer.transform) { case 'function': break - default: throw new TypeError('Expected opts.transform[].transform to be function') + default: throw new TypeError('Expected opts.transformer[].transform to be function') } - return transform - }) -} - -S3Storage.prototype._handleFile = function (req, file, cb) { - collect(this, req, file, function (err, opts) { - if (err) return cb(err) - - if (opts.shouldTransform) { - this.transformUpload(opts, req, file, cb) - } else { - this.directUpload(opts, file, cb) - } + return transformer }) } @@ -292,28 +291,26 @@ S3Storage.prototype.directUpload = function (opts, file, cb, piper, key, id) { S3Storage.prototype.transformUpload = function (opts, req, file, cb) { var storage = this - var transforms = {} - var transformsCount = 0 + var transformations = {} - parallel( - storage.transforms.map(function (transform) { + waterfall( + storage.transformers.map(function (transform) { return transform.key.bind(storage, req, file) }), function (err, keys) { if (err) return cb(err) keys.forEach(function (key, i) { - storage.transforms[i].transform(req, file, function (err, piper) { + storage.transformers[i].transform(req, file, function (err, piper) { if (err) return cb(err) - var id = storage.transforms[i].id || i + var id = storage.transformers[i].id || i storage.directUpload(opts, file, function (err, result) { if (err) return cb(err) - transforms[id] = result - transformsCount++ - if (transformsCount === keys.length) { - cb(null, { transforms }) + transformations[id] = result + if (i === keys.length - 1) { + cb(null, { transformations: transformations }) } }, piper, key, id) }) @@ -322,6 +319,18 @@ S3Storage.prototype.transformUpload = function (opts, req, file, cb) { ) } +S3Storage.prototype._handleFile = function (req, file, cb) { + collect(this, req, file, function (err, opts) { + if (err) return cb(err) + + if (opts.shouldTransform) { + this.transformUpload(opts, req, file, cb) + } else { + this.directUpload(opts, file, cb) + } + }) +} + S3Storage.prototype._removeFile = function (req, file, cb) { this.s3.send( new DeleteObjectCommand({ diff --git a/package-lock.json b/package-lock.json index e66958a..e30e90a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,9 +10,7 @@ "license": "MIT", "dependencies": { "@aws-sdk/lib-storage": "^3.46.0", - "file-type": "^16.5.4", - "html-comment-regex": "^1.1.2", - "run-parallel": "^1.1.6" + "file-type": "^16.5.4" }, "devDependencies": { "express": "^4.13.1", @@ -3548,11 +3546,6 @@ "node": ">=0.10.0" } }, - "node_modules/html-comment-regex": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.2.tgz", - "integrity": "sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ==" - }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -4859,6 +4852,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, "funding": [ { "type": "github", @@ -5265,6 +5259,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, "funding": [ { "type": "github", @@ -9016,11 +9011,6 @@ } } }, - "html-comment-regex": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.2.tgz", - "integrity": "sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ==" - }, "http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -10104,7 +10094,8 @@ "queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true }, "range-parser": { "version": "1.2.1", @@ -10427,6 +10418,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, "requires": { "queue-microtask": "^1.2.2" } diff --git a/package.json b/package.json index fd96f82..7cfcceb 100644 --- a/package.json +++ b/package.json @@ -27,9 +27,7 @@ "homepage": "https://github.com/badunk/multer-s3#readme", "dependencies": { "@aws-sdk/lib-storage": "^3.46.0", - "file-type": "^16.5.4", - "html-comment-regex": "^1.1.2", - "run-parallel": "^1.1.6" + "file-type": "^16.5.4" }, "peerDependencies": { "@aws-sdk/client-s3": "^3.0.0" @@ -43,4 +41,4 @@ "standard": "^5.4.1", "xtend": "^4.0.1" } -} \ No newline at end of file +} diff --git a/test/basic.js b/test/basic.js index 39dbec9..1171182 100644 --- a/test/basic.js +++ b/test/basic.js @@ -291,7 +291,7 @@ describe('Multer S3', function () { }) }) - it('uploads PNG file with transforms', function (done) { + it('uploads PNG file with transformers', function (done) { var s3 = mockS3() var form = new FormData() var storage = multerS3({ @@ -299,7 +299,7 @@ describe('Multer S3', function () { bucket: 'test', contentType: multerS3.AUTO_CONTENT_TYPE, shouldTransform: true, - transforms: [ + transformers: [ { id: 'original', key: 'original', @@ -333,14 +333,14 @@ describe('Multer S3', function () { assert.equal(req.file.fieldname, 'image') assert.equal(req.file.mimetype, 'image/png') assert.equal(req.file.originalname, 'ffffff.png') - assert.equal(req.file.transforms.original.size, 68) - assert.equal(req.file.transforms.original.bucket, 'test') - assert.equal(req.file.transforms.original.etag, 'mock-etag') - assert.equal(req.file.transforms.original.location, 'https//test.hostname/original') - assert.equal(req.file.transforms.thumbnail.size, 68) - assert.equal(req.file.transforms.thumbnail.bucket, 'test') - assert.equal(req.file.transforms.thumbnail.etag, 'mock-etag') - assert.equal(req.file.transforms.thumbnail.location, 'https//test.hostname/thumbnail') + assert.equal(req.file.transformations.original.size, 68) + assert.equal(req.file.transformations.original.bucket, 'test') + assert.equal(req.file.transformations.original.etag, 'mock-etag') + assert.equal(req.file.transformations.original.location, 'https//test.hostname/original') + assert.equal(req.file.transformations.thumbnail.size, 68) + assert.equal(req.file.transformations.thumbnail.bucket, 'test') + assert.equal(req.file.transformations.thumbnail.etag, 'mock-etag') + assert.equal(req.file.transformations.thumbnail.location, 'https//test.hostname/thumbnail') done() }) @@ -354,7 +354,7 @@ describe('Multer S3', function () { // bucket: 'test', // contentType: multerS3.AUTO_CONTENT_TYPE, // shouldTransform: true, - // transforms: [ + // transformers: [ // { // id: 'original', // key: 'original', @@ -390,14 +390,14 @@ describe('Multer S3', function () { // assert.equal(req.files[0].fieldname, 'images') // assert.equal(req.files[0].mimetype, 'image/png') // assert.equal(req.files[0].originalname, 'ffffff.png') - // assert.equal(req.files[0].transforms.original.size, 68) - // assert.equal(req.files[0].transforms.original.bucket, 'test') - // assert.equal(req.files[0].transforms.original.etag, 'mock-etag') - // assert.equal(req.files[0].transforms.original.location, 'https//test.hostname/original') - // assert.equal(req.files[0].transforms.thumbnail.size, 68) - // assert.equal(req.files[0].transforms.thumbnail.bucket, 'test') - // assert.equal(req.files[0].transforms.thumbnail.etag, 'mock-etag') - // assert.equal(req.files[0].transforms.thumbnail.location, 'https//test.hostname/thumbnail') + // assert.equal(req.files[0].transformations.original.size, 68) + // assert.equal(req.files[0].transformations.original.bucket, 'test') + // assert.equal(req.files[0].transformations.original.etag, 'mock-etag') + // assert.equal(req.files[0].transformations.original.location, 'https//test.hostname/original') + // assert.equal(req.files[0].transformations.thumbnail.size, 68) + // assert.equal(req.files[0].transformations.thumbnail.bucket, 'test') + // assert.equal(req.files[0].transformations.thumbnail.etag, 'mock-etag') + // assert.equal(req.files[0].transformations.thumbnail.location, 'https//test.hostname/thumbnail') // assert.equal(req.files[1].fieldname, 'images') // assert.equal(req.files[1].mimetype, 'image/svg+xml') From 26edd56fc1a218b370a9c493c1d1344b23b52783 Mon Sep 17 00:00:00 2001 From: Halim Date: Sun, 3 Dec 2023 14:18:28 +0200 Subject: [PATCH 7/9] Fix concurrency issue when the last transformer finishes before the others --- index.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 8db327e..d84bff5 100644 --- a/index.js +++ b/index.js @@ -292,6 +292,7 @@ S3Storage.prototype.directUpload = function (opts, file, cb, piper, key, id) { S3Storage.prototype.transformUpload = function (opts, req, file, cb) { var storage = this var transformations = {} + var pending = storage.transformers.length waterfall( storage.transformers.map(function (transform) { @@ -309,9 +310,7 @@ S3Storage.prototype.transformUpload = function (opts, req, file, cb) { if (err) return cb(err) transformations[id] = result - if (i === keys.length - 1) { - cb(null, { transformations: transformations }) - } + if (--pending === 0) cb(null, { transformations: transformations }) }, piper, key, id) }) }) From 2011a5912c5a32eb2ab0d2f52ce67c00547e36fb Mon Sep 17 00:00:00 2001 From: Halim Date: Mon, 4 Dec 2023 13:17:22 +0200 Subject: [PATCH 8/9] Refactor autoContentType function to use stream.pipeline --- index.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index d84bff5..4969711 100644 --- a/index.js +++ b/index.js @@ -59,6 +59,12 @@ function autoContentType (req, file, cb) { return svgRegex.test(svg) } + var outStream = new stream.PassThrough() + + stream.pipeline(file.stream, outStream, function (err) { + if (err) return cb(err) + }) + file.stream.once('data', function (firstChunk) { fileType.fromBuffer(firstChunk).then(function (type) { var mime = 'application/octet-stream' // default type @@ -70,11 +76,6 @@ function autoContentType (req, file, cb) { mime = type.mime } - var outStream = new stream.PassThrough() - - outStream.write(firstChunk) - file.stream.pipe(outStream) - cb(null, mime, outStream) }) }) From 2c949416dbf565d753eaad2a5e7acbd0736a9864 Mon Sep 17 00:00:00 2001 From: Halim Date: Mon, 4 Dec 2023 13:50:50 +0200 Subject: [PATCH 9/9] Add separate content type for transformers with fallback to the global content type --- index.js | 44 ++++++++--- test/basic.js | 207 ++++++++++++++++++++++++++------------------------ 2 files changed, 139 insertions(+), 112 deletions(-) diff --git a/index.js b/index.js index 4969711..052489c 100644 --- a/index.js +++ b/index.js @@ -148,8 +148,9 @@ function S3Storage (opts) { switch (typeof opts.contentType) { case 'function': this.getContentType = opts.contentType; break + case 'string': this.getContentType = staticValue(opts.contentType); break case 'undefined': this.getContentType = defaultContentType; break - default: throw new TypeError('Expected opts.contentType to be undefined or function') + default: throw new TypeError('Expected opts.contentType to be undefined, string or function') } switch (typeof opts.metadata) { @@ -231,16 +232,23 @@ function S3Storage (opts) { default: throw new TypeError('Expected opts.transformer[].transform to be function') } + switch (typeof transformer.contentType) { + case 'function': transformer.getContentType = transformer.contentType; break + case 'string': transformer.getContentType = staticValue(transformer.contentType); break + case 'undefined': transformer.getContentType = staticValue(undefined); break + default: throw new TypeError('Expected opts.transformer[].contentType to be undefined, string or function') + } + return transformer }) } -S3Storage.prototype.directUpload = function (opts, file, cb, piper, key, id) { +S3Storage.prototype.directUpload = function (opts, file, cb) { var currentSize = 0 var params = { Bucket: opts.bucket, - Key: key || opts.key, + Key: opts.key, ACL: opts.acl, CacheControl: opts.cacheControl, ContentType: opts.contentType, @@ -248,7 +256,7 @@ S3Storage.prototype.directUpload = function (opts, file, cb, piper, key, id) { StorageClass: opts.storageClass, ServerSideEncryption: opts.serverSideEncryption, SSEKMSKeyId: opts.sseKmsKeyId, - Body: piper ? (opts.replacementStream || file.stream).pipe(piper) : (opts.replacementStream || file.stream) + Body: opts.piper ? (opts.replacementStream || file.stream).pipe(opts.piper) : (opts.replacementStream || file.stream) } if (opts.contentDisposition) { @@ -272,10 +280,10 @@ S3Storage.prototype.directUpload = function (opts, file, cb, piper, key, id) { if (err) return cb(err) cb(null, { - id: id, + id: opts.id, size: currentSize, bucket: opts.bucket, - key: key || opts.key, + key: opts.key, acl: opts.acl, contentType: opts.contentType, contentDisposition: opts.contentDisposition, @@ -303,16 +311,28 @@ S3Storage.prototype.transformUpload = function (opts, req, file, cb) { if (err) return cb(err) keys.forEach(function (key, i) { - storage.transformers[i].transform(req, file, function (err, piper) { + var transform = storage.transformers[i].transform.bind(storage, req, file) + var getContentType = storage.transformers[i].getContentType.bind(storage, req, file) + + getContentType(function (err, contentType) { if (err) return cb(err) - var id = storage.transformers[i].id || i - storage.directUpload(opts, file, function (err, result) { + transform(function (err, piper) { if (err) return cb(err) - transformations[id] = result - if (--pending === 0) cb(null, { transformations: transformations }) - }, piper, key, id) + var transformerOpts = Object.assign({}, opts) + transformerOpts.id = storage.transformers[i].id || i + transformerOpts.key = key + transformerOpts.contentType = contentType || transformerOpts.contentType + transformerOpts.piper = piper + + storage.directUpload(transformerOpts, file, function (err, result) { + if (err) return cb(err) + + transformations[transformerOpts.id] = result + if (--pending === 0) cb(null, { transformations: transformations }) + }) + }) }) }) } diff --git a/test/basic.js b/test/basic.js index 1171182..a6f3ae6 100644 --- a/test/basic.js +++ b/test/basic.js @@ -314,7 +314,8 @@ describe('Multer S3', function () { }, transform: function (req, file, cb) { cb(null, new stream.PassThrough()) - } + }, + contentType: 'image/webp' } ] }) @@ -337,112 +338,118 @@ describe('Multer S3', function () { assert.equal(req.file.transformations.original.bucket, 'test') assert.equal(req.file.transformations.original.etag, 'mock-etag') assert.equal(req.file.transformations.original.location, 'https//test.hostname/original') + assert.equal(req.file.transformations.original.contentType, 'image/png') assert.equal(req.file.transformations.thumbnail.size, 68) assert.equal(req.file.transformations.thumbnail.bucket, 'test') assert.equal(req.file.transformations.thumbnail.etag, 'mock-etag') assert.equal(req.file.transformations.thumbnail.location, 'https//test.hostname/thumbnail') + assert.equal(req.file.transformations.thumbnail.contentType, 'image/webp') done() }) }) - // it('uploads a PNG and SVG file with transforms', function (done) { - // var s3 = mockS3() - // var form = new FormData() - // var storage = multerS3({ - // s3: s3, - // bucket: 'test', - // contentType: multerS3.AUTO_CONTENT_TYPE, - // shouldTransform: true, - // transformers: [ - // { - // id: 'original', - // key: 'original', - // transform: function (req, file, cb) { - // cb(null, new stream.PassThrough()) - // } - // }, - // { - // id: 'thumbnail', - // key: function (req, file, cb) { - // cb(null, 'thumbnail') - // }, - // transform: function (req, file, cb) { - // cb(null, new stream.PassThrough()) - // } - // } - // ] - // }) - // var upload = multer({ storage: storage }) - // var parser = upload.array('images', 2) - // var image1 = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) - // var image2 = fs.createReadStream(path.join(__dirname, 'files', 'test2.svg')) - - // form.append('name', 'Multer') - // form.append('images', image1) - // form.append('images', image2) - - // submitForm(parser, form, function (err, req) { - // assert.ifError(err) - - // assert.equal(req.body.name, 'Multer') - - // assert.equal(req.files[0].fieldname, 'images') - // assert.equal(req.files[0].mimetype, 'image/png') - // assert.equal(req.files[0].originalname, 'ffffff.png') - // assert.equal(req.files[0].transformations.original.size, 68) - // assert.equal(req.files[0].transformations.original.bucket, 'test') - // assert.equal(req.files[0].transformations.original.etag, 'mock-etag') - // assert.equal(req.files[0].transformations.original.location, 'https//test.hostname/original') - // assert.equal(req.files[0].transformations.thumbnail.size, 68) - // assert.equal(req.files[0].transformations.thumbnail.bucket, 'test') - // assert.equal(req.files[0].transformations.thumbnail.etag, 'mock-etag') - // assert.equal(req.files[0].transformations.thumbnail.location, 'https//test.hostname/thumbnail') - - // assert.equal(req.files[1].fieldname, 'images') - // assert.equal(req.files[1].mimetype, 'image/svg+xml') - // assert.equal(req.files[1].originalname, 'test2.svg') - // }) - // }) - - // it('uploads a PNG and SVG file', function (done) { - // var s3 = mockS3() - // var form = new FormData() - // var storage = multerS3({ - // s3: s3, - // bucket: 'test', - // key: 'mock-location', - // contentType: multerS3.AUTO_CONTENT_TYPE - // }) - // var upload = multer({ storage: storage }) - // var parser = upload.array('images', 2) - // var image1 = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) - // var image2 = fs.createReadStream(path.join(__dirname, 'files', 'test2.svg')) - - // form.append('name', 'Multer') - // form.append('images', image1) - // form.append('images', image2) - - // submitForm(parser, form, function (err, req) { - // assert.ifError(err) - - // assert.equal(req.body.name, 'Multer') - - // assert.equal(req.files[0].fieldname, 'images') - // assert.equal(req.files[0].mimetype, 'image/png') - // assert.equal(req.files[0].originalname, 'ffffff.png') - // assert.equal(req.files[0].size, 68) - // assert.equal(req.files[0].bucket, 'test') - // assert.equal(req.files[0].etag, 'mock-etag') - // assert.equal(req.files[0].location, 'https//test.hostname/mock-location') - - // assert.equal(req.files[1].fieldname, 'images') - // assert.equal(req.files[1].mimetype, 'image/svg+xml') - // assert.equal(req.files[1].originalname, 'test2.svg') - // assert.equal(req.files[1].size, 285) - // assert.equal(req.files[1].bucket, 'test') - // assert.equal(req.files[1].etag, 'mock-etag') - // assert.equal(req.files[1].location, 'https//test.hostname/mock-location') - // }) - // }) + it('uploads a PNG and SVG file with transforms', function (done) { + var s3 = mockS3() + var form = new FormData() + var storage = multerS3({ + s3: s3, + bucket: 'test', + contentType: multerS3.AUTO_CONTENT_TYPE, + shouldTransform: true, + transformers: [ + { + id: 'original', + key: 'original', + transform: function (req, file, cb) { + cb(null, new stream.PassThrough()) + } + }, + { + id: 'thumbnail', + key: function (req, file, cb) { + cb(null, 'thumbnail') + }, + transform: function (req, file, cb) { + cb(null, new stream.PassThrough()) + } + } + ] + }) + var upload = multer({ storage: storage }) + var parser = upload.array('images', 2) + var image1 = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) + var image2 = fs.createReadStream(path.join(__dirname, 'files', 'test2.svg')) + + form.append('name', 'Multer') + form.append('images', image1) + form.append('images', image2) + + submitForm(parser, form, function (err, req) { + assert.ifError(err) + + assert.equal(req.body.name, 'Multer') + + assert.equal(req.files[0].fieldname, 'images') + assert.equal(req.files[0].mimetype, 'image/png') + assert.equal(req.files[0].originalname, 'ffffff.png') + assert.equal(req.files[0].transformations.original.size, 68) + assert.equal(req.files[0].transformations.original.bucket, 'test') + assert.equal(req.files[0].transformations.original.etag, 'mock-etag') + assert.equal(req.files[0].transformations.original.location, 'https//test.hostname/original') + assert.equal(req.files[0].transformations.thumbnail.size, 68) + assert.equal(req.files[0].transformations.thumbnail.bucket, 'test') + assert.equal(req.files[0].transformations.thumbnail.etag, 'mock-etag') + assert.equal(req.files[0].transformations.thumbnail.location, 'https//test.hostname/thumbnail') + + assert.equal(req.files[1].fieldname, 'images') + assert.equal(req.files[1].mimetype, 'image/svg+xml') + assert.equal(req.files[1].originalname, 'test2.svg') + + done() + }) + }) + + it('uploads a PNG and SVG file', function (done) { + var s3 = mockS3() + var form = new FormData() + var storage = multerS3({ + s3: s3, + bucket: 'test', + key: 'mock-location', + contentType: multerS3.AUTO_CONTENT_TYPE + }) + var upload = multer({ storage: storage }) + var parser = upload.array('images', 2) + var image1 = fs.createReadStream(path.join(__dirname, 'files', 'ffffff.png')) + var image2 = fs.createReadStream(path.join(__dirname, 'files', 'test2.svg')) + + form.append('name', 'Multer') + form.append('images', image1) + form.append('images', image2) + + submitForm(parser, form, function (err, req) { + assert.ifError(err) + + assert.equal(req.body.name, 'Multer') + + assert.equal(req.files[0].fieldname, 'images') + assert.equal(req.files[0].mimetype, 'image/png') + assert.equal(req.files[0].originalname, 'ffffff.png') + assert.equal(req.files[0].size, 68) + assert.equal(req.files[0].bucket, 'test') + assert.equal(req.files[0].etag, 'mock-etag') + assert.equal(req.files[0].location, 'https//test.hostname/mock-location') + + assert.equal(req.files[1].fieldname, 'images') + assert.equal(req.files[1].mimetype, 'image/svg+xml') + assert.equal(req.files[1].originalname, 'test2.svg') + assert.equal(req.files[1].size, 285) + assert.equal(req.files[1].bucket, 'test') + assert.equal(req.files[1].etag, 'mock-etag') + assert.equal(req.files[1].location, 'https//test.hostname/mock-location') + + done() + }) + }) })