From 49eb08e5048fee6a8398bf815300a813b05c8c7f Mon Sep 17 00:00:00 2001 From: bikeath1337 Date: Wed, 11 Oct 2017 07:14:37 -0400 Subject: [PATCH 1/8] Attempts to make checksum calculation cooperative in a web worker. --- evaporate.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evaporate.js b/evaporate.js index 945e5a4a..9dd0381c 100644 --- a/evaporate.js +++ b/evaporate.js @@ -1381,8 +1381,8 @@ return new Promise(function (resolve, reject) { if (self.con.computeContentMd5 && !part.md5_digest) { self.getPayload() - .then(function (data) { - var md5_digest = self.con.cryptoMd5Method(data); + .then(self.con.cryptoMd5Method) + .then(function (md5_digest) { if (self.partNumber === 1 && self.con.computeContentMd5 && typeof self.fileUpload.firstMd5Digest === "undefined") { self.fileUpload.firstMd5Digest = md5_digest; self.fileUpload.updateUploadFile({firstMd5Digest: md5_digest}) From 9a23bb96484de1cad9cef7334772f93e1214d681 Mon Sep 17 00:00:00 2001 From: bikeath1337 Date: Sun, 22 Oct 2017 08:14:10 -0400 Subject: [PATCH 2/8] Adds new option (default: true) `enablePartSizeOptimization` and sets this option to `false` so that existing specs pass --- evaporate.js | 1 + test/helpers/browser-env.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/evaporate.js b/evaporate.js index 9dd0381c..11b1ae5c 100644 --- a/evaporate.js +++ b/evaporate.js @@ -60,6 +60,7 @@ logging: true, maxConcurrentParts: 5, partSize: 6 * 1024 * 1024, + enablePartSizeOptimization: true, retryBackoffPower: 2, maxRetryBackoffSecs: 300, progressIntervalMS: 1000, diff --git a/test/helpers/browser-env.js b/test/helpers/browser-env.js index 98473f67..9ab5349d 100644 --- a/test/helpers/browser-env.js +++ b/test/helpers/browser-env.js @@ -45,7 +45,8 @@ const baseConfig = { bucket: AWS_BUCKET, logging: false, maxRetryBackoffSecs: 0.1, - abortCompletionThrottlingMs: 0 + abortCompletionThrottlingMs: 0, + enablePartSizeOptimization: false } function LocalStorage() { From ce55ae654c95cd37de55fa6d21c4a0d1a838f1fa Mon Sep 17 00:00:00 2001 From: bikeath1337 Date: Sun, 22 Oct 2017 15:41:32 -0400 Subject: [PATCH 3/8] Adds new spec helper mapping for the new PUT OBJECT request --- test/helpers/browser-env.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/helpers/browser-env.js b/test/helpers/browser-env.js index 9ab5349d..74465b5c 100644 --- a/test/helpers/browser-env.js +++ b/test/helpers/browser-env.js @@ -75,7 +75,8 @@ let requestMap = { 'POST:uploads': 'initiate', 'POST:uploadId': 'complete', 'DELETE:uploadId': 'cancel', - 'GET:uploadId': 'check for parts' + 'GET:uploadId': 'check for parts', + 'PUT': 'put object' } global.requestOrder = function (t) { From 4b95c4813759708937b7860209aa52ca22c7b444 Mon Sep 17 00:00:00 2001 From: bikeath1337 Date: Sun, 22 Oct 2017 15:42:22 -0400 Subject: [PATCH 4/8] Adds spec coverage for PUT OBJECT, making sure that initiate and complete requests do not fire but the callbacks do --- test/optimized-part-size.spec.js | 173 +++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 test/optimized-part-size.spec.js diff --git a/test/optimized-part-size.spec.js b/test/optimized-part-size.spec.js new file mode 100644 index 00000000..bf57806d --- /dev/null +++ b/test/optimized-part-size.spec.js @@ -0,0 +1,173 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import test from 'ava' + +// constants + +let server + + +function testCommon(t, addCfg, initConfig) { + let addConfig = Object.assign({}, { file: new File({ + path: '/tmp/file', + size: 50, + name: 'tests' + })}, addCfg) + + let evapConfig = Object.assign({}, {awsSignatureVersion: '2', enablePartSizeOptimization: true}, initConfig) + return testBase(t, addConfig, evapConfig) +} +function testMd5V2(t) { + return testCommon(t, {}, { awsSignatureVersion: '2', computeContentMd5: true }) +} + +function testMd5V4(t) { + return testCommon(t, {}, { + computeContentMd5: true, + cryptoHexEncodedHash256: function (d) { return d; } + }) +} + + + +test.before(() => { + sinon.xhr.supportsCORS = true + global.XMLHttpRequest = sinon.useFakeXMLHttpRequest() + global.window = { + localStorage: {}, + console: console + }; + + server = serverCommonCase() +}) + +test.beforeEach((t) => { + beforeEachSetup(t) +}) + +// Callbacks +test('should call a callback on successful add()', (t) => { + return testCommon(t) + .then(function () { + expect(t.context.config.started.withArgs('bucket/' + t.context.requestedAwsObjectKey).calledOnce).to.be.true + }) +}) +test('should call a progress with stats callback on successful add()', (t) => { + return testCommon(t, {progress: sinon.spy()}) + .then(function () { + expect(t.context.config.progress.firstCall.args.length).to.equal(2) + expect(typeof t.context.config.progress.firstCall.args[1]).to.equal('object') + }) +}) +test('should return the object key in the complete callback', (t) => { + let complete_id + + let config = Object.assign({}, {}, { + name: AWS_UPLOAD_KEY, + complete: sinon.spy(function (xhr, name) { complete_id = name; }) + }) + + return testCommon(t, config) + .then(function () { + expect(complete_id).to.equal(config.name) + expect(t.context.config.complete.firstCall.args.length).to.equal(3) + expect(t.context.config.complete.firstCall.args[0]).to.be.undefined + expect(typeof t.context.config.complete.firstCall.args[1]).to.equal('string') + expect(typeof t.context.config.complete.firstCall.args[2]).to.equal('object') + }) + +}) + + +// Default Setup: V2 signatures: Common Case +test('should not call cryptoMd5 upload a file with defaults and V2 signature', (t) => { + return testCommon(t, {}, { awsSignatureVersion: '2' }) + .then(function () { + expect(t.context.cryptoMd5.callCount).to.equal(0) + }) +}) +test('should upload a file with S3 requests in the correct order', (t) => { + return testCommon(t) + .then(function () { + expect(requestOrder(t)).to.equal('put object') + }) +}) +test('should upload a file and return the correct file upload ID', (t) => { + return testCommon(t) + .then(function () { + expect(t.context.completedAwsKey).to.equal(t.context.requestedAwsObjectKey) + }) +}) +test('should upload a file and callback complete once', (t) => { + return testCommon(t) + .then(function () { + expect(t.context.config.complete.calledOnce).to.be.true + }) +}) +test('should upload a file and callback complete with second param the awsKey', (t) => { + return testCommon(t) + .then(function () { + expect(t.context.config.complete.firstCall.args[1]).to.equal(t.context.requestedAwsObjectKey) + }) +}) +test('should upload a file and not callback with a changed object name', (t) => { + return testCommon(t, {nameChanged: sinon.spy()}) + .then(function () { + expect(t.context.config.nameChanged.callCount).to.equal(0) + }) +}) + +// md5Digest tests +test('V2 should call cryptoMd5 when uploading a file with defaults', (t) => { + return testMd5V2(t) + .then(function () { + expect(t.context.cryptoMd5.callCount).to.equal(1) + }) +}) +test('V2 should upload a file with MD5Digests with S3 requests in the correct order', (t) => { + return testMd5V2(t) + .then(function () { + expect(requestOrder(t)).to.equal('put object') + }) +}) +test('V2 should upload a file and return the correct file upload ID', (t) => { + return testMd5V2(t) + .then(function () { + expect(t.context.completedAwsKey).to.equal(t.context.requestedAwsObjectKey) + }) +}) + +test('V4 should call cryptoMd5 when uploading a file with defaults', (t) => { + return testMd5V4(t) + .then(function () { + expect(t.context.cryptoMd5.callCount).to.equal(1) + }) +}) +test('V4 should upload a file with MD5Digests with S3 requests in the correct order', (t) => { + return testMd5V4(t) + .then(function () { + expect(requestOrder(t)).to.equal('put object') + }) +}) +test('V4 should upload a file and return the correct file upload ID', (t) => { + return testMd5V4(t) + .then(function () { + expect(t.context.completedAwsKey).to.equal(t.context.requestedAwsObjectKey) + }) +}) + +test('should retry Upload Part', (t) => { + t.context.retry = function (type) { + return type === 'part' + } + + return testCommon(t, { file: new File({ + path: '/tmp/file', + size: 50, + name: 'tests' + }) + }) + .then(function () { + expect(requestOrder(t)).to.equal('put object,put object') + }) +}) From c0cfb4b6683b4f0d067a8429319c23fdf8c2536e Mon Sep 17 00:00:00 2001 From: bikeath1337 Date: Sun, 22 Oct 2017 15:42:49 -0400 Subject: [PATCH 5/8] We only need to calculate the number of parts once. --- evaporate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaporate.js b/evaporate.js index 11b1ae5c..2b8ddbb7 100644 --- a/evaporate.js +++ b/evaporate.js @@ -460,6 +460,7 @@ this.id = decodeURIComponent(this.con.bucket + '/' + this.name); this.signParams = con.signParams; + this.numParts = Math.ceil(this.sizeBytes / this.con.partSize) || 1; // issue #58 } FileUpload.prototype.con = undefined; FileUpload.prototype.evaporate = undefined; @@ -678,7 +679,6 @@ }); }; FileUpload.prototype.makeParts = function (firstPart) { - this.numParts = Math.ceil(this.sizeBytes / this.con.partSize) || 1; // issue #58 var partsDeferredPromises = []; var self = this; From 99465fa7a2662baf158142d604472116af9bf0ad Mon Sep 17 00:00:00 2001 From: bikeath1337 Date: Sun, 22 Oct 2017 15:44:51 -0400 Subject: [PATCH 6/8] Adds new subclass for Initiate and Complete so that they become NOOPs when used for files of one part. This new class simply resolves its promise rather than make any requests --- evaporate.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/evaporate.js b/evaporate.js index 2b8ddbb7..7b467daf 100644 --- a/evaporate.js +++ b/evaporate.js @@ -1210,6 +1210,20 @@ return [ABORTED, CANCELED].indexOf(this.fileUpload.status) > -1; }; + function CancelableS3MultipartRequest(fileUpload, request) { + CancelableS3AWSRequest.call(this, fileUpload, request); + } + CancelableS3MultipartRequest.prototype = Object.create(CancelableS3AWSRequest.prototype); + CancelableS3MultipartRequest.prototype.constructor = CancelableS3MultipartRequest; + CancelableS3MultipartRequest.prototype.send = function () { + if (this.fileUpload.numParts === 1 && this.con.enablePartSizeOptimization) { + this.awsDeferred.resolve(); + } else { + this.trySend(); + } + return this.awsDeferred.promise; + }; + function SignedS3AWSRequestWithRetryLimit(fileUpload, request, maxRetries) { if (maxRetries > -1) { this.maxRetries = maxRetries; @@ -1244,10 +1258,10 @@ response_match: '(.+)<\/UploadId>' }; - CancelableS3AWSRequest.call(this, fileUpload, request); + CancelableS3MultipartRequest.call(this, fileUpload, request); this.awsKey = awsKey; } - InitiateMultipartUpload.prototype = Object.create(CancelableS3AWSRequest.prototype); + InitiateMultipartUpload.prototype = Object.create(CancelableS3MultipartRequest.prototype); InitiateMultipartUpload.prototype.constructor = InitiateMultipartUpload; InitiateMultipartUpload.prototype.success = function () { var match = this.currentXhr.response.match(new RegExp(this.request.response_match)); @@ -1268,9 +1282,9 @@ x_amz_headers: fileUpload.xAmzHeadersCommon || fileUpload.xAmzHeadersAtComplete, step: 'complete' }; - CancelableS3AWSRequest.call(this, fileUpload, request); + CancelableS3MultipartRequest.call(this, fileUpload, request); } - CompleteMultipartUpload.prototype = Object.create(CancelableS3AWSRequest.prototype); + CompleteMultipartUpload.prototype = Object.create(CancelableS3MultipartRequest.prototype); CompleteMultipartUpload.prototype.constructor = CompleteMultipartUpload; CompleteMultipartUpload.prototype.getPayload = function () { return Promise.resolve(this.fileUpload.getCompletedPayload()); From 009d080f2cdd5072fa2a3a0a1af87b9b9096d4b4 Mon Sep 17 00:00:00 2001 From: bikeath1337 Date: Sun, 22 Oct 2017 15:46:10 -0400 Subject: [PATCH 7/8] Creates a new subclass of PutPart, PutObject, that implements http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html --- evaporate.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/evaporate.js b/evaporate.js index 7b467daf..f34aff7d 100644 --- a/evaporate.js +++ b/evaporate.js @@ -1592,6 +1592,27 @@ }; + //http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html + function PutObject(fileUpload, part) { + this.part = part; + + this.partNumber = 1; + this.start = 0; + this.end = fileUpload.sizeBytes; + + var request = { + method: 'PUT', + step: 'upload #' + this.partNumber, + x_amz_headers: fileUpload.xAmzHeadersCommon || fileUpload.xAmzHeadersAtUpload, + onProgress: this.onProgress.bind(this) + }; + + SignedS3AWSRequest.call(this, fileUpload, request); + } + PutObject.prototype = Object.create(PutPart.prototype); + PutObject.prototype.constructor = PutObject; + + //http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadAbort.html function DeleteMultipartUpload(fileUpload) { fileUpload.info('will attempt to abort the upload'); From d519655750199cf17ffdc2438b0593501b55d7a7 Mon Sep 17 00:00:00 2001 From: bikeath1337 Date: Sun, 22 Oct 2017 15:47:15 -0400 Subject: [PATCH 8/8] Conditionally applies PutObject to use when a the optimization is enabled and the file has only one part. --- evaporate.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/evaporate.js b/evaporate.js index f34aff7d..3c15c223 100644 --- a/evaporate.js +++ b/evaporate.js @@ -712,7 +712,7 @@ } else { s3Part = this.makePart(part, PENDING, this.sizeBytes); } - s3Part.awsRequest = new PutPart(this, s3Part); + s3Part.awsRequest = this.numParts === 1 && this.con.enablePartSizeOptimization ? new PutObject(this, s3Part) : new PutPart(this, s3Part); s3Part.awsRequest.awsDeferred.promise .then(resolve(s3Part), reject(s3Part)); @@ -863,7 +863,11 @@ .send() .then( function (xhr) { - self.eTag = elementText(xhr.responseText, "ETag").replace(/"/g, '"'); + if (self.numParts === 1 && self.con.enablePartSizeOptimization) { + self.eTag = self.partsOnS3[0].eTag; + } else { + self.eTag = elementText(xhr.responseText, "ETag").replace(/"/g, '"'); + } self.completeUploadFile(xhr); }); };