Adds Automatic Support to Not Use Multipart Uploads If File Size <= partSize

evaporate.js

@@ -60,6 +60,7 @@
       logging: true,
       maxConcurrentParts: 5,
       partSize: 6 * 1024 * 1024,
+      enablePartSizeOptimization: true,
       retryBackoffPower: 2,
       maxRetryBackoffSecs: 300,
       progressIntervalMS: 1000,
@@ -459,6 +460,7 @@
     this.id = decodeURIComponent(this.con.bucket + '/' + this.name);
 
     this.signParams = con.signParams;
+    this.numParts = Math.ceil(this.sizeBytes / this.con.partSize) || 1; // issue #58
   }
   FileUpload.prototype.con = undefined;
   FileUpload.prototype.evaporate = undefined;
@@ -677,7 +679,6 @@
     });
   };
   FileUpload.prototype.makeParts = function (firstPart) {
-    this.numParts = Math.ceil(this.sizeBytes / this.con.partSize) || 1; // issue #58
     var partsDeferredPromises = [];
 
     var self = this;
@@ -711,7 +712,7 @@
       } else {
         s3Part = this.makePart(part, PENDING, this.sizeBytes);
       }
-      s3Part.awsRequest = new PutPart(this, s3Part);
+      s3Part.awsRequest = this.numParts === 1 && this.con.enablePartSizeOptimization ?  new PutObject(this, s3Part) : new PutPart(this, s3Part);
       s3Part.awsRequest.awsDeferred.promise
           .then(resolve(s3Part), reject(s3Part));
 
@@ -862,7 +863,11 @@
         .send()
         .then(
             function (xhr) {
-              self.eTag = elementText(xhr.responseText, "ETag").replace(/"/g, '"');
+              if (self.numParts === 1 && self.con.enablePartSizeOptimization) {
+                self.eTag = self.partsOnS3[0].eTag;
+              } else {
+                self.eTag = elementText(xhr.responseText, "ETag").replace(/"/g, '"');
+              }
               self.completeUploadFile(xhr);
             });
   };
@@ -1209,6 +1214,20 @@
     return [ABORTED, CANCELED].indexOf(this.fileUpload.status) > -1;
   };
 
+  function CancelableS3MultipartRequest(fileUpload, request) {
+    CancelableS3AWSRequest.call(this, fileUpload, request);
+  }
+  CancelableS3MultipartRequest.prototype = Object.create(CancelableS3AWSRequest.prototype);
+  CancelableS3MultipartRequest.prototype.constructor = CancelableS3MultipartRequest;
+  CancelableS3MultipartRequest.prototype.send = function () {
+    if (this.fileUpload.numParts === 1 && this.con.enablePartSizeOptimization) {
+      this.awsDeferred.resolve();
+    } else {
+      this.trySend();
+    }
+    return this.awsDeferred.promise;
+  };
+
   function SignedS3AWSRequestWithRetryLimit(fileUpload, request, maxRetries) {
     if (maxRetries > -1) {
       this.maxRetries = maxRetries;
@@ -1243,10 +1262,10 @@
       response_match: '<UploadId>(.+)<\/UploadId>'
     };
 
-    CancelableS3AWSRequest.call(this, fileUpload, request);
+    CancelableS3MultipartRequest.call(this, fileUpload, request);
     this.awsKey = awsKey;
   }
-  InitiateMultipartUpload.prototype = Object.create(CancelableS3AWSRequest.prototype);
+  InitiateMultipartUpload.prototype = Object.create(CancelableS3MultipartRequest.prototype);
   InitiateMultipartUpload.prototype.constructor = InitiateMultipartUpload;
   InitiateMultipartUpload.prototype.success = function () {
     var match = this.currentXhr.response.match(new RegExp(this.request.response_match));
@@ -1267,9 +1286,9 @@
       x_amz_headers: fileUpload.xAmzHeadersCommon || fileUpload.xAmzHeadersAtComplete,
       step: 'complete'
     };
-    CancelableS3AWSRequest.call(this, fileUpload, request);
+    CancelableS3MultipartRequest.call(this, fileUpload, request);
   }
-  CompleteMultipartUpload.prototype = Object.create(CancelableS3AWSRequest.prototype);
+  CompleteMultipartUpload.prototype = Object.create(CancelableS3MultipartRequest.prototype);
   CompleteMultipartUpload.prototype.constructor = CompleteMultipartUpload;
   CompleteMultipartUpload.prototype.getPayload = function () {
     return Promise.resolve(this.fileUpload.getCompletedPayload());
@@ -1381,8 +1400,8 @@
     return new Promise(function (resolve, reject) {
       if (self.con.computeContentMd5 && !part.md5_digest) {
         self.getPayload()
-            .then(function (data) {
-              var md5_digest = self.con.cryptoMd5Method(data);
+            .then(self.con.cryptoMd5Method)
+            .then(function (md5_digest) {
               if (self.partNumber === 1 && self.con.computeContentMd5 && typeof self.fileUpload.firstMd5Digest === "undefined") {
                 self.fileUpload.firstMd5Digest = md5_digest;
                 self.fileUpload.updateUploadFile({firstMd5Digest: md5_digest})
@@ -1577,6 +1596,27 @@
   };
 
 
+  //http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
+  function PutObject(fileUpload, part) {
+    this.part = part;
+
+    this.partNumber = 1;
+    this.start = 0;
+    this.end = fileUpload.sizeBytes;
+
+    var request = {
+      method: 'PUT',
+      step: 'upload #' + this.partNumber,
+      x_amz_headers: fileUpload.xAmzHeadersCommon || fileUpload.xAmzHeadersAtUpload,
+      onProgress: this.onProgress.bind(this)
+    };
+
+    SignedS3AWSRequest.call(this, fileUpload, request);
+  }
+  PutObject.prototype = Object.create(PutPart.prototype);
+  PutObject.prototype.constructor = PutObject;
+
+
   //http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadAbort.html
   function DeleteMultipartUpload(fileUpload) {
     fileUpload.info('will attempt to abort the upload');

test/helpers/browser-env.js

@@ -45,7 +45,8 @@ const baseConfig = {
   bucket: AWS_BUCKET,
   logging: false,
   maxRetryBackoffSecs: 0.1,
-  abortCompletionThrottlingMs: 0
+  abortCompletionThrottlingMs: 0,
+  enablePartSizeOptimization: false
 }
 
 function LocalStorage() {
@@ -74,7 +75,8 @@ let requestMap = {
   'POST:uploads': 'initiate',
   'POST:uploadId': 'complete',
   'DELETE:uploadId': 'cancel',
-  'GET:uploadId': 'check for parts'
+  'GET:uploadId': 'check for parts',
+  'PUT': 'put object'
 }
 
 global.requestOrder = function (t) {

test/optimized-part-size.spec.js

@@ -0,0 +1,173 @@
+import { expect } from 'chai'
+import sinon from 'sinon'
+import test from 'ava'
+
+// constants
+
+let server
+
+
+function testCommon(t, addCfg, initConfig) {
+  let addConfig = Object.assign({}, { file: new File({
+    path: '/tmp/file',
+    size: 50,
+    name: 'tests'
+  })}, addCfg)
+
+  let evapConfig = Object.assign({}, {awsSignatureVersion: '2', enablePartSizeOptimization: true}, initConfig)
+  return testBase(t, addConfig, evapConfig)
+}
+function testMd5V2(t) {
+  return testCommon(t, {}, { awsSignatureVersion: '2', computeContentMd5: true })
+}
+
+function testMd5V4(t) {
+  return testCommon(t, {}, {
+    computeContentMd5: true,
+    cryptoHexEncodedHash256: function (d) { return d; }
+  })
+}
+
+
+
+test.before(() => {
+  sinon.xhr.supportsCORS = true
+  global.XMLHttpRequest = sinon.useFakeXMLHttpRequest()
+  global.window = {
+    localStorage: {},
+    console: console
+  };
+
+  server = serverCommonCase()
+})
+
+test.beforeEach((t) => {
+  beforeEachSetup(t)
+})
+
+// Callbacks
+test('should call a callback on successful add()', (t) => {
+  return testCommon(t)
+      .then(function () {
+        expect(t.context.config.started.withArgs('bucket/' + t.context.requestedAwsObjectKey).calledOnce).to.be.true
+      })
+})
+test('should call a progress with stats callback on successful add()', (t) => {
+  return testCommon(t, {progress: sinon.spy()})
+      .then(function () {
+        expect(t.context.config.progress.firstCall.args.length).to.equal(2)
+        expect(typeof t.context.config.progress.firstCall.args[1]).to.equal('object')
+      })
+})
+test('should return the object key in the complete callback', (t) => {
+  let complete_id
+
+  let config = Object.assign({}, {}, {
+    name:  AWS_UPLOAD_KEY,
+    complete: sinon.spy(function (xhr, name) { complete_id = name; })
+  })
+
+  return testCommon(t, config)
+      .then(function () {
+        expect(complete_id).to.equal(config.name)
+        expect(t.context.config.complete.firstCall.args.length).to.equal(3)
+        expect(t.context.config.complete.firstCall.args[0]).to.be.undefined
+        expect(typeof t.context.config.complete.firstCall.args[1]).to.equal('string')
+        expect(typeof t.context.config.complete.firstCall.args[2]).to.equal('object')
+      })
+
+})
+
+
+// Default Setup: V2 signatures: Common Case
+test('should not call cryptoMd5 upload a file with defaults and V2 signature', (t) => {
+  return testCommon(t, {}, { awsSignatureVersion: '2' })
+      .then(function () {
+        expect(t.context.cryptoMd5.callCount).to.equal(0)
+      })
+})
+test('should upload a file with S3 requests in the correct order', (t) => {
+  return testCommon(t)
+      .then(function () {
+        expect(requestOrder(t)).to.equal('put object')
+      })
+})
+test('should upload a file and return the correct file upload ID', (t) => {
+  return testCommon(t)
+      .then(function () {
+        expect(t.context.completedAwsKey).to.equal(t.context.requestedAwsObjectKey)
+      })
+})
+test('should upload a file and callback complete once', (t) => {
+  return testCommon(t)
+      .then(function () {
+        expect(t.context.config.complete.calledOnce).to.be.true
+      })
+})
+test('should upload a file and callback complete with second param the awsKey', (t) => {
+  return testCommon(t)
+      .then(function () {
+        expect(t.context.config.complete.firstCall.args[1]).to.equal(t.context.requestedAwsObjectKey)
+      })
+})
+test('should upload a file and not callback with a changed object name', (t) => {
+  return testCommon(t, {nameChanged: sinon.spy()})
+      .then(function () {
+        expect(t.context.config.nameChanged.callCount).to.equal(0)
+      })
+})
+
+// md5Digest tests
+test('V2 should call cryptoMd5 when uploading a file with defaults', (t) => {
+  return testMd5V2(t)
+      .then(function () {
+        expect(t.context.cryptoMd5.callCount).to.equal(1)
+      })
+})
+test('V2 should upload a file with MD5Digests with S3 requests in the correct order', (t) => {
+  return testMd5V2(t)
+      .then(function () {
+        expect(requestOrder(t)).to.equal('put object')
+      })
+})
+test('V2 should upload a file and return the correct file upload ID', (t) => {
+  return testMd5V2(t)
+      .then(function () {
+        expect(t.context.completedAwsKey).to.equal(t.context.requestedAwsObjectKey)
+      })
+})
+
+test('V4 should call cryptoMd5 when uploading a file with defaults', (t) => {
+  return testMd5V4(t)
+      .then(function () {
+        expect(t.context.cryptoMd5.callCount).to.equal(1)
+      })
+})
+test('V4 should upload a file with MD5Digests with S3 requests in the correct order', (t) => {
+  return testMd5V4(t)
+      .then(function () {
+        expect(requestOrder(t)).to.equal('put object')
+      })
+})
+test('V4 should upload a file and return the correct file upload ID', (t) => {
+  return testMd5V4(t)
+      .then(function () {
+        expect(t.context.completedAwsKey).to.equal(t.context.requestedAwsObjectKey)
+      })
+})
+
+test('should retry Upload Part', (t) => {
+  t.context.retry = function (type) {
+    return type === 'part'
+  }
+
+  return testCommon(t, { file: new File({
+        path: '/tmp/file',
+        size: 50,
+        name: 'tests'
+      })
+    })
+      .then(function () {
+        expect(requestOrder(t)).to.equal('put object,put object')
+      })
+})