Add rate limiter to imports (#1101)

* Add rate limiter to imports

* Reduce limit to 50 requests per sec

lib/utils/import/index.js

@@ -1,3 +1,4 @@
+import Bottleneck from 'bottleneck';
 import { EventEmitter } from 'events';
 import { isEmpty, get, pick } from 'lodash';
 
@@ -11,11 +12,21 @@ const propertyWhitelist = [
   'tags',
 ];
 
+const MAX_REQUESTS_PER_SEC = 50;
+
 class CoreImporter extends EventEmitter {
   constructor({ noteBucket, tagBucket }) {
     super();
     this.noteBucket = noteBucket;
     this.tagBucket = tagBucket;
+
+    // Rate limiter for adding new notes to the bucket. Without this, the
+    // server may return 503 errors, which will result in unsynced notes.
+    this.limiter = new Bottleneck({
+      reservoir: MAX_REQUESTS_PER_SEC,
+      reservoirRefreshAmount: MAX_REQUESTS_PER_SEC,
+      reservoirRefreshInterval: 1000, // Must be divisible by 250. See https://github.com/SGrondin/bottleneck/issues/88
+    });
   }
 
   importNote = (note, { isTrashed = false, isMarkdown = false } = {}) => {
@@ -65,7 +76,10 @@ class CoreImporter extends EventEmitter {
       });
     }
 
-    return this.noteBucket.add(importedNote);
+    // Add to note bucket with rate limiting
+    return this.limiter
+      .schedule(() => this.noteBucket.add.bind(this.noteBucket)(importedNote))
+      .catch(console.log);
   };
 
   importNotes = (notes = {}, options) => {

lib/utils/import/test.js

@@ -17,19 +17,19 @@ describe('CoreImporter', () => {
   describe('importNote', () => {
     it('should call noteBucket.add() with a note containing the required properties', () => {
       const note = {};
-      importer.importNote(note);
+      return importer.importNote(note).then(() => {
+        const passedNote = noteBucketAdd.mock.calls[0][0];
 
-      const passedNote = noteBucketAdd.mock.calls[0][0];
-
-      // Conforms to schema
-      expect(passedNote.publishURL).toBe('');
-      expect(passedNote.shareURL).toBe('');
-      expect(passedNote.deleted).toBe(false);
-      expect(passedNote.tags).toEqual([]);
-      expect(passedNote.systemTags).toEqual([]);
-      expect(passedNote.creationDate).toEqual(expect.any(Number));
-      expect(passedNote.modificationDate).toEqual(expect.any(Number));
-      expect(passedNote.content).toBe('');
+        // Conforms to schema
+        expect(passedNote.publishURL).toBe('');
+        expect(passedNote.shareURL).toBe('');
+        expect(passedNote.deleted).toBe(false);
+        expect(passedNote.tags).toEqual([]);
+        expect(passedNote.systemTags).toEqual([]);
+        expect(passedNote.creationDate).toEqual(expect.any(Number));
+        expect(passedNote.modificationDate).toEqual(expect.any(Number));
+        expect(passedNote.content).toBe('');
+      });
     });
   });
 

package.json

@@ -100,6 +100,7 @@
   "dependencies": {
     "@babel/polyfill": "7.0.0",
     "@material-ui/core": "3.3.1",
+    "bottleneck": "2.14.0",
     "cookie": "0.3.1",
     "core-js": "2.5.7",
     "create-hash": "1.1.3",