diff --git a/src/util/actor.js b/src/util/actor.js
index 12c066d931a..0f10ab51593 100644
--- a/src/util/actor.js
+++ b/src/util/actor.js
@@ -113,7 +113,8 @@ class Actor {
                 // We're using a MessageChannel object to get throttle the process() flow to one at a time.
                 const callback = this.callbacks[id];
                 const metadata = (callback && callback.metadata) || {type: "message"};
-                this.cancelCallbacks[id] = this.scheduler.add(() => this.processTask(id, data), metadata);
+                const cancel = this.scheduler.add(() => this.processTask(id, data), metadata);
+                if (cancel) this.cancelCallbacks[id] = cancel;
             } else {
                 // In the main thread, process messages immediately so that other work does not slip in
                 // between getting partial data back from workers.
@@ -123,6 +124,8 @@ class Actor {
     }
 
     processTask(id: number, task: any) {
+        // Always delete since we are no longer cancellable
+        delete this.cancelCallbacks[id];
         if (task.type === '<response>') {
             // The done() function in the counterpart has been called, and we are now
             // firing the callback in the originating actor, if there is one.
@@ -139,7 +142,6 @@ class Actor {
         } else {
             const buffers: Set<Transferable> = new Set();
             const done = task.hasCallback ? (err: ?Error, data: mixed) => {
-                delete this.cancelCallbacks[id];
                 this.target.postMessage({
                     id,
                     type: '<response>',
diff --git a/src/util/scheduler.js b/src/util/scheduler.js
index 1b603d59431..0f7112a936c 100644
--- a/src/util/scheduler.js
+++ b/src/util/scheduler.js
@@ -38,7 +38,7 @@ class Scheduler {
         this.nextId = 0;
     }
 
-    add(fn: TaskFunction, metadata: TaskMetadata): Cancelable {
+    add(fn: TaskFunction, metadata: TaskMetadata): Cancelable | null {
         const id = this.nextId++;
         const priority = getPriority(metadata);
 
@@ -50,9 +50,8 @@ class Scheduler {
             } finally {
                 if (m) PerformanceUtils.endMeasure(m);
             }
-            return {
-                cancel: () => {}
-            };
+            // Don't return an empty cancel because we can't actually be cancelled
+            return null;
         }
 
         this.tasks[id] = {fn, metadata, priority, id};