diff --git a/CHANGES.rst b/CHANGES.rst
index 1b94727c6..588bfa388 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -26,7 +26,7 @@ New Features
 
   - Added new ``PSFPhotometry``, ``IterativePSFPhotometry``, and
     ``SourceGrouper`` classes. [#1558, #1559, #1563, #1566, #1567,
-    #1581, #1586, #1590, #1594, #1603]
+    #1581, #1586, #1590, #1594, #1603, #1604]
 
   - Added a ``GriddedPSFModel`` ``fill_value`` attribute, [#1583]
 
diff --git a/photutils/psf/photometry.py b/photutils/psf/photometry.py
index 4e5bb7fa5..9ff04e2ee 100644
--- a/photutils/psf/photometry.py
+++ b/photutils/psf/photometry.py
@@ -1060,7 +1060,8 @@ def make_residual_image(self, data, psf_shape):
             if isinstance(data, u.Quantity):
                 unit = data.unit
                 data = data.value
-            residual = data - self.make_model_image(data.shape, psf_shape)
+            residual = -self.make_model_image(data.shape, psf_shape)
+            residual += data
 
             if unit is not None:
                 residual <<= unit