Merge pull request #731 from ComputationalCryoEM/develop

Develop ~> Master v0.10.0 Release

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.2
+current_version = 0.10.0
 commit = True
 tag = True
 

CODEOWNERS

@@ -6,7 +6,7 @@
 # @global-owner1 and @global-owner2 will be requested for
 # review when someone opens a pull request.
 # *       @global-owner1 @global-owner2
-*        @janden
+*        @janden @garrettwrong
 
 # Order is important; the last matching pattern takes the most
 # precedence. When someone opens a pull request that only

README.md

@@ -5,7 +5,7 @@
 [![codecov](https://codecov.io/gh/ComputationalCryoEM/ASPIRE-Python/branch/master/graph/badge.svg?token=3XFC4VONX0)](https://codecov.io/gh/ComputationalCryoEM/ASPIRE-Python)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5657281.svg)](https://doi.org/10.5281/zenodo.5657281)
 
-# ASPIRE - Algorithms for Single Particle Reconstruction - v0.9.2
+# ASPIRE - Algorithms for Single Particle Reconstruction - v0.10.0
 
 This is the Python version to supersede the [Matlab ASPIRE](https://github.com/PrincetonUniversity/aspire).
 
@@ -20,7 +20,7 @@ For more information about the project, algorithms, and related publications ple
 Please cite using the following DOI. This DOI represents all versions, and will always resolve to the latest one.
 
 ```
-ComputationalCryoEM/ASPIRE-Python: v0.9.2 https://doi.org/10.5281/zenodo.5657281
+ComputationalCryoEM/ASPIRE-Python: v0.10.0 https://doi.org/10.5281/zenodo.5657281
 
 ```
 

azure-pipelines.yml

@@ -9,9 +9,9 @@ schedules:
 
 jobs:
 - job:
-  displayName: ubuntu-latest
+  displayName: ubuntu-22.04
   pool:
-    vmImage: 'ubuntu-latest'
+    vmImage: 'ubuntu-22.04'
   strategy:
     matrix:
       Python37:
@@ -31,9 +31,9 @@ jobs:
     displayName: pytest
 
 - job:
-  displayName: ubuntu-18.04
+  displayName: ubuntu-20.04
   pool:
-    vmImage: 'ubuntu-18.04'
+    vmImage: 'ubuntu-20.04'
   strategy:
     matrix:
       Python37:
@@ -86,9 +86,9 @@ jobs:
     displayName: pytest
 
 - job:
-  displayName: macOS-10.15
+  displayName: macOS-11
   pool:
-    vmImage: 'macOS-10.15'
+    vmImage: 'macOS-11'
   strategy:
     matrix:
       Python37:

docs/source/conf.py

@@ -74,14 +74,14 @@
 # built documents.
 #
 # The full version, including alpha/beta/rc tags.
-release = version = "0.9.2"
+release = version = "0.10.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

docs/source/index.rst

@@ -1,4 +1,4 @@
-Aspire v0.9.2
+Aspire v0.10.0
 ==============
 
 Algorithms for Single Particle Reconstruction

docs/source/quickstart.rst

@@ -9,7 +9,7 @@ CRYO projections (MRC files).
     aspire <command>
 
 Running the ``aspire`` module as a script allows one to run different stages of the Cryo-EM data pipeline.
-Substitute ``<command>`` with one of the available ``aspire`` commands. Use the ``-h`` flag to display all available commands.
+Substitute ``<command>`` with one of the available ``aspire`` commands. Use ``aspire --help`` to display all available commands and ``aspire <command> --help`` to display configurable options for a particular ``<command>``.
 
 Currently, the following operations can be run with ASPIRE:
 
@@ -28,73 +28,94 @@ For example, to run the command on sample data included in ASPIRE (a single ``sa
     mkdir apple_output
     aspire apple --mrc_path /path/to/aspire/data/sample.mrc --output_dir particles
 
-Use the ``--help`` argument with the command to see the several options associated with this command.
+2. Particle Extraction
+######################
 
-2. Simulation
-#############
+Given a dataset of full micrographs (``*.mrc`` file) and corresponding coordinate files containing the locations
+of picked particles in the ``*.mrc``, the ``extract-particles`` command extracts the particles into one or more ``.mrcs``
+stacks and generates a ``.star`` file.
 
-The ``simulation`` command simulates a virtual particle made up of multiple gaussian blobs, generates of set of (noisy) images,
-runs the ASPIRE pipeline to determine the estimated mean volume and estimated covariance on the mean volume,
-and runs evaluations on these estimated quantities (against the `true` values which we know from the simulation).
+Example usage:
+
+.. code-block::
+
+    aspire extract-particles --mrc_paths=my/data/sample.mrc --coord_paths=my/data/coords/sample.coord --starfile_out=my_dataset_stack.star --particle_size=256 --centers
+
+3. Estimate Contrast Transfer Function
+######################################
+
+The ``estimate-ctf`` command estimates the CTF from experimental data and saves the CTF as an ``.mrc`` file.  For example,
 
 .. code-block:: console
 
-    aspire simulation
+      aspire estimate-ctf --data_folder path_to_input_data_folder
 
-Use the ``--help`` argument to look for configurable options. You can select the no. of distinct gaussian blobs, the no. of images,
-the resolution of the (square) images generated etc.
+.. note::
 
-3. Reconstructing a mean volume with covariance
-###############################################
+    This command expects data files are in the directory prescribed by ``--data_folder``,
+    and will process all files with the extension ``.mrc`` and ``.mrcs`` contained there.
+    This command will output ``.mrc`` files to a ``--output_dir`` (``./results`` by default).
 
-The ``cov3d`` command takes in a ``*.star`` file, processes the images (``*.mrcs`` files) found in the starfile, and runs the ASPIRE pipeline
-to determine the estimated mean volume and estimated covariance on the mean volume. No results are saved currently, but this command is
-a good way to exercise most parts of the ASPIRE pipeline.
+4. Image Preprocessing
+######################
 
-For example, to run the command on a sample data included in ASPIRE:
+The ``preprocess`` command takes in a ``*.star`` file representing particle stacks and applies a selection of preprocessing
+methods such as phase flipping, downsampling, normalization to background noise, noise whitening, and contrast inversion.
+Resulting images are saved as a starfile.
+
+For example, to run the command on sample data included in ASPIRE:
 
 .. code-block:: console
 
-    aspire cov3d --starfile /path/to/aspire/data/sample_relion_data.star --pixel_size 1.338 --max_resolution 8 --cg_tol 0.2
+   aspire preprocess --starfile_in path/to/aspire/data/sample_relion_data.star --starfile_out preprocess_output.star --downsample 8
 
-.. note::
+5. Image Denoising
+##################
 
-    Pay special attention to the flags specified in the example above. The ``--max_resolution 8``
-    flag down-samples images to 8x8 pixels (needed otherwise you may run out of memory, and/or the script may take way
-    too long to execute). ``--cg_tol 0.2`` sets very liberal (and unrealistic) limits on optimization convergence
-    tolerance, which is needed for such a small dataset. For real datasets, you typically *do not* want to override this
-    parameter.
+The ``denoise`` command takes in a ``*.star`` file, downsamples the images (``*.mrcs`` files) found in the starfile
+to a desired resolution, then estimates the noise of the images and whitens that noise using the covariance
+Weiner filtering method. The denoised images (``*.mrcs``) and a corresponding starfile are saved in an output folder.
 
-Use the ``--help`` argument to look for configurable options.
+For example, to run the command on sample data included in ASPIRE:
 
-4. Crop a set of projections
-############################
+.. code-block:: console
 
-The ``crop`` command crops a stack of projections of an mrc file to squares of a given size (in pixels). For example,
+   mkdir denoise_output
+   aspire denoise --starfile_in path/to/aspire/data/sample_relion_data.star --starfile_out denoise_output/denoised_images.star
 
-.. code-block:: console
+6. Orientation Estimation
+#########################
 
-      aspire --debug -v 3 crop demo.mrc 42
+The ``orient3d`` command takes in a ``*.star`` file contaning images and performs an orientation estimation using the
+common lines algorithm employing synchronization and voting described at :cite:`DBLP:journals/siamis/ShkolniskyS12`.
+The estimated rotations are saved in a starfile along with the original images.
 
-.. note::
+For example, to run the command on sample data included in ASPIRE:
 
-    This command will crop images found in `demo.mrc` to images of size 42x42, in debug mode and with maximum verbosity.
+.. code-block:: console
 
-5. Estimate Contrast Transfer Function
-######################################
+   aspire orient3d --starfile_in path/to/aspire/data/sample_relion_data.star --starfile_out orient3d_output.star
 
-The ``estimate-ctf`` command estimates the CTF from experimental data and returns the CTF as a mrc file.  For example,
+7. Reconstructing a mean volume with covariance
+###############################################
+
+The ``cov3d`` command takes in a ``*.star`` file, processes the images (``*.mrcs`` files) found in the starfile, and runs the ASPIRE pipeline
+to determine the estimated mean volume and estimated covariance on the mean volume. No results are saved currently, but this command is
+a good way to exercise most parts of the ASPIRE pipeline.
+
+For example, to run the command on a sample data included in ASPIRE:
 
 .. code-block:: console
 
-      python -m aspire estimate-ctf --data_folder path_to_input_data_folder
+    aspire cov3d --starfile /path/to/aspire/data/sample_relion_data.star --pixel_size 1.338 --max_resolution 8 --cg_tol 0.2
 
 .. note::
 
-    This command expects data files are in the directory prescribed by ``--data_folder``,
-    and will process all files with the extension ``.mrc`` and ``.mrcs`` contained there.
-    This command will output mrc files to a ``--output_dir``, set to ``./results`` by default.
-
+    Pay special attention to the flags specified in the example above. The ``--max_resolution 8``
+    flag down-samples images to 8x8 pixels (needed otherwise you may run out of memory, and/or the script may take way
+    too long to execute). ``--cg_tol 0.2`` sets very liberal (and unrealistic) limits on optimization convergence
+    tolerance, which is needed for such a small dataset. For real datasets, you typically *do not* want to override this
+    parameter.
 
 Arguments, options and flags
 ############################

docs/source/references.bib

@@ -45,3 +45,19 @@ @article{DBLP:journals/siamis/SingerS11
   biburl    = {https://dblp.org/rec/bib/journals/siamis/SingerS11},
   bibsource = {dblp computer science bibliography, https://dblp.org}
 }
+
+@article{DBLP:journals/siamis/ShkolniskyS12,
+  author    = {Yoel Shkolnisky and
+               Amit Singer},
+  title     = {Viewing Direction Estimation in Cryo-EM Using Synchronization},
+  journal   = {{SIAM} J. Imaging Sci.},
+  volume    = {5},
+  number    = {3},
+  pages     = {1088--1110},
+  year      = {2012},
+  url       = {https://doi.org/10.1137/120863642},
+  doi       = {10.1137/120863642},
+  timestamp = {Tue, 14 Jul 2020 14:30:31 +0200},
+  biburl    = {https://dblp.org/rec/journals/siamis/ShkolniskyS12.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}

gallery/experiments/experimental_abinitio_pipeline.py

@@ -70,7 +70,7 @@
 
 # Peek
 if interactive:
-    src.images(0, 10).show()
+    src.images[:10].show()
 
 # Use phase_flip to attempt correcting for CTF.
 logger.info("Perform phase flip to input images.")
@@ -87,7 +87,7 @@
 
 # Peek, what do the whitened images look like...
 if interactive:
-    src.images(0, 10).show()
+    src.images[:10].show()
 
 # # Optionally invert image contrast, depends on data convention.
 # # This is not needed for 10028, but included anyway.
@@ -117,7 +117,7 @@
     classification_src = cwf_denoiser.denoise()
     # Peek, what do the denoised images look like...
     if interactive:
-        classification_src.images(0, 10).show()
+        classification_src.images[:10].show()
 
     # Use regular `src` for the alignment and composition (averaging).
     composite_basis = FFBBasis2D((src.L,) * 2, dtype=src.dtype)
@@ -147,7 +147,7 @@
 classes, reflections, distances = rir.classify()
 avgs = rir.averages(classes, reflections, distances)
 if interactive:
-    avgs.images(0, 10).show()
+    avgs.images[:10].show()
 
 # %%
 # Common Line Estimation
@@ -172,7 +172,7 @@
 logger.info("Begin Volume reconstruction")
 
 # Assign the estimated rotations to the class averages
-avgs.rots = rots_est
+avgs.rotations = rots_est
 
 # Create a reasonable Basis for the 3d Volume
 basis = FFBBasis3D((img_size,) * 3, dtype=src.dtype)

gallery/experiments/preprocess_imgs_exp.py.dontrun

@@ -28,29 +28,29 @@ source = RelionSource(
 nimgs_ext = 1
 
 print('Obtain original images')
-imgs_od = source.images(start=0, num=nimgs_ext)
+imgs_od = source.images[:nimgs_ext]
 
 print('Perform phase flip to input images')
 source.phase_flip()
-imgs_pf = source.images(start=0, num=nimgs_ext)
+imgs_pf = source.images[:nimgs_ext]
 
 max_resolution = 60
 print(f'Downsample resolution to {max_resolution} X {max_resolution}')
 source.downsample(max_resolution)
-imgs_ds = source.images(start=0, num=nimgs_ext)
+imgs_ds = source.images[:nimgs_ext]
 
 print('Normalize images to noise background')
 source.normalize_background()
-imgs_nb = source.images(start=0, num=nimgs_ext)
+imgs_nb = source.images[:nimgs_ext]
 
 print('Whiten noise of images')
 noise_estimator = WhiteNoiseEstimator(source)
 source.whiten(noise_estimator.filter)
-imgs_wt = source.images(start=0, num=nimgs_ext)
+imgs_wt = source.images[:nimgs_ext]
 
 print('Invert global density contrast')
 source.invert_contrast()
-imgs_rc = source.images(start=0, num=nimgs_ext)
+imgs_rc = source.images[:nimgs_ext]
 
 # plot the first images
 print('plot the first images')

gallery/experiments/simulated_abinitio_pipeline.py

@@ -106,10 +106,11 @@ def noise_function(x, y):
     vols=v,
     noise_filter=custom_noise_filter,
     unique_filters=ctf_filters,
+    dtype=v.dtype,
 )
 # Peek
 if interactive:
-    src.images(0, 10).show()
+    src.images[:10].show()
 
 # Use phase_flip to attempt correcting for CTF.
 logger.info("Perform phase flip to input images.")
@@ -126,7 +127,7 @@ def noise_function(x, y):
 
 # Peek, what do the whitened images look like...
 if interactive:
-    src.images(0, 10).show()
+    src.images[:10].show()
 
 # Cache to memory for some speedup
 src.cache()
@@ -154,7 +155,7 @@ def noise_function(x, y):
     classification_src = cwf_denoiser.denoise()
     # Peek, what do the denoised images look like...
     if interactive:
-        classification_src.images(0, 10).show()
+        classification_src.images[:10].show()
 
     # Use regular `src` for the alignment and composition (averaging).
     composite_basis = FFBBasis2D((src.L,) * 2, dtype=src.dtype)
@@ -184,7 +185,7 @@ def noise_function(x, y):
 classes, reflections, distances = rir.classify()
 avgs = rir.averages(classes, reflections, distances)
 if interactive:
-    avgs.images(0, 10).show()
+    avgs.images[:10].show()
 
 # %%
 # Common Line Estimation
@@ -197,7 +198,7 @@ def noise_function(x, y):
 
 # Stash true rotations for later comparison,
 #   note this line only works with naive class selection...
-true_rotations = src.rots[:n_classes]
+true_rotations = src.rotations[:n_classes]
 
 orient_est = CLSyncVoting(avgs, n_theta=36)
 # Get the estimated rotations
@@ -222,7 +223,7 @@ def noise_function(x, y):
 logger.info("Begin Volume reconstruction")
 
 # Assign the estimated rotations to the class averages
-avgs.rots = rots_est
+avgs.rotations = rots_est
 
 # Create a reasonable Basis for the 3d Volume
 basis = FFBBasis3D((v.resolution,) * 3, dtype=v.dtype)