Skip to content

Commit

Permalink
Merge pull request #80 from KamitaniLab/update-async-features-get
Browse files Browse the repository at this point in the history
Update asynchronous Features.get()
  • Loading branch information
ShuntaroAoki authored Dec 14, 2023
2 parents 260569b + 0096d3a commit 9ed810d
Showing 1 changed file with 15 additions and 6 deletions.
21 changes: 15 additions & 6 deletions bdpy/dataform/features.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,11 @@ def get(self, layer: str, label: Union[str, List[str], None] = None) -> np.ndarr
num_labels = len(labels)
num_parallel = _determine_num_parallel(num_labels)
path_iterator = map(lambda label: self.__feature_file_table[layer][label], labels)
with Pool(processes=num_parallel) as pool:
features = np.concatenate(pool.map(partial(_load_array_with_key, 'feat'), path_iterator), axis=0)
if num_parallel == 1:
features = np.concatenate(list(map(partial(_load_array_with_key, 'feat'), path_iterator)), axis=0)
else:
with Pool(processes=num_parallel) as pool:
features = np.concatenate(pool.map(partial(_load_array_with_key, 'feat'), path_iterator), axis=0)

if self.__feat_index_table is not None:
# Select features by index
Expand Down Expand Up @@ -215,8 +218,11 @@ def get_features(self, layer: str) -> np.ndarray:
num_labels = len(self.__labels)
num_parallel = _determine_num_parallel(num_labels)
path_iterator = map(lambda label: self.__feature_file_table[layer][label], self.__labels)
with Pool(processes=num_parallel) as pool:
features = np.concatenate(pool.map(partial(_load_array_with_key, 'feat'), path_iterator), axis=0)
if num_parallel == 1:
features = np.concatenate(list(map(partial(_load_array_with_key, 'feat'), path_iterator)), axis=0)
else:
with Pool(processes=num_parallel) as pool:
features = np.concatenate(pool.map(partial(_load_array_with_key, 'feat'), path_iterator), axis=0)
self.__features = features

self.__c_feature_name = layer
Expand Down Expand Up @@ -376,8 +382,11 @@ def get(self, layer=None, subject=None, roi=None, fold=None, label=None, image=N
features: np.ndarray

Check warning on line 382 in bdpy/dataform/features.py

View workflow job for this annotation

GitHub Actions / type-check

Error code: `annotation-unchecked`

By default the bodies of untyped functions are not checked, consider using --check-untyped-defs

Check warning on line 382 in bdpy/dataform/features.py

View workflow job for this annotation

GitHub Actions / type-check

Error code: `annotation-unchecked`

By default the bodies of untyped functions are not checked, consider using --check-untyped-defs
num_files = len(files)
num_parallel = _determine_num_parallel(num_files)
with Pool(processes=num_parallel) as pool:
features = np.concatenate(pool.map(partial(_load_array_with_key, self.__file_key), files), axis=0)
if num_parallel == 1:
features = np.concatenate(list(map(partial(_load_array_with_key, self.__file_key), files)), axis=0)
else:
with Pool(processes=num_parallel) as pool:
features = np.concatenate(pool.map(partial(_load_array_with_key, self.__file_key), files), axis=0)

if self.__squeeze:
features = np.squeeze(features)
Expand Down

4 comments on commit 9ed810d

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Coverage

Coverage Report
FileStmtsMissCoverMissing
bdpy
   __init__.py330%9–11
bdpy/bdata
   __init__.py220%7–8
   bdata.py3983980%26–919
   featureselector.py64640%8–124
   metadata.py67670%8–154
   utils.py1131130%4–263
bdpy/dataform
   __init__.py440%7–10
   datastore.py1071070%7–265
   features.py2982980%8–549
   pd.py990%7–44
   sparse.py67670%6–126
   utils.py12120%3–18
bdpy/dataset
   utils.py45450%3–98
bdpy/distcomp
   __init__.py110%6
   distcomp.py92920%7–127
bdpy/dl
   caffe.py60600%4–129
bdpy/dl/torch
   __init__.py220%1–2
   base.py43430%6–105
   models.py3323320%4–874
   torch.py1091090%3–258
bdpy/evals
   metrics.py95950%3–179
bdpy/feature
   __init__.py110%3
   feature.py30300%1–74
bdpy/fig
   __init__.py440%6–9
   draw_group_image_set.py90900%3–182
   fig.py88880%16–164
   makeplots.py3363360%1–729
   tile_images.py59590%1–193
bdpy/ml
   __init__.py770%8–14
   crossvalidation.py59590%7–196
   ensemble.py13130%5–46
   learning.py3083080%4–613
   model.py1401400%4–285
   regress.py11110%6–38
   searchlight.py16160%4–51
bdpy/mri
   __init__.py770%7–13
   fmriprep.py4974970%4–866
   glm.py40400%4–95
   image.py24240%4–54
   load_epi.py28280%7–88
   load_mri.py19190%4–36
   roi.py2482480%4–499
   spm.py1581580%1–300
bdpy/opendata
   __init__.py110%1
   openneuro.py2102100%1–329
bdpy/preproc
   __init__.py330%8–10
   interface.py52520%8–217
   preprocessor.py1291290%8–236
   select_top.py22220%8–61
   util.py660%6–22
bdpy/recon
   utils.py55550%4–146
bdpy/recon/torch
   __init__.py110%1
   icnn.py1611610%15–478
bdpy/stats
   __init__.py110%13
   corr.py43430%6–112
bdpy/util
   __init__.py330%7–9
   info.py47470%4–79
   math.py13130%4–38
   utils.py36360%7–145
TOTAL488948890% 

Tests Skipped Failures Errors Time
114 0 💤 13 ❌ 6 🔥 9.600s ⏱️

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Coverage

Coverage Report
FileStmtsMissCoverMissing
bdpy
   __init__.py330%9–11
bdpy/bdata
   __init__.py220%7–8
   bdata.py3983980%26–919
   featureselector.py64640%8–124
   metadata.py67670%8–154
   utils.py1131130%4–263
bdpy/dataform
   __init__.py440%7–10
   datastore.py1071070%7–265
   features.py2982980%8–549
   pd.py990%7–44
   sparse.py67670%6–126
   utils.py12120%3–18
bdpy/dataset
   utils.py45450%3–98
bdpy/distcomp
   __init__.py110%6
   distcomp.py92920%7–127
bdpy/dl
   caffe.py60600%4–129
bdpy/dl/torch
   __init__.py220%1–2
   base.py43430%6–105
   models.py3323320%4–874
   torch.py1091090%3–258
bdpy/evals
   metrics.py95950%3–179
bdpy/feature
   __init__.py110%3
   feature.py30300%1–74
bdpy/fig
   __init__.py440%6–9
   draw_group_image_set.py90900%3–182
   fig.py88880%16–164
   makeplots.py3363360%1–729
   tile_images.py59590%1–193
bdpy/ml
   __init__.py770%8–14
   crossvalidation.py59590%7–196
   ensemble.py13130%5–46
   learning.py3083080%4–613
   model.py1401400%4–285
   regress.py11110%6–38
   searchlight.py16160%4–51
bdpy/mri
   __init__.py770%7–13
   fmriprep.py4974970%4–866
   glm.py40400%4–95
   image.py24240%4–54
   load_epi.py28280%7–88
   load_mri.py19190%4–36
   roi.py2482480%4–499
   spm.py1581580%1–300
bdpy/opendata
   __init__.py110%1
   openneuro.py2102100%1–329
bdpy/preproc
   __init__.py330%8–10
   interface.py52520%8–217
   preprocessor.py1291290%8–236
   select_top.py22220%8–61
   util.py660%6–22
bdpy/recon
   utils.py55550%4–146
bdpy/recon/torch
   __init__.py110%1
   icnn.py1611610%15–478
bdpy/stats
   __init__.py110%13
   corr.py43430%6–112
bdpy/util
   __init__.py330%7–9
   info.py47470%4–79
   math.py13130%4–38
   utils.py36360%7–145
TOTAL488948890% 

Tests Skipped Failures Errors Time
114 0 💤 13 ❌ 6 🔥 9.410s ⏱️

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Coverage

Coverage Report
FileStmtsMissCoverMissing
bdpy
   __init__.py330%9–11
bdpy/bdata
   __init__.py220%7–8
   bdata.py3983980%26–919
   featureselector.py64640%8–124
   metadata.py67670%8–154
   utils.py1131130%4–263
bdpy/dataform
   __init__.py440%7–10
   datastore.py1071070%7–265
   features.py2982980%8–549
   pd.py990%7–44
   sparse.py67670%6–126
   utils.py12120%3–18
bdpy/dataset
   utils.py45450%3–98
bdpy/distcomp
   __init__.py110%6
   distcomp.py92920%7–127
bdpy/dl
   caffe.py60600%4–129
bdpy/dl/torch
   __init__.py220%1–2
   base.py43430%6–105
   models.py3323320%4–874
   torch.py1091090%3–258
bdpy/evals
   metrics.py95950%3–179
bdpy/feature
   __init__.py110%3
   feature.py30300%1–74
bdpy/fig
   __init__.py440%6–9
   draw_group_image_set.py90900%3–182
   fig.py88880%16–164
   makeplots.py3363360%1–729
   tile_images.py59590%1–193
bdpy/ml
   __init__.py770%8–14
   crossvalidation.py59590%7–196
   ensemble.py13130%5–46
   learning.py3083080%4–613
   model.py1401400%4–285
   regress.py11110%6–38
   searchlight.py16160%4–51
bdpy/mri
   __init__.py770%7–13
   fmriprep.py4974970%4–866
   glm.py40400%4–95
   image.py24240%4–54
   load_epi.py28280%7–88
   load_mri.py19190%4–36
   roi.py2482480%4–499
   spm.py1581580%1–300
bdpy/opendata
   __init__.py110%1
   openneuro.py2102100%1–329
bdpy/preproc
   __init__.py330%8–10
   interface.py52520%8–217
   preprocessor.py1291290%8–236
   select_top.py22220%8–61
   util.py660%6–22
bdpy/recon
   utils.py55550%4–146
bdpy/recon/torch
   __init__.py110%1
   icnn.py1611610%15–478
bdpy/stats
   __init__.py110%13
   corr.py43430%6–112
bdpy/util
   __init__.py330%7–9
   info.py47470%4–79
   math.py13130%4–38
   utils.py36360%7–145
TOTAL488948890% 

Tests Skipped Failures Errors Time
114 0 💤 13 ❌ 6 🔥 9.120s ⏱️

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Coverage

Coverage Report
FileStmtsMissCoverMissing
bdpy
   __init__.py330%9–11
bdpy/bdata
   __init__.py220%7–8
   bdata.py3983980%26–919
   featureselector.py64640%8–124
   metadata.py67670%8–154
   utils.py1131130%4–263
bdpy/dataform
   __init__.py440%7–10
   datastore.py1071070%7–265
   features.py2982980%8–549
   pd.py990%7–44
   sparse.py67670%6–126
   utils.py12120%3–18
bdpy/dataset
   utils.py45450%3–98
bdpy/distcomp
   __init__.py110%6
   distcomp.py92920%7–127
bdpy/dl
   caffe.py60600%4–129
bdpy/dl/torch
   __init__.py220%1–2
   base.py43430%6–105
   models.py3323320%4–874
   torch.py1091090%3–258
bdpy/evals
   metrics.py95950%3–179
bdpy/feature
   __init__.py110%3
   feature.py30300%1–74
bdpy/fig
   __init__.py440%6–9
   draw_group_image_set.py90900%3–182
   fig.py88880%16–164
   makeplots.py3363360%1–729
   tile_images.py59590%1–193
bdpy/ml
   __init__.py770%8–14
   crossvalidation.py59590%7–196
   ensemble.py13130%5–46
   learning.py3083080%4–613
   model.py1401400%4–285
   regress.py11110%6–38
   searchlight.py16160%4–51
bdpy/mri
   __init__.py770%7–13
   fmriprep.py4974970%4–866
   glm.py40400%4–95
   image.py24240%4–54
   load_epi.py28280%7–88
   load_mri.py19190%4–36
   roi.py2482480%4–499
   spm.py1581580%1–300
bdpy/opendata
   __init__.py110%1
   openneuro.py2102100%1–329
bdpy/preproc
   __init__.py330%8–10
   interface.py52520%8–217
   preprocessor.py1291290%8–236
   select_top.py22220%8–61
   util.py660%6–22
bdpy/recon
   utils.py55550%4–146
bdpy/recon/torch
   __init__.py110%1
   icnn.py1611610%15–478
bdpy/stats
   __init__.py110%13
   corr.py43430%6–112
bdpy/util
   __init__.py330%7–9
   info.py47470%4–79
   math.py13130%4–38
   utils.py36360%7–145
TOTAL488948890% 

Tests Skipped Failures Errors Time
114 0 💤 13 ❌ 6 🔥 9.737s ⏱️

Please sign in to comment.