From a1a59fe1507f5b9a58231056951fc176d7477bc8 Mon Sep 17 00:00:00 2001 From: Yida Wang Date: Tue, 7 Feb 2017 21:41:02 -0800 Subject: [PATCH 1/2] replace pathos.multiprocessing with multiprocess; use with to wrap Pool to prevent Too many open files error --- brainiak/fcma/voxelselector.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/brainiak/fcma/voxelselector.py b/brainiak/fcma/voxelselector.py index 16f086918..06390fc7a 100644 --- a/brainiak/fcma/voxelselector.py +++ b/brainiak/fcma/voxelselector.py @@ -28,7 +28,7 @@ from . import fcma_extension from . import cython_blas as blas import logging -import pathos.multiprocessing +import multiprocess logger = logging.getLogger(__name__) @@ -412,11 +412,11 @@ def _cross_validation_for_one_voxel(vid, num_folds, inlist = [(i + task[0], self.num_folds, data[i, :, :], self.labels) for i in range(task[1])] - pool = pathos.multiprocessing.ProcessingPool(None) - results = list(pool.map( - lambda x: _cross_validation_for_one_voxel - (x[0], x[1], x[2], x[3]), - inlist)) + with multiprocess.Pool(None) as pool: + results = list(pool.map( + lambda x: _cross_validation_for_one_voxel + (x[0], x[1], x[2], x[3]), + inlist)) else: results = [] for i in range(task[1]): From d9f9e602453bd4b16fadaaf0fb75706f0d3e3113 Mon Sep 17 00:00:00 2001 From: Yida Wang Date: Wed, 8 Feb 2017 10:22:15 -0800 Subject: [PATCH 2/2] address review comments --- brainiak/fcma/voxelselector.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/brainiak/fcma/voxelselector.py b/brainiak/fcma/voxelselector.py index 06390fc7a..5fa4304da 100644 --- a/brainiak/fcma/voxelselector.py +++ b/brainiak/fcma/voxelselector.py @@ -413,10 +413,8 @@ def _cross_validation_for_one_voxel(vid, num_folds, self.labels) for i in range(task[1])] with multiprocess.Pool(None) as pool: - results = list(pool.map( - lambda x: _cross_validation_for_one_voxel - (x[0], x[1], x[2], x[3]), - inlist)) + results = list(pool.starmap(_cross_validation_for_one_voxel, + inlist)) else: results = [] for i in range(task[1]):