Adding a progress message indicator.

* Include dependence of pcigale.analysis_modules.utils.Counter class, to perform the progress message.
* Fixed error in the way to pass "outdir" parameter to the pool in chi2, and pdf.

pcigale_plots/plot_types/chi2.py

@@ -17,18 +17,35 @@ import matplotlib.pyplot as plt
 import multiprocessing as mp
 import numpy as np
 from pcigale.utils import read_table
+from pcigale.analysis_modules.utils import Counter, nothread
+
+
+def pool_initializer(counter):
+    """Initializer of the pool of processes to share variables between workers.
+    Parameters
+    ----------
+    :param counter: Counter class object for the number of models plotted
+    """
+    global gbl_counter
+    # Limit the number of threads to 1 if we use MKL in order to limit the
+    # oversubscription of the CPU/RAM.
+    nothread()
+    gbl_counter = counter
 
 
 def chi2(config, outdir):
     """Plot the χ² values of analysed variables.
     """
-    input_data = read_table(path.join(path.dirname(outdir), config.configuration['data_file']))
+    file = path.join(path.dirname(outdir), config.configuration['data_file'])
+    input_data = read_table(file)
     chi2_vars = config.configuration['analysis_params']['variables']
     chi2_vars += [band for band in config.configuration['bands']
                   if band.endswith('_err') is False]
 
-    with mp.Pool(processes=config.configuration['cores']) as pool:
-        items = product(input_data['id'], chi2_vars, outdir)
+    items = list(product(input_data['id'], chi2_vars, [outdir]))
+    counter = Counter(len(items))
+    with mp.Pool(processes=config.configuration['cores'], initializer=pool_initializer,
+                 initargs=(counter,)) as pool:
         pool.starmap(_chi2_worker, items)
         pool.close()
         pool.join()
@@ -47,6 +64,7 @@ def _chi2_worker(obj_name, var_name, outdir):
         The absolute path to outdir
 
     """
+    gbl_counter.inc()
     figure = plt.figure()
     ax = figure.add_subplot(111)
 

pcigale_plots/plot_types/mock.py

@@ -17,11 +17,26 @@ import multiprocessing as mp
 import numpy as np
 import pkg_resources
 from scipy import stats
+from pcigale.analysis_modules.utils import Counter, nothread
 
 # Name of the file containing the best models information
 BEST_RESULTS = "results.fits"
 MOCK_RESULTS = "results_mock.fits"
 
+
+def pool_initializer(counter):
+    """Initializer of the pool of processes to share variables between workers.
+    Parameters
+    ----------
+    :param counter: Counter class object for the number of models plotted
+    """
+    global gbl_counter
+    # Limit the number of threads to 1 if we use MKL in order to limit the
+    # oversubscription of the CPU/RAM.
+    nothread()
+    gbl_counter = counter
+
+
 def mock(config, nologo, outdir):
     """Plot the comparison of input/output values of analysed variables.
     """
@@ -50,10 +65,12 @@ def mock(config, nologo, outdir):
     logo = False if nologo else plt.imread(pkg_resources.resource_filename(__name__,
                                                                            "../resources/CIGALE.png"))
 
-    arguments = ((exact["best."+param], estimated["bayes."+param], param, logo, outdir)
-                 for param in params)
+    arguments = [(exact["best."+param], estimated["bayes."+param], param, logo, outdir)
+                 for param in params]
 
-    with mp.Pool(processes=config.configuration['cores']) as pool:
+    counter = Counter(len(arguments))
+    with mp.Pool(processes=config.configuration['cores'], initializer=pool_initializer,
+                 initargs=(counter,)) as pool:
         pool.starmap(_mock_worker, arguments)
         pool.close()
         pool.join()
@@ -76,7 +93,7 @@ def _mock_worker(exact, estimated, param, logo, outdir):
         The absolute path to outdir
 
     """
-
+    gbl_counter.inc()
     range_exact = np.linspace(np.min(exact), np.max(exact), 100)
 
     # We compute the linear regression

pcigale_plots/plot_types/pdf.py

@@ -16,6 +16,20 @@ import matplotlib.pyplot as plt
 import multiprocessing as mp
 import numpy as np
 from pcigale.utils import read_table
+from pcigale.analysis_modules.utils import Counter, nothread
+
+
+def pool_initializer(counter):
+    """Initializer of the pool of processes to share variables between workers.
+    Parameters
+    ----------
+    :param counter: Counter class object for the number of models plotted
+    """
+    global gbl_counter
+    # Limit the number of threads to 1 if we use MKL in order to limit the
+    # oversubscription of the CPU/RAM.
+    nothread()
+    gbl_counter = counter
 
 
 def pdf(config, outdir):
@@ -26,8 +40,10 @@ def pdf(config, outdir):
     pdf_vars += [band for band in config.configuration['bands']
                  if band.endswith('_err') is False]
 
-    with mp.Pool(processes=config.configuration['cores']) as pool:
-        items = product(input_data['id'], pdf_vars, outdir)
+    items = list(product(input_data['id'], pdf_vars, [outdir]))
+    counter = Counter(len(items))
+    with mp.Pool(processes=config.configuration['cores'], initializer=pool_initializer,
+                 initargs=(counter,)) as pool:
         pool.starmap(_pdf_worker, items)
         pool.close()
         pool.join()
@@ -46,6 +62,7 @@ def _pdf_worker(obj_name, var_name, outdir):
         The absolute path to outdir
 
     """
+    gbl_counter.inc()
     var_name = var_name.replace('/', '_')
     if var_name.endswith('_log'):
         fnames = glob.glob("{}/{}_{}_chi2-block-*.npy".format(outdir, obj_name,

pcigale_plots/plot_types/sed.py

@@ -22,6 +22,7 @@ from scipy.constants import c
 from pcigale.data import Database
 from pcigale.utils import read_table
 import matplotlib.gridspec as gridspec
+from pcigale.analysis_modules.utils import Counter, nothread
 
 # Name of the file containing the best models information
 BEST_RESULTS = "results.fits"
@@ -32,6 +33,19 @@ PLOT_L_MIN = 0.1
 PLOT_L_MAX = 5e5
 
 
+def pool_initializer(counter):
+    """Initializer of the pool of processes to share variables between workers.
+    Parameters
+    ----------
+    :param counter: Counter class object for the number of models plotted
+    """
+    global gbl_counter
+    # Limit the number of threads to 1 if we use MKL in order to limit the
+    # oversubscription of the CPU/RAM.
+    nothread()
+    gbl_counter = counter
+
+
 def sed(config, sed_type, nologo, outdir):
     """Plot the best SED with associated observed and modelled fluxes.
     """
@@ -46,7 +60,9 @@ def sed(config, sed_type, nologo, outdir):
     logo = False if nologo else plt.imread(pkg_resources.resource_filename(__name__,
                                                                "../resources/CIGALE.png"))
 
-    with mp.Pool(processes=config.configuration['cores']) as pool:
+    counter = Counter(len(obs))
+    with mp.Pool(processes=config.configuration['cores'], initializer=pool_initializer,
+                 initargs=(counter,)) as pool:
         pool.starmap(_sed_worker, zip(obs, mod, repeat(filters),
                                       repeat(sed_type), repeat(logo), repeat(outdir)))
         pool.close()
@@ -77,6 +93,8 @@ def _sed_worker(obs, mod, filters, sed_type, logo, outdir):
         The absolute path to outdir
 
     """
+    gbl_counter.inc()
+
     id_best_model_file = path.join(outdir, '{}_best_model.fits'.format(obs['id']))
     if path.isfile(id_best_model_file):