Adding a progress message indicator.

* Include dependence of pcigale.analysis_modules.utils.Counter class, to perform the progress message.
* Fixed error in the way to pass "outdir" parameter to the pool in chi2, and pdf.
parent 5cf4de19
......@@ -17,18 +17,35 @@ import matplotlib.pyplot as plt
import multiprocessing as mp
import numpy as np
from pcigale.utils import read_table
from pcigale.analysis_modules.utils import Counter, nothread
def pool_initializer(counter):
"""Initializer of the pool of processes to share variables between workers.
Parameters
----------
:param counter: Counter class object for the number of models plotted
"""
global gbl_counter
# Limit the number of threads to 1 if we use MKL in order to limit the
# oversubscription of the CPU/RAM.
nothread()
gbl_counter = counter
def chi2(config, outdir):
"""Plot the χ² values of analysed variables.
"""
input_data = read_table(path.join(path.dirname(outdir), config.configuration['data_file']))
file = path.join(path.dirname(outdir), config.configuration['data_file'])
input_data = read_table(file)
chi2_vars = config.configuration['analysis_params']['variables']
chi2_vars += [band for band in config.configuration['bands']
if band.endswith('_err') is False]
with mp.Pool(processes=config.configuration['cores']) as pool:
items = product(input_data['id'], chi2_vars, outdir)
items = list(product(input_data['id'], chi2_vars, [outdir]))
counter = Counter(len(items))
with mp.Pool(processes=config.configuration['cores'], initializer=pool_initializer,
initargs=(counter,)) as pool:
pool.starmap(_chi2_worker, items)
pool.close()
pool.join()
......@@ -47,6 +64,7 @@ def _chi2_worker(obj_name, var_name, outdir):
The absolute path to outdir
"""
gbl_counter.inc()
figure = plt.figure()
ax = figure.add_subplot(111)
......
......@@ -17,11 +17,26 @@ import multiprocessing as mp
import numpy as np
import pkg_resources
from scipy import stats
from pcigale.analysis_modules.utils import Counter, nothread
# Name of the file containing the best models information
BEST_RESULTS = "results.fits"
MOCK_RESULTS = "results_mock.fits"
def pool_initializer(counter):
"""Initializer of the pool of processes to share variables between workers.
Parameters
----------
:param counter: Counter class object for the number of models plotted
"""
global gbl_counter
# Limit the number of threads to 1 if we use MKL in order to limit the
# oversubscription of the CPU/RAM.
nothread()
gbl_counter = counter
def mock(config, nologo, outdir):
"""Plot the comparison of input/output values of analysed variables.
"""
......@@ -50,10 +65,12 @@ def mock(config, nologo, outdir):
logo = False if nologo else plt.imread(pkg_resources.resource_filename(__name__,
"../resources/CIGALE.png"))
arguments = ((exact["best."+param], estimated["bayes."+param], param, logo, outdir)
for param in params)
arguments = [(exact["best."+param], estimated["bayes."+param], param, logo, outdir)
for param in params]
with mp.Pool(processes=config.configuration['cores']) as pool:
counter = Counter(len(arguments))
with mp.Pool(processes=config.configuration['cores'], initializer=pool_initializer,
initargs=(counter,)) as pool:
pool.starmap(_mock_worker, arguments)
pool.close()
pool.join()
......@@ -76,7 +93,7 @@ def _mock_worker(exact, estimated, param, logo, outdir):
The absolute path to outdir
"""
gbl_counter.inc()
range_exact = np.linspace(np.min(exact), np.max(exact), 100)
# We compute the linear regression
......
......@@ -16,6 +16,20 @@ import matplotlib.pyplot as plt
import multiprocessing as mp
import numpy as np
from pcigale.utils import read_table
from pcigale.analysis_modules.utils import Counter, nothread
def pool_initializer(counter):
"""Initializer of the pool of processes to share variables between workers.
Parameters
----------
:param counter: Counter class object for the number of models plotted
"""
global gbl_counter
# Limit the number of threads to 1 if we use MKL in order to limit the
# oversubscription of the CPU/RAM.
nothread()
gbl_counter = counter
def pdf(config, outdir):
......@@ -26,8 +40,10 @@ def pdf(config, outdir):
pdf_vars += [band for band in config.configuration['bands']
if band.endswith('_err') is False]
with mp.Pool(processes=config.configuration['cores']) as pool:
items = product(input_data['id'], pdf_vars, outdir)
items = list(product(input_data['id'], pdf_vars, [outdir]))
counter = Counter(len(items))
with mp.Pool(processes=config.configuration['cores'], initializer=pool_initializer,
initargs=(counter,)) as pool:
pool.starmap(_pdf_worker, items)
pool.close()
pool.join()
......@@ -46,6 +62,7 @@ def _pdf_worker(obj_name, var_name, outdir):
The absolute path to outdir
"""
gbl_counter.inc()
var_name = var_name.replace('/', '_')
if var_name.endswith('_log'):
fnames = glob.glob("{}/{}_{}_chi2-block-*.npy".format(outdir, obj_name,
......
......@@ -22,6 +22,7 @@ from scipy.constants import c
from pcigale.data import Database
from pcigale.utils import read_table
import matplotlib.gridspec as gridspec
from pcigale.analysis_modules.utils import Counter, nothread
# Name of the file containing the best models information
BEST_RESULTS = "results.fits"
......@@ -32,6 +33,19 @@ PLOT_L_MIN = 0.1
PLOT_L_MAX = 5e5
def pool_initializer(counter):
"""Initializer of the pool of processes to share variables between workers.
Parameters
----------
:param counter: Counter class object for the number of models plotted
"""
global gbl_counter
# Limit the number of threads to 1 if we use MKL in order to limit the
# oversubscription of the CPU/RAM.
nothread()
gbl_counter = counter
def sed(config, sed_type, nologo, outdir):
"""Plot the best SED with associated observed and modelled fluxes.
"""
......@@ -46,7 +60,9 @@ def sed(config, sed_type, nologo, outdir):
logo = False if nologo else plt.imread(pkg_resources.resource_filename(__name__,
"../resources/CIGALE.png"))
with mp.Pool(processes=config.configuration['cores']) as pool:
counter = Counter(len(obs))
with mp.Pool(processes=config.configuration['cores'], initializer=pool_initializer,
initargs=(counter,)) as pool:
pool.starmap(_sed_worker, zip(obs, mod, repeat(filters),
repeat(sed_type), repeat(logo), repeat(outdir)))
pool.close()
......@@ -77,6 +93,8 @@ def _sed_worker(obs, mod, filters, sed_type, logo, outdir):
The absolute path to outdir
"""
gbl_counter.inc()
id_best_model_file = path.join(outdir, '{}_best_model.fits'.format(obs['id']))
if path.isfile(id_best_model_file):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment