configuration.py 15 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5
6

import pkgutil
7
from collections import Iterable, OrderedDict
8
import multiprocessing as mp
9
from textwrap import wrap
10
11
12
13
14
15

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import pkg_resources
import numpy as np

16
from ..handlers.parameters_handler import ParametersHandler
17
from ..data import Database
18
from ..utils import read_table
19
20
from .. import creation_modules
from .. import analysis_modules
21
22
from ..warehouse import SedWarehouse

23

24
25
26
# Limit the redshift to this number of decimals
REDSHIFT_DECIMALS = 2

27

Yannick Roehlly's avatar
Yannick Roehlly committed
28
29
30
31
32
33
def evaluate_description(description):
    """Evaluate a description from the config file as a list.

    The description is read from the config file by configobj that transforms
    coma separated value in a list. From this description, this function try
    to evaluate the desired list of values:
34
35
36
    - If the description is a string beginning with 'eval ', then its content
      (without 'eval ') is evaluated as Python code and its result returned.
      An array is expected.
37
38
39
    - If the description is a string beginning by 'range', the start, step and
      stop values are then expected and the range is evaluated (stop included
      if reached.
Yannick Roehlly's avatar
Yannick Roehlly committed
40
41
    - Then the function tries to evaluate the description as a Numpy array of
      float and returns the mere list if this fails.
42

Yannick Roehlly's avatar
Yannick Roehlly committed
43
    Parameters
44
    ----------
45
    description: string or list
46
47
48
49
        The description to be evaluated.

    Returns
    -------
50
     results: list
Yannick Roehlly's avatar
Yannick Roehlly committed
51
        The evaluated list of values.
52
53

    """
54
55
56
57
58
    results = description
    if type(description) == str:
        if description.startswith('eval '):
            results = eval(description[4:])
            # If the evaluation lead to a single value, we put it in a list.
59
            if not isinstance(results, Iterable):
60
61
62
63
64
65
66
67
                results = [results]
        elif description.startswith('range '):
            start, stop, step = [float(item) for item
                                 in description[5:].split()]
            results = np.arange(start, stop+step, step)
        else:
            # We need to return a list to combine the list of possible values
            # for each parameter.
68
            results = [results]
69
70
71
72
73
74
75

    # We prefer to evaluate the parameter as a numpy array of floats if
    # possible.
    try:
        results = np.array(results, float)
    except ValueError:
        pass
76

Yannick Roehlly's avatar
Yannick Roehlly committed
77
    return results
78
79
80
81
82
83
84
85
86


class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
87
        Parameters
88
        ----------
89
        filename: string
90
91
92
93
94
            Name of the configuration file (pcigale.conf by default).

        """
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
95
96
                                          indent_type='  ',
                                          encoding='UTF8')
97
98
99
100
101
102
103
104
105
106
107
108

    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
109
110
111
112
113
114
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
            "the filter names for the fluxes, and the filter names with the "
            "'_err' suffix for the uncertainties. The fluxes and the "
            "uncertainties must be in mJy. This file is optional to generate "
            "the configuration file, in particular for the savefluxes module.")
115

116
117
118
119
120
121
122
123
124
125
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
            "line behind a different model. The columns must be in the order "
            "the modules will be called. The redshift column must be the last "
            "one. Finally, if this parameters is not left empty, cigale will "
            "not interpret the configuration parameters given in pcigale.ini. "
            "They will be given only for information.")

126
        self.config['creation_modules'] = []
127
128
129
130
131
132
133
134
135
136
        self.config.comments['creation_modules'] = ([""] +
            ["Order of the modules use for SED creation. Available modules:"] +
            ["SFH: sfh2exp, sfhdelayed, sfhfromfile, sfhperiodic"] +
            ["SSP: bc03, m2005"] +
            ["Nebular emission: nebular"] +
            ["Dust attenuation: dustatt_calzleit, dustatt_powerlaw"] +
            ["Dust emission: casey2012, dale2014, dl2007, dl2014"] +
            ["AGN: dale2014, fritz2006"] +
            ["Radio: radio"] +
            ["Redshift: redshifting (mandatory!)"])
137
138
139
140

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
141
            "pdf_analysis, savefluxes.")
142

143
144
145
146
147
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
            "Number of CPU cores available. This computer has {} cores."
            .format(mp.cpu_count()))

148
149
150
151
152
153
154
155
156
157
158
159
        self.config.write()

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """

        # Getting the list of the filters available in pcigale database
160
        with Database() as base:
161
            filter_list = base.get_filter_names()
162

163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

            # Check that the id and redshift columns are present in the input
            # file
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
            column_list = []
            for column in obs_table.columns:
                filter_name = column[:-4] if column.endswith('_err') else column
                if filter_name in filter_list:
                    column_list.append(column)

            # Check that we don't have an error column without the associated
            # flux
            for column in column_list:
Médéric Boquien's avatar
Médéric Boquien committed
183
184
                if column.endswith('_err') and (column[:-4]
                                                not in column_list):
185
186
                    raise Exception("The observation table as a {} column "
                                    "but no {} column.".format(column,
Médéric Boquien's avatar
Médéric Boquien committed
187
                                                               column[:-4]))
188
189
190
191

            self.config['column_list'] = column_list
        else:
            self.config['column_list'] = ''
192
193
194
195
196
        self.config.comments['column_list'] = [""] + wrap(
            "List of the columns in the observation data file to use for "
            "the fitting.")

        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
197
        # the configuration section from its parameter list.
198
199
200
201
        self.config['sed_creation_modules'] = {}
        self.config.comments['sed_creation_modules'] = ["", ""] + wrap(
            "Configuration of the SED creation modules.")

202
        for module_name in self.config['creation_modules']:
203
204
205
            self.config["sed_creation_modules"][module_name] = {}
            sub_config = self.config["sed_creation_modules"][module_name]

206
            for name, (typ, description, default) in \
207
208
209
                    creation_modules.get_module(
                        module_name,
                        blank=True).parameter_list.items():
210
211
212
213
214
215
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)

            self.config['sed_creation_modules'].comments[module_name] = [
216
                creation_modules.get_module(module_name, blank=True).comments]
217

218
219
        self.check_modules()

220
221
222
223
224
        # Configuration for the analysis method
        self.config['analysis_configuration'] = {}
        self.config.comments['analysis_configuration'] = ["", ""] + wrap(
            "Configuration of the statistical analysis method.")
        module_name = self.config['analysis_method']
225
        for name, (typ, desc, default) in \
226
                analysis_modules.get_module(module_name).parameter_list.items():
227
228
229
230
231
232
            if default is None:
                default = ''
            self.config['analysis_configuration'][name] = default
            self.config['analysis_configuration'].comments[name] = wrap(desc)

        self.config.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
233

234
235
    @property
    def configuration(self):
Yannick Roehlly's avatar
Yannick Roehlly committed
236
        """Returns a dictionary for the session configuration.
Yannick Roehlly's avatar
Yannick Roehlly committed
237
238
239

        Returns
        -------
240
        configuration['data_file']: string
Yannick Roehlly's avatar
Yannick Roehlly committed
241
            File containing the observations to fit.
242
        configuration['column_list']: list of strings
Yannick Roehlly's avatar
Yannick Roehlly committed
243
            List of the columns of data_file to use in the fitting.
244
        configuration['creation_modules']: list of strings
Yannick Roehlly's avatar
Yannick Roehlly committed
245
            List of the modules (in the right order) used to create the SEDs.
246
        configuration['creation_modules_params']: list of dictionaries
Yannick Roehlly's avatar
Yannick Roehlly committed
247
248
            Configuration parameters for each module. To each parameter, the
            dictionary associates a list of possible values (possibly only
Yannick Roehlly's avatar
Yannick Roehlly committed
249
            one).
250
        configuration['analysis_method']: string
Yannick Roehlly's avatar
Yannick Roehlly committed
251
            Statistical analysis module used to fit the data.
252
        configuration['analysis_method_params']: dictionary
Yannick Roehlly's avatar
Yannick Roehlly committed
253
            Parameters for the statistical analysis module. To each parameter
Yannick Roehlly's avatar
Yannick Roehlly committed
254
255
256
257
            is associated a list of possible values.
        """
        configuration = {}

258
259
        # Before building the configuration dictionary, we ensure that all the
        # fields are filled
260
261
        if not self.config['parameters_file']:
            self.complete_redshifts()
262

263
264
        for section in ['data_file', 'parameters_file', 'column_list',
                        'creation_modules', 'analysis_method']:
Yannick Roehlly's avatar
Yannick Roehlly committed
265
            configuration[section] = self.config[section]
266
        configuration['cores'] = int(self.config['cores'])
Yannick Roehlly's avatar
Yannick Roehlly committed
267

Yannick Roehlly's avatar
Yannick Roehlly committed
268
        # Parsing the SED modules parameters
269
270
        configuration['creation_modules_params'] = []
        for module in self.config['creation_modules']:
271
            module_params = {}
Yannick Roehlly's avatar
Yannick Roehlly committed
272
273
            for key, value in \
                    self.config['sed_creation_modules'][module].items():
Yannick Roehlly's avatar
Yannick Roehlly committed
274
                module_params[key] = evaluate_description(value)
275
            configuration['creation_modules_params'].append(module_params)
276

277
278
279
        if (self.config['analysis_method'] == 'savefluxes' and
            not self.config['analysis_configuration']['variables']):
            warehouse = SedWarehouse()
280
            params = ParametersHandler(configuration)
281
            sed = warehouse.get_sed(params.modules,
282
283
284
285
286
287
288
                                    params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
            self.config['analysis_configuration']['variables'] = info
        elif (self.config['analysis_method'] == 'pdf_analysis' and
              not self.config['analysis_configuration']['analysed_variables']):
            warehouse = SedWarehouse()
289
            params = ParametersHandler(configuration)
290
            sed = warehouse.get_sed(params.modules,
291
292
293
294
295
                                    params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
            self.config['analysis_configuration']['analysed_variables'] = info

296
        # Analysis method parameters
297
298
        configuration['analysis_method_params'] = \
            self.config['analysis_configuration']
Yannick Roehlly's avatar
Yannick Roehlly committed
299
300

        return configuration
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhfromfile',
                                        'sfhperiodic']),
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
                                                     'dustatt_powerlaw']),
                               ('dust emission', ['casey2012', 'dale2014',
                                                  'dl2007', 'dl2014']),
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
321
322
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
323
324
325
326
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
327
                    'dust emission': "No dust emission module found.",
328
329
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
Médéric Boquien's avatar
Médéric Boquien committed
330
                    'redshift': "ERROR! No redshifting module found."}
331
332
333
334
335
336

        for module in modules:
            if all([user_module not in modules[module] for user_module in
                    self.config['creation_modules']]):
                print("{} Options are: {}.".
                      format(comments[module], ', '.join(modules[module])))
337

338
339
340
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
341
342
343
344
345
346
347
348
349
350
351
352
        """

        z_mod = self.config['sed_creation_modules']['redshifting']['redshift']
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
                z = np.unique(np.around(obs_table['redshift'],
                                        decimals=REDSHIFT_DECIMALS))
                self.config['sed_creation_modules']['redshifting']['redshift'] = z
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")