configuration.py 16.8 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5

6
from collections import OrderedDict
7
import multiprocessing as mp
8
9
import os.path
import sys
10
from textwrap import wrap
11
12
13
14

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import numpy as np
15
import validate
16

17
from ..managers.parameters import ParametersManager
18
from ..data import Database
19
from ..utils import read_table
20
from .. import sed_modules
21
from .. import analysis_modules
22
from ..warehouse import SedWarehouse
23
from . import validation
24

25
26
27
28
29
30
31
32

class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
33
        Parameters
34
        ----------
35
        filename: string
36
37
38
            Name of the configuration file (pcigale.conf by default).

        """
39
40
41
42
43
44
        self.spec = configobj.ConfigObj(filename+'.spec',
                                        write_empty_values=True,
                                        indent_type='  ',
                                        encoding='UTF8',
                                        list_values=False,
                                        _inspec=True)
45
46
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
47
                                          indent_type='  ',
48
49
50
51
52
53
54
                                          encoding='UTF8',
                                          configspec=self.spec)

        # We validate the configuration so that the variables are converted to
        # the expected that. We do not handle errors at the point but only when
        # we actually return the configuration file from the property() method.
        self.config.validate(validate.Validator(validation.functions))
55

56
57
        self.pcigaleini_exists = os.path.isfile(filename)

58
59
60
61
62
63
64
65
66
67
68
    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
69
70
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
71
            "'distance' (Mpc, optional, if present it will be used in lieu "
72
73
74
75
76
77
            "of the distance computed from the redshift), the filter names for "
            "the fluxes, and the filter names with the '_err' suffix for the "
            "uncertainties. The fluxes and the uncertainties must be in mJy "
            "for broadband data and in W/m² for emission lines. This file is "
            "optional to generate the configuration file, in particular for "
            "the savefluxes module.")
78
        self.spec['data_file'] = "string()"
79

80
81
82
83
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
84
            "line being a different model. The columns must be in the order "
85
            "the modules will be called. The redshift column must be the last "
86
87
            "one. Finally, if this parameter is not empty, cigale will not "
            "interpret the configuration parameters given in pcigale.ini. "
88
89
90
91
            "They will be given only for information. Note that this module "
            "should only be used in conjonction with the savefluxes module. "
            "Using it with the pdf_analysis module will yield incorrect "
            "results.")
92
        self.spec['parameters_file'] = "string()"
93

94
95
        self.config['sed_modules'] = []
        self.config.comments['sed_modules'] = ([""] +
96
97
98
99
100
101
102
            ["Avaiable modules to compute the models. The order must be kept."
            ] +
            ["SFH:"] +
            ["* sfh2exp (double exponential)"] +
            ["* sfhdelayed (delayed SFH with optional exponential burst)"] +
            ["* sfhdelayedbq (delayed SFH with optional constant burst/quench)"
            ] +
Médéric Boquien's avatar
Médéric Boquien committed
103
            ["* sfhfromfile (arbitrary SFH read from an input file)"] +
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
            ["* sfhperiodic (periodic SFH, exponential, rectangle or delayed"
             ")"] +
            ["SSP:"] +
            ["* bc03 (Bruzual and Charlot 2003)"] +
            ["* m2005 (Maraston 2005)"] +
            ["Nebular emission:"] +
            ["* nebular (continuum and line nebular emission)"] +
            ["Dust attenuation:"] +
            ["* dustatt_modified_CF00 (modified Charlot & Fall 2000 "
             "attenuation law)"] +
            ["* dustatt_modified_starburst (modified starburst attenuaton law)"
            ] +
            ["Dust emission:"] +
            ["* casey2012 (Casey 2012 dust emission models)"] +
            ["* dale2014 (Dale et al. 2014 dust emission templates)"] +
            ["* dl2007 (Draine & Li 2007 dust emission models)"] +
            ["* dl2014 (Draine et al. 2014 update of the previous models)"] +
            ["* themis (Themis dust emission models from Jones et al. 2017)"] +
            ["AGN:"] +
            ["* fritz2006 (AGN models from Fritz et al. 2006)"] +
            ["Radio:"] +
            ["* radio (synchrotron emission)"] +
            ["Restframe parameters:"] +
            ["* restframe_parameters (UV slope, IRX-beta, D4000, EW, etc.)"] +
            ["Redshift+IGM:"] +
            ["* redshifting (mandatory, also includes the IGM from Meiksin "
             "2006)"]
        )
132
        self.spec['sed_modules'] = "cigale_string_list()"
133
134
135
136

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
137
            "pdf_analysis, savefluxes.")
138
        self.spec['analysis_method'] = "string()"
139

140
141
142
143
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
            "Number of CPU cores available. This computer has {} cores."
            .format(mp.cpu_count()))
144
        self.spec['cores'] = "integer(min=1)"
145

146
        self.config.write()
147
        self.spec.write()
148
149
150
151
152
153
154
155
156

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """
157
158
159
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)
160
161

        # Getting the list of the filters available in pcigale database
162
        with Database() as base:
163
            filter_list = base.get_filter_names()
164

165
166
167
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

168
169
170
171
172
173
            # Check that the the file was correctly read and that the id and
            # redshift columns are present in the input file
            if 'col1' in obs_table.columns:
                raise Exception("The input could not be read properly. Verify "
                                "its format and that it does not have two "
                                "columns with the same name.")
174
175
176
177
178
179
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
180
181
182
            bands = []
            for band in obs_table.columns:
                filter_name = band[:-4] if band.endswith('_err') else band
183
                if filter_name in filter_list:
184
                    bands.append(band)
185

186
187
188
189
            # Check that we don't have an band error without the associated
            # band
            for band in bands:
                if band.endswith('_err') and (band[:-4] not in bands):
190
                    raise Exception("The observation table as a {} column "
191
192
                                    "but no {} column.".format(band,
                                                               band[:-4]))
193

194
            self.config['bands'] = bands
195
        else:
196
197
198
199
200
            self.config['bands'] = ''
        self.config.comments['bands'] = [""] + wrap("Bands to consider. To "
            "consider uncertainties too, the name of the band must be "
            "indicated with the _err suffix. For instance: FUV, FUV_err.")
        self.spec['bands'] = "cigale_string_list()"
201

202
        self.config['properties'] = ''
203
204
205
206
        self.config.comments['properties'] = [""] + wrap("Properties to be "
            "considered. All properties are to be given in the rest frame "
            "rather than the observed frame. This is the case for instance "
            "the equivalent widths and for luminosity densities.")
207
208
        self.spec['properties'] = "cigale_string_list()"

209
        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
210
        # the configuration section from its parameter list.
211
212
        self.config['sed_modules_params'] = {}
        self.config.comments['sed_modules_params'] = ["", ""] + wrap(
213
            "Configuration of the SED creation modules.")
214
        self.spec['sed_modules_params'] = {}
215

216
        for module_name in self.config['sed_modules']:
217
218
219
220
            self.config['sed_modules_params'][module_name] = {}
            self.spec['sed_modules_params'][module_name] = {}
            sub_config = self.config['sed_modules_params'][module_name]
            sub_spec = self.spec['sed_modules_params'][module_name]
221

222
            for name, (typ, description, default) in \
223
                    sed_modules.get_module(
224
225
                        module_name,
                        blank=True).parameter_list.items():
226
227
228
229
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)
230
231
                sub_spec[name] = typ
            self.config['sed_modules_params'].comments[module_name] = [
232
                sed_modules.get_module(module_name, blank=True).comments]
233

234
235
        self.check_modules()

236
        # Configuration for the analysis method
237
238
        self.config['analysis_params'] = {}
        self.config.comments['analysis_params'] = ["", ""] + wrap(
239
            "Configuration of the statistical analysis method.")
240
241
        self.spec['analysis_params'] = {}

242
        module_name = self.config['analysis_method']
243
        for name, (typ, desc, default) in \
244
                analysis_modules.get_module(module_name).parameter_list.items():
245
246
            if default is None:
                default = ''
247
248
249
            self.config['analysis_params'][name] = default
            self.config['analysis_params'].comments[name] = wrap(desc)
            self.spec['analysis_params'][name] = typ
250
251

        self.config.write()
252
        self.spec.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
253

254
255
    @property
    def configuration(self):
256
257
        """Returns a dictionary for the session configuration if it is valid.
        Otherwise, print the erroneous keys.
Yannick Roehlly's avatar
Yannick Roehlly committed
258
259
260

        Returns
        -------
261
262
        configuration: dictionary
            Dictionary containing the information provided in pcigale.ini.
Yannick Roehlly's avatar
Yannick Roehlly committed
263
        """
264
265
266
267
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)

268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
        self.complete_redshifts()
        self.complete_analysed_parameters()

        vdt = validate.Validator(validation.functions)
        validity = self.config.validate(vdt, preserve_errors=True)

        if validity is not True:
            print("The following issues have been found in pcigale.ini:")
            for module, param, message in configobj.flatten_errors(self.config,
                                                                   validity):
                if len(module) > 0:
                    print("Module {}, parameter {}: {}".format('/'.join(module),
                                                               param, message))
                else:
                    print("Parameter {}: {}".format(param, message))
283
            print("Run the same command after having fixed pcigale.ini.")
284

285
            return None
Yannick Roehlly's avatar
Yannick Roehlly committed
286

287
        return self.config.copy()
288
289
290
291
292
293
294
295

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

296
297
        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhdelayedbq',
                                        'sfhfromfile', 'sfhperiodic']),
298
299
300
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
301
                                                     'dustatt_powerlaw',
302
303
304
                                                     'dustatt_2powerlaws',
                                                     'dustatt_modified_CF00',
                                                     'dustatt_modified_starburst']),
305
                               ('dust emission', ['casey2012', 'dale2014',
306
307
                                                  'dl2007', 'dl2014',
                                                  'themis']),
308
309
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
310
311
                               ('restframe_parameters',
                                ['restframe_parameters']),
312
313
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
314
315
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
316
317
318
319
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
320
                    'dust emission': "No dust emission module found.",
321
322
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
323
                    'restframe_parameters': "No restframe parameters module "
324
                                            "found",
Médéric Boquien's avatar
Médéric Boquien committed
325
                    'redshift': "ERROR! No redshifting module found."}
326
327
328

        for module in modules:
            if all([user_module not in modules[module] for user_module in
329
                    self.config['sed_modules']]):
330
331
                print("{} Options are: {}.".
                      format(comments[module], ', '.join(modules[module])))
332

333
334
335
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
336
337
        """

338
        z_mod = self.config['sed_modules_params']['redshifting']['redshift']
339
340
341
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
342
343
344
345
346
347
348
349
350
                if 'redshift_decimals' in self.config['analysis_params']:
                    decimals = self.config['analysis_params']['redshift_decimals']
                    if decimals < 0:
                        z = list(np.unique(obs_table['redshift']))
                    else:
                        z = list(np.unique(np.around(obs_table['redshift'],
                                                     decimals=decimals)))
                else:
                    z = list(np.unique(obs_table['redshift']))
351
                self.config['sed_modules_params']['redshifting']['redshift'] = z
352
353
354
            elif self.config['parameters_file']:
                # The entry will be ignored anyway. Just pass a dummy list
                self.config['sed_modules_params']['redshifting']['redshift'] = []
355
356
357
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")
358
359
360

    def complete_analysed_parameters(self):
        """Complete the configuration when the variables are missing from the
361
        configuration file and must be extracted from a dummy run."""
362
        if not self.config['analysis_params']['variables']:
363
            warehouse = SedWarehouse()
364
            params = ParametersManager(self.config.dict())
365
366
367
            sed = warehouse.get_sed(params.modules, params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
368
            self.config['analysis_params']['variables'] = info