configuration.py 16.9 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5

6
from collections import OrderedDict
7
import multiprocessing as mp
8
9
import os.path
import sys
10
from textwrap import wrap
11
12
13
14

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import numpy as np
15
import validate
16

17
from ..managers.parameters import ParametersManager
18
from ..data import Database
19
from ..utils.io import read_table
20
from .. import sed_modules
21
from .. import analysis_modules
22
from ..warehouse import SedWarehouse
23
from . import validation
24

25
26
27
28
29
30
31
32

class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
33
        Parameters
34
        ----------
35
        filename: string
36
37
38
            Name of the configuration file (pcigale.conf by default).

        """
39
40
41
42
43
44
        self.spec = configobj.ConfigObj(filename+'.spec',
                                        write_empty_values=True,
                                        indent_type='  ',
                                        encoding='UTF8',
                                        list_values=False,
                                        _inspec=True)
45
46
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
47
                                          indent_type='  ',
48
49
50
51
52
53
54
                                          encoding='UTF8',
                                          configspec=self.spec)

        # We validate the configuration so that the variables are converted to
        # the expected that. We do not handle errors at the point but only when
        # we actually return the configuration file from the property() method.
        self.config.validate(validate.Validator(validation.functions))
55

56
57
        self.pcigaleini_exists = os.path.isfile(filename)

58
59
60
61
62
63
64
65
66
67
68
    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
69
70
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
71
            "'distance' (Mpc, optional, if present it will be used in lieu "
72
73
74
75
76
77
            "of the distance computed from the redshift), the filter names for "
            "the fluxes, and the filter names with the '_err' suffix for the "
            "uncertainties. The fluxes and the uncertainties must be in mJy "
            "for broadband data and in W/m² for emission lines. This file is "
            "optional to generate the configuration file, in particular for "
            "the savefluxes module.")
78
        self.spec['data_file'] = "string()"
79

80
81
82
83
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
84
            "line being a different model. The columns must be in the order "
85
            "the modules will be called. The redshift column must be the last "
86
87
            "one. Finally, if this parameter is not empty, cigale will not "
            "interpret the configuration parameters given in pcigale.ini. "
88
89
90
91
            "They will be given only for information. Note that this module "
            "should only be used in conjonction with the savefluxes module. "
            "Using it with the pdf_analysis module will yield incorrect "
            "results.")
92
        self.spec['parameters_file'] = "string()"
93

94
95
        self.config['sed_modules'] = []
        self.config.comments['sed_modules'] = ([""] +
96
97
98
99
100
101
102
            ["Avaiable modules to compute the models. The order must be kept."
            ] +
            ["SFH:"] +
            ["* sfh2exp (double exponential)"] +
            ["* sfhdelayed (delayed SFH with optional exponential burst)"] +
            ["* sfhdelayedbq (delayed SFH with optional constant burst/quench)"
            ] +
Médéric Boquien's avatar
Médéric Boquien committed
103
            ["* sfhfromfile (arbitrary SFH read from an input file)"] +
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
            ["* sfhperiodic (periodic SFH, exponential, rectangle or delayed"
             ")"] +
            ["SSP:"] +
            ["* bc03 (Bruzual and Charlot 2003)"] +
            ["* m2005 (Maraston 2005)"] +
            ["Nebular emission:"] +
            ["* nebular (continuum and line nebular emission)"] +
            ["Dust attenuation:"] +
            ["* dustatt_modified_CF00 (modified Charlot & Fall 2000 "
             "attenuation law)"] +
            ["* dustatt_modified_starburst (modified starburst attenuaton law)"
            ] +
            ["Dust emission:"] +
            ["* casey2012 (Casey 2012 dust emission models)"] +
            ["* dale2014 (Dale et al. 2014 dust emission templates)"] +
            ["* dl2007 (Draine & Li 2007 dust emission models)"] +
            ["* dl2014 (Draine et al. 2014 update of the previous models)"] +
            ["* themis (Themis dust emission models from Jones et al. 2017)"] +
            ["AGN:"] +
            ["* fritz2006 (AGN models from Fritz et al. 2006)"] +
            ["Radio:"] +
            ["* radio (synchrotron emission)"] +
            ["Restframe parameters:"] +
            ["* restframe_parameters (UV slope, IRX-beta, D4000, EW, etc.)"] +
            ["Redshift+IGM:"] +
            ["* redshifting (mandatory, also includes the IGM from Meiksin "
             "2006)"]
        )
132
        self.spec['sed_modules'] = "cigale_string_list()"
133
134
135
136

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
137
            "pdf_analysis, savefluxes.")
138
        self.spec['analysis_method'] = "string()"
139

140
141
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
142
143
            f"Number of CPU cores available. This computer has "
            f"{mp.cpu_count()} cores.")
144
        self.spec['cores'] = "integer(min=1)"
145

146
        self.config.write()
147
        self.spec.write()
148
149
150
151
152
153
154
155
156

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """
157
158
159
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)
160
161

        # Getting the list of the filters available in pcigale database
162
        with Database() as base:
163
            filter_list = base.get_filter_names()
164

165
166
167
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

168
169
170
171
172
173
            # Check that the the file was correctly read and that the id and
            # redshift columns are present in the input file
            if 'col1' in obs_table.columns:
                raise Exception("The input could not be read properly. Verify "
                                "its format and that it does not have two "
                                "columns with the same name.")
174
175
176
177
178
179
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
180
181
182
            bands = []
            for band in obs_table.columns:
                filter_name = band[:-4] if band.endswith('_err') else band
183
                if filter_name in filter_list:
184
                    bands.append(band)
185

186
187
188
189
            # Check that we don't have an band error without the associated
            # band
            for band in bands:
                if band.endswith('_err') and (band[:-4] not in bands):
190
191
                    raise Exception(f"The observation table as a {band} column "
                                    f"but no {band[:-4]} column.")
192

193
            self.config['bands'] = bands
194
        else:
195
196
197
198
199
            self.config['bands'] = ''
        self.config.comments['bands'] = [""] + wrap("Bands to consider. To "
            "consider uncertainties too, the name of the band must be "
            "indicated with the _err suffix. For instance: FUV, FUV_err.")
        self.spec['bands'] = "cigale_string_list()"
200

201
        self.config['properties'] = ''
202
203
204
205
        self.config.comments['properties'] = [""] + wrap("Properties to be "
            "considered. All properties are to be given in the rest frame "
            "rather than the observed frame. This is the case for instance "
            "the equivalent widths and for luminosity densities.")
206
207
        self.spec['properties'] = "cigale_string_list()"

208
        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
209
        # the configuration section from its parameter list.
210
211
        self.config['sed_modules_params'] = {}
        self.config.comments['sed_modules_params'] = ["", ""] + wrap(
212
            "Configuration of the SED creation modules.")
213
        self.spec['sed_modules_params'] = {}
214

215
        for module_name in self.config['sed_modules']:
216
217
218
219
            self.config['sed_modules_params'][module_name] = {}
            self.spec['sed_modules_params'][module_name] = {}
            sub_config = self.config['sed_modules_params'][module_name]
            sub_spec = self.spec['sed_modules_params'][module_name]
220

221
            for name, (typ, description, default) in \
222
                    sed_modules.get_module(
223
224
                        module_name,
                        blank=True).parameter_list.items():
225
226
227
228
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)
229
230
                sub_spec[name] = typ
            self.config['sed_modules_params'].comments[module_name] = [
231
                sed_modules.get_module(module_name, blank=True).comments]
232

233
234
        self.check_modules()

235
        # Configuration for the analysis method
236
237
        self.config['analysis_params'] = {}
        self.config.comments['analysis_params'] = ["", ""] + wrap(
238
            "Configuration of the statistical analysis method.")
239
240
        self.spec['analysis_params'] = {}

241
        module_name = self.config['analysis_method']
242
        for name, (typ, desc, default) in \
243
                analysis_modules.get_module(module_name).parameter_list.items():
244
245
            if default is None:
                default = ''
246
247
248
            self.config['analysis_params'][name] = default
            self.config['analysis_params'].comments[name] = wrap(desc)
            self.spec['analysis_params'][name] = typ
249

250
251
252
253
254
        if 'pdf_analysis' == module_name:
            bands = [band for band in self.config['bands']
                     if not band.endswith('_err')]
            self.config['analysis_params']['bands'] = bands

255
        self.config.write()
256
        self.spec.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
257

258
259
    @property
    def configuration(self):
260
261
        """Returns a dictionary for the session configuration if it is valid.
        Otherwise, print the erroneous keys.
Yannick Roehlly's avatar
Yannick Roehlly committed
262
263
264

        Returns
        -------
265
266
        configuration: dictionary
            Dictionary containing the information provided in pcigale.ini.
Yannick Roehlly's avatar
Yannick Roehlly committed
267
        """
268
269
270
271
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)

272
273
274
275
276
277
278
279
280
281
282
        self.complete_redshifts()
        self.complete_analysed_parameters()

        vdt = validate.Validator(validation.functions)
        validity = self.config.validate(vdt, preserve_errors=True)

        if validity is not True:
            print("The following issues have been found in pcigale.ini:")
            for module, param, message in configobj.flatten_errors(self.config,
                                                                   validity):
                if len(module) > 0:
283
284
                    print(f"Module {'/'.join(module)}, parameter {param}: "
                          f"{message}")
285
                else:
286
                    print(f"Parameter {param}: {message}")
287
            print("Run the same command after having fixed pcigale.ini.")
288

289
            return None
Yannick Roehlly's avatar
Yannick Roehlly committed
290

291
        return self.config.copy()
292
293
294
295
296
297
298
299

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

300
301
        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhdelayedbq',
                                        'sfhfromfile', 'sfhperiodic']),
302
303
304
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
305
                                                     'dustatt_powerlaw',
306
307
308
                                                     'dustatt_2powerlaws',
                                                     'dustatt_modified_CF00',
                                                     'dustatt_modified_starburst']),
309
                               ('dust emission', ['casey2012', 'dale2014',
310
311
                                                  'dl2007', 'dl2014',
                                                  'themis']),
312
313
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
314
315
                               ('restframe_parameters',
                                ['restframe_parameters']),
316
317
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
318
319
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
320
321
322
323
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
324
                    'dust emission': "No dust emission module found.",
325
326
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
327
                    'restframe_parameters': "No restframe parameters module "
328
                                            "found",
Médéric Boquien's avatar
Médéric Boquien committed
329
                    'redshift': "ERROR! No redshifting module found."}
330
331
332

        for module in modules:
            if all([user_module not in modules[module] for user_module in
333
                    self.config['sed_modules']]):
334
335
                print(f"{comments[module]} Options are: "
                      f"{', '.join(modules[module])}.")
336

337
338
339
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
340
341
        """

342
        z_mod = self.config['sed_modules_params']['redshifting']['redshift']
343
344
345
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
346
347
348
349
350
351
352
353
354
                if 'redshift_decimals' in self.config['analysis_params']:
                    decimals = self.config['analysis_params']['redshift_decimals']
                    if decimals < 0:
                        z = list(np.unique(obs_table['redshift']))
                    else:
                        z = list(np.unique(np.around(obs_table['redshift'],
                                                     decimals=decimals)))
                else:
                    z = list(np.unique(obs_table['redshift']))
355
                self.config['sed_modules_params']['redshifting']['redshift'] = z
356
357
358
            elif self.config['parameters_file']:
                # The entry will be ignored anyway. Just pass a dummy list
                self.config['sed_modules_params']['redshifting']['redshift'] = []
359
360
361
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")
362
363
364

    def complete_analysed_parameters(self):
        """Complete the configuration when the variables are missing from the
365
        configuration file and must be extracted from a dummy run."""
366
        if not self.config['analysis_params']['variables']:
367
            warehouse = SedWarehouse()
368
            params = ParametersManager(self.config.dict())
369
370
371
            sed = warehouse.get_sed(params.modules, params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
372
            self.config['analysis_params']['variables'] = info