configuration.py 13.7 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5
6

import pkgutil
7
from collections import Iterable, OrderedDict
8
import multiprocessing as mp
9
from textwrap import wrap
10
11
12
13
14

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import pkg_resources
import numpy as np
15
import validate
16

17
from ..handlers.parameters_handler import ParametersHandler
18
from ..data import Database
19
from ..utils import read_table
20
from .. import sed_modules
21
from .. import analysis_modules
22
from ..warehouse import SedWarehouse
23
from . import validation
24

25

26
27
28
# Limit the redshift to this number of decimals
REDSHIFT_DECIMALS = 2

29
30
31
32
33
34
35
36

class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
37
        Parameters
38
        ----------
39
        filename: string
40
41
42
            Name of the configuration file (pcigale.conf by default).

        """
43
44
45
46
47
48
        self.spec = configobj.ConfigObj(filename+'.spec',
                                        write_empty_values=True,
                                        indent_type='  ',
                                        encoding='UTF8',
                                        list_values=False,
                                        _inspec=True)
49
50
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
51
                                          indent_type='  ',
52
53
54
55
56
57
58
                                          encoding='UTF8',
                                          configspec=self.spec)

        # We validate the configuration so that the variables are converted to
        # the expected that. We do not handle errors at the point but only when
        # we actually return the configuration file from the property() method.
        self.config.validate(validate.Validator(validation.functions))
59
60
61
62
63
64
65
66
67
68
69
70

    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
71
72
73
74
75
76
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
            "the filter names for the fluxes, and the filter names with the "
            "'_err' suffix for the uncertainties. The fluxes and the "
            "uncertainties must be in mJy. This file is optional to generate "
            "the configuration file, in particular for the savefluxes module.")
77
        self.spec['data_file'] = "string"
78

79
80
81
82
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
83
            "line being a different model. The columns must be in the order "
84
            "the modules will be called. The redshift column must be the last "
85
86
            "one. Finally, if this parameter is not empty, cigale will not "
            "interpret the configuration parameters given in pcigale.ini. "
87
88
89
90
            "They will be given only for information. Note that this module "
            "should only be used in conjonction with the savefluxes module. "
            "Using it with the pdf_analysis module will yield incorrect "
            "results.")
91
        self.spec['parameters_file'] = "string()"
92

93
94
        self.config['sed_modules'] = []
        self.config.comments['sed_modules'] = ([""] +
95
96
97
98
            ["Order of the modules use for SED creation. Available modules:"] +
            ["SFH: sfh2exp, sfhdelayed, sfhfromfile, sfhperiodic"] +
            ["SSP: bc03, m2005"] +
            ["Nebular emission: nebular"] +
99
100
            ["Dust attenuation: dustatt_calzleit, dustatt_powerlaw, "
             "dustatt_2powerlaws"] +
101
102
103
104
            ["Dust emission: casey2012, dale2014, dl2007, dl2014"] +
            ["AGN: dale2014, fritz2006"] +
            ["Radio: radio"] +
            ["Redshift: redshifting (mandatory!)"])
105
        self.spec['sed_modules'] = "cigale_string_list()"
106
107
108
109

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
110
            "pdf_analysis, savefluxes.")
111
        self.spec['analysis_method'] = "string()"
112

113
114
115
116
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
            "Number of CPU cores available. This computer has {} cores."
            .format(mp.cpu_count()))
117
        self.spec['cores'] = "integer(min=1)"
118

119
        self.config.write()
120
        self.spec.write()
121
122
123
124
125
126
127
128
129
130
131

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """

        # Getting the list of the filters available in pcigale database
132
        with Database() as base:
133
            filter_list = base.get_filter_names()
134

135
136
137
138
139
140
141
142
143
144
145
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

            # Check that the id and redshift columns are present in the input
            # file
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
146
147
148
            bands = []
            for band in obs_table.columns:
                filter_name = band[:-4] if band.endswith('_err') else band
149
                if filter_name in filter_list:
150
                    bands.append(band)
151

152
153
154
155
            # Check that we don't have an band error without the associated
            # band
            for band in bands:
                if band.endswith('_err') and (band[:-4] not in bands):
156
                    raise Exception("The observation table as a {} column "
157
158
                                    "but no {} column.".format(band,
                                                               band[:-4]))
159

160
            self.config['bands'] = bands
161
        else:
162
163
164
165
166
            self.config['bands'] = ''
        self.config.comments['bands'] = [""] + wrap("Bands to consider. To "
            "consider uncertainties too, the name of the band must be "
            "indicated with the _err suffix. For instance: FUV, FUV_err.")
        self.spec['bands'] = "cigale_string_list()"
167
168

        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
169
        # the configuration section from its parameter list.
170
171
        self.config['sed_modules_params'] = {}
        self.config.comments['sed_modules_params'] = ["", ""] + wrap(
172
            "Configuration of the SED creation modules.")
173
        self.spec['sed_modules_params'] = {}
174

175
        for module_name in self.config['sed_modules']:
176
177
178
179
            self.config['sed_modules_params'][module_name] = {}
            self.spec['sed_modules_params'][module_name] = {}
            sub_config = self.config['sed_modules_params'][module_name]
            sub_spec = self.spec['sed_modules_params'][module_name]
180

181
            for name, (typ, description, default) in \
182
                    sed_modules.get_module(
183
184
                        module_name,
                        blank=True).parameter_list.items():
185
186
187
188
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)
189
190
                sub_spec[name] = typ
            self.config['sed_modules_params'].comments[module_name] = [
191
                sed_modules.get_module(module_name, blank=True).comments]
192

193
194
        self.check_modules()

195
        # Configuration for the analysis method
196
197
        self.config['analysis_params'] = {}
        self.config.comments['analysis_params'] = ["", ""] + wrap(
198
            "Configuration of the statistical analysis method.")
199
200
        self.spec['analysis_params'] = {}

201
        module_name = self.config['analysis_method']
202
        for name, (typ, desc, default) in \
203
                analysis_modules.get_module(module_name).parameter_list.items():
204
205
            if default is None:
                default = ''
206
207
208
            self.config['analysis_params'][name] = default
            self.config['analysis_params'].comments[name] = wrap(desc)
            self.spec['analysis_params'][name] = typ
209
210

        self.config.write()
211
        self.spec.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
212

213
214
    @property
    def configuration(self):
215
216
        """Returns a dictionary for the session configuration if it is valid.
        Otherwise, print the erroneous keys.
Yannick Roehlly's avatar
Yannick Roehlly committed
217
218
219

        Returns
        -------
220
221
        configuration: dictionary
            Dictionary containing the information provided in pcigale.ini.
Yannick Roehlly's avatar
Yannick Roehlly committed
222
        """
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
        self.complete_redshifts()
        self.complete_analysed_parameters()

        vdt = validate.Validator(validation.functions)
        validity = self.config.validate(vdt, preserve_errors=True)

        if validity is not True:
            print("The following issues have been found in pcigale.ini:")
            for module, param, message in configobj.flatten_errors(self.config,
                                                                   validity):
                if len(module) > 0:
                    print("Module {}, parameter {}: {}".format('/'.join(module),
                                                               param, message))
                else:
                    print("Parameter {}: {}".format(param, message))
238
            print("Run the same command after having fixed pcigale.ini.")
239

240
            return None
Yannick Roehlly's avatar
Yannick Roehlly committed
241

242
        return self.config.dict()
243
244
245
246
247
248
249
250
251
252
253
254
255

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhfromfile',
                                        'sfhperiodic']),
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
256
257
                                                     'dustatt_powerlaw',
                                                     'dustatt_2powerlaws']),
258
259
260
261
262
263
                               ('dust emission', ['casey2012', 'dale2014',
                                                  'dl2007', 'dl2014']),
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
264
265
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
266
267
268
269
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
270
                    'dust emission': "No dust emission module found.",
271
272
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
Médéric Boquien's avatar
Médéric Boquien committed
273
                    'redshift': "ERROR! No redshifting module found."}
274
275
276

        for module in modules:
            if all([user_module not in modules[module] for user_module in
277
                    self.config['sed_modules']]):
278
279
                print("{} Options are: {}.".
                      format(comments[module], ', '.join(modules[module])))
280

281
282
283
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
284
285
        """

286
        z_mod = self.config['sed_modules_params']['redshifting']['redshift']
287
288
289
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
290
291
292
                z = list(np.unique(np.around(obs_table['redshift'],
                                        decimals=REDSHIFT_DECIMALS)))
                self.config['sed_modules_params']['redshifting']['redshift'] = z
293
294
295
            elif self.config['parameters_file']:
                # The entry will be ignored anyway. Just pass a dummy list
                self.config['sed_modules_params']['redshifting']['redshift'] = []
296
297
298
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")
299
300
301

    def complete_analysed_parameters(self):
        """Complete the configuration when the variables are missing from the
302
        configuration file and must be extracted from a dummy run."""
303
        if not self.config['analysis_params']['variables']:
304
305
306
307
308
            warehouse = SedWarehouse()
            params = ParametersHandler(self.config.dict())
            sed = warehouse.get_sed(params.modules, params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
309
            self.config['analysis_params']['variables'] = info