configuration.py 13.2 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5
6

import pkgutil
7
from collections import Iterable, OrderedDict
8
import multiprocessing as mp
9
from textwrap import wrap
10
11
12
13
14

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import pkg_resources
import numpy as np
15
import validate
16

17
from ..handlers.parameters_handler import ParametersHandler
18
from ..data import Database
19
from ..utils import read_table
20
from .. import sed_modules
21
from .. import analysis_modules
22
from ..warehouse import SedWarehouse
23
from . import validation
24

25

26
27
28
# Limit the redshift to this number of decimals
REDSHIFT_DECIMALS = 2

29
30
31
32
33
34
35
36

class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
37
        Parameters
38
        ----------
39
        filename: string
40
41
42
            Name of the configuration file (pcigale.conf by default).

        """
43
44
45
46
47
48
        self.spec = configobj.ConfigObj(filename+'.spec',
                                        write_empty_values=True,
                                        indent_type='  ',
                                        encoding='UTF8',
                                        list_values=False,
                                        _inspec=True)
49
50
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
51
                                          indent_type='  ',
52
53
54
55
56
57
58
                                          encoding='UTF8',
                                          configspec=self.spec)

        # We validate the configuration so that the variables are converted to
        # the expected that. We do not handle errors at the point but only when
        # we actually return the configuration file from the property() method.
        self.config.validate(validate.Validator(validation.functions))
59
60
61
62
63
64
65
66
67
68
69
70

    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
71
72
73
74
75
76
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
            "the filter names for the fluxes, and the filter names with the "
            "'_err' suffix for the uncertainties. The fluxes and the "
            "uncertainties must be in mJy. This file is optional to generate "
            "the configuration file, in particular for the savefluxes module.")
77
        self.spec['data_file'] = "string"
78

79
80
81
82
83
84
85
86
87
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
            "line behind a different model. The columns must be in the order "
            "the modules will be called. The redshift column must be the last "
            "one. Finally, if this parameters is not left empty, cigale will "
            "not interpret the configuration parameters given in pcigale.ini. "
            "They will be given only for information.")
88
        self.spec['parameters_file'] = "string()"
89

90
91
        self.config['sed_modules'] = []
        self.config.comments['sed_modules'] = ([""] +
92
93
94
95
96
97
98
99
100
            ["Order of the modules use for SED creation. Available modules:"] +
            ["SFH: sfh2exp, sfhdelayed, sfhfromfile, sfhperiodic"] +
            ["SSP: bc03, m2005"] +
            ["Nebular emission: nebular"] +
            ["Dust attenuation: dustatt_calzleit, dustatt_powerlaw"] +
            ["Dust emission: casey2012, dale2014, dl2007, dl2014"] +
            ["AGN: dale2014, fritz2006"] +
            ["Radio: radio"] +
            ["Redshift: redshifting (mandatory!)"])
101
        self.spec['sed_modules'] = "cigale_string_list()"
102
103
104
105

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
106
            "pdf_analysis, savefluxes.")
107
        self.spec['analysis_method'] = "string()"
108

109
110
111
112
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
            "Number of CPU cores available. This computer has {} cores."
            .format(mp.cpu_count()))
113
        self.spec['cores'] = "integer(min=1)"
114

115
        self.config.write()
116
        self.spec.write()
117
118
119
120
121
122
123
124
125
126
127

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """

        # Getting the list of the filters available in pcigale database
128
        with Database() as base:
129
            filter_list = base.get_filter_names()
130

131
132
133
134
135
136
137
138
139
140
141
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

            # Check that the id and redshift columns are present in the input
            # file
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
142
143
144
            bands = []
            for band in obs_table.columns:
                filter_name = band[:-4] if band.endswith('_err') else band
145
                if filter_name in filter_list:
146
                    bands.append(band)
147

148
149
150
151
            # Check that we don't have an band error without the associated
            # band
            for band in bands:
                if band.endswith('_err') and (band[:-4] not in bands):
152
                    raise Exception("The observation table as a {} column "
153
154
                                    "but no {} column.".format(band,
                                                               band[:-4]))
155

156
            self.config['bands'] = bands
157
        else:
158
159
160
161
162
            self.config['bands'] = ''
        self.config.comments['bands'] = [""] + wrap("Bands to consider. To "
            "consider uncertainties too, the name of the band must be "
            "indicated with the _err suffix. For instance: FUV, FUV_err.")
        self.spec['bands'] = "cigale_string_list()"
163
164

        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
165
        # the configuration section from its parameter list.
166
167
        self.config['sed_modules_params'] = {}
        self.config.comments['sed_modules_params'] = ["", ""] + wrap(
168
            "Configuration of the SED creation modules.")
169
        self.spec['sed_modules_params'] = {}
170

171
        for module_name in self.config['sed_modules']:
172
173
174
175
            self.config['sed_modules_params'][module_name] = {}
            self.spec['sed_modules_params'][module_name] = {}
            sub_config = self.config['sed_modules_params'][module_name]
            sub_spec = self.spec['sed_modules_params'][module_name]
176

177
            for name, (typ, description, default) in \
178
                    sed_modules.get_module(
179
180
                        module_name,
                        blank=True).parameter_list.items():
181
182
183
184
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)
185
186
                sub_spec[name] = typ
            self.config['sed_modules_params'].comments[module_name] = [
187
                sed_modules.get_module(module_name, blank=True).comments]
188

189
190
        self.check_modules()

191
        # Configuration for the analysis method
192
193
        self.config['analysis_params'] = {}
        self.config.comments['analysis_params'] = ["", ""] + wrap(
194
            "Configuration of the statistical analysis method.")
195
196
        self.spec['analysis_params'] = {}

197
        module_name = self.config['analysis_method']
198
        for name, (typ, desc, default) in \
199
                analysis_modules.get_module(module_name).parameter_list.items():
200
201
            if default is None:
                default = ''
202
203
204
            self.config['analysis_params'][name] = default
            self.config['analysis_params'].comments[name] = wrap(desc)
            self.spec['analysis_params'][name] = typ
205
206

        self.config.write()
207
        self.spec.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
208

209
210
    @property
    def configuration(self):
211
212
        """Returns a dictionary for the session configuration if it is valid.
        Otherwise, print the erroneous keys.
Yannick Roehlly's avatar
Yannick Roehlly committed
213
214
215

        Returns
        -------
216
217
        configuration: dictionary
            Dictionary containing the information provided in pcigale.ini.
Yannick Roehlly's avatar
Yannick Roehlly committed
218
        """
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
        self.complete_redshifts()
        self.complete_analysed_parameters()

        vdt = validate.Validator(validation.functions)
        validity = self.config.validate(vdt, preserve_errors=True)

        if validity is not True:
            print("The following issues have been found in pcigale.ini:")
            for module, param, message in configobj.flatten_errors(self.config,
                                                                   validity):
                if len(module) > 0:
                    print("Module {}, parameter {}: {}".format('/'.join(module),
                                                               param, message))
                else:
                    print("Parameter {}: {}".format(param, message))
234
            print("Run the same command after having fixed pcigale.ini.")
235

236
            return None
Yannick Roehlly's avatar
Yannick Roehlly committed
237

238
        return self.config.dict()
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhfromfile',
                                        'sfhperiodic']),
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
                                                     'dustatt_powerlaw']),
                               ('dust emission', ['casey2012', 'dale2014',
                                                  'dl2007', 'dl2014']),
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
259
260
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
261
262
263
264
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
265
                    'dust emission': "No dust emission module found.",
266
267
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
Médéric Boquien's avatar
Médéric Boquien committed
268
                    'redshift': "ERROR! No redshifting module found."}
269
270
271

        for module in modules:
            if all([user_module not in modules[module] for user_module in
272
                    self.config['sed_modules']]):
273
274
                print("{} Options are: {}.".
                      format(comments[module], ', '.join(modules[module])))
275

276
277
278
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
279
280
        """

281
        z_mod = self.config['sed_modules_params']['redshifting']['redshift']
282
283
284
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
285
286
287
                z = list(np.unique(np.around(obs_table['redshift'],
                                        decimals=REDSHIFT_DECIMALS)))
                self.config['sed_modules_params']['redshifting']['redshift'] = z
288
289
290
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")
291
292
293
294

    def complete_analysed_parameters(self):
        """Complete the configuration when the variables are missing from the
        configuration file and must be extract from a dummy run."""
295
        if not self.config['analysis_params']['variables']:
296
297
298
299
300
            warehouse = SedWarehouse()
            params = ParametersHandler(self.config.dict())
            sed = warehouse.get_sed(params.modules, params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
301
            self.config['analysis_params']['variables'] = info