configuration.py 15.7 KB
Newer Older
1
# -*- coding: utf-8 -*-
2 3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5

6
from collections import OrderedDict
7
import multiprocessing as mp
8 9
import os.path
import sys
10
from textwrap import wrap
11 12 13 14

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import numpy as np
15
import validate
16

17
from ..managers.parameters import ParametersManager
18
from ..data import Database
19
from ..utils import read_table
20
from .. import sed_modules
21
from .. import analysis_modules
22
from ..warehouse import SedWarehouse
23
from . import validation
24

25 26 27 28 29 30 31 32

class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
33
        Parameters
34
        ----------
35
        filename: string
36 37 38
            Name of the configuration file (pcigale.conf by default).

        """
39 40 41 42 43 44
        self.spec = configobj.ConfigObj(filename+'.spec',
                                        write_empty_values=True,
                                        indent_type='  ',
                                        encoding='UTF8',
                                        list_values=False,
                                        _inspec=True)
45 46
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
47
                                          indent_type='  ',
48 49 50 51 52 53 54
                                          encoding='UTF8',
                                          configspec=self.spec)

        # We validate the configuration so that the variables are converted to
        # the expected that. We do not handle errors at the point but only when
        # we actually return the configuration file from the property() method.
        self.config.validate(validate.Validator(validation.functions))
55

56 57
        self.pcigaleini_exists = os.path.isfile(filename)

58 59 60 61 62 63 64 65 66 67 68
    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
69 70
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
71
            "'distance' (Mpc, optional, if present it will be used in lieu "
72 73 74 75 76 77
            "of the distance computed from the redshift), the filter names for "
            "the fluxes, and the filter names with the '_err' suffix for the "
            "uncertainties. The fluxes and the uncertainties must be in mJy "
            "for broadband data and in W/m² for emission lines. This file is "
            "optional to generate the configuration file, in particular for "
            "the savefluxes module.")
78
        self.spec['data_file'] = "string()"
79

80 81 82 83
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
84
            "line being a different model. The columns must be in the order "
85
            "the modules will be called. The redshift column must be the last "
86 87
            "one. Finally, if this parameter is not empty, cigale will not "
            "interpret the configuration parameters given in pcigale.ini. "
88 89 90 91
            "They will be given only for information. Note that this module "
            "should only be used in conjonction with the savefluxes module. "
            "Using it with the pdf_analysis module will yield incorrect "
            "results.")
92
        self.spec['parameters_file'] = "string()"
93

94 95
        self.config['sed_modules'] = []
        self.config.comments['sed_modules'] = ([""] +
96
            ["Order of the modules use for SED creation. Available modules:"] +
97 98
            ["SFH: sfh2exp, sfhdelayed, sfhdelayedbq, sfhfromfile, "
             "sfhperiodic"] +
99 100
            ["SSP: bc03, m2005"] +
            ["Nebular emission: nebular"] +
101 102
            ["Dust attenuation: dustatt_calzleit, dustatt_powerlaw, "
             "dustatt_2powerlaws"] +
103
            ["Dust emission: casey2012, dale2014, dl2007, dl2014, themis"] +
104 105
            ["AGN: dale2014, fritz2006"] +
            ["Radio: radio"] +
106
            ["Restframe parameters: restframe_parameters"] +
107
            ["Redshift: redshifting (mandatory!)"])
108
        self.spec['sed_modules'] = "cigale_string_list()"
109 110 111 112

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
113
            "pdf_analysis, savefluxes.")
114
        self.spec['analysis_method'] = "string()"
115

116 117 118 119
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
            "Number of CPU cores available. This computer has {} cores."
            .format(mp.cpu_count()))
120
        self.spec['cores'] = "integer(min=1)"
121

122
        self.config.write()
123
        self.spec.write()
124 125 126 127 128 129 130 131 132

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """
133 134 135
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)
136 137

        # Getting the list of the filters available in pcigale database
138
        with Database() as base:
139
            filter_list = base.get_filter_names()
140

141 142 143
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

144 145 146 147 148 149
            # Check that the the file was correctly read and that the id and
            # redshift columns are present in the input file
            if 'col1' in obs_table.columns:
                raise Exception("The input could not be read properly. Verify "
                                "its format and that it does not have two "
                                "columns with the same name.")
150 151 152 153 154 155
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
156 157 158
            bands = []
            for band in obs_table.columns:
                filter_name = band[:-4] if band.endswith('_err') else band
159
                if filter_name in filter_list:
160
                    bands.append(band)
161

162 163 164 165
            # Check that we don't have an band error without the associated
            # band
            for band in bands:
                if band.endswith('_err') and (band[:-4] not in bands):
166
                    raise Exception("The observation table as a {} column "
167 168
                                    "but no {} column.".format(band,
                                                               band[:-4]))
169

170
            self.config['bands'] = bands
171
        else:
172 173 174 175 176
            self.config['bands'] = ''
        self.config.comments['bands'] = [""] + wrap("Bands to consider. To "
            "consider uncertainties too, the name of the band must be "
            "indicated with the _err suffix. For instance: FUV, FUV_err.")
        self.spec['bands'] = "cigale_string_list()"
177

178
        self.config['properties'] = ''
179 180 181 182
        self.config.comments['properties'] = [""] + wrap("Properties to be "
            "considered. All properties are to be given in the rest frame "
            "rather than the observed frame. This is the case for instance "
            "the equivalent widths and for luminosity densities.")
183 184
        self.spec['properties'] = "cigale_string_list()"

185
        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
186
        # the configuration section from its parameter list.
187 188
        self.config['sed_modules_params'] = {}
        self.config.comments['sed_modules_params'] = ["", ""] + wrap(
189
            "Configuration of the SED creation modules.")
190
        self.spec['sed_modules_params'] = {}
191

192
        for module_name in self.config['sed_modules']:
193 194 195 196
            self.config['sed_modules_params'][module_name] = {}
            self.spec['sed_modules_params'][module_name] = {}
            sub_config = self.config['sed_modules_params'][module_name]
            sub_spec = self.spec['sed_modules_params'][module_name]
197

198
            for name, (typ, description, default) in \
199
                    sed_modules.get_module(
200 201
                        module_name,
                        blank=True).parameter_list.items():
202 203 204 205
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)
206 207
                sub_spec[name] = typ
            self.config['sed_modules_params'].comments[module_name] = [
208
                sed_modules.get_module(module_name, blank=True).comments]
209

210 211
        self.check_modules()

212
        # Configuration for the analysis method
213 214
        self.config['analysis_params'] = {}
        self.config.comments['analysis_params'] = ["", ""] + wrap(
215
            "Configuration of the statistical analysis method.")
216 217
        self.spec['analysis_params'] = {}

218
        module_name = self.config['analysis_method']
219
        for name, (typ, desc, default) in \
220
                analysis_modules.get_module(module_name).parameter_list.items():
221 222
            if default is None:
                default = ''
223 224 225
            self.config['analysis_params'][name] = default
            self.config['analysis_params'].comments[name] = wrap(desc)
            self.spec['analysis_params'][name] = typ
226 227

        self.config.write()
228
        self.spec.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
229

230 231
    @property
    def configuration(self):
232 233
        """Returns a dictionary for the session configuration if it is valid.
        Otherwise, print the erroneous keys.
Yannick Roehlly's avatar
Yannick Roehlly committed
234 235 236

        Returns
        -------
237 238
        configuration: dictionary
            Dictionary containing the information provided in pcigale.ini.
Yannick Roehlly's avatar
Yannick Roehlly committed
239
        """
240 241 242 243
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)

244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
        self.complete_redshifts()
        self.complete_analysed_parameters()

        vdt = validate.Validator(validation.functions)
        validity = self.config.validate(vdt, preserve_errors=True)

        if validity is not True:
            print("The following issues have been found in pcigale.ini:")
            for module, param, message in configobj.flatten_errors(self.config,
                                                                   validity):
                if len(module) > 0:
                    print("Module {}, parameter {}: {}".format('/'.join(module),
                                                               param, message))
                else:
                    print("Parameter {}: {}".format(param, message))
259
            print("Run the same command after having fixed pcigale.ini.")
260

261
            return None
Yannick Roehlly's avatar
Yannick Roehlly committed
262

263
        return self.config.copy()
264 265 266 267 268 269 270 271

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

272 273
        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhdelayedbq',
                                        'sfhfromfile', 'sfhperiodic']),
274 275 276
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
277
                                                     'dustatt_powerlaw',
278 279 280
                                                     'dustatt_2powerlaws',
                                                     'dustatt_modified_CF00',
                                                     'dustatt_modified_starburst']),
281
                               ('dust emission', ['casey2012', 'dale2014',
282 283
                                                  'dl2007', 'dl2014',
                                                  'themis']),
284 285
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
286 287
                               ('restframe_parameters',
                                ['restframe_parameters']),
288 289
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
290 291
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
292 293 294 295
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
296
                    'dust emission': "No dust emission module found.",
297 298
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
299
                    'restframe_parameters': "No restframe parameters module "
300
                                            "found",
Médéric Boquien's avatar
Médéric Boquien committed
301
                    'redshift': "ERROR! No redshifting module found."}
302 303 304

        for module in modules:
            if all([user_module not in modules[module] for user_module in
305
                    self.config['sed_modules']]):
306 307
                print("{} Options are: {}.".
                      format(comments[module], ', '.join(modules[module])))
308

309 310 311
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
312 313
        """

314
        z_mod = self.config['sed_modules_params']['redshifting']['redshift']
315 316 317
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
318 319 320 321 322 323 324 325 326
                if 'redshift_decimals' in self.config['analysis_params']:
                    decimals = self.config['analysis_params']['redshift_decimals']
                    if decimals < 0:
                        z = list(np.unique(obs_table['redshift']))
                    else:
                        z = list(np.unique(np.around(obs_table['redshift'],
                                                     decimals=decimals)))
                else:
                    z = list(np.unique(obs_table['redshift']))
327
                self.config['sed_modules_params']['redshifting']['redshift'] = z
328 329 330
            elif self.config['parameters_file']:
                # The entry will be ignored anyway. Just pass a dummy list
                self.config['sed_modules_params']['redshifting']['redshift'] = []
331 332 333
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")
334 335 336

    def complete_analysed_parameters(self):
        """Complete the configuration when the variables are missing from the
337
        configuration file and must be extracted from a dummy run."""
338
        if not self.config['analysis_params']['variables']:
339
            warehouse = SedWarehouse()
340
            params = ParametersManager(self.config.dict())
341 342 343
            sed = warehouse.get_sed(params.modules, params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
344
            self.config['analysis_params']['variables'] = info