configuration.py 16.7 KB
Newer Older
1
# -*- coding: utf-8 -*-
2 3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5

6
from collections import OrderedDict
7
import multiprocessing as mp
8 9
import os.path
import sys
10
from textwrap import wrap
11 12 13 14

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import numpy as np
15
import validate
16

17
from ..managers.parameters import ParametersManager
18
from ..data import Database
19
from ..utils import read_table
20
from .. import sed_modules
21
from .. import analysis_modules
22
from ..warehouse import SedWarehouse
23
from . import validation
24

25 26 27 28 29 30 31 32

class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

33
        Parameters
34
        ----------
35
        filename: string
36 37 38
            Name of the configuration file (pcigale.conf by default).

        """
39 40 41 42 43 44
        self.spec = configobj.ConfigObj(filename+'.spec',
                                        write_empty_values=True,
                                        indent_type='  ',
                                        encoding='UTF8',
                                        list_values=False,
                                        _inspec=True)
45 46
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
47
                                          indent_type='  ',
48 49 50 51 52 53 54
                                          encoding='UTF8',
                                          configspec=self.spec)

        # We validate the configuration so that the variables are converted to
        # the expected that. We do not handle errors at the point but only when
        # we actually return the configuration file from the property() method.
        self.config.validate(validate.Validator(validation.functions))
55

56 57
        self.pcigaleini_exists = os.path.isfile(filename)

58 59 60 61 62 63 64 65 66 67 68
    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
69 70
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
71
            "'distance' (Mpc, optional, if present it will be used in lieu "
72 73 74 75 76 77
            "of the distance computed from the redshift), the filter names for "
            "the fluxes, and the filter names with the '_err' suffix for the "
            "uncertainties. The fluxes and the uncertainties must be in mJy "
            "for broadband data and in W/m² for emission lines. This file is "
            "optional to generate the configuration file, in particular for "
            "the savefluxes module.")
78
        self.spec['data_file'] = "string()"
79

80 81 82 83
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
84
            "line being a different model. The columns must be in the order "
85
            "the modules will be called. The redshift column must be the last "
86 87
            "one. Finally, if this parameter is not empty, cigale will not "
            "interpret the configuration parameters given in pcigale.ini. "
88 89 90 91
            "They will be given only for information. Note that this module "
            "should only be used in conjonction with the savefluxes module. "
            "Using it with the pdf_analysis module will yield incorrect "
            "results.")
92
        self.spec['parameters_file'] = "string()"
93

94 95
        self.config['sed_modules'] = []
        self.config.comments['sed_modules'] = ([""] +
96 97 98 99 100 101 102
            ["Avaiable modules to compute the models. The order must be kept."
            ] +
            ["SFH:"] +
            ["* sfh2exp (double exponential)"] +
            ["* sfhdelayed (delayed SFH with optional exponential burst)"] +
            ["* sfhdelayedbq (delayed SFH with optional constant burst/quench)"
            ] +
Médéric Boquien's avatar
Médéric Boquien committed
103
            ["* sfhfromfile (arbitrary SFH read from an input file)"] +
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
            ["* sfhperiodic (periodic SFH, exponential, rectangle or delayed"
             ")"] +
            ["SSP:"] +
            ["* bc03 (Bruzual and Charlot 2003)"] +
            ["* m2005 (Maraston 2005)"] +
            ["Nebular emission:"] +
            ["* nebular (continuum and line nebular emission)"] +
            ["Dust attenuation:"] +
            ["* dustatt_modified_CF00 (modified Charlot & Fall 2000 "
             "attenuation law)"] +
            ["* dustatt_modified_starburst (modified starburst attenuaton law)"
            ] +
            ["Dust emission:"] +
            ["* casey2012 (Casey 2012 dust emission models)"] +
            ["* dale2014 (Dale et al. 2014 dust emission templates)"] +
            ["* dl2007 (Draine & Li 2007 dust emission models)"] +
            ["* dl2014 (Draine et al. 2014 update of the previous models)"] +
            ["* themis (Themis dust emission models from Jones et al. 2017)"] +
            ["AGN:"] +
            ["* fritz2006 (AGN models from Fritz et al. 2006)"] +
            ["Radio:"] +
            ["* radio (synchrotron emission)"] +
            ["Restframe parameters:"] +
            ["* restframe_parameters (UV slope, IRX-beta, D4000, EW, etc.)"] +
            ["Redshift+IGM:"] +
            ["* redshifting (mandatory, also includes the IGM from Meiksin "
             "2006)"]
        )
132
        self.spec['sed_modules'] = "cigale_string_list()"
133 134 135 136

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
137
            "pdf_analysis, savefluxes.")
138
        self.spec['analysis_method'] = "string()"
139

140 141
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
142 143
            f"Number of CPU cores available. This computer has "
            f"{mp.cpu_count()} cores.")
144
        self.spec['cores'] = "integer(min=1)"
145

146
        self.config.write()
147
        self.spec.write()
148 149 150 151 152 153 154 155 156

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """
157 158 159
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)
160 161

        # Getting the list of the filters available in pcigale database
162
        with Database() as base:
163
            filter_list = base.get_filter_names()
164

165 166 167
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

168 169 170 171 172 173
            # Check that the the file was correctly read and that the id and
            # redshift columns are present in the input file
            if 'col1' in obs_table.columns:
                raise Exception("The input could not be read properly. Verify "
                                "its format and that it does not have two "
                                "columns with the same name.")
174 175 176 177 178 179
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
180 181 182
            bands = []
            for band in obs_table.columns:
                filter_name = band[:-4] if band.endswith('_err') else band
183
                if filter_name in filter_list:
184
                    bands.append(band)
185

186 187 188 189
            # Check that we don't have an band error without the associated
            # band
            for band in bands:
                if band.endswith('_err') and (band[:-4] not in bands):
190 191
                    raise Exception(f"The observation table as a {band} column "
                                    f"but no {band[:-4]} column.")
192

193
            self.config['bands'] = bands
194
        else:
195 196 197 198 199
            self.config['bands'] = ''
        self.config.comments['bands'] = [""] + wrap("Bands to consider. To "
            "consider uncertainties too, the name of the band must be "
            "indicated with the _err suffix. For instance: FUV, FUV_err.")
        self.spec['bands'] = "cigale_string_list()"
200

201
        self.config['properties'] = ''
202 203 204 205
        self.config.comments['properties'] = [""] + wrap("Properties to be "
            "considered. All properties are to be given in the rest frame "
            "rather than the observed frame. This is the case for instance "
            "the equivalent widths and for luminosity densities.")
206 207
        self.spec['properties'] = "cigale_string_list()"

208
        # SED creation modules configurations. For each module, we generate
209
        # the configuration section from its parameter list.
210 211
        self.config['sed_modules_params'] = {}
        self.config.comments['sed_modules_params'] = ["", ""] + wrap(
212
            "Configuration of the SED creation modules.")
213
        self.spec['sed_modules_params'] = {}
214

215
        for module_name in self.config['sed_modules']:
216 217 218 219
            self.config['sed_modules_params'][module_name] = {}
            self.spec['sed_modules_params'][module_name] = {}
            sub_config = self.config['sed_modules_params'][module_name]
            sub_spec = self.spec['sed_modules_params'][module_name]
220

221
            for name, (typ, description, default) in \
222
                    sed_modules.get_module(
223 224
                        module_name,
                        blank=True).parameter_list.items():
225 226 227 228
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)
229 230
                sub_spec[name] = typ
            self.config['sed_modules_params'].comments[module_name] = [
231
                sed_modules.get_module(module_name, blank=True).comments]
232

233 234
        self.check_modules()

235
        # Configuration for the analysis method
236 237
        self.config['analysis_params'] = {}
        self.config.comments['analysis_params'] = ["", ""] + wrap(
238
            "Configuration of the statistical analysis method.")
239 240
        self.spec['analysis_params'] = {}

241
        module_name = self.config['analysis_method']
242
        for name, (typ, desc, default) in \
243
                analysis_modules.get_module(module_name).parameter_list.items():
244 245
            if default is None:
                default = ''
246 247 248
            self.config['analysis_params'][name] = default
            self.config['analysis_params'].comments[name] = wrap(desc)
            self.spec['analysis_params'][name] = typ
249 250

        self.config.write()
251
        self.spec.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
252

253 254
    @property
    def configuration(self):
255 256
        """Returns a dictionary for the session configuration if it is valid.
        Otherwise, print the erroneous keys.
Yannick Roehlly's avatar
Yannick Roehlly committed
257 258 259

        Returns
        -------
260 261
        configuration: dictionary
            Dictionary containing the information provided in pcigale.ini.
Yannick Roehlly's avatar
Yannick Roehlly committed
262
        """
263 264 265 266
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)

267 268 269 270 271 272 273 274 275 276 277
        self.complete_redshifts()
        self.complete_analysed_parameters()

        vdt = validate.Validator(validation.functions)
        validity = self.config.validate(vdt, preserve_errors=True)

        if validity is not True:
            print("The following issues have been found in pcigale.ini:")
            for module, param, message in configobj.flatten_errors(self.config,
                                                                   validity):
                if len(module) > 0:
278 279
                    print(f"Module {'/'.join(module)}, parameter {param}: "
                          f"{message}")
280
                else:
281
                    print(f"Parameter {param}: {message}")
282
            print("Run the same command after having fixed pcigale.ini.")
283

284
            return None
Yannick Roehlly's avatar
Yannick Roehlly committed
285

286
        return self.config.copy()
287 288 289 290 291 292 293 294

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

295 296
        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhdelayedbq',
                                        'sfhfromfile', 'sfhperiodic']),
297 298 299
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
300
                                                     'dustatt_powerlaw',
301 302 303
                                                     'dustatt_2powerlaws',
                                                     'dustatt_modified_CF00',
                                                     'dustatt_modified_starburst']),
304
                               ('dust emission', ['casey2012', 'dale2014',
305 306
                                                  'dl2007', 'dl2014',
                                                  'themis']),
307 308
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
309 310
                               ('restframe_parameters',
                                ['restframe_parameters']),
311 312
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
313 314
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
315 316 317 318
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
319
                    'dust emission': "No dust emission module found.",
320 321
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
322
                    'restframe_parameters': "No restframe parameters module "
323
                                            "found",
Médéric Boquien's avatar
Médéric Boquien committed
324
                    'redshift': "ERROR! No redshifting module found."}
325 326 327

        for module in modules:
            if all([user_module not in modules[module] for user_module in
328
                    self.config['sed_modules']]):
329 330
                print(f"{comments[module]} Options are: "
                      f"{', '.join(modules[module])}.")
331

332 333 334
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
335 336
        """

337
        z_mod = self.config['sed_modules_params']['redshifting']['redshift']
338 339 340
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
341 342 343 344 345 346 347 348 349
                if 'redshift_decimals' in self.config['analysis_params']:
                    decimals = self.config['analysis_params']['redshift_decimals']
                    if decimals < 0:
                        z = list(np.unique(obs_table['redshift']))
                    else:
                        z = list(np.unique(np.around(obs_table['redshift'],
                                                     decimals=decimals)))
                else:
                    z = list(np.unique(obs_table['redshift']))
350
                self.config['sed_modules_params']['redshifting']['redshift'] = z
351 352 353
            elif self.config['parameters_file']:
                # The entry will be ignored anyway. Just pass a dummy list
                self.config['sed_modules_params']['redshifting']['redshift'] = []
354 355 356
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")
357 358 359

    def complete_analysed_parameters(self):
        """Complete the configuration when the variables are missing from the
360
        configuration file and must be extracted from a dummy run."""
361
        if not self.config['analysis_params']['variables']:
362
            warehouse = SedWarehouse()
363
            params = ParametersManager(self.config.dict())
364 365 366
            sed = warehouse.get_sed(params.modules, params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
367
            self.config['analysis_params']['variables'] = info