configuration.py 15.2 KB
Newer Older
1
# -*- coding: utf-8 -*-
2 3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5

6
from collections import OrderedDict
7
import multiprocessing as mp
8 9
import os.path
import sys
10
from textwrap import wrap
11 12 13 14

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import numpy as np
15
import validate
16

17
from ..managers.parameters import ParametersManager
18
from ..data import Database
19
from ..utils import read_table
20
from .. import sed_modules
21
from .. import analysis_modules
22
from ..warehouse import SedWarehouse
23
from . import validation
24

25 26 27 28 29 30 31 32

class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
33
        Parameters
34
        ----------
35
        filename: string
36 37 38
            Name of the configuration file (pcigale.conf by default).

        """
39 40 41 42 43 44
        self.spec = configobj.ConfigObj(filename+'.spec',
                                        write_empty_values=True,
                                        indent_type='  ',
                                        encoding='UTF8',
                                        list_values=False,
                                        _inspec=True)
45 46
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
47
                                          indent_type='  ',
48 49 50 51 52 53 54
                                          encoding='UTF8',
                                          configspec=self.spec)

        # We validate the configuration so that the variables are converted to
        # the expected that. We do not handle errors at the point but only when
        # we actually return the configuration file from the property() method.
        self.config.validate(validate.Validator(validation.functions))
55

56 57
        self.pcigaleini_exists = os.path.isfile(filename)

58 59 60 61 62 63 64 65 66 67 68
    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
69 70
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
71 72 73 74 75 76
            "'distance' (Mpc, optional, if present it will be used in lieu "
            "of the distance computed from the redshift), the filter names for"
            " the fluxes, and the filter names with the '_err' suffix for the "
            "uncertainties. The fluxes and the uncertainties must be in mJy. "
            "This file is optional to generate the configuration file, in "
            "particular for the savefluxes module.")
77
        self.spec['data_file'] = "string()"
78

79 80 81 82
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
83
            "line being a different model. The columns must be in the order "
84
            "the modules will be called. The redshift column must be the last "
85 86
            "one. Finally, if this parameter is not empty, cigale will not "
            "interpret the configuration parameters given in pcigale.ini. "
87 88 89 90
            "They will be given only for information. Note that this module "
            "should only be used in conjonction with the savefluxes module. "
            "Using it with the pdf_analysis module will yield incorrect "
            "results.")
91
        self.spec['parameters_file'] = "string()"
92

93 94
        self.config['sed_modules'] = []
        self.config.comments['sed_modules'] = ([""] +
95 96 97 98
            ["Order of the modules use for SED creation. Available modules:"] +
            ["SFH: sfh2exp, sfhdelayed, sfhfromfile, sfhperiodic"] +
            ["SSP: bc03, m2005"] +
            ["Nebular emission: nebular"] +
99 100
            ["Dust attenuation: dustatt_calzleit, dustatt_powerlaw, "
             "dustatt_2powerlaws"] +
101
            ["Dust emission: casey2012, dale2014, dl2007, dl2014, themis"] +
102 103
            ["AGN: dale2014, fritz2006"] +
            ["Radio: radio"] +
104
            ["Restframe parameters: restframe_parameters"] +
105
            ["Redshift: redshifting (mandatory!)"])
106
        self.spec['sed_modules'] = "cigale_string_list()"
107 108 109 110

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
111
            "pdf_analysis, savefluxes.")
112
        self.spec['analysis_method'] = "string()"
113

114 115 116 117
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
            "Number of CPU cores available. This computer has {} cores."
            .format(mp.cpu_count()))
118
        self.spec['cores'] = "integer(min=1)"
119

120
        self.config.write()
121
        self.spec.write()
122 123 124 125 126 127 128 129 130

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """
131 132 133
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)
134 135

        # Getting the list of the filters available in pcigale database
136
        with Database() as base:
137
            filter_list = base.get_filter_names()
138

139 140 141
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

142 143 144 145 146 147
            # Check that the the file was correctly read and that the id and
            # redshift columns are present in the input file
            if 'col1' in obs_table.columns:
                raise Exception("The input could not be read properly. Verify "
                                "its format and that it does not have two "
                                "columns with the same name.")
148 149 150 151 152 153
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
154 155 156
            bands = []
            for band in obs_table.columns:
                filter_name = band[:-4] if band.endswith('_err') else band
157
                if filter_name in filter_list:
158
                    bands.append(band)
159

160 161 162 163
            # Check that we don't have an band error without the associated
            # band
            for band in bands:
                if band.endswith('_err') and (band[:-4] not in bands):
164
                    raise Exception("The observation table as a {} column "
165 166
                                    "but no {} column.".format(band,
                                                               band[:-4]))
167

168
            self.config['bands'] = bands
169
        else:
170 171 172 173 174
            self.config['bands'] = ''
        self.config.comments['bands'] = [""] + wrap("Bands to consider. To "
            "consider uncertainties too, the name of the band must be "
            "indicated with the _err suffix. For instance: FUV, FUV_err.")
        self.spec['bands'] = "cigale_string_list()"
175

176 177 178 179
        self.config['properties'] = ''
        self.config.comments['properties'] = [""] + wrap("Properties to be considered.")
        self.spec['properties'] = "cigale_string_list()"

180
        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
181
        # the configuration section from its parameter list.
182 183
        self.config['sed_modules_params'] = {}
        self.config.comments['sed_modules_params'] = ["", ""] + wrap(
184
            "Configuration of the SED creation modules.")
185
        self.spec['sed_modules_params'] = {}
186

187
        for module_name in self.config['sed_modules']:
188 189 190 191
            self.config['sed_modules_params'][module_name] = {}
            self.spec['sed_modules_params'][module_name] = {}
            sub_config = self.config['sed_modules_params'][module_name]
            sub_spec = self.spec['sed_modules_params'][module_name]
192

193
            for name, (typ, description, default) in \
194
                    sed_modules.get_module(
195 196
                        module_name,
                        blank=True).parameter_list.items():
197 198 199 200
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)
201 202
                sub_spec[name] = typ
            self.config['sed_modules_params'].comments[module_name] = [
203
                sed_modules.get_module(module_name, blank=True).comments]
204

205 206
        self.check_modules()

207
        # Configuration for the analysis method
208 209
        self.config['analysis_params'] = {}
        self.config.comments['analysis_params'] = ["", ""] + wrap(
210
            "Configuration of the statistical analysis method.")
211 212
        self.spec['analysis_params'] = {}

213
        module_name = self.config['analysis_method']
214
        for name, (typ, desc, default) in \
215
                analysis_modules.get_module(module_name).parameter_list.items():
216 217
            if default is None:
                default = ''
218 219 220
            self.config['analysis_params'][name] = default
            self.config['analysis_params'].comments[name] = wrap(desc)
            self.spec['analysis_params'][name] = typ
221 222

        self.config.write()
223
        self.spec.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
224

225 226
    @property
    def configuration(self):
227 228
        """Returns a dictionary for the session configuration if it is valid.
        Otherwise, print the erroneous keys.
Yannick Roehlly's avatar
Yannick Roehlly committed
229 230 231

        Returns
        -------
232 233
        configuration: dictionary
            Dictionary containing the information provided in pcigale.ini.
Yannick Roehlly's avatar
Yannick Roehlly committed
234
        """
235 236 237 238
        if self.pcigaleini_exists is False:
            print("Error: pcigale.ini could not be found.")
            sys.exit(1)

239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
        self.complete_redshifts()
        self.complete_analysed_parameters()

        vdt = validate.Validator(validation.functions)
        validity = self.config.validate(vdt, preserve_errors=True)

        if validity is not True:
            print("The following issues have been found in pcigale.ini:")
            for module, param, message in configobj.flatten_errors(self.config,
                                                                   validity):
                if len(module) > 0:
                    print("Module {}, parameter {}: {}".format('/'.join(module),
                                                               param, message))
                else:
                    print("Parameter {}: {}".format(param, message))
254
            print("Run the same command after having fixed pcigale.ini.")
255

256
            return None
Yannick Roehlly's avatar
Yannick Roehlly committed
257

258
        return self.config.copy()
259 260 261 262 263 264 265 266 267 268 269 270 271

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhfromfile',
                                        'sfhperiodic']),
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
272 273
                                                     'dustatt_powerlaw',
                                                     'dustatt_2powerlaws']),
274
                               ('dust emission', ['casey2012', 'dale2014',
275 276
                                                  'dl2007', 'dl2014',
                                                  'themis']),
277 278
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
279 280
                               ('restframe_parameters',
                                ['restframe_parameters']),
281 282
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
283 284
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
285 286 287 288
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
289
                    'dust emission': "No dust emission module found.",
290 291
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
292
                    'restframe_parameters': "No restframe parameters module "
293
                                            "found",
Médéric Boquien's avatar
Médéric Boquien committed
294
                    'redshift': "ERROR! No redshifting module found."}
295 296 297

        for module in modules:
            if all([user_module not in modules[module] for user_module in
298
                    self.config['sed_modules']]):
299 300
                print("{} Options are: {}.".
                      format(comments[module], ', '.join(modules[module])))
301

302 303 304
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
305 306
        """

307
        z_mod = self.config['sed_modules_params']['redshifting']['redshift']
308 309 310
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
311 312 313 314 315 316 317 318 319
                if 'redshift_decimals' in self.config['analysis_params']:
                    decimals = self.config['analysis_params']['redshift_decimals']
                    if decimals < 0:
                        z = list(np.unique(obs_table['redshift']))
                    else:
                        z = list(np.unique(np.around(obs_table['redshift'],
                                                     decimals=decimals)))
                else:
                    z = list(np.unique(obs_table['redshift']))
320
                self.config['sed_modules_params']['redshifting']['redshift'] = z
321 322 323
            elif self.config['parameters_file']:
                # The entry will be ignored anyway. Just pass a dummy list
                self.config['sed_modules_params']['redshifting']['redshift'] = []
324 325 326
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")
327 328 329

    def complete_analysed_parameters(self):
        """Complete the configuration when the variables are missing from the
330
        configuration file and must be extracted from a dummy run."""
331
        if not self.config['analysis_params']['variables']:
332
            warehouse = SedWarehouse()
333
            params = ParametersManager(self.config.dict())
334 335 336
            sed = warehouse.get_sed(params.modules, params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
337
            self.config['analysis_params']['variables'] = info