configuration.py 13.6 KB
Newer Older
1
# -*- coding: utf-8 -*-
2 3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5 6

import pkgutil
7
from collections import Iterable, OrderedDict
8
import multiprocessing as mp
9
from textwrap import wrap
10 11 12 13 14

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import pkg_resources
import numpy as np
15
import validate
16

17
from ..handlers.parameters_handler import ParametersHandler
18
from ..data import Database
19
from ..utils import read_table
20 21
from .. import creation_modules
from .. import analysis_modules
22
from ..warehouse import SedWarehouse
23
from . import validation
24

25

26 27 28
# Limit the redshift to this number of decimals
REDSHIFT_DECIMALS = 2

29 30 31 32 33 34 35 36

class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
37
        Parameters
38
        ----------
39
        filename: string
40 41 42
            Name of the configuration file (pcigale.conf by default).

        """
43 44 45 46 47 48
        self.spec = configobj.ConfigObj(filename+'.spec',
                                        write_empty_values=True,
                                        indent_type='  ',
                                        encoding='UTF8',
                                        list_values=False,
                                        _inspec=True)
49 50
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
51
                                          indent_type='  ',
52 53 54 55 56 57 58
                                          encoding='UTF8',
                                          configspec=self.spec)

        # We validate the configuration so that the variables are converted to
        # the expected that. We do not handle errors at the point but only when
        # we actually return the configuration file from the property() method.
        self.config.validate(validate.Validator(validation.functions))
59 60 61 62 63 64 65 66 67 68 69 70

    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
71 72 73 74 75 76
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
            "the filter names for the fluxes, and the filter names with the "
            "'_err' suffix for the uncertainties. The fluxes and the "
            "uncertainties must be in mJy. This file is optional to generate "
            "the configuration file, in particular for the savefluxes module.")
77
        self.spec['data_file'] = "string"
78

79 80 81 82 83 84 85 86 87
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
            "line behind a different model. The columns must be in the order "
            "the modules will be called. The redshift column must be the last "
            "one. Finally, if this parameters is not left empty, cigale will "
            "not interpret the configuration parameters given in pcigale.ini. "
            "They will be given only for information.")
88
        self.spec['parameters_file'] = "string()"
89

90
        self.config['creation_modules'] = []
91 92 93 94 95 96 97 98 99 100
        self.config.comments['creation_modules'] = ([""] +
            ["Order of the modules use for SED creation. Available modules:"] +
            ["SFH: sfh2exp, sfhdelayed, sfhfromfile, sfhperiodic"] +
            ["SSP: bc03, m2005"] +
            ["Nebular emission: nebular"] +
            ["Dust attenuation: dustatt_calzleit, dustatt_powerlaw"] +
            ["Dust emission: casey2012, dale2014, dl2007, dl2014"] +
            ["AGN: dale2014, fritz2006"] +
            ["Radio: radio"] +
            ["Redshift: redshifting (mandatory!)"])
101
        self.spec['creation_modules'] = "cigale_string_list()"
102 103 104 105

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
106
            "pdf_analysis, savefluxes.")
107
        self.spec['analysis_method'] = "string()"
108

109 110 111 112
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
            "Number of CPU cores available. This computer has {} cores."
            .format(mp.cpu_count()))
113
        self.spec['cores'] = "integer(min=1)"
114

115
        self.config.write()
116
        self.spec.write()
117 118 119 120 121 122 123 124 125 126 127

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """

        # Getting the list of the filters available in pcigale database
128
        with Database() as base:
129
            filter_list = base.get_filter_names()
130

131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

            # Check that the id and redshift columns are present in the input
            # file
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
            column_list = []
            for column in obs_table.columns:
                filter_name = column[:-4] if column.endswith('_err') else column
                if filter_name in filter_list:
                    column_list.append(column)

            # Check that we don't have an error column without the associated
            # flux
            for column in column_list:
Médéric Boquien's avatar
Médéric Boquien committed
151 152
                if column.endswith('_err') and (column[:-4]
                                                not in column_list):
153 154
                    raise Exception("The observation table as a {} column "
                                    "but no {} column.".format(column,
Médéric Boquien's avatar
Médéric Boquien committed
155
                                                               column[:-4]))
156 157 158 159

            self.config['column_list'] = column_list
        else:
            self.config['column_list'] = ''
160 161 162
        self.config.comments['column_list'] = [""] + wrap(
            "List of the columns in the observation data file to use for "
            "the fitting.")
163
        self.spec['column_list'] = "cigale_string_list()"
164 165

        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
166
        # the configuration section from its parameter list.
167 168
        self.config['sed_modules_params'] = {}
        self.config.comments['sed_modules_params'] = ["", ""] + wrap(
169
            "Configuration of the SED creation modules.")
170
        self.spec['sed_modules_params'] = {}
171

172
        for module_name in self.config['creation_modules']:
173 174 175 176
            self.config['sed_modules_params'][module_name] = {}
            self.spec['sed_modules_params'][module_name] = {}
            sub_config = self.config['sed_modules_params'][module_name]
            sub_spec = self.spec['sed_modules_params'][module_name]
177

178
            for name, (typ, description, default) in \
179 180 181
                    creation_modules.get_module(
                        module_name,
                        blank=True).parameter_list.items():
182 183 184 185
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)
186 187
                sub_spec[name] = typ
            self.config['sed_modules_params'].comments[module_name] = [
188
                creation_modules.get_module(module_name, blank=True).comments]
189

190 191
        self.check_modules()

192
        # Configuration for the analysis method
193 194
        self.config['analysis_params'] = {}
        self.config.comments['analysis_params'] = ["", ""] + wrap(
195
            "Configuration of the statistical analysis method.")
196 197
        self.spec['analysis_params'] = {}

198
        module_name = self.config['analysis_method']
199
        for name, (typ, desc, default) in \
200
                analysis_modules.get_module(module_name).parameter_list.items():
201 202
            if default is None:
                default = ''
203 204 205
            self.config['analysis_params'][name] = default
            self.config['analysis_params'].comments[name] = wrap(desc)
            self.spec['analysis_params'][name] = typ
206 207

        self.config.write()
208
        self.spec.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
209

210 211
    @property
    def configuration(self):
212 213
        """Returns a dictionary for the session configuration if it is valid.
        Otherwise, print the erroneous keys.
Yannick Roehlly's avatar
Yannick Roehlly committed
214 215 216

        Returns
        -------
217 218
        configuration: dictionary
            Dictionary containing the information provided in pcigale.ini.
Yannick Roehlly's avatar
Yannick Roehlly committed
219
        """
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
        self.complete_redshifts()
        self.complete_analysed_parameters()

        vdt = validate.Validator(validation.functions)
        validity = self.config.validate(vdt, preserve_errors=True)

        if validity is not True:
            print("The following issues have been found in pcigale.ini:")
            for module, param, message in configobj.flatten_errors(self.config,
                                                                   validity):
                if len(module) > 0:
                    print("Module {}, parameter {}: {}".format('/'.join(module),
                                                               param, message))
                else:
                    print("Parameter {}: {}".format(param, message))
            print("Run the same command after having fixed pcigale.ini. If you"
                  " want to disable error checking, simply remove the "
                  "pcigale.ini.spec file.")
238

239
            return None
Yannick Roehlly's avatar
Yannick Roehlly committed
240

241
        return self.config.dict()
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhfromfile',
                                        'sfhperiodic']),
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
                                                     'dustatt_powerlaw']),
                               ('dust emission', ['casey2012', 'dale2014',
                                                  'dl2007', 'dl2014']),
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
262 263
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
264 265 266 267
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
268
                    'dust emission': "No dust emission module found.",
269 270
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
Médéric Boquien's avatar
Médéric Boquien committed
271
                    'redshift': "ERROR! No redshifting module found."}
272 273 274 275 276 277

        for module in modules:
            if all([user_module not in modules[module] for user_module in
                    self.config['creation_modules']]):
                print("{} Options are: {}.".
                      format(comments[module], ', '.join(modules[module])))
278

279 280 281
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
282 283
        """

284
        z_mod = self.config['sed_modules_params']['redshifting']['redshift']
285 286 287
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
288 289 290
                z = list(np.unique(np.around(obs_table['redshift'],
                                        decimals=REDSHIFT_DECIMALS)))
                self.config['sed_modules_params']['redshifting']['redshift'] = z
291 292 293
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311

    def complete_analysed_parameters(self):
        """Complete the configuration when the variables are missing from the
        configuration file and must be extract from a dummy run."""
        if self.config['analysis_method'] == 'savefluxes':
            name = 'variables'
        elif self.config['analysis_method'] == 'pdf_analysis':
            name = 'analysed_variables'
        else:
            raise Exception("Unknown analysis method")

        if not self.config['analysis_params'][name]:
            warehouse = SedWarehouse()
            params = ParametersHandler(self.config.dict())
            sed = warehouse.get_sed(params.modules, params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
            self.config['analysis_params'][name] = info