configuration.py 15.7 KB
Newer Older
1
# -*- coding: utf-8 -*-
2 3
# Copyright (C) 2012, 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
Yannick Roehlly's avatar
Yannick Roehlly committed
4
# Author: Yannick Roehlly
5 6

import pkgutil
7
from collections import Iterable, OrderedDict
8
import multiprocessing as mp
9
from textwrap import wrap
10 11 12 13 14 15

import configobj
from glob import glob  # To allow the use of glob() in "eval..."
import pkg_resources
import numpy as np

16
from ..handlers.parameters_handler import ParametersHandler
17
from ..data import Database
18
from ..utils import read_table
19 20
from .. import creation_modules
from .. import analysis_modules
21 22
from ..warehouse import SedWarehouse

23

24 25 26
# Limit the redshift to this number of decimals
REDSHIFT_DECIMALS = 2

27 28 29 30

def list_modules(package_name):
    """Lists the modules available in a package

Yannick Roehlly's avatar
Yannick Roehlly committed
31
    Parameters
32
    ----------
33
    package_name: string
34
        Name of the package (e.g. pcigale.creation_modules).
35 36 37

    Returns
    -------
38
    module_name: array of strings
39 40 41 42 43 44 45 46 47
        List of the available modules.

    """
    directory = pkg_resources.resource_filename(package_name, '')
    module_names = [name for _, name, _ in pkgutil.iter_modules([directory])]

    return module_names


Yannick Roehlly's avatar
Yannick Roehlly committed
48 49 50 51 52 53
def evaluate_description(description):
    """Evaluate a description from the config file as a list.

    The description is read from the config file by configobj that transforms
    coma separated value in a list. From this description, this function try
    to evaluate the desired list of values:
54 55 56
    - If the description is a string beginning with 'eval ', then its content
      (without 'eval ') is evaluated as Python code and its result returned.
      An array is expected.
57 58 59
    - If the description is a string beginning by 'range', the start, step and
      stop values are then expected and the range is evaluated (stop included
      if reached.
Yannick Roehlly's avatar
Yannick Roehlly committed
60 61
    - Then the function tries to evaluate the description as a Numpy array of
      float and returns the mere list if this fails.
62

Yannick Roehlly's avatar
Yannick Roehlly committed
63
    Parameters
64
    ----------
65
    description: string or list
66 67 68 69
        The description to be evaluated.

    Returns
    -------
70
     results: list
Yannick Roehlly's avatar
Yannick Roehlly committed
71
        The evaluated list of values.
72 73

    """
74 75 76 77 78
    results = description
    if type(description) == str:
        if description.startswith('eval '):
            results = eval(description[4:])
            # If the evaluation lead to a single value, we put it in a list.
79
            if not isinstance(results, Iterable):
80 81 82 83 84 85 86 87
                results = [results]
        elif description.startswith('range '):
            start, stop, step = [float(item) for item
                                 in description[5:].split()]
            results = np.arange(start, stop+step, step)
        else:
            # We need to return a list to combine the list of possible values
            # for each parameter.
88
            results = [results]
89 90 91 92 93 94 95

    # We prefer to evaluate the parameter as a numpy array of floats if
    # possible.
    try:
        results = np.array(results, float)
    except ValueError:
        pass
96

Yannick Roehlly's avatar
Yannick Roehlly committed
97
    return results
98 99 100 101 102 103 104 105 106


class Configuration(object):
    """This class manages the configuration of pcigale.
    """

    def __init__(self, filename="pcigale.ini"):
        """Initialise a pcigale configuration.

Yannick Roehlly's avatar
Yannick Roehlly committed
107
        Parameters
108
        ----------
109
        filename: string
110 111 112 113 114
            Name of the configuration file (pcigale.conf by default).

        """
        self.config = configobj.ConfigObj(filename,
                                          write_empty_values=True,
115 116
                                          indent_type='  ',
                                          encoding='UTF8')
117 118 119 120 121 122 123 124 125 126 127 128

    def create_blank_conf(self):
        """Create the initial configuration file

        Write the initial pcigale configuration file where the user can state
        which data file to use, which modules to use for the SED creation, as
        well as the method selected for statistical analysis.

        """

        self.config['data_file'] = ""
        self.config.comments['data_file'] = wrap(
129 130 131 132 133 134
            "File containing the input data. The columns are 'id' (name of the"
            " object), 'redshift' (if 0 the distance is assumed to be 10 pc), "
            "the filter names for the fluxes, and the filter names with the "
            "'_err' suffix for the uncertainties. The fluxes and the "
            "uncertainties must be in mJy. This file is optional to generate "
            "the configuration file, in particular for the savefluxes module.")
135

136 137 138 139 140 141 142 143 144 145
        self.config['parameters_file'] = ""
        self.config.comments['parameters_file'] = [""] + wrap(
            "Optional file containing the list of physical parameters. Each "
            "column must be in the form module_name.parameter_name, with each "
            "line behind a different model. The columns must be in the order "
            "the modules will be called. The redshift column must be the last "
            "one. Finally, if this parameters is not left empty, cigale will "
            "not interpret the configuration parameters given in pcigale.ini. "
            "They will be given only for information.")

146
        self.config['creation_modules'] = []
147 148 149 150 151 152 153 154 155 156
        self.config.comments['creation_modules'] = ([""] +
            ["Order of the modules use for SED creation. Available modules:"] +
            ["SFH: sfh2exp, sfhdelayed, sfhfromfile, sfhperiodic"] +
            ["SSP: bc03, m2005"] +
            ["Nebular emission: nebular"] +
            ["Dust attenuation: dustatt_calzleit, dustatt_powerlaw"] +
            ["Dust emission: casey2012, dale2014, dl2007, dl2014"] +
            ["AGN: dale2014, fritz2006"] +
            ["Radio: radio"] +
            ["Redshift: redshifting (mandatory!)"])
157 158 159 160

        self.config['analysis_method'] = ""
        self.config.comments['analysis_method'] = [""] + wrap(
            "Method used for statistical analysis. Available methods: "
161
            "pdf_analysis, savefluxes.")
162

163 164 165 166 167
        self.config['cores'] = ""
        self.config.comments['cores'] = [""] + wrap(
            "Number of CPU cores available. This computer has {} cores."
            .format(mp.cpu_count()))

168 169 170 171 172 173 174 175 176 177 178 179
        self.config.write()

    def generate_conf(self):
        """Generate the full configuration file

        Reads the user entries in the initial configuration file and add the
        configuration options of all selected modules as well as the filter
        selection based on the filters identified in the data table file.

        """

        # Getting the list of the filters available in pcigale database
180
        with Database() as base:
181
            filter_list = base.get_filter_names()
182

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
        if self.config['data_file'] != '':
            obs_table = read_table(self.config['data_file'])

            # Check that the id and redshift columns are present in the input
            # file
            if 'id' not in obs_table.columns:
                raise Exception("Column id not present in input file")
            if 'redshift' not in obs_table.columns:
                raise Exception("Column redshift not present in input file")

            # Finding the known filters in the data table
            column_list = []
            for column in obs_table.columns:
                filter_name = column[:-4] if column.endswith('_err') else column
                if filter_name in filter_list:
                    column_list.append(column)

            # Check that we don't have an error column without the associated
            # flux
            for column in column_list:
Médéric Boquien's avatar
Médéric Boquien committed
203 204
                if column.endswith('_err') and (column[:-4]
                                                not in column_list):
205 206
                    raise Exception("The observation table as a {} column "
                                    "but no {} column.".format(column,
Médéric Boquien's avatar
Médéric Boquien committed
207
                                                               column[:-4]))
208 209 210 211

            self.config['column_list'] = column_list
        else:
            self.config['column_list'] = ''
212 213 214 215 216
        self.config.comments['column_list'] = [""] + wrap(
            "List of the columns in the observation data file to use for "
            "the fitting.")

        # SED creation modules configurations. For each module, we generate
Yannick Roehlly's avatar
Yannick Roehlly committed
217
        # the configuration section from its parameter list.
218 219 220 221
        self.config['sed_creation_modules'] = {}
        self.config.comments['sed_creation_modules'] = ["", ""] + wrap(
            "Configuration of the SED creation modules.")

222
        for module_name in self.config['creation_modules']:
223 224 225
            self.config["sed_creation_modules"][module_name] = {}
            sub_config = self.config["sed_creation_modules"][module_name]

226
            for name, (typ, description, default) in \
227 228 229
                    creation_modules.get_module(
                        module_name,
                        blank=True).parameter_list.items():
230 231 232 233 234 235
                if default is None:
                    default = ''
                sub_config[name] = default
                sub_config.comments[name] = wrap(description)

            self.config['sed_creation_modules'].comments[module_name] = [
236
                creation_modules.get_module(module_name, blank=True).comments]
237

238 239
        self.check_modules()

240 241 242 243 244
        # Configuration for the analysis method
        self.config['analysis_configuration'] = {}
        self.config.comments['analysis_configuration'] = ["", ""] + wrap(
            "Configuration of the statistical analysis method.")
        module_name = self.config['analysis_method']
245
        for name, (typ, desc, default) in \
246
                analysis_modules.get_module(module_name).parameter_list.items():
247 248 249 250 251 252
            if default is None:
                default = ''
            self.config['analysis_configuration'][name] = default
            self.config['analysis_configuration'].comments[name] = wrap(desc)

        self.config.write()
Yannick Roehlly's avatar
Yannick Roehlly committed
253

254 255
    @property
    def configuration(self):
Yannick Roehlly's avatar
Yannick Roehlly committed
256
        """Returns a dictionary for the session configuration.
Yannick Roehlly's avatar
Yannick Roehlly committed
257 258 259

        Returns
        -------
260
        configuration['data_file']: string
Yannick Roehlly's avatar
Yannick Roehlly committed
261
            File containing the observations to fit.
262
        configuration['column_list']: list of strings
Yannick Roehlly's avatar
Yannick Roehlly committed
263
            List of the columns of data_file to use in the fitting.
264
        configuration['creation_modules']: list of strings
Yannick Roehlly's avatar
Yannick Roehlly committed
265
            List of the modules (in the right order) used to create the SEDs.
266
        configuration['creation_modules_params']: list of dictionaries
Yannick Roehlly's avatar
Yannick Roehlly committed
267 268
            Configuration parameters for each module. To each parameter, the
            dictionary associates a list of possible values (possibly only
Yannick Roehlly's avatar
Yannick Roehlly committed
269
            one).
270
        configuration['analysis_method']: string
Yannick Roehlly's avatar
Yannick Roehlly committed
271
            Statistical analysis module used to fit the data.
272
        configuration['analysis_method_params']: dictionary
Yannick Roehlly's avatar
Yannick Roehlly committed
273
            Parameters for the statistical analysis module. To each parameter
Yannick Roehlly's avatar
Yannick Roehlly committed
274 275 276 277
            is associated a list of possible values.
        """
        configuration = {}

278 279
        # Before building the configuration dictionary, we ensure that all the
        # fields are filled
280 281
        if not self.config['parameters_file']:
            self.complete_redshifts()
282

283 284
        for section in ['data_file', 'parameters_file', 'column_list',
                        'creation_modules', 'analysis_method']:
Yannick Roehlly's avatar
Yannick Roehlly committed
285
            configuration[section] = self.config[section]
286
        configuration['cores'] = int(self.config['cores'])
Yannick Roehlly's avatar
Yannick Roehlly committed
287

Yannick Roehlly's avatar
Yannick Roehlly committed
288
        # Parsing the SED modules parameters
289 290
        configuration['creation_modules_params'] = []
        for module in self.config['creation_modules']:
291
            module_params = {}
Yannick Roehlly's avatar
Yannick Roehlly committed
292 293
            for key, value in \
                    self.config['sed_creation_modules'][module].items():
Yannick Roehlly's avatar
Yannick Roehlly committed
294
                module_params[key] = evaluate_description(value)
295
            configuration['creation_modules_params'].append(module_params)
296

297 298 299
        if (self.config['analysis_method'] == 'savefluxes' and
            not self.config['analysis_configuration']['variables']):
            warehouse = SedWarehouse()
300
            params = ParametersHandler(configuration)
301
            sed = warehouse.get_sed(params.modules,
302 303 304 305 306 307 308
                                    params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
            self.config['analysis_configuration']['variables'] = info
        elif (self.config['analysis_method'] == 'pdf_analysis' and
              not self.config['analysis_configuration']['analysed_variables']):
            warehouse = SedWarehouse()
309
            params = ParametersHandler(configuration)
310
            sed = warehouse.get_sed(params.modules,
311 312 313 314 315 316 317 318 319
                                    params.from_index(0))
            info = list(sed.info.keys())
            info.sort()
            self.config['analysis_configuration']['analysed_variables'] = info
        else:
            raise Exception("Cannot determine which physical variables are to"
                            "be computed with the {} module.").format(
                            configuration['analysis_method'])

320
        # Analysis method parameters
321 322
        configuration['analysis_method_params'] = \
            self.config['analysis_configuration']
Yannick Roehlly's avatar
Yannick Roehlly committed
323 324

        return configuration
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344

    def check_modules(self):
        """Make a basic check to ensure that some required modules are present.
        Otherwise we emit a warning so the user knows their list of modules is
        suspicious. We do not emit an exception as they may be using an
        unofficial module that is not in our list
        """

        modules = OrderedDict((('SFH', ['sfh2exp', 'sfhdelayed', 'sfhfromfile',
                                        'sfhperiodic']),
                               ('SSP', ['bc03', 'm2005']),
                               ('nebular', ['nebular']),
                               ('dust attenuation', ['dustatt_calzleit',
                                                     'dustatt_powerlaw']),
                               ('dust emission', ['casey2012', 'dale2014',
                                                  'dl2007', 'dl2014']),
                               ('AGN', ['dale2014', 'fritz2006']),
                               ('radio', ['radio']),
                               ('redshift', ['redshifting'])))

Médéric Boquien's avatar
Médéric Boquien committed
345 346
        comments = {'SFH': "ERROR! Choosing one SFH module is mandatory.",
                    'SSP': "ERROR! Choosing one SSP module is mandatory.",
347 348 349 350
                    'nebular': "WARNING! Choosing the nebular module is "
                               "recommended. Without it the Lyman continuum "
                               "is left untouched.",
                    'dust attenuation': "No dust attenuation module found.",
351
                    'dust emission': "No dust emission module found.",
352 353
                    'AGN': "No AGN module found.",
                    'radio': "No radio module found.",
Médéric Boquien's avatar
Médéric Boquien committed
354
                    'redshift': "ERROR! No redshifting module found."}
355 356 357 358 359 360

        for module in modules:
            if all([user_module not in modules[module] for user_module in
                    self.config['creation_modules']]):
                print("{} Options are: {}.".
                      format(comments[module], ', '.join(modules[module])))
361

362 363 364
    def complete_redshifts(self):
        """Complete the configuration when the redshifts are missing from the
        configuration file and must be extracted from the input flux file.
365 366 367 368 369 370 371 372 373 374 375 376
        """

        z_mod = self.config['sed_creation_modules']['redshifting']['redshift']
        if type(z_mod) is str and not z_mod:
            if self.config['data_file']:
                obs_table = read_table(self.config['data_file'])
                z = np.unique(np.around(obs_table['redshift'],
                                        decimals=REDSHIFT_DECIMALS))
                self.config['sed_creation_modules']['redshifting']['redshift'] = z
            else:
                raise Exception("No flux file and no redshift indicated. "
                                "The spectra cannot be computed. Aborting.")