Abstract the call to the parameters handler to allow for the creation of...

Abstract the call to the parameters handler to allow for the creation of different handlers depending on how the parameters are provided, for instance either a systematic grid by giving then in pcigale.ini or through a file with each line providing a the parameters for a different model. The change is transparent to the rest of pcigale so it does not matter which handler is actually used.

pcigale/__init__.py

@@ -9,7 +9,7 @@ import sys
 
 from .session.configuration import Configuration
 from .analysis_modules import get_module
-from .analysis_modules.utils import ParametersHandler
+from .handlers.parameters_handler import ParametersHandler
 
 __version__ = "0.1-alpha"
 
@@ -38,9 +38,7 @@ def check(config):
     # given for each module.
     configuration = config.configuration
     print("With this configuration cigale will compute {} "
-          "models.".format(ParametersHandler(
-                             configuration['creation_modules'],
-                             configuration['creation_modules_params']).size))
+          "models.".format(ParametersHandler(configuration).size))
 
 
 def run(config):

pcigale/analysis_modules/pdf_analysis/__init__.py

@@ -41,7 +41,8 @@ from .workers import sed as worker_sed
 from .workers import init_sed as init_worker_sed
 from .workers import init_analysis as init_worker_analysis
 from .workers import analysis as worker_analysis
-from ..utils import ParametersHandler, backup_dir
+from ..utils import backup_dir
+from ...handlers.parameters_handler import ParametersHandler
 
 
 # Tolerance threshold under which any flux or error is considered as 0.
@@ -144,7 +145,7 @@ class PdfAnalysis(AnalysisModule):
         # a list of parameters as they are computed on-the-fly. It also has
         # nice goodies such as finding the index of the first parameter to
         # have changed between two indices or the number of models.
-        params = ParametersHandler(creation_modules, creation_modules_params)
+        params = ParametersHandler(conf)
         n_params = params.size
 
         # Retrieve an arbitrary SED to obtain the list of output parameters

pcigale/analysis_modules/savefluxes/__init__.py

@@ -24,10 +24,11 @@ import time
 import numpy as np
 
 from .. import AnalysisModule
-from ..utils import ParametersHandler, backup_dir, save_fluxes
+from ..utils import backup_dir, save_fluxes
 from ...utils import read_table
 from .workers import init_fluxes as init_worker_fluxes
 from .workers import fluxes as worker_fluxes
+from ...handlers.parameters_handler import ParametersHandler
 
 
 class SaveFluxes(AnalysisModule):
@@ -80,8 +81,6 @@ class SaveFluxes(AnalysisModule):
 
         # Rename the output directory if it exists
         backup_dir()
-        creation_modules = conf['creation_modules']
-        creation_modules_params = conf['creation_modules_params']
         out_file = conf['analysis_method_params']['output_file']
         out_format = conf['analysis_method_params']['output_format']
         save_sed = conf['analysis_method_params']['save_sed'].lower() == "true"
@@ -95,7 +94,7 @@ class SaveFluxes(AnalysisModule):
         # a list of parameters as they are computed on-the-fly. It also has
         # nice goodies such as finding the index of the first parameter to
         # have changed between two indices or the number of models.
-        params = ParametersHandler(creation_modules, creation_modules_params)
+        params = ParametersHandler(conf)
         n_params = params.size
 
         info = conf['analysis_method_params']['variables']

pcigale/analysis_modules/utils.py

@@ -23,112 +23,6 @@ log.setLevel('ERROR')
 OUT_DIR = "out/"
 
 
-class ParametersHandler(object):
-    """Class to handle the parameters to generate a parameter list on-the-fly.
-    """
-
-    def __init__(self, modules, params):
-        """Instantiate the class.
-
-        Parameters
-        ----------
-        modules: list
-            Contains the modules in the order they are called
-        params: list of dictionaries
-            Contains a dictionary of parameters for each module
-
-        """
-        self.modules = modules
-        self.parameters = [self._param_dict_combine(dictionary)
-                           for dictionary in params]
-        self.shape = tuple(len(parameter) for parameter in self.parameters)
-        self.size = int(np.product(self.shape))
-
-    def _param_dict_combine(self, dictionary):
-        """Given a dictionary associating to each key an array, returns all the
-        possible dictionaries associating a single element to each key.
-
-        Parameters
-        ----------
-        dictionary: dict
-            Dictionary associating an array to its (or some of its) keys.
-
-        Returns
-        -------
-        combination_list: list of dictionaries
-            List of dictionaries with the same keys but associating one element
-            to each.
-
-        """
-        # We make a copy of the dictionary as we are modifying it.
-        dictionary = dict(dictionary)
-
-        # First, we must ensure that all values are lists; when a value is a
-        # single element, we put it in a list.
-        # We must take a special care of strings, because they are iterable.
-
-        for key, value in dictionary.items():
-            if ((not isinstance(value, collections.Iterable)) or
-                    isinstance(value, str)):
-                dictionary[key] = [value]
-
-        # We use itertools.product to make all the possible combinations from
-        # the value lists.
-        key_list = dictionary.keys()
-        value_array_list = [dictionary[key] for key in key_list]
-        combination_list = [dict(zip(key_list, combination))
-                            for combination in
-                            itertools.product(*value_array_list)]
-
-        return combination_list
-
-    def from_index(self, index):
-        """Provides the parameters of a model given a 1D index.
-
-        Parameters
-        ----------
-        index: int
-            1D index of the model for which we want the parameters
-
-        Returns
-        -------
-        params: list
-            Parameters of the model corresponding to the index
-
-        """
-        # Our problem is isomorph to the conversion between a linear and an nD
-        # index of an nD array. Thankfully numpy's unravel_index does the
-        # conversion from a 1D index to nD indices.
-        indices = np.unravel_index(index, self.shape)
-        params = [self.parameters[module][param_idx]
-                  for module, param_idx in enumerate(indices)]
-
-        return params
-
-    def index_module_changed(self, index1, index2):
-        """Find the index of the first module affected by a change of parameters.
-
-        Parameters
-        ----------
-        index1: int
-            First index
-        index2: int
-            Second index
-
-        Returns
-        -------
-        module_idx: int
-            Index of the first module that has a different parameter
-
-        """
-        indices = np.unravel_index((index1, index2), self.shape)
-        for module_idx, (i, j) in enumerate(indices):
-            if i != j:
-                return module_idx
-
-        return len(self.shape)
-
-
 def backup_dir(directory=OUT_DIR):
     if os.path.exists(directory):
         new_name = datetime.now().strftime("%Y%m%d%H%M") + "_" + directory

pcigale/handlers/parameters_handler.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2014 University of Cambridge
+# Copyright (C) 2016 Universidad de Antofagasta
+# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
+# Author: Médéric Boquien
+
+import collections
+import itertools
+import numpy as np
+
+
+class ParametersHandler(object):
+    """Class to abstract the call to the relevant parameters handler depending
+    how the physical parameters of the models are provided (directly in the
+    pcigale.ini file ).
+
+    A ParametersHandler allows to generate a list containing the parameters for
+    a modules whose index is passed as an argument. It also allows to know what
+    modules have changed their parameters given their indices. Because the
+    order of the modules is optimised to minimise the computations, this allows
+    to keep the cache of partially computed models in SedWarehouse as small as
+    possible by weeding out partial models that will not be used anymore."""
+
+    def __new__(object, configuration):
+            return ParametersHandlerGrid(configuration)
+
+
+class ParametersHandlerGrid(object):
+    """Class to generate a parameters handler for a systematic grid using the
+    parameters given in the pcigale.ini file."""
+
+    def __init__(self, configuration):
+        """Instantiate the class.
+
+        Parameters
+        ----------
+        configuration: dictionary
+            Contains the modules in the order they are called
+
+        """
+        self.modules = configuration['creation_modules']
+        self.parameters = [self._param_dict_combine(dictionary) for dictionary
+                           in configuration['creation_modules_params']]
+        self.shape = tuple(len(parameter) for parameter in self.parameters)
+        self.size = int(np.product(self.shape))
+
+    def _param_dict_combine(self, dictionary):
+        """Given a dictionary associating to each key an array, returns all the
+        possible dictionaries associating a single element to each key.
+
+        Parameters
+        ----------
+        dictionary: dict
+            Dictionary associating an array to its (or some of its) keys.
+
+        Returns
+        -------
+        combination_list: list of dictionaries
+            List of dictionaries with the same keys but associating one element
+            to each.
+
+        """
+        # We make a copy of the dictionary as we are modifying it.
+        dictionary = dict(dictionary)
+
+        # First, we must ensure that all values are lists; when a value is a
+        # single element, we put it in a list.
+        # We must take a special care of strings, because they are iterable.
+
+        for key, value in dictionary.items():
+            if ((not isinstance(value, collections.Iterable)) or
+                    isinstance(value, str)):
+                dictionary[key] = [value]
+
+        # We use itertools.product to make all the possible combinations from
+        # the value lists.
+        key_list = dictionary.keys()
+        value_array_list = [dictionary[key] for key in key_list]
+        combination_list = [dict(zip(key_list, combination))
+                            for combination in
+                            itertools.product(*value_array_list)]
+
+        return combination_list
+
+    def from_index(self, index):
+        """Provides the parameters of a model given a 1D index.
+
+        Parameters
+        ----------
+        index: int
+            1D index of the model for which we want the parameters
+
+        Returns
+        -------
+        params: list
+            Parameters of the model corresponding to the index
+
+        """
+        # Our problem is isomorph to the conversion between a linear and an nD
+        # index of an nD array. Thankfully numpy's unravel_index does the
+        # conversion from a 1D index to nD indices.
+        indices = np.unravel_index(index, self.shape)
+        params = [self.parameters[module][param_idx]
+                  for module, param_idx in enumerate(indices)]
+
+        return params
+
+    def index_module_changed(self, index1, index2):
+        """Find the index of the first module affected by a change of parameters.
+
+        Parameters
+        ----------
+        index1: int
+            First index
+        index2: int
+            Second index
+
+        Returns
+        -------
+        module_idx: int
+            Index of the first module that has a different parameter
+
+        """
+        indices = np.unravel_index((index1, index2), self.shape)
+        for module_idx, (i, j) in enumerate(indices):
+            if i != j:
+                return module_idx
+
+        return len(self.shape)

pcigale/session/configuration.py

@@ -13,7 +13,7 @@ from glob import glob  # To allow the use of glob() in "eval..."
 import pkg_resources
 import numpy as np
 
-from ..analysis_modules.utils import ParametersHandler
+from ..handlers.parameters_handler import ParametersHandler
 from ..data import Database
 from ..utils import read_table
 from .. import creation_modules
@@ -286,8 +286,7 @@ class Configuration(object):
         if (self.config['analysis_method'] == 'savefluxes' and
             not self.config['analysis_configuration']['variables']):
             warehouse = SedWarehouse()
-            params = ParametersHandler(configuration['creation_modules'],
-                                       configuration['creation_modules_params'])
+            params = ParametersHandler(configuration)
             sed = warehouse.get_sed(configuration['creation_modules'],
                                     params.from_index(0))
             info = list(sed.info.keys())
@@ -296,8 +295,7 @@ class Configuration(object):
         elif (self.config['analysis_method'] == 'pdf_analysis' and
               not self.config['analysis_configuration']['analysed_variables']):
             warehouse = SedWarehouse()
-            params = ParametersHandler(configuration['creation_modules'],
-                                       configuration['creation_modules_params'])
+            params = ParametersHandler(configuration)
             sed = warehouse.get_sed(configuration['creation_modules'],
                                     params.from_index(0))
             info = list(sed.info.keys())