Commit 52a4866c authored by Yannick Roehlly's avatar Yannick Roehlly

warehouse to a module

Convert the warehouse a its own module and use submodules for each
"back-end".
Also, make the back-end configurable with a parameter of the psum
analysis module.
parent 9f4a4f12
......@@ -26,7 +26,7 @@ from scipy import stats
from progressbar import ProgressBar
from matplotlib import pyplot as plt
from . import common
from ..sed.warehouse import SedWarehouse
from ..warehouse import SedWarehouse
from ..sed.modules.common import get_module
from ..data import Database
......@@ -88,6 +88,11 @@ class Module(common.AnalysisModule):
"If there are less values, the probability is given for each "
"one.",
50
),
"storage_type": (
"string",
"Type of storage used to cache the generate SED.",
"memory"
)
}
......@@ -129,7 +134,9 @@ class Module(common.AnalysisModule):
sys.exit()
# Open the warehouse
sed_warehouse = SedWarehouse()
# TODO Why is parameters["storage_type"] an array?
sed_warehouse = SedWarehouse(
cache_type=parameters["storage_type"][0])
# Get the parameters
analysed_variables = parameters["analysed_variables"]
......@@ -235,9 +242,6 @@ class Module(common.AnalysisModule):
progress_bar.finish()
#Save the numpy table
np.save(OUT_DIR + "comp_table.npy", comp_table)
# Loop over the observations to find the best fitting model and
# compute the parametre statistics.
for obs_index, obs_name in enumerate(obs_table['id']):
......@@ -432,6 +436,8 @@ class Module(common.AnalysisModule):
results[variable + '_err'])
result_table.write(OUT_DIR + RESULT_FILE, verbose=False)
sed_warehouse.close()
def adjust_errors(flux, error, default_error=0.1, systematic_deviation=0.1):
"""Adjust the errors replacing the 0 values by the default error and
......
......@@ -6,10 +6,9 @@ Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
@author: Yannick Roehlly <yannick.roehlly@oamp.fr>
"""
from copy import deepcopy
from json import JSONEncoder
from . import SED
from .modules import common as sed_modules
from ..sed import SED
from ..sed.modules import common as sed_modules
class SedWarehouse(object):
......@@ -28,39 +27,11 @@ class SedWarehouse(object):
Type of cache used. For now, only in memory caching.
"""
if cache_type == "memory":
dict_cache = {}
from .store.memory import SedStore
elif cache_type == "shelf":
from .store.shelf import SedStore
def get_from_cache(key):
"""Return the value corresponding to the key in the cache.
If the key is not in the cache, returns None.
Parameters
----------
key : any immutable
Returns
-------
object
"""
# We return a copy not to modify the stored object.
return deepcopy(dict_cache.get(key))
def add_to_cache(key, value):
"""Add a new key, value pair to the cache.
Parameters
----------
key : any immutable
value : object
"""
# We store a copy not to modify the stored object.
dict_cache[key] = deepcopy(value)
self.get_from_cache = get_from_cache
self.add_to_cache = add_to_cache
self.storage = SedStore()
def get_sed(self, module_list, parameter_list):
"""Get the SED corresponding to the module and parameter lists
......@@ -92,7 +63,7 @@ class SedWarehouse(object):
encoder = JSONEncoder()
sed_key = encoder.encode((module_list, parameter_list))
sed = self.get_from_cache(sed_key)
sed = self.storage.get(sed_key)
if not sed:
mod = sed_modules.get_module(module_list.pop())
......@@ -104,6 +75,10 @@ class SedWarehouse(object):
sed = self.get_sed(module_list, parameter_list)
mod.process(sed)
self.add_to_cache(sed_key, sed)
self.storage.add(sed_key, sed)
return sed
def close(self):
""" Close the underlying storage if needed """
self.storage.close()
# -*- coding: utf-8 -*-
"""
Copyright (C) 2013 Centre de données Astrophysiques de Marseille
Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
@author: Yannick Roehlly <yannick.roehlly@oamp.fr>
"""
# -*- coding: utf-8 -*-
"""
Copyright (C) 2013 Centre de données Astrophysiques de Marseille
Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
@author: Yannick Roehlly <yannick.roehlly@oamp.fr>
"""
from copy import deepcopy
class SedStore(object):
"""In memory storage"""
def __init__(self):
self.dictionary = {}
def get(self, key):
"""Get a value from the cache dictionary
Parameters
----------
key : any immutable
Returns
-------
object
"""
# We return a copy not to modify the stored object.
return deepcopy(self.dictionary.get(key))
def add(self, key, value):
"""Add a new key, value pair to the cache.
Parameters
----------
key : any immutable
value : object
"""
# We store a copy not to modify the stored object.
self.dictionary[key] = deepcopy(value)
def close(self):
"""Do nothing"""
pass
# -*- coding: utf-8 -*-
"""
Copyright (C) 2013 Centre de données Astrophysiques de Marseille
Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
@author: Yannick Roehlly <yannick.roehlly@oamp.fr>
"""
import shelve
from copy import deepcopy
class SedStore(object):
"""Shelf (dictionary into a file) storage"""
def __init__(self, filename="pcigale.cache"):
self.shelf_storage = shelve.open(filename)
def get(self, key):
"""Get a value from the shelf
Parameters
----------
key : any immutable
Returns
-------
object
"""
# We return a copy not to modify the stored object.
return deepcopy(self.shelf_storage.get(key))
def add(self, key, value):
"""Add a new key, value pair to the shelf.
Parameters
----------
key : any immutable
value : object
"""
# We store a copy not to modify the stored object.
self.shelf_storage[key] = deepcopy(value)
def close(self):
"""Close the shelf file"""
self.shelf_storage.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment