Commit ee7a6c0a authored by Médéric Boquien's avatar Médéric Boquien
Browse files

Simplify the cache architecture as we always use the in-memory one. The...

Simplify the cache architecture as we always use the in-memory one. The removes one level of indirection and makes the model generation faster by 1%.
parent d620d530
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
- Corrected a typo that prevented `restframe\_parameters` from being listed among the available modules. (Médéric Boquien) - Corrected a typo that prevented `restframe\_parameters` from being listed among the available modules. (Médéric Boquien)
### Optimised ### Optimised
- The cache architecture has been simplified, making it somewhat faster. It speeds up the model generation by ~1%. (Médéric Boquien)
## 0.11.0 (2017-02-10) ## 0.11.0 (2017-02-10)
### Added ### Added
......
...@@ -16,22 +16,16 @@ class SedWarehouse(object): ...@@ -16,22 +16,16 @@ class SedWarehouse(object):
cache or a database. cache or a database.
""" """
def __init__(self, cache_type="memory", nocache=None): def __init__(self, nocache=None):
"""Instantiate an SED warehouse """Instantiate an SED warehouse
Parameters Parameters
---------- ----------
cache_type: string
Type of cache used. For now, only in memory caching.
nocache: list nocache: list
SED module or list of the SED modules that are not to be cached, SED module or list of the SED modules that are not to be cached,
trading CPU for memory. trading CPU for memory.
"""
if cache_type == "memory":
from .store.memory import SedStore
elif cache_type == "shelf":
from .store.shelf import SedStore
"""
if nocache is None: if nocache is None:
self.nocache = [] self.nocache = []
elif isinstance(nocache, list) is True: elif isinstance(nocache, list) is True:
...@@ -41,16 +35,14 @@ class SedWarehouse(object): ...@@ -41,16 +35,14 @@ class SedWarehouse(object):
else: else:
raise TypeError("The nocache argument must be a list or an str.") raise TypeError("The nocache argument must be a list or an str.")
self.storage = SedStore() self.sed_cache = {}
# Cache for modules
self.module_cache = {} self.module_cache = {}
def __enter__(self): def __enter__(self):
return self return self
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
self.close() pass
def get_module_cached(self, name, **kwargs): def get_module_cached(self, name, **kwargs):
"""Get the SED module using the internal cache. """Get the SED module using the internal cache.
...@@ -67,7 +59,6 @@ class SedWarehouse(object): ...@@ -67,7 +59,6 @@ class SedWarehouse(object):
a pcigale.sed_modules.Module instance a pcigale.sed_modules.Module instance
""" """
if name in self.nocache: if name in self.nocache:
module = sed_modules.get_module(name, **kwargs) module = sed_modules.get_module(name, **kwargs)
else: else:
...@@ -100,10 +91,9 @@ class SedWarehouse(object): ...@@ -100,10 +91,9 @@ class SedWarehouse(object):
""" """
if n_modules_max > -1: if n_modules_max > -1:
for k in list(self.storage.dictionary.keys()): for k in list(self.sed_cache.keys()):
list_modules = marshal.loads(k)[0] if len(marshal.loads(k)[0]) > n_modules_max:
if len(list_modules) > n_modules_max: del self.sed_cache[k]
self.storage.delete(k)
def get_sed(self, module_list, parameter_list): def get_sed(self, module_list, parameter_list):
"""Get the SED corresponding to the module and parameter lists """Get the SED corresponding to the module and parameter lists
...@@ -132,11 +122,10 @@ class SedWarehouse(object): ...@@ -132,11 +122,10 @@ class SedWarehouse(object):
# Marshal a tuple (module_list, parameter_list) to be used as a key # Marshal a tuple (module_list, parameter_list) to be used as a key
# for storing the SED in the cache. # for storing the SED in the cache.
sed_key = marshal.dumps((module_list, parameter_list)) key = marshal.dumps((module_list, parameter_list))
sed = self.storage.get(sed_key)
if not sed: sed = self.sed_cache.get(key)
if sed is None:
mod = self.get_module_cached(module_list.pop(), mod = self.get_module_cached(module_list.pop(),
**parameter_list.pop()) **parameter_list.pop())
...@@ -146,18 +135,6 @@ class SedWarehouse(object): ...@@ -146,18 +135,6 @@ class SedWarehouse(object):
sed = self.get_sed(module_list, parameter_list) sed = self.get_sed(module_list, parameter_list)
mod.process(sed) mod.process(sed)
self.storage.add(sed_key, sed) self.sed_cache[key] = sed
return sed
def sed_generator(self, module_list, list_of_parameter_list):
"""Generator to yield SED corresponding to a module list and a list
of parameter lists, one at a time.
"""
for parameter_list in list_of_parameter_list:
yield self.get_sed(module_list, parameter_list)
def close(self): return sed.copy()
""" Close the underlying storage if needed """
self.storage.close()
# -*- coding: utf-8 -*-
# Copyright (C) 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
# Author: Yannick Roehlly
# -*- coding: utf-8 -*-
# Copyright (C) 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
# Author: Yannick Roehlly
class SedStore(object):
"""In memory storage"""
def __init__(self):
self.dictionary = {}
def get(self, key):
"""Get a value from the cache dictionary
Parameters
----------
key: any immutable
Returns
-------
object
"""
# We return a copy not to modify the stored object.
sed = self.dictionary.get(key)
if sed is not None:
return sed.copy()
else:
return None
def add(self, key, value):
"""Add a new key, value pair to the cache.
Parameters
----------
key: any immutable
value: object
"""
# We store a copy not to modify the stored object.
self.dictionary[key] = value.copy()
def delete(self, key):
"""Delete a key, value pair from the cache
Parameters
----------
key: key of the element to be deleted
"""
del self.dictionary[key]
def close(self):
"""Do nothing"""
pass
# -*- coding: utf-8 -*-
# Copyright (C) 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
# Author: Yannick Roehlly
import shelve
from copy import deepcopy
class SedStore(object):
"""Shelf (dictionary into a file) storage"""
def __init__(self, filename="pcigale.cache"):
self.shelf_storage = shelve.open(filename)
def get(self, key):
"""Get a value from the shelf
Parameters
----------
key: any immutable
Returns
-------
object
"""
# We return a copy not to modify the stored object.
return deepcopy(self.shelf_storage.get(key))
def add(self, key, value):
"""Add a new key, value pair to the shelf.
Parameters
----------
key: any immutable
value: object
"""
# We store a copy not to modify the stored object.
self.shelf_storage[key] = deepcopy(value)
def delete(self, key):
"""Delete a key, value pair from the cache
Parameters
----------
key: key of the element to be deleted
"""
del self.dictionary[key]
def close(self):
"""Close the shelf file"""
self.shelf_storage.close()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment