Commit ee7a6c0a authored by Médéric Boquien's avatar Médéric Boquien

Simplify the cache architecture as we always use the in-memory one. The...

Simplify the cache architecture as we always use the in-memory one. The removes one level of indirection and makes the model generation faster by 1%.
parent d620d530
......@@ -9,6 +9,7 @@
- Corrected a typo that prevented `restframe\_parameters` from being listed among the available modules. (Médéric Boquien)
### Optimised
- The cache architecture has been simplified, making it somewhat faster. It speeds up the model generation by ~1%. (Médéric Boquien)
## 0.11.0 (2017-02-10)
### Added
......
......@@ -16,22 +16,16 @@ class SedWarehouse(object):
cache or a database.
"""
def __init__(self, cache_type="memory", nocache=None):
def __init__(self, nocache=None):
"""Instantiate an SED warehouse
Parameters
----------
cache_type: string
Type of cache used. For now, only in memory caching.
nocache: list
SED module or list of the SED modules that are not to be cached,
trading CPU for memory.
"""
if cache_type == "memory":
from .store.memory import SedStore
elif cache_type == "shelf":
from .store.shelf import SedStore
"""
if nocache is None:
self.nocache = []
elif isinstance(nocache, list) is True:
......@@ -41,16 +35,14 @@ class SedWarehouse(object):
else:
raise TypeError("The nocache argument must be a list or an str.")
self.storage = SedStore()
# Cache for modules
self.sed_cache = {}
self.module_cache = {}
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
pass
def get_module_cached(self, name, **kwargs):
"""Get the SED module using the internal cache.
......@@ -67,7 +59,6 @@ class SedWarehouse(object):
a pcigale.sed_modules.Module instance
"""
if name in self.nocache:
module = sed_modules.get_module(name, **kwargs)
else:
......@@ -100,10 +91,9 @@ class SedWarehouse(object):
"""
if n_modules_max > -1:
for k in list(self.storage.dictionary.keys()):
list_modules = marshal.loads(k)[0]
if len(list_modules) > n_modules_max:
self.storage.delete(k)
for k in list(self.sed_cache.keys()):
if len(marshal.loads(k)[0]) > n_modules_max:
del self.sed_cache[k]
def get_sed(self, module_list, parameter_list):
"""Get the SED corresponding to the module and parameter lists
......@@ -132,11 +122,10 @@ class SedWarehouse(object):
# Marshal a tuple (module_list, parameter_list) to be used as a key
# for storing the SED in the cache.
sed_key = marshal.dumps((module_list, parameter_list))
sed = self.storage.get(sed_key)
key = marshal.dumps((module_list, parameter_list))
if not sed:
sed = self.sed_cache.get(key)
if sed is None:
mod = self.get_module_cached(module_list.pop(),
**parameter_list.pop())
......@@ -146,18 +135,6 @@ class SedWarehouse(object):
sed = self.get_sed(module_list, parameter_list)
mod.process(sed)
self.storage.add(sed_key, sed)
return sed
def sed_generator(self, module_list, list_of_parameter_list):
"""Generator to yield SED corresponding to a module list and a list
of parameter lists, one at a time.
"""
for parameter_list in list_of_parameter_list:
yield self.get_sed(module_list, parameter_list)
self.sed_cache[key] = sed
def close(self):
""" Close the underlying storage if needed """
self.storage.close()
return sed.copy()
# -*- coding: utf-8 -*-
# Copyright (C) 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
# Author: Yannick Roehlly
# -*- coding: utf-8 -*-
# Copyright (C) 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
# Author: Yannick Roehlly
class SedStore(object):
"""In memory storage"""
def __init__(self):
self.dictionary = {}
def get(self, key):
"""Get a value from the cache dictionary
Parameters
----------
key: any immutable
Returns
-------
object
"""
# We return a copy not to modify the stored object.
sed = self.dictionary.get(key)
if sed is not None:
return sed.copy()
else:
return None
def add(self, key, value):
"""Add a new key, value pair to the cache.
Parameters
----------
key: any immutable
value: object
"""
# We store a copy not to modify the stored object.
self.dictionary[key] = value.copy()
def delete(self, key):
"""Delete a key, value pair from the cache
Parameters
----------
key: key of the element to be deleted
"""
del self.dictionary[key]
def close(self):
"""Do nothing"""
pass
# -*- coding: utf-8 -*-
# Copyright (C) 2013 Centre de données Astrophysiques de Marseille
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
# Author: Yannick Roehlly
import shelve
from copy import deepcopy
class SedStore(object):
"""Shelf (dictionary into a file) storage"""
def __init__(self, filename="pcigale.cache"):
self.shelf_storage = shelve.open(filename)
def get(self, key):
"""Get a value from the shelf
Parameters
----------
key: any immutable
Returns
-------
object
"""
# We return a copy not to modify the stored object.
return deepcopy(self.shelf_storage.get(key))
def add(self, key, value):
"""Add a new key, value pair to the shelf.
Parameters
----------
key: any immutable
value: object
"""
# We store a copy not to modify the stored object.
self.shelf_storage[key] = deepcopy(value)
def delete(self, key):
"""Delete a key, value pair from the cache
Parameters
----------
key: key of the element to be deleted
"""
del self.dictionary[key]
def close(self):
"""Close the shelf file"""
self.shelf_storage.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment