Commit 0ae585f3 authored by Médéric Boquien's avatar Médéric Boquien

Implement the possibility of not caching a module in order to save memory. A...

Implement the possibility of not caching a module in order to save memory. A typical use case is when having many different stellar populations at different metallicities. It can take a substantial amount of memory, especially when using many parallel processes.
parent 24e81d3e
......@@ -2,6 +2,8 @@
## Unreleased
### Added
- Provide the possibility not to store a given module in cache. This can be useful on computers with a limited amount of memory. The downside is that when not caching the model generation will be slower. (Médéric Boquien)
### Changed
### Fixed
- Corrected a typo that prevented `restframe\_parameters` from being listed among the available modules. (Médéric Boquien)
......
......@@ -16,19 +16,31 @@ class SedWarehouse(object):
cache or a database.
"""
def __init__(self, cache_type="memory"):
def __init__(self, cache_type="memory", nocache=None):
"""Instantiate a SED warehouse
Parameters
----------
cache_type: string
Type of cache used. For now, only in memory caching.
nocache: list
SED module or list of the SED modules that are not to be cached,
trading CPU for memory.
"""
if cache_type == "memory":
from .store.memory import SedStore
elif cache_type == "shelf":
from .store.shelf import SedStore
if nocache is None:
self.nocache = []
elif isinstance(nocache, list) is True:
self.nocache = nocache
elif isinstance(nocache, str) is True:
self.nocache = [nocache]
else:
raise TypeError("The nocache argument must be a list or an str.")
self.storage = SedStore()
# Cache for modules
......@@ -55,15 +67,18 @@ class SedWarehouse(object):
a pcigale.sed_modules.Module instance
"""
# Marshal a tuple (name, parameters) to be used as a key for storing
# the module in the cache.
module_key = marshal.dumps((name, kwargs))
if module_key in self.module_cache:
module = self.module_cache[module_key]
else:
if name in self.nocache:
module = sed_modules.get_module(name, **kwargs)
self.module_cache[module_key] = module
else:
# Marshal a tuple (name, parameters) to be used as a key for
# storing the module in the cache.
module_key = marshal.dumps((name, kwargs))
if module_key in self.module_cache:
module = self.module_cache[module_key]
else:
module = sed_modules.get_module(name, **kwargs)
self.module_cache[module_key] = module
return module
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment