Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
CIGALE
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
12
Issues
12
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Packages & Registries
Packages & Registries
Package Registry
Container Registry
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
cigale
CIGALE
Commits
3ea2300b
Commit
3ea2300b
authored
Apr 03, 2013
by
Yannick Roehlly
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Implement memory caching in the SED warehouse
parent
bd120196
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
113 additions
and
26 deletions
+113
-26
pcigale/sed/warehouse.py
pcigale/sed/warehouse.py
+107
-23
pcigale/stats/psum.py
pcigale/stats/psum.py
+6
-3
No files found.
pcigale/sed/warehouse.py
View file @
3ea2300b
# -*- coding: utf-8 -*-
"""
Copyright (C) 2012 Centre de données Astrophysiques de Marseille
Copyright (C) 2012
, 2013
Centre de données Astrophysiques de Marseille
Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
@author: Yannick Roehlly <yannick.roehlly@oamp.fr>
"""
from
copy
import
deepcopy
from
json
import
JSONEncoder
from
.
import
SED
from
.modules
import
common
as
sed_modules
def
create_sed
(
module_list
,
parameter_lis
t
):
"""Create
a new SED using the given modules and parameters
class
SedWarehouse
(
objec
t
):
"""Create
, cache and store SED
Parameters
----------
module_list : list
List of module names in the order they have to be used to create
the SED.
parameter_list : list of dictionaries
List of the parameter dictionaries corresponding to each module of
the module_list list.
This object is responsible for creating SED and storing them in a memory
cache or a database.
"""
Returns
-------
sed : pcigale.sed
The SED made from the given modules with the given parameters.
def
__init__
(
self
,
cache_type
=
"memory"
):
"""Instantiate a SED warehouse
"""
Parameters
----------
cache_type : string
Type of cache used. For now, only in memory caching.
"""
if
cache_type
==
"memory"
:
dict_cache
=
{}
def
is_cached
(
key
):
"""Return true is the key is in the cache.
Parameters
----------
key : any immutable
Returns
-------
boolean
"""
return
(
key
in
dict_cache
)
def
get_from_cache
(
key
):
"""Return the value corresponding to the key in the cache.
If the key is not in the cache, returns None.
Parameters
----------
key : any immutable
Returns
-------
object
"""
# We return a copy not to modify the stored object.
return
deepcopy
(
dict_cache
.
get
(
key
))
def
add_to_cache
(
key
,
value
):
"""Add a new key, value pair to the cache.
Parameters
----------
key : any immutable
value : object
"""
# We store a copy not to modify the stored object.
dict_cache
[
key
]
=
deepcopy
(
value
)
self
.
is_cached
=
is_cached
self
.
get_from_cache
=
get_from_cache
self
.
add_to_cache
=
add_to_cache
def
get_sed
(
self
,
module_list
,
parameter_list
):
"""Get the SED corresponding to the module and parameter lists
If the SED was cached, get it from the cache. If it is not, create it
and add it the the cache. The method is recursive to permit caching
partial SED.
Parameters
----------
module_list : iterable
List of module names in the order they have to be used to
create the SED.
parameter_list : iterable
List of the parameter dictionaries corresponding to each
module of the module_list list.
Returns
-------
sed : pcigale.sed
The SED made from the given modules with the given parameters.
"""
module_list
=
list
(
module_list
)
parameter_list
=
list
(
parameter_list
)
# JSon representation of the tuple (module_list, parameter_list)
# used as a key for storing the SED in the cache.
encoder
=
JSONEncoder
()
sed_key
=
encoder
.
encode
((
module_list
,
parameter_list
))
sed
=
self
.
get_from_cache
(
sed_key
)
if
not
sed
:
mod
=
sed_modules
.
get_module
(
module_list
.
pop
())
mod
.
parameters
=
parameter_list
.
pop
()
# We start from an empty SED.
sed
=
SED
()
if
(
len
(
module_list
)
==
0
):
sed
=
SED
()
else
:
sed
=
self
.
get_sed
(
module_list
,
parameter_list
)
for
(
module
,
parameters
)
in
zip
(
module_list
,
parameter_list
):
mod
=
sed_modules
.
get_module
(
module
)
mod
.
parameters
=
parameters
mod
.
process
(
sed
)
mod
.
process
(
sed
)
self
.
add_to_cache
(
sed_key
,
sed
)
return
sed
return
sed
pcigale/stats/psum.py
View file @
3ea2300b
...
...
@@ -26,7 +26,7 @@ from scipy import stats
from
progressbar
import
ProgressBar
from
matplotlib
import
pyplot
as
plt
from
.
import
common
from
..sed.warehouse
import
create_sed
from
..sed.warehouse
import
SedWarehouse
from
..sed.modules.common
import
get_module
from
..data
import
Database
...
...
@@ -128,6 +128,9 @@ class Module(common.AnalysisModule):
"it yet exists."
.
format
(
OUT_DIR
))
sys
.
exit
()
# Open the warehouse
sed_warehouse
=
SedWarehouse
()
# Get the parameters
analysed_variables
=
parameters
[
"analysed_variables"
]
save_best_sed
=
parameters
[
"save_best_sed"
]
...
...
@@ -191,7 +194,7 @@ class Module(common.AnalysisModule):
# We loop over all the possible theoretical SEDs
progress_bar
=
ProgressBar
(
maxval
=
len
(
sed_modules_params
)).
start
()
for
model_index
,
parameters
in
enumerate
(
sed_modules_params
):
sed
=
create
_sed
(
sed_modules
,
parameters
)
sed
=
sed_warehouse
.
get
_sed
(
sed_modules
,
parameters
)
# Compute the reduced Chi-square, the galaxy mass (normalisation
# factor) and probability for each observed SEDs. Add these and
...
...
@@ -254,7 +257,7 @@ class Module(common.AnalysisModule):
best_chi2
=
comp_table
[
best_index
,
obs_index
,
0
]
best_norm_factor
=
comp_table
[
best_index
,
obs_index
,
2
]
best_params
=
sed_modules_params
[
best_index
]
best_sed
=
create
_sed
(
sed_modules
,
best_params
)
best_sed
=
sed_warehouse
.
get
_sed
(
sed_modules
,
best_params
)
# Save best SED
# TODO: For now, we only save the lambda vs fnu table. Once
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment