Commit 6d6839bf authored by Médéric Boquien's avatar Médéric Boquien

Speedup the construction of the database. One of the issues is that models...

Speedup the construction of the database. One of the issues is that models where inserted and committed one by one, which is very slow with SQLite. Now we rather insert all the models at once (for instance all the DL07 models) and commit only when this is done. Even on an SSD, the total run time of python setup.py build goes from 5m20s to 2m42s. The speedup should be even more spectacular on a rotating hard drive.
parent 1b131250
......@@ -137,6 +137,7 @@ def read_bc03_ssp(filename):
def build_filters(base):
filters = []
filters_dir = os.path.join(os.path.dirname(__file__), 'filters/')
for filter_file in glob.glob(filters_dir + '*.dat'):
with open(filter_file, 'r') as filter_file_read:
......@@ -165,8 +166,9 @@ def build_filters(base):
new_filter.effective_wavelength = np.mean(
filter_table[0][filter_table[1] > 0]
)
filters.append(new_filter)
base.add_filter(new_filter)
base.add_filters(filters)
def build_m2005(base):
......@@ -329,6 +331,7 @@ def build_bc2003(base):
def build_dale2014(base):
models = []
dale2014_dir = os.path.join(os.path.dirname(__file__), 'dale2014/')
# Getting the alpha grid for the templates
......@@ -371,8 +374,7 @@ def build_dale2014(base):
norm = np.trapz(lumin, x=wave)
lumin /= norm
base.add_dale2014(Dale2014(fraction, alpha_grid[al-1], wave, lumin))
models.append(Dale2014(fraction, alpha_grid[al-1], wave, lumin))
# Emission from dust heated by AGN - Quasar template
filename = dale2014_dir + "shi_agn.regridded.extended.dat"
print("Importing {}...".format(filename))
......@@ -383,10 +385,13 @@ def build_dale2014(base):
norm = np.trapz(lumin_quasar, x=wave)
lumin_quasar /= norm
base.add_dale2014(Dale2014(1.0, 0.0, wave, lumin_quasar))
models.append(Dale2014(1.0, 0.0, wave, lumin_quasar))
base.add_dale2014(models)
def build_dl2007(base):
models = []
dl2007_dir = os.path.join(os.path.dirname(__file__), 'dl2007/')
qpah = {
......@@ -442,7 +447,7 @@ def build_dl2007(base):
# Conversion from Jy cm² sr¯¹ H¯¹to W nm¯¹ (kg of dust)¯¹
lumin *= conv/MdMH[model]
base.add_dl2007(DL2007(qpah[model], umin, umin, wave, lumin))
models.append(DL2007(qpah[model], umin, umin, wave, lumin))
for umax in umaximum:
filename = dl2007_dir + "U{}/U{}_{}_MW3.1_{}.txt".format(umin,
umin,
......@@ -459,10 +464,12 @@ def build_dl2007(base):
# Conversion from Jy cm² sr¯¹ H¯¹to W nm¯¹ (kg of dust)¯¹
lumin *= conv/MdMH[model]
base.add_dl2007(DL2007(qpah[model], umin, umax, wave, lumin))
models.append(DL2007(qpah[model], umin, umax, wave, lumin))
base.add_dl2007(models)
def build_dl2014(base):
models = []
dl2014_dir = os.path.join(os.path.dirname(__file__), 'dl2014/')
qpah = {"000": 0.47, "010": 1.12, "020": 1.77, "030": 2.50, "040": 3.19,
......@@ -515,7 +522,7 @@ def build_dl2014(base):
# Conversion from Jy cm² sr¯¹ H¯¹to W nm¯¹ (kg of dust)¯¹
lumin *= conv/MdMH[model]
base.add_dl2014(DL2014(qpah[model], umin, umin, 1.0, wave, lumin))
models.append(DL2014(qpah[model], umin, umin, 1.0, wave, lumin))
for al in alpha:
filename = (dl2014_dir + "U{}_1e7_MW3.1_{}/spec_{}.dat"
.format(umin, model, al))
......@@ -529,11 +536,12 @@ def build_dl2014(base):
# Conversion from Jy cm² sr¯¹ H¯¹to W nm¯¹ (kg of dust)¯¹
lumin *= conv/MdMH[model]
base.add_dl2014(DL2014(qpah[model], umin, 1e7, al, wave,
lumin))
models.append(DL2014(qpah[model], umin, 1e7, al, wave, lumin))
base.add_dl2014(models)
def build_fritz2006(base):
models = []
fritz2006_dir = os.path.join(os.path.dirname(__file__), 'fritz2006/')
# Parameters of Fritz+2006
......@@ -594,12 +602,15 @@ def build_fritz2006(base):
lumin_scatt /= norm
lumin_agn /= norm
base.add_fritz2006(Fritz2006(params[4], params[3], params[2],
models.append(Fritz2006(params[4], params[3], params[2],
params[1], params[0], psy[n], wave,
lumin_therm, lumin_scatt, lumin_agn))
base.add_fritz2006(models)
def build_nebular(base):
models_lines = []
models_cont = []
lines_dir = os.path.join(os.path.dirname(__file__), 'nebular/')
# Number of Lyman continuum photon to normalize the nebular continuum
......@@ -628,14 +639,9 @@ def build_nebular(base):
ratio2 = ratio2/ratio2[w]
ratio3 = ratio3/ratio3[w]
lines = NebularLines(np.float(Z), -3., wave, ratio1)
base.add_nebular_lines(lines)
lines = NebularLines(np.float(Z), -2., wave, ratio2)
base.add_nebular_lines(lines)
lines = NebularLines(np.float(Z), -1., wave, ratio3)
base.add_nebular_lines(lines)
models_lines.append(NebularLines(np.float(Z), -3., wave, ratio1))
models_lines.append(NebularLines(np.float(Z), -2., wave, ratio2))
models_lines.append(NebularLines(np.float(Z), -1., wave, ratio3))
filename = "{}continuum_{}.dat".format(lines_dir, Z)
print("Importing {}...".format(filename))
......@@ -651,15 +657,12 @@ def build_nebular(base):
cont2 *= conv
cont3 *= conv
cont = NebularContinuum(np.float(Z), -3., wave, cont1)
base.add_nebular_continuum(cont)
cont = NebularContinuum(np.float(Z), -2., wave, cont2)
base.add_nebular_continuum(cont)
cont = NebularContinuum(np.float(Z), -1., wave, cont3)
base.add_nebular_continuum(cont)
models_cont.append(NebularContinuum(np.float(Z), -3., wave, cont1))
models_cont.append(NebularContinuum(np.float(Z), -2., wave, cont2))
models_cont.append(NebularContinuum(np.float(Z), -1., wave, cont3))
base.add_nebular_continuum(models_cont)
base.add_nebular_lines(models_lines)
def build_base():
base = Database(writable=True)
......
......@@ -404,17 +404,18 @@ class Database(object):
"""
return self._get_parameters(_BC03)
def add_dl2007(self, model):
def add_dl2007(self, models):
"""
Add a Draine and Li (2007) model to the database.
Add a list of Draine and Li (2007) models to the database.
Parameters
----------
model: pcigale.data.DL2007
models: list of pcigale.data.DL2007 objects
"""
if self.is_writable:
self.session.add(_DL2007(model))
for model in models:
self.session.add(_DL2007(model))
try:
self.session.commit()
except exc.IntegrityError:
......@@ -424,6 +425,7 @@ class Database(object):
else:
raise Exception('The database is not writable.')
def get_dl2007(self, qpah, umin, umax):
"""
Get the Draine and Li (2007) model corresponding to the given set of
......@@ -471,17 +473,18 @@ class Database(object):
"""
return self._get_parameters(_DL2007)
def add_dl2014(self, model):
def add_dl2014(self, models):
"""
Add an updated Draine and Li (2007) model to the database.
Add a list of updated Draine and Li (2007) models to the database.
Parameters
----------
model: pcigale.data.DL2014
models: list of pcigale.data.DL2014 objects
"""
if self.is_writable:
self.session.add(_DL2014(model))
for model in models:
self.session.add(_DL2014(model))
try:
self.session.commit()
except exc.IntegrityError:
......@@ -542,19 +545,19 @@ class Database(object):
"""
return self._get_parameters(_DL2014)
def add_dale2014(self, iragn):
def add_dale2014(self, models):
"""
Add Dale et al (2014) templates the collection.
Parameters
----------
iragn: pcigale.data.Dale2014
models: list of pcigale.data.Dale2014 objects
"""
if self.is_writable:
template = _Dale2014(iragn)
self.session.add(template)
for model in models:
self.session.add(_Dale2014(model))
try:
self.session.commit()
except exc.IntegrityError:
......@@ -609,17 +612,18 @@ class Database(object):
"""
return self._get_parameters(_Dale2014)
def add_fritz2006(self, agn):
def add_fritz2006(self, models):
"""
Add a Fritz et al. (2006) AGN model to the database.
Parameters
----------
agn: pcigale.data.Fritz2006
models: list of pcigale.data.Fritz2006 objects
"""
if self.is_writable:
self.session.add(_Fritz2006(agn))
for model in models:
self.session.add(_Fritz2006(model))
try:
self.session.commit()
except exc.IntegrityError:
......@@ -695,12 +699,13 @@ class Database(object):
"""
return self._get_parameters(_Fritz2006)
def add_nebular_lines(self, nebular_lines):
def add_nebular_lines(self, models):
"""
Add ultraviolet and optical line templates to the database.
"""
if self.is_writable:
self.session.add(_NebularLines(nebular_lines))
for model in models:
self.session.add(_NebularLines(model))
try:
self.session.commit()
except exc.IntegrityError:
......@@ -740,12 +745,13 @@ class Database(object):
"""
return self._get_parameters(_NebularLines)
def add_nebular_continuum(self, nebular_continuum):
def add_nebular_continuum(self, models):
"""
Add nebular continuum templates to the database.
"""
if self.is_writable:
self.session.add(_NebularContinuum(nebular_continuum))
for model in models:
self.session.add(_NebularContinuum(model))
try:
self.session.commit()
except exc.IntegrityError:
......@@ -817,6 +823,25 @@ class Database(object):
else:
raise Exception('The database is not writable.')
def add_filters(self, pcigale_filters):
"""
Add a list of filters to the pcigale database.
Parameters
----------
pcigale_filters: list of pcigale.data.Filter objects
"""
if self.is_writable:
for pcigale_filter in pcigale_filters:
self.session.add(_Filter(pcigale_filter))
try:
self.session.commit()
except exc.IntegrityError:
self.session.rollback()
raise DatabaseInsertError('The filter is already in the base.')
else:
raise Exception('The database is not writable.')
def del_filter(self, name):
"""
Delete a filter from the pcigale database.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment