camel.py 70.7 KB
Newer Older
1
#!/usr/bin/env python
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# -*- coding: utf-8 -*-

"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.

Created on 12/2014
@author: Benoît Epinat
"""

import time
23 24
import os
import sys
25 26
import copy
import numpy as np
27 28
#import pyfits as pf
import astropy.io.fits as fits
29 30
from cap_mpfit import mpfit  # python 2.7 & 3!
#from mpfit import mpfit  # python 2.7...
bepinat's avatar
bepinat committed
31
import ipdb
32 33 34 35
#from matplotlib import pyplot as plt
from scipy import ndimage
from scipy import constants as ct

bepinat's avatar
bepinat committed
36
import argparse
bepinat's avatar
bepinat committed
37 38 39 40 41
import logging

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('CAMEL')

42 43 44 45 46 47 48 49 50 51 52 53 54

def compute_rv(z, zcosmo=None):
    """This function computes the velocity from local and global redshifts

    Parameters
    ----------
    z: float (or numpy array of floats)
        local redshift (local velocity)
    zcosmo: float
        cosmological redshift (sytemic velocity)

    """

55
    if zcosmo is None:
56 57 58 59 60
        v = ((z + 1) ** 2 - 1) / ((z + 1) ** 2 + 1) * ct.c * 1e-3
    else:
        v = (z - zcosmo) / (1 + z) * ct.c * 1e-3
    return v

61

62 63 64 65 66 67 68 69 70 71 72 73 74 75
def compute_fwhm(dz, z=None, zcosmo=0):
    """This function computes the velocity variation

    Parameters
    ----------
    dz: float (or numpy array of floats)
        local redshift variation (local velocity dispersion)
    z: float (or numpy array of floats)
        local redshift (local velocity)
    zcosmo: float
        cosmological redshift (sytemic velocity)

    """

76 77 78
    if z is None:
        z = zcosmo
    dv = dz * (1 + zcosmo) / (1 + z) ** 2 * ct.c * 1e-3
79 80
    return dv

81

82 83 84 85 86 87 88 89 90 91 92 93 94
def readconfig(filename, config):
    """This function reads the configuration file and returns a configuration dictionnary

    Parameters
    ----------
    filename: string
        name of the configuration file
    config: dictionnary
        contains options with priority on the configuration file

    """

    conf = {}
95
    if filename is not None:
bepinat's avatar
message  
bepinat committed
96 97 98 99 100
        try:
            data = open(filename)
        except:
            logger.info('Camel is not able to open ' + filename)
            data = open(filename)
bepinat's avatar
bepinat committed
101 102 103
        count_extral = 0
        for raw in data:
            keyvalcom = raw.split('= ')
104
            key = keyvalcom[0].rstrip()   # on vire les espaces de fin de keyword
bepinat's avatar
bepinat committed
105
            value = keyvalcom[1].split('/ ')[0].rstrip('\t').rstrip()  # on vire les tabultation et les espaces de fin de keyword
106 107 108 109 110 111 112 113 114 115
            value = value.replace("'", '')
            value = value.replace('"', '')
            if value == '':
                value = None
            elif value.upper() == 'NONE':
                value = None
            elif value.upper() == 'FALSE':
                value = False
            elif value.upper() == 'TRUE':
                value = True
bepinat's avatar
bepinat committed
116 117 118 119 120 121 122 123 124 125 126 127 128 129
            #elif (value.count('.') == 1) & (value.replace('.', '').isdigit()):
                #value = float(value)
            elif (value.count('.') == 1):
                try:
                    value = float(value)
                except:
                    value = value
            #elif value.isdigit():
                #value = int(value)
            elif (value.count('.') == 0):
                try:
                    value = int(value)
                except:
                    value = value
130
            if (key == 'EXTRAL') & (value is not None):
bepinat's avatar
bepinat committed
131
                if count_extral == 0:
132 133
                    conf[key] = list()
                conf[key].append([value])
bepinat's avatar
bepinat committed
134 135 136
                count_extral += 1
            else:
                conf[key] = value
137 138

    # add input options to configuration dictionnary + check needed keywords
139
    if config is not None:
140
        for key in config:
141
            if config[key] is not None:
142
                conf[key] = config[key]
143
            elif not(key in conf.keys()):
144 145 146
                conf[key] = None
                # XXX or give an error message but in that case, must test that the keywords have correct values
            
147
    # XXX more keywords? add default values?
bepinat's avatar
message  
bepinat committed
148
    needed_keys = ['FITSFILE', 'OUTPUT', 'SKYFILE', 'HALPHA', 'NII6548', 'NII6583', 'SII6716', 'SII6731', 'OIII4959', 'OIII5007', 'HBETA', 'OII3729', 'OII3726', 'EXTRAL', 'COMMW', 'REDSHIFT', 'REDMIN', 'REDMAX', 'INITW', 'WMIN', 'WMAX', 'DFIT', 'DGCTNUM', 'MFIT', 'SCLIP', 'XYCLIP', 'NCLIP', 'SPSF', 'WSMOOTH', 'SSMOOTH', 'THRES', 'MEDIAN', 'FITSPSF', 'XMIN', 'YMIN', 'ZMIN', 'XMAX', 'YMAX', 'ZMAX']
149
    for key in needed_keys:
150
        if not(key in conf.keys()):
151
            conf[key] = None
152
    
153
    if 'OII' in conf.keys():
bepinat's avatar
bepinat committed
154
        conf['OII3729'] = conf['OII3726'] = conf.pop('OII')
155 156 157 158 159
    if conf['SPSF'] is None:
        conf['SPSF'] = 0.
    if conf['COMMW'] is None:
        conf['COMMW'] = False
        # XXX or give an error message but in that case, must test that the keywords have correct values
160 161 162
    
    return conf

163

164 165 166 167 168 169 170 171 172 173
def readcubes(conf, multi_ext=False):
    """This function reads the input cubes (cube and variance)

    Parameters
    ----------
    conf: configuration dictionnary
    multi_ext: bool
        keyword to indicate if input fits has multiple extensions
    
    """
174
    
175
    if multi_ext:
176
        hdul = fits.open(conf['FITSFILE'])
177 178 179 180 181 182
        cube = hdul[1].data
        hdr = hdul[1].header
        if conf['SKYFILE'] == conf['FITSFILE']:
            var = hdul[2].data
            varhdr = hdul[2].header
        elif (not conf['SKYFILE']):
bepinat's avatar
bepinat committed
183
            var = None
184 185
            varhdr = ''
        else:
186
            hdul = fits.open(conf['SKYFILE'])
187 188 189
            var = hdul[0].data
            varhdr = hdul[0].header
    else:
190
        hdul = fits.open(conf['FITSFILE'])
bepinat's avatar
bepinat committed
191 192
        cube = hdul[0].data
        hdr = hdul[0].header
193
        if conf['SKYFILE'] == conf['FITSFILE']:
194
            var = np.zeros(cube.shape, dtype='>f4')
195
            if conf['XMIN'] is None:
196
                conf['XMIN'] = 0
197
            if conf['YMIN'] is None:
198
                conf['YMIN'] = 0
199
            if conf['XMAX'] is None:
200
                conf['XMAX'] = cube.shape[2] - 1
201
            if conf['YMAX'] is None:
202 203
                conf['YMAX'] = cube.shape[1] - 1
            for i in np.arange(cube.shape[0]):
204
                var[i, :, :] = np.var(cube[i, conf['YMIN']:conf['YMAX'], conf['XMIN']:conf['XMAX']])
205 206 207
            varhdr = hdul[0].header
            var = writedata(var, varhdr, conf['OUTPUT'] + '_variance.fits')
        elif (not conf['SKYFILE']):
bepinat's avatar
bepinat committed
208
            var = None
209 210
            varhdr = ''
        else:
211
            hdul = fits.open(conf['SKYFILE'])
212 213 214 215
            var = hdul[0].data
            varhdr = hdul[0].header
    return cube, hdr, var, varhdr

216

217 218 219 220 221 222 223 224 225 226 227
def testcubes(cube, var):
    """This function tests the sizes of the variance data

    Parameters
    ----------
    cube: ndarray
        input data cube
    var: ndarray
        input variance data
    
    """
228
    if var is None:
bepinat's avatar
bepinat committed
229
        return
230 231 232 233 234 235 236 237
    if var.ndim == 3:
        if var.shape != cube.shape:
            sys.exit(2)
    if var.ndim == 1:
        if var.shape[0] != cube.shape[0]:
            sys.exit(2)
    return

238

239 240 241 242 243 244 245 246 247 248 249 250 251
def writedata(data, hdr, filename):
    """This function writes the output data

    Parameters
    ----------
    data: ndarray
        data to be written
    hdr: fits header
        fits header to be written
    filename: string
        name of the output
    
    """
252 253
    hdu = fits.PrimaryHDU(data, hdr)
    hdulist = fits.HDUList(hdu)
bepinat's avatar
bepinat committed
254
    hdulist.writeto(filename, overwrite=True, output_verify='fix')
255 256
    return hdulist[0].header

257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273

def muse_whitelight_image(cube, hdr, filename):
    """This function creates and writes a white light image

    Parameters
    ----------
    data: ndarray
        cube to be collapsed
    hdr: fits header
        fits header of the cube
    filename: string
        name of the output
    
    """
    
    data = np.sum(cube, axis=0)
    hdrw = copy.deepcopy(hdr)
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
    if 'CTYPE3' in hdr.keys():
        del hdrw['CTYPE3']
    if 'CRVAL3' in hdr.keys():
        del hdrw['CRVAL3']
    if 'CRPIX3' in hdr.keys():
        del hdrw['CRPIX3']
    if 'CUNIT3' in hdr.keys():
        del hdrw['CUNIT3']
    if 'CD3_3' in hdr.keys():
        del hdrw['CD3_3']
    if 'CD3_2' in hdr.keys():
        del hdrw['CD3_2']
    if 'CD3_1' in hdr.keys():
        del hdrw['CD3_1']
    if 'CD2_3' in hdr.keys():
        del hdrw['CD2_3']
    if 'CD1_3' in hdr.keys():
        del hdrw['CD1_3']
292 293 294
    wlh = writedata(data, hdrw, filename)
    return wlh

295

296 297 298 299 300 301 302 303 304 305 306 307 308
def cutcube(cube, hdr, conf):
    """This function cuts the cube according to the limits requested

    Parameters
    ----------
    cube: array
        input data cube or variance array
    hdr: fits header
        input data cube header
    conf: configuration dictionnary

    """
    
309
    if conf['XMIN'] is None:
310
        conf['XMIN'] = 0
311
    if conf['YMIN'] is None:
312
        conf['YMIN'] = 0
313
    if conf['ZMIN'] is None:
314
        conf['ZMIN'] = 0
315
    if conf['XMAX'] is None:
316
        conf['XMAX'] = cube.shape[2] - 1
317
    if conf['YMAX'] is None:
318
        conf['YMAX'] = cube.shape[1] - 1
319
    if conf['ZMAX'] is None:
320 321
        conf['ZMAX'] = cube.shape[0] - 1

322 323 324 325
    conf['ZMIN'], conf['YMIN'], conf['XMIN'] = np.max([[conf['ZMIN'], conf['YMIN'], conf['XMIN']], [0, 0, 0]], axis=0)
    conf['ZMAX'], conf['YMAX'], conf['XMAX'] = np.min([[conf['ZMAX'], conf['YMAX'], conf['XMAX']], np.array(cube.shape) - 1], axis=0)
    conf['ZMIN'], conf['YMIN'], conf['XMIN'] = np.min([[conf['ZMIN'], conf['YMIN'], conf['XMIN']], [conf['ZMAX'], conf['YMAX'], conf['XMAX']]], axis=0)
    conf['ZMAX'], conf['YMAX'], conf['XMAX'] = np.max([[conf['ZMAX'], conf['YMAX'], conf['XMAX']], [conf['ZMIN'], conf['YMIN'], conf['XMIN']]], axis=0)
326
    if cube.ndim == 3:
327
        cubecut = cube[int(conf['ZMIN']):int(conf['ZMAX']) + 1, int(conf['YMIN']):int(conf['YMAX']) + 1, int(conf['XMIN']):int(conf['XMAX']) + 1]
328 329 330 331
        hdr['CRPIX1'] -= conf['XMIN']
        hdr['CRPIX2'] -= conf['YMIN']
        hdr['CRPIX3'] -= conf['ZMIN']
    elif cube.ndim == 1:
332
        cubecut = cube[int(conf['ZMIN']):int(conf['ZMAX']) + 1]
333 334 335
        hdr['CRPIX1'] -= conf['ZMIN']
    return cubecut, hdr

336

bepinat's avatar
bepinat committed
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357
def remove_nanvar(var):
    """This function removes nan in variance cube if any and replaces them by median variance (spectrally)
    
    Parameters
    ----------
    var: array
        input variance cube
    
    Returns
    -------
    corrected variance

    """
    
    cnan = np.isnan(var)
    nanmap = cnan.prod(axis=0)
    indnan = np.where(nanmap == 1)
    indnum = np.where(nanmap == 0)
    qvar = var[:, indnum[0], indnum[1]].reshape(var.shape[0], np.size(indnum[0]))
    medianvar = np.median(qvar, axis=1)
    if np.size(indnan[0]) == 0:
bepinat's avatar
message  
bepinat committed
358
        logger.info('Variance cube contains NaN and has been corrected')
bepinat's avatar
bepinat committed
359 360 361 362
    for ind in range(np.size(indnan[0])):
        var[:, indnan[0][ind], indnan[1][ind]] = medianvar
    return var

363

364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
def clipcube(cube, hdr, xy=3, sclip=3, npass=10):
    """This function performs a sigma clipping on the cube

    Parameters
    ----------
    cube: array
        input datacube
    hdr: fits header
        input data cube header
    xy: integer
        size of the box in pixels in which the clipping is done
    sclip: float
        value of the clipping (n sigma)
    npass: integer
        number of pass to perform the clipping
bepinat's avatar
bepinat committed
379
    
380 381
    """

382
    cube1 = np.zeros(cube.shape, dtype='>f4')   # initialization of output cube
383 384
    for z in range(cube.shape[0]):  # loop on the z range
        im = cube[z, :]
385 386 387 388 389 390 391 392 393
        count = 0
        ok = False
        while count < npass | ok:
            count += 1
            im1 = ndimage.median_filter(im, xy)
            diff = im - im1
            bad = np.abs(diff) > (sclip * np.std(diff))
            im[bad] = im1[bad]
            if np.size(bad) == 0:
394 395
                ok = True
        cube1[z, :] = im
396 397 398 399 400
    hdr.append(('XYCLIP', xy, 'Size of the clipping box'))
    hdr.append(('SCLIP', sclip, 'n-sigma clipping'))
    hdr.append(('NCLIP', npass, 'Number of pass of the clipping process'))
    return cube1, hdr

401

402 403 404 405 406 407 408 409 410 411 412 413 414
def spatialsmooth(cube, hdr, fwhm):
    """This function performs a Gaussian spatial smoothing on the cube

    Parameters
    ----------
    cube: array
        input datacube
    hdr: fits header
        input data cube header
    fwhm: float
        full width half maximum of the 2D Gaussian kernel
    """
    
415
    cube1 = np.zeros(cube.shape, dtype='>f4')   # initialization of output cube
416
    sigma = fwhm / (2. * np.sqrt(2. * np.log(2.)))
417 418
    for z in range(cube.shape[0]):  # loop on the z range
        cube1[z, :] = ndimage.gaussian_filter(cube[z, :], sigma)
419 420 421
    hdr.append(('SSMOOTH', fwhm, 'FWHM in pixels of 2D Gaussian spatial smoothing'))
    return cube1, hdr

422

423 424 425 426 427 428 429 430 431 432 433 434 435
def spectralsmooth(cube, hdr, fwhm):
    """This function performs a Gaussian spectral smoothing on the cube

    Parameters
    ----------
    cube: array
        input datacube
    hdr: fits header
        input data cube header
    fwhm: float
        full width half maximum of the 1D Gaussian kernel
    """
    
436
    cube1 = np.zeros(cube.shape, dtype='>f4')   # initialization of output cube
437
    sigma = fwhm / (2. * np.sqrt(2. * np.log(2.)))
438 439
    for x in range(cube.shape[2]):  # loop on the x range
        for y in range(cube.shape[1]):  # loop on the y range
440 441 442 443 444
            cube1[:, y, x] = ndimage.gaussian_filter(cube[:, y, x], sigma)
    hdr.append(('WSMOOTH', fwhm, 'FWHM in pixels of Gaussian spectral smoothing'))
    # XXX add Hanning, Gauss, ...
    return cube1, hdr

445

446 447 448 449 450 451 452 453 454
class line:
    """This class is the basic for defining a line. A line is defined by its name, its wavelength and the reference line to which it is attached if the ratio has to be constrained.
    
    """
    def __init__(self, name, wave, ref=None, fit=False, low=0, up=None, th=None, index=None):
        self.index = index
        self.name = name
        self.wave = wave
        self.fit = fit
455 456
        if ref is None:
            ref = name
457 458 459 460 461 462 463 464 465 466
        self.ref = ref
        if ref == name:
            self.low = 1
            self.up = 1
            self.th = 1
        else:
            self.low = low
            self.up = up
            self.th = th

467

468 469 470 471 472 473 474 475 476 477 478 479 480
class lines:
    """This class enables to deal with lines.  A dictionnary stored in lines will contain informations on each lines.
    
    """
    
    def append(self, line):
        self.lines[line.name] = line
        self.lines[line.name].index = self.index
        self.index += 1
        
    def __init__(self):
        self.index = 0
        self.lines = {}
bepinat's avatar
bepinat committed
481 482
        self.append(line('HALPHA', 6562.801, ref='HBETA', low=2.75, th=2.85))
        self.append(line('HBETA', 4861.363))
bepinat's avatar
bepinat committed
483
        self.append(line('HGAMMA', 4340.47, ref='HBETA', low=0.44, up=0.5, th=0.468))
bepinat's avatar
bepinat committed
484 485
        self.append(line('HDELTA', 4101.73, ref='HBETA', low=0.23, up=0.29, th=0.259))
        self.append(line('HEPS', 3970.07, ref='HBETA', low=0.13, up=0.19, th=0.159))
bepinat's avatar
bepinat committed
486
        self.append(line('NII6583', 6583.45, ref='NII6548', low=2.7, up=3.3, th=3.))
bepinat's avatar
bepinat committed
487 488 489 490
        #self.append(line('NII6583', 6583., ref='NII6548', low=2.7, up=3.3, th=3.))
        self.append(line('NII6548', 6548.05))
        self.append(line('SII6731', 6730.82))
        self.append(line('SII6716', 6716.44, ref='SII6731', low=0.45, up=1.45, th=1.))
bepinat's avatar
bepinat committed
491
        self.append(line('OIII5007', 5006.843, ref='OIII4959', low=2.7, up=3.3, th=3.))
492
        #self.append(line('OIII5007', 5006.843, ref='OIII4959', low=2.9, up=3.1, th=3.))
bepinat's avatar
bepinat committed
493
        self.append(line('OIII4959', 4958.911))
494 495
        self.append(line('OIII4363', 4363.21, ref='OIII4959'))  # XXX low=, up=, th=
        self.append(line('OII3729', 3728.80, ref='OII3726', low=0.35, up=1.5, th=1.))
496 497 498
        #self.append(line('OII3729', 3728.80, ref='OII3726', low=0.7, up=1.5, th=1.))
        #self.append(line('OII3729', 3728.80, ref='OII3726', low=0.95, up=1.05, th=1.))
        #self.append(line('OII3729', 3728.80, ref='OII3726', low=1.3, up=1.5, th=1.4))
bepinat's avatar
bepinat committed
499
        self.append(line('OII3726', 3726.04))
500 501 502
        self.append(line('HEI4471', 4471.))
        self.append(line('HEI5876', 5876., ref='HEI4471', low=2.5, th=2.5))
        self.append(line('HEII4686', 4686.))
bepinat's avatar
bepinat committed
503
        self.append(line('OI6300', 6300.3))
504 505
        self.append(line('NEIII3868', 3868.))

506
        #self['Ha'] =
bepinat's avatar
bepinat committed
507
        #self.names = ['Ha', 'Hb', 'Hga', 'Hd', 'Heps', 'NII6583', 'NII6548', 'SII6731', 'SII6716', 'OIII5007', 'OIII4959', 'OIII4363', 'OII3729', 'OII3726', 'HeI4471', 'HeI5876', 'HeII4686', 'OI6300', 'NeIII3868']
508 509 510
        #self.waves = [6562.8, 4861., 4340., 4101., 3968., 6583., 6548., 6731., 6717., 5007., 4959., 4363., 3729., 3726., 4471., 5876., 4686., 6300., 3868.]
        #self.ref = ['Hb', 'Hb', 'Hb', 'Hb', 'Hb', 'NII6548', 'NII6548', 'SII6731', 'SII6731', 'OIII4959', 'OIII4959', 'OIII4959', 'OII3726', 'OII3726', 'HeI4471', 'HeI4471', 'HeII4686', 'OI6300', 'NeIII3868']
        
bepinat's avatar
bepinat committed
511
        #lines={'Ha':6562.8,'Hb':4861.,'Hga':4340.,'Hd':4101.,'Heps':3968.,'NII6583':6583.,'NII6548':6548.,'SII6731':6731.,'SII6716':6717.,'OIII5007':5007.,'OIII4959':4959.,'OIII4363':4363.,'OII3729':3729.,'OII3726':3726.,'HeI4471':4471.,'HeI5876':5876.,'HeII4686':4686.,'OI6300':6300.,'NeIII3868':3868.}
512
        ##Initialisation of groups of lines
bepinat's avatar
bepinat committed
513
        #sgroup={'Ha':'Hb','Hb':'Hb','Hga':'Hb','Hd':'Hb','Heps':'Hb','NII6583':'NII6548','NII6548':'NII6548','SII6731':'SII6731','SII6716':'SII6731','OIII5007':'OIII4959','OIII4959':'OIII4959','OIII4363':'OIII4959','OII3729':'OII3726','OII3726':'OII3726','HeI4471':'HeI4471','HeI5876':'HeI4471','HeII4686':'HeII4686','OI6300':'OI6300','NeIII3868':'NeIII3868'}
514

515

516 517 518 519 520 521
def waveindgen(hdr):
    '''
    This function returns the wavelength index in angstroms from the header information and the conversion factor from header data to angstroms.
    '''
    
    cunit = hdr['CUNIT3']
522 523 524 525 526
    if cunit.strip().lower() in {'microns', 'micron', 'micrometers', 'mum'}:
        lconvfac = 1e4
    if cunit.strip().lower() in {'angstroms', 'angstrom'}:
        lconvfac = 1
    if cunit.strip().lower() in {'nanometers', 'nm'}:
527 528 529
        lconvfac = 1e1
    if cunit.strip().lower() in {'centimeters', 'cm'}:
        lconvfac = 1e8
bepinat's avatar
bepinat committed
530
    if 'CD3_3' in hdr.keys():
531
        sclp = hdr['CD3_3']
bepinat's avatar
bepinat committed
532 533
    elif 'CDELT3' in hdr.keys():
        sclp = hdr['CDELT3']
534 535 536
    wave = lconvfac * (sclp * (np.arange(hdr['NAXIS3']) - hdr['CRPIX3'] + 1) + hdr['CRVAL3'])
    return wave, lconvfac, sclp

537

538 539 540 541 542 543 544 545
def elspectrum(p, wave=None, lines=None, psf=None, degc=None, zind=None, sigind=None, intind=None, ratioind=None, contind=None):
    wavel = wave.reshape(wave.size, 1)
    deg = np.arange(degc + 1)
    coefs = p[contind]
    continuum = (coefs * (wavel - wave[0]) ** deg).sum(axis=1)
    wavel1 = (lines * (p[zind] + 1))
    dlines = (p[sigind] * lines)
    coefs = (p[intind] * p[ratioind])
546
    slines = (coefs * np.exp(-0.5 * (wavel - wavel1) ** 2 / (dlines ** 2 + psf ** 2))).sum(axis=1)
547
    
548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573
    #wavel = wave.reshape(wave.size, 1)
    #deg = np.arange(degc + 1)
    #coefs = p[lines.size * 4:lines.size * 4 + degc + 1]
    #continuum = (coefs * (wavel - wave[0]) ** deg).sum(axis=1)
    #wavel1 = (lines * (p[0:lines.size] + 1))
    #dlines = (p[lines.size:lines.size * 2] * lines)
    #coefs = (p[lines.size * 2:lines.size * 3] * p[lines.size * 3:lines.size * 4])
    #slines = (coefs * np.exp( -0.5 * (wavel - wavel1) ** 2 / (dlines ** 2 + psf ** 2))).sum(axis=1)
    
    #wavel = wave
    #deg = np.arange(degc + 1).reshape(degc + 1, 1)
    #coefs = p[lines.size * 4:lines.size * 4 + degc + 1].reshape(degc + 1, 1)
    #continuum = (coefs * (wavel - wave[0]) ** deg).sum(axis=0)
    #wavel1 = (lines * (p[0:lines.size] + 1)).reshape(lines.size, 1)
    #dlines = (p[lines.size:lines.size * 2] * lines).reshape(lines.size, 1)
    #coefs = (p[lines.size * 2:lines.size * 3] * p[lines.size * 3:lines.size * 4]).reshape(lines.size, 1)
    #slines = (coefs * np.exp( -0.5 * (wavel - wavel1) ** 2 / (dlines ** 2 + psf ** 2))).sum(axis=0)

    #wavel = wave.reshape(1, wave.size)
    #deg = np.arange(degc + 1).reshape(degc + 1, 1)
    #coefs = p[lines.size * 4:lines.size * 4 + degc + 1].reshape(degc + 1, 1)
    #continuum = (coefs * (wavel - wave[0]) ** deg).sum(axis=0)
    #wavel1 = (lines * (p[0:lines.size] + 1)).reshape(lines.size, 1)
    #dlines = (p[lines.size:lines.size * 2] * lines).reshape(lines.size, 1)
    #coefs = (p[lines.size * 2:lines.size * 3] * p[lines.size * 3:lines.size * 4]).reshape(lines.size, 1)
    #slines = (coefs * np.exp( -0.5 * (wavel - wavel1) ** 2 / (dlines ** 2 + psf ** 2))).sum(axis=0)
574 575
    return (continuum + slines)

576

577
def myelspectrum(p, fjac=None, wave=None, spectrum=None, err=None, lines=None, psf=None, degc=None, zind=None, sigind=None, intind=None, ratioind=None, contind=None):
bepinat's avatar
bepinat committed
578
    #t1=time.time()
579
    model = elspectrum(p, wave=wave, lines=lines, psf=psf, degc=degc, zind=zind, sigind=sigind, intind=intind, ratioind=ratioind, contind=contind)
580
    status = 0
bepinat's avatar
bepinat committed
581 582
    #t2=time.time()
    #print(t2-t1)
583 584
    return [status, (spectrum - model) / err]

585 586 587 588 589 590

def build_parinfo_lines(els, linename, conf, free_ratio, comind, multi_comp=1):
    """
    els: emission lines (not organised)
    linename: name of lines organised
    conf: configuration parameters
591
    free_ratio:
592 593 594 595 596 597
    comind: Index of the reference line for redshift and eventually dispersion
    """
    zcosmo = conf['REDSHIFT']
    zmax = conf['REDMAX']
    zmin = conf['REDMIN']
    
598
    parbase = {'value': 0., 'fixed': 1, 'limited': [0, 0], 'limits': [0., 0.], 'tied': '', 'mpmaxstep': 0, 'mpminstep': 0, 'parname': ''}
599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
    parinfo = []
    
    #Creation of parameter constraints for all lines
    for i in range(els.index * 4):
        parinfo.append(copy.deepcopy(parbase))
        
    #We set the redshift as free parameter for the reference line
    parinfo[comind]['fixed'] = 0
    
    #When common dispersion, we set the dispersion as free parameter for the reference line
    if conf['COMMW']:
        parinfo[els.index + comind]['fixed'] = 0

    for i in range(els.index):
        name = linename[i]
        j = els.lines[els.lines[name].ref].index
        
        #Redshift
        parinfo[i]['limited'] = [1, 1]
        parinfo[i]['limits'] = [zmin, zmax]
        #Redshift is common for all lines: tied to the first free line
        if (i != comind) & (els.lines[name].fit is True):
621
            parinfo[i]['tied'] = 'p[%i]' % (comind + multi_comp * len(parinfo))
622 623 624
        
        #Dispersion
        parinfo[els.index + i]['limited'] = [1, 1]
625
        parinfo[els.index + i]['limits'] = [(conf['WMIN'] * 1e3 / ct.c + zcosmo) / (1 - conf['WMIN'] * 1e3 / ct.c) - zcosmo, (conf['WMAX'] * 1e3 / ct.c + zcosmo) / (1 - conf['WMAX'] * 1e3 / ct.c) - zcosmo]  # in redshift unit
626 627 628 629
        parinfo[els.index + i]['value'] = (conf['INITW'] * 1e3 / ct.c + zcosmo) / (1 - conf['INITW'] * 1e3 / ct.c) - zcosmo
        
        #When dispersion is common for all lines, widths are tied to the first free line (the first free width is free)
        if (conf['COMMW']) & (i != comind) & (els.lines[name].fit is True):
630
            parinfo[els.index + i]['tied'] = 'p[%i]' % (els.index + comind + multi_comp * len(parinfo))
631 632 633
        
        #When dispersion is independant for all lines (common by groups), widths are tied to the reference line and the ref width is free
        if (not conf['COMMW']) & (i != j):
634
            parinfo[els.index + i]['tied'] = 'p[%i]' % (els.index + j + multi_comp * len(parinfo))
635
            if els.lines[name].fit:
636 637 638
                parinfo[els.index + j]['fixed'] = 0  # the width of the reference line is free
        if (not conf['COMMW']) & (i == j) & (els.lines[name].fit is True):
            parinfo[els.index + i]['fixed'] = 0  # the width of the reference line is free
639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
            
        #XXX When dispersion is completely independant, widths are not tied and are free
        #if (not conf['COMMW']):
            #if els.lines[name].fit: parinfo[els.index + i]['fixed'] = 0 # the width of the reference line is free
        
        #Intensity
        parinfo[2 * els.index + i]['limited'][0] = 1
        parinfo[2 * els.index + i]['limits'][0] = 0.
        
        #When line ratio is free: all ratios are fixed (= 1) and lines intensities are not tied and free parameters
        if free_ratio:
            parinfo[3 * els.index + i]['value'] = 1
            if els.lines[name].fit is True:
                parinfo[2 * els.index + i]['fixed'] = 0
        
        #When line ratio is constrained: line intensities are tied and ratios are free, except for the reference line (the ref intensity is free, the ref ratio is fixed = 1)
        if (not free_ratio) & (i != j) & (els.lines[name].fit is True):
656
            parinfo[2 * els.index + i]['tied'] = 'p[%i]' % (2 * els.index + j + multi_comp * len(parinfo))  # line intensity is tied to reference
657 658
            #Ratio limits
            parinfo[3 * els.index + i]['limited'][0] = 1
659
            if els.lines[name].low is not None:
660
                parinfo[3 * els.index + i]['limits'][0] = els.lines[name].low
661
            if els.lines[name].up is not None:
662 663
                parinfo[3 * els.index + i]['limited'][1] = 1
                parinfo[3 * els.index + i]['limits'][1] = els.lines[name].up
664
            if els.lines[name].th is not None:
665 666 667 668 669 670 671 672
                parinfo[3 * els.index + i]['value'] = els.lines[name].th
                #params[3 * els.index + i] = els.lines[name].th
            else:
                parinfo[3 * els.index + i]['value'] = 1
                #params[3 * els.index + i] = 1
            
            #Free parameters
            if els.lines[name].fit is True:
673 674
                parinfo[2 * els.index + j]['fixed'] = 0  # the intensity of the reference line is free
                parinfo[3 * els.index + i]['fixed'] = 0  # line ratio is free
675 676 677 678 679 680 681 682 683 684 685 686 687
                #XXX When reference line is not fitted, we have to find a solution if several lines of a group are fitted
            
        if (not free_ratio) & (i == j) & (els.lines[name].fit is True):
            #When line is reference line, the ratio is fixed equal to 1
            parinfo[3 * els.index + i]['limited'] = [0, 0]
            parinfo[3 * els.index + i]['value'] = 1
            #params[3 * els.index + i] = 1
            parinfo[2 * els.index + i]['fixed'] = 0
                
        #XXX When line ratio is fixed: all ratios are fixed equal to theoretical values and intensities are tied (the ref intensity is free)
        #if (not free_ratio) & (i!=j):
            #parinfo[2 * els.index + i]['tied'] = 'p[%i]'%(2 * els.index + j + multi_comp * len(parinfo))
            #parinfo[3 * els.index + i]['limited'][0] = 1
688
            #if els.lines[name].low is not None:
689
                #parinfo[3 * els.index + i]['limits'][0] = els.lines[name].low
690
            #if els.lines[name].up is not None:
691 692
                #parinfo[3 * els.index + i]['limited'][1] = 1
                #parinfo[3 * els.index + i]['limits'][1] = els.lines[name].up
693
            #if els.lines[name].th is not None:
694 695 696 697 698 699 700 701 702 703 704 705 706
                #parinfo[3 * els.index + i]['value'] = params[3 * els.index + i] = els.lines[name].th
            #else:
                #params[3 * els.index + i] = 1
            #if els.lines[name].fit is True:
                #parinfo[2 * els.index + j]['fixed'] = 0 # the intensity of the reference line is free
            
        #if (not free_ratio) & (i == j) & (els.lines[name].fit is True):
            ##When line is reference line, the ratio is fixed equal to 1
            #parinfo[3 * els.index + i]['limited'] = [0, 0]
            #parinfo[3 * els.index + i]['value'] = params[3 * els.index + i] = 1
            #parinfo[2 * els.index + i]['fixed'] = 0
    return parinfo

707

708
def buildmaps(conf, cube, hdr, var=None, plot=False, debug=False, free_ratio=False, multi_comp=1, factor=10, ftol=1e-7, gtol=1e-7, xtol=1e-7, maxiter=200):
709 710 711 712 713
    """
    """

    #Initialisation of lines
    els = lines()
bepinat's avatar
bepinat committed
714
    for lline in els.lines:
715
        #check the line name is set in the configuration file
716
        if lline in conf.keys():
bepinat's avatar
bepinat committed
717
            els.lines[lline].fit = conf[lline]
718

719
    if conf['EXTRAL'] is not None:
720
        for i in np.arange(np.shape(conf['EXTRAL'])[0]):
721
            if np.shape(conf['EXTRAL'][i])[0] == 1:
722
                els.append(line('EXTRAL%i' % (i+1), float(conf['EXTRAL'][i][0]), fit=True))
723
            if np.shape(conf['EXTRAL'][i])[0] == 2:
724
                els.append(line('EXTRAL%i' % (i+1), float(conf['EXTRAL'][i][0]), fit=True, ref=conf['EXTRAL'][i][1]))
725
            if np.shape(conf['EXTRAL'][i])[0] == 3:
726
                els.append(line('EXTRAL%i' % (i+1), float(conf['EXTRAL'][i][0]), fit=True, ref=conf['EXTRAL'][i][1], th=float(conf['EXTRAL'][i][2])))
727
            if np.shape(conf['EXTRAL'][i])[0] == 4:
728
                els.append(line('EXTRAL%i' % (i+1), float(conf['EXTRAL'][i][0]), fit=True, ref=conf['EXTRAL'][i][1], th=float(conf['EXTRAL'][i][2]), low=float(conf['EXTRAL'][i][3])))
729
            if np.shape(conf['EXTRAL'][i])[0] == 5:
730 731 732
                els.append(line('EXTRAL%i' % (i+1), float(conf['EXTRAL'][i][0]), fit=True, ref=conf['EXTRAL'][i][1], th=float(conf['EXTRAL'][i][2]), low=float(conf['EXTRAL'][i][3]), up=float(conf['EXTRAL'][i][4])))
            #if np.shape(conf['EXTRAL'][i])[0] == 6:
                #els.append(line(conf['EXTRAL'][i][5], float(conf['EXTRAL'][i][0]), fit=True, ref=conf['EXTRAL'][i][1], th=float(conf['EXTRAL'][i][2]), low=float(conf['EXTRAL'][i][3]), up=float(conf['EXTRAL'][i][4])))
733 734 735 736 737 738 739 740 741
    
    #Construction of wavelength index
    wave0, lconvfac, sclp = waveindgen(hdr)  # wavelength array in Angstrom and convertion factor

    #Condition on the wavelength to be within the redshift cuts
    #XXX try zmin > zmax?
    zcosmo = conf['REDSHIFT']
    zmax = conf['REDMAX']
    zmin = conf['REDMIN']
bepinat's avatar
bepinat committed
742
    zrange = zmax - zmin
bepinat's avatar
bepinat committed
743
    
744 745 746 747 748
    argsorted = np.argsort(np.array([els.lines[i].index for i in els.lines]))
    lineindex = np.array([els.lines[i].index for i in els.lines])[argsorted]
    linefit = np.array([els.lines[i].fit for i in els.lines])[argsorted]
    linewave = np.array([els.lines[i].wave for i in els.lines])[argsorted]
    linename = np.array([els.lines[i].name for i in els.lines])[argsorted]
749
    indok = np.zeros(0, dtype=int)
750 751 752 753
    firstind = np.zeros(linefit.size, dtype=int)
    lastind = np.zeros(linefit.size, dtype=int)
    firstind0 = np.zeros(linefit.size, dtype=int)
    lastind0 = np.zeros(linefit.size, dtype=int)
754
    for i in range(linefit.size):
755 756 757 758
        if (not linefit[i]):
            continue
        linf = np.max([(zmin - zrange / 2. + 1) * linewave[i], np.min(wave0)])
        lsup = np.min([(zmax + zrange / 2. + 1) * linewave[i], np.max(wave0)])
759 760 761 762 763 764 765 766 767
        condo = np.where((wave0 >= linf) & (wave0 <= lsup))
        if np.size(condo) != 0:
            firstind0[i] = condo[0][0]
            lastind0[i] = condo[0][np.size(condo) - 1]
            indok = np.append(indok, condo)
    indok = np.unique(indok)
    wave = wave0[indok]
    
    for i in range(linefit.size):
768 769 770 771
        if (not linefit[i]):
            continue
        linf = np.max([(zmin - zrange / 2. + 1) * linewave[i], np.min(wave0)])
        lsup = np.min([(zmax + zrange / 2. + 1) * linewave[i], np.max(wave0)])
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800
        condo = np.where((wave >= linf) & (wave <= lsup))
        if np.size(condo) != 0:
            firstind[i] = condo[0][0]
            lastind[i] = condo[0][np.size(condo) - 1]
    
    #firstind = np.zeros(np.size(cond))
    #lastind = np.zeros(np.size(cond))
    #firstind0 = np.zeros(np.size(cond))
    #lastind0 = np.zeros(np.size(cond))
    #for i in range(np.size(cond)):
        #linf = np.max([(zmin - zrange / 2. + 1) * linewave[cond[0][i]] , np.min(wave0)])
        #lsup = np.min([(zmax + zrange / 2. + 1) * linewave[cond[0][i]] , np.max(wave0)])
        #condo = np.where((wave0 >= linf) & (wave0 <= lsup))
        #if np.size(condo) != 0:
            #firstind0[i] = condo[0][0]
            #lastind0[i] = condo[0][np.size(condo) - 1]
            #indok = np.append(indok, condo)
    #indok = np.unique(indok)
    #wave = wave0[indok]
    
    #for i in range(np.size(cond)):
        #linf = np.max([(zmin - zrange / 2. + 1) * linewave[cond[0][i]] , np.min(wave0)])
        #lsup = np.min([(zmax + zrange / 2. + 1) * linewave[cond[0][i]] , np.max(wave0)])
        #condo = np.where((wave >= linf) & (wave <= lsup))
        #if np.size(condo) != 0:
            #firstind[i] = condo[0][0]
            #lastind[i] = condo[0][np.size(condo) - 1]

    #Weights
801
    #if var is None: var = np.ones([indok.size, cube.shape[1], cube.shape[2]])
802 803 804 805 806
    if var is None:
        var = np.ones(cube.shape, dtype='>f4')
    if var.ndim == 1:
        var = np.tile(var.reshape(var.size, 1, 1), (1, cube.shape[1], cube.shape[2]))
    #if var.ndim == 3: weight = ((1. / var[indok, :, :]) / np.sum(1. / var[indok, :, :], axis=0)) * indok.size
807 808 809 810
    # weight inverse variance
    #if var.ndim == 3: weight = ((1. / var) / np.sum(1. / var, axis=0)) * var.shape[0]
    # weight inverse standard deviation
    #var += 0
811 812
    if var.ndim == 3:
        weight = ((1. / np.sqrt(var)) / np.sum(1. / np.sqrt(var), axis=0)) * var.shape[0]
813 814 815
    
    #Redshift step
    zstep = (conf['DFIT'] * 1e3 / ct.c + zcosmo) / (1 - conf['DFIT'] * 1e3 / ct.c) - zcosmo
bepinat's avatar
bepinat committed
816

bepinat's avatar
bepinat committed
817
    #if zstep >= (zmax - zmin) / 2: zstep = (zmax - zmin) / 2.
bepinat's avatar
bepinat committed
818
    zmean = (zmax + zmin) / 2.
bepinat's avatar
bepinat committed
819 820
    zzmax = zmean + np.floor((zmax - zmean) / zstep + 1) * zstep
    zzmin = zmean + np.ceil((zmin - zmean) / zstep) * zstep
bepinat's avatar
bepinat committed
821

822 823 824 825
    #Parameters contraints and informations: each line has z, sigma, intensity and ratio constraint

    #Index of the reference line for redshift and eventually dispersion
    comind = np.min(lineindex[np.where(linefit)])
bepinat's avatar
bepinat committed
826
    
827
    #parinfo = build_parinfo_lines(els, linename, conf, free_ratio, comind)
828
    parinfo = []
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844
    zind = []
    sigind = []
    intind = []
    ratioind = []
    #Lines parameters
    linewaves = np.array(())
    components = np.zeros(0, dtype='i')
    for comp in range(multi_comp):
        zind.extend(range(len(parinfo) + 0 * len(linewave), len(parinfo) + 1 * len(linewave)))
        sigind.extend(range(len(parinfo) + 1 * len(linewave), len(parinfo) + 2 * len(linewave)))
        intind.extend(range(len(parinfo) + 2 * len(linewave), len(parinfo) + 3 * len(linewave)))
        ratioind.extend(range(len(parinfo) + 3 * len(linewave), len(parinfo) + 4 * len(linewave)))
        parinfom = build_parinfo_lines(els, linename, conf, free_ratio, comind, multi_comp=comp)
        parinfo.extend(parinfom)
        linewaves = np.append(linewaves, linewave)
        components = np.append(components, np.ones(len(parinfom), dtype='i') * (comp + 1))
bepinat's avatar
bepinat committed
845

846
    #Continuum parametres
847
    parbase = {'value': 0., 'fixed': 0, 'limited': [0, 0], 'limits': [0., 0.], 'tied': '', 'mpmaxstep': 0, 'mpminstep': 0, 'parname': ''}
848 849 850 851 852 853
    if conf['DGCTNUM'] == -1:  # case without continuum
        conf['DGCTNUM'] = 0
        parbase['fixed'] = 1
        contfixed = True
    else:
        contfixed = False
854
    contind = [i for i in range(len(parinfo), len(parinfo) + conf['DGCTNUM'] + 1)]
855
    for i in range(conf['DGCTNUM'] + 1):
856
        parinfo.append(copy.deepcopy(parbase))
857
        components = np.append(components, -1)
858 859 860 861
    params = np.zeros(len(parinfo), dtype='>f4')
    for i in range(len(parinfo)):
        params[i] = parinfo[i]['value']
    
862
    logger.info(' Buildmaps: initial parameters % s' % (str(params)))
863

864
    # Output initialisation to be filled
865 866 867 868 869 870
    paramscube = np.zeros(np.append(len(params), cube.shape[1:]), dtype='>f4')
    perrorcube = np.zeros(np.append(len(params), cube.shape[1:]), dtype='>f4')
    statusmap = np.zeros(cube.shape[1:], dtype='>f4')
    dofmap = np.zeros(cube.shape[1:], dtype='>f4')
    fnormmap = np.zeros(cube.shape[1:], dtype='>f4')
    modcube = np.zeros(cube.shape, dtype='>f4')
871 872 873
    
    #List of extra parameters
    #xxx Ajuster wave et spectrum pour ne garder que ce qui nous intéresse?
874
    fa = {'wave': wave, 'spectrum': None, 'err': None, 'lines': linewaves, 'psf': conf['SPSF'], 'degc': conf['DGCTNUM'], 'zind': zind, 'sigind': sigind, 'intind': intind, 'ratioind': ratioind, 'contind': contind}
875 876

    #Loop on pixels
bepinat's avatar
bepinat committed
877
    counter = 0
bepinat's avatar
bepinat committed
878 879
    fac_prev = -1
    number_of_pixels = np.float(cube.shape[2] * cube.shape[1])
bepinat's avatar
bepinat committed
880 881
    tii = time.time()
    tt = 0
882 883
    for x in range(cube.shape[2]):
        for y in range(cube.shape[1]):
bepinat's avatar
bepinat committed
884 885 886 887
            if np.int((counter / number_of_pixels) * 100) != fac_prev:
                fac_prev = np.int((counter / number_of_pixels) * 100)
                sys.stdout.write("Progress => {:3d}%\r".format(fac_prev))
                sys.stdout.flush()
888
            #if (counter / 10.) == np.int(counter / 10.): logger.debug('pixel %i / %i'%(counter, number_of_pixels))
bepinat's avatar
bepinat committed
889
            
bepinat's avatar
bepinat committed
890
            counter += 1
891 892 893 894 895
            fa['spectrum'] = cube[indok, y, x]  # Spectrum
            fa['err'] = np.sqrt(var[indok, y, x])  # Error
            #fa['weight'] = np.sqrt(weight[indok, y, x])  # Weight
            if (np.min(fa['spectrum']) == np.max(fa['spectrum'])) & (np.min(fa['spectrum']) == 0):
                continue  # No need for fitting
896 897 898 899 900 901 902 903 904
            
            minfnorm = np.infty
            #index = None
            
            std = np.std(fa['spectrum'])
            
            #Parameters initialisation
            p = np.copy(params)
            
905
            #Line intensity initialisation
906
            for i in intind:
907
                if parinfo[i]['fixed'] == 0:
908
                    p[i] = 5 * std * 10 ** (- components[i] + 1.)
909 910 911

            #for i in range(els.index):
                #if parinfo[2 * els.index + i]['fixed'] == 0:  p[2 * els.index + i] = 2 * std
912

913
            #Inititalisation of continuum (degree 0)
914 915
            if not contfixed:
                p[contind[0]] = np.median(fa['spectrum'])
916
            #p[els.index * 4] = np.median(fa['spectrum'])
bepinat's avatar
bepinat committed
917
            for z in np.arange(zzmin, zzmax, zstep):
918 919 920
                #p[comind] = z
                p[zind] = z
                #p[:els.index] = z
921
                pstart = np.copy(p)
bepinat's avatar
bepinat committed
922
                ti = time.time()
923 924
                fit0 = mpfit(myelspectrum, xall=pstart, functkw=fa, quiet=True, parinfo=parinfo, ftol=ftol, xtol=xtol, gtol=gtol, maxiter=maxiter, factor=factor)
                #fit0 = mpfit.mpfit(myelspectrum, xall=pstart, functkw=fa, quiet=True, parinfo=parinfo, ftol=ftol, xtol=xtol, gtol=gtol, maxiter=maxiter, factor=factor)
bepinat's avatar
bepinat committed
925 926
                tf = time.time()
                tt += tf - ti
927 928 929 930
                if fit0.status == 0:
                    continue
                if fit0.status == -16:
                    continue
931 932 933
                if fit0.fnorm <= minfnorm:
                    minfnorm = fit0.fnorm
                    fit = copy.copy(fit0)
bepinat's avatar
bepinat committed
934
                #print(z,tf - ti)
935 936
            if minfnorm == np.infty:
                continue
bepinat's avatar
message  
bepinat committed
937
            #logger.debug(fit.params[2 * els.index + 1 + 12], parinfo[2 * els.index + 1 + 12]['limits'], fit.params[2 * els.index + 1 + 13])
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953
            ####################################
            # sort of the components based on their flux. In a second pass, we could start from the output parameters and set constraints on e.g. sigma, intensity (e.g. int[comp2] < int[comp1]), etc.)
            # tie intensities to impose one lower than the other?
            if multi_comp > 1:
                ftot = np.zeros(multi_comp)
                for comp in range(multi_comp):
                    ind = (components == (comp + 1))
                    ftot[comp] = (fit.params[np.array(intind)[ind[intind]]] * fit.params[np.array(ratioind)[ind[ratioind]]] * fit.params[np.array(sigind)[ind[sigind]]]).sum()
                sortind = ftot[::-1].argsort() + 1
                paramssort = np.copy(fit.params)
                perrorsort = np.copy(fit.perror)
                for comp in range(multi_comp):
                    paramssort[components == (comp + 1)] = fit.params[components == sortind[comp]]
                    perrorsort[components == (comp + 1)] = fit.perror[components == sortind[comp]]
                fit.params = np.copy(paramssort)
                fit.perror = np.copy(perrorsort)
954 955 956 957 958
            paramscube[:, y, x] = fit.params[:]
            perrorcube[:, y, x] = fit.perror[:]
            fnormmap[y, x] = fit.fnorm
            dofmap[y, x] = fit.dof
            statusmap[y, x] = fit.status
959
            modcube[indok, y, x] = elspectrum(fit.params, wave=wave, lines=linewaves, psf=conf['SPSF'], degc=conf['DGCTNUM'], zind=zind, sigind=sigind, intind=intind, ratioind=ratioind, contind=contind)
960
            
bepinat's avatar
bepinat committed
961
    tff = time.time()
962
    logger.debug('Execution time: %3.1f' % (tff - tii))
bepinat's avatar
bepinat committed
963 964
    #print(tt)

965 966 967 968 969 970
    #Creating outputs
    #Error normalisation by reduced chi2
    perrorcube = perrorcube * np.sqrt(fnormmap / dofmap)

    #Continuum
    #if we want the coefficient, we must multiply them by factor + give lambda[0]
971 972
    contcube = np.zeros(cube.shape, dtype='>f4')
    econtcube = np.zeros(cube.shape, dtype='>f4')
973
    for i in range(conf['DGCTNUM'] + 1):
974 975
        contcube += paramscube[contind[i], :, :] * (wave0 - wave[0]).reshape(wave0.size, 1, 1) ** i
        econtcube += perrorcube[contind[i], :, :] * (wave0 - wave[0]).reshape(wave0.size, 1, 1) ** i
976 977 978 979 980 981 982
        #contmap
        #; continuum is computed at the longest wavelength, not necessarily optimal
        #cont[m,n]+=pfinal[2*n_elements(l)+1+i]*double(lambda[n_elements(lambda)-1]-lambda[0])^i/double(lambda[n_elements(lambda)-1]-lambda[0])
        #econt[m,n]+=(paramerr[2*n_elements(l)+1+i]*double(lambda[n_elements(lambda)-1]-lambda[0])^i/double(lambda[n_elements(lambda)-1]-lambda[0]))^2
    
    #Residual cube
    #residual
983
    residualcube = np.zeros(cube.shape, dtype='>f4')
984
    residualcube[indok, :, :] = cube[indok, :, :] - modcube[indok, :, :]
985
    #dispcont
986
    dispcontmap = np.std(cube[indok, :, :] - modcube[indok, :, :], axis=0)
987 988 989 990
    
    #Chi2
    chi2map = fnormmap / dofmap
    
991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
    wavemaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    ewavemaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    wavedispmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    ewavedispmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    dispmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    edispmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    intmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    eintmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    fluxmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    efluxmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    snrmaps = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    zmap = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    ezmap = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    rvmap = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
    ervmap = np.zeros(np.append(els.index * multi_comp, cube.shape[1:]), dtype='>f4')
1006
    
1007 1008
    for comp in range(multi_comp):
        for i in range(els.index):
1009 1010
            if (not linefit[i]):
                continue
1011 1012 1013 1014
            name = linename[i]
            j = els.lines[els.lines[name].ref].index
            
            #Velocity field
1015 1016 1017 1018 1019 1020
            zmap[comp * els.index + i, :, :] = paramscube[zind[comp * els.index + i], :, :]
            ezmap[comp * els.index + i, :, :] = perrorcube[zind[comp * els.index + i], :, :]
            rvmap[comp * els.index + i, :, :] = compute_rv(zmap[comp * els.index + i, :, :], zcosmo=zcosmo)
            ervmap[comp * els.index + i, :, :] = compute_fwhm(ezmap[comp * els.index + i, :, :], z=zmap[comp * els.index + i, :, :], zcosmo=zcosmo)
            wavemaps[comp * els.index + i, :, :] = (zmap[comp * els.index + i, :, :] + 1) * linewave[i]
            ewavemaps[comp * els.index + i, :, :] = ezmap[comp * els.index + i, :, :] * linewave[i]
1021 1022
            
            #Velocity dispersion
1023 1024 1025 1026 1027 1028
            dzmap = paramscube[sigind[comp * els.index + i], :, :]
            edzmap = perrorcube[sigind[comp * els.index + i], :, :]
            wavedispmaps[comp * els.index + i, :, :] = dzmap * linewave[i]
            ewavedispmaps[comp * els.index + i, :, :] = edzmap * linewave[i]
            dispmaps[comp * els.index + i, :, :] = compute_fwhm(dzmap, z=zmap[comp * els.index + i, :, :], zcosmo=zcosmo)
            edispmaps[comp * els.index + i, :, :] = compute_fwhm(edzmap, z=zmap[comp * els.index + i, :, :], zcosmo=zcosmo)
1029 1030
            
            #Flux
1031 1032 1033
            intmaps[comp * els.index + i, :, :] = paramscube[intind[comp * els.index + i], :, :] * paramscube[ratioind[comp * els.index + i], :, :]
            fluxmaps[comp * els.index + i, :, :] = np.sum(intmaps[comp * els.index + i, :, :] * np.exp(-0.5 * (wave.reshape(wave.size, 1, 1) - linewave[i] * (1 + zmap[comp * els.index + i, :, :])) ** 2 / ((dzmap * linewave[i]) ** 2 + conf['SPSF'] ** 2)) * sclp * lconvfac, axis=0)
            snrmaps[comp * els.index + i, :, :] = np.sum(weight[firstind0[i]:lastind0[i], :, :] * (modcube[firstind0[i]:lastind0[i], :, :] - contcube[firstind0[i]:lastind0[i], :, :]), axis=0) * sclp * lconvfac / (np.sqrt(np.pi * 2) * np.sqrt((linewave[i] * dzmap) ** 2 + conf['SPSF'] ** 2) * np.sqrt(np.sum(weight[firstind0[i]:lastind0[i], :, :] * (cube[firstind0[i]:lastind0[i], :, :] - modcube[firstind0[i]:lastind0[i], :, :]) ** 2, axis=0) / np.sum(weight[firstind0[i]:lastind0[i], :, :], axis=0))) * (lastind0[i] - firstind0[i]) / np.sum(weight[firstind0[i]:lastind0[i], :, :], axis=0)
1034
            if free_ratio:
1035
                eintmaps[comp * els.index + i, :, :] = perrorcube[intind[comp * els.index + i], :, :]
1036
            elif i != j:
1037
                eintmaps[comp * els.index + i, :, :] = np.sqrt((paramscube[intind[comp * els.index + i], :, :] * perrorcube[ratioind[comp * els.index + i], :, :]) ** 2 + (perrorcube[intind[comp * els.index + j], :, :] * paramscube[ratioind[comp * els.index + i], :, :]) ** 2)
1038
            else:
1039
                eintmaps[comp * els.index + i, :, :] = perrorcube[intind[comp * els.index + i], :, :]
1040
            
1041
            efluxmaps[comp * els.index + i, :, :] = np.sqrt(np.pi * 2) * np.sqrt((eintmaps[comp * els.index + i, :, :] * np.sqrt((linewave[i] * dzmap) ** 2 + conf['SPSF'] ** 2)) ** 2 + (linewave[i] * dzmap / np.sqrt((linewave[i] * dzmap) ** 2 + conf['SPSF'] ** 2) * intmaps[comp * els.index + i, :, :] * edzmap) ** 2)
1042 1043
            
        # This definition suffers inclusion of dz and z, especially when multiple components are fitted
1044
        snrmap = np.sum(weight[indok, :, :] * (modcube[indok, :, :] - contcube[indok, :, :]), axis=0) * sclp * lconvfac * np.size(indok) / (np.sum(weight[indok, :, :], axis=0)) / (np.sqrt(np.pi * 2) * np.sqrt((dzmap * np.median(wave) / (1 + zmap[comind, :, :])) ** 2 + conf['SPSF'] ** 2) * np.sqrt(np.sum(weight[indok, :, :] * (cube[indok, :, :] - modcube[indok, :, :]) ** 2, axis=0) / np.sum(weight[indok, :, :], axis=0)))
1045 1046

        ##Velocity field
1047 1048 1049 1050
        #zmap[comp, :, :] = paramscube[comp * els.index + comind, :, :]
        #ezmap[comp, :, :] = perrorcube[comp * els.index + comind, :, :]
        #rvmap[comp, :, :] = compute_rv(zmap[comp, :, :], zcosmo=zcosmo)
        #ervmap[comp, :, :] = compute_fwhm(ezmap[comp, :, :], z=zmap[comp, :, :], zcosmo=zcosmo)
1051
        
1052
        ##Lines
1053
        
1054 1055
        ## dzmap and edzmap are intermediate products
        #if conf['COMMW']:
1056 1057
            #dzmap = paramscube[(comp + 1) * els.index + comind, :, :]
            #edzmap = perrorcube[(comp + 1) * els.index + comind, :, :]
1058
        
1059 1060 1061 1062 1063 1064
        #for i in range(els.index):
            #if (not linefit[i]): continue
            #name = linename[i]
            #j = els.lines[els.lines[name].ref].index + comp * els.index
            
            ##if (not conf['COMMW']) & (i != j):
1065 1066
                ##dzmap = paramscube[comp * els.index + 1 + j, :, :]
                ##edzmap = perrorcube[comp * els.index + 1 + j, :, :]
1067
            #if (not conf['COMMW']):
1068 1069
                #dzmap = paramscube[(comp + 1) * els.index + j, :, :]
                #edzmap = perrorcube[(comp + 1) * els.index + j, :, :]
1070
            
1071 1072 1073 1074 1075 1076
            #wavemaps[i, :, :] = (zmap + 1) * linewave[i]
            #ewavemaps[i, :, :] = ezmap * linewave[i]
            #wavedispmaps[i, :, :] = dzmap * linewave[i]
            #ewavedispmaps[i, :, :] = edzmap * linewave[i]
            #dispmaps[i, :, :] = compute_fwhm(dzmap, z=zmap, zcosmo=zcosmo)
            #edispmaps[i, :, :] = compute_fwhm(edzmap, z=zmap, zcosmo=zcosmo)
1077
            
1078 1079 1080
            #intmaps[i, :, :] = paramscube[2 * els.index + i, :, :] * paramscube[3 * els.index + i, :, :]
            #fluxmaps[i, :, :] = np.sum(intmaps[i, :, :] * np.exp(-0.5 * (wave.reshape(wave.size, 1, 1) - linewave[i] * (1 + zmap)) ** 2 / ((dzmap * linewave[i]) ** 2 + conf['SPSF'] ** 2)) * sclp * lconvfac, axis=0)
            #snrmaps[i, :, :] = np.sum(weight[firstind0[i]:lastind0[i], :, :] * (modcube[firstind0[i]:lastind0[i], :, :] - contcube[firstind0[i]:lastind0[i], :, :]), axis=0) * sclp * lconvfac / (np.sqrt(np.pi * 2) * np.sqrt((linewave[i] * dzmap) ** 2 + conf['SPSF'] ** 2) * np.sqrt(np.sum(weight[firstind0[i]:lastind0[i], :, :] * (cube[firstind0[i]:lastind0[i], :, :] - modcube[firstind0[i]:lastind0[i], :, :]) ** 2, axis=0) / np.sum(weight[firstind0[i]:lastind0[i], :, :], axis=0))) * (lastind0[i] - firstind0[i]) / np.sum(weight[firstind0[i]:lastind0[i], :, :], axis=0)
1081
            #if free_ratio:
1082
                #eintmaps[i, :, :] = perrorcube[2 * els.index + i, :, :]
1083
            #elif i != j:
1084
                #eintmaps[i, :, :] = np.sqrt((paramscube[2 * els.index + i, :, :] * perrorcube[3 * els.index + i, :, :]) ** 2 + (perrorcube[2 * els.index + j, :, :] * paramscube[3 * els.index + i, :, :]) ** 2)
1085
            #else:
1086
                #eintmaps[i, :, :] = perrorcube[2 * els.index + i, :, :]
1087
            
1088
            #efluxmaps[i, :, :] = np.sqrt(np.pi * 2) * np.sqrt((eintmaps[i, :, :] * np.sqrt((linewave[i] * dzmap) ** 2 + conf['SPSF'] ** 2)) ** 2 + (linewave[i] * dzmap / np.sqrt((linewave[i] * dzmap) ** 2 + conf['SPSF'] ** 2) * intmaps[i, :, :] * edzmap) ** 2)
1089 1090
        
        ## XXX Pour les raies individuelles, ne devrait-on pas calculer le flux dans la raie uniquement?
1091
    
1092
        ## XXX ajouter un cas pour lequel on ajoute un niveau seuil a la variance (e.g. sky spectrum pour lequel on veut rajouter un bruit de detecteur, ou si on estime que le niveau bas est trop bas pour que les weights soient realistes)
1093
    
1094
        ## case SNR weighted by inverse of variance / standard deviation (normalised)
1095
        ##snrmap = np.sum(weight[indok, :, :] * (modcube[indok, :, :] - contcube[indok, :, :]), axis = 0) * sclp * lconvfac / (np.sqrt(np.pi * 2) * np.sqrt((dzmap * np.median(wave) / (1 + zmap)) ** 2 + conf['SPSF'] ** 2) * np.sqrt( np.sum(weight[indok, :, :] * (cube[indok, :, :] - modcube[indok, :, :]) ** 2, axis=0) / np.sum(weight[indok, :, :], axis=0)))
1096
    
1097
        ## case SNR weighted by inverse of variance / standard deviation (normalised) + normalised to number of elements
1098
        #snrmap = np.sum(weight[indok, :, :] * (modcube[indok, :, :] - contcube[indok, :, :]), axis = 0) * sclp * lconvfac * np.size(indok) / (np.sum(weight[indok, :, :], axis=0)) /(np.sqrt(np.pi * 2) * np.sqrt((dzmap * np.median(wave) / (1 + zmap)) ** 2 + conf['SPSF'] ** 2) * np.sqrt( np.sum(weight[indok, :, :] * (cube[indok, :, :] - modcube[indok, :, :]) ** 2, axis=0) / np.sum(weight[indok, :, :], axis=0)))
1099 1100
        
        ## case SNR using standard deviation as estimate of noise (might be incorrect after smoothing?)
1101
        ##snrmap = np.sum((modcube[indok, :, :] - contcube[indok, :, :]) / np.sqrt(var[indok, :, :]), axis = 0) * sclp * lconvfac / (np.sqrt(np.pi * 2) * np.sqrt((dzmap * np.median(wave) / (1 + zmap)) ** 2 + conf['SPSF'] ** 2)) #  * np.sqrt( np.sum((cube[indok, :, :] - modcube[indok, :, :]) ** 2 / np.sqrt(var[indok, :, :]), axis=0) / np.sum(var[indok, :, :], axis=0)))
1102 1103
        
        ## case SNR without variance. No weight, noise estimated as the dispersion of the residuals
1104
        ##snrmap = np.sum((modcube[indok, :, :] - contcube[indok, :, :]), axis = 0) * sclp * lconvfac / (np.sqrt(np.pi * 2) * np.sqrt((dzmap * np.median(wave) / (1 + zmap)) ** 2 + conf['SPSF'] ** 2) * np.sqrt( np.sum((cube[indok, :, :] - modcube[indok, :, :]) ** 2, axis=0) / np.size(indok)))
1105
        
1106 1107 1108 1109 1110 1111 1112 1113
    #Output writing
    #Cubes
    hh = writedata(residualcube, hdr, conf['OUTPUT'] + '_residualcube.fits')
    hh = writedata(modcube, hdr, conf['OUTPUT'] + '_modcube.fits')
    hh = writedata(contcube, hdr, conf['OUTPUT'] + '_contcube.fits')
    hh = writedata(econtcube, hdr, conf['OUTPUT'] + '_econtcube.fits')
    
    #Images
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
    if 'CTYPE3' in hdr.keys():
        del hdr['CTYPE3']
    if 'CRVAL3' in hdr.keys():
        del hdr['CRVAL3']
    if 'CRPIX3' in hdr.keys():
        del hdr['CRPIX3']
    if 'CUNIT3' in hdr.keys():
        del hdr['CUNIT3']
    if 'CD3_3' in hdr.keys():
        del hdr['CD3_3']
    if 'CD3_2' in hdr.keys():
        del hdr['CD3_2']
    if 'CD3_1' in hdr.keys():
        del hdr['CD3_1']
    if 'CD2_3' in hdr.keys():
        del hdr['CD2_3']
    if 'CD1_3' in hdr.keys():
        del hdr['CD1_3']
    if 'CDELT3' in hdr.keys():
        del hdr['CDELT3']
1134
    
1135
    suff = ''
1136
    if conf['COMMW']:
1137
        suff = '_common'
1138
    else:
1139
        suff = '_indep'
1140 1141 1142
  
    #paramscube
    #perrorcube
1143 1144 1145 1146 1147 1148 1149 1150
    hh = writedata(dofmap, hdr, conf['OUTPUT'] + '_dof' + suff + '.fits')
    hh = writedata(fnormmap, hdr, conf['OUTPUT'] + '_fnorm' + suff + '.fits')
    hh = writedata(statusmap, hdr, conf['OUTPUT'] + '_status' + suff + '.fits')
    hh = writedata(chi2map, hdr, conf['OUTPUT'] + '_chi2' + suff + '.fits')
    hh = writedata(dispcontmap, hdr, conf['OUTPUT'] + '_dispcont' + suff + '.fits')
    hh = writedata(snrmap, hdr, conf['OUTPUT'] + '_snr' + suff + '.fits')
    
    for comp in range(multi_comp):
1151 1152 1153 1154 1155 1156 1157 1158
        if multi_comp == 1:
            suffixe = suff
        else:
            suffixe = suff + '_comp%i' % (comp + 1)
        hh = writedata(zmap[comp * els.index + comind, :, :], hdr, conf['OUTPUT'] + '_z' + suffixe + '.fits')
        hh = writedata(ezmap[comp * els.index + comind, :, :], hdr, conf['OUTPUT'] + '_ez' + suffixe + '.fits')
        hh = writedata(rvmap[comp * els.index + comind, :, :], hdr, conf['OUTPUT'] + '_vel' + suffixe + '.fits')
        hh = writedata(ervmap[comp * els.index + comind, :, :], hdr, conf['OUTPUT'] + '_evel' + suffixe + '.fits')
1159 1160
        
        for i in range(els.index):
1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
            if (not linefit[i]):
                continue
            hh = writedata(wavemaps[comp * els.index + i, :, :], hdr, conf['OUTPUT'] + '_wave' + suffixe + '_' + linename[i] + '.fits')
            hh = writedata(ewavemaps[comp * els.index + i, :, :], hdr, conf['OUTPUT'] + '_ewave' + suffixe + '_' + linename[i] + '.fits')
            hh = writedata(wavedispmaps[comp * els.index + i, :, :], hdr, conf['OUTPUT'] + '_wavedisp' + suffixe + '_' + linename[i] + '.fits')
            hh = writedata(ewavedispmaps[comp * els.index + i, :, :], hdr, conf['OUTPUT'] + '_ewavedisp' + suffixe + '_' + linename[i] + '.fits')
            hh = writedata(intmaps[comp * els.index + i, :, :], hdr, conf['OUTPUT'] + '_int' + suffixe + '_' + linename[i] + '.fits')
            hh = writedata(eintmaps[comp * els.index + i, :, :], hdr, conf['OUTPUT'] + '_eint' + suffixe + '_' + linename[i] + '.fits')
            hh = writedata(fluxmaps[comp * els.index + i, :, :], hdr, conf['OUTPUT'] + '_flux' + suffixe + '_' + linename[i] + '.fits')
            hh = writedata(efluxmaps[comp * els.index + i, :, :], <