utils.py 2.89 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
# -*- coding: utf-8 -*-
# Copyright (C) 2018 Universidad de Antofagasta
# Licensed under the CeCILL-v2 licence - see Licence_CeCILL_V2-en.txt
# Author: Héctor Salas

"""
Various utility functions for pcigale manager modules
"""

import ctypes
import numpy as np

from multiprocessing.sharedctypes import RawArray
from ..warehouse import SedWarehouse

16

17 18 19 20 21 22 23 24 25
def get_info(cls):
    warehouse = SedWarehouse()
    sed = warehouse.get_sed(cls.conf['sed_modules'],
                            cls.params.from_index(0))
    info = list(sed.info.keys())
    info.sort()

    return (info, sed.mass_proportional_info)

26

27
class SharedArray(object):
28
    """Class to create a shared array that can be read/written by parallel
29 30 31 32 33 34
    processes, were data related to the models is going to be stored. For
    memory efficiency reasons, we use RawArrays that will be passed in argument
    to the pool. Each worker will fill a part of the RawArrays. It is
    important that there is no conflict and that two different workers do
    not write on the same section.

35 36 37 38
    To simplify the interface, from the point of view of the rest of the code,
    this will behave like a regular Numpy array. This is a minimal
    implementation and if new operations are done on these arrays, it may be
    necessary to define them here.
39
    """
40
    def __init__(self, size):
41 42 43 44 45 46 47 48 49 50 51
        """The RawArray is stored in raw, which is protected by a setter and
        a getter. The array property returns raw as a regular Numpy array. It
        is important to access both the RawArray and the Numpy array forms. The
        conversion from a RawArray to a Numpy array can be costly, in
        particular if it is to just set or get an element. Conversely a
        RawArray is dramatically slower when using slices. To address this
        issue we selectively work with array or raw depending on whether the
        operation is done with a slice or not.
        """
        self.raw = RawArray(ctypes.c_double, size)
        self.size = size
52 53 54 55
        # By default RawArray initialises all the elements to 0. Setting them to
        # np.nan is preferanble to in case for a reason some elements are never
        # assigned a value during a run
        self.array[:] = np.nan
56 57 58 59 60

    def __setitem__(self, idx, data):
        if isinstance(idx, slice):
            self.array[idx] = data
        else:
61
            self._raw[idx] = data
62 63 64 65

    def __getitem__(self, idx):
        if isinstance(idx, slice):
            return self.array[idx]
66
        return self._raw[idx]
67 68 69 70 71 72

    def __len__(self):
        return self.size

    def __rmul__(self, other):
        return other * self.array
73 74

    @property
75
    def array(self):
76
        return np.ctypeslib.as_array(self._raw)
77 78 79 80

    @property
    def raw(self):
        return self._raw
81

82 83 84 85
    @raw.setter
    def raw(self, raw):
        if isinstance(raw, ctypes.Array):
            self._raw = raw
86 87
        else:
            raise TypeError("Type must be RawArray.")