<gh_stars>0
"""
<NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This code is used to train retention time predictors and store
predictions from a CV procedure for further analysis.

This project was made possible by MASSTRPLAN. MASSTRPLAN received funding
from the Marie Sklodowska-Curie EU Framework for Research and Innovation
Horizon 2020, under Grant Agreement No. 675132.
"""

from pygam import LinearGAM, s
import pandas as pd
from matplotlib import pyplot as plt
import numpy as np
from scipy.stats import pearsonr, spearmanr

def call_ghostbusters(infile_known="temp/tempKnownsl2.csv",infile_unknown="temp/tempUnknownsl2.csv",fold_list="temp/tempFolds.txt"): #df_known,df_unknown,
    """
    Get the dataframe associated with this analysis

    Parameters
    ----------
    infile_known : str
        location of a file with known retention time, for Layer 2
    infile_unknown : str
        location of a file with umknown retention time, for Layer 2
    fold_list : str
        the folds to be used in Layer 2

    Returns
    -------
    pd.DataFrame
        test predictions
	pd.DataFrame
		train predictions
    """
    cmd = "Rscript makeGAM.R %s %s %s" % (infile_known,infile_unknown,fold_list)
    print("Executing: ",cmd)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
    out, err = p.communicate()

    preds = pd.read_csv("GAMpredTemp.csv")
    train_preds = pd.read_csv("GAMtrainTemp.csv")

    return(preds,train_preds)

def apply_l2(known_all,unknown_all,name="cake",ignore_cols=["IDENTIFIER","time"],cv_list=None,min_cor=0.75):
    """
    Get the dataframe associated with this analysis

    Parameters
    ----------
    known_all : pd.DataFrame
        dataframe with known retention time, for Layer 2
	unknown_all : pd.DataFrame
		dataframe with unknown retention time, for Layer 2
    ignore_cols : list
        ignore these columns
    cv_list : list
        the folds to be used in Layer 2

    Returns
    -------
    pd.DataFrame
        test predictions
	pd.DataFrame
		train predictions
    """
    ret_preds = []
    ret_preds_train = []
    cnames = []

    known_all.index = known_all["IDENTIFIER"]
    unknown_all.index = unknown_all["IDENTIFIER"]

    df_return_train = pd.DataFrame(np.zeros((len(known_all.index), len(known_all.columns))))
    df_return_train.columns = known_all.columns
    df_return_train.index = known_all.index
    df_return_train["IDENTIFIER"] = known_all["IDENTIFIER"]
    df_return_train["time"] = known_all["time"]

    df_return_test = pd.DataFrame(np.zeros((len(unknown_all.index), len(unknown_all.columns))))
    df_return_test.columns = unknown_all.columns
    df_return_test.index = unknown_all.index
    df_return_test["IDENTIFIER"] = unknown_all["IDENTIFIER"]

    for c in known_all.columns:
        try:
            if c in ["IDENTIFIER","time"]:
                continue
            try:
                cor = spearmanr(known_all[c], known_all["time"])[0]
            except:
                continue

            if abs(cor) < min_cor:
                continue

            if cor < 0.0:
                constr = "monotonic_dec"
            else:
                constr = "monotonic_inc"

            unique_cv = list(set(cv_list))

            for cv_num in unique_cv:
                selection_instances = [True if fold_num != cv_num else False for fold_num in cv_list]
                selection_instances_test = [False if fold_num != cv_num else True for fold_num in cv_list]

                X_train = known_all.loc[selection_instances,c]
                y_train = known_all.loc[selection_instances,"time"]

                X_test = known_all.loc[selection_instances_test,c]
                y_test = known_all.loc[selection_instances_test,"time"]

                gam_model_cv = LinearGAM(s(0, constraints=constr, n_splines=10), verbose=True).fit(X_train, y_train)
                df_return_train.loc[selection_instances_test,c] = list(gam_model_cv.predict(X_test))

            print("--------------------------------")
            gam_model = LinearGAM(s(0, constraints=constr, n_splines=10), verbose=True).fit(known_all[c], known_all["time"])
            df_return_test.loc[:,c] = list(gam_model.predict(unknown_all[c]))
        except KeyError:
            continue
    non_feature_cols = ["IDENTIFIER","time"]

    df_return_test.columns = [c+"+RtGAM" if c not in non_feature_cols else c for c in df_return_test.columns]
    df_return_train.columns = [c+"+RtGAM" if c not in non_feature_cols else c for c in df_return_train.columns]

    return(df_return_test,df_return_train)


<filename>nga_binaural/binaural_wrapper.py<gh_stars>1-10
import copy
from ear.options import Option, OptionsHandler
from ear.core.metadata_input import ObjectRenderingItem
from ear.core import point_source
from ear.fileio.adm.elements import ObjectPolarPosition
import numpy as np
from scipy import signal
from . import sofa, binaural_point_source
from .matrix_convolver import MatrixBlockConvolver
from .convolver import VariableBlockSizeAdapter
from .align_irs import align_irs
from .binaural_layout import BinauralOutput

binaural_output_options = OptionsHandler(
    block_size=Option(
        default=512,
        description="block size for convolution",
    ),
    virtual_layout_hrir=Option(
        default=("binaural", "all_defined"),
        description="loudspeaker layout to render to before applying HRIRs",
    ),
    virtual_layout_brir=Option(
        default=("binaural", "BRIR"),
        description="loudspeaker layout to render to before applying BRIRs",
    ),
    hrir_file=Option(
        default="resource:data/HRIR_FULL2DEG.sofa",
        description="SOFA file to get HRIRs from",
    ),
    brir_file=Option(
        default=
        "resource:data/BRIR_KU100_60ms.sofa",
        description="SOFA file to get BRIRs from",
    ),
)

class BinauralWrapper(object):
    """Wrapper around multiple loudspeaker renderers which returns the binaural rendering."""
    @binaural_output_options.with_defaults
    def __init__(self,
                 renderer_cls,
                 layout,
                 virtual_layout,
                 sr,
                 block_size,
                 virtual_layout_hrir,
                 virtual_layout_brir,
                 hrir_file,
                 brir_file,
                 renderer_opts={}):

        point_source.configure = binaural_point_source.configure

        """load layouts for all three renderings"""

        if virtual_layout is None:
            hrir_layout = sofa.get_binaural_layout(virtual_layout_hrir)
        else:
            hrir_layout = sofa.get_binaural_layout(('bs2051', virtual_layout))
        if len(hrir_layout.channels) < 22:
            brir_layout = hrir_layout
        else:
            brir_layout = sofa.get_binaural_layout(virtual_layout_brir)
        dirir_layout = sofa.get_binaural_layout(("binaural", "binaural_direct"))

        """define three renderers"""
        self.renderer_hrir = renderer_cls(hrir_layout, **renderer_opts)
        self.renderer_brir = renderer_cls(brir_layout, **renderer_opts)
        self.renderer_direct = renderer_cls(dirir_layout, **renderer_opts)

        """load impulse responses according to the prior defined layouts and apply gain and delay for correct summation"""
        hrir_sofa_file = sofa.SOFAFileHRIR(sofa.load_hdf5(hrir_file))
        hrirs = hrir_sofa_file.irs_for_positions(hrir_layout.positions)
        hrirs = align_irs(hrirs)
        if hrir_sofa_file.check_fs() != sr:
            hrirs = signal.resample(
                hrirs, int(len(hrirs) / hrir_sofa_file.check_fs() * sr))
        hrirs = hrirs / sofa.calc_gain_of_irs(hrirs) * 0.20885643426029013 / 2

        brir_sofa_file = sofa.SOFAFileHRIR(sofa.load_hdf5(brir_file))
        brirs = brir_sofa_file.irs_for_positions(brir_layout.positions)
        if brir_sofa_file.check_fs() != sr:
            brirs = signal.resample(
                brirs, int(len(brirs) / brir_sofa_file.check_fs() * sr))
        brirs = brirs / sofa.calc_gain_of_irs(
            brirs) * 0.05542830927315457 / 2
        brirs = np.concatenate(
            (np.zeros([len(brirs), 2,
                       int(sofa.calc_delay_of_irs(hrirs)) - 1]), brirs), axis=2)

        dirirs = np.concatenate((np.zeros(
            [2, 2, int(sofa.calc_delay_of_irs(hrirs)) - 1]), np.ones([2, 2, 1])), axis=2)
        dirirs = dirirs * 0.37

        """prepare filter matrix for MatrixBlockConvolver"""
        filter_matrix_hrir = [(in_ch, out_ch, ir)
                              for in_ch, ir_pair in enumerate(hrirs)
                              for out_ch, ir in enumerate(ir_pair)]

        filter_matrix_brir = [(in_ch, out_ch, ir)
                              for in_ch, ir_pair in enumerate(brirs)
                              for out_ch, ir in enumerate(ir_pair)]

        filter_matrix_dirir = [(in_ch, out_ch, ir)
                               for in_ch, ir_pair in enumerate(dirirs)
                               for out_ch, ir in enumerate(ir_pair)]

        """define convolver for all three parts"""
        convolver_hrir = MatrixBlockConvolver(block_size,
                                              len(hrir_layout.channels), 2,
                                              filter_matrix_hrir)

        convolver_brir = MatrixBlockConvolver(block_size,
                                              len(brir_layout.channels), 2,
                                              filter_matrix_brir)

        convolver_dirir = MatrixBlockConvolver(block_size,
                                               len(dirir_layout.channels), 2,
                                               filter_matrix_dirir)

        """convolution with variable block size"""
        self.convolver_vbs_hrir = VariableBlockSizeAdapter(
            block_size, (len(hrir_layout.channels), 2),
            convolver_hrir.filter_block)

        self.convolver_vbs_brir = VariableBlockSizeAdapter(
            block_size, (len(brir_layout.channels), 2),
            convolver_brir.filter_block)

        self.convolver_vbs_dirir = VariableBlockSizeAdapter(
            block_size, (len(dirir_layout.channels), 2),
            convolver_dirir.filter_block)

    """filter items to be rendered for different renderers (binaural, non-binaural)"""
    def filter_rendering_items_hrir(self, rendering_items):
        rendering_items_hrir = copy.deepcopy(rendering_items)

        for objects in enumerate(rendering_items_hrir):
            if isinstance(objects[1], ObjectRenderingItem):

                for audioBlock in objects[1].adm_path.audioChannelFormat.audioBlockFormats[:]:
                    if isinstance(audioBlock.position, ObjectPolarPosition):
                        if audioBlock.position.distance <= 0.3:
                            audioBlock.gain *= (audioBlock.position.distance / 0.3)
                        audioBlock.position.distance = 1

        return rendering_items_hrir

    def filter_rendering_items_brir(self, rendering_items):
        rendering_items_brir = copy.deepcopy(rendering_items)

        for objects in enumerate(rendering_items_brir):
            if isinstance(objects[1], ObjectRenderingItem):

                for audioBlock in objects[1].adm_path.audioChannelFormat.audioBlockFormats[:]:
                    if isinstance(audioBlock.position, ObjectPolarPosition):
                        if audioBlock.position.distance <= 1 and audioBlock.position.distance > 0.2:
                            audioBlock.gain *= (audioBlock.position.distance - 0.2) / 0.8
                        if audioBlock.position.distance <= 0.2:
                            audioBlock.gain = 0
                        audioBlock.position.distance = 1

        return rendering_items_brir

    def filter_rendering_items_direct(self, rendering_items):
        rendering_items_direct = copy.deepcopy(rendering_items)

        for objects in enumerate(rendering_items_direct):
            if isinstance(objects[1], ObjectRenderingItem):

                for audioBlock in objects[1].adm_path.audioChannelFormat.audioBlockFormats[:]:
                    if isinstance(audioBlock.position, ObjectPolarPosition):
                        if audioBlock.position.distance > 0.3:
                            audioBlock.gain = 0
                        if audioBlock.position.distance <= 0.3:
                            audioBlock.gain *= 1 - (audioBlock.position.distance / 0.3)
                        audioBlock.position.distance = 1
                    else:
                        audioBlock.gain = 0

            else:
                rendering_items_direct = []

        return rendering_items_direct

    """sets rendering items and applies filtering"""
    def set_rendering_items(self, rendering_items):

        self.renderer_brir.set_rendering_items(self.filter_rendering_items_brir(rendering_items))
        self.renderer_hrir.set_rendering_items(self.filter_rendering_items_hrir(rendering_items))
        self.renderer_direct.set_rendering_items(self.filter_rendering_items_direct(rendering_items))

    @property
    def overall_delay(self):
        """check delays for all renderers"""

        return self.convolver_vbs_hrir.delay(self.renderer_hrir.overall_delay)

    """take output of all rednerers and convolve accordingly, return complete summed rendering"""
    def render(self, sample_rate, start_sample, samples):

        loudspeaker_signals_brir = self.renderer_brir.render(
            sample_rate, start_sample, samples)
        brir_rendering = self.convolver_vbs_brir.process(
            loudspeaker_signals_brir)

        loudspeaker_signals_hrir = self.renderer_hrir.render(
            sample_rate, start_sample, samples)
        hrir_rendering = self.convolver_vbs_hrir.process(
            loudspeaker_signals_hrir)

        loudspeaker_signals_direct = self.renderer_direct.render(
            sample_rate, start_sample, samples)
        direct_rendering = self.convolver_vbs_dirir.process(
            loudspeaker_signals_direct)

        rendering = (hrir_rendering + brir_rendering + direct_rendering) / 2

        return rendering


<filename>bounding_phase.py<gh_stars>0
import math
import sympy as sym

print("BOUNDING PHASE METHOD")
x = sym.Symbol('x')

### Function ###

f = x**2 - 5*x + x**2

################

def substitute(k):
    return (f.subs(x,k))

while(1):
    delta = float(input("Enter the delta value : "))
    x_o = 0.6 #initial guess
    x_minus_delta = x_o - delta
    x_plus_delta = x_o + delta

    f_minus_delta = substitute(x_minus_delta)
    f_plus_delta = substitute(x_plus_delta)
    f_o = substitute(x_o)
    print("x_minus_delta = {}  x = {}  x_plus_delta = {}".format(x_minus_delta,x_o,x_plus_delta))
    print("f(x-$) = {}  f(x) = {}  f(x+$) = {}".format(f_minus_delta,f_o,f_plus_delta))

    if(f_minus_delta>=f_o and f_o>=f_plus_delta):
        delta = delta
        break
    elif(f_minus_delta<=f_o and f_o<=f_plus_delta):
        delta = -delta
        break
    else:
        continue

a = []
a.append(x_o)
k = 0
f_o_plus_1 = f_o - 1
while(f_o_plus_1 < f_o):
    x_o_plus_1 = x_o + (2**k)*delta
    a.append(x_o_plus_1)
    k = k + 1
    f_o_plus_1 = substitute(x_o_plus_1)
    f_o = substitute(x_o)
    print("step :", k)
    print("x_o = {} , x_o_plus_1 = {}".format(x_o,x_o_plus_1))
    print("f_o = {}  f_o_plus_1 = {}".format(f_o,f_o_plus_1))
    x_o = x_o_plus_1

print("( {} , {}  )".format(a[-3],x_o_plus_1))


<filename>DeformationTransfer.py<gh_stars>1-10
#Code by <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, and <NAME>
import sys
import numpy as np
from scipy import sparse
import scipy.io as sio
from scipy.linalg import norm
from scipy.sparse.linalg import lsqr

def saveOffFileExternal(filename, VPos, VColors, ITris):
    #Save off file given buffers, not necessarily in the PolyMesh object
    nV = VPos.shape[0]
    nF = ITris.shape[0]
    fout = open(filename, "w")
    if VColors.size == 0:
        fout.write("OFF\n%i %i %i\n"%(nV, nF, 0))
    else:
        fout.write("COFF\n%i %i %i\n"%(nV, nF, 0))
    for i in range(nV):
        fout.write("%g %g %g"%tuple(VPos[i, :]))
        if VColors.size > 0:
            fout.write(" %g %g %g"%tuple(VColors[i, :]))
        fout.write("\n")
    for i in range(nF):
        fout.write("3 %i %i %i\n"%tuple(ITris[i, :]))
    fout.close()

#Return VPos, VColors, and ITris without creating any structure
#(Assumes triangle mesh)
def loadOffFileExternal(filename):
    fin = open(filename, 'r')
    nVertices = 0
    nFaces = 0
    lineCount = 0
    face = 0
    vertex = 0
    divideColor = False
    VPos = np.zeros((0, 3))
    VColors = np.zeros((0, 3))
    ITris = np.zeros((0, 3))
    for line in fin:
        lineCount = lineCount+1
        fields = line.split() #Splits whitespace by default
        if len(fields) == 0: #Blank line
            continue
        if fields[0][0] in ['#', '\0', ' '] or len(fields[0]) == 0:
            continue
        #Check section
        if nVertices == 0:
            if fields[0] == "OFF" or fields[0] == "COFF":
                if len(fields) > 2:
                    fields[1:4] = [int(field) for field in fields]
                    [nVertices, nFaces, nEdges] = fields[1:4]
                    print "nVertices = %i, nFaces = %i"%(nVertices, nFaces)
                    #Pre-allocate vertex arrays
                    VPos = np.zeros((nVertices, 3))
                    VColors = np.zeros((nVertices, 3))
                    ITris = np.zeros((nFaces, 3))
                if fields[0] == "COFF":
                    divideColor = True
            else:
                fields[0:3] = [int(field) for field in fields]
                [nVertices, nFaces, nEdges] = fields[0:3]
                VPos = np.zeros((nVertices, 3))
                VColors = np.zeros((nVertices, 3))
                ITris = np.zeros((nFaces, 3))
        elif vertex < nVertices:
            fields = [float(i) for i in fields]
            P = [fields[0],fields[1], fields[2]]
            color = np.array([0.5, 0.5, 0.5]) #Gray by default
            if len(fields) >= 6:
                #There is color information
                if divideColor:
                    color = [float(c)/255.0 for c in fields[3:6]]
                else:
                    color = [float(c) for c in fields[3:6]]
            VPos[vertex, :] = P
            VColors[vertex, :] = color
            vertex = vertex+1
        elif face < nFaces:
            #Assume the vertices are specified in CCW order
            fields = [int(i) for i in fields]
            ITris[face, :] = fields[1:fields[0]+1]
            face = face+1
    fin.close()
    VPos = np.array(VPos, np.float64)
    VColors = np.array(VColors, np.float64)
    ITris = np.array(ITris, np.int32)
    return (VPos, VColors, ITris)

def loadBaselKeypointMesh():
    (VPos, VColors, ITris) = loadOffFileExternal("BUMesh.off")
    return (VPos, ITris)

def getBaselBUKeypointsIdx():
    idx = sio.loadmat("BaselBUKeypointsIdx")['idx']-1
    idx = idx.flatten()
    return idx

class VideoMesh(object):
    def __init__(self):
        self.Frames = np.array([])
        self.ITris = np.array([])

    #Initialize the basel video with the first (neutral) frame filled in
    #and the rest blank
    def initBaselVideo(self, filename, NFrames):
        (a, b, self.ITris) = loadOffFileExternal("BUMesh.off")
        #Grab the keypoints of the chosen basel model
        shape = sio.loadmat(filename)['shape']
        shape = np.reshape(shape, [len(shape)/3, 3])
        idx = getBaselBUKeypointsIdx()
        shape = shape[idx, :]
        self.Frames = np.zeros((NFrames, shape.shape[0], shape.shape[1]))
        self.Frames[0, :, :] = shape

    #Load in a bunch of bnd files, assuming the first one is a neutral
    #expression
    def initBUVideo(self, paths):
        (a, b, self.ITris) = loadOffFileExternal("BUMesh.off")
        X1 = np.loadtxt(paths[0])
        X1 = X1[:, 1::]
        NFrames = len(paths)
        self.Frames = np.zeros((NFrames, X1.shape[0], X1.shape[1]))
        self.Frames[0, :, :] = X1
        for i in range(1, NFrames):
            X = np.loadtxt(paths[i])
            X = X[:, 1::]
            self.Frames[i, :, :] = X

    def saveFramesOff(self, prefix):
        for i in range(self.Frames.shape[0]):
            VPos = self.Frames[i, :, :]
            fout = open("%s%i.off"%(prefix, i), "w")
            fout.write("OFF\n%i %i 0\n"%(VPos.shape[0], self.ITris.shape[0]))
            for i in range(VPos.shape[0]):
                fout.write("%g %g %g\n"%(VPos[i, 0], VPos[i, 1], VPos[i, 2]))
            for i in range(self.ITris.shape[0]):
                fout.write("3 %g %g %g\n"%(self.ITris[i, 0], self.ITris[i, 1], self.ITris[i, 2]))
            fout.close()

def getLaplacianMatrixCotangent(VPos, ITris, anchorsIdx, anchorWeights = 1):
    N = VPos.shape[0]
    M = ITris.shape[0]
    #Allocate space for the sparse array storage, with 2 entries for every
    #edge for every triangle (6 entries per triangle); one entry for directed
    #edge ij and ji.  Note that this means that edges with two incident triangles
    #will have two entries per directed edge, but sparse array will sum them
    I = np.zeros(M*6)
    J = np.zeros(M*6)
    V = np.zeros(M*6)

    #Keep track of areas of incident triangles and the number of incident triangles
    IA = np.zeros(M*3)
    VA = np.zeros(M*3) #Incident areas
    VC = 1.0*np.ones(M*3) #Number of incident triangles

    #Step 1: Compute cotangent weights
    for shift in range(3):
        #For all 3 shifts of the roles of triangle vertices
        #to compute different cotangent weights
        [i, j, k] = [shift, (shift+1)%3, (shift+2)%3]
        dV1 = VPos[ITris[:, i], :] - VPos[ITris[:, k], :]
        dV2 = VPos[ITris[:, j], :] - VPos[ITris[:, k], :]
        Normal = np.cross(dV1, dV2)
        #Cotangent is dot product / mag cross product
        NMag = np.sqrt(np.sum(Normal**2, 1))
        cotAlpha = np.sum(dV1*dV2, 1)/NMag
        I[shift*M*2:shift*M*2+M] = ITris[:, i]
        J[shift*M*2:shift*M*2+M] = ITris[:, j]
        V[shift*M*2:shift*M*2+M] = cotAlpha
        I[shift*M*2+M:shift*M*2+2*M] = ITris[:, j]
        J[shift*M*2+M:shift*M*2+2*M] = ITris[:, i]
        V[shift*M*2+M:shift*M*2+2*M] = cotAlpha
        if shift == 0:
            #Compute contribution of this triangle to each of the vertices
            for k in range(3):
                IA[k*M:(k+1)*M] = ITris[:, k]
                VA[k*M:(k+1)*M] = 0.5*NMag

    #Step 2: Create laplacian matrix
    L = sparse.coo_matrix((V, (I, J)), shape=(N, N)).tocsr()
    #Create the diagonal by summing the rows and subtracting off the nondiagonal entries
    L = sparse.dia_matrix((L.sum(1).flatten(), 0), L.shape) - L

    #Step 3: Add anchors
    L = L.tocoo()
    I = L.row.tolist()
    J = L.col.tolist()
    V = L.data.tolist()
    I = I + range(N, N+len(anchorsIdx))
    J = J + anchorsIdx.tolist()
    V = V + [anchorWeights]*len(anchorsIdx)
    L = sparse.coo_matrix((V, (I, J)), shape=(N+len(anchorsIdx), N)).tocsr()
    return L

def solveLaplacianMesh(VPos, ITris, anchorsIdx, anchors):
    N = VPos.shape[0]
    L = getLaplacianMatrixCotangent(VPos, ITris, anchorsIdx)
    delta = L.dot(VPos)
    delta[N:, :] = anchors
    sio.savemat("System.mat", {"L":L, "delta":delta})
    VPosNew = np.zeros((N, 3))
    for k in range(3):
        print "Solving Laplacian mesh coordinates %i of %i..."%(k+1, 3)
        VPosNew[:, k] = lsqr(L, delta[:, k])[0]
    return VPosNew

class DeformationTransferer:
    def __init__(self, origVideo, warpedVideo):
        self.origVideo = origVideo
        self.warpedVideo = warpedVideo
        self.origFrames = self.origVideo.Frames
        self.warpedFrames = self.warpedVideo.Frames
        self.NFrames = self.origFrames.shape[0]
        self.ITris = self.origVideo.ITris
        self.NFaces = self.ITris.shape[0]

        self.count = 0
        self.NVertices = self.origFrames.shape[1]
        self.NVertices4 = self.NVertices + self.NFaces #original vertices plus 1 new vertex (4th vector) for each face
        # Tris4 is Tris plus 4th col indexing 4th vector (which should be mapped to the N to N+F-1 index of VPos4)
        self.Tris4 = np.hstack((self.ITris,
                                    np.reshape(np.arange(self.NVertices, self.NVertices4), (self.NFaces, 1))))
        print "#####debug info: initial values#########"
        print "origFrame shape (NFrames x NVertices x 3):", self.origFrames.shape
        print "warpedFrame shape (NFrames x NVertices x 3): ", self.warpedFrames.shape
        print "ITris shape:", self.ITris.shape
        print "#####end: initial values#########"

    def beginDeformationTransfer(self):
        resultFrames = np.empty([self.NFrames, self.NVertices, 3])  # this is result array to fill in
        resultFrames[0, :, :] = self.warpedFrames[0, :, :]
        origOldVPos4 = self.getVPos4(self.origFrames[0, :, :], self.ITris)  # old VPos with extra NFaces vectors
        warpedOldVPos4 = self.getVPos4(self.warpedFrames[0, :, :], self.ITris)
        for i in range(1, self.NFrames):
            # 1 orig: get newVPos4
            origNewVPos4 = self.getVPos4(self.origFrames[i, :, :], self.ITris)
            # 2 orig: use old and new VPos4 to get S-matrix which shape is 3 x 3NFaces
            S = self.getSMatrix(origOldVPos4, origNewVPos4, self.Tris4)
            # 3 warped: use old VPos4 to get A (coefficient) sparse matrix which shape is 3NFaces x NVertices
            A = self.getAMatrix(warpedOldVPos4, self.Tris4)
            origOldVPos4 = origNewVPos4
            warpedOldVPos4[:, 0] = lsqr(A, S[0, :])[0]
            warpedOldVPos4[:, 1] = lsqr(A, S[1, :])[0]
            warpedOldVPos4[:, 2] = lsqr(A, S[2, :])[0]
           # print "new VPos4 shape:", warpedOldVPos4[np.arange(self.NVertices), :].shape
            resultFrames[i, :, :] = warpedOldVPos4[np.arange(self.NVertices), :]
        self.warpedVideo.Frames = resultFrames

    #get VPos4 (each face has 4 vertices) from VPos3 (each face 3 vertices) with mesh topology given
    def getVPos4(self, VPos3, ITris3):
        V4 = self.get4thVertex(VPos3, ITris3)
        VPos4 = np.vstack((VPos3, V4))
        return VPos4

    # get4thVertex for each face, aka face normal scaled by reciprocal of sqrt of its length
    # (3 vertices's index are stored in every row in ITris)
    def get4thVertex(self, VPos3, ITris3):
        V1 = VPos3[ITris3[:, 1], :] - VPos3[ITris3[:, 0], :]
        V2 = VPos3[ITris3[:, 2], :] - VPos3[ITris3[:, 0], :]
        FNormals = np.cross(V1, V2)

        FNormalsSqrtLength = np.sqrt(np.sum(FNormals**2, 1))[:, None]
        F = FNormals/FNormalsSqrtLength
        Vertex4 = VPos3[ITris3[:, 0], :] + F
        return Vertex4

    def getSMatrix(self, oldVPos4, newVPos4, Tris4):
        v2subv1 = oldVPos4[Tris4[:, 1], :] - oldVPos4[Tris4[:, 0], :]
        v3subv1 = oldVPos4[Tris4[:, 2], :] - oldVPos4[Tris4[:, 0], :]
        v4subv1 = oldVPos4[Tris4[:, 3], :] - oldVPos4[Tris4[:, 0], :]
        tildev2subv1 = newVPos4[Tris4[:, 1], :] - newVPos4[Tris4[:, 0], :]
        tildev3subv1 = newVPos4[Tris4[:, 2], :] - newVPos4[Tris4[:, 0], :]
        tildev4subv1 = newVPos4[Tris4[:, 3], :] - newVPos4[Tris4[:, 0], :]
        assert self.NFaces == Tris4.shape[0]
        S = np.zeros((3, 0))
        for i in range(0, self.NFaces):
            vInv = np.linalg.inv((np.vstack((v2subv1[i, :], v3subv1[i, :], v4subv1[i, :]))).T)
            tildev = (np.vstack((tildev2subv1[i, :], tildev3subv1[i, :], tildev4subv1[i, :]))).T
            S = np.hstack((S, np.dot(tildev, vInv)))
        return S

    def getAMatrix(self, VPos4, Tris4):
        # I, J, and V are parallel numpy arrays that hold the rows, columns, and values of nonzero elements
        I = []
        J = []
        V = []
        v2subv1 = VPos4[Tris4[:, 1], :] - VPos4[Tris4[:, 0], :]
        v3subv1 = VPos4[Tris4[:, 2], :] - VPos4[Tris4[:, 0], :]
        v4subv1 = VPos4[Tris4[:, 3], :] - VPos4[Tris4[:, 0], :]
        assert self.NFaces == Tris4.shape[0]

        for i in range(0, self.NFaces):
            idxRow = i * 3
            vInv = np.linalg.inv((np.vstack((v2subv1[i, :], v3subv1[i, :], v4subv1[i, :]))).T)  # 3x3
            sumOfNegativevInv = np.sum(-1 * vInv, axis = 0) # shape is (3,)
            ###################   ######
            # -A-D-G, A, D, G #   # x1 #
            # -B-E-H, B, E, H # X # x2 #
            # -C-F-I, C, F, I #   # x3 #
            ###################   # x4 #
                                  ######

            # sumOfNegativevInv current looks like this, take care when fill in I, J, V
            ##########################
            # -A-D-G, -B-E-H, -C-F-I #
            ##########################
            for j in range(0, 3):
                I.append(idxRow + j)
                J.append(Tris4[i, 0])
                V.append(sumOfNegativevInv[j])
            # vInv current looks like this. Same, be careful.
            ###########
            # A, B, C #
            # D, E, F #
            # G, H, I #
            ###########
            for j in range(0, 3):
                for k in range(0, 3):
                    I.append(idxRow + k)
                    J.append(Tris4[i, j + 1])
                    V.append(vInv[j, k])
        A = sparse.coo_matrix((V, (I, J)), shape = (3 * self.NFaces, self.NVertices4)).tocsr()
        return A

if __name__ == '__main__':
    #Load in BU bnd files
    buVideo = VideoMesh()
    buVideo.initBUVideo(["bu3/F0012/F0012_AN01WH_F3D.bnd", "bu3/F0012/F0012_HA04WH_F3D.bnd"])
    NFrames = buVideo.Frames.shape[0]

    #Load in basel mesh
    baselVertsFile = "BaselVerts.mat"
    ITris = sio.loadmat("BaselTris.mat")['ITris']
    VPos = sio.loadmat(baselVertsFile)['shape']
    VPos = np.reshape(VPos, [len(VPos)/3, 3])

    #Create basel video placeholder
    baselVideo = VideoMesh()
    baselVideo.initBaselVideo(baselVertsFile, NFrames)

    #Do coarse deformation transfer
    T = DeformationTransferer(buVideo, baselVideo)
    T.beginDeformationTransfer()
    #Save coarse frames to hard drive
    baselVideo.saveFramesOff("Basel")

    #Do fine deformation transfer with Laplacian mesh using coarse
    #vertices as anchors
    idx = getBaselBUKeypointsIdx()
    L = getLaplacianMatrixCotangent(VPos, ITris, idx)
    VPosNew = solveLaplacianMesh(VPos, ITris, idx, baselVideo.Frames[1, :, :])
    saveOffFileExternal("BaselTransfer.off", VPosNew, np.array([]), ITris)


<filename>astrobase/lcmath.py<gh_stars>10-100
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# lcmath.py - <NAME> (<EMAIL>) - Feb 2015

'''
Contains various useful tools for calculating various things related to
lightcurves (like phasing, sigma-clipping, finding and filling gaps, etc.)

'''

#############
## LOGGING ##
#############

import logging
from astrobase import log_sub, log_fmt, log_date_fmt

DEBUG = False
if DEBUG:
    level = logging.DEBUG
else:
    level = logging.INFO
LOGGER = logging.getLogger(__name__)
logging.basicConfig(
    level=level,
    style=log_sub,
    format=log_fmt,
    datefmt=log_date_fmt,
)

LOGDEBUG = LOGGER.debug
LOGINFO = LOGGER.info
LOGWARNING = LOGGER.warning
LOGERROR = LOGGER.error
LOGEXCEPTION = LOGGER.exception

#############
## IMPORTS ##
#############

import numpy as np
from numpy import (
    isfinite as npisfinite, median as npmedian, mean as npmean,
    abs as npabs, std as npstddev
)

from scipy.spatial import cKDTree
from scipy.signal import savgol_filter
import scipy.stats

############################
## NORMALIZING MAG SERIES ##
############################

def find_lc_timegroups(lctimes, mingap=4.0):
    '''Finds gaps in the provided time-series and indexes them into groups.

    This finds the gaps in the provided `lctimes` array, so we can figure out
    which times are for consecutive observations and which represent gaps
    between seasons or observing eras.

    Parameters
    ----------

    lctimes : array-like
        This contains the times to analyze for gaps; assumed to be some form of
        Julian date.

    mingap : float
        This defines how much the difference between consecutive measurements is
        allowed to be to consider them as parts of different timegroups. By
        default it is set to 4.0 days.

    Returns
    -------

    tuple
        A tuple of the form: `(ngroups, [slice(start_ind_1, end_ind_1), ...])`
        is returned.  This contains the number of groups as the first element,
        and a list of Python `slice` objects for each time-group found. These
        can be used directly to index into the array of times to quickly get
        measurements associated with each group.

    '''

    lc_time_diffs = np.diff(lctimes)
    group_start_indices = np.where(lc_time_diffs > mingap)[0]

    if len(group_start_indices) > 0:

        group_indices = []

        for i, gindex in enumerate(group_start_indices):

            if i == 0:
                group_indices.append(slice(0,gindex+1))
            else:
                group_indices.append(slice(group_start_indices[i-1]+1,gindex+1))

        # at the end, add the slice for the last group to the end of the times
        # array
        group_indices.append(slice(group_start_indices[-1]+1,len(lctimes)))

    # if there's no large gap in the LC, then there's only one group to worry
    # about
    else:
        group_indices = [slice(0,len(lctimes))]

    return len(group_indices), group_indices

def normalize_magseries(times,
                        mags,
                        mingap=4.0,
                        normto='globalmedian',
                        magsarefluxes=False,
                        debugmode=False):
    '''This normalizes the magnitude time-series to a specified value.

    This is used to normalize time series measurements that may have large time
    gaps and vertical offsets in mag/flux measurement between these
    'timegroups', either due to instrument changes or different filters.

    NOTE: this works in-place! The mags array will be replaced with normalized
    mags when this function finishes.

    Parameters
    ----------

    times,mags : array-like
        The times (assumed to be some form of JD) and mags (or flux)
        measurements to be normalized.

    mingap : float
        This defines how much the difference between consecutive measurements is
        allowed to be to consider them as parts of different timegroups. By
        default it is set to 4.0 days.

    normto : {'globalmedian', 'zero'} or a float
        Specifies the normalization type::

          'globalmedian' -> norms each mag to the global median of the LC column
          'zero'         -> norms each mag to zero
          a float        -> norms each mag to this specified float value.

    magsarefluxes : bool
        Indicates if the input `mags` array is actually an array of flux
        measurements instead of magnitude measurements. If this is set to True,
        then:

        - if `normto` is 'zero', then the median flux is divided from each
          observation's flux value to yield normalized fluxes with 1.0 as the
          global median.

        - if `normto` is 'globalmedian', then the global median flux value
          across the entire time series is multiplied with each measurement.

        - if `norm` is set to a `float`, then this number is multiplied with the
          flux value for each measurement.

    debugmode : bool
        If this is True, will print out verbose info on each timegroup found.

    Returns
    -------

    times,normalized_mags : np.arrays
        Normalized magnitude values after normalization. If normalization fails
        for some reason, `times` and `normalized_mags` will both be None.

    '''

    ngroups, timegroups = find_lc_timegroups(times,
                                             mingap=mingap)

    # find all the non-nan indices
    finite_ind = np.isfinite(mags)

    if any(finite_ind):

        # find the global median
        global_mag_median = np.median(mags[finite_ind])

        # go through the groups and normalize them to the median for
        # each group
        for tgind, tg in enumerate(timegroups):

            finite_ind = np.isfinite(mags[tg])

            # find this timegroup's median mag and normalize the mags in
            # it to this median
            group_median = np.median((mags[tg])[finite_ind])

            if magsarefluxes:
                mags[tg] = mags[tg]/group_median
            else:
                mags[tg] = mags[tg] - group_median

            if debugmode:
                LOGDEBUG('group %s: elems %s, '
                         'finite elems %s, median mag %s' %
                         (tgind,
                          len(mags[tg]),
                          len(finite_ind),
                          group_median))

        # now that everything is normalized to 0.0, add the global median
        # offset back to all the mags and write the result back to the dict
        if isinstance(normto, str) and normto == 'globalmedian':

            if magsarefluxes:
                mags = mags * global_mag_median
            else:
                mags = mags + global_mag_median

        # if the normto is a float, add everything to that float and return
        elif isinstance(normto, float):

            if magsarefluxes:
                mags = mags * normto
            else:
                mags = mags + normto

        # anything else just returns the normalized mags as usual
        return times, mags

    else:
        LOGERROR('measurements are all nan!')
        return None, None

####################
## SIGMA-CLIPPING ##
####################

def sigclip_magseries(times, mags, errs,
                      sigclip=None,
                      iterative=False,
                      niterations=None,
                      meanormedian='median',
                      magsarefluxes=False):
    '''Sigma-clips a magnitude or flux time-series.

    Selects the finite times, magnitudes (or fluxes), and errors from the passed
    values, and apply symmetric or asymmetric sigma clipping to them.

    Parameters
    ----------

    times,mags,errs : np.array
        The magnitude or flux time-series arrays to sigma-clip. This doesn't
        assume all values are finite or if they're positive/negative. All of
        these arrays will have their non-finite elements removed, and then will
        be sigma-clipped based on the arguments to this function.

        `errs` is optional. Set it to None if you don't have values for these. A
        'faked' `errs` array will be generated if necessary, which can be
        ignored in the output as well.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    iterative : bool
        If this is set to True, will perform iterative sigma-clipping. If
        `niterations` is not set and this is True, sigma-clipping is iterated
        until no more points are removed.

    niterations : int
        The maximum number of iterations to perform for sigma-clipping. If None,
        the `iterative` arg takes precedence, and `iterative=True` will
        sigma-clip until no more points are removed.  If `niterations` is not
        None and `iterative` is False, `niterations` takes precedence and
        iteration will occur for the specified number of iterations.

    meanormedian : {'mean', 'median'}
        Use 'mean' for sigma-clipping based on the mean value, or 'median' for
        sigma-clipping based on the median value.  Default is 'median'.

    magsareflux : bool
        True if your "mags" are in fact fluxes, i.e. if "fainter" corresponds to
        `mags` getting smaller.

    Returns
    -------

    (stimes, smags, serrs) : tuple
        The sigma-clipped and nan-stripped time-series.

    '''

    returnerrs = True

    # fake the errors if they don't exist
    # this is inconsequential to sigma-clipping
    # we don't return these dummy values if the input errs are None
    if errs is None:
        # assume 0.1% errors if not given
        # this should work for mags and fluxes
        errs = 0.001*mags
        returnerrs = False

    # filter the input times, mags, errs; do sigclipping and normalization
    find = npisfinite(times) & npisfinite(mags) & npisfinite(errs)
    ftimes, fmags, ferrs = times[find], mags[find], errs[find]

    # get the center value and stdev
    if meanormedian == 'median':  # stddev = 1.483 x MAD

        center_mag = npmedian(fmags)
        stddev_mag = (npmedian(npabs(fmags - center_mag))) * 1.483

    elif meanormedian == 'mean':

        center_mag = npmean(fmags)
        stddev_mag = npstddev(fmags)

    else:
        LOGWARNING("unrecognized meanormedian value given to "
                   "sigclip_magseries: %s, defaulting to 'median'" %
                   meanormedian)
        meanormedian = 'median'
        center_mag = npmedian(fmags)
        stddev_mag = (npmedian(npabs(fmags - center_mag))) * 1.483

    # sigclip next for a single sigclip value
    if sigclip and isinstance(sigclip, (float, int)):

        if not iterative and niterations is None:

            sigind = (npabs(fmags - center_mag)) < (sigclip * stddev_mag)

            stimes = ftimes[sigind]
            smags = fmags[sigind]
            serrs = ferrs[sigind]

        else:

            #
            # iterative version adapted from scipy.stats.sigmaclip
            #

            # First, if niterations is not set, iterate until covergence
            if niterations is None:

                delta = 1

                this_times = ftimes
                this_mags = fmags
                this_errs = ferrs

                while delta:

                    if meanormedian == 'mean':
                        this_center = npmean(this_mags)
                        this_stdev = npstddev(this_mags)
                    elif meanormedian == 'median':
                        this_center = npmedian(this_mags)
                        this_stdev = (
                            npmedian(npabs(this_mags - this_center))
                        ) * 1.483
                    this_size = this_mags.size

                    # apply the sigclip
                    tsi = (
                        (npabs(this_mags - this_center)) <
                        (sigclip * this_stdev)
                    )

                    # update the arrays
                    this_times = this_times[tsi]
                    this_mags = this_mags[tsi]
                    this_errs = this_errs[tsi]

                    # update delta and go to the top of the loop
                    delta = this_size - this_mags.size

            else:  # If iterating only a certain number of times

                this_times = ftimes
                this_mags = fmags
                this_errs = ferrs

                iter_num = 0
                delta = 1
                while iter_num < niterations and delta:

                    if meanormedian == 'mean':

                        this_center = npmean(this_mags)
                        this_stdev = npstddev(this_mags)

                    elif meanormedian == 'median':

                        this_center = npmedian(this_mags)
                        this_stdev = (npmedian(npabs(this_mags -
                                                     this_center))) * 1.483
                    this_size = this_mags.size

                    # apply the sigclip
                    tsi = (
                        (npabs(this_mags - this_center)) <
                        (sigclip * this_stdev)
                    )

                    # update the arrays
                    this_times = this_times[tsi]
                    this_mags = this_mags[tsi]
                    this_errs = this_errs[tsi]

                    # update the number of iterations and delta and
                    # go to the top of the loop
                    delta = this_size - this_mags.size
                    iter_num += 1

            # final sigclipped versions
            stimes, smags, serrs = this_times, this_mags, this_errs

    # this handles sigclipping for asymmetric +ve and -ve clip values
    elif sigclip and isinstance(sigclip, (list,tuple)) and len(sigclip) == 2:

        # sigclip is passed as [dimmingclip, brighteningclip]
        dimmingclip = sigclip[0]
        brighteningclip = sigclip[1]

        if not iterative and niterations is None:

            if magsarefluxes:
                nottoodimind = (
                    (fmags - center_mag) > (-dimmingclip*stddev_mag)
                )
                nottoobrightind = (
                    (fmags - center_mag) < (brighteningclip*stddev_mag)
                )
            else:
                nottoodimind = (
                    (fmags - center_mag) < (dimmingclip*stddev_mag)
                )
                nottoobrightind = (
                    (fmags - center_mag) > (-brighteningclip*stddev_mag)
                )

            sigind = nottoodimind & nottoobrightind

            stimes = ftimes[sigind]
            smags = fmags[sigind]
            serrs = ferrs[sigind]

        else:

            #
            # iterative version adapted from scipy.stats.sigmaclip
            #
            if niterations is None:

                delta = 1

                this_times = ftimes
                this_mags = fmags
                this_errs = ferrs

                while delta:

                    if meanormedian == 'mean':

                        this_center = npmean(this_mags)
                        this_stdev = npstddev(this_mags)

                    elif meanormedian == 'median':
                        this_center = npmedian(this_mags)
                        this_stdev = (npmedian(npabs(this_mags -
                                                     this_center))) * 1.483
                    this_size = this_mags.size

                    if magsarefluxes:
                        nottoodimind = (
                            (this_mags - this_center) >
                            (-dimmingclip*this_stdev)
                        )
                        nottoobrightind = (
                            (this_mags - this_center) <
                            (brighteningclip*this_stdev)
                        )
                    else:
                        nottoodimind = (
                            (this_mags - this_center) <
                            (dimmingclip*this_stdev)
                        )
                        nottoobrightind = (
                            (this_mags - this_center) >
                            (-brighteningclip*this_stdev)
                        )

                    # apply the sigclip
                    tsi = nottoodimind & nottoobrightind

                    # update the arrays
                    this_times = this_times[tsi]
                    this_mags = this_mags[tsi]
                    this_errs = this_errs[tsi]

                    # update delta and go to top of the loop
                    delta = this_size - this_mags.size

            else:  # If iterating only a certain number of times
                this_times = ftimes
                this_mags = fmags
                this_errs = ferrs

                iter_num = 0
                delta = 1

                while iter_num < niterations and delta:

                    if meanormedian == 'mean':
                        this_center = npmean(this_mags)
                        this_stdev = npstddev(this_mags)
                    elif meanormedian == 'median':
                        this_center = npmedian(this_mags)
                        this_stdev = (npmedian(npabs(this_mags -
                                                     this_center))) * 1.483
                    this_size = this_mags.size

                    if magsarefluxes:
                        nottoodimind = (
                            (this_mags - this_center) >
                            (-dimmingclip*this_stdev)
                        )
                        nottoobrightind = (
                            (this_mags - this_center) <
                            (brighteningclip*this_stdev)
                        )
                    else:
                        nottoodimind = (
                            (this_mags - this_center) < (dimmingclip*this_stdev)
                        )
                        nottoobrightind = (
                            (this_mags - this_center) >
                            (-brighteningclip*this_stdev)
                        )

                    # apply the sigclip
                    tsi = nottoodimind & nottoobrightind

                    # update the arrays
                    this_times = this_times[tsi]
                    this_mags = this_mags[tsi]
                    this_errs = this_errs[tsi]

                    # update the number of iterations and delta
                    # and go to top of the loop
                    delta = this_size - this_mags.size
                    iter_num += 1

            # final sigclipped versions
            stimes, smags, serrs = this_times, this_mags, this_errs

    else:

        stimes = ftimes
        smags = fmags
        serrs = ferrs

    if returnerrs:
        return stimes, smags, serrs
    else:
        return stimes, smags, None

def sigclip_magseries_with_extparams(times, mags, errs, extparams,
                                     sigclip=None,
                                     iterative=False,
                                     magsarefluxes=False):
    '''Sigma-clips a magnitude or flux time-series and associated measurement
    arrays.

    Selects the finite times, magnitudes (or fluxes), and errors from the passed
    values, and apply symmetric or asymmetric sigma clipping to them.  Uses the
    same array indices as these values to filter out the values of all arrays in
    the `extparams` list. This can be useful for simultaneously sigma-clipping a
    magnitude/flux time-series along with their associated values of external
    parameters, such as telescope hour angle, zenith distance, temperature, moon
    phase, etc.

    Parameters
    ----------

    times,mags,errs : np.array
        The magnitude or flux time-series arrays to sigma-clip. This doesn't
        assume all values are finite or if they're positive/negative. All of
        these arrays will have their non-finite elements removed, and then will
        be sigma-clipped based on the arguments to this function.

        `errs` is optional. Set it to None if you don't have values for these. A
        'faked' `errs` array will be generated if necessary, which can be
        ignored in the output as well.

    extparams : list of np.array
        This is a list of all external parameter arrays to simultaneously filter
        along with the magnitude/flux time-series. All of these arrays should
        have the same length as the `times`, `mags`, and `errs` arrays.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    iterative : bool
        If this is set to True, will perform iterative sigma-clipping. If
        `niterations` is not set and this is True, sigma-clipping is iterated
        until no more points are removed.

    magsareflux : bool
        True if your "mags" are in fact fluxes, i.e. if "fainter" corresponds to
        `mags` getting smaller.

    Returns
    -------

    (stimes, smags, serrs) : tuple
        The sigma-clipped and nan-stripped time-series in `stimes`, `smags`,
        `serrs` and the associated values of the `extparams` in `sextparams`.

    '''

    returnerrs = True

    # fake the errors if they don't exist
    # this is inconsequential to sigma-clipping
    # we don't return these dummy values if the input errs are None
    if errs is None:
        # assume 0.1% errors if not given
        # this should work for mags and fluxes
        errs = 0.001*mags
        returnerrs = False

    # filter the input times, mags, errs; do sigclipping and normalization
    find = npisfinite(times) & npisfinite(mags) & npisfinite(errs)
    ftimes, fmags, ferrs = times[find], mags[find], errs[find]

    # apply the same indices to the external parameters
    for epi, eparr in enumerate(extparams):
        extparams[epi] = eparr[find]

    # get the median and stdev = 1.483 x MAD
    median_mag = npmedian(fmags)
    stddev_mag = (npmedian(npabs(fmags - median_mag))) * 1.483

    # sigclip next for a single sigclip value
    if sigclip and isinstance(sigclip, (float, int)):

        if not iterative:

            sigind = (npabs(fmags - median_mag)) < (sigclip * stddev_mag)

            stimes = ftimes[sigind]
            smags = fmags[sigind]
            serrs = ferrs[sigind]

            # apply the same indices to the external parameters
            for epi, eparr in enumerate(extparams):
                extparams[epi] = eparr[sigind]

        else:

            #
            # iterative version adapted from scipy.stats.sigmaclip
            #
            delta = 1

            this_times = ftimes
            this_mags = fmags
            this_errs = ferrs

            while delta:

                this_median = npmedian(this_mags)
                this_stdev = (npmedian(npabs(this_mags - this_median))) * 1.483
                this_size = this_mags.size

                # apply the sigclip
                tsi = (npabs(this_mags - this_median)) < (sigclip * this_stdev)

                # update the arrays
                this_times = this_times[tsi]
                this_mags = this_mags[tsi]
                this_errs = this_errs[tsi]

                # apply the same indices to the external parameters
                for epi, eparr in enumerate(extparams):
                    extparams[epi] = eparr[tsi]

                # update delta and go to the top of the loop
                delta = this_size - this_mags.size

            # final sigclipped versions
            stimes, smags, serrs = this_times, this_mags, this_errs

    # this handles sigclipping for asymmetric +ve and -ve clip values
    elif sigclip and isinstance(sigclip, (list, tuple)) and len(sigclip) == 2:

        # sigclip is passed as [dimmingclip, brighteningclip]
        dimmingclip = sigclip[0]
        brighteningclip = sigclip[1]

        if not iterative:

            if magsarefluxes:
                nottoodimind = (
                    (fmags - median_mag) > (-dimmingclip*stddev_mag)
                )
                nottoobrightind = (
                    (fmags - median_mag) < (brighteningclip*stddev_mag)
                )
            else:
                nottoodimind = (
                    (fmags - median_mag) < (dimmingclip*stddev_mag)
                )
                nottoobrightind = (
                    (fmags - median_mag) > (-brighteningclip*stddev_mag)
                )

            sigind = nottoodimind & nottoobrightind

            stimes = ftimes[sigind]
            smags = fmags[sigind]
            serrs = ferrs[sigind]

            # apply the same indices to the external parameters
            for epi, eparr in enumerate(extparams):
                extparams[epi] = eparr[sigind]

        else:

            #
            # iterative version adapted from scipy.stats.sigmaclip
            #
            delta = 1

            this_times = ftimes
            this_mags = fmags
            this_errs = ferrs

            while delta:

                this_median = npmedian(this_mags)
                this_stdev = (npmedian(npabs(this_mags - this_median))) * 1.483
                this_size = this_mags.size

                if magsarefluxes:
                    nottoodimind = (
                        (this_mags - this_median) > (-dimmingclip*this_stdev)
                    )
                    nottoobrightind = (
                        (this_mags - this_median) < (brighteningclip*this_stdev)
                    )
                else:
                    nottoodimind = (
                        (this_mags - this_median) < (dimmingclip*this_stdev)
                    )
                    nottoobrightind = (
                        (this_mags - this_median) >
                        (-brighteningclip*this_stdev)
                    )

                # apply the sigclip
                tsi = nottoodimind & nottoobrightind

                # update the arrays
                this_times = this_times[tsi]
                this_mags = this_mags[tsi]
                this_errs = this_errs[tsi]

                # apply the same indices to the external parameters
                for epi, eparr in enumerate(extparams):
                    extparams[epi] = eparr[tsi]

                # update delta and go to top of the loop
                delta = this_size - this_mags.size

            # final sigclipped versions
            stimes, smags, serrs = this_times, this_mags, this_errs

    else:

        stimes = ftimes
        smags = fmags
        serrs = ferrs

    if returnerrs:
        return stimes, smags, serrs, extparams
    else:
        return stimes, smags, None, extparams

#################
## PHASING LCS ##
#################

def phase_magseries(times, mags, period, epoch, wrap=True, sort=True):
    '''Phases a magnitude/flux time-series using a given period and epoch.

    The equation used is::

        phase = (times - epoch)/period - floor((times - epoch)/period)

    This phases the given magnitude timeseries using the given period and
    epoch. If wrap is True, wraps the result around 0.0 (and returns an array
    that has twice the number of the original elements). If sort is True,
    returns the magnitude timeseries in phase sorted order.

    Parameters
    ----------

    times,mags : np.array
        The magnitude/flux time-series values to phase using the provided
        `period` and `epoch`. Non-fiinite values will be removed.

    period : float
        The period to use to phase the time-series.

    epoch : float
        The epoch to phase the time-series. This is usually the time-of-minimum
        or time-of-maximum of some periodic light curve
        phenomenon. Alternatively, one can use the minimum time value in
        `times`.

    wrap : bool
        If this is True, the returned phased time-series will be wrapped around
        phase 0.0, which is useful for plotting purposes. The arrays returned
        will have twice the number of input elements because of this wrapping.

    sort : bool
        If this is True, the returned phased time-series will be sorted in
        increasing phase order.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'phase': the phase values,
             'mags': the mags/flux values at each phase,
             'period': the input `period` used to phase the time-series,
             'epoch': the input `epoch` used to phase the time-series}

    '''

    # find all the finite values of the magnitudes and times
    finiteind = np.isfinite(mags) & np.isfinite(times)

    finite_times = times[finiteind]
    finite_mags = mags[finiteind]

    magseries_phase = (
        (finite_times - epoch)/period -
        np.floor(((finite_times - epoch)/period))
    )

    outdict = {'phase':magseries_phase,
               'mags':finite_mags,
               'period':period,
               'epoch':epoch}

    if sort:
        sortorder = np.argsort(outdict['phase'])
        outdict['phase'] = outdict['phase'][sortorder]
        outdict['mags'] = outdict['mags'][sortorder]

    if wrap:
        outdict['phase'] = np.concatenate((outdict['phase']-1.0,
                                           outdict['phase']))
        outdict['mags'] = np.concatenate((outdict['mags'],
                                          outdict['mags']))

    return outdict

def phase_magseries_with_errs(times, mags, errs, period, epoch,
                              wrap=True, sort=True):
    '''Phases a magnitude/flux time-series using a given period and epoch.

    The equation used is::

        phase = (times - epoch)/period - floor((times - epoch)/period)

    This phases the given magnitude timeseries using the given period and
    epoch. If wrap is True, wraps the result around 0.0 (and returns an array
    that has twice the number of the original elements). If sort is True,
    returns the magnitude timeseries in phase sorted order.

    Parameters
    ----------

    times,mags,errs : np.array
        The magnitude/flux time-series values and associated measurement errors
        to phase using the provided `period` and `epoch`. Non-fiinite values
        will be removed.

    period : float
        The period to use to phase the time-series.

    epoch : float
        The epoch to phase the time-series. This is usually the time-of-minimum
        or time-of-maximum of some periodic light curve
        phenomenon. Alternatively, one can use the minimum time value in
        `times`.

    wrap : bool
        If this is True, the returned phased time-series will be wrapped around
        phase 0.0, which is useful for plotting purposes. The arrays returned
        will have twice the number of input elements because of this wrapping.

    sort : bool
        If this is True, the returned phased time-series will be sorted in
        increasing phase order.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'phase': the phase values,
             'mags': the mags/flux values at each phase,
             'errs': the err values at each phase,
             'period': the input `period` used to phase the time-series,
             'epoch': the input `epoch` used to phase the time-series}

    '''

    # find all the finite values of the magnitudes and times
    finiteind = np.isfinite(mags)
    finite_times = times[finiteind]
    finite_mags = mags[finiteind]
    finite_errs = errs[finiteind]

    magseries_phase = (
        (finite_times - epoch)/period -
        np.floor(((finite_times - epoch)/period))
    )

    outdict = {'phase':magseries_phase,
               'mags':finite_mags,
               'errs':finite_errs,
               'period':period,
               'epoch':epoch}

    if sort:
        sortorder = np.argsort(outdict['phase'])
        outdict['phase'] = outdict['phase'][sortorder]
        outdict['mags'] = outdict['mags'][sortorder]
        outdict['errs'] = outdict['errs'][sortorder]

    if wrap:
        outdict['phase'] = np.concatenate((outdict['phase']-1.0,
                                           outdict['phase']))
        outdict['mags'] = np.concatenate((outdict['mags'],
                                          outdict['mags']))
        outdict['errs'] = np.concatenate((outdict['errs'],
                                          outdict['errs']))

    return outdict

#################
## BINNING LCs ##
#################

def time_bin_magseries(times, mags,
                       binsize=540.0,
                       minbinelems=7):
    '''Bins the given mag/flux time-series in time using the bin size given.

    Parameters
    ----------

    times,mags : np.array
        The magnitude/flux time-series to bin in time. Non-finite elements will
        be removed from these arrays. At least 10 elements in each array are
        required for this function to operate.

    binsize : float
        The bin size to use to group together measurements closer than this
        amount in time. This is in seconds.

    minbinelems : int
        The minimum number of elements required per bin to include it in the
        output.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'jdbin_indices': a list of the index arrays into the nan-filtered
                              input arrays per each bin,
             'jdbins': list of bin boundaries for each bin,
             'nbins': the number of bins generated,
             'binnedtimes': the time values associated with each time bin;
                            this is the median of the times in each bin,
             'binnedmags': the mag/flux values associated with each time bin;
                           this is the median of the mags/fluxes in each bin}

    '''

    # check if the input arrays are ok
    if not(times.shape and mags.shape and len(times) > 9 and len(mags) > 9):

        LOGERROR("input time/mag arrays don't have enough elements")
        return

    # find all the finite values of the magnitudes and times
    finiteind = np.isfinite(mags) & np.isfinite(times)
    finite_times = times[finiteind]
    finite_mags = mags[finiteind]

    # convert binsize in seconds to JD units
    binsizejd = binsize/(86400.0)
    nbins = int(np.ceil((np.nanmax(finite_times) -
                         np.nanmin(finite_times))/binsizejd) + 1)

    minjd = np.nanmin(finite_times)
    jdbins = [(minjd + x*binsizejd) for x in range(nbins)]

    # make a KD-tree on the JDs so we can do fast distance calculations.  we
    # need to add a bogus y coord to make this a problem that KD-trees can
    # solve.
    time_coords = np.array([[x,1.0] for x in finite_times])
    jdtree = cKDTree(time_coords)
    binned_finite_timeseries_indices = []

    collected_binned_mags = {}

    for jd in jdbins:
        # find all bin indices close to within binsizejd of this point
        # using the cKDTree query. we use the p-norm = 1 (I think this
        # means straight-up pairwise distance? FIXME: check this)
        bin_indices = jdtree.query_ball_point(np.array([jd,1.0]),
                                              binsizejd/2.0, p=1.0)

        # if the bin_indices have already been collected, then we're
        # done with this bin, move to the next one. if they haven't,
        # then this is the start of a new bin.
        if (bin_indices not in binned_finite_timeseries_indices and
            len(bin_indices) >= minbinelems):

            binned_finite_timeseries_indices.append(bin_indices)

    # convert to ndarrays
    binned_finite_timeseries_indices = [np.array(x) for x in
                                        binned_finite_timeseries_indices]

    collected_binned_mags['jdbins_indices'] = binned_finite_timeseries_indices
    collected_binned_mags['jdbins'] = jdbins
    collected_binned_mags['nbins'] = len(binned_finite_timeseries_indices)

    # collect the finite_times
    binned_jd = np.array([np.median(finite_times[x])
                          for x in binned_finite_timeseries_indices])
    collected_binned_mags['binnedtimes'] = binned_jd
    collected_binned_mags['binsize'] = binsize

    # median bin the magnitudes according to the calculated indices
    collected_binned_mags['binnedmags'] = (
        np.array([np.median(finite_mags[x])
                  for x in binned_finite_timeseries_indices])
    )

    return collected_binned_mags

def time_bin_magseries_with_errs(times, mags, errs,
                                 binsize=540.0,
                                 minbinelems=7):
    '''Bins the given mag/flux time-series in time using the bin size given.

    Parameters
    ----------

    times,mags,errs : np.array
        The magnitude/flux time-series and associated measurement errors to bin
        in time. Non-finite elements will be removed from these arrays. At least
        10 elements in each array are required for this function to operate.

    binsize : float
        The bin size to use to group together measurements closer than this
        amount in time. This is in seconds.

    minbinelems : int
        The minimum number of elements required per bin to include it in the
        output.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'jdbin_indices': a list of the index arrays into the nan-filtered
                              input arrays per each bin,
             'jdbins': list of bin boundaries for each bin,
             'nbins': the number of bins generated,
             'binnedtimes': the time values associated with each time bin;
                            this is the median of the times in each bin,
             'binnedmags': the mag/flux values associated with each time bin;
                           this is the median of the mags/fluxes in each bin,
             'binnederrs': the err values associated with each time bin;
                           this is the median of the errs in each bin}

    '''

    # check if the input arrays are ok
    if not(times.shape and mags.shape and errs.shape and
           len(times) > 9 and len(mags) > 9):

        LOGERROR("input time/mag/err arrays don't have enough elements")
        return

    # find all the finite values of the magnitudes and times
    finiteind = np.isfinite(mags) & np.isfinite(times) & np.isfinite(errs)
    finite_times = times[finiteind]
    finite_mags = mags[finiteind]
    finite_errs = errs[finiteind]

    # convert binsize in seconds to JD units
    binsizejd = binsize/(86400.0)
    nbins = int(np.ceil((np.nanmax(finite_times) -
                         np.nanmin(finite_times))/binsizejd) + 1)

    minjd = np.nanmin(finite_times)
    jdbins = [(minjd + x*binsizejd) for x in range(nbins)]

    # make a KD-tree on the JDs so we can do fast distance calculations.  we
    # need to add a bogus y coord to make this a problem that KD-trees can
    # solve.
    time_coords = np.array([[x,1.0] for x in finite_times])
    jdtree = cKDTree(time_coords)
    binned_finite_timeseries_indices = []

    collected_binned_mags = {}

    for jd in jdbins:

        # find all bin indices close to within binsize of this point using the
        # cKDTree query. we use the p-norm = 1 for pairwise Euclidean distance.
        bin_indices = jdtree.query_ball_point(np.array([jd,1.0]),
                                              binsizejd/2.0, p=1.0)

        # if the bin_indices have already been collected, then we're
        # done with this bin, move to the next one. if they haven't,
        # then this is the start of a new bin.
        if (bin_indices not in binned_finite_timeseries_indices and
            len(bin_indices) >= minbinelems):

            binned_finite_timeseries_indices.append(bin_indices)

    # convert to ndarrays
    binned_finite_timeseries_indices = [np.array(x) for x in
                                        binned_finite_timeseries_indices]

    collected_binned_mags['jdbins_indices'] = binned_finite_timeseries_indices
    collected_binned_mags['jdbins'] = np.array(jdbins)
    collected_binned_mags['nbins'] = len(binned_finite_timeseries_indices)

    # collect the finite_times
    binned_jd = np.array([np.median(finite_times[x])
                          for x in binned_finite_timeseries_indices])
    collected_binned_mags['binnedtimes'] = binned_jd
    collected_binned_mags['binsize'] = binsize

    # median bin the magnitudes according to the calculated indices
    collected_binned_mags['binnedmags'] = (
        np.array([np.median(finite_mags[x])
                  for x in binned_finite_timeseries_indices])
    )

    # FIXME: calculate the error in the median-binned magnitude correctly
    # for now, just take the median of the errors in this bin
    collected_binned_mags['binnederrs'] = (
        np.array([np.median(finite_errs[x])
                  for x in binned_finite_timeseries_indices])
    )

    return collected_binned_mags

def phase_bin_magseries(phases, mags,
                        binsize=0.005,
                        minbinelems=7):
    '''Bins a phased magnitude/flux time-series using the bin size provided.

    Parameters
    ----------

    phases,mags : np.array
        The phased magnitude/flux time-series to bin in phase. Non-finite
        elements will be removed from these arrays. At least 10 elements in each
        array are required for this function to operate.

    binsize : float
        The bin size to use to group together measurements closer than this
        amount in phase. This is in units of phase.

    minbinelems : int
        The minimum number of elements required per bin to include it in the
        output.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'phasebin_indices': a list of the index arrays into the
                                 nan-filtered input arrays per each bin,
             'phasebins': list of bin boundaries for each bin,
             'nbins': the number of bins generated,
             'binnedphases': the phase values associated with each phase bin;
                            this is the median of the phase value in each bin,
             'binnedmags': the mag/flux values associated with each phase bin;
                           this is the median of the mags/fluxes in each bin}

    '''

    # check if the input arrays are ok
    if not(phases.shape and mags.shape and len(phases) > 10 and len(mags) > 10):

        LOGERROR("input time/mag arrays don't have enough elements")
        return

    # find all the finite values of the magnitudes and phases
    finiteind = np.isfinite(mags) & np.isfinite(phases)
    finite_phases = phases[finiteind]
    finite_mags = mags[finiteind]

    nbins = int(np.ceil((np.nanmax(finite_phases) -
                         np.nanmin(finite_phases))/binsize) + 1)

    minphase = np.nanmin(finite_phases)
    phasebins = [(minphase + x*binsize) for x in range(nbins)]

    # make a KD-tree on the PHASEs so we can do fast distance calculations.  we
    # need to add a bogus y coord to make this a problem that KD-trees can
    # solve.
    time_coords = np.array([[x,1.0] for x in finite_phases])
    phasetree = cKDTree(time_coords)
    binned_finite_phaseseries_indices = []

    collected_binned_mags = {}

    for phase in phasebins:

        # find all bin indices close to within binsize of this point using the
        # cKDTree query. we use the p-norm = 1 for pairwise Euclidean distance.
        bin_indices = phasetree.query_ball_point(np.array([phase,1.0]),
                                                 binsize/2.0, p=1.0)

        # if the bin_indices have already been collected, then we're
        # done with this bin, move to the next one. if they haven't,
        # then this is the start of a new bin.
        if (bin_indices not in binned_finite_phaseseries_indices and
            len(bin_indices) >= minbinelems):

            binned_finite_phaseseries_indices.append(bin_indices)

    # convert to ndarrays
    binned_finite_phaseseries_indices = [np.array(x) for x in
                                         binned_finite_phaseseries_indices]

    collected_binned_mags['phasebins_indices'] = (
        binned_finite_phaseseries_indices
    )
    collected_binned_mags['phasebins'] = phasebins
    collected_binned_mags['nbins'] = len(binned_finite_phaseseries_indices)

    # collect the finite_phases
    binned_phase = np.array([np.median(finite_phases[x])
                             for x in binned_finite_phaseseries_indices])
    collected_binned_mags['binnedphases'] = binned_phase
    collected_binned_mags['binsize'] = binsize

    # median bin the magnitudes according to the calculated indices
    collected_binned_mags['binnedmags'] = (
        np.array([np.median(finite_mags[x])
                  for x in binned_finite_phaseseries_indices])
    )

    return collected_binned_mags

def phase_bin_magseries_with_errs(phases, mags, errs,
                                  binsize=0.005,
                                  minbinelems=7,
                                  weights=None):
    '''Bins a phased magnitude/flux time-series using the bin size provided.

    Parameters
    ----------

    phases,mags,errs : np.array
        The phased magnitude/flux time-series and associated errs to bin in
        phase. Non-finite elements will be removed from these arrays. At least
        10 elements in each array are required for this function to operate.

    binsize : float
        The bin size to use to group together measurements closer than this
        amount in phase. This is in units of phase.

    minbinelems : int
        The minimum number of elements required per bin to include it in the
        output.

    weights : np.array or None
        Optional weight vector to be applied during binning. If if is passed,
        `np.average` is used to bin, rather than `np.median`. A good choice
        would be to pass ``weights=1/errs**2``, to weight by the inverse variance.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'phasebin_indices': a list of the index arrays into the
                                 nan-filtered input arrays per each bin,
             'phasebins': list of bin boundaries for each bin,
             'nbins': the number of bins generated,
             'binnedphases': the phase values associated with each phase bin;
                            this is the median of the phase value in each bin,
             'binnedmags': the mag/flux values associated with each phase bin;
                           this is the median of the mags/fluxes in each bin,
             'binnederrs': the err values associated with each phase bin;
                           this is the median of the errs in each bin}

    '''

    # check if the input arrays are ok
    if not(phases.shape and mags.shape and len(phases) > 10 and len(mags) > 10):

        LOGERROR("input time/mag arrays don't have enough elements")
        return

    # find all the finite values of the magnitudes and phases
    finiteind = np.isfinite(mags) & np.isfinite(phases) & np.isfinite(errs)
    finite_phases = phases[finiteind]
    finite_mags = mags[finiteind]
    finite_errs = errs[finiteind]
    if weights is not None and len(weights) > 10:
        finite_weights = weights[finiteind]
    else:
        finite_weights = None

    nbins = int(np.ceil((np.nanmax(finite_phases) -
                         np.nanmin(finite_phases))/binsize) + 1)

    minphase = np.nanmin(finite_phases)
    phasebins = [(minphase + x*binsize) for x in range(nbins)]

    # make a KD-tree on the PHASEs so we can do fast distance calculations.  we
    # need to add a bogus y coord to make this a problem that KD-trees can
    # solve.
    time_coords = np.array([[x,1.0] for x in finite_phases])
    phasetree = cKDTree(time_coords)
    binned_finite_phaseseries_indices = []

    collected_binned_mags = {}

    for phase in phasebins:

        # find all bin indices close to within binsize of this point using the
        # cKDTree query. we use the p-norm = 1 for pairwise Euclidean distance.
        bin_indices = phasetree.query_ball_point(np.array([phase,1.0]),
                                                 binsize/2.0, p=1.0)

        # if the bin_indices have already been collected, then we're
        # done with this bin, move to the next one. if they haven't,
        # then this is the start of a new bin.
        if (bin_indices not in binned_finite_phaseseries_indices and
            len(bin_indices) >= minbinelems):

            binned_finite_phaseseries_indices.append(bin_indices)

    # convert to ndarrays
    binned_finite_phaseseries_indices = [np.array(x) for x in
                                         binned_finite_phaseseries_indices]

    collected_binned_mags['phasebins_indices'] = (
        binned_finite_phaseseries_indices
    )
    collected_binned_mags['phasebins'] = phasebins
    collected_binned_mags['nbins'] = len(binned_finite_phaseseries_indices)

    # collect the finite_phases
    binned_phase = np.array([np.median(finite_phases[x])
                             for x in binned_finite_phaseseries_indices])
    collected_binned_mags['binnedphases'] = binned_phase
    collected_binned_mags['binsize'] = binsize

    # median bin the magnitudes according to the calculated indices
    if finite_weights is None:
        collected_binned_mags['binnedmags'] = (
            np.array([np.median(finite_mags[x])
                      for x in binned_finite_phaseseries_indices])
        )
    else:
        collected_binned_mags['binnedmags'] = (
            np.array([np.average(finite_mags[x], weights=finite_weights[x])
                      for x in binned_finite_phaseseries_indices])
        )

    collected_binned_mags['binnederrs'] = (
        np.array([np.median(finite_errs[x])
                  for x in binned_finite_phaseseries_indices])
    )

    return collected_binned_mags

#############################
## FILLING TIMESERIES GAPS ##
#############################

def fill_magseries_gaps(times, mags, errs,
                        fillgaps=0.0,
                        sigclip=3.0,
                        magsarefluxes=False,
                        filterwindow=11,
                        forcetimebin=None,
                        verbose=True):
    '''This fills in gaps in a light curve.

    This is mainly intended for use in ACF period-finding, but maybe useful
    otherwise (i.e. when we figure out ARMA stuff for LCs). The main steps here
    are:

    - normalize the light curve to zero
    - remove giant outliers
    - interpolate gaps in the light curve
      (since ACF requires evenly spaced sampling)

    From McQuillan+ 2013a (https://doi.org/10.1093/mnras/stt536):

        "The ACF calculation requires the light curves to be regularly sampled
        and normalized to zero. We divided the flux in each quarter by its
        median and subtracted unity. Gaps in the light curve longer than the
        Kepler long cadence were filled using linear interpolation with added
        white Gaussian noise. This noise level was estimated using the variance
        of the residuals following subtraction of a smoothed version of the
        flux. To smooth the flux, we applied an iterative non-linear filter
        which consists of a median filter followed by a boxcar filter, both with
        11-point windows, with iterative 3σ clipping of outliers."

    Parameters
    ----------

    times,mags,errs : np.array
        The magnitude/flux time-series and associated measurement errors to
        operate on. Non-finite elements will be removed from these arrays. At
        least 10 elements in each array are required for this function to
        operate.

    fillgaps : {'noiselevel', 'nan'} or float
        If `fillgap='noiselevel'`, fills the gaps with the noise level obtained
        via the procedure above. If `fillgaps='nan'`, fills the gaps with
        `np.nan`. Otherwise, if `fillgaps` is a float, will use that value to
        fill the gaps. The default is to fill the gaps with 0.0 (as in
        McQuillan+ 2014) to "...prevent them contributing to the ACF".

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    magsareflux : bool
        True if your "mags" are in fact fluxes, i.e. if "fainter" corresponds to
        `mags` getting smaller.

    filterwindow : int
        The number of time-series points to include in the Savitsky-Golay filter
        operation when smoothing the light curve. This should be an odd integer.

    forcetimebin : float or None
        If `forcetimebin` is a float, this value will be used to generate the
        interpolated time series, effectively binning the light curve to this
        cadence. If `forcetimebin` is None, the mode of the gaps (the forward
        difference between successive time values in `times`) in the provided
        light curve will be used as the effective cadence. NOTE: `forcetimebin`
        must be in the same units as `times`, e.g. if times are JD then
        `forcetimebin` must be in days as well

    verbose : bool
        If this is True, will indicate progress at various stages in the
        operation.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'itimes': the interpolated time values after gap-filling,
             'imags': the interpolated mag/flux values after gap-filling,
             'ierrs': the interpolated mag/flux values after gap-filling,
             'cadence': the cadence of the output mag/flux time-series}

    '''

    # remove nans
    finind = np.isfinite(times) & np.isfinite(mags) & np.isfinite(errs)
    ftimes, fmags, ferrs = times[finind], mags[finind], errs[finind]

    # remove zero errs
    nzind = np.nonzero(ferrs)
    ftimes, fmags, ferrs = ftimes[nzind], fmags[nzind], ferrs[nzind]

    # sigma-clip
    stimes, smags, serrs = sigclip_magseries(ftimes, fmags, ferrs,
                                             magsarefluxes=magsarefluxes,
                                             sigclip=sigclip)

    # normalize to zero
    if magsarefluxes:
        smags = smags / np.median(smags) - 1.0
    else:
        smags = smags - np.median(smags)

    if isinstance(fillgaps, float):

        gapfiller = fillgaps

    elif isinstance(fillgaps, str) and fillgaps == 'noiselevel':

        # figure out the gaussian noise level by subtracting a Savitsky-Golay
        # filtered version of the light curve
        smoothed = smags - savgol_filter(smags, filterwindow, 2)
        noiselevel = 1.483 * np.median(np.abs(smoothed - np.median(smoothed)))
        gapfiller = noiselevel

    elif isinstance(fillgaps, str) and fillgaps == 'nan':

        gapfiller = np.nan

    # figure out the gap size and where to interpolate. we do this by figuring
    # out the most common gap (this should be the cadence). to do this, we need
    # to calculate the mode of the gap distribution.

    # get the gaps
    gaps = np.diff(stimes)

    # just use scipy.stats.mode instead of our hacked together nonsense earlier.
    gapmoderes = scipy.stats.mode(gaps)
    gapmode = gapmoderes[0].item()

    LOGINFO('auto-cadence for mag series: %.5f' % gapmode)

    # sort the gaps
    if forcetimebin:
        LOGWARNING('forcetimebin is set, forcing cadence to %.5f' %
                   forcetimebin)
        gapmode = forcetimebin

    if gapmode == 0.0:
        LOGERROR('the smallest cadence of this light curve appears to be 0.0, '
                 'the automatic cadence finder probably failed. '
                 'try setting forcetimebin?')
        return None

    starttime, endtime = np.min(stimes), np.max(stimes)
    ntimes = int(np.ceil((endtime - starttime)/gapmode) + 1)
    if verbose:
        LOGINFO('generating new time series with %s measurements' % ntimes)

    # first, generate the full time series
    interpolated_times = np.linspace(starttime, endtime, ntimes)
    interpolated_mags = np.full_like(interpolated_times, gapfiller)
    interpolated_errs = np.full_like(interpolated_times, gapfiller)

    for ind, itime in enumerate(interpolated_times[:-1]):

        nextitime = itime + gapmode
        # find the mags between this and the next time bin
        itimeind = np.where((stimes > itime) & (stimes < nextitime))

        # if there's more than one elem in this time bin, median them
        if itimeind[0].size > 1:

            interpolated_mags[ind] = np.median(smags[itimeind[0]])
            interpolated_errs[ind] = np.median(serrs[itimeind[0]])

        # otherwise, if there's only one elem in this time bin, take it
        elif itimeind[0].size == 1:

            interpolated_mags[ind] = smags[itimeind[0]]
            interpolated_errs[ind] = serrs[itimeind[0]]

    return {'itimes':interpolated_times,
            'imags':interpolated_mags,
            'ierrs':interpolated_errs,
            'cadence':gapmode}