/*
 * Copyright 2015, Yahoo Inc.
 * Copyrights licensed under the GPL License.
 * See the accompanying LICENSE file for terms.
 */

// Provides the auto-sensitivity solution for EGADS
// using the bootstrapping framework.

package com.yahoo.egads.utilities;
import java.util.ArrayList;
import org.apache.commons.lang.ArrayUtils;
import java.util.Arrays;
import java.util.Collections;

public class AutoSensitivity {
    static org.apache.logging.log4j.Logger logger = org.apache.logging.log4j.LogManager.getLogger(AutoSensitivity.class.getName());
    // Computes sensitivity based on the density distribution.
    // Assumes that anomalies constitute at most 5% of the data.
    public static Float getLowDensitySensitivity(Float[] data, float sDAutoSensitivy, float amntAutoSensitivity) {
        Float toReturn = Float.POSITIVE_INFINITY;
        Arrays.sort(data, Collections.reverseOrder());
        while (data.length > 0) {     
        	
            ArrayList fData = new ArrayList();
            fData.add(data[0]);
            data = ((Float[]) ArrayUtils.remove(data, 0));
            
            Float centroid = (float) fData.get(0);
            Float maxDelta = (float) sDAutoSensitivy * StatsUtils.getSD(data, StatsUtils.getMean(data));
            
            logger.debug("AutoSensitivity: Adding: " + fData.get(0) + " SD: " + maxDelta);
            
            // Add points while it's in the same cluster or not part of the other cluster.
            String localDebug = null;
            while (data.length > 0 &&
                   (centroid - data[0]) <= ((float) (maxDelta))) {
            	float maxDeltaInit = maxDelta;
                fData.add(data[0]);
                data = ((Float[]) ArrayUtils.remove(data, 0));
                Float[] tmp = new Float[fData.size()];
                tmp = fData.toArray(tmp);
                centroid = StatsUtils.getMean(tmp);
                
                if (data.length > 0) {
                    Float sdOtherCluster = (float) StatsUtils.getSD(data, StatsUtils.getMean(data));
                    maxDelta = sDAutoSensitivy * sdOtherCluster;
                    logger.debug("AutoSensitivity: Adding: " + data[0] + " SD: " + maxDeltaInit + " SD': " + maxDelta);
                }
            }
            if (data.length > 0) {
                logger.debug("AutoSensitivity: Next Point I would have added is " + data[0]);
            }
                        
            if (((double) fData.size() / (double) data.length) > amntAutoSensitivity) {
                // Cannot do anomaly detection.
                logger.debug("AutoSensitivity: Returning " + toReturn + " data size: " + data.length + " fData.size: " + fData.size());
                return toReturn;
            }

            toReturn = fData.get(fData.size() - 1);
            logger.debug("AutoSensitivity: Updating toReturn:  " + toReturn + " SD: " + maxDelta);
            return toReturn;
        }
        return toReturn;
    }
    
    // Uses the simple KSigma rule to get the anoamly sensitivity.
    // Assumes that we have a normal distribution.
    public static Float getKSigmaSensitivity(Float[] data, float sDAutoSensitivity) {
         Float mean = StatsUtils.getMean(data);
         Float sd = StatsUtils.getSD(data, mean);
         return (mean + (sd * sDAutoSensitivity));
    }

    // Uses the mean as the base to find the static threshold.
    public static Float[] getAdaptiveKSigmaSensitivity(Float[] data, float amntAutoSens) {
         Float mean = StatsUtils.getMean(data);
         Float sd = StatsUtils.getSD(data, mean);
         if (sd == (float) 0.0) {
             sd = (float) 1.0;
         }
         Float[] ret = null;
         float k = (float) 1;
         float incr = (float) 1;
         
         Float max = null;
         Float min = null;
         float thresh = mean + Math.abs(sd * k);
         int howMany = howManyGreater(data, thresh);

         while (((float) howMany / (float) data.length) > amntAutoSens) {
             k += incr;
             thresh = mean + Math.abs(sd * k);
             howMany = howManyGreater(data, thresh);
         } 
         if (((float) howMany / (float) data.length) <= amntAutoSens) {
             max = thresh;
         } 
         k = 1;
         thresh = mean - Math.abs(sd * k);
         howMany = howManyLess(data, thresh);
        
         while (((float) howMany / (float) data.length) > amntAutoSens) {
             k += incr;
             thresh = mean - Math.abs(sd * k);
             howMany = howManyLess(data, thresh);
         } 
         if (((float) howMany / (float) data.length) <= amntAutoSens) {
           min = thresh;
         }
         ret = new Float[]{max, min};
         return ret; 
    }
    
    // Uses the max/min as the base to find the static threshold.
    public static Float[] getAdaptiveMaxMinSigmaSensitivity(Float[] data, float amntAutoSens, float k) {
    	Arrays.sort(data);
    	Float mean = StatsUtils.getMean(data);
        Float sd = StatsUtils.getSD(data, mean);
        if (sd == (float) 0.0) {
            sd = (float) 1.0;
        }
        Float[] ret = null;
        
        Float max = null;
        Float min = null;
        int i = 0;
        float thresh = data[i] + Math.abs(sd * k);
        int howMany = howManyLess(data, thresh);
        while (((float) howMany / (float) data.length) <= amntAutoSens) {
        	min = thresh;
        	i++;
            thresh = data[i] + Math.abs(sd * k);
            howMany = howManyLess(data, thresh);
        } 
        i = data.length - 1;
        thresh = data[i] - Math.abs(sd * k);
        howMany = howManyGreater(data, thresh);
        while (((float) howMany / (float) data.length) <= amntAutoSens) {
        	max = thresh;
            i--;
            thresh = data[i] - Math.abs(sd * k);
            howMany = howManyGreater(data, thresh);
        } 

        ret = new Float[]{max, min};
        return ret; 
   }

    private static int howManyGreater(Float[] data, Float value) {
        int numgreater = 0;
        for (Float f : data) {
            if (value <= f) {
                numgreater++;
            }
        }
        return numgreater;
    }

    private static int howManyLess(Float[] data, Float value) {
        int numless = 0;
        for (Float f : data) {
            if (value >= f) {
                numless++;
            }
        }
        return numless;
    }
}

 

Ads help maintain this website.