package brn.analysis.dump;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import jist.swans.Constants;
import jist.swans.mac.MacDcfMessage;
import jist.swans.misc.MessageBytes;
import jist.swans.misc.Pickle;

import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;

import brn.analysis.Statistics;
import brn.sim.data.Line;
import brn.sim.data.TimeLine;
import brn.sim.data.XplotSerializer;
import brn.sim.data.dump.WiresharkDump;
import brn.sim.data.dump.WiresharkMessage;
import brn.swans.radio.biterrormodels.BitErrorMasks;
import brn.swans.radio.biterrormodels.BitErrorMask.SpecifiedErrors;

public class BitErrorAggregator {

  public static final Logger log = Logger.getLogger(BitErrorAggregator.class
      .getName());

  protected static final String[] rateString = {
    "1Mbps",
    "2Mbps",
    "5Mbps",
    "11Mbps",
    "6Mbps",
    "9Mbps",
    "12Mbps",
    "18Mbps",
    "24Mbps",
    "36Mbps",
    "48Mbps",
    "54Mbps"
  };

  protected static final String[] rateColor = {
    "red", // 1
    "blue", // 2
    "yellow", // 5
    "green", // 11
    "white", // 6
    "magenta", // 9
    "orange", // 12
    "purple", // 18
    "red", // 24
    "blue", // 36
    "yellow", // 48
    "green", // 54
  };

  protected static final String[] rateShape = {
    "dot", // 1
    "dot", // 2
    "dot", // 5
    "dot", // 11
    "x", // 6
    "x", // 9
    "x", // 12
    "x", // 18
    "x", // 24
    "x", // 36
    "x", // 48
    "x", // 54
  };


  protected static final int MTU = 4000;
  protected static final int DATA_OFFSET = AthdescHeader.HEADER_SIZE +
      MacDcfMessage.Data.getHeaderSize();
  protected static final int AUTOCORRELLATION_SIZE = 4000;
  protected static final int CONDITIONAL_LENGTH = 12000;
  protected static final double CONDITIONAL_THRES = .05;

  protected static final String[] statusString = {
    "ok", "crc", "phy", "all" };

  public static class IEEE80211Header {
    public static short getSeqId(byte[] msg, int offset) {
      if (offset + 24 > msg.length)
        return -1;
      return (short)(Pickle.arrayToUShort(msg, offset + 22) >> 4);
    }
  }

  private static class BaseCtx {
    int pktPerRate[] = new int[rateString.length];
    int pktProcessedPerRate[] = new int[rateString.length];

    int badBytesPerPosRate[][] = new int[MTU][rateString.length];
    int badBitsPerPosRate[][] = new int[MTU*8][rateString.length];
    int missingBytesPerPosRate[][] = new int[MTU][rateString.length];
    int pktPerBadBitsRate[][] = new int[MTU*8][rateString.length];

    // Note: all values are summed up and divided by pktPerRate at the end
    double bitAutoCorrelation[][] = new double[rateString.length][AUTOCORRELLATION_SIZE+1];
    // TODO remove test code -- begin
//    double bitAutoCorrelation2[][] = new double[rateString.length][AUTOCORRELLATION_SIZE+1];
    // TODO remove test code -- end
    double[][] bitAllanVarCuml = new double[rateString.length][];
    int[] bitAllanVarCount = new int[rateString.length]; // No. entries in bitAllanVarCuml

    double pdrRatioDist[][] = new double[4][41];
  }

  private static class GlobalCtx extends BaseCtx {
    StringBuilder builder = new StringBuilder("node\trate\trate\tcorr_arrived\tcorr_correct\n");

    double pktArrivedAutoCorrelation[][] = new double[rateString.length][2*AUTOCORRELLATION_SIZE];
    double pktCorrectAutoCorrelation[][] = new double[rateString.length][2*AUTOCORRELLATION_SIZE];

//    double[][] pktArrivedAllanVar = new double[rateString.length][];
//    double[][] pktCorrectAllanVar = new double[rateString.length][];
  }

  private static class LinkCtx extends BaseCtx {
    private WiresharkDump dump = null;
    private String fileName = null;

    Map mapTimestampToPacket = new HashMap();

    List pktArrivedCorrect[] = new ArrayList[rateString.length];
    List pktArrived[] = new ArrayList[rateString.length];
    double[][][] condLow = new double[rateString.length][2][CONDITIONAL_LENGTH]; 
    double[][][] condHigh = new double[rateString.length][2][CONDITIONAL_LENGTH]; 
    double[][] correctRunLength = new double[rateString.length][MTU*8];

    TimeLine[][] tlPDR = new TimeLine[statusString.length][rateString.length];

    public LinkCtx() {
      for (int i = 0; i < pktArrivedCorrect.length; i++) {
        pktArrivedCorrect[i] = new ArrayList();
        pktArrived[i] = new ArrayList();
      }
      for (int i = 0; i < statusString.length; i++)
        for (int j = 0; j < rateString.length; j++)
          tlPDR[i][j] = new TimeLine("PDR " + rateString[j] + " " + statusString[i]);
    }
  }

  private static class ReferenceCtx {
    private WiresharkDump dump = null;
    private String fileName = null;

    int pktRate[] = new int[rateString.length];
  }

  private LinkCtx link;

  private ReferenceCtx ref;

  private GlobalCtx glob;

  private boolean createMasks;

  private boolean doStatistics;
  
  private boolean doSave;

  private BitErrorMasks errorMasks;

  public BitErrorMasks getErrorMasks() {
    return errorMasks;
  }

  /**
   * Constructor.
   *
   * @param fileName file name of the reference stream.
   * @throws IOException
   */
  public BitErrorAggregator(String fileName) throws IOException {
    this(fileName, false, false, true);
  }
  
  public BitErrorAggregator(String fileName, boolean doStatistics, boolean createMasks, boolean doSave) {
    this.createMasks = createMasks;
    this.doStatistics = doStatistics;
    this.doSave = doSave;
    this.link = null;
    this.ref = new ReferenceCtx();
    ref.fileName = fileName;

    this.glob = new GlobalCtx();
    if (createMasks)
      errorMasks = new BitErrorMasks();
    else
      errorMasks = null;
    if (createMasks && doStatistics)
      log.warn("Creating masks and statistics. Prepare to see the results in your next life.");
  }

  /**
   * Analyze an input dump file.
   *
   * @param fileName
   * @throws IOException
   */
  public void analyze(String fileName) throws IOException {
    log.info("Analyzing " + fileName);

    this.link = new LinkCtx();

    link.fileName = fileName;
    link.dump = new WiresharkDump();
    link.dump.open(fileName);

    // Iterate through the dump, put all packets in a hash
    for (WiresharkMessage msg = link.dump.read(); null != msg; msg = link.dump.read()) {
      analyzePacket(msg);
    }

    // compare against reference
    analyzeRef(ref.fileName);

    if (doSave)
      save();

    if (doStatistics) {
      for (int rate = 0; rate < rateString.length; rate++) {
        glob.pktPerRate[rate] += link.pktPerRate[rate];
        glob.pktProcessedPerRate[rate] += link.pktProcessedPerRate[rate];

        for (int i = 0; i < MTU*8; i++) {
          glob.badBitsPerPosRate[i][rate] += link.badBitsPerPosRate[i][rate];
          glob.pktPerBadBitsRate[i][rate] += link.pktPerBadBitsRate[i][rate];
        }
        for (int i = 0; i < MTU; i++) {
          glob.badBytesPerPosRate[i][rate] += link.badBytesPerPosRate[i][rate];
          glob.missingBytesPerPosRate[i][rate] += link.missingBytesPerPosRate[i][rate];
        }
      }
    }

    link = null;
  }


  
  
  /**
   * Put all packets into a hash map.
   *
   * @param curr
   */
  private void analyzePacket(WiresharkMessage curr) {
    byte[] currBytes = ((MessageBytes) curr.getPayload()).getBytes();
    int idx = AthdescHeader.getExtRxRateIdx(currBytes, 0);
    link.pktPerRate[idx]++;

    if (currBytes.length <= DATA_OFFSET + 8)
      return;

    long currTime = Pickle.arrayToInteger(currBytes, DATA_OFFSET);
    currTime = currTime * Constants.SECOND; // in us!
    currTime += Pickle.arrayToInteger(currBytes, DATA_OFFSET + 4) * Constants.MICRO_SECOND;
    link.mapTimestampToPacket.put(new Long(currTime), curr);
  }

  /**
   * Analyze the reference file.
   *
   * @param fileName
   * @throws IOException
   */
  private void analyzeRef(String fileName) throws IOException {
    log.info("Analyzing " + fileName);

    ref = new ReferenceCtx();

    ref.fileName = fileName;
    ref.dump = new WiresharkDump();
    ref.dump.open(fileName);

    // Iterate through the dump
    // iterate through all ref packets
    WiresharkMessage refMsg = null;
    for(refMsg = ref.dump.read(); refMsg != null; refMsg = ref.dump.read()) {
      if (createMasks)
        analyzeRefPacketForMasks(refMsg);
      else
        analyzeRefPacket(refMsg);
    }
  }

  private void analyzeRefPacketForMasks(WiresharkMessage msg) {
    byte[] refBytes = ((MessageBytes) msg.getPayload()).getBytes();
    int idx = AthdescHeader.getExtTxRateIdx(refBytes, 0);

    ref.pktRate[idx] ++;

    // compare in-packet timestamps
    long refTime = Pickle.arrayToInteger(refBytes, DATA_OFFSET);
    refTime = refTime * Constants.SECOND; // in us!
    refTime += Pickle.arrayToInteger(refBytes, DATA_OFFSET + 4) * Constants.MICRO_SECOND;
    WiresharkMessage curr = (WiresharkMessage) link.mapTimestampToPacket.get(new Long(refTime));
    if (null == curr) {
      link.tlPDR[0][idx].add(refTime, .0);
      link.tlPDR[1][idx].add(refTime, .0);
      link.tlPDR[2][idx].add(refTime, .0);
      link.tlPDR[3][idx].add(refTime, .0);
      link.pktArrivedCorrect[idx].add(new Double(0));
      link.pktArrived[idx].add(new Double(0));
      return;
    }

    link.pktProcessedPerRate[idx]++;
    int bitErrors = 0;
    byte[] currBytes = ((MessageBytes) curr.getPayload()).getBytes();
    int bitLength = curr.getPayload().getSize() * 8;
    SpecifiedErrors errors = new SpecifiedErrors(bitLength);

    byte status = AthdescHeader.getExtRxStatus(currBytes, 0);
    byte rssi = AthdescHeader.getExtRxRssi(currBytes, 0);
    byte rate = AthdescHeader.getExtRxRateMbps(currBytes, 0);
    link.tlPDR[0][idx].add(refTime, (0 == status) ? 1. : .0);
    link.tlPDR[1][idx].add(refTime, (1 == status) ? 1. : .0);
    link.tlPDR[2][idx].add(refTime, (2 == status) ? 1. : .0);
    link.tlPDR[3][idx].add(refTime, 1.);

    // generate error mask (using xor), ignore athdesc header
    for (int i = AthdescHeader.HEADER_SIZE; i < refBytes.length; i++) {
      if (i == AthdescHeader.HEADER_SIZE + 22
          ||i == AthdescHeader.HEADER_SIZE + 23)
        continue; // skip seq id, it is not set in tx feedback

      if (i < currBytes.length) {
        refBytes[i] ^= currBytes[i];

        if (0 != refBytes[i]) {
          link.badBytesPerPosRate[i][idx]++;
          for (int j = 0; j < Byte.SIZE; j++)
            if (0 != (refBytes[i] & (1 << j))) {
              link.badBitsPerPosRate[j + i*Byte.SIZE][idx]++;
              errors.setError(j + i*Byte.SIZE);
              bitErrors++;
            }
        }
      } else { // missing part
        link.missingBytesPerPosRate[i][idx]++;
      }
    }
    link.pktArrived[idx].add(new Double(1));
    link.pktArrivedCorrect[idx].add(new Double(0 == bitErrors ? 1 : 0));

    // store number of bit errors per packet
    link.pktPerBadBitsRate[bitErrors][idx]++;

    // calculate the auto correlation and allan deviation
    // this is unbelievably slow - don't switch on masks and statistics at the same time
    if (0 != bitErrors && doStatistics) {
      double[] correlation = new double[AUTOCORRELLATION_SIZE];
      Statistics.autoCorrelate(errors, bitLength, correlation, 0.01);
      for (int i = 0; i < correlation.length; i++) {
        link.bitAutoCorrelation[idx][i] += correlation[i];
        glob.bitAutoCorrelation[idx][i] += link.bitAutoCorrelation[idx][i];
      }

      double[] allanDev = Statistics.allanVariance(errors, bitLength);
      if (null == link.bitAllanVarCuml[idx])
        link.bitAllanVarCuml[idx] = new double[allanDev.length];
      else if (link.bitAllanVarCuml[idx].length < allanDev.length) {
        double d[] = link.bitAllanVarCuml[idx];
        link.bitAllanVarCuml[idx] = new double[allanDev.length];
        System.arraycopy(d, 0, link.bitAllanVarCuml[idx], 0, d.length);
      }
      if (null == glob.bitAllanVarCuml[idx])
        glob.bitAllanVarCuml[idx] = new double[allanDev.length];
      else if (glob.bitAllanVarCuml[idx].length < allanDev.length) {
        double d[] = glob.bitAllanVarCuml[idx];
        glob.bitAllanVarCuml[idx] = new double[allanDev.length];
        System.arraycopy(d, 0, glob.bitAllanVarCuml[idx], 0, d.length);
      }
      for (int i = 0; i < allanDev.length; i++) {
        link.bitAllanVarCuml[idx][i] += allanDev[i]; link.bitAllanVarCount[idx]++;
        glob.bitAllanVarCuml[idx][i] += allanDev[i]; glob.bitAllanVarCount[idx]++;
      }
    }
      
    errorMasks.insert(errors, bitLength, rssi, rate);
 
  }
  
  /**
   * Compares the reference against the received packet.
   *
   */
  private void analyzeRefPacket(WiresharkMessage msg) {
    byte[] refBytes = ((MessageBytes) msg.getPayload()).getBytes();
    int idx = AthdescHeader.getExtTxRateIdx(refBytes, 0);

    ref.pktRate[idx] ++;

    // compare in-packet timestamps
    long refTime = Pickle.arrayToInteger(refBytes, DATA_OFFSET);
    refTime = refTime * Constants.SECOND; // in us!
    refTime += Pickle.arrayToInteger(refBytes, DATA_OFFSET + 4) * Constants.MICRO_SECOND;
    WiresharkMessage curr = (WiresharkMessage) link.mapTimestampToPacket.get(new Long(refTime));
    if (null == curr) {
      link.tlPDR[0][idx].add(refTime, .0);
      link.tlPDR[1][idx].add(refTime, .0);
      link.tlPDR[2][idx].add(refTime, .0);
      link.tlPDR[3][idx].add(refTime, .0);
      link.pktArrivedCorrect[idx].add(new Double(0));
      link.pktArrived[idx].add(new Double(0));
      return;
    }

    link.pktProcessedPerRate[idx]++;
    int bitErrors = 0;
    byte[] currBytes = ((MessageBytes) curr.getPayload()).getBytes();
    double[] errors = new double[currBytes.length * 8];
    int posLastBitError = -1; 
    Arrays.fill(errors, 0);

    byte status = AthdescHeader.getExtRxStatus(currBytes, 0);
    link.tlPDR[0][idx].add(refTime, (0 == status) ? 1. : .0);
    link.tlPDR[1][idx].add(refTime, (1 == status) ? 1. : .0);
    link.tlPDR[2][idx].add(refTime, (2 == status) ? 1. : .0);
    link.tlPDR[3][idx].add(refTime, 1.);

    // generate error mask (using xor), ignore athdesc header
    for (int i = AthdescHeader.HEADER_SIZE; i < refBytes.length; i++) {
      if (i == AthdescHeader.HEADER_SIZE + 22
          ||i == AthdescHeader.HEADER_SIZE + 23)
        continue; // skip seq id, it is not set in tx feedback

      if (i < currBytes.length) {
        refBytes[i] ^= currBytes[i];

        if (0 != refBytes[i]) {
          link.badBytesPerPosRate[i][idx]++;
          for (int j = 0; j < Byte.SIZE; j++) {
            if (0 != (refBytes[i] & (1 << j))) {
              int pos = j + i*Byte.SIZE;
              link.badBitsPerPosRate[pos][idx]++;
              errors[pos] = 1;
              int runLength = pos - posLastBitError - 1;
              link.correctRunLength [idx][runLength]++;
              posLastBitError = pos;
              bitErrors++;
            }
          }
        }
      } else { // missing part
        link.missingBytesPerPosRate[i][idx]++;
      }
    }
    int runLength = currBytes.length*8 - posLastBitError - 1;
    link.correctRunLength [idx][runLength]++;
    link.pktArrived[idx].add(new Double(1));
    link.pktArrivedCorrect[idx].add(new Double(0 == bitErrors ? 1 : 0));

    // create conditional error probability
    double ber = bitErrors / (double)errors.length;
    for (int i = 0; i < errors.length && bitErrors > 0; i++) {
      if (errors[i] == 1) {
       for (int j = 0; j < CONDITIONAL_LENGTH && i - j >= 0; j++){
         if (ber < CONDITIONAL_THRES) {
           link.condLow[idx][0][j] += errors[i-j];
           link.condLow[idx][1][j] += 1;
         } else {
           link.condHigh[idx][0][j] += errors[i-j];
           link.condHigh[idx][1][j] += 1;
         }
       }
      }
    }

    // store number of bit errors per packet
    link.pktPerBadBitsRate[bitErrors][idx]++;

    // calculate the auto correlation and allan deviation
    if (0 != bitErrors && doStatistics) {
      double[] correlation = new double[AUTOCORRELLATION_SIZE];
      Statistics.autoCorrelate(errors, correlation, 0.01);

//      // TODO remove test code -- begin
//      double[] correlation2 = new double[AUTOCORRELLATION_SIZE];
//      Statistics.correlate(errors, errors, correlation2, 0.01);
//      // TODO remove test code -- end

      for (int i = 0; i < correlation.length; i++) {
//        // TODO remove test code -- begin
//        link.bitAutoCorrelation2[idx][i] += correlation2[i];
//        if (Math.abs(correlation[i] - correlation2[i]) > 0.0001)
//          System.out.println("Error at " + i);
//        // TODO remove test code -- end

        link.bitAutoCorrelation[idx][i] += correlation[i];
        glob.bitAutoCorrelation[idx][i] += link.bitAutoCorrelation[idx][i];
      }

      double[] allanDev = Statistics.allanVariance(errors);
      if (null == link.bitAllanVarCuml[idx])
        link.bitAllanVarCuml[idx] = new double[allanDev.length];
      else if (link.bitAllanVarCuml[idx].length < allanDev.length) {
        double d[] = link.bitAllanVarCuml[idx];
        link.bitAllanVarCuml[idx] = new double[allanDev.length];
        System.arraycopy(d, 0, link.bitAllanVarCuml[idx], 0, d.length);
      }
      if (null == glob.bitAllanVarCuml[idx])
        glob.bitAllanVarCuml[idx] = new double[allanDev.length];
      else if (glob.bitAllanVarCuml[idx].length < allanDev.length) {
        double d[] = glob.bitAllanVarCuml[idx];
        glob.bitAllanVarCuml[idx] = new double[allanDev.length];
        System.arraycopy(d, 0, glob.bitAllanVarCuml[idx], 0, d.length);
      }
      for (int i = 0; i < allanDev.length; i++) {
        link.bitAllanVarCuml[idx][i] += allanDev[i]; link.bitAllanVarCount[idx]++;
        glob.bitAllanVarCuml[idx][i] += allanDev[i]; glob.bitAllanVarCount[idx]++;
      }

      // TODO comment the following sections in for a dump of bit error pattern.
//      StringBuilder b = new StringBuilder(rateString[idx] + "\t"+ bitErrors + ":\t");
//      for (int i = 0; i < errors.length; i++) {
//        b.append(errors[i]==.0 ? "0" : "1");
//      }
//      System.out.println(b.toString());
    }

    // TODO dump out error mask as packet
  }


  public void finished() throws IOException {
    log.info("Writing global results");
    String fileName = "global";
    GlobalCtx global = this.glob;

    saveGlobal(global, fileName);

    System.out.println(global.builder.toString());
  }

  private void save() throws IOException {
    log.info("Writing results");
    File file = new File(this.link.fileName);
    String fileName = file.getName();
    LinkCtx global = this.link;

    int max_size = global.pktArrived[0].size();
    for (int i = 1; i < global.pktArrived.length; i++)
      max_size = Math.max(max_size, global.pktArrived[i].size());
    double pktArrivedCorrect[][] = new double[rateString.length][max_size];
    double pktArrived[][] = new double[rateString.length][max_size];

    // save packet error autocorrelation
    XplotSerializer seri = new XplotSerializer(fileName + " autocorrelation packet loss",
        "lag", "autocorrelation");
    for (int rate = 0; rate < rateString.length; rate++) {
      if (0 == global.pktArrivedCorrect[rate].size())
        continue;

      for (int i = 0; i < global.pktArrivedCorrect[rate].size(); i++) {
        pktArrivedCorrect[rate][i] = ((Double)global.pktArrivedCorrect[rate].get(i)).doubleValue();
        pktArrived[rate][i] = ((Double)global.pktArrived[rate].get(i)).doubleValue();
      }

      double pktCorrectAutoCorrelation[] = new double[2*AUTOCORRELLATION_SIZE];
      double pktArrivedAutoCorrelation[] = new double[2*AUTOCORRELLATION_SIZE];
      Statistics.autoCorrelate(pktArrivedCorrect[rate], pktCorrectAutoCorrelation, 0.01);
      Statistics.autoCorrelate(pktArrived[rate], pktArrivedAutoCorrelation, 0.01);

      Line correctCorrelation = new Line("autocorrelation packet correct-" + rateString[rate]);
      Line arrivedCorrelation = new Line("autocorrelation packet arrived-" + rateString[rate]);

      for (int lag = pktCorrectAutoCorrelation.length-1; lag >= 0 ; lag--) {
        correctCorrelation.add(lag, pktCorrectAutoCorrelation[lag]);
        arrivedCorrelation.add(lag, pktArrivedAutoCorrelation[lag]);

        // Add to global packet loss autocorrelation
        glob.pktCorrectAutoCorrelation[rate][lag] += pktCorrectAutoCorrelation[lag];
        glob.pktArrivedAutoCorrelation[rate][lag] += pktArrivedAutoCorrelation[lag];
      }

      seri.addLine(correctCorrelation, rateColor[rate]);
      seri.addPoints(arrivedCorrelation, rateColor[rate]);
    }
    seri.saveToFile(fileName + "-pktAutoCorrelation.xpl");

    // save conditional bit error probability
    {
      seri = new XplotSerializer(fileName + " conditional bit error probability",
          "lag", "P(e[j+k]|e[j])");
      for (int rate = 0; rate < rateString.length; rate++) {
        if (global.condLow[rate][1][1] <= 0)
          continue;
        Line line = new Line("p_cond_" + rateString[rate]);
        for (int i = 0; i < global.condLow[rate][0].length; i++) {
          if (global.condLow[rate][1][i] <= 0)
            continue;
          line.add(i, global.condLow[rate][0][i] / global.condLow[rate][1][i]);
        }
        seri.addLine(line, rateColor[rate]);
      }
      seri.saveToFile(fileName + "-condLowBER.xpl");

      seri = new XplotSerializer(fileName + " conditional bit error probability",
          "lag", "P(e[j+k]|e[j])");
      for (int rate = 0; rate < rateString.length; rate++) {
        if (global.condHigh[rate][1][1] <= 0)
          continue;
        Line line = new Line("p_cond_" + rateString[rate]);
        for (int i = 0; i < global.condHigh[rate][0].length; i++) {
          if (global.condHigh[rate][1][i] <= 0)
            continue;
          line.add(i, global.condHigh[rate][0][i] / global.condHigh[rate][1][i]);
        }
        seri.addLine(line, rateColor[rate]);
      }
      seri.saveToFile(fileName + "-condHighBER.xpl");

      seri = new XplotSerializer(fileName + " conditional bit error probability",
          "lag", "P(e[j+k]|e[j])");
      for (int rate = 0; rate < rateString.length; rate++) {
        if (global.condLow[rate][1][1] + global.condHigh[rate][1][1] <= 0)
          continue;
        Line line = new Line("p_cond_" + rateString[rate]);
        for (int i = 0; i < global.condLow[rate][0].length; i++) {
          if (global.condLow[rate][1][i] + global.condHigh[rate][1][i] <= 0)
            continue;
          line.add(i, (global.condLow[rate][0][i] + global.condHigh[rate][0][i]) 
             / (global.condLow[rate][1][i] + global.condHigh[rate][1][i]));
        }
        seri.addLine(line, rateColor[rate]);
      }
      seri.saveToFile(fileName + "-condBER.xpl");
    }
    
    {
      int start = 10;
      seri = new XplotSerializer(fileName + " ccdf correct run length",
          "length of correct part", "cdf");
      for (int rate = 0; rate < rateString.length; rate++) {
        Line cdf = new Line("ccdf correctRunLength-" + rateString[rate]);
        double cuml = .0;
        double sum = .0;

        for (int bitPos = start; bitPos < MTU*8; bitPos++) {
          sum += global.correctRunLength[rate][bitPos];
        }

        if (0 >= sum)
          continue;

        for (int bitPos = start; bitPos < MTU*8; bitPos++) {
          if (0 == global.correctRunLength[rate][bitPos])
            continue;

          // generate cdf
          cuml += global.correctRunLength[rate][bitPos] / sum;
          //cuml = global.correctRunLength[rate][bitPos] / link.pktProcessedPerRate[rate];
          cdf.add(bitPos, 1-cuml);
        }

        seri.addLine(cdf, rateColor[rate]);
      }
      seri.saveToFile(fileName + "-correctRunLength.xpl");

      // usable information
      seri = new XplotSerializer(fileName + " ccdf usable information",
          "length of correct part", "ccdf");
      for (int rate = 0; rate < rateString.length; rate++) {
        Line cdf = new Line("ccdf usableInformation-" + rateString[rate]);
        double cuml = .0;
        double sum = .0;

        // determine highest product for normalization
        for (int bitPos = MTU*8-1; bitPos >= 0 && sum == 0; bitPos--) {
          sum += global.correctRunLength[rate][bitPos] * bitPos;
        }

        if (0 >= sum)
          continue;

        for (int bitPos = MTU*8-1; bitPos >= 0; bitPos--) {
          if (0 == global.correctRunLength[rate][bitPos])
            continue;

          // generate cdf
          cuml += global.correctRunLength[rate][bitPos] * bitPos / sum;
          cdf.add(bitPos, cuml);
        }

        seri.addLine(cdf, rateColor[rate]);
      }
      seri.saveToFile(fileName + "-usableInformation.xpl");
    }
    
    // calc variance time plot for packet loss
    seri = new XplotSerializer(fileName + " time-variance packet loss",
        "log2(sample period)", "log2(variance)");
    for (int rate = 0; rate < rateString.length; rate++) {
      if (0 == global.pktArrivedCorrect[rate].size())
        continue;

      double[] VTP = Statistics.timeCovariance(pktArrivedCorrect[rate],
          pktArrivedCorrect[rate]);
      Line varianceTime = new Line("variance-time correct-" + rateString[rate]);
      for (int i = 0; i < VTP.length; i++) {
        if (VTP[i] <= .0)
          continue;
        varianceTime.add(i, Math.log(VTP[i]) / Math.log(2));
      }

      seri.addLine(varianceTime, rateColor[rate]);
      if (rate < 4) // 802.11b rates
        seri.addPoints(varianceTime, rateColor[rate], "x");

      VTP = Statistics.timeCovariance(pktArrived[rate], pktArrived[rate]);
      varianceTime = new Line("variance-time arrived-" + rateString[rate]);
      for (int i = 0; i < VTP.length; i++) {
        if (VTP[i] <= .0)
          continue;
        varianceTime.add(i, Math.log(VTP[i]) / Math.log(2));
      }

      seri.addLine(varianceTime, rateColor[rate]);
      if (rate < 4) // 802.11b rates
        seri.addPoints(varianceTime, rateColor[rate], "diamond");
      else
        seri.addPoints(varianceTime, rateColor[rate], "box");
    }
    seri.saveToFile(fileName + "-pktVarianceTime.xpl");

    // save packet loss allan deviation
    seri = new XplotSerializer(fileName + " allan dev packet loss",
        "log2(sample period)", "log2(allan dev)");
    for (int rate = 0; rate < rateString.length; rate++) {
      if (0 == global.pktArrivedCorrect[rate].size())
        continue;

      double[][] ADEV = Statistics.allanDeviation(pktArrivedCorrect[rate], 1);
      Line allanDev = new Line("allan dev correct-" + rateString[rate]);
      for (int i = 0; i < ADEV[1].length; i++) {
        if (ADEV[1][i] <= .0)
          continue;
        allanDev.add(i+1, Math.log(ADEV[1][i]) / Math.log(2));
      }

      seri.addLine(allanDev, rateColor[rate]);
      if (rate < 4) // 802.11b rates
        seri.addPoints(allanDev, rateColor[rate], "x");

      ADEV = Statistics.allanDeviation(pktArrived[rate], 1);
      allanDev = new Line("allan dev-arrived" + rateString[rate]);
      for (int i = 0; i < ADEV[1].length; i++) {
        if (ADEV[1][i] <= .0)
          continue;
        allanDev.add(i+1, Math.log(ADEV[1][i]) / Math.log(2));
      }

      seri.addLine(allanDev, rateColor[rate]);
      if (rate < 4) // 802.11b rates
        seri.addPoints(allanDev, rateColor[rate], "diamond");
      else
        seri.addPoints(allanDev, rateColor[rate], "box");
    }
    seri.saveToFile(fileName + "-pktAllanDev.xpl");

    // Correlate the arrived/correct packets againt each other considering the rates
    seri = new XplotSerializer(fileName + " pkt loss COR",
        "log2(sample period)", "pkt loss COR");
    for (int idx1 = 0; idx1 < rateString.length; idx1++) {
      for (int idx2 = idx1; idx2 < rateString.length; idx2++) {
        double corrArrived = Statistics.correlationCoefficient(pktArrived[idx1], pktArrived[idx2]);
        double corrCorrect = Statistics.correlationCoefficient(pktArrivedCorrect[idx1], pktArrivedCorrect[idx2]);

        this.glob.builder.append(link.fileName);
        this.glob.builder.append("\t");
        this.glob.builder.append(rateString[idx1]);
        this.glob.builder.append("\t");
        this.glob.builder.append(rateString[idx2]);
        this.glob.builder.append("\t");
        this.glob.builder.append(corrArrived);
        this.glob.builder.append("\t");
        this.glob.builder.append(corrCorrect);
        this.glob.builder.append("\n");
      }
      int idx2 = idx1 + 1;
      if (idx2 < rateString.length) {
        double[] values = Statistics.timeCorrelation(pktArrivedCorrect[idx1], pktArrivedCorrect[idx2]);
        Line timeCOR = new Line("pkt loss arrived COR-" + rateString[idx1] + "-"
            + rateString[idx2]);
        for (int i = 0; i < values.length; i++)
          timeCOR.add(i, values[i]);
        seri.addLine(timeCOR, rateColor[idx1]);
        if (idx1 < 4) // 802.11b rates
          seri.addPoints(timeCOR, rateColor[idx1], "x");

        values = Statistics.timeCorrelation(pktArrived[idx1],
            pktArrived[idx2]);
        timeCOR = new Line("pkt loss correct COR-" + rateString[idx1] + "-"
            + rateString[idx2]);
        for (int i = 0; i < values.length; i++)
          timeCOR.add(i, values[i]);
        seri.addLine(timeCOR, rateColor[idx1]);
        if (idx1 < 4) // 802.11b rates
          seri.addPoints(timeCOR, rateColor[idx1], "diamond");
        else
          seri.addPoints(timeCOR, rateColor[idx1], "box");
      }
    }
    seri.saveToFile(fileName + "-pktTimeCOR.xpl");

    // generate scatter plots
    double[][][] pktArrCorrAvg = new double[4][rateString.length][];
    double[][][] pktArrAvg = new double[4][rateString.length][];
    for (int idx1 = 0; idx1 < rateString.length-1; idx1++) {
      int idx2 = idx1 + 1;
      for (int level = 16, lidx = 0; level < 128; level *= 2, lidx++) {
        pktArrCorrAvg[lidx][idx1] = Statistics.average(pktArrivedCorrect[idx1], level);
        pktArrCorrAvg[lidx][idx2] = Statistics.average(pktArrivedCorrect[idx2], level);

        pktArrAvg[lidx][idx1] = Statistics.average(pktArrived[idx1], level);
        pktArrAvg[lidx][idx2] = Statistics.average(pktArrived[idx2], level);

        if (null == pktArrCorrAvg[lidx][idx1] && null == pktArrCorrAvg[lidx][idx2]
            &&null == pktArrAvg[lidx][idx1] && null == pktArrAvg[lidx][idx2])
          continue;

        seri = new XplotSerializer(fileName + " scatterplot pkt loss adj bitrates",
            "PDR " + rateString[idx1], "PDR " + rateString[idx2]);
        Line scatterplot = new Line("scatterplot correct-" + rateString[idx1] + "-"
            + rateString[idx2]);
        for (int i = 0; i < Math.min(pktArrCorrAvg[lidx][idx1].length, pktArrCorrAvg[lidx][idx2].length); i++)
          scatterplot.add(pktArrCorrAvg[lidx][idx1][i], pktArrCorrAvg[lidx][idx2][i]);
        seri.addPoints(scatterplot, rateColor[0], "x");

        scatterplot = new Line("scatterplot arrived-" + rateString[idx1] + "-"
            + rateString[idx2]);
        for (int i = 0; i < Math.min(pktArrAvg[lidx][idx1].length, pktArrAvg[lidx][idx2].length); i++)
          scatterplot.add(pktArrAvg[lidx][idx1][i], pktArrAvg[lidx][idx2][i]);
        seri.addPoints(scatterplot, rateColor[1], "box");

        seri.saveToFile(fileName + "-pktScatter"  + rateString[idx1] +
            rateString[idx2] + level + ".xpl");
      }
    }

    // generate pdr ratio distribution
    for (int idx1 = 0; idx1 < rateString.length-1; idx1++) {
      int idx2 = idx1 + 1;
      if (3 == idx1) // 11 <-> 6
        continue;

      int dest = idx1 < 3 ? 0 : 1;
      int min = Math.min(pktArrCorrAvg[2][idx1].length, pktArrCorrAvg[2][idx2].length);
      for (int i = 0; i < min; i++) {
        int yi = 0;
        for (double y = -1.0; y < 1.01; y += .05, yi++) {
          if(pktArrCorrAvg[2][idx1][i] >= pktArrCorrAvg[2][idx2][i] + y) {
            glob.pdrRatioDist[dest][yi]++;
            global.pdrRatioDist[dest][yi]++;
          }
        }
      }

      dest = idx1 < 3 ? 2 : 3;
      min = Math.min(pktArrAvg[2][idx1].length, pktArrAvg[2][idx2].length);
      for (int i = 0; i < min; i++) {
        int yi = 0;
        for (double y = -1.0; y < 1.01; y += .05, yi++) {
          if(pktArrAvg[2][idx1][i] >= pktArrAvg[2][idx2][i] + y) {
            glob.pdrRatioDist[dest][yi]++;
            global.pdrRatioDist[dest][yi]++;
          }
        }
      }
    }

//    double[][][] x = new double[12][3][];
//    double[][][] y = new double[12][3][];
    long[][][][] timeRateStatus = new long[12][3][2][];

    //int xMin = Integer.MAX_VALUE;
    for (int rate = 0; rate < rateString.length; rate++) {
      for (int status = 0; status < statusString.length - 1; status++) {
        double[] x = global.tlPDR[status][rate].getX();
        double[] y = global.tlPDR[status][rate].getY();
        timeRateStatus[rate][status][0] = new long[x.length];
        timeRateStatus[rate][status][1] = new long[y.length];
        for (int i=0; i < x.length; i++)
          timeRateStatus[rate][status][0][i] = (long)(x[i] * Constants.SECOND);
        for (int i=0; i < x.length; i++)
          timeRateStatus[rate][status][1][i] = (long)(y[i]);
//        x[rate][status] = global.tlPDR[status][rate].getX();
//        y[rate][status] = global.tlPDR[status][rate].getY();
//        xMin = Math.min(xMin, x.length);
      }
    }

//    double[][][][] timeRateStatus = new double[xMin][12][3][2];
//    for (int j = 0; j < xMin; j++) {
//      for (int rate = 0; rate < rateString.length; rate++) {
//        for (int status = 0; status < statusString.length - 1; status++) {
//          timeRateStatus[j][rate][status][0] = x[rate][status][j];
//          timeRateStatus[j][rate][status][1] = y[rate][status][j];
//        }
//      }
//    }
    OutputStream fos = new FileOutputStream(fileName + ".jobi");
    ObjectOutputStream out = new ObjectOutputStream(fos);
    out.writeObject(timeRateStatus);
    out.close();
    //Util.writeObject(fileName + ".xml", timeRateStatus);

    saveGlobal(this.link, fileName);
  }

  /**
   *
   * @param global
   * @param fileName
   * @throws IOException
   */
  private void saveGlobal(BaseCtx global, String fileName) throws IOException {

    // process badBytesPerPosRate
    XplotSerializer seri = new XplotSerializer(fileName + " badByte",
        "byte position", "pdf");
    for (int rate = 0; rate < rateString.length; rate++) {
      Line pdf = new Line("pdf badByte-" + rateString[rate]);
//      Line cdf = new Line("cdf badByte-" + rateString[rate]);
//      double cuml = .0;

      for (int bytePos = 0; bytePos < MTU; bytePos++) {
        if (0 == global.pktProcessedPerRate[rate]
          || 0 == global.badBytesPerPosRate[bytePos][rate])
          continue;

        // generate pdf
        double value = global.badBytesPerPosRate[bytePos][rate] /
          (double)global.pktProcessedPerRate[rate];
        pdf.add(bytePos, value);

        // generate cdf
//        cuml += value / MTU;
//        cdf.add(bytePos, cuml);
      }

      seri.addPoints(pdf, rateColor[rate], rateShape[rate]);
//      seri.addLine(cdf, rateColor[rate]);
    }
    seri.saveToFile(fileName + "-badBytes.xpl");

    // process badBitsPerPosRate
    seri = new XplotSerializer(fileName + " pdf/cdf badBits",
        "bit position", "pdf");
    for (int rate = 0; rate < rateString.length; rate++) {
      Line pdf = new Line("pdf badBit-" + rateString[rate]);
//      Line cdf = new Line("cdf badBit-" + rateString[rate]);
//      double cuml = .0;

      for (int bitPos = 0; bitPos < MTU*8; bitPos++) {
        if (0 == global.pktProcessedPerRate[rate]
          || 0 == global.badBitsPerPosRate[bitPos][rate])
          continue;

        // generate pdf
        double value = global.badBitsPerPosRate[bitPos][rate] /
          (double)global.pktProcessedPerRate[rate];
        pdf.add(bitPos, value);

        // generate cdf
//        cuml += value / MTU / 8;
//        cdf.add(bitPos, cuml);
      }

      seri.addPoints(pdf, rateColor[rate], rateShape[rate]);
//      seri.addLine(cdf, rateColor[rate]);
    }
    seri.saveToFile(fileName + "-badBits.xpl");

    // process missingBytesPerPosRate
    seri = new XplotSerializer(fileName + " pdf missingBytes",
        "bit position", "pdf");
    for (int rate = 0; rate < rateString.length; rate++) {
      Line pdf = new Line("pdf missingByte-" + rateString[rate]);

      for (int bytePos = 0; bytePos < MTU; bytePos++) {
        if (0 == global.pktProcessedPerRate[rate]
          || 0 == global.missingBytesPerPosRate[bytePos][rate])
          continue;

        // generate pdf
        double value = global.missingBytesPerPosRate[bytePos][rate] /
          (double)global.pktProcessedPerRate[rate];
        pdf.add(bytePos, value);
      }

      seri.addPoints(pdf, rateColor[rate], rateShape[rate]);
    }
    seri.saveToFile(fileName + "-missingBytes.xpl");

    // process pktPerBadBitsRate
    seri = new XplotSerializer(fileName + " pdf/cdf no. bad bits per packet",
        "no. bad bits", "pdf / cdf");
    XplotSerializer seri2 = new XplotSerializer(fileName + " pdf/cdf no. bad bits per packet",
        "no. bad bits", "pdf / cdf");
    for (int rate = 0; rate < rateString.length; rate++) {
      Line pdf = new Line("pdf badBitsPerPacket-" + rateString[rate]);
      Line cdf = new Line("cdf badBitsPerPacket-" + rateString[rate]);
      double cuml = .0;
      double sum = .0;
      Line pdf2 = new Line("pdf badBitsPerPacket-" + rateString[rate]);
      Line cdf2 = new Line("cdf badBitsPerPacket-" + rateString[rate]);
      double cuml2 = .0;
      double sum2 = .0;

      sum += global.pktPerBadBitsRate[0][rate];
      for (int bitPos = 1; bitPos < MTU*8; bitPos++) {
        sum += global.pktPerBadBitsRate[bitPos][rate];
        sum2 += global.pktPerBadBitsRate[bitPos][rate];
      }

      if (0 >= sum)
        continue;

      if (0 != global.pktPerBadBitsRate[0][rate]) {
        // generate pdf
        double value = global.pktPerBadBitsRate[0][rate] / sum;
        pdf.add(0, value);

        // generate cdf
        cuml += value;
        cdf.add(0, cuml);
      }

      for (int bitPos = 1; bitPos < MTU*8; bitPos++) {
        if (0 == global.pktPerBadBitsRate[bitPos][rate])
          continue;

        // generate pdf
        double value = global.pktPerBadBitsRate[bitPos][rate] / sum;
        double value2 = global.pktPerBadBitsRate[bitPos][rate] / sum2;
        pdf.add(bitPos, value);
        pdf2.add(bitPos, value2);

        // generate cdf
        cuml += value;
        cuml2 += value2;
        cdf.add(bitPos, cuml);
        cdf2.add(bitPos, cuml2);
      }

      seri.addPoints(pdf, rateColor[rate], rateShape[rate]);
      seri.addLine(cdf, rateColor[rate]);
      seri2.addPoints(pdf2, rateColor[rate], rateShape[rate]);
      seri2.addLine(cdf2, rateColor[rate]);
    }
    seri.saveToFile(fileName + "-badBitsPerPacket.xpl");
    seri2.saveToFile(fileName + "-badBitsPerPacket2.xpl");

    // save bit error autocorrelation
    seri = new XplotSerializer(fileName + " autocorrelation bit errors",
        "lag", "autocorrelation");
    for (int rate = 0; rate < rateString.length; rate++) {
      if (0 == global.bitAutoCorrelation[rate][0])
        continue;

      Line correlation = new Line("autocorrelation bit errors-" + rateString[rate]);
      for (int lag = global.bitAutoCorrelation[rate].length-1; lag >= 0; lag--) {
        double value = global.bitAutoCorrelation[rate][lag] / global.bitAutoCorrelation[rate][0];
        correlation.add(lag, value);

//        // TODO remove test code -- begin
//        double value2 = link.bitAutoCorrelation2[rate][lag] / link.bitAutoCorrelation2[rate][0];
//        if (Math.abs(value - value2) > 0.0001)
//          System.out.println("Error at " + lag);
//        // TODO remove test code -- end
      }

      seri.addLine(correlation, rateColor[rate]);
      if (rate < 4) // 802.11b rates
        seri.addPoints(correlation, rateColor[rate], "x");
    }
    seri.saveToFile(fileName + "-bitsAutoCorrelation.xpl");

    // save bit error allan deviation
    seri = new XplotSerializer(fileName + " allan dev bit errors",
        "log2(sample period)", "log2(allan dev)");
    for (int rate = 0; rate < rateString.length; rate++) {
      if (0 == global.bitAllanVarCount[rate])
        continue;

      Line allanDev = new Line("allan dev-" + rateString[rate]);
      for (int i = 0; i < global.bitAllanVarCuml[rate].length; i++) {
        allanDev.add(i+1, Math.log(Math.sqrt(global.bitAllanVarCuml[rate][i] /
            global.bitAllanVarCount[rate])) / Math.log(2));
      }

      seri.addLine(allanDev, rateColor[rate]);
      if (rate < 4) // 802.11b rates
        seri.addPoints(allanDev, rateColor[rate], "x");
    }
    seri.saveToFile(fileName + "-bitsAllanDev.xpl");

    // save packet error autocorrelation
    seri = new XplotSerializer(fileName + " autocorrelation packet loss",
        "lag", "autocorrelation");
    for (int rate = 0; rate < rateString.length; rate++) {
      if (0 != glob.pktCorrectAutoCorrelation[rate][0]) {
        Line correctCorrelation = new Line("autocorrelation packet correct-" +
            rateString[rate]);

        for (int lag = glob.pktCorrectAutoCorrelation[rate].length-1; lag >= 0 ; lag--)
          correctCorrelation.add(lag, glob.pktCorrectAutoCorrelation[rate][lag] /
              glob.pktCorrectAutoCorrelation[rate][0]);

        seri.addLine(correctCorrelation, rateColor[rate]);
      }

      if (0 != glob.pktArrivedAutoCorrelation[rate][0]) {
        Line arrivedCorrelation = new Line("autocorrelation packet arrived-" +
            rateString[rate]);

        for (int lag = glob.pktArrivedAutoCorrelation[rate].length-1; lag >= 0 ; lag--)
          arrivedCorrelation.add(lag, glob.pktArrivedAutoCorrelation[rate][lag] /
              glob.pktArrivedAutoCorrelation[rate][0]);

        seri.addPoints(arrivedCorrelation, rateColor[rate]);
      }
    }
    seri.saveToFile(fileName + "-pktAutoCorrelation.xpl");

    // save pdr ratio distribution
    seri = new XplotSerializer(fileName + " pdr difference distribution",
        "difference between neighboring rates", "percentage of links");
    for (int dest = 0; dest < global.pdrRatioDist.length; dest++) {
      String title = "pdr ratio distribution-";
      switch(dest) {
      case 0: title += "corr-b"; break;
      case 1: title += "corr-g"; break;
      case 2: title += "arr-b"; break;
      case 3: title += "arr-g"; break;
      }
      Line pdrratiodist = new Line(title);

      for (int y = global.pdrRatioDist[dest].length-1; y >= 0; y--)
        global.pdrRatioDist[dest][y] /= global.pdrRatioDist[dest][0];

      int yi = 0;
      for (double y = -1.0; y < 1.01; y += .05, yi++)
        pdrratiodist.add(y, global.pdrRatioDist[dest][yi]);

      seri.addLine(pdrratiodist, rateColor[dest]);
    }
    seri.saveToFile(fileName + "-pktPdrDifferenceDistribution.xpl");
}


  /**
   * Main function
   *
   * @param args
   * @throws IOException
   */
  public static void main(String[] args) throws IOException {
    PatternLayout layout = new PatternLayout( "%d{ISO8601} %-5p [%t] %c: %m%n" );
    ConsoleAppender consoleAppender = new ConsoleAppender( layout );
    log.addAppender( consoleAppender );
    log.setLevel(Level.ALL);

    BitErrorAggregator p = new BitErrorAggregator(args[0]);
    for (int i = 1; i < args.length; i++) {
      p.analyze(args[i]);
    }
    p.finished();
  }

}
