/*
 * ExtractData.java
 *
 * Created on March 10, 2008, 6:36 PM
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */

package sidnet.batch;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import sidnet.core.misc.FileUtils;

/**
 *
 * @author Oliver
 */
public class ExtractData
{
    public static final String FILE_PREFIX_NAME ="run";
    public static final String FILE_EXTENSION   =".log";
    public static final String OUTPUT_DELIMITER ="\t"; // for tab separated output files
    //public static final String OUTPUT_DELIMITER =","; // for comma separated output files
     enum MATH
     {
         AVG,
         MIN,
         MAX
     }
     
      static String path;
      static String outputFileName;
      static String xAxisTagName;
      static String yAxisTagName;
      static MATH mathOperation;
      static String categoryOn;
     
      static Map<String,List<Double[]>> dataTable;
      static Map<String,Double> contorTable; // keeps track on how many rows has been collapsed in one - used for computing AVG
      
     /**
     * @param args the command line arguments
     */
     public static void main(String[] cmdLineArgs) throws Exception 
    {
         List<File> fileList;
         
         parseCommandLineArguments(cmdLineArgs);

         // init data table
         dataTable   = new HashMap<String,List<Double[]>>();
         contorTable = new HashMap<String,Double>();
         
         // retrieve the list of files that will be processed
         fileList = buildInputLogFileList(new File(path));
         
         // process files
         processFiles(fileList);

         // post processing (necessary for AVG, for example)
         postProcessing();
         
         // write aggregated results back to CSV file
         writeResultsBack(outputFileName);
         
         System.out.println("\n[ExtractData] Completed succesfully!");
     }
     
     public static void parseCommandLineArguments(String[] cmdLineArgs)
     {
        if (cmdLineArgs.length < 6)
        {
            System.out.println("<ExtractData>[ERROR]: Invalid number of arguments specified");
            System.out.println("syntax: ExtractData <path> <output file name> <xAxisTag> <yAxisTag>  -average -categoryOn=... ");
            return;
        }
          
        path           = cmdLineArgs[0];
        outputFileName = cmdLineArgs[1];
        xAxisTagName   = cmdLineArgs[2];
        yAxisTagName   = cmdLineArgs[3];
        categoryOn     = cmdLineArgs[4];
        if (cmdLineArgs[5].equals("average") || cmdLineArgs[5].equals("avg"))
            mathOperation  = MATH.AVG;
        if (cmdLineArgs[5].equals("minimum") || cmdLineArgs[5].equals("min"))
            mathOperation  = MATH.AVG;
        if (cmdLineArgs[5].equals("maximum") || cmdLineArgs[5].equals("max"))
            mathOperation  = MATH.AVG;
        
        System.out.println("Path = " + path);
        System.out.println("Output Filename = " + outputFileName);
        System.out.println("X-Axis Tag Name = " + xAxisTagName);
        System.out.println("Y-Axis Tag Name = " + yAxisTagName);
        System.out.println("Category On     = " + categoryOn);
        System.out.println("Math Operation  = " + cmdLineArgs[5]);
     }
     
     public static List<File> buildInputLogFileList(File rootPath) throws FileNotFoundException 
     {
        FileUtils.validateDirectory(rootPath);
        List<File> result = new ArrayList<File>();

        File[] filesAndDirs = rootPath.listFiles();
        List<File> filesDirs = Arrays.asList(filesAndDirs);
        for(File file : filesDirs) 
        {
            if (file.getName().startsWith(FILE_PREFIX_NAME) && file.getName().endsWith(FILE_EXTENSION))
                result.add(file); //always add, even if directory
            if ( ! file.isFile() ) 
            {
                //must be a directory
                //recursive call!
                List<File> deeperList = buildInputLogFileList(file);
                result.addAll(deeperList);
            }
        }
        //Collections.sort(result);
        return result;
     }  
     
     public static void processFiles(List<File> fileList)
     {
        // for (int i = 0; i < ((ArrayList)fileList).size(); i++)
         for (int i = 0; i < fileList.size(); i++)
         {
             File currentFile = (File)((ArrayList)fileList).get(i);
             System.out.println("\n[ExtractData] Processing file: " + currentFile);
             
             List<Double[]> newDataSet   = processFile(currentFile, xAxisTagName, yAxisTagName);
             if (newDataSet == null)
                 continue;
             
             String key = retrieveDistinctLabel(currentFile, categoryOn);
             
             if (key != null)
                 System.out.println("\tkey = " + key);
             else
                 key = "nolabel";
             
             List<Double[]> resultSet = collapseRows(dataTable.get(key), newDataSet, mathOperation);
             if (contorTable.get(key) != null)
                 contorTable.put(key, contorTable.get(key) + 1); // increment the number of collapsed rows
             else
                 contorTable.put(key, new Double(1));
             
             dataTable.put(key, resultSet);
           
             //System.out.println("[" + xAxisTagName + "]\t" + "[" + yAxisTagName + "]" );
             //System.out.println("dataSet size = " + newDataSet.size());
             //for(Double[] dataPair:newDataSet)
             //{
             //    System.out.println(dataPair[0] + "\t" + dataPair[1]);
             //}
         }
     }
     
     public static List<Double[]> processFile(File file, String xAxisTag, String yAxisTag)
     {
          int xAxisIndex = 0, yAxisIndex = 0;
          BufferedReader input = null;          
          String nextLine;
          List<Double[]> rowList = new LinkedList<Double[]>();
          
          try{
               input = new BufferedReader(new FileReader(file));
          } catch(FileNotFoundException e){e.printStackTrace();System.exit(1);};

          try{
              while((nextLine = input.readLine()) != null)
              {
                 
                  if (nextLine.contains("<header>"))
                  {
                      String[] header = nextLine.split("\t");
                      for (int i = 1; i < header.length; i++) // start at 1 since the 0th is <header>
                      {
                          //System.out.println("[" + i + "]: |" +  header[i] + "|\n");
                          if (xAxisIndex == 0 && header[i].trim().equals(xAxisTag))
                              xAxisIndex = i;
                          if (yAxisIndex == 0 && header[i].trim().equals(yAxisTag))
                              yAxisIndex = i;
                      }

                      if (xAxisIndex == 0)
                      {
                          System.err.println("[ExtractData] ERROR: xAxisTagName (\"" + xAxisTag + "\") has not been found ");
                          System.exit(1);
                      }
                      if (yAxisIndex == 0)
                      {
                          System.err.println("[ExtractData] ERROR: yAxisTagName (\"" + yAxisTag + "\") has not been found ");
                          System.exit(1);
                      }
                      if (xAxisIndex == yAxisIndex)
                      {
                          System.err.println("[ExtractData] ERROR: xAxisTagName (\"" + xAxisTag + "\") cannot have the same name with yAxisTagName (\"" + yAxisTagName + "\")");
                          System.exit(1);
                      }
                      //System.out.println("xAxisIndex = " + xAxisIndex);
                      //System.out.println("yAxisIndex = " + yAxisIndex);
                  }
                  if (nextLine.contains("<row>"))
                  {
                      if(xAxisIndex == 0 || yAxisIndex == 0)
                      {
                          System.err.println("[ExtractData] ERROR: <row> element found before the <header> element. I don't know where to place the data. Aborting!");
                          System.exit(1);
                      }
                      
                      //System.out.println(nextLine);

                      //nextLine.replaceAll("s+"," ");
                      nextLine = nextLine.replaceFirst(" ","");
                      nextLine = nextLine.replaceAll("\t"," ");
                      String[] rowEntries = nextLine.split(" ");
                      //System.out.println("row length = " + rowEntries.length);
                      //for (int i =0; i < rowEntries.length; i++)
                        //  System.out.println("[" + i + "]: " + rowEntries[i]);
                      //System.out.println("convert Pair: " + rowEntries[xAxisIndex] + " and " + rowEntries[yAxisIndex]);
                      //System.out.println("convert Pair: " + rowEntries[xAxisIndex].trim() + " and " + rowEntries[yAxisIndex].trim());
                      Double[] pair = new Double[2];
                      pair[0] = Double.parseDouble(rowEntries[xAxisIndex].trim());
                      pair[1] = Double.parseDouble(rowEntries[yAxisIndex].trim());
                      rowList.add(pair);
                  }  
              }
          }catch(IOException ioe){ioe.printStackTrace(); System.exit(1);};
          if (rowList.size() == 0)
              return null;
          return rowList;
     }
     
      public static String retrieveDistinctLabel(File file, String categoryOn)
     {
         BufferedReader input = null;    
         String nextLine;
         
         // ????? 
         if (file.getName().contains("bezier"))
             return "bezier";
         if (file.getName().contains("short"))
             return "k-short";
         // ?????
         
          try{
               input = new BufferedReader(new FileReader(file));
          } catch(FileNotFoundException e){e.printStackTrace();System.exit(1);};
          
          try
          {
              while((nextLine = input.readLine()) != null)
              {
                  nextLine = nextLine.replaceAll(" ", "");
                  nextLine = nextLine.replaceAll("\t", "");
                  //System.out.println("> " + nextLine);
                  if (nextLine.contains(categoryOn + ":"))
                  {
                      //System.out.println(nextLine);
                      return nextLine.split(":")[1].trim();
                  }
                  if (nextLine.contains("<"+categoryOn+">"))
                  {
                      //System.out.println(nextLine);
                      return nextLine.split(">")[1].trim();
                  }

              }
          }
          catch(IOException ioe){ioe.printStackTrace(); System.exit(1);};
          
          return null;
     }
      
      public static void postProcessing()
      {
          if (mathOperation == MATH.AVG)
          {
                postProcessingAVG();
          }
      }
      
      public static void postProcessingAVG()
      {
           Object[] keySet = dataTable.keySet().toArray();
           for (int i = 0; i < keySet.length; i++)
           {
               int numRows = dataTable.get(keySet[i]).size();
               for (int j = 0; j < numRows; j++)
               {
                   Double[] row  = dataTable.get(keySet[i]).get(j);
                   Double contor = contorTable.get(keySet[i]);
                   for (int k = 1; k < row.length; k++) // we don't average over the x-axis, so k>0
                       row[k] = row[k] / contor;
               }
           }
      }
      
     public static void writeResultsBack(String filename)
     {
           File outputFile = new File(filename);
           String header = xAxisTagName;
           String nextLine;
           String row;
           
           FileWriter fileWriter = null;
           BufferedWriter bw = null;
            
            try
            {
                File file = new File(filename); 
                fileWriter = new FileWriter(file, false); // false - overwrite mode
                bw = new BufferedWriter(fileWriter);
                //Map<String,List<Double[]>> dataTable;
                Object[] keySet = dataTable.keySet().toArray();
                for (int i = 0; i < keySet.length; i++)
                    header += OUTPUT_DELIMITER + yAxisTagName + "-" + keySet[i];
                bw.write(header);
                bw.newLine();
                
                // Write the data
                int numRows = dataTable.get(keySet[0]).size();
                for (int i = 0; i < numRows; i++)
                {
                    row = dataTable.get(keySet[0]).get(i)[0].toString(); 
                    for (int j = 0; j < keySet.length; j++)
                        row += OUTPUT_DELIMITER + dataTable.get(keySet[j]).get(i)[1].toString(); 
                    bw.write(row);
                    bw.newLine();
                }
                bw.flush();
                bw.close();
                fileWriter.close();
            } catch (IOException ioe) {
                ioe.printStackTrace();
            } finally {                       // always close the file
                if (bw != null) try {
                   bw.close();
                   fileWriter.close();
                } catch (IOException ioe2) {
                   // just ignore it
                }
            } // end try/catch/finally
     }
     
     public static List<Double[]> collapseRows(List<Double[]> storedDataSet, List<Double[]> newDataSet, MATH mathOperation)
     {
         if (storedDataSet == null)
             return newDataSet;
         
         if (((LinkedList<Double[]>)storedDataSet).size() != ((LinkedList<Double[]>)newDataSet).size())
         {
             System.err.println("[ExtractData] <ERROR> Distinct lengths data sets encountered");
             System.exit(1);
         }
         int index = 0;
         
         for(Double[] row: storedDataSet)
         {
             if (!row[0].equals(newDataSet.get(index)[0]))
             {
                 System.err.println("[ExtractData] <ERROR> xAxis values are out of sync: (" + row[0] + " != " +newDataSet.get(index)[0] + "). They must equal pairwise. Aborting!");
                 System.exit(1);
             }
             switch(mathOperation)
             {
                 case AVG    : row[1] = row[1] + newDataSet.get(index)[1];        ; break;
                 case MIN    : row[1] = Math.min(row[1], newDataSet.get(index)[1]); break;
                 case MAX    : row[1] = Math.max(row[1], newDataSet.get(index)[1]); break;
             }
             index++;
         }
         
         return storedDataSet;
     }
     /**
     * Creates a new instance of ExtractData
     */
    public ExtractData()
    {
    }
} 
