/*
 * SIDnetCSVRunner.java
 *
 * Created on March 3, 2008, 7:38 PM
 */

package sidnet.batch;

import au.com.bytecode.opencsv.CSVReader;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import org.apache.log4j.*;
import sidnet.core.misc.FileUtils;

/**
 *
 * @author  Oliver
 */
public class SIDnetCSVRunner extends javax.swing.JFrame
{
    private static final String DETAILED_LOG_FILENAME_PREFIX = "DetailedExperimentReport";
    private static final String SUMMARY_LOG_FILENAME_PREFIX  = "SummaryExperimentReport";
    private static final String EXPERIMENT_CONTOR_FILE       = "runcontor.cnt";
    private static       String experimentsTargetDirectory   = "";
    private static Logger detailedLog;
    private static Logger summaryLog;
    
    private static Process p = null; 
    private static int totalNumberAttemptedRuns = 0;
    private static int totalNumberAbortedRuns = 0;
    private static List<Integer> experimentsMatrix = new LinkedList<Integer>();
    private static LinkedList<Integer> launchedExperiments = new LinkedList<Integer>();
    private static LinkedList<Long> launchedExperimentsIds = new LinkedList<Long>();

    private static String experimentsStartDateAndTime = getDateTime();
    private static long startTime;
    private static long runId = 0;
    private static int degreeOfParallelism = 1;
    private static int instanceContor = 0;
    private static LinkedList<SIDnetInstanceLauncher> SIDnetInstanceList;
    private static long lastCommitedInstanceIndex = 0;
    private static long userExperimentId    = -1;
    private static long userRunId           = -1;
    private static long currentExperimentId = -1;
    
    private static boolean started = false;
    private static boolean quitted = false;
    private static long totalNumberOfExperiments = 0;
    
    private static boolean linuxEnv = false;
    
    private static long passedExperimentsCount = 0;
    private static long failedExperimentsCount = 0;
    private static long remainingExperimentsCount = 0;

  
    /**
     * Creates new form SIDnetCSVRunner
     */
    public SIDnetCSVRunner()
    {
        initComponents();
        jButtonQuit.setEnabled(false);
    }
    
    /** This method is called from within the constructor to
     * initialize the form.
     * WARNING: Do NOT modify this code. The content of this method is
     * always regenerated by the Form Editor.
     */
    // <editor-fold defaultstate="collapsed" desc=" Generated Code ">//GEN-BEGIN:initComponents
    private void initComponents()
    {
        jMainPanel = new javax.swing.JPanel();
        jButtonStart = new javax.swing.JButton();
        jButtonQuit = new javax.swing.JButton();
        jLabel1 = new javax.swing.JLabel();
        jTextRunId = new javax.swing.JTextField();
        jLabel3 = new javax.swing.JLabel();
        jTextDegreeParallelism = new javax.swing.JTextField();
        jRuntimeProgress = new javax.swing.JPanel();
        jProgressBar1 = new javax.swing.JProgressBar();
        jLabel4 = new javax.swing.JLabel();
        jTextElapsedTime = new javax.swing.JTextField();
        jLabel5 = new javax.swing.JLabel();
        jExperimentsCount = new javax.swing.JTextField();
        jLabel6 = new javax.swing.JLabel();
        jPassedExperimentsCount = new javax.swing.JTextField();
        jLabel7 = new javax.swing.JLabel();
        jFailedExperimentsCount = new javax.swing.JTextField();
        jLabel2 = new javax.swing.JLabel();
        jRemainingExperimentsCount = new javax.swing.JTextField();

        setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);
        setTitle("SIDnet CSV Runner v1.0");
        setResizable(false);
        jMainPanel.setBorder(javax.swing.BorderFactory.createEtchedBorder());
        jButtonStart.setText("START");
        jButtonStart.addActionListener(new java.awt.event.ActionListener()
        {
            public void actionPerformed(java.awt.event.ActionEvent evt)
            {
                jButtonStartActionPerformed(evt);
            }
        });

        jButtonQuit.setText("Safe Interrupt");
        jButtonQuit.addActionListener(new java.awt.event.ActionListener()
        {
            public void actionPerformed(java.awt.event.ActionEvent evt)
            {
                jButtonQuitActionPerformed(evt);
            }
        });

        jLabel1.setText("Run ID");

        jTextRunId.setEditable(false);
        jTextRunId.setText("jTextField1");
        jTextRunId.setBorder(null);

        jLabel3.setText("Degree of parallelism");

        jTextDegreeParallelism.setEditable(false);
        jTextDegreeParallelism.setText("jTextField3");
        jTextDegreeParallelism.setBorder(null);

        jRuntimeProgress.setBorder(javax.swing.BorderFactory.createTitledBorder(""));
        jRuntimeProgress.setToolTipText("Experiment Progress Window");
        org.jdesktop.layout.GroupLayout jRuntimeProgressLayout = new org.jdesktop.layout.GroupLayout(jRuntimeProgress);
        jRuntimeProgress.setLayout(jRuntimeProgressLayout);
        jRuntimeProgressLayout.setHorizontalGroup(
            jRuntimeProgressLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
            .add(0, 401, Short.MAX_VALUE)
        );
        jRuntimeProgressLayout.setVerticalGroup(
            jRuntimeProgressLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
            .add(0, 0, Short.MAX_VALUE)
        );

        jProgressBar1.setFocusable(false);

        jLabel4.setText("Elapsed Time");

        jTextElapsedTime.setEditable(false);
        jTextElapsedTime.setText("N/A");
        jTextElapsedTime.setBorder(null);

        jLabel5.setText("No. of Experiments");

        jExperimentsCount.setEditable(false);
        jExperimentsCount.setText("N/A");
        jExperimentsCount.setAutoscrolls(false);
        jExperimentsCount.setBorder(null);

        jLabel6.setText("#Passed");

        jPassedExperimentsCount.setEditable(false);
        jPassedExperimentsCount.setText("N/A");
        jPassedExperimentsCount.setBorder(null);

        jLabel7.setText("#Failed");

        jFailedExperimentsCount.setEditable(false);
        jFailedExperimentsCount.setText("N/A");
        jFailedExperimentsCount.setBorder(null);

        jLabel2.setText("#Remaining");

        jRemainingExperimentsCount.setEditable(false);
        jRemainingExperimentsCount.setText("N/A");
        jRemainingExperimentsCount.setBorder(null);

        org.jdesktop.layout.GroupLayout jMainPanelLayout = new org.jdesktop.layout.GroupLayout(jMainPanel);
        jMainPanel.setLayout(jMainPanelLayout);
        jMainPanelLayout.setHorizontalGroup(
            jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
            .add(jMainPanelLayout.createSequentialGroup()
                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
                    .add(jMainPanelLayout.createSequentialGroup()
                        .addContainerGap()
                        .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
                            .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
                                .add(org.jdesktop.layout.GroupLayout.TRAILING, jRuntimeProgress, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                                .add(jProgressBar1, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, 417, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                            .add(jMainPanelLayout.createSequentialGroup()
                                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
                                    .add(jButtonStart, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, 201, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                                    .add(jMainPanelLayout.createSequentialGroup()
                                        .add(jLabel1)
                                        .add(21, 21, 21)
                                        .add(jTextRunId, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)))
                                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
                                    .add(jMainPanelLayout.createSequentialGroup()
                                        .add(jLabel3)
                                        .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                                        .add(jTextDegreeParallelism, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                                    .add(jButtonQuit, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, 210, Short.MAX_VALUE)))))
                    .add(jMainPanelLayout.createSequentialGroup()
                        .add(50, 50, 50)
                        .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
                            .add(jLabel4)
                            .add(jLabel5)
                            .add(jLabel6)
                            .add(jLabel7)
                            .add(jLabel2))
                        .add(7, 7, 7)
                        .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING, false)
                            .add(jRemainingExperimentsCount, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                            .add(jFailedExperimentsCount, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                            .add(jPassedExperimentsCount)
                            .add(jExperimentsCount, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                            .add(jTextElapsedTime, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))))
                .addContainerGap())
        );
        jMainPanelLayout.setVerticalGroup(
            jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
            .add(jMainPanelLayout.createSequentialGroup()
                .addContainerGap()
                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.BASELINE)
                    .add(jLabel1)
                    .add(jTextRunId, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                    .add(jLabel3)
                    .add(jTextDegreeParallelism, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING, false)
                    .add(jButtonQuit, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                    .add(jButtonStart, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, 46, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                .add(jRuntimeProgress, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                .add(jProgressBar1, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, 27, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.BASELINE)
                    .add(jLabel4)
                    .add(jTextElapsedTime, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.BASELINE)
                    .add(jLabel5)
                    .add(jExperimentsCount, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.BASELINE)
                    .add(jLabel6)
                    .add(jPassedExperimentsCount, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.BASELINE)
                    .add(jLabel7)
                    .add(jFailedExperimentsCount, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                .addPreferredGap(org.jdesktop.layout.LayoutStyle.RELATED)
                .add(jMainPanelLayout.createParallelGroup(org.jdesktop.layout.GroupLayout.BASELINE)
                    .add(jLabel2)
                    .add(jRemainingExperimentsCount, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE))
                .addContainerGap(org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
        );

        org.jdesktop.layout.GroupLayout layout = new org.jdesktop.layout.GroupLayout(getContentPane());
        getContentPane().setLayout(layout);
        layout.setHorizontalGroup(
            layout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
            .add(layout.createSequentialGroup()
                .addContainerGap()
                .add(jMainPanel, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                .addContainerGap())
        );
        layout.setVerticalGroup(
            layout.createParallelGroup(org.jdesktop.layout.GroupLayout.LEADING)
            .add(layout.createSequentialGroup()
                .addContainerGap()
                .add(jMainPanel, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE, org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, org.jdesktop.layout.GroupLayout.PREFERRED_SIZE)
                .addContainerGap(org.jdesktop.layout.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
        );
        pack();
    }// </editor-fold>//GEN-END:initComponents

    private void jButtonStartActionPerformed(java.awt.event.ActionEvent evt)//GEN-FIRST:event_jButtonStartActionPerformed
    {//GEN-HEADEREND:event_jButtonStartActionPerformed
        started = true;
        jButtonStart.setEnabled(false);
        jButtonQuit.setEnabled(true);
        
    }//GEN-LAST:event_jButtonStartActionPerformed

    private void jButtonQuitActionPerformed(java.awt.event.ActionEvent evt)//GEN-FIRST:event_jButtonQuitActionPerformed
    {//GEN-HEADEREND:event_jButtonQuitActionPerformed
        quitted = true;
        jButtonQuit.setEnabled(false);
    }//GEN-LAST:event_jButtonQuitActionPerformed
    
 
    
    // Variables declaration - do not modify//GEN-BEGIN:variables
    private javax.swing.JButton jButtonQuit;
    private javax.swing.JButton jButtonStart;
    private javax.swing.JTextField jExperimentsCount;
    private javax.swing.JTextField jFailedExperimentsCount;
    private javax.swing.JLabel jLabel1;
    private javax.swing.JLabel jLabel2;
    private javax.swing.JLabel jLabel3;
    private javax.swing.JLabel jLabel4;
    private javax.swing.JLabel jLabel5;
    private javax.swing.JLabel jLabel6;
    private javax.swing.JLabel jLabel7;
    private javax.swing.JPanel jMainPanel;
    private javax.swing.JTextField jPassedExperimentsCount;
    private javax.swing.JProgressBar jProgressBar1;
    private javax.swing.JTextField jRemainingExperimentsCount;
    private javax.swing.JPanel jRuntimeProgress;
    private javax.swing.JTextField jTextDegreeParallelism;
    private javax.swing.JTextField jTextElapsedTime;
    private javax.swing.JTextField jTextRunId;
    // End of variables declaration//GEN-END:variables
    
      /**
     * @param args the command line arguments
     */
     public static void main(String[] cmdLineArgs) throws Exception 
    {
         SIDnetCSVRunner runnerInterface = new SIDnetCSVRunner();
         runnerInterface.setVisible(true);

        if (cmdLineArgs.length == 0)
        {
            System.out.println("<SIDnetCSVRunner>[ERROR]: Invalid number of arguments specified");
            System.out.println("syntax: SIDnetCSVRunner <fileName>.csv [-runid=#] [-experimentid=#] [ -demo | -experiment ] -parallelism=#");
            return;
        }

        if(!cmdLineArgs[0].endsWith(".csv"))
        {
            System.out.println("<SIDnetCSVRunner>[ERROR]: Invalid filename. Only .csv files supported. The filename must have the .csv extension");
            System.out.println("syntax: SIDnetCSVRunner <fileName>.csv [-runid=#] [experimentid=#] [ -demo | -experiment ]");
            return;
        }    

        for (int i = 1; i < cmdLineArgs.length; i++)
        {
            if (cmdLineArgs[i].toLowerCase().startsWith("-parallelism") || cmdLineArgs[i].toLowerCase().startsWith("parallelism"))
            {   
                String substring;
                try{
                    substring = cmdLineArgs[i].split("=", 2)[1];
                }catch(Exception e)
                {
                    i++;
                    substring = cmdLineArgs[i];
                }

                degreeOfParallelism = Integer.parseInt(substring);
            }
            if (cmdLineArgs[i].toLowerCase().startsWith("-experimentid") || cmdLineArgs[i].toLowerCase().startsWith("experimentid"))
            {   
                String substring;
                try{
                    substring = cmdLineArgs[i].split("=", 2)[1];
                }catch(Exception e)
                {
                    i++;
                    substring = cmdLineArgs[i];
                }

                userExperimentId = Long.parseLong(substring);
            }
            if (cmdLineArgs[i].toLowerCase().startsWith("-runid") || cmdLineArgs[i].toLowerCase().startsWith("runid"))
            {   
                String substring;
                try{
                    substring = cmdLineArgs[i].split("=", 2)[1];
                }catch(Exception e)
                {
                    i++;
                    substring = cmdLineArgs[i];
                }

                userRunId = Long.parseLong(substring);
            }
            if (cmdLineArgs[i].toLowerCase().equals("linux") || cmdLineArgs[i].toLowerCase().equals("-linux")) 
                linuxEnv = true;
        }

        if (userRunId == -1)
            runId = retrieveRunId();
        else
            runId = userRunId;  
         
       runnerInterface.jTextRunId.setText("" + runId);
       runnerInterface.jTextDegreeParallelism.setText("" + degreeOfParallelism);

       

        System.out.println("\n\nSIDnetCSVRunner started!");
        System.out.println("**************************************************************************************");
        if (linuxEnv)
            System.out.println("*\n* Operating System: Linux");
        else
            System.out.println("*\n* Operating System: Windows baby");
        System.out.println("*\n* Working/Current directory: " + (new File(".")).getAbsolutePath());

       

        // Create a dedicated directory
        if (!linuxEnv)
            experimentsTargetDirectory += ".\\run" + runId + "\\";
        else
            experimentsTargetDirectory += "" +
                    "run" + runId + "/";

        // Create the experiments target directory, if not existant
        if (new File(experimentsTargetDirectory).exists())
            System.out.println("*\n* Target directory: " + experimentsTargetDirectory + " already exists. Only experiments associated to the missing experiments log files will be run!");
        else
        {
            boolean success = (new File(experimentsTargetDirectory)).mkdir();
            if (success) {
              System.out.println("*\n* New target directory: " + experimentsTargetDirectory + " succesfully created");
            }    
            else 
            {
                System.out.println("*\n* Cannot create experiment target directory: " + experimentsTargetDirectory);
                System.exit(1);
            }
        }

        System.out.println("*\n* Degree of parallelism: " + degreeOfParallelism);

        System.out.println("*\n**************************************************************************************");

        configureSummaryLogger("run" + runId + "-" + SUMMARY_LOG_FILENAME_PREFIX);
        configureDetailedLogger("run" + runId + "-" + DETAILED_LOG_FILENAME_PREFIX);

        CSVReader reader = null;
        try{
             reader = new CSVReader(new FileReader(cmdLineArgs[0]));
        }catch(IOException e){e.printStackTrace();}
        String [] nextLine;
        int lineNumber = -1; // 0 is header

        /* ************** Detailed logger *************** */
        detailedLog.info("Detailed Report for SIDnet Experiments Run #" + runId +"\n\n");
        detailedLog.info("Date/Time                       : " + experimentsStartDateAndTime);
        detailedLog.info("CSV file                        : " + cmdLineArgs[0]);
        detailedLog.info("********************************************************** ");

        SIDnetInstanceList = new LinkedList<SIDnetInstanceLauncher>();
        if (userExperimentId == -1)
        {
            // determine the number of experiments
            lineNumber=-1;
            while ((nextLine = reader.readNext()) != null)
            {
                lineNumber ++;
                if (lineNumber == 0)
                    continue; // skip the header
                // determine if this particular experiment needs to be performed or not
                //System.out.println("lineNumber = " + lineNumber);
                //for (int i = 0; i < nextLine.length; i++)
                    //System.out.println(nextLine[i]);
                if (1 > nextLine.length)
                {
                    System.err.println("Invalid content found at the end of the .csv file. Please check and erase the possible extra lines at the end of the .csv file");
                    System.exit(1);
                }
                currentExperimentId = Long.parseLong(nextLine[1]);
                File logFile = new File(experimentsTargetDirectory, "err-Exp" + currentExperimentId + ".tmp");
                if (!logFile.exists())
                    totalNumberOfExperiments++;
            }
        }
        else
            totalNumberOfExperiments = 1;
        
        runnerInterface.jExperimentsCount.setText("" + totalNumberOfExperiments);
        runnerInterface.jTextElapsedTime.setText("0s");
        runnerInterface.jProgressBar1.setMaximum((int)totalNumberOfExperiments);
        runnerInterface.jProgressBar1.setString("Experiment Count");
        runnerInterface.jProgressBar1.setStringPainted(true);
        runnerInterface.jPassedExperimentsCount.setText("0");
        runnerInterface.jFailedExperimentsCount.setText("0");
        runnerInterface.jRemainingExperimentsCount.setText("" + totalNumberOfExperiments);
        
        
        while (!started && !quitted)
            Thread.sleep(100);

        if (quitted)
            System.exit(0);
        
        startTime = System.currentTimeMillis();
        
        // reopen the reader
        try{
             reader = new CSVReader(new FileReader(cmdLineArgs[0]));
        }catch(IOException e){e.printStackTrace();}
        
        lineNumber = -1;
        while ((nextLine = reader.readNext()) != null && !quitted) {
            
            lineNumber++;      
            if (lineNumber == 0)
                System.out.println("<"+cmdLineArgs[0] + "> Header read succesfully");
            else
            {
                while(SIDnetInstanceList.size() >= degreeOfParallelism && !quitted)
                {
                    SIDnetInstanceLauncher removable = null;

                    // check for SIDnet instance terminations
                    for(SIDnetInstanceLauncher SIDnetInstance: SIDnetInstanceList)
                        if (!SIDnetInstance.isAlive())
                        {
                            removable = SIDnetInstance;
                            FileUtils.appendToFilesWithPrefix(experimentsTargetDirectory, "run" + SIDnetInstance.getRunNumber() + "-exp" + SIDnetInstance.getExperimentId(), "\n<elapsedTimeMinutes>   " + (System.currentTimeMillis() - SIDnetInstance.getStartTimeMillis())/60000 + "   </elapsedTimeMinutes>\n");
                            break;
                        }
                    if (removable != null)
                        SIDnetInstanceList.remove(removable);
                    try{
                        runnerInterface.jTextElapsedTime.setText(elapsedFormatedTime(System.currentTimeMillis() - startTime));
                        Thread.sleep(1000);
                    }catch(Exception e){e.printStackTrace();};
                }

                currentExperimentId = Long.parseLong(nextLine[1]);
                if (userExperimentId != -1 && userExperimentId != currentExperimentId /*expId*/)
                    continue;

                if (quitted)
                    break;

            File logFile = new File(experimentsTargetDirectory, "err-Exp" + currentExperimentId + ".tmp");
            if (logFile.exists())
                {
                    System.out.println("\n---------------------\n Skipping SIDnet [" + currentExperimentId + "] because experiments results already exist in target directory for run# " + runId + " --------------------------------------------\n---------------------\n");
                    // check to see if the log file is empty or not
                    if (FileUtils.isEmpty(logFile))
                        passedExperimentsCount++;
                    else
                        failedExperimentsCount++;
                    continue;
                }
                else
                {
                    // remove any existing log file for this runid/experimentid
                    FileUtils.deleteFilesWithPrefix(experimentsTargetDirectory, "run" + runId + "-exp" + currentExperimentId);
                }

                totalNumberAttemptedRuns ++;
                
                remainingExperimentsCount = totalNumberOfExperiments - totalNumberAttemptedRuns;
                
                runnerInterface.jPassedExperimentsCount.setText("" + passedExperimentsCount);
                runnerInterface.jFailedExperimentsCount.setText("" + failedExperimentsCount);
                runnerInterface.jRemainingExperimentsCount.setText("" + remainingExperimentsCount);
                

                System.out.println("\n---------------------\n Launching SIDnet [" + currentExperimentId + "] --------------------------------------------\n---------------------\n");

                String arguments = "";
                for (int i = 1; i < nextLine.length; i++)
                    arguments += nextLine[i] + " ";    

                System.out.println("> java jist.runtime.Main jist.swans.Main " + nextLine[0] + " " + experimentsTargetDirectory + " " + runId + " " + arguments + " \n");

                runnerInterface.jProgressBar1.setValue(lineNumber);
                runnerInterface.jProgressBar1.setString("#"+lineNumber);
                
                SIDnetInstanceList.add(new SIDnetInstanceLauncher("java jist.runtime.Main jist.swans.Main " + nextLine[0] + " " + experimentsTargetDirectory + " " + runId + " " + arguments, runId, currentExperimentId, experimentsTargetDirectory));
                //launchedExperiments.add(lineNumber);
                //launchedExperimentsIds.add(currentExperimentId);
                SIDnetInstanceList.getLast().start();
            }   
        }

          while(SIDnetInstanceList.size() > 0) // this must not be checked for quitted
                {
                    SIDnetInstanceLauncher removable = null;

                    // check for SIDnet instance terminations
                    for(SIDnetInstanceLauncher SIDnetInstance: SIDnetInstanceList)
                    {
                        if (!SIDnetInstance.isAlive())
                        {                        
                            removable = SIDnetInstance;
                            // commit temp error logs
                            //commitTempErrorFile(experimentsTargetDirectory, "err-Exp" + launchedExperiments.getFirst() + ".tmp", consecutiveExperimentsNumber, launchedExperimentsIds.getFirst());
                            //launchedExperiments.removeFirst();
                            //launchedExperimentsIds.removeFirst();
                            FileUtils.appendToFilesWithPrefix(experimentsTargetDirectory, "run" + SIDnetInstance.getRunNumber() + "-exp" + SIDnetInstance.getExperimentId(), "\n<elapsedTimeMinutes>" + (System.currentTimeMillis() - SIDnetInstance.getStartTimeMillis())/60000 + "</minutes>\n");
                            break;       
                        } 
                        else
                            if (quitted)
                                SIDnetInstance.quit();
                    }
                    if (removable != null)
                        SIDnetInstanceList.remove(removable);
                }

        if (totalNumberAttemptedRuns == 0)
        {
            if (lineNumber <= 1)
                System.out.println("[WARNING] The indicated .csv file does not contain any experiments parameters");
            else
                if (userExperimentId != -1)
                    System.out.println("[WARNING] The indicated experiment ID# ["  + userExperimentId + "] cannot be found in the .csv");
        }

         /* ************** Summary logger *************** */
        long elapsedTime = System.currentTimeMillis() - startTime;
        summaryLog.info("Summary Report for SIDnet Experiments Run #" + runId + "\n\n");
        summaryLog.info("Date/Time                       : " + experimentsStartDateAndTime);
        summaryLog.info("CSV file                        : " + cmdLineArgs[0]);
        summaryLog.info("Total Number Attempted Runs     : " + totalNumberAttemptedRuns);
        summaryLog.info("Total Number Aborted Runs       : " + totalNumberAbortedRuns);
        if (totalNumberAttemptedRuns != 0)
            summaryLog.info("Percentage successful runs      : " + (totalNumberAttemptedRuns - totalNumberAbortedRuns) * 100 / totalNumberAttemptedRuns + " %");
        else
            summaryLog.info("Percentage successful runs      : 0 %");
        summaryLog.info("Experiments ended at            : " + getDateTime());
        summaryLog.info("Experiment duration             : " + elapsedFormatedTime(elapsedTime));
        if (totalNumberAttemptedRuns != 0)
            summaryLog.info("Average time per run            : " + elapsedFormatedTime(elapsedTime/totalNumberAttemptedRuns));
        else
            summaryLog.info("Average time per run            : N/A");
        summaryLog.info("\n\nExperiments Matrix");
        String str="";
        int count = 0;
        int interval = 20;
        // build horizontal ruller
        for (int i = 0; i < interval; i++ )
            if (i < 10)
                str += i + "  ";
            else
                str += i + " ";

        summaryLog.info("\t\t" + str);
        str = "";

        for (Integer errorCode: experimentsMatrix)
        {
            if (count % interval == 0)
            {
                if (count != 0)
                {
                    String padding = "";
                    String header = "[" + (count/interval-1)*interval + " ... " + (count/interval)*interval + "]: ";
                    for (int i = 0; i < 16 - header.length(); i++)
                        padding +=" ";
                    summaryLog.info( header + padding + str);
                }
                str = "";
            }
            if (errorCode == 0)
                str += ".  ";
            else
            {
                if (errorCode < 10)
                    str += errorCode + "  ";
                else
                    str += errorCode + " ";
            }
            count++;
        }
        if ((count-1) % interval != 0)
        {
            String padding = "";
            String header = "[" + (count/interval)*interval + " ... " + (count/interval+1)*interval + "]: ";
            for (int i = 0; i < 16 - header.length(); i++)
                 padding +=" ";
            summaryLog.info(header + padding + str);
        }
     
            
        System.exit(0);
    }

    
    public static void commitTempErrorFile(String experimentsTargetDirectory, String filename, long runNumber, long runId)
    {
        int MAX_LENGTH = 30;
        
        detailedLog.info("\n---------------\n Experiment #" + runId + " --------------------------------------------------\n---------------\n");
         
        File tmpFile = new File(experimentsTargetDirectory + filename);
        String nextLine;
        
        try{
            // If the file does not exists, create one
            if (!tmpFile.exists())
            {
                System.out.println("[FATAL ERROR]<SIDnetCSVRunner> - cannot locate temporary error log files \"err-Exp#.tmp\" which should have been created automatically by now");
                detailedLog.info("[FATAL ERROR]<SIDnetCSVRunner> - cannot locate temporary error log files \"err-Exp#.tmp\" which should have been created automatically by now");        
            }
            else
            {
                // Here BufferedInputStream is added for fast reading.
                BufferedReader input = new BufferedReader(new FileReader(tmpFile));
                
                while((nextLine = input.readLine()) != null)
                    detailedLog.info(nextLine);

                tmpFile.delete();
            }
        }catch(Exception e){e.printStackTrace();detailedLog.info("[EXCEPTION]<SIDnetCSVRunner>(CommitTempErrorFile)");};
    }
    
    public static String elapsedFormatedTime(long elapsedTimeMillis)
    {
        String str = "";
        long hours, minutes, seconds, milliseconds;
        
        hours = elapsedTimeMillis/(1000*60*60);
        elapsedTimeMillis -= hours * (1000*60*60);
        
        minutes = elapsedTimeMillis/(1000*60);
        elapsedTimeMillis -= minutes * (1000*60);
        
        seconds = elapsedTimeMillis/(1000);
        elapsedTimeMillis -= seconds * (1000);
        
        milliseconds       = elapsedTimeMillis;
        
        str = "" + hours + "h " + minutes + "m " + seconds + "s " + milliseconds +"ms";
        
        return str;
    }
    
    private static void configureSummaryLogger(String fileNamePrefix)
    {
        // Configure the summary report logger
        String filename = fileNamePrefix + getDateTime() + ".log";
        
        FileAppender appender; 
        try
        {
            PatternLayout layout = new PatternLayout("%m %n");
            appender = new FileAppender(layout, experimentsTargetDirectory + filename, true);
        }
        catch(IOException e)
        {
            e.printStackTrace();
            throw new RuntimeException("Unable to configure loggin property");
        }

        summaryLog = Logger.getLogger(filename);
        summaryLog.addAppender(appender);
        summaryLog.setLevel((Level)Level.INFO);      
        summaryLog.setAdditivity(false);
    }

    private static void configureDetailedLogger(String fileNamePrefix)
    {
        // Configure the summary report logger
        String filename = fileNamePrefix  + getDateTime() + ".log";
        
        FileAppender appender; 
        try
        {
            PatternLayout layout = new PatternLayout("%m %n");
            appender = new FileAppender(layout, experimentsTargetDirectory + filename, true);
        }
        catch(IOException e)
        {
            e.printStackTrace();
            throw new RuntimeException("Unable to configure loggin property");
        }

        detailedLog = Logger.getLogger(filename);
        detailedLog.addAppender(appender);
        detailedLog.setLevel((Level)Level.INFO);      
        detailedLog.setAdditivity(false);
    }
    
    
    
    public static long retrieveRunId()
    {
        long experimentSuiteIdNumber = 1;
        // Generate the experimentSuite#
        File experimentSuiteFile = new File(EXPERIMENT_CONTOR_FILE);
        
        try{
            // If the file does not exists, create one
            if (!experimentSuiteFile.exists())
            {
                System.out.println("*\n* Experiment contor tracker file (" + EXPERIMENT_CONTOR_FILE + ") does not exists. Creating one that starts at #1");

                experimentSuiteFile.createNewFile();

                FileOutputStream fout = new FileOutputStream(experimentSuiteFile);
                new PrintStream(fout).println(experimentSuiteIdNumber);
                fout.close();
            }
            else
            {
                // Here BufferedInputStream is added for fast reading.
                BufferedReader input = new BufferedReader(new FileReader(experimentSuiteFile));

                // dis.available() returns 0 if the file does not have more lines.
                long lastExperimentSuiteId = Long.parseLong((String)input.readLine());
                experimentSuiteIdNumber = lastExperimentSuiteId + 1;

                // write back the experiment suite id
                System.out.println("*\n* Experiment contor tracker file found. Executing experiment suite #" + experimentSuiteIdNumber);

                experimentSuiteFile.delete();

                experimentSuiteFile.createNewFile();

                FileOutputStream fout = new FileOutputStream(experimentSuiteFile);
                new PrintStream(fout).println(experimentSuiteIdNumber);
                fout.close();
                input.close();
            }
        }catch(Exception e){e.printStackTrace();};
        return experimentSuiteIdNumber;
    }

    
     public static String getDateTime() {
        DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd--HH-mm-ss");
        Date date = new Date();
        return dateFormat.format(date);
    }
}

 class SIDnetInstanceLauncher extends Thread
 {
     private String command;
     private Logger errorLog;
     private Process p = null;
     private long runNumber;
     private long experimentId;
     private String experimentsTargetDirectory;
     private boolean quitted = false;
     private long startTimeMillis;
     
     public SIDnetInstanceLauncher(String command, long runNumber, long experimentId, String experimentsTargetDirectory)
     {
         this.command = command;
         this.runNumber = runNumber;
         this.experimentId = experimentId;
         this.experimentsTargetDirectory = experimentsTargetDirectory;
         errorLog = configureTempErrorLogger(experimentsTargetDirectory, "err-Exp" + experimentId);
     }
     
     public void run()
     {
          try{
            startTimeMillis = System.currentTimeMillis();
            p = Runtime.getRuntime().exec(command, null);

            // any error message?
            StreamGobbler errorGobbler = new 
                StreamGobbler(p.getErrorStream(), "ERROR", p, errorLog);            

            // any output?
            StreamGobbler outputGobbler = new 
                StreamGobbler(p.getInputStream(), "OUTPUT", p, null);

            // kick them off
            errorGobbler.start();  
            outputGobbler.start();

            int exitValue = p.waitFor();

            errorGobbler.stopGlobber();
            outputGobbler.stopGlobber();

            //experimentsMatrix.add(new Integer(exitValue));

            switch(exitValue)
            {
                case 0: System.out.println("\tSIDnet terminated normally!");break;
                case 1: System.out.println("\t! ! ! SIDnet terminated abnormally. Exit code: " + exitValue);break;
            }
            if (quitted)
            {
                 System.out.println("SIDnet experimentId#" + experimentId + " quitted. Remove associated log files");
                 errorLog.removeAllAppenders();
                 FileUtils.deleteFilesWithPrefix(experimentsTargetDirectory, "run" + runNumber + "-exp" + experimentId);
                 FileUtils.deleteFilesWithPrefix(experimentsTargetDirectory, "err-Exp" + experimentId);
            }
          }
            catch(Throwable e)
            {
                //totalNumberAbortedRuns++;
                System.out.println("Aborting current run due to run-time errors");
                e.printStackTrace();
                p.destroy();
            }
     }
          
      public void quit()
      {
          quitted = true;
          p.destroy();
      }
     
     
     public long getRunNumber()
     {
         return runNumber;
     }
     
     public long getExperimentId()
     {
         return experimentId;
     }
     
     public long getStartTimeMillis()
     {
         return startTimeMillis;
     }
     
    private Logger configureTempErrorLogger(String experimentsTargetDirectory, String fileNamePrefix)
    {
        // Configure the summary report logger
        Logger errorLog;
        String filename = fileNamePrefix  + ".tmp";
        
        FileAppender appender; 
        try
        {
            PatternLayout layout = new PatternLayout("%m %n");
            appender = new FileAppender(layout, experimentsTargetDirectory + filename, true);
        }
        catch(IOException e)
        {
            e.printStackTrace();
            throw new RuntimeException("Unable to configure loggin property");
        }

        errorLog = Logger.getLogger(filename);
        errorLog.addAppender(appender);
        errorLog.setLevel((Level)Level.ERROR);      
        errorLog.setAdditivity(false);
        
        return errorLog;
    }
 }
 


 class StreamGobbler extends Thread
    {
        InputStream is;
        String type;
        Process p = null;
        Logger log = null;
        boolean terminated = false;
        

        StreamGobbler(InputStream is, String type, Process p, Logger log)
        {
            this.is = is;
            this.type = type;
            this.p = p;
            this.log = log;
        }
        
        public void stopGlobber()
        {
            while(!terminated)
            {
                try{
                    this.sleep(100);
                }catch(Exception e){e.printStackTrace();};
            }
        }

        public void run()
        {
            try
            {
                InputStreamReader isr = new InputStreamReader(is);
                BufferedReader br = new BufferedReader(isr);
                String line=null;
                while ( (line = br.readLine()) != null)
                {   
                    System.out.println(line);    
                    if (type.equals("ERROR") && p != null)
                    {
                         if (log != null)
                            log.error(line); 
                         if (!br.ready())
                            p.destroy();
                    }
                }
                } catch (IOException ioe)
                  {
                        if (log != null)
                            log.error(ioe);
                        ioe.printStackTrace();
                        p.destroy();
                  }
                terminated = true;  
        }
}
