Examples of org.encog.ml.data.buffer.BufferedMLDataSet

org.encog.ml.data.buffer.BufferedMLDataSet
This class is not memory based, so very long files can be used, without running out of memory. This dataset uses a Encog binary training file as a buffer. When used with a slower access dataset, such as CSV, XML or SQL, where parsing must occur, this dataset can be used to load from the slower dataset and train at much higher speeds. This class makes use of Java file channels for maximum file access performance. If you are going to create a binary file, by using the add methods, you must call beginLoad to cause Encog to open an output file. Once the data has been loaded, call endLoad. You can also use the BinaryDataLoader class, with a CODEC, to load many other popular external formats. The binary files produced by this class are in the Encog binary training format, and can be used with any Encog platform. Encog binary files are stored using "little endian" numbers.

    
        BasicMLData inputData = new BasicMLData(inputCount);
        BasicMLData idealData = new BasicMLData(idealCount);


    ReadCSV csv = null;
    BufferedMLDataSet buffer = new BufferedMLDataSet(file);
    buffer.beginLoad(inputCount,idealCount);


    try {
      csv = new ReadCSV(getInputFilename().toString(),
          isExpectInputHeaders(), getFormat());


      resetStatus();
      final int outputLength = this.analyst.determineTotalColumns();


      // write file contents
      while (csv.next() && !shouldStop()) {
        updateStatus(false);


        double[] output = AnalystNormalizeCSV.extractFields(
            this.analyst, this.analystHeaders, csv, outputLength,
            false);


        if (this.series.getTotalDepth() > 1) {
          output = this.series.process(output);
        }
        
        // copy the input
        int idx = 0;
        for(int i=0;i<inputData.size();i++) {
          inputData.setData(i, output[idx++]);
        }
        
        for(int i=0;i<idealData.size();i++) {
          idealData.setData(i, output[idx++]);
        }


        if (output != null) {
          buffer.add(inputData,idealData);
        }
      }
    } finally {
      reportDone(false);
      if (csv != null) {
        try {
          csv.close();
        } catch (final Exception ex) {
          EncogLogging.log(ex);
        }
      }


      if (buffer != null) {
        try {
          buffer.endLoad();
        } catch (final Exception ex) {
          EncogLogging.log(ex);
        }
      }
    }

View Full Code Here

   */
  @Override
  public void performJobUnit(final JobUnitContext context) {


    final BasicNetwork network = (BasicNetwork) context.getJobUnit();
    BufferedMLDataSet buffer = null;
    MLDataSet useTraining = this.training;


    if (this.training instanceof BufferedMLDataSet) {
      buffer = (BufferedMLDataSet) this.training;
      useTraining = buffer.openAdditional();
    }


    // train the neural network


    double error = Double.POSITIVE_INFINITY;
    for (int z = 0; z < this.weightTries; z++) {
      network.reset();
      final Propagation train = new ResilientPropagation(network,
          useTraining);
      final StopTrainingStrategy strat = new StopTrainingStrategy(0.001,
          5);


      train.addStrategy(strat);
      train.setThreadCount(1); // force single thread mode


      for (int i = 0; (i < this.iterations) && !getShouldStop()
          && !strat.shouldStop(); i++) {
        train.iteration();
      }


      error = Math.min(error, train.getError());
    }


    if (buffer != null) {
      buffer.close();
    }


    if (!getShouldStop()) {
      // update min and max

View Full Code Here

      final File binFile, final int inputCount, final int outputCount,
      final boolean headers) {
    binFile.delete();
    final CSVNeuralDataSet csv = new CSVNeuralDataSet(csvFile.toString(),
        inputCount, outputCount, false);
    final BufferedMLDataSet buffer = new BufferedMLDataSet(binFile);
    buffer.beginLoad(inputCount, outputCount);
    for (final MLDataPair pair : csv) {
      buffer.add(pair);
    }
    buffer.endLoad();
  }

View Full Code Here

  private EncogUtility() {


  }


  public static MLDataSet loadEGB2Memory(File filename) {
    BufferedMLDataSet buffer = new BufferedMLDataSet(filename);
    MLDataSet result = buffer.loadToMemory();
    buffer.close();
    return result;
  }

View Full Code Here

    {


        (new File(binFile)).delete();
        CSVNeuralDataSet csv = new CSVNeuralDataSet(csvFile.toString(),
               inputCount, outputCount, headers);
        BufferedMLDataSet buffer = new BufferedMLDataSet(new File(binFile));
        buffer.beginLoad(inputCount, outputCount);
        for(MLDataPair pair : csv)
        {
            buffer.add(pair);
        }
        buffer.endLoad();
    }

View Full Code Here

   {


       binFile.delete();
       ReadCSV csv = new ReadCSV(csvFile.toString(), headers, format);
       
       BufferedMLDataSet buffer = new BufferedMLDataSet(binFile);
       buffer.beginLoad(input.length, ideal.length);
       while(csv.next())
       {
         BasicMLData inputData = new BasicMLData(input.length);
         BasicMLData idealData = new BasicMLData(ideal.length);
         
         // handle input data
         for(int i=0;i<input.length;i++) {
           inputData.setData(i, csv.getDouble(input[i]));
         }
         
         // handle input data
         for(int i=0;i<ideal.length;i++) {
           idealData.setData(i, csv.getDouble(ideal[i]));
         }
         
         // add to dataset
         
           buffer.add(inputData,idealData);
       }
       buffer.endLoad();
   }

View Full Code Here

   * Save a training set to an EGB file.
   * @param f
   * @param data
   */
  public static void saveEGB(File f, MLDataSet data) {
    BufferedMLDataSet binary = new BufferedMLDataSet(f);
    binary.load(data);
    data.close();    
  }

View Full Code Here

public class TestNEAT extends TestCase {
  public final TempDir TEMP_DIR = new TempDir();
  public final File EGB_FILENAME = TEMP_DIR.createFile("encogtest.egb");


  public void testNEATBuffered() {
    BufferedMLDataSet buffer = new BufferedMLDataSet(EGB_FILENAME);
    buffer.beginLoad(2, 1);
    for(int i=0;i<XOR.XOR_INPUT.length;i++) {
      buffer.add(new BasicMLDataPair(
          new BasicMLData(XOR.XOR_INPUT[i]),
          new BasicMLData(XOR.XOR_IDEAL[i])));
    }
    buffer.endLoad();
    
    NEATPopulation pop = new NEATPopulation(2,1,1000);
    pop.setInitialConnectionDensity(1.0);// not required, but speeds training
    pop.reset();

View Full Code Here

  
  
  public void testBufferData() throws Exception 
  {
    new File(FILENAME).delete();
    BufferedMLDataSet set = new BufferedMLDataSet(new File(FILENAME));
    set.beginLoad(2, 1);
    for(int i=0;i<XOR.XOR_INPUT.length;i++) {
      BasicMLData input = new BasicMLData(XOR.XOR_INPUT[i]);
      BasicMLData ideal = new BasicMLData(XOR.XOR_IDEAL[i]);
      set.add(input,ideal);
    }
    set.endLoad();
    
    XOR.testXORDataSet(set);
    
  }

View Full Code Here

        1000, 10000, 10, 10, -1, 1);


    // create the binary file


    file.delete();
    BufferedMLDataSet training2 = new BufferedMLDataSet(file);
    training2.load(training);


    final long start = System.currentTimeMillis();
    final long stop = start + (10 * Evaluate.MILIS);
    int record = 0;


    MLDataPair pair = BasicMLDataPair.createPair(10, 10);


    int iterations = 0;
    while (System.currentTimeMillis() < stop) {
      iterations++;
      training2.getRecord(record++, pair);
      if (record >= training2.getRecordCount())
        record = 0;
    }


    training.close();
    iterations /= 100000;

View Full Code Here

TOP

Related Classes of org.encog.ml.data.buffer.BufferedMLDataSet

org.encog.app.analyst.csv.normalize.AnalystNormalizeToEGB

org.encog.ml.data.basic.BasicMLDataSet

org.encog.ml.data.MLDataError

org.encog.ml.data.MLDataPair

org.encog.neural.data.buffer.TestBufferedNeuralDataSet

org.encog.neural.networks.training.TestNEAT

org.encog.neural.prune.PruneIncremental

org.encog.util.benchmark.EncogBenchmark

org.encog.util.simple.EncogUtility

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.