Examples of SplitAlgorithm

net.sourceforge.align.filter.modifier.modify.split.SplitAlgorithm
Represents modify algorithm splitting single segment into a list of segments. @author Jarek Lipski (loomchild)
org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm

Examples of net.sourceforge.align.filter.modifier.modify.split.SplitAlgorithm

   * @param alignmentList input alignment list
   * @return modified alignment list
   */
  private List<Alignment> unifyRareWords(List<Alignment> alignmentList) {
    
    SplitAlgorithm splitAlgorithm = DEFAULT_TOKENIZE_ALGORITHM;


    Vocabulary sourceVocabulary = new Vocabulary();
    Vocabulary targetVocabulary = new Vocabulary();
    List<List<Integer>> sourceWidList = new ArrayList<List<Integer>>(); 
    List<List<Integer>> targetWidList = new ArrayList<List<Integer>>();

View Full Code Here

Examples of net.sourceforge.align.filter.modifier.modify.split.SplitAlgorithm

  }
  
  private Pair<ModifyAlgorithm, ModifyAlgorithm> createUnifyRareWordsAlgorithms(
      CommandLine commandLine, List<Alignment> alignmentList) {


    SplitAlgorithm splitAlgorithm = DEFAULT_TOKENIZE_ALGORITHM;
    int maxWordCount = createInt(commandLine, "max-word-count", 
        DEFAULT_MAX_WORD_COUNT);
    int minOccurenceCount = createInt(commandLine, "min-occurrence-count", 
        DEFAULT_MIN_OCCURRENCE_COUNT);

View Full Code Here

Examples of net.sourceforge.align.filter.modifier.modify.split.SplitAlgorithm

      //    TranslationModelUtil.DEFAULT_TRAIN_ITERATION_COUNT);
  
      //Prosty algorytm tokenizujący dlatego że wejście powinno być już 
      //stokenizowane - tokeny rozdzielone spacją.
      //TODO: Czy dobrze? nie może być tokenów ze spacją w środku.
      SplitAlgorithm splitAlgorithm = 
        VocabularyUtil.DEFAULT_TOKENIZE_ALGORITHM;
      Vocabulary sourceVocabulary = new Vocabulary();
      Vocabulary targetVocabulary = new Vocabulary();
      
      Parser parser = new AlParser(getIn());

View Full Code Here

Examples of net.sourceforge.align.filter.modifier.modify.split.SplitAlgorithm

   * {@link SeparatorMergeAlgorithm}, applies the filter and checks 
   * if the results are correct.
   */
  @Test
  public void testRunAllFilters() {
    SplitAlgorithm splitAlgorithm = new SplitAlgorithmMock(1);
    AlignAlgorithm alignAlgorithm = new AlignAlgorithmMock(2);
    MergeAlgorithm mergeAlgorithm = new SeparatorMergeAlgorithm();
    List<Filter> filterList = new ArrayList<Filter>();
    filterList.add(new Modifier(splitAlgorithm, splitAlgorithm));
    filterList.add(new Aligner(alignAlgorithm));

View Full Code Here

Examples of org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm

    @Override
    public void run() {
      long startTime, endTime;
      HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(TABLE_NAME));
      desc.addFamily(new HColumnDescriptor(COLUMN_NAME));
      SplitAlgorithm algo = new RegionSplitter.HexStringSplit();
      byte[][] splits = algo.split(REGION_COUNT);


      LOG.info(String.format("Creating table %s with %d splits.",
        TABLE_NAME, REGION_COUNT));
      startTime = System.currentTimeMillis();
      try {

View Full Code Here

Examples of org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm

    @Override
    public void run() {
      long startTime, endTime;
      HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
      desc.addFamily(new HColumnDescriptor(COLUMN_NAME));
      SplitAlgorithm algo = new RegionSplitter.HexStringSplit();
      byte[][] splits = algo.split(REGION_COUNT);


      LOG.info(String.format("Creating table %s with %d splits.",
        TABLE_NAME, REGION_COUNT));
      startTime = System.currentTimeMillis();
      try {

View Full Code Here

Examples of org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm

    @Override
    public void run() {
      long startTime, endTime;
      HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
      desc.addFamily(new HColumnDescriptor(COLUMN_NAME));
      SplitAlgorithm algo = new RegionSplitter.HexStringSplit();
      byte[][] splits = algo.split(REGION_COUNT);


      LOG.info(String.format("Creating table %s with %d splits.",
        TABLE_NAME, REGION_COUNT));
      startTime = System.currentTimeMillis();
      try {

View Full Code Here

Examples of org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm

                new byte[] {(byte)0xef, xFF, xFF, xFF, xFF, xFF, xFF, xFF});
    }


  @Test
  public void testUserInput() {
    SplitAlgorithm algo = new HexStringSplit();
    assertFalse(splitFailsPrecondition(algo)); // default settings are fine
    assertFalse(splitFailsPrecondition(algo, "00", "AA")); // custom is fine
    assertTrue(splitFailsPrecondition(algo, "AA", "00")); // range error
    assertTrue(splitFailsPrecondition(algo, "AA", "AA")); // range error
    assertFalse(splitFailsPrecondition(algo, "0", "2", 3)); // should be fine

View Full Code Here

Examples of org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm

    private void preSplitTableAndVerify(List<byte[]> expectedBounds,
            String splitClass, String tableName) throws Exception {
        final int numRegions = expectedBounds.size()-1;
        final Configuration conf = UTIL.getConfiguration();
        conf.setInt("split.count", numRegions);
        SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
        RegionSplitter.createPresplitTable(tableName, splitAlgo,
                new String[] {CF_NAME}, conf);
        verifyBounds(expectedBounds, tableName);
    }

View Full Code Here

Examples of org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm

            List<byte[]> expectedBounds)  throws Exception {
        final Configuration conf = UTIL.getConfiguration();


        // Set this larger than the number of splits so RegionSplitter won't block
        conf.setInt("split.outstanding", 5);
        SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
        RegionSplitter.rollingSplit(tableName, splitAlgo, conf);
        verifyBounds(expectedBounds, tableName);
    }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.