Examples of SortInfo


Examples of org.apache.hadoop.zebra.types.SortInfo

   * @return the zebra tables's SortInfo; null if the table is unsorted.
   */
  public static SortInfo getSortInfo(JobConf conf) throws IOException
  {
    TableExpr expr = getInputExpr(conf);
    SortInfo result = null;
    int sortSize = 0;
    if (expr instanceof BasicTableExpr)
    {
      BasicTable.Reader reader = new BasicTable.Reader(((BasicTableExpr) expr).getPath(), conf);
      SortInfo sortInfo = reader.getSortInfo();
      reader.close();
      result = sortInfo;
    } else {
      List<LeafTableInfo> leaves = expr.getLeafTables(null);
      for (Iterator<LeafTableInfo> it = leaves.iterator(); it.hasNext(); )
      {
        LeafTableInfo leaf = it.next();
        BasicTable.Reader reader = new BasicTable.Reader(leaf.getPath(), conf);
        SortInfo sortInfo = reader.getSortInfo();
        reader.close();
        if (sortSize == 0)
        {
          sortSize = sortInfo.size();
          result = sortInfo;
        } else if (sortSize != sortInfo.size()) {
          throw new IOException("Tables of the table union do not possess the same sort property.");
        }
      }
    }
    return result;
View Full Code Here

Examples of org.apache.hadoop.zebra.types.SortInfo

   */
  public static SortInfo getSortInfo(JobContext jobContext) throws IOException
  {
      Configuration conf = jobContext.getConfiguration();
    TableExpr expr = getInputExpr(jobContext);
    SortInfo result = null;
    int sortSize = 0;
    if (expr instanceof BasicTableExpr)
    {
      BasicTable.Reader reader = new BasicTable.Reader(((BasicTableExpr) expr).getPath(), conf);
      SortInfo sortInfo = reader.getSortInfo();
      reader.close();
      result = sortInfo;
    } else {
      List<LeafTableInfo> leaves = expr.getLeafTables(null);
      for (Iterator<LeafTableInfo> it = leaves.iterator(); it.hasNext(); )
      {
        LeafTableInfo leaf = it.next();
        BasicTable.Reader reader = new BasicTable.Reader(leaf.getPath(), conf);
        SortInfo sortInfo = reader.getSortInfo();
        reader.close();
        if (sortSize == 0)
        {
          sortSize = sortInfo.size();
          result = sortInfo;
        } else if (sortSize != sortInfo.size()) {
          throw new IOException("Tables of the table union do not possess the same sort property.");
        }
      }
    }
    return result;
View Full Code Here

Examples of org.apache.lucene.search.suggest.Sort.SortInfo

  }

  @Test
  public void testIntermediateMerges() throws Exception {
    // Sort 20 mb worth of data with 1mb buffer, binary merging.
    SortInfo info = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), 2),
        generateRandom((int)Sort.MB * 20));
    assertTrue(info.mergeRounds > 10);
  }
View Full Code Here

Examples of org.apache.lucene.search.suggest.Sort.SortInfo

  }

  @Test
  public void testSmallRandom() throws Exception {
    // Sort 20 mb worth of data with 1mb buffer.
    SortInfo sortInfo = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
        generateRandom((int)Sort.MB * 20));
    assertEquals(1, sortInfo.mergeRounds);
  }
View Full Code Here

Examples of org.apache.lucene.search.suggest.Sort.SortInfo

    Arrays.sort(data, unsignedByteOrderComparator);
    File golden = writeAll("golden", data);

    File sorted = new File(tempDir, "sorted");
    SortInfo sortInfo = sort.sort(unsorted, sorted);
    //System.out.println("Input size [MB]: " + unsorted.length() / (1024 * 1024));
    //System.out.println(sortInfo);

    assertFilesIdentical(golden, sorted);
    return sortInfo;
View Full Code Here

Examples of org.apache.lucene.search.suggest.fst.Sort.SortInfo

      }
      writer.close();

      // We don't know the distribution of scores and we need to bucket them, so we'll sort
      // and divide into equal buckets.
      SortInfo info = new Sort().sort(tempInput, tempSorted);
      tempInput.delete();
      FSTCompletionBuilder builder = new FSTCompletionBuilder(
          buckets, new ExternalRefSorter(new Sort()), sharedTailLength);

      final int inputLines = info.lines;
View Full Code Here

Examples of org.apache.lucene.util.OfflineSorter.SortInfo

      }
      writer.close();

      // We don't know the distribution of scores and we need to bucket them, so we'll sort
      // and divide into equal buckets.
      SortInfo info = new OfflineSorter().sort(tempInput, tempSorted);
      tempInput.delete();
      FSTCompletionBuilder builder = new FSTCompletionBuilder(
          buckets, sorter = new ExternalRefSorter(new OfflineSorter()), sharedTailLength);

      final int inputLines = info.lines;
View Full Code Here

Examples of org.apache.pig.SortInfo

                schema.getField(sortColIndex).alias;
            sortColInfoList.add(new SortColInfo(sortColName, sortColIndex,
                    mAscCols.get(i)? SortColInfo.Order.ASCENDING :
                        SortColInfo.Order.DESCENDING));
        }
        return new SortInfo(sortColInfoList);
    }
View Full Code Here

Examples of org.apache.pig.SortInfo

                int errCode = 2051;
                String msg = "Did not find a predecessor for Store." ;
                throw new FrontendException(msg, errCode, PigException.BUG);   
            }
           
            SortInfo sortInfo = null;
            if(storePred instanceof LOLimit) {
                storePred = store.getPlan().getPredecessors(storePred).get(0);
            } else if (storePred instanceof LOSplitOutput) {
                LOSplitOutput splitOutput = (LOSplitOutput)storePred;
                // We assume this is the LOSplitOutput we injected for this case:
View Full Code Here

Examples of org.apache.pig.SortInfo

                schema.getField(sortColIndex).alias;
            sortColInfoList.add(new SortColInfo(sortColName, sortColIndex,
                    mAscCols.get(i)? SortColInfo.Order.ASCENDING :
                        SortColInfo.Order.DESCENDING));
        }
        return new SortInfo(sortColInfoList);
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.