Examples of Partitioner


Examples of com.linkedin.camus.etl.Partitioner

    public static void setRunTrackingPost(JobContext job, boolean value) {
        job.getConfiguration().setBoolean(ETL_RUN_TRACKING_POST, value);
    }

    public static String getWorkingFileName(JobContext context, EtlKey key) throws IOException {
        Partitioner partitioner = getPartitioner(context, key.getTopic());
        return partitioner.getWorkingFileName(context, key.getTopic(), key.getLeaderId(), key.getPartition(), partitioner.encodePartition(context, key));
    }
View Full Code Here

Examples of com.netflix.astyanax.partitioner.Partitioner

    public QueryResult listRows(String cursor, Integer rowLimit, Integer columnLimit) throws PaasException {
        try {
            invariant();
           
            // Execute the query
            Partitioner partitioner = keyspace.getPartitioner();
            Rows<ByteBuffer, ByteBuffer> result = keyspace
                .prepareQuery(columnFamily)
                .getKeyRange(null,  null, cursor != null ? cursor : partitioner.getMinToken(),  partitioner.getMaxToken(),  rowLimit)
                .execute()
                .getResult();
           
            // Convert raw data into a simple sparse tree
            SchemalessRows.Builder builder = SchemalessRows.builder();
            for (Row<ByteBuffer, ByteBuffer> row : result) {
                Map<String, String> columns = Maps.newHashMap();
                for (Column<ByteBuffer> column : row.getColumns()) {
                    columns.put(serializers.columnAsString(column.getRawName()), serializers.valueAsString(column.getRawName(), column.getByteBufferValue()));
                }
                builder.addRow(serializers.keyAsString(row.getKey()), columns);
            }
           
            QueryResult dr = new QueryResult();
            dr.setSrows(builder.build());
           
            if (!result.isEmpty()) {
                dr.setCursor(partitioner.getTokenForKey(Iterables.getLast(result).getKey()));
            }
            return dr;
        } catch (ConnectionException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
View Full Code Here

Examples of com.netflix.astyanax.partitioner.Partitioner

    }

    @Override
    public void executeWithCallback(final RowCallback<K, C> callback) throws ConnectionException {
        final ThriftKeyspaceImpl keyspace = query.keyspace;
        final Partitioner partitioner = keyspace.getPartitioner();
        final AtomicReference<ConnectionException> error = new AtomicReference<ConnectionException>();
        final boolean bIgnoreTombstones = shouldIgnoreEmptyRows();

        List<Pair<String, String>> ranges;
        if (this.getConcurrencyLevel() != null) {
            ranges = Lists.newArrayList();
            int nThreads = this.getConcurrencyLevel();
            List<TokenRange> tokens = partitioner.splitTokenRange(
                    startToken == null ? partitioner.getMinToken() : startToken,
                    endToken == null   ? partitioner.getMaxToken() : endToken,
                    nThreads);
            for (TokenRange range : tokens) {
                try {
                    String currentToken = checkpointManager.getCheckpoint(range.getStartToken());
                    if (currentToken == null) {
                        currentToken = range.getStartToken();
                    }
                    else if (currentToken.equals(range.getEndToken())) {
                        continue;
                    }
                    ranges.add(Pair.create(currentToken, range.getEndToken()));
                } catch (Exception e) {
                    throw ThriftConverter.ToConnectionPoolException(e);
                }
            }
        }
        else {
            ranges = Lists.transform(keyspace.describeRing(true), new Function<TokenRange, Pair<String, String>> () {
                @Override
                public Pair<String, String> apply(TokenRange input) {
                    return Pair.create(input.getStartToken(), input.getEndToken());
                }
            });
        }
        final CountDownLatch doneSignal = new CountDownLatch(ranges.size());

        for (final Pair<String, String> tokenPair : ranges) {
            // Prepare the range of tokens for this token range
            final KeyRange range = new KeyRange()
                    .setCount(getBlockSize())
                    .setStart_token(tokenPair.left)
                    .setEnd_token(tokenPair.right);

            query.executor.submit(new Callable<Void>() {
                private boolean firstBlock = true;
               
                @Override
                public Void call() throws Exception {
                    if (error.get() == null && internalRun()) {
                        query.executor.submit(this);
                    }
                    else {
                        doneSignal.countDown();
                    }
                    return null;
                }

                private boolean internalRun() throws Exception {
                    try {
                        // Get the next block
                        List<KeySlice> ks = keyspace.connectionPool.executeWithFailover(
                                new AbstractKeyspaceOperationImpl<List<KeySlice>>(keyspace.tracerFactory
                                        .newTracer(CassandraOperationType.GET_ROWS_RANGE,
                                                columnFamily), query.pinnedHost, keyspace
                                        .getKeyspaceName()) {
                                    @Override
                                    public List<KeySlice> internalExecute(Client client, ConnectionContext context)
                                            throws Exception {
                                        return client.get_range_slices(new ColumnParent()
                                                .setColumn_family(columnFamily.getName()),
                                                predicate, range, ThriftConverter
                                                        .ToThriftConsistencyLevel(query.consistencyLevel));
                                    }

                                    @Override
                                    public ByteBuffer getRowKey() {
                                        if (range.getStart_key() != null)
                                            return ByteBuffer.wrap(range.getStart_key());
                                        return null;
                                    }
                                }, query.retry.duplicate()).getResult();

                        // Notify the callback
                        if (!ks.isEmpty()) {
                            KeySlice lastRow = Iterables.getLast(ks);
                            boolean bContinue = (ks.size() == getBlockSize());

                            if (getRepeatLastToken()) {
                                if (firstBlock) {
                                    firstBlock = false;
                                }
                                else {
                                    ks.remove(0);
                                }
                            }
                           
                            if (bIgnoreTombstones) {
                                Iterator<KeySlice> iter = ks.iterator();
                                while (iter.hasNext()) {
                                    if (iter.next().getColumnsSize() == 0)
                                        iter.remove();
                                }
                            }
                            Rows<K, C> rows = new ThriftRowsSliceImpl<K, C>(ks, columnFamily
                                    .getKeySerializer(), columnFamily.getColumnSerializer());
                            try {
                                callback.success(rows);
                            }
                            catch (Throwable t) {
                                ConnectionException ce = ThriftConverter.ToConnectionPoolException(t);
                                error.set(ce);
                                return false;
                            }
                           
                            if (bContinue) {
                                // Determine the start token for the next page
                                String token = partitioner.getTokenForKey(lastRow.bufferForKey()).toString();
                                checkpointManager.trackCheckpoint(tokenPair.left, token);
                                if (getRepeatLastToken()) {
                                    range.setStart_token(partitioner.getTokenMinusOne(token));
                                }
                                else {
                                    range.setStart_token(token);
                                }
                            }
View Full Code Here

Examples of com.netflix.astyanax.partitioner.Partitioner

        return this;
    }
   
    @Override
    public Partitioner getPartitioner(String partitionerName) throws Exception {
        Partitioner partitioner = partitioners.get(partitionerName);
        if (partitioner == null)
            throw new Exception("Unsupported partitioner " + partitionerName);
        return partitioner;
    }
View Full Code Here

Examples of com.thinkaurelius.titan.diskstorage.cassandra.AbstractCassandraStoreManager.Partitioner

    public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) throws StorageException {
        // this query could only be done when byte-ordering partitioner is used
        // because Cassandra operates on tokens internally which means that even contiguous
        // range of keys (e.g. time slice) with random partitioner could produce disjoint set of tokens
        // returning ambiguous results to the user.
        Partitioner partitioner = storeManager.getPartitioner();
        if (partitioner != Partitioner.BYTEORDER)
            throw new PermanentStorageException("getKeys(KeyRangeQuery could only be used with byte-ordering partitioner.");

        ByteBuffer start = query.getKeyStart().asByteBuffer(), end = query.getKeyEnd().asByteBuffer();
        int limit = (query.hasLimit()) ? query.getLimit() : Integer.MAX_VALUE;
View Full Code Here

Examples of edu.cmu.sphinx.decoder.search.Partitioner

  private void performTestPartitionSizes(int absoluteBeamWidth,
      int tokenListSize, boolean tokenListLarger) {

    Random random = new Random(System.currentTimeMillis());
    Partitioner partitioner = new Partitioner();

    Token parent = new Token(null, 0);
    Token[] tokens = new Token[tokenListSize];

    for (int i = 0; i < tokens.length; i++) {
      float logTotalScore = random.nextFloat();
      tokens[i] = new Token(parent, null, logTotalScore, 0.0f, 0.0f, i);
    }

    final int r = partitioner.partition(tokens, tokens.length,
        absoluteBeamWidth);

    if (tokenListLarger) {
      Assert.assertEquals(r, absoluteBeamWidth - 1);
    } else {
View Full Code Here

Examples of edu.cmu.sphinx.decoder.search.Partitioner

  @Test
  public void testPartitionOrders() {
    int p;
    Token[] tokens = new Token[100000];
    Partitioner partitioner = new Partitioner();

    for (int i = 0; i < 100000; i++)
      tokens[i] = new Token(null, null, 1 - i, 0, 0, 0);
    p = partitioner.partition(tokens, 100000, 3000);
    Assert.assertEquals(p, 2999);
    testSorted(tokens, p);

    for (int i = 0; i < 100000; i++)
      tokens[i] = new Token(null, null, i, 0, 0, 0);
    p = partitioner.partition(tokens, 100000, 3000);
    Assert.assertEquals(p, 2999);
    testSorted(tokens, p);

    for (int i = 0; i < 100000; i++)
      tokens[i] = new Token(null, null, 0, 0, 0, 0);
    p = partitioner.partition(tokens, 100000, 3000);
    Assert.assertEquals(p, 2999);
    testSorted(tokens, p);

    for (int i = 0; i < 100000; i++)
      tokens[i] = new Token(null, null, (float) Math.random(), 0, 0, 0);
    p = partitioner.partition(tokens, 100000, 3000);
    Assert.assertEquals(p, 2999);
    testSorted(tokens, p);
  }
View Full Code Here

Examples of eu.admire.gateway.location.Partitioner

            Map<String, String> allResults = new HashMap<String, String>();
            Map<String, String> allExternalTransfers = new HashMap<String, String>();
            Map<String, String> allManagedTransfers = new HashMap<String, String>();
            try
            {
                Partitioner partitioner =
                    GatewayBeanFactory.getInstance().getPartitioner();
                for (Graph graph : executionGraphs)
                {
                    LOG.debug("Partitioning graph");
                    Map<GatewayLocation, Graph> partitions =
                        partitioner.partition(graph);
                   
                    DistributedDISPELProcessor processor =
                        new DistributedDISPELProcessor();
                    processor.registerRemoteProcessListener(this);
                    LOG.debug("Initialising distributed processors.");
View Full Code Here

Examples of org.apache.accumulo.core.util.LocalityGroupUtil.Partitioner

    groups[0].put(new ArrayByteSequence("cf2"), new MutableLong(1));
   
    groups[1] = new HashMap<ByteSequence,MutableLong>();
    groups[1].put(new ArrayByteSequence("cf3"), new MutableLong(1));
   
    Partitioner p1 = new Partitioner(groups);
   
    Mutation m1 = new Mutation("r1");
    m1.put("cf1", "cq1", "v1");
   
    Mutation m2 = new Mutation("r2");
    m2.put("cf1", "cq1", "v2");
    m2.put("cf2", "cq2", "v3");
   
    Mutation m3 = new Mutation("r3");
    m3.put("cf1", "cq1", "v4");
    m3.put("cf3", "cq2", "v5");
   
    Mutation m4 = new Mutation("r4");
    m4.put("cf1", "cq1", "v6");
    m4.put("cf3", "cq2", "v7");
    m4.put("cf5", "cq3", "v8");
   
    Mutation m5 = new Mutation("r5");
    m5.put("cf5", "cq3", "v9");
   
    List<Mutation> mutations = Arrays.asList(m1, m2, m3, m4, m5);
    @SuppressWarnings("unchecked")
    List<Mutation>[] partitioned = new List[3];
   
    for (int i = 0; i < partitioned.length; i++) {
      partitioned[i] = new ArrayList<Mutation>();
    }
   
    p1.partition(mutations, partitioned);
   
    m1 = new Mutation("r1");
    m1.put("cf1", "cq1", "v1");
   
    m2 = new Mutation("r2");
View Full Code Here

Examples of org.apache.hadoop.mapred.Partitioner

      if (s.getPath().getName().contains("part-"))
        numPartitions++;
    }
    conf.setInt("NodeCount", nodeCount);

    Partitioner p = null;

    if (useRange) {
      p = new RangePartitioner<IntWritable, Writable>();
      p.configure(conf);
    } else {
      p = new HashPartitioner<WritableComparable, Writable>();
    }

    // this is really annoying: the mapping between the partition numbers on
    // disk (i.e., part-XXXX) and what partition the file contains (i.e.,
    // key.hash % #reducer) is arbitrary... so this means that we need to
    // open up each partition, peek inside to find out.
    IntWritable key = new IntWritable();
    HITSNode value = new HITSNode();
    FileStatus[] status = fs.listStatus(new Path(inputPath));

    StringBuilder sb = new StringBuilder();

    for (FileStatus f : status) {
      if (f.getPath().getName().contains("_logs"))
        continue;

      SequenceFile.Reader reader = new SequenceFile.Reader(fs, f
          .getPath(), conf);

      reader.next(key, value);
      @SuppressWarnings("unchecked")
      int np = p.getPartition(key, value, numPartitions);
      reader.close();

      sLogger.info(f.getPath() + "\t" + np);
      sb.append(np + "=" + f.getPath() + "\t");
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.