Examples of SplitSource

com.facebook.presto.execution.SplitSource
com.facebook.presto.spi.SplitSource
net.sf.kpex.builtins.SplitSource
Splits a (finite) Source in two new ones which inherit the current state of the parent.

Examples of com.facebook.presto.spi.SplitSource

            throws InterruptedException
    {
        AtomicInteger nextTaskId = new AtomicInteger(0);
        long getSplitStart = System.nanoTime();


        SplitSource splitSource = this.dataSource.get();
        while (!splitSource.isFinished()) {
            getSplitDistribution.add(System.nanoTime() - getSplitStart);


            // if query has been canceled, exit cleanly; query will never run regardless
            if (getState().isDone()) {
                break;
            }


            Set<Split> pendingSplits = ImmutableSet.copyOf(splitSource.getNextBatch(splitBatchSize));
            while (!pendingSplits.isEmpty() && !getState().isDone()) {
                Multimap<Node, Split> splitAssignment = nodeSelector.computeAssignments(pendingSplits);
                pendingSplits = ImmutableSet.copyOf(Sets.difference(pendingSplits, ImmutableSet.copyOf(splitAssignment.values())));


                assignSplits(nextTaskId, splitAssignment);

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource

            List<Partition> partitions = FluentIterable.from(getPartitions(node))
                    .filter(materializedViewPartitionPredicate)
                    .toList();


            // get dataSource for table
            SplitSource splitSource = splitManager.getPartitionSplits(node.getTable(), partitions);


            return new NodeSplits(node.getId(), splitSource);
        }

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource

                    return node.getSource().accept(this, materializedViewPartitionPredicate);


                case SYSTEM: {
                    NodeSplits nodeSplits = node.getSource().accept(this, materializedViewPartitionPredicate);
                    if (nodeSplits.getDataSource().isPresent()) {
                        SplitSource sampledSplitSource = new SampledSplitSource(nodeSplits.getDataSource().get(), node.getSampleRatio());
                        return new NodeSplits(node.getId(), sampledSplitSource);
                    }
                    else {
                        // table sampling on a sub query without splits is meaningless
                        return nodeSplits;

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource

            MaterializedViewWriter materializedViewWriter = new MaterializedViewWriter(node, shardManager);


            // get source splits
            NodeSplits nodeSplits = node.getSource().accept(this, materializedViewWriter.getPartitionPredicate());
            checkState(nodeSplits.getDataSource().isPresent(), "No splits present for import");
            SplitSource splitSource = nodeSplits.getDataSource().get();


            // record output
            outputReceivers.put(node.getId(), materializedViewWriter.getOutputReceiver());


            // wrap splits with table writer info

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource

        List<ColumnHandle> columnHandles = ImmutableList.copyOf(client.getColumnHandles(table).values());
        Map<String, Integer> columnIndex = indexColumns(columnHandles);


        PartitionResult partitionResult = client.getPartitions(table, TupleDomain.all());
        assertEquals(partitionResult.getPartitions().size(), 1);
        SplitSource splitSource = client.getPartitionSplits(table, partitionResult.getPartitions());


        long sum = 0;
        for (Split split : getAllSplits(splitSource)) {
            try (RecordCursor cursor = client.getRecordSet(split, columnHandles).cursor()) {
                while (cursor.advanceNextPosition()) {

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource

        List<ColumnHandle> columnHandles = ImmutableList.copyOf(client.getColumnHandles(tableHandle).values());


        // verify the data
        PartitionResult partitionResult = client.getPartitions(tableHandle, TupleDomain.all());
        assertEquals(partitionResult.getPartitions().size(), 1);
        SplitSource splitSource = client.getPartitionSplits(tableHandle, partitionResult.getPartitions());
        Split split = getOnlyElement(getAllSplits(splitSource));


        try (RecordCursor cursor = client.getRecordSet(split, columnHandles).cursor()) {
            assertTrue(cursor.advanceNextPosition());
            assertEquals(cursor.getLong(0), 1);

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource

    public void testGetPartitionSplitsBatch()
            throws Exception
    {
        TableHandle tableHandle = getTableHandle(table);
        PartitionResult partitionResult = splitManager.getPartitions(tableHandle, TupleDomain.all());
        SplitSource splitSource = splitManager.getPartitionSplits(tableHandle, partitionResult.getPartitions());


        assertEquals(getSplitCount(splitSource), partitions.size());
    }

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource

    public void testGetPartitionSplitsBatchUnpartitioned()
            throws Exception
    {
        TableHandle tableHandle = getTableHandle(tableUnpartitioned);
        PartitionResult partitionResult = splitManager.getPartitions(tableHandle, TupleDomain.all());
        SplitSource splitSource = splitManager.getPartitionSplits(tableHandle, partitionResult.getPartitions());


        assertEquals(getSplitCount(splitSource), 1);
    }

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource


    @Test
    public void testGetPartitionSplitsEmpty()
            throws Exception
    {
        SplitSource splitSource = splitManager.getPartitionSplits(invalidTableHandle, ImmutableList.<Partition>of());
        // fetch full list
        getSplitCount(splitSource);
    }

View Full Code Here

Examples of com.facebook.presto.spi.SplitSource

        assertPrimitiveField(columnMap, 0, "sales", ColumnType.LONG, false);


        // verify the data
        PartitionResult partitionResult = splitManager.getPartitions(tableHandle, TupleDomain.all());
        assertEquals(partitionResult.getPartitions().size(), 1);
        SplitSource splitSource = splitManager.getPartitionSplits(tableHandle, partitionResult.getPartitions());
        Split split = getOnlyElement(getAllSplits(splitSource));


        try (RecordCursor cursor = recordSetProvider.getRecordSet(split, columnHandles).cursor()) {
            assertRecordCursorType(cursor, "rcfile-binary");

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.