Examples of AggregateProcessor

org.apache.camel.processor.aggregate.AggregateProcessor
.apache.org/aggregator2.html">Aggregator pattern where a batch of messages are processed (up to a maximum amount or until some timeout is reached) and messages for the same correlation key are combined together using some kind of {@link AggregationStrategy}(by default the latest message is used) to compress many message exchanges into a smaller number of exchanges.
A good example of this is stock market data; you may be receiving 30,000 messages/second and you may want to throttle it right down so that multiple messages for the same stock are combined (or just the latest message is used and older prices are discarded). Another idea is to combine line item messages together into a single invoice message.
storm.trident.planner.processor.AggregateProcessor

Examples of org.apache.camel.processor.aggregate.AggregateProcessor

            // we do not run in parallel mode, but use a synchronous executor, so we run in current thread
            threadPool = new SynchronousExecutorService();
            shutdownThreadPool = true;
        }


        AggregateProcessor answer = new AggregateProcessor(routeContext.getCamelContext(), internal,
                correlation, strategy, threadPool, shutdownThreadPool);


        AggregationRepository repository = createAggregationRepository(routeContext);
        if (repository != null) {
            answer.setAggregationRepository(repository);
        }


        // this EIP supports using a shared timeout checker thread pool or fallback to create a new thread pool
        boolean shutdownTimeoutThreadPool = false;
        ScheduledExecutorService timeoutThreadPool = timeoutCheckerExecutorService;
        if (timeoutThreadPool == null && timeoutCheckerExecutorServiceRef != null) {
            // lookup existing thread pool
            timeoutThreadPool = routeContext.getCamelContext().getRegistry().lookupByNameAndType(timeoutCheckerExecutorServiceRef, ScheduledExecutorService.class);
            if (timeoutThreadPool == null) {
                // then create a thread pool assuming the ref is a thread pool profile id
                timeoutThreadPool = routeContext.getCamelContext().getExecutorServiceManager().newScheduledThreadPool(this,
                        AggregateProcessor.AGGREGATE_TIMEOUT_CHECKER, timeoutCheckerExecutorServiceRef);
                if (timeoutThreadPool == null) {
                    throw new IllegalArgumentException("ExecutorServiceRef " + timeoutCheckerExecutorServiceRef + " not found in registry or as a thread pool profile.");
                }
                shutdownTimeoutThreadPool = true;
            }
        }
        answer.setTimeoutCheckerExecutorService(timeoutThreadPool);
        answer.setShutdownTimeoutCheckerExecutorService(shutdownTimeoutThreadPool);


        // set other options
        answer.setParallelProcessing(isParallelProcessing());
        answer.setOptimisticLocking(isOptimisticLocking());
        if (getCompletionPredicate() != null) {
            Predicate predicate = getCompletionPredicate().createPredicate(routeContext);
            answer.setCompletionPredicate(predicate);
        }
        if (getCompletionTimeoutExpression() != null) {
            Expression expression = getCompletionTimeoutExpression().createExpression(routeContext);
            answer.setCompletionTimeoutExpression(expression);
        }
        if (getCompletionTimeout() != null) {
            answer.setCompletionTimeout(getCompletionTimeout());
        }
        if (getCompletionInterval() != null) {
            answer.setCompletionInterval(getCompletionInterval());
        }
        if (getCompletionSizeExpression() != null) {
            Expression expression = getCompletionSizeExpression().createExpression(routeContext);
            answer.setCompletionSizeExpression(expression);
        }
        if (getCompletionSize() != null) {
            answer.setCompletionSize(getCompletionSize());
        }
        if (getCompletionFromBatchConsumer() != null) {
            answer.setCompletionFromBatchConsumer(isCompletionFromBatchConsumer());
        }
        if (getEagerCheckCompletion() != null) {
            answer.setEagerCheckCompletion(isEagerCheckCompletion());
        }
        if (getIgnoreInvalidCorrelationKeys() != null) {
            answer.setIgnoreInvalidCorrelationKeys(isIgnoreInvalidCorrelationKeys());
        }
        if (getCloseCorrelationKeyOnCompletion() != null) {
            answer.setCloseCorrelationKeyOnCompletion(getCloseCorrelationKeyOnCompletion());
        }
        if (getDiscardOnCompletionTimeout() != null) {
            answer.setDiscardOnCompletionTimeout(isDiscardOnCompletionTimeout());
        }
        if (getForceCompletionOnStop() != null) {
            answer.setForceCompletionOnStop(getForceCompletionOnStop());
        }
        if (optimisticLockRetryPolicy == null) {
            if (getOptimisticLockRetryPolicyDefinition() != null) {
                answer.setOptimisticLockRetryPolicy(getOptimisticLockRetryPolicyDefinition().createOptimisticLockRetryPolicy());
            }
        } else {
            answer.setOptimisticLockRetryPolicy(optimisticLockRetryPolicy);
        }
        return answer;
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor


        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionSize(10);
        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");
        e2.getIn().setHeader("id", 456);


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("END");
        e3.getIn().setHeader("id", 123);


        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("END");
        e4.getIn().setHeader("id", 456);


        ap.process(e1);
        ap.process(e2);
        ap.process(e3);
        ap.process(e4);


        assertEquals("should not have completed yet", 0, mock.getExchanges().size());


        ap.forceCompletionOfAllGroups();


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor

        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();
        Predicate complete = body().contains("END");


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionPredicate(complete);
        ap.setEagerCheckCompletion(false);
        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");
        e2.getIn().setHeader("id", 123);


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("END");
        e3.getIn().setHeader("id", 123);


        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("D");
        e4.getIn().setHeader("id", 123);


        ap.process(e1);
        ap.process(e2);
        ap.process(e3);
        ap.process(e4);


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor

        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();
        Predicate complete = body().isEqualTo("END");


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionPredicate(complete);
        ap.setEagerCheckCompletion(true);
        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");
        e2.getIn().setHeader("id", 123);


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("END");
        e3.getIn().setHeader("id", 123);


        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("D");
        e4.getIn().setHeader("id", 123);


        ap.process(e1);
        ap.process(e2);
        ap.process(e3);
        ap.process(e4);


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor


        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionSize(3);
        ap.setEagerCheckCompletion(eager);
        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");
        e2.getIn().setHeader("id", 123);


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("C");
        e3.getIn().setHeader("id", 123);


        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("D");
        e4.getIn().setHeader("id", 123);


        ap.process(e1);
        ap.process(e2);
        ap.process(e3);
        ap.process(e4);


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor


        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionTimeout(3000);
        ap.setEagerCheckCompletion(eager);
        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");
        e2.getIn().setHeader("id", 123);


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("C");
        e3.getIn().setHeader("id", 123);


        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("D");
        e4.getIn().setHeader("id", 123);


        ap.process(e1);


        Thread.sleep(250);
        ap.process(e2);


        Thread.sleep(500);
        ap.process(e3);


        Thread.sleep(5000);
        ap.process(e4);


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor


        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionInterval(3000);
        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");
        e2.getIn().setHeader("id", 123);


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("C");
        e3.getIn().setHeader("id", 123);


        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("D");
        e4.getIn().setHeader("id", 123);


        ap.process(e1);
        ap.process(e2);
        ap.process(e3);


        Thread.sleep(5000);
        ap.process(e4);


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor

        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();
        Predicate complete = body().contains("END");


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionPredicate(complete);
        ap.setIgnoreInvalidCorrelationKeys(true);


        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("C");
        e3.getIn().setHeader("id", 123);


        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("END");
        e4.getIn().setHeader("id", 123);


        ap.process(e1);
        ap.process(e2);
        ap.process(e3);
        ap.process(e4);


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor

        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();
        Predicate complete = body().contains("END");


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionPredicate(complete);


        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("C");
        e3.getIn().setHeader("id", 123);




        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("END");
        e4.getIn().setHeader("id", 123);


        ap.process(e1);


        ap.process(e2);
        Exception e = e2.getException();
        assertNotNull(e);
        assertEquals("Invalid correlation key. Exchange[Message: B]", e.getMessage());


        ap.process(e3);
        ap.process(e4);


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

Examples of org.apache.camel.processor.aggregate.AggregateProcessor

        Processor done = new SendProcessor(context.getEndpoint("mock:result"));
        Expression corr = header("id");
        AggregationStrategy as = new BodyInAggregatingStrategy();
        Predicate complete = body().contains("END");


        AggregateProcessor ap = new AggregateProcessor(context, done, corr, as, executorService, true);
        ap.setCompletionPredicate(complete);
        ap.setCloseCorrelationKeyOnCompletion(1000);


        ap.start();


        Exchange e1 = new DefaultExchange(context);
        e1.getIn().setBody("A");
        e1.getIn().setHeader("id", 123);


        Exchange e2 = new DefaultExchange(context);
        e2.getIn().setBody("B");
        e2.getIn().setHeader("id", 123);


        Exchange e3 = new DefaultExchange(context);
        e3.getIn().setBody("END");
        e3.getIn().setHeader("id", 123);


        Exchange e4 = new DefaultExchange(context);
        e4.getIn().setBody("C");
        e4.getIn().setHeader("id", 123);


        ap.process(e1);
        ap.process(e2);
        ap.process(e3);


        ap.process(e4);
        Exception e = e4.getException();
        assertNotNull(e);
        assertEquals("The correlation key [123] has been closed. Exchange[Message: C]", e.getMessage());


        assertMockEndpointsSatisfied();


        ap.stop();
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.