Package org.elasticsearch.client.action.bulk

Examples of org.elasticsearch.client.action.bulk.BulkRequestBuilder


        System.out.println("Indexing [" + COUNT + "] ...");
        long ITERS = COUNT / BATCH;
        long i = 1;
        int counter = 0;
        for (; i <= ITERS; i++) {
            BulkRequestBuilder request = client1.prepareBulk();
            for (int j = 0; j < BATCH; j++) {
                counter++;
                request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter)).source(source(Integer.toString(counter), "test" + counter)));
            }
            BulkResponse response = request.execute().actionGet();
            if (response.hasFailures()) {
                System.err.println("failures...");
            }
            if (((i * BATCH) % 10000) == 0) {
                System.out.println("Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime());
View Full Code Here


            System.out.println("--> Indexing [" + COUNT + "] ...");
            long ITERS = COUNT / BATCH;
            long i = 1;
            int counter = 0;
            for (; i <= ITERS; i++) {
                BulkRequestBuilder request = client.prepareBulk();
                for (int j = 0; j < BATCH; j++) {
                    counter++;
                    request.add(Requests.indexRequest("test").type("parent").id(Integer.toString(counter))
                            .source(parentSource(Integer.toString(counter), "test" + counter)));
                    for (int k = 0; k < CHILD_COUNT; k++) {
                        request.add(Requests.indexRequest("test").type("child").id(Integer.toString(counter) + "_" + k)
                                .parent(Integer.toString(counter))
                                .source(childSource(Integer.toString(counter), "tag" + k)));
                    }
                }
                BulkResponse response = request.execute().actionGet();
                if (response.hasFailures()) {
                    System.err.println("--> failures...");
                }
                if (((i * BATCH) % 10000) == 0) {
                    System.out.println("--> Indexed " + (i * BATCH) * (1 + CHILD_COUNT) + " took " + stopWatch.stop().lastTaskTime());
View Full Code Here

        }

        Client client = nodes.length == 1 ? nodes[0].client() : nodes[1].client();

        while (true) {
            BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (int i = 0; i < BATCH; i++) {
                bulkRequest.add(Requests.indexRequest("test" + ThreadLocalRandom.current().nextInt(NUMBER_OF_INDICES)).type("type").source("field", "value"));
            }
            bulkRequest.execute().actionGet();
        }
    }
View Full Code Here

            logger.info("*** ROUND {}", ++numberOfRounds);
            // bulk index data
            int numberOfBulks = numberOfDocsPerRound / bulkSize;
            for (int b = 0; b < numberOfBulks; b++) {
                BulkRequestBuilder bulk = client.client().prepareBulk();
                for (int k = 0; k < bulkSize; k++) {
                    StringBuffer sb = new StringBuffer();
                    XContentBuilder json = XContentFactory.jsonBuilder().startObject()
                            .field("field", "value" + ThreadLocalRandom.current().nextInt());

                    int fields = ThreadLocalRandom.current().nextInt() % numberOfFields;
                    for (int i = 0; i < fields; i++) {
                        json.field("num_" + i, ThreadLocalRandom.current().nextDouble());
                        int tokens = ThreadLocalRandom.current().nextInt() % textTokens;
                        sb.setLength(0);
                        for (int j = 0; j < tokens; j++) {
                            sb.append(UUID.randomBase64UUID()).append(' ');
                        }
                        json.field("text_" + i, sb.toString());
                    }

                    json.endObject();

                    bulk.add(Requests.indexRequest("test" + (Math.abs(ThreadLocalRandom.current().nextInt()) % numberOfIndices)).type("type1").source(json));
                    indexCounter.incrementAndGet();
                }
                bulk.execute().actionGet();
            }

            client.client().admin().indices().prepareGatewaySnapshot().execute().actionGet();

            client.close();
View Full Code Here

            System.out.println("--> Indexing [" + COUNT + "] ...");
            long ITERS = COUNT / BATCH;
            long i = 1;
            int counter = 0;
            for (; i <= ITERS; i++) {
                BulkRequestBuilder request = client.prepareBulk();
                for (int j = 0; j < BATCH; j++) {
                    counter++;
                    XContentBuilder source = jsonBuilder().startObject()
                            .field("id", Integer.valueOf(counter))
                            .field("l_value", lValues[counter % lValues.length])
                            .field("date", new Date())
                            .endObject();
                    request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter))
                            .source(source));
                }
                BulkResponse response = request.execute().actionGet();
                if (response.hasFailures()) {
                    System.err.println("--> failures...");
                }
                if (((i * BATCH) % 10000) == 0) {
                    System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime());
View Full Code Here

            System.out.println("--> Indexing [" + COUNT + "] ...");
            long ITERS = COUNT / BATCH;
            long i = 1;
            int counter = 0;
            for (; i <= ITERS; i++) {
                BulkRequestBuilder request = client.prepareBulk();
                for (int j = 0; j < BATCH; j++) {
                    counter++;

                    XContentBuilder builder = jsonBuilder().startObject();
                    builder.field("id", Integer.toString(counter));
                    builder.field("s_value", sValues[counter % sValues.length]);
                    builder.field("l_value", lValues[counter % lValues.length]);

                    builder.startArray("sm_value");
                    for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) {
                        builder.value(sValues[ThreadLocalRandom.current().nextInt(sValues.length)]);
                    }
                    builder.endArray();

                    builder.startArray("lm_value");
                    for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) {
                        builder.value(lValues[ThreadLocalRandom.current().nextInt(sValues.length)]);
                    }
                    builder.endArray();

                    builder.endObject();

                    request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter))
                            .source(builder));
                }
                BulkResponse response = request.execute().actionGet();
                if (response.hasFailures()) {
                    System.err.println("--> failures...");
                }
                if (((i * BATCH) % 10000) == 0) {
                    System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime());
View Full Code Here

            System.out.println("--> Indexing [" + COUNT + "] ...");
            long ITERS = COUNT / BATCH;
            long i = 1;
            int counter = 0;
            for (; i <= ITERS; i++) {
                BulkRequestBuilder request = client.prepareBulk();
                for (int j = 0; j < BATCH; j++) {
                    counter++;

                    XContentBuilder builder = jsonBuilder().startObject();
                    builder.field("id", Integer.toString(counter));
                    builder.field("l_value", lValues[counter % lValues.length]);

                    builder.endObject();

                    request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter))
                            .source(builder));
                }
                BulkResponse response = request.execute().actionGet();
                if (response.hasFailures()) {
                    System.err.println("--> failures...");
                }
                if (((i * BATCH) % 10000) == 0) {
                    System.out.println("--> Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime());
View Full Code Here

                    .setPercolate("color:green").execute().actionGet();
            assertThat(index.matches().size(), equalTo(0));
        }

        // test bulk
        BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
        for (int i = 0; i < 10; i++) {
            bulkRequestBuilder.add(client.prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "value1")
                    .setPercolate("*"));
        }
        BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
        assertThat(bulkResponse.hasFailures(), equalTo(false));
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            IndexResponse index = bulkItemResponse.response();
            assertThat(index.matches().size(), equalTo(1));
            assertThat(index.matches(), hasItem("kuku"));
View Full Code Here

    @Override public DeleteRequestBuilder prepareDelete(String index, String type, String id) {
        return prepareDelete().setIndex(index).setType(type).setId(id);
    }

    @Override public BulkRequestBuilder prepareBulk() {
        return new BulkRequestBuilder(this);
    }
View Full Code Here

                    }

                    if (task != null && task.getBody() != null) {
                        final List<Long> deliveryTags = Lists.newArrayList();

                        BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();

                        try {
                            bulkRequestBuilder.add(task.getBody(), 0, task.getBody().length, false);
                        } catch (Exception e) {
                            logger.warn("failed to parse request for delivery tag [{}], ack'ing...", e, task.getEnvelope().getDeliveryTag());
                            try {
                                channel.basicAck(task.getEnvelope().getDeliveryTag(), false);
                            } catch (IOException e1) {
                                logger.warn("failed to ack [{}]", e1, task.getEnvelope().getDeliveryTag());
                            }
                            continue;
                        }

                        deliveryTags.add(task.getEnvelope().getDeliveryTag());

                        if (bulkRequestBuilder.numberOfActions() < bulkSize) {
                            // try and spin some more of those without timeout, so we have a bigger bulk (bounded by the bulk size)
                            try {
                                while ((task = consumer.nextDelivery(bulkTimeout.millis())) != null) {
                                    try {
                                        bulkRequestBuilder.add(task.getBody(), 0, task.getBody().length, false);
                                    } catch (Exception e) {
                                        logger.warn("failed to parse request for delivery tag [{}], ack'ing...", e, task.getEnvelope().getDeliveryTag());
                                        try {
                                            channel.basicAck(task.getEnvelope().getDeliveryTag(), false);
                                        } catch (IOException e1) {
                                            logger.warn("failed to ack on failure [{}]", e1, task.getEnvelope().getDeliveryTag());
                                        }
                                    }
                                    deliveryTags.add(task.getEnvelope().getDeliveryTag());
                                    if (bulkRequestBuilder.numberOfActions() >= bulkSize) {
                                        break;
                                    }
                                }
                            } catch (InterruptedException e) {
                                if (closed) {
                                    break;
                                }
                            }
                        }

                        if (logger.isTraceEnabled()) {
                            logger.trace("executing bulk with [{}] actions", bulkRequestBuilder.numberOfActions());
                        }

                        if (ordered) {
                            try {
                                BulkResponse response = bulkRequestBuilder.execute().actionGet();
                                if (response.hasFailures()) {
                                    // TODO write to exception queue?
                                    logger.warn("failed to execute" + response.buildFailureMessage());
                                }
                                for (Long deliveryTag : deliveryTags) {
                                    try {
                                        channel.basicAck(deliveryTag, false);
                                    } catch (IOException e1) {
                                        logger.warn("failed to ack [{}]", e1, deliveryTag);
                                    }
                                }
                            } catch (Exception e) {
                                logger.warn("failed to execute bulk", e);
                            }
                        } else {
                            bulkRequestBuilder.execute(new ActionListener<BulkResponse>() {
                                @Override public void onResponse(BulkResponse response) {
                                    if (response.hasFailures()) {
                                        // TODO write to exception queue?
                                        logger.warn("failed to execute" + response.buildFailureMessage());
                                    }
View Full Code Here

TOP

Related Classes of org.elasticsearch.client.action.bulk.BulkRequestBuilder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.