Package org.elasticsearch.action.bulk

Examples of org.elasticsearch.action.bulk.BulkRequestBuilder


      } else {
        if (isClosed())
          throw new InterruptedException("Interrupted because River is closed");

        Date firstDocumentUpdatedDate = null;
        BulkRequestBuilder esBulk = esIntegrationComponent.prepareESBulkRequestBuilder();
        for (Map<String, Object> document : res.getDocuments()) {
          String documentId = documentIndexStructureBuilder.extractDocumentId(document);
          if (documentId == null) {
            throw new IllegalArgumentException("Document ID not found in remote system response for Space " + spaceKey
                + " within data: " + document);
View Full Code Here


    if (scrollResp.getHits().getTotalHits() > 0) {
      if (isClosed())
        throw new InterruptedException("Interrupted because River is closed");
      scrollResp = esIntegrationComponent.executeESScrollSearchNextRequest(scrollResp);
      BulkRequestBuilder esBulk = esIntegrationComponent.prepareESBulkRequestBuilder();
      while (scrollResp.getHits().getHits().length > 0) {
        for (SearchHit hit : scrollResp.getHits()) {
          logger.debug("Go to delete indexed document for ES document id {}", hit.getId());
          if (documentIndexStructureBuilder.deleteESDocument(esBulk, hit)) {
            indexingInfo.documentsDeleted++;
View Full Code Here

     *
     * @param tweetIds
     * @param tweets
     */
    public void bulkIndex(List<Long> tweetIds, List<String> tweets) {
        BulkRequestBuilder requestBuilder = client.prepareBulk();
        for(int i = 0; i < tweetIds.size(); i++) {
            XContentBuilder builder;
            try {
                builder = jsonBuilder()
                        .startObject()
                        .field("text", tweets.get(i))
                        .field("id", tweetIds.get(i))
                        .endObject();
            } catch (IOException e) {
                continue;
            }
            IndexRequestBuilder request = client.prepareIndex("hackaton", "tweets")
                    .setIndex("hackaton")
                    .setType("tweets")
                    .setSource(builder);
            requestBuilder.add(request);
        }
        BulkResponse bulkResponse = requestBuilder.execute().actionGet();
        int items = bulkResponse.getItems().length;
        System.err.print("indexed [" + items + "] items, with failures? [" + bulkResponse.hasFailures()  + "]");
    }
View Full Code Here

        }
    }

    @Override
    public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, TransactionHandle tx) throws StorageException {
        BulkRequestBuilder brb = client.prepareBulk();
        int bulkrequests = 0;
        try {
            for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
                String storename = stores.getKey();
                for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
                    String docid = entry.getKey();
                    IndexMutation mutation = entry.getValue();
                    Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
                    Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
                    Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());

                    //Deletions first
                    if (mutation.hasDeletions()) {
                        if (mutation.isDeleted()) {
                            log.trace("Deleting entire document {}", docid);
                            brb.add(new DeleteRequest(indexName, storename, docid));
                            bulkrequests++;
                        } else {
                            Set<String> deletions = Sets.newHashSet(mutation.getDeletions());
                            if (mutation.hasAdditions()) {
                                for (IndexEntry ie : mutation.getAdditions()) {
                                    deletions.remove(ie.key);
                                }
                            }
                            if (!deletions.isEmpty()) {
                                //TODO make part of batch mutation if/when possible
                                StringBuilder script = new StringBuilder();
                                for (String key : deletions) {
                                    script.append("ctx._source.remove(\"" + key + "\"); ");
                                }
                                log.trace("Deleting individual fields [{}] for document {}", deletions, docid);
                                client.prepareUpdate(indexName, storename, docid).setScript(script.toString()).execute().actionGet();
                            }
                        }
                    }

                    if (mutation.hasAdditions()) {
                        if (mutation.isNew()) { //Index
                            log.trace("Adding entire document {}", docid);
                            brb.add(new IndexRequest(indexName, storename, docid).source(getContent(mutation.getAdditions())));
                            bulkrequests++;
                        } else { //Update: TODO make part of batch mutation if/when possible
                            boolean needUpsert = !mutation.hasDeletions();
                            XContentBuilder builder = getContent(mutation.getAdditions());
                            UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setDoc(builder);
                            if (needUpsert) update.setUpsert(builder);
                            log.trace("Updating document {} with upsert {}", docid, needUpsert);
                            update.execute().actionGet();
                        }
                    }

                }
            }
            if (bulkrequests > 0) brb.execute().actionGet();
        } catch (Exception e) {
            throw convert(e);
        }
    }
View Full Code Here

        Map<Object, Set<Object>> joinTableRecords = joinTableData.getJoinTableRecords();

        Set<Object> joinKeys = joinTableRecords.keySet();

        BulkRequestBuilder bulkRequest = txClient.prepareBulk();

        /**
         * 1_p => 1_a1,1_a2 1_a1=> 1_p,1_p1
         *
         * Example: join table : PERSON_ADDRESS join column : PERSON_ID (1_p)
         * inverse join column : ADDRESS_ID (1_a) store in ES: schema name:
         * PERSON_ADDRESS type: PERSON id: 1_p\0011_a
         *
         * PERSON_ADDRESS:1_p_1_a PERSON_ID 1_p ADDRESS_ID 1_a
         *
         * source: (PERSON_ID, 1_p) (ADDRESS_ID, 1_a)
         *
         * embeddable keys over many to many does not work.
         */

        boolean found = false;
        for (Object key : joinKeys)
        {
            Set<Object> inversejoinTableRecords = joinTableRecords.get(key);
            Map<String, Object> source = new HashMap<String, Object>();

            for (Object inverseObj : inversejoinTableRecords)
            {
                source = new HashMap<String, Object>();
                source.put(joinTableData.getJoinColumnName(), key);
                source.put(joinTableData.getInverseJoinColumnName(), inverseObj);

                String joinKeyAsStr = PropertyAccessorHelper.getString(key);
                String inverseKeyAsStr = PropertyAccessorHelper.getString(inverseObj);

                String keyAsString = joinKeyAsStr + KEY_SEPERATOR + inverseKeyAsStr;
                IndexRequest request = new IndexRequest(schemaName.toLowerCase(), tableName, keyAsString)
                        .source(source);
                found = true;
                bulkRequest.add(request);
            }
        }

        // check made, as bulk request throws an error, in case no request is
        // present.
        if (found)
        {
            bulkRequest.execute().actionGet();
        }

    }
View Full Code Here

    }

    @Override
    public int executeBatch()
    {
        BulkRequestBuilder bulkRequest = txClient.prepareBulk();

        try
        {
            for (Node node : nodes)
            {
                if (node.isDirty())
                {
                    node.handlePreEvent();
                    Object entity = node.getData();
                    Object id = node.getEntityId();
                    EntityMetadata metadata = KunderaMetadataManager.getEntityMetadata(kunderaMetadata,
                            node.getDataClass());

                    MetamodelImpl metaModel = (MetamodelImpl) kunderaMetadata.getApplicationMetadata().getMetamodel(
                            metadata.getPersistenceUnit());

                    EntityType entityType = metaModel.entity(metadata.getEntityClazz());

                    String key = getKeyAsString(id, metadata, metaModel);

                    if (node.isInState(RemovedState.class))
                    {
                        // create a delete request.

                        DeleteRequest request = new DeleteRequest(metadata.getSchema().toLowerCase(),
                                metadata.getTableName(), key);
                        bulkRequest.add(request);

                    }
                    else if (node.isUpdate())
                    {
                        Map<String, Object> values = new HashMap<String, Object>();
                        List<RelationHolder> relationHolders = getRelationHolders(node);

                        addSource(entity, values, entityType);

                        addRelations(relationHolders, values);

                        UpdateRequest request = new UpdateRequest(metadata.getSchema().toLowerCase(),
                                metadata.getTableName(), key).doc(values);
                        bulkRequest.add(request);
                    }
                    else
                    {
                        // create an insert request.
                        Map<String, Object> values = new HashMap<String, Object>();
                        List<RelationHolder> relationHolders = getRelationHolders(node);

                        addSource(entity, values, entityType);

                        addRelations(relationHolders, values);

                        IndexRequest request = new IndexRequest(metadata.getSchema().toLowerCase(),
                                metadata.getTableName(), key).source(values);
                        bulkRequest.add(request);

                    }

                }
            }

            BulkResponse response = null;
            if (nodes != null && !nodes.isEmpty())
            {
                // bulkRequest.setRefresh(true);
                response = bulkRequest.execute().actionGet();
            }
            return response != null ? response.getItems().length : 0;
        }
        finally
        {
View Full Code Here

    }


    private void indexSetupStateForElasticSearch() throws Exception {
        new ElasticSearchTestHelper(client).deleteIndexIfItExists("test");
        BulkRequest bulkRequest = new BulkRequestBuilder(client).request();
        URL bulkIndexRequest = this.getClass().getResource("es-numericbulkindex.json");
        byte[] data = ByteStreams.toByteArray(bulkIndexRequest.openStream());
        bulkRequest.add(data, 0, data.length, true);
        BulkResponse bulkResponse = client.bulk(bulkRequest).actionGet();
        if (bulkResponse.hasFailures()) {
View Full Code Here

    }


    private void indexSetupStateForElasticSearch() throws Exception {
        new ElasticSearchTestHelper(client).deleteIndexIfItExists("test");
        BulkRequest bulkRequest = new BulkRequestBuilder(client).request();
        URL bulkIndexRequest = this.getClass().getResource("es-bulkindex.json");
        byte[] data = ByteStreams.toByteArray(bulkIndexRequest.openStream());
        bulkRequest.add(data, 0, data.length, true);
        BulkResponse bulkResponse = client.bulk(bulkRequest).actionGet();
        if (bulkResponse.hasFailures()) {
View Full Code Here

    @BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
    @Test
    public void testESBulkDelete() {
        HashMap<String, String> ids = createSampleData();
        Client client = getClient(false);
        BulkRequestBuilder request = new BulkRequestBuilder(client);

        for(String id: ids.values()){
            DeleteRequest deleteRequest = new DeleteRequest("users", "default", id);
            request.add(deleteRequest);
        }
        request.execute().actionGet();
        refresh(client());
    }
View Full Code Here

                    }
                }
            }
        } else {
            logger.info("Index [{}] docs async: [{}] bulk: [{}]", list.size(), false, true);
            BulkRequestBuilder bulkBuilder = client().prepareBulk();
            for (IndexRequestBuilder indexRequestBuilder : list) {
                bulkBuilder.add(indexRequestBuilder);
            }
            BulkResponse actionGet = bulkBuilder.execute().actionGet();
            assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false));
        }
        for (CountDownLatch countDownLatch : latches) {
            countDownLatch.await();
        }
View Full Code Here

TOP

Related Classes of org.elasticsearch.action.bulk.BulkRequestBuilder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.