Package org.elasticsearch.action.bulk

Examples of org.elasticsearch.action.bulk.BulkRequest


        // get the type of Solr update handler we want to mock, default to xml
        final String handler = request.hasParam("handler") ? request.param("handler").toLowerCase() : "xml";

        // Requests are typically sent to Solr in batches of documents
        // We can copy that by submitting batch requests to Solr
        BulkRequest bulkRequest = Requests.bulkRequest();

        // parse and handle the content
        if (handler.equals("xml")) {
            // XML Content
            try {
                // create parser for the content
                XMLStreamReader parser = inputFactory.createXMLStreamReader(new StringReader(request.content().toUtf8()));

                // parse the xml
                // we only care about doc and delete tags for now
                boolean stop = false;
                while (!stop) {
                    // get the xml "event"
                    int event = parser.next();
                    switch (event) {
                        case XMLStreamConstants.END_DOCUMENT :
                            // this is the end of the document
                            // close parser and exit while loop
                            parser.close();
                            stop = true;
                            break;
                        case XMLStreamConstants.START_ELEMENT :
                            // start of an xml tag
                            // determine if we need to add or delete a document
                            String currTag = parser.getLocalName();
                            if ("doc".equals(currTag)) {
                                // add a document
                                Map<String, Object> doc = parseXmlDoc(parser);
                                if (doc != null) {
                                    bulkRequest.add(getIndexRequest(doc, request));
                                }
                            } else if ("delete".equals(currTag)) {
                                // delete a document
                                String docid = parseXmlDelete(parser);
                                if (docid != null) {
                                    bulkRequest.add(getDeleteRequest(docid, request));
                                }
                            }
                            break;
                    }
                }
            } catch (Exception e) {
                // some sort of error processing the xml input
                try {
                    logger.error("Error processing xml input", e);
                    channel.sendResponse(new XContentThrowableRestResponse(request, e));
                } catch (IOException e1) {
                    logger.error("Failed to send error response", e1);
                }
            }
        } else if (handler.equals("javabin")) {
            // JavaBin Content
            try {
                // We will use the JavaBin codec from solrj
                // unmarshal the input to a SolrUpdate request
                JavaBinUpdateRequestCodec codec = new JavaBinUpdateRequestCodec();
                UpdateRequest req = codec.unmarshal(new ByteArrayInputStream(request.content().array()), null);

                // Get the list of documents to index out of the UpdateRequest
                // Add each document to the bulk request
                // convert the SolrInputDocument into a map which will be used as the ES source field
                List<SolrInputDocument> docs = req.getDocuments();
                if (docs != null) {
                    for (SolrInputDocument doc : docs) {
                        bulkRequest.add(getIndexRequest(convertToMap(doc), request));
                    }
                }

                // See if we have any documents to delete
                // if yes, add them to the bulk request
                if (req.getDeleteById() != null) {
                    for (String id : req.getDeleteById()) {
                        bulkRequest.add(getDeleteRequest(id, request));
                    }
                }
            } catch (Exception e) {
                // some sort of error processing the javabin input
                try {
                    logger.error("Error processing javabin input", e);
                    channel.sendResponse(new XContentThrowableRestResponse(request, e));
                } catch (IOException e1) {
                    logger.error("Failed to send error response", e1);
                }
            }
        }

        // only submit the bulk request if there are index/delete actions
        // it is possible not to have any actions when parsing xml due to the
        // commit and optimize messages that will not generate documents
        if (bulkRequest.numberOfActions() > 0) {
            client.bulk(bulkRequest, new ActionListener<BulkResponse>() {

                // successful bulk request
                public void onResponse(BulkResponse response) {
                    logger.info("Bulk request completed");
View Full Code Here


    /**
     * Creats a new bulk request.
     */
    public static BulkRequest bulkRequest() {
        return new BulkRequest();
    }
View Full Code Here

* @author kimchy (shay.banon)
*/
public class BulkRequestBuilder extends BaseRequestBuilder<BulkRequest, BulkResponse> {

    public BulkRequestBuilder(Client client) {
        super(client, new BulkRequest());
    }
View Full Code Here

        controller.registerHandler(POST, "/_bulk", this);
        controller.registerHandler(PUT, "/_bulk", this);
    }

    @Override public void handleRequest(final RestRequest request, final RestChannel channel) {
        BulkRequest bulkRequest = Requests.bulkRequest();

        String replicationType = request.param("replication");
        if (replicationType != null) {
            bulkRequest.replicationType(ReplicationType.fromString(replicationType));
        }
        String consistencyLevel = request.param("consistency");
        if (consistencyLevel != null) {
            bulkRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
        }
        bulkRequest.refresh(request.paramAsBoolean("refresh", bulkRequest.refresh()));
        try {
            bulkRequest.add(request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength(), request.contentUnsafe());
        } catch (Exception e) {
            try {
                XContentBuilder builder = restContentBuilder(request);
                channel.sendResponse(new XContentRestResponse(request, BAD_REQUEST, builder.startObject().field("error", e.getMessage()).endObject()));
            } catch (IOException e1) {
View Full Code Here

        execute(channel);
    }

    // (currently) needs to be executed under a lock
    private void execute(final InteractiveChannel channel) {
        final BulkRequest bulkRequest = new BulkRequest().add(bulk);
        bulk.clear();

        final long executionId = executionIdGen.incrementAndGet();

        if (concurrentRequests == 0) {
View Full Code Here

        int msgCount = 100;
        for (int i = 0; i < msgCount; ++i) {
            msgList.add(new Message(routingKey, jsonMapper.writeValueAsBytes(msg)));
        }

        BulkRequest request = sink.createBulkRequest(msgList);
        for (int i = 0; i < msgCount; ++i) {
            sink.recover(i, request);
        }

        node.client().admin().indices().prepareRefresh(index).execute().actionGet();
View Full Code Here

        indexProducts(products, index, null);
    }

    private void indexProducts(List<Map<String, Object>> products, String index, String routing) throws Exception {
        long currentCount = getCurrentDocumentCount(index);
        BulkRequest bulkRequest = new BulkRequest();
        for (Map<String, Object> product : products) {
            IndexRequest indexRequest = new IndexRequest(index, "product", (String)product.get("ProductId"));
            indexRequest.source(product);
            if (Strings.hasLength(routing)) {
                indexRequest.routing(routing);
            }
            bulkRequest.add(indexRequest);
        }
        bulkRequest.refresh(true);
        BulkResponse response = client().bulk(bulkRequest).actionGet();
        if (response.hasFailures()) {
            fail("Error in creating products: " + response.buildFailureMessage());
        }
View Full Code Here

    // get the type of Solr update handler we want to mock, default to xml
    final String handler = request.hasParam("handler") ? request.param("handler").toLowerCase() : "xml";

    // Requests are typically sent to Solr in batches of documents
    // We can copy that by submitting batch requests to Solr
    BulkRequest bulkRequest = Requests.bulkRequest();

    // parse and handle the content
    if (handler.equals("xml")) {
      // XML Content
      try {
        // create parser for the content
        XMLStreamReader parser = inputFactory.createXMLStreamReader(new StringReader(request.contentAsString()));

        // parse the xml
        // we only care about doc and delete tags for now
        boolean stop = false;
        while (!stop) {
          // get the xml "event"
          int event = parser.next();
          switch (event) {
          case XMLStreamConstants.END_DOCUMENT:
            // this is the end of the document
            // close parser and exit while loop
            parser.close();
            stop = true;
            break;
          case XMLStreamConstants.START_ELEMENT:
            // start of an xml tag
            // determine if we need to add or delete a document
            String currTag = parser.getLocalName();
            if ("doc".equals(currTag)) {
              // add a document
              Map<String, Object> doc = parseXmlDoc(parser);
              if (doc != null) {
                bulkRequest.add(getIndexRequest(doc, request));
              }
            } else if ("delete".equals(currTag)) {
              // delete a document
              String docid = parseXmlDelete(parser);
              if (docid != null) {
                bulkRequest.add(getDeleteRequest(docid, request));
              }
            }
            break;
          }
        }
      } catch (Exception e) {
        // some sort of error processing the xml input
        try {
          logger.error("Error processing xml input", e);
          channel.sendResponse(new XContentThrowableRestResponse(request, e));
        } catch (IOException e1) {
          logger.error("Failed to send error response", e1);
        }
      }
    } else if (handler.equals("javabin")) {
      // JavaBin Content
      try {
        // We will use the JavaBin codec from solrj
        // unmarshal the input to a SolrUpdate request
        JavaBinUpdateRequestCodec codec = new JavaBinUpdateRequestCodec();
        UpdateRequest req = codec.unmarshal(new ByteArrayInputStream(request.contentByteArray()), null);

        // Get the list of documents to index out of the UpdateRequest
        // Add each document to the bulk request
        // convert the SolrInputDocument into a map which will be used as the ES source field
        List<SolrInputDocument> docs = req.getDocuments();
        if (docs != null) {
          for (SolrInputDocument doc : docs) {
            bulkRequest.add(getIndexRequest(convertToMap(doc), request));
          }
        }

        // See if we have any documents to delete
        // if yes, add them to the bulk request
        if (req.getDeleteById() != null) {
          for (String id : req.getDeleteById()) {
            bulkRequest.add(getDeleteRequest(id, request));
          }
        }
      } catch (Exception e) {
        // some sort of error processing the javabin input
        try {
          logger.error("Error processing javabin input", e);
          channel.sendResponse(new XContentThrowableRestResponse(request, e));
        } catch (IOException e1) {
          logger.error("Failed to send error response", e1);
        }
      }
    }

    // only submit the bulk request if there are index/delete actions
    // it is possible not to have any actions when parsing xml due to the
    // commit and optimize messages that will not generate documents
    if (bulkRequest.numberOfActions() > 0) {
      client.bulk(bulkRequest, new ActionListener<BulkResponse>() {

        // successful bulk request
        public void onResponse(BulkResponse response) {
          logger.info("Bulk request completed");
View Full Code Here

        }
    }

    @Override
    protected void write(List<Message> msgList) throws IOException {
        final BulkRequest request = createBulkRequest(msgList);

        senders.execute(createRunnable(request));
    }
View Full Code Here

        senders.execute(createRunnable(request));
    }

    @VisibleForTesting
    protected BulkRequest createBulkRequest(List<Message> msgList) {
        final BulkRequest request = new BulkRequest();
        for (Message m : msgList) {
            IndexRequest indexRequest = createIndexRequest(m);
            if (indexRequest != null) {
                request.add(indexRequest, m);
            }
        }
        return request;
    }
View Full Code Here

TOP

Related Classes of org.elasticsearch.action.bulk.BulkRequest

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.