Package org.elasticsearch.node

Examples of org.elasticsearch.node.Node


   * Opens a local discovery node for talking to an elasticsearch server running
   * in the same JVM
   */
  private void openLocalDiscoveryClient() {
    logger.info("Using ElasticSearch AutoDiscovery mode");
    Node node = NodeBuilder.nodeBuilder().client(true).local(true).node();
    if (client != null) {
      client.close();
    }
    client = node.client();
  }
View Full Code Here


   * Opens a local discovery node for talking to an elasticsearch server running
   * in the same JVM
   */
  private void openLocalDiscoveryClient() {
    logger.info("Using ElasticSearch AutoDiscovery mode");
    Node node = NodeBuilder.nodeBuilder().client(true).local(true).node();
    if (client != null) {
      client.close();
    }
    client = node.client();
  }
View Full Code Here

        Node[] nodes = new Node[1];
        for (int i = 0; i < nodes.length; i++) {
            nodes[i] = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node" + i)).node();
        }

        Node client = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "client")).client(true).node();

        Client client1 = client.client();

        Thread.sleep(1000);
        client1.admin().indices().prepareCreate("test").setSettings(settings).addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1")
                .startObject("_source").field("enabled", false).endObject()
                .startObject("_all").field("enabled", false).endObject()
                .startObject("_type").field("index", "no").endObject()
                .startObject("_id").field("index", "no").endObject()
                .startObject("properties")
                .startObject("field").field("type", "string").field("index", "not_analyzed").field("omit_norms", true).endObject()
//                .startObject("field").field("index", "analyzed").field("omit_norms", false).endObject()
                .endObject()
                .endObject().endObject()).execute().actionGet();
        Thread.sleep(5000);

        StopWatch stopWatch = new StopWatch().start();
        long COUNT = SizeValue.parseSizeValue("2m").singles();
        int BATCH = 500;
        System.out.println("Indexing [" + COUNT + "] ...");
        long ITERS = COUNT / BATCH;
        long i = 1;
        int counter = 0;
        for (; i <= ITERS; i++) {
            BulkRequestBuilder request = client1.prepareBulk();
            for (int j = 0; j < BATCH; j++) {
                counter++;
                request.add(Requests.indexRequest("test").type("type1").id(Integer.toString(counter)).source(source(Integer.toString(counter), "test" + counter)));
            }
            BulkResponse response = request.execute().actionGet();
            if (response.hasFailures()) {
                System.err.println("failures...");
            }
            if (((i * BATCH) % 10000) == 0) {
                System.out.println("Indexed " + (i * BATCH) + " took " + stopWatch.stop().lastTaskTime());
                stopWatch.start();
            }
        }
        System.out.println("Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) COUNT) / stopWatch.totalTime().secondsFrac()));

        client.client().admin().indices().prepareRefresh().execute().actionGet();
        System.out.println("Count: " + client.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().count());

        client.close();

        for (Node node : nodes) {
            node.close();
        }
    }
View Full Code Here

        Settings settings = ImmutableSettings.settingsBuilder()
                .put("gateway.type", "none")
                .build();

        Node node = NodeBuilder.nodeBuilder().settings(settings).node();
        final Client client = node.client();

        client.admin().indices().prepareCreate("test").execute().actionGet();
        client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();

        final AtomicBoolean stop = new AtomicBoolean();
View Full Code Here

        Node[] nodes = new Node[1];
        for (int i = 0; i < nodes.length; i++) {
            nodes[i] = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node" + i)).node();
        }

        Node client = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "client")).client(true).node();

        Client client1 = client.client();

        Thread.sleep(1000);
        client1.admin().indices().create(createIndexRequest("test")).actionGet();
        Thread.sleep(5000);

        StopWatch stopWatch = new StopWatch().start();
        int COUNT = 200000;
        System.out.println("Indexing [" + COUNT + "] ...");
        int i = 1;
        for (; i <= COUNT; i++) {
//            client1.admin().cluster().preparePingSingle("test", "type1", Integer.toString(i)).execute().actionGet();
            client1.prepareIndex("test", "type1").setId(Integer.toString(i)).setSource(source(Integer.toString(i), "test" + i))
                    .setCreate(false).execute().actionGet();
            if ((i % 10000) == 0) {
                System.out.println("Indexed " + i + " took " + stopWatch.stop().lastTaskTime());
                stopWatch.start();
            }
        }
        System.out.println("Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) COUNT) / stopWatch.totalTime().secondsFrac()));

        client.client().admin().indices().prepareRefresh().execute().actionGet();
        System.out.println("Count: " + client.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet().count());

        client.close();

        for (Node node : nodes) {
            node.close();
        }
    }
View Full Code Here

                .put("gateway.type", "local")
                .put(SETTING_NUMBER_OF_SHARDS, 2)
                .put(SETTING_NUMBER_OF_REPLICAS, 1)
                .build();

        Node node1 = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node1")).node();
        Node node2 = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node2")).node();

        Node clientNode = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "client")).client(true).node();

        Client client = clientNode.client();

        long COUNT = SizeValue.parseSizeValue("1m").singles();
        int CHILD_COUNT = 5;
        int BATCH = 100;
        int QUERY_WARMUP = 20;
        int QUERY_COUNT = 50;

        Thread.sleep(10000);
        try {
            client.admin().indices().create(createIndexRequest("test")).actionGet();
            client.admin().indices().preparePutMapping("test").setType("child").setSource(XContentFactory.jsonBuilder().startObject().startObject("type")
                    .startObject("_parent").field("type", "parent").endObject()
                    .endObject().endObject()).execute().actionGet();
            Thread.sleep(5000);

            StopWatch stopWatch = new StopWatch().start();

            System.out.println("--> Indexing [" + COUNT + "] ...");
            long ITERS = COUNT / BATCH;
            long i = 1;
            int counter = 0;
            for (; i <= ITERS; i++) {
                BulkRequestBuilder request = client.prepareBulk();
                for (int j = 0; j < BATCH; j++) {
                    counter++;
                    request.add(Requests.indexRequest("test").type("parent").id(Integer.toString(counter))
                            .source(parentSource(Integer.toString(counter), "test" + counter)));
                    for (int k = 0; k < CHILD_COUNT; k++) {
                        request.add(Requests.indexRequest("test").type("child").id(Integer.toString(counter) + "_" + k)
                                .parent(Integer.toString(counter))
                                .source(childSource(Integer.toString(counter), "tag" + k)));
                    }
                }
                BulkResponse response = request.execute().actionGet();
                if (response.hasFailures()) {
                    System.err.println("--> failures...");
                }
                if (((i * BATCH) % 10000) == 0) {
                    System.out.println("--> Indexed " + (i * BATCH) * (1 + CHILD_COUNT) + " took " + stopWatch.stop().lastTaskTime());
                    stopWatch.start();
                }
            }
            System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (COUNT * (1 + CHILD_COUNT))) / stopWatch.totalTime().secondsFrac()));
        } catch (Exception e) {
            System.out.println("--> Index already exists, ignoring indexing phase, waiting for green");
            ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet();
            if (clusterHealthResponse.timedOut()) {
                System.err.println("--> Timed out waiting for cluster health");
            }
        }
        client.admin().indices().prepareRefresh().execute().actionGet();
        System.out.println("--> Number of docs in index: " + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().count());

        System.out.println("--> Running just child query");
        // run just the child query, warm up first
        for (int j = 0; j < QUERY_WARMUP; j++) {
            SearchResponse searchResponse = client.prepareSearch().setQuery(termQuery("child.tag", "tag1")).execute().actionGet();
            if (j == 0) {
                System.out.println("--> Warmup took: " + searchResponse.took());
            }
            if (searchResponse.hits().totalHits() != COUNT) {
                System.err.println("--> mismatch on hits");
            }
        }

        long totalQueryTime = 0;
        for (int j = 0; j < QUERY_COUNT; j++) {
            SearchResponse searchResponse = client.prepareSearch().setQuery(termQuery("child.tag", "tag1")).execute().actionGet();
            if (searchResponse.hits().totalHits() != COUNT) {
                System.err.println("--> mismatch on hits");
            }
            totalQueryTime += searchResponse.tookInMillis();
        }
        System.out.println("--> Just Child Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");

        System.out.println("--> Running has_child query");
        // run parent child constant query
        for (int j = 0; j < QUERY_WARMUP; j++) {
            SearchResponse searchResponse = client.prepareSearch().setQuery(hasChildQuery("child", termQuery("tag", "tag1"))).execute().actionGet();
            if (searchResponse.failedShards() > 0) {
                System.err.println("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
            }
            if (searchResponse.hits().totalHits() != COUNT) {
                System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.hits().totalHits() + "], expected [" + COUNT + "]");
            }
        }

        totalQueryTime = 0;
        for (int j = 0; j < QUERY_COUNT; j++) {
            SearchResponse searchResponse = client.prepareSearch().setQuery(hasChildQuery("child", termQuery("tag", "tag1"))).execute().actionGet();
            if (searchResponse.failedShards() > 0) {
                System.err.println("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
            }
            if (searchResponse.hits().totalHits() != COUNT) {
                System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.hits().totalHits() + "], expected [" + COUNT + "]");
            }
            totalQueryTime += searchResponse.tookInMillis();
        }
        System.out.println("--> has_child Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");

        System.out.println("--> Running top_children query");
        // run parent child score query
        for (int j = 0; j < QUERY_WARMUP; j++) {
            SearchResponse searchResponse = client.prepareSearch().setQuery(topChildrenQuery("child", termQuery("tag", "tag1"))).execute().actionGet();
            // we expect to have mismatch on hits here
//            if (searchResponse.hits().totalHits() != COUNT) {
//                System.err.println("mismatch on hits");
//            }
        }

        totalQueryTime = 0;
        for (int j = 0; j < QUERY_COUNT; j++) {
            SearchResponse searchResponse = client.prepareSearch().setQuery(topChildrenQuery("child", termQuery("tag", "tag1"))).execute().actionGet();
            // we expect to have mismatch on hits here
//            if (searchResponse.hits().totalHits() != COUNT) {
//                System.err.println("mismatch on hits");
//            }
            totalQueryTime += searchResponse.tookInMillis();
        }
        System.out.println("--> top_children Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");

        clientNode.close();

        node1.close();
        node2.close();
    }
View Full Code Here

        Settings settings = settingsBuilder()
                .put("gateway.type", "none")
                .build();

        Node node1 = nodeBuilder().settings(settings).node();
        Node node2 = nodeBuilder().settings(settings).node();
        final Node client = nodeBuilder().settings(settings).client(true).node();

        final int NUMBER_OF_DOCS = 10000;
        final int NUMBER_OF_THREADS = 10;
        final long NUMBER_OF_ITERATIONS = SizeValue.parseSizeValue("10k").singles();
        final long DELETE_EVERY = 10;

        final CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS);
        Thread[] threads = new Thread[NUMBER_OF_THREADS];
        for (int i = 0; i < threads.length; i++) {
            threads[i] = new Thread() {
                @Override public void run() {
                    try {
                        for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) {
                            if ((i % DELETE_EVERY) == 0) {
                                client.client().prepareDelete("test", "type1", Integer.toString(ThreadLocalRandom.current().nextInt(NUMBER_OF_DOCS))).execute().actionGet();
                            } else {
                                client.client().prepareIndex("test", "type1", Integer.toString(ThreadLocalRandom.current().nextInt(NUMBER_OF_DOCS))).setSource("field1", "value1").execute().actionGet();
                            }
                        }
                    } finally {
                        latch.countDown();
                    }
                }
            };
        }

        for (Thread thread : threads) {
            thread.start();
        }

        latch.await();
        System.out.println("done indexing, verifying docs");
        client.client().admin().indices().prepareRefresh().execute().actionGet();
        for (int i = 0; i < NUMBER_OF_DOCS; i++) {
            String id = Integer.toString(i);
            for (int j = 0; j < 5; j++) {
                SearchResponse response = client.client().prepareSearch().setQuery(QueryBuilders.termQuery("_id", id)).execute().actionGet();
                if (response.hits().totalHits() > 1) {
                    System.err.println("[" + i + "] FAIL, HITS [" + response.hits().totalHits() + "]");
                }
            }
            GetResponse getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
            if (getResponse.exists()) {
                long version = getResponse.version();
                for (int j = 0; j < 5; j++) {
                    getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
                    if (!getResponse.exists()) {
                        System.err.println("[" + i + "] FAIL, EXISTED, and NOT_EXISTED");
                        break;
                    }
                    if (version != getResponse.version()) {
                        System.err.println("[" + i + "] FAIL, DIFFERENT VERSIONS: [" + version + "], [" + getResponse.version() + "]");
                        break;
                    }
                }
            } else {
                for (int j = 0; j < 5; j++) {
                    getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
                    if (getResponse.exists()) {
                        System.err.println("[" + i + "] FAIL, EXISTED, and NOT_EXISTED");
                        break;
                    }
                }
            }
        }
        System.out.println("done.");

        client.close();
        node1.close();
        node2.close();
    }
View Full Code Here

        Settings settings = ImmutableSettings.settingsBuilder()
                .put("index.shard.check_index", false)
                .put("gateway.type", "fs")
                .put("index.number_of_shards", 1)
                .build();
        Node node = NodeBuilder.nodeBuilder().settings(settings).node();

        for (int i = 0; i < numberOfIndices; i++) {
            logger.info("START index [{}] ...", i);
            node.client().admin().indices().prepareCreate("index_" + i).execute().actionGet();

            for (int j = 0; j < numberOfDocs; j++) {
                node.client().prepareIndex("index_" + i, "type").setSource("field1", "test", "field2", 2, "field3", new Date()).execute().actionGet();
            }
            logger.info("DONE  index [{}] ...", i);
        }

        node.client().admin().indices().prepareGatewaySnapshot().execute().actionGet();

        logger.info("closing node...");
        node.close();
        logger.info("node closed");

        logger.info("starting node...");
        node = NodeBuilder.nodeBuilder().settings(settings).node();

        ClusterHealthResponse health = node.client().admin().cluster().prepareHealth().setTimeout("5m").setWaitForYellowStatus().execute().actionGet();
        logger.info("health: " + health.status());
        logger.info("active shards: " + health.activeShards());
        logger.info("active primary shards: " + health.activePrimaryShards());
        if (health.timedOut()) {
            logger.error("Timed out on health...");
        }

        ClusterState clusterState = node.client().admin().cluster().prepareState().execute().actionGet().state();
        for (int i = 0; i < numberOfIndices; i++) {
            if (clusterState.blocks().indices().containsKey("index_" + i)) {
                logger.error("index [{}] has blocks: {}", i, clusterState.blocks().indices().get("index_" + i));
            }
        }

        for (int i = 0; i < numberOfIndices; i++) {
            long count = node.client().prepareCount("index_" + i).setQuery(matchAllQuery()).execute().actionGet().count();
            if (count == numberOfDocs) {
                logger.info("VERIFIED [{}], count [{}]", i, count);
            } else {
                logger.error("FAILED [{}], expected [{}], got [{}]", i, numberOfDocs, count);
            }
        }

        logger.info("closing node...");
        node.close();
        logger.info("node closed");
    }
View Full Code Here

        int numberOfReplicas = 1;
        int numberOfIndices = 1000;
        int numberOfDocs = 1;

        Client client;
        Node node = null;
        if (true) {
            client = new TransportClient().addTransportAddress(new InetSocketTransportAddress("localhost", 9300));
        } else {
            node = NodeBuilder.nodeBuilder().client(true).node();
            client = node.client();
        }

        for (int i = 0; i < numberOfIndices; i++) {
            logger.info("START index [{}] ...", i);
            client.admin().indices().prepareCreate("index_" + i)
                    .setSettings(ImmutableSettings.settingsBuilder().put("index.number_of_shards", numberOfShards).put("index.number_of_replicas", numberOfReplicas))
                    .execute().actionGet();

            for (int j = 0; j < numberOfDocs; j++) {
                client.prepareIndex("index_" + i, "type").setSource("field1", "test", "field2", 2, "field3", new Date()).execute().actionGet();
            }
            logger.info("DONE  index [{}]", i);
        }

        client.admin().indices().prepareGatewaySnapshot().execute().actionGet();

        logger.info("closing node...");
        if (node != null) {
            node.close();
        }
        logger.info("node closed");
    }
View Full Code Here

        while (true) {
            Node[] nodes = new Node[numberOfNodes];
            for (int i = 0; i < nodes.length; i++) {
                nodes[i] = NodeBuilder.nodeBuilder().settings(settings).node();
            }
            Node client = NodeBuilder.nodeBuilder().settings(settings).client(true).node();

            // verify that the indices are there
            for (int i = 0; i < numberOfIndices; i++) {
                try {
                    client.client().admin().indices().prepareCreate("test" + i).execute().actionGet();
                } catch (Exception e) {
                    // might already exists, fine
                }
            }

            logger.info("*** Waiting for GREEN status");
            try {
                ClusterHealthResponse clusterHealth = client.client().admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("10m").execute().actionGet();
                if (clusterHealth.timedOut()) {
                    logger.warn("timed out waiting for green status....");
                }
            } catch (Exception e) {
                logger.warn("failed to execute cluster health....");
            }

            CountResponse count = client.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet();
            logger.info("*** index_count [{}], expected_count [{}]", count.count(), indexCounter.get());
            // verify count
            for (int i = 0; i < (nodes.length * 5); i++) {
                count = client.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet();
                logger.debug("index_count [{}], expected_count [{}]", count.count(), indexCounter.get());
                if (count.count() != indexCounter.get()) {
                    logger.warn("!!! count does not match, index_count [{}], expected_count [{}]", count.count(), indexCounter.get());
                    throw new Exception("failed test, count does not match...");
                }
            }

            // verify search
            for (int i = 0; i < (nodes.length * 5); i++) {
                // do a search with norms field, so we don't rely on match all filtering cache
                SearchResponse search = client.client().prepareSearch().setQuery(matchAllQuery().normsField("field")).execute().actionGet();
                logger.debug("index_count [{}], expected_count [{}]", search.hits().totalHits(), indexCounter.get());
                if (count.count() != indexCounter.get()) {
                    logger.warn("!!! search does not match, index_count [{}], expected_count [{}]", search.hits().totalHits(), indexCounter.get());
                    throw new Exception("failed test, count does not match...");
                }
            }

            logger.info("*** ROUND {}", ++numberOfRounds);
            // bulk index data
            int numberOfBulks = numberOfDocsPerRound / bulkSize;
            for (int b = 0; b < numberOfBulks; b++) {
                BulkRequestBuilder bulk = client.client().prepareBulk();
                for (int k = 0; k < bulkSize; k++) {
                    StringBuffer sb = new StringBuffer();
                    XContentBuilder json = XContentFactory.jsonBuilder().startObject()
                            .field("field", "value" + ThreadLocalRandom.current().nextInt());

                    int fields = ThreadLocalRandom.current().nextInt() % numberOfFields;
                    for (int i = 0; i < fields; i++) {
                        json.field("num_" + i, ThreadLocalRandom.current().nextDouble());
                        int tokens = ThreadLocalRandom.current().nextInt() % textTokens;
                        sb.setLength(0);
                        for (int j = 0; j < tokens; j++) {
                            sb.append(UUID.randomBase64UUID()).append(' ');
                        }
                        json.field("text_" + i, sb.toString());
                    }

                    json.endObject();

                    bulk.add(Requests.indexRequest("test" + (Math.abs(ThreadLocalRandom.current().nextInt()) % numberOfIndices)).type("type1").source(json));
                    indexCounter.incrementAndGet();
                }
                bulk.execute().actionGet();
            }

            client.client().admin().indices().prepareGatewaySnapshot().execute().actionGet();

            client.close();
            for (Node node : nodes) {
                File nodeWork = ((InternalNode) node).injector().getInstance(NodeEnvironment.class).nodeDataLocation();
                node.close();
                if (clearNodeWork && !settings.get("gateway.type").equals("local")) {
                    FileSystemUtils.deleteRecursively(nodeWork);
View Full Code Here

TOP

Related Classes of org.elasticsearch.node.Node

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.