Package org.elasticsearch.hadoop.rest

Examples of org.elasticsearch.hadoop.rest.RestRepository


        if (input == null) {
            // get original copy
            Settings settings = CascadingUtils.addDefaultsToSettings(CascadingUtils.extractOriginalProperties(flowProcess.getConfigCopy()), tapProperties, log);

            // will be closed by the query is finished
            RestRepository client = new RestRepository(settings);
            Field mapping = client.getMapping();
            Collection<String> fields = CascadingUtils.fieldToAlias(settings, getSourceFields());

            // validate if possible
            FieldPresenceValidation validation = settings.getFieldExistanceValidation();
            if (validation.isRequired()) {
View Full Code Here


    private void initClient(Properties props, boolean read) {
        if (client == null) {
            Settings settings = CascadingUtils.addDefaultsToSettings(props, this.props, LogFactory.getLog(EsTap.class));
            CascadingUtils.init(settings, host, port, resource, query, read);
            client = new RestRepository(settings);
        }
    }
View Full Code Here

    private void init(Configuration cfg) throws IOException {
        Settings settings = HadoopSettingsManager.loadFrom(cfg);
        Assert.hasText(settings.getResourceWrite(), String.format("No resource ['%s'] (index/query/location) specified", ES_RESOURCE));

        // lazy-init
        RestRepository client = null;

        InitializationUtils.checkIdForOperation(settings);
        InitializationUtils.checkIndexExistence(settings, client);

        if (HadoopCfgUtils.getReduceTasks(cfg) != null) {
View Full Code Here

    @Test
    public void testBulkWrite() throws Exception {
        TestSettings testSettings = new TestSettings("rest/savebulk");
        //testSettings.setPort(9200)
        testSettings.setProperty(ConfigurationOptions.ES_SERIALIZATION_WRITER_VALUE_CLASS, JdkValueWriter.class.getName());
        RestRepository client = new RestRepository(testSettings);

        Scanner in = new Scanner(getClass().getResourceAsStream("/artists.dat")).useDelimiter("\\n|\\t");

        Map<String, String> line = new LinkedHashMap<String, String>();

        for (; in.hasNextLine();) {
            // ignore number
            in.next();
            line.put("name", in.next());
            line.put("url", in.next());
            line.put("picture", in.next());
            client.writeToIndex(line);
            line.clear();
        }

        client.close();
    }
View Full Code Here

    @Test
    public void testEmptyBulkWrite() throws Exception {
        TestSettings testSettings = new TestSettings("rest/emptybulk");
        testSettings.setProperty(ConfigurationOptions.ES_SERIALIZATION_WRITER_VALUE_CLASS, JdkValueWriter.class.getName());
        RestRepository restRepo = new RestRepository(testSettings);
        RestClient client = restRepo.getRestClient();
        client.bulk(new Resource(testSettings, false), new TrackingBytesArray(new BytesArray("{}")));
        restRepo.waitForYellow();
        restRepo.close();
        client.close();
    }
View Full Code Here

    public void start() throws IOException {
        settings = new TestSettings("rest/savebulk");
        //testSettings.setPort(9200)
        settings.setProperty(ConfigurationOptions.ES_SERIALIZATION_WRITER_VALUE_CLASS, JdkValueWriter.class.getName());
        settings.setProperty(ConfigurationOptions.ES_SERIALIZATION_WRITER_VALUE_CLASS, JdkValueWriter.class.getName());
        client = new RestRepository(settings);
        client.waitForYellow();
    }
View Full Code Here

    private void init(Configuration cfg) throws IOException {
        Settings settings = SettingsManager.loadFrom(cfg);
        Assert.hasText(settings.getResourceWrite(), String.format("No resource ['%s'] (index/query/location) specified", ES_RESOURCE));

        // lazy-init
        RestRepository client = null;

        InitializationUtils.checkIdForOperation(settings);
        InitializationUtils.checkIndexExistence(settings, client);

        if (HadoopCfgUtils.getReduceTasks(cfg) != null) {
View Full Code Here

        if (input == null) {
            // get original copy
            Settings settings = CascadingUtils.addDefaultsToSettings(CascadingUtils.extractOriginalProperties(flowProcess.getConfigCopy()), tapProperties, log);

            // will be closed by the query is finished
            RestRepository client = new RestRepository(settings);
            Field mapping = client.getMapping();
            Collection<String> fields = CascadingUtils.fieldToAlias(settings, getSourceFields());

            // validate if possible
            FieldPresenceValidation validation = settings.getFieldExistanceValidation();
            if (validation.isRequired()) {
View Full Code Here

        private void initSingleIndex(Settings settings, int currentInstance) throws IOException {
            if (log.isDebugEnabled()) {
                log.debug(String.format("Resource [%s] resolves as a single index", resource));
            }

            repository = new RestRepository(settings);
            // create the index if needed
            if (repository.touch()) {
                if (repository.waitForYellow()) {
                    log.warn(String.format("Timed out waiting for index [%s] to reach yellow health", resource));
                }
            }

            Map<Shard, Node> targetShards = repository.getWriteTargetPrimaryShards();
            repository.close();

            List<Shard> orderedShards = new ArrayList<Shard>(targetShards.keySet());
            // make sure the order is strict
            Collections.sort(orderedShards);
            if (log.isTraceEnabled()) {
                log.trace(String.format("ESRecordWriter instance [%s] discovered %s primary shards %s", currentInstance, orderedShards.size(), orderedShards));
            }

            // if there's no task info, just pick a random bucket
            if (currentInstance <= 0) {
                currentInstance = new Random().nextInt(targetShards.size()) + 1;
            }
            int bucket = currentInstance % targetShards.size();
            Shard chosenShard = orderedShards.get(bucket);
            Node targetNode = targetShards.get(chosenShard);

            // override the global settings to communicate directly with the target node
            settings.setHosts(targetNode.getIpAddress()).setPort(targetNode.getHttpPort());
            repository = new RestRepository(settings);
            uri = SettingsUtils.nodes(settings).get(0);

            if (log.isDebugEnabled()) {
                log.debug(String.format("EsRecordWriter instance [%s] assigned to primary shard [%s] at address [%s]", currentInstance, chosenShard.getName(), uri));
            }
View Full Code Here

            if (log.isDebugEnabled()) {
                log.debug(String.format("EsRecordWriter instance [%s] assigned to [%s]", currentInstance, uri));
            }

            repository = new RestRepository(settings);
        }
View Full Code Here

TOP

Related Classes of org.elasticsearch.hadoop.rest.RestRepository

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.