Package org.elasticsearch.hadoop.serialization

Examples of org.elasticsearch.hadoop.serialization.ScrollReader

82
83
84
85
86
87
88
89
90
            if (validation.isRequired()) {
                MappingUtils.validateMapping(fields, mapping, validation, log);
            }

            input = QueryBuilder.query(settings).fields(StringUtils.concatenateAndUriEncode(fields,  ",")).build(client,
                          new ScrollReader(new JdkValueReader(), mapping, settings.getReadMetadata(), settings.getReadMetadataField()));
        }
        return new TupleEntrySchemeIterator<Properties, ScrollQuery>(flowProcess, getScheme(), input, getIdentifier());
    }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
        assertEquals(Short.valueOf((short) 125), value);
    }

    @Test
    public void testScrollWithSource() throws IOException {
        ScrollReader reader = new ScrollReader(new JdkValueReader(), null, readMetadata, metadataField);
        InputStream stream = getClass().getResourceAsStream("scroll-source.json");
        List<Object[]> read = reader.read(stream);
        assertEquals(3, read.size());
        Object[] objects = read.get(0);
        assertTrue(((Map) objects[1]).containsKey("source"));
    }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
        assertTrue(((Map) objects[1]).containsKey("source"));
    }

    @Test
    public void testScrollWithoutSource() throws IOException {
        ScrollReader reader = new ScrollReader(new JdkValueReader(), null, readMetadata, metadataField);
        InputStream stream = getClass().getResourceAsStream("empty-source.json");
        List<Object[]> read = reader.read(stream);
        assertEquals(2, read.size());
        Object[] objects = read.get(0);
        if (readMetadata) {
            assertTrue(((Map) objects[1]).containsKey(metadataField));
        }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
        }
    }

    @Test
    public void testScrollMultiValueList() throws IOException {
        ScrollReader reader = new ScrollReader(new JdkValueReader(), null, readMetadata, metadataField);
        InputStream stream = getClass().getResourceAsStream("list-with-null.json");
        List<Object[]> read = reader.read(stream);
        assertEquals(1, read.size());
        Object[] objects = read.get(0);
        Map map = (Map) read.get(0)[1];
        List links = (List) map.get("links");
        assertTrue(links.contains(null));
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    public void testQueryBuilder() throws Exception {
        Settings sets = settings.copy();
        sets.setProperty(ConfigurationOptions.ES_QUERY, "?q=me*");
        QueryBuilder qb = QueryBuilder.query(sets);
        Field mapping = client.getMapping();
        ScrollReader reader = new ScrollReader(new JdkValueReader(), mapping, true, "_metadata");

        int count = 0;
        for (ScrollQuery query = qb.build(client, reader); query.hasNext();) {
            Object[] next = query.next();
            assertNotNull(next);
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
    @Test
    public void testQueryShards() throws Exception {
        Map<Shard, Node> targetShards = client.getReadTargetShards();

        Field mapping = client.getMapping();
        ScrollReader reader = new ScrollReader(new JdkValueReader(), mapping, true, "_metadata");

        Settings sets = settings.copy();
        sets.setProperty(ConfigurationOptions.ES_QUERY, "?q=me*");

        String nodeId = targetShards.values().iterator().next().getId();
View Full Code Here
287
288
289
290
291
292
293
294
295
296
297
        }
        else {
            log.warn(String.format("No mapping found for [%s] - either no index exists or the partition configuration has been corrupted", partition));
        }

        ScrollReader scrollReader = new ScrollReader(reader, fieldMapping, settings.getReadMetadata(), settings.getReadMetadataField());

        // initialize REST client
        RestRepository client = new RestRepository(settings);

        QueryBuilder queryBuilder = QueryBuilder.query(settings).shard(partition.shardId).onlyNode(partition.nodeId);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
        this.metadataField = metadataField;
    }

    @Test
    public void testScrollWithFields() throws IOException {
        ScrollReader reader = new ScrollReader(new JdkValueReader(), null, readMetadata, metadataField);
        InputStream stream = getClass().getResourceAsStream("scroll-fields.json");
        List<Object[]> read = reader.read(stream);
        assertEquals(3, read.size());
        Object[] objects = read.get(0);
        assertTrue(((Map) objects[1]).containsKey("fields"));
    }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
        assertTrue(((Map) objects[1]).containsKey("fields"));
    }

    @Test
    public void testScrollWithMatchedQueries() throws IOException {
        ScrollReader reader = new ScrollReader(new JdkValueReader(), null, readMetadata, metadataField);
        InputStream stream = getClass().getResourceAsStream("scroll-matched-queries.json");
        List<Object[]> read = reader.read(stream);
        assertEquals(3, read.size());
        Object[] objects = read.get(0);
        assertTrue(((Map) objects[1]).containsKey("foo"));
    }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    @Test
    public void testScrollWithNestedFields() throws IOException {
        InputStream stream = getClass().getResourceAsStream("scroll-source-mapping.json");
        Field fl = Field.parseField(new ObjectMapper().readValue(stream, Map.class));
        ScrollReader reader = new ScrollReader(new JdkValueReader(), fl, readMetadata, metadataField);
        stream = getClass().getResourceAsStream("scroll-source.json");

        List<Object[]> read = reader.read(stream);

        assertEquals(3, read.size());
        Object[] objects = read.get(0);
        assertTrue(((Map) objects[1]).containsKey("source"));
        Map map = (Map) read.get(2)[1];
View Full Code Here
TOP

Related Classes of org.elasticsearch.hadoop.serialization.ScrollReader

  • org.elasticsearch.hadoop.cascading.EsLocalTap
  • org.elasticsearch.hadoop.hive.HiveValueReaderTest
  • org.elasticsearch.hadoop.integration.rest.AbstractRestQueryTest
  • org.elasticsearch.hadoop.mr.EsInputFormat$ShardRecordReader
  • org.elasticsearch.hadoop.rest.RestService
  • org.elasticsearch.hadoop.rest.ScrollReaderTest
  • org.elasticsearch.hadoop.serialization.json.JacksonJsonParser
  • org.elasticsearch.hadoop.serialization.Parser.NumberType
  • org.elasticsearch.hadoop.serialization.Parser.Token
  • org.elasticsearch.hadoop.util.BytesArray
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.