Package org.springframework.data.hadoop.store.input

Examples of org.springframework.data.hadoop.store.input.TextFileReader

53
54
55
56
57
58
59
60
61
62
63
  @Test
  public void testWithOneReader() throws Exception {
    Path path = new Path("/syarn-tmp/DataStoreItemReaderTests-testWithOneReader/data.txt");
    createTestData(path);
    DataStoreReader<String> reader = new TextFileReader(configuration, path, null, null, null);
    DataStoreItemReader<String> itemReader = new DataStoreItemReader<String>();
    itemReader.setDataStoreReader(reader);
    itemReader.setLineDataMapper(new PassThroughLineDataMapper());
    int count = 0;
    while (itemReader.read() != null) {
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    StaticLengthSplitter splitter = new StaticLengthSplitter(getConfiguration(), 1500);
    List<Split> splits = splitter.getSplits(path);
    assertThat(splits.size(), is(2));

    DataStoreReader<String> reader = new TextFileReader(configuration, path, null, splits.get(0), null);
    DataStoreItemReader<String> itemReader = new DataStoreItemReader<String>();
    itemReader.setDataStoreReader(reader);
    itemReader.setLineDataMapper(new PassThroughLineDataMapper());
    int count = 0;
    while (itemReader.read() != null) {
      count++;
    }

    reader = new TextFileReader(configuration, path, null, splits.get(1), null);
    itemReader = new DataStoreItemReader<String>();
    itemReader.setDataStoreReader(reader);
    itemReader.setLineDataMapper(new PassThroughLineDataMapper());
    while (itemReader.read() != null) {
      count++;
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
  @Test
  public void testRestore() throws Exception {
    Path path = new Path("/syarn-tmp/DataStoreItemReaderTests-testRestore/data.txt");
    createTestData(path);
    DataStoreReader<String> reader = new TextFileReader(configuration, path, null, null, null);
    DataStoreItemReader<String> itemReader = new DataStoreItemReader<String>();
    itemReader.setDataStoreReader(reader);
    itemReader.setLineDataMapper(new PassThroughLineDataMapper());

    ExecutionContext context = new ExecutionContext();
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
  @Test
  public void testRestoreToEnd() throws Exception {
    Path path = new Path("/syarn-tmp/DataStoreItemReaderTests-testRestoreToEnd/data.txt");
    createTestData(path);
    DataStoreReader<String> reader = new TextFileReader(configuration, path, null, null, null);
    DataStoreItemReader<String> itemReader = new DataStoreItemReader<String>();
    itemReader.setDataStoreReader(reader);
    itemReader.setLineDataMapper(new PassThroughLineDataMapper());

    ExecutionContext context = new ExecutionContext();
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
  @Test(expected = ItemStreamException.class)
  public void testRestoreFailure() throws Exception {
    Path path = new Path("/syarn-tmp/DataStoreItemReaderTests-testRestoreFailure/data.txt");
    createTestData(path);
    DataStoreReader<String> reader = new TextFileReader(configuration, path, null, null, null);
    DataStoreItemReader<String> itemReader = new DataStoreItemReader<String>();
    itemReader.setDataStoreReader(reader);
    itemReader.setLineDataMapper(new PassThroughLineDataMapper());

    ExecutionContext context = new ExecutionContext();
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
  @Test
  public void testSavePosition() throws Exception {
    Path path = new Path("/syarn-tmp/DataStoreItemReaderTests-testSavePosition/data.txt");
    createTestData(path);
    DataStoreReader<String> reader = new TextFileReader(configuration, path, null, null, null);
    DataStoreItemReader<String> itemReader = new DataStoreItemReader<String>();
    itemReader.setDataStoreReader(reader);
    itemReader.setLineDataMapper(new PassThroughLineDataMapper());
    int count = 0;
    while (itemReader.read() != null) {
View Full Code Here
45
46
47
48
49
50
51
52
53
54
    TextFileWriter writer = context.getBean("writer", TextFileWriter.class);

    TestUtils.writeData(writer, DATA09ARRAY, false, false);
    writer.flush();

    TextFileReader reader = new TextFileReader(getConfiguration(), writer.getPath(), null);
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
    writer.close();
    reader.close();
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
  public void testWriteAppendReadTextManyLines() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    writer.setAppendable(true);
    TestUtils.writeData(writer, DATA09ARRAY, false);

    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
    writer.close();
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
  @Test
  public void testWriteAppendReopen() throws IOException {
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    writer.setAppendable(true);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TextFileReader reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, DATA09ARRAY);
    writer.close();
    reader.close();

    writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    writer.setAppendable(true);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    reader = new TextFileReader(getConfiguration(), testDefaultPath, null);
    TestUtils.readDataAndAssert(reader, StringUtils.concatenateStringArrays(DATA09ARRAY, DATA09ARRAY));
    writer.close();
    reader.close();
  }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
    writer.setAppendable(true);

    writer.write(dataArray[0], headers);
    writer.flush();

    TextFileReader reader = new TextFileReader(getConfiguration(), new Path(testDefaultPath, "foo/" + nowYYYYMM + "/0_hash/jee_list/10_range/bar"), null);
    TestUtils.readDataAndAssert(reader, dataArray);
    writer.close();
    reader.close();
  }
View Full Code Here
TOP

Related Classes of org.springframework.data.hadoop.store.input.TextFileReader

  • org.apache.hadoop.io.Text
  • org.apache.hadoop.util.LineReader
  • org.springframework.data.hadoop.store.PartitionTextFileWriterTests
  • org.springframework.data.hadoop.store.RawStreamStoreTests
  • org.springframework.data.hadoop.store.TextFileStoreAppendCtxTests
  • org.springframework.data.hadoop.store.TextFileStoreAppendTests
  • org.springframework.data.hadoop.store.TextFileStoreCtxTests
  • org.springframework.data.hadoop.store.TextFileStoreSplitTests
  • org.springframework.data.hadoop.store.TextFileStoreTests
  • org.springframework.yarn.batch.item.DataStoreItemReaderTests
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.