Package org.springframework.data.hadoop.fs

Examples of org.springframework.data.hadoop.fs.FsShell.ls()

84
85
86
87
88
89
90
91
92
93
94
  }

  protected void listFiles(Configuration configuration) {
    @SuppressWarnings("resource")
    FsShell shell = new FsShell(configuration);
    for (FileStatus s : shell.ls(true, "/")) {
      log.info("status " + s);
    }
  }

  protected void catFile(String path) {
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
    };
    writer.flush();
    writer.close();

    FsShell fsShell = new FsShell(getConfiguration());
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=05/*.avro").size() >= 2);
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=11/*.avro").size() >= 2);
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size() >= 2);
  }

  @Test
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
    writer.flush();
    writer.close();

    FsShell fsShell = new FsShell(getConfiguration());
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=05/*.avro").size() >= 2);
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=11/*.avro").size() >= 2);
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size() >= 2);
  }

  @Test
  public void runSettingWriterCacheSize() throws IOException {
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
    writer.close();

    FsShell fsShell = new FsShell(getConfiguration());
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=05/*.avro").size() >= 2);
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=11/*.avro").size() >= 2);
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size() >= 2);
  }

  @Test
  public void runSettingWriterCacheSize() throws IOException {
    PartitionStrategy partitionStrategy =
View Full Code Here
133
134
135
136
137
138
139
140
141
142
    };
    writer.flush();
    writer.close();

    FsShell fsShell = new FsShell(getConfiguration());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=05/*.avro").size());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=11/*.avro").size());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size());
  }
}
 
View Full Code Here
134
135
136
137
138
139
140
141
142
    writer.flush();
    writer.close();

    FsShell fsShell = new FsShell(getConfiguration());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=05/*.avro").size());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=11/*.avro").size());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size());
  }
}
 
View Full Code Here
135
136
137
138
139
140
141
142
    writer.close();

    FsShell fsShell = new FsShell(getConfiguration());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=05/*.avro").size());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=11/*.avro").size());
    assertEquals(1, fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size());
  }
}
 
View Full Code Here
22
23
24
25
26
27
28
29
30
31
32
    log.info("Hello from HelloPojo");
    log.info("About to list from hdfs root content");

    @SuppressWarnings("resource")
    FsShell shell = new FsShell(configuration);
    for (FileStatus s : shell.ls(false, "/")) {
      log.info(s);
    }
  }

}
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
        if (!fsShell.test(getBasePath())) {
          fsShell.mkdir(getBasePath());
        }
        int maxCounter = 0;
        boolean foundFile = false;
        Collection<FileStatus> fileStats = fsShell.ls(this.getBasePath());
        for (FileStatus fileStatus : fileStats) {
          String shortName = fileStatus.getPath().getName();
          int counterFromName = getCounterFromName(shortName);
          if (counterFromName != -1) {
            foundFile = true;
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
      FsShell fsShell = new FsShell(getFileSystem().getConf(), getFileSystem());

      if (getFileSystem().exists(new Path(getBasePath()))) {
        int maxCounter = 0;
        boolean foundFile = false;
        Collection<FileStatus> fileStats = fsShell.ls(this.getBasePath());
        for (FileStatus fileStatus : fileStats) {
          String shortName = fileStatus.getPath().getName();
          int counterFromName = getCounterFromName(shortName);
          if (counterFromName != -1) {
            foundFile = true;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.