Package org.apache.hadoop.zebra.mapred

Examples of org.apache.hadoop.zebra.mapred.TableInputFormat$DummyFileInputFormat$MultiPathFilter

86
87
88
89
90
91
92
    private org.apache.hadoop.zebra.schema.Schema projectionSchema;
  /**
   * default constructor
   */
  public TableLoader() {
    inputFormat = new TableInputFormat();
  }
View Full Code Here
94
95
96
97
98
99
100
101
  /**
   * @param projectionStr
   *       projection string passed from pig query.
   */
  public TableLoader(String projectionStr) {
    inputFormat = new TableInputFormat();
    projectionString = projectionStr;   
  }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
   *       projection string passed from pig query.
   * @param sorted
   *      need sorted table(s)?
   */
  public TableLoader(String projectionStr, String sorted) throws IOException {
      inputFormat = new TableInputFormat();
      if (projectionStr != null && !projectionStr.isEmpty())
        projectionString = projectionStr;   
      if (sorted.equalsIgnoreCase("sorted"))
        this.sorted = true;
      else
View Full Code Here
516
517
518
519
520
521
522
523
524
525
526
        projection = TableInputFormat.getProjection(conf);
      } catch (ParseException e) {
        throw new IOException("Schema parsing failed :"+e.getMessage());
      }
      numProjCols = Projection.getNumColumns(projection);
      TableInputFormat inputFormat = new TableInputFormat();
      if (sorted)
        TableInputFormat.requireSortedTable(conf, null);
      scanner = inputFormat.getRecordReader(split, conf, Reporter.NULL);
      key = new BytesWritable();
    }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    BasicTable.drop(path1, conf);
    BasicTable.drop(path2, conf);
    int total1 = TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path1, true);   
    int total2 = TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path2, true);   

    TableInputFormat inputFormat = new TableInputFormat();
    TableInputFormat.setInputPaths(jobConf, path1, path2);
    TableInputFormat.setProjection(jobConf, "source_table");
    InputSplit[] splits = inputFormat.getSplits(jobConf, -1);
    Assert.assertEquals(splits.length, 2);
    for (int i = 0; i < 2; i++)
    {
      int count = 0;
      RowTableSplit split = (RowTableSplit) splits[i];
      TableRecordReader rr = (TableRecordReader) inputFormat.getRecordReader(split, jobConf, null);
      Tuple t = TypesUtils.createTuple(1);
      BytesWritable key = new BytesWritable();
      while (rr.next(key, t)) {
        int idx= (Integer) t.get(0);
        Assert.assertEquals(idx, i);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
  public void testTfileSplit1()
          throws IOException, ParseException {
    BasicTable.drop(path, conf);
    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);   

    TableInputFormat inputFormat = new TableInputFormat();
    JobConf jobConf = new JobConf(conf);
    inputFormat.setInputPaths(jobConf, path);
    inputFormat.setMinSplitSize(jobConf, 100);
    inputFormat.setProjection(jobConf, "aa");
    InputSplit[] splits = inputFormat.getSplits(jobConf, 40);
   
    RowTableSplit split = (RowTableSplit) splits[0];
    String str = split.getSplit().toString();
    StringTokenizer tokens = new StringTokenizer(str, "\n");
    str = tokens.nextToken();
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
    BasicTable.drop(path, conf);
    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);   
    BasicTable.dropColumnGroup(path, conf, "CG0");
    BasicTable.dropColumnGroup(path, conf, "CG2");
   
    TableInputFormat inputFormat = new TableInputFormat();
    JobConf jobConf = new JobConf(conf);
    inputFormat.setInputPaths(jobConf, path);
    inputFormat.setMinSplitSize(jobConf, 100);
    InputSplit[] splits = inputFormat.getSplits(jobConf, 40);
   
    RowTableSplit split = (RowTableSplit) splits[0];
    String str = split.getSplit().toString();
    StringTokenizer tokens = new StringTokenizer(str, "\n");
    str = tokens.nextToken();
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
    TestBasicTable.createBasicTable(1, 100, "a, b, c, d, e, f", "[a, b]; [c, d]", null, path, true);   
    BasicTable.dropColumnGroup(path, conf, "CG0");
    BasicTable.dropColumnGroup(path, conf, "CG1");
    BasicTable.dropColumnGroup(path, conf, "CG2");
   
    TableInputFormat inputFormat = new TableInputFormat();
    JobConf jobConf = new JobConf(conf);
    inputFormat.setInputPaths(jobConf, path);
    inputFormat.setMinSplitSize(jobConf, 100);
    InputSplit[] splits = inputFormat.getSplits(jobConf, 40);
   
    Assert.assertEquals(splits.length, 0);
  }
View Full Code Here
70
71
72
73
74
75
76
 
  /**
   * default constructor
   */
  public TableLoader() {
    inputFormat = new TableInputFormat();
  }
View Full Code Here
78
79
80
81
82
83
84
85
  /**
   * @param projectionStr
   *       projection string passed from pig query.
   */
  public TableLoader(String projectionStr) {
    inputFormat = new TableInputFormat();
    projectionString = projectionStr;   
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.zebra.mapred.TableInputFormat$DummyFileInputFormat$MultiPathFilter

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.PathFilter
  • org.apache.hadoop.io.BytesWritable
  • org.apache.hadoop.io.file.tfile.RawComparable
  • org.apache.hadoop.mapred.FileSplit
  • org.apache.hadoop.zebra.io.BasicTable.Reader.RowSplit
  • org.apache.hadoop.zebra.io.BasicTableStatus
  • org.apache.hadoop.zebra.io.BlockDistribution
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.