Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FSDataInputStream.readInt()

66
67
68
69
70
71
72
73
74
75
76
    offsets = new int[blocks];
    filenos = new short[blocks];

    for (int i = 0; i < blocks; i++) {
      docnos[i] = in.readInt();
      offsets[i] = in.readInt();
      filenos[i] = in.readShort();
    }

    in.close();
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
  }

  public TTable_sliced(FileSystem fs, Path p) throws IOException {
    _fs = fs; _datapath = p;
    FSDataInputStream in = _fs.open(_datapath.suffix(Path.SEPARATOR + "metadata.bin"));
    _data = new IndexedFloatArray[in.readInt()];
    in.close();
  }

  public TTable_sliced(int e_voc_size, FileSystem fs, Path p) {
    _fs = fs; _datapath = p;
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
  static public String[] readDocnoData(Path p, FileSystem fs) throws IOException {
    LOG.warn("p: " + p);
    FSDataInputStream in = fs.open(p);

    // Docnos start at one, so we need an array that's one larger than number of docs.
    int sz = in.readInt() + 1;
    LOG.warn("creating array of length: " + sz);
    String[] arr = new String[sz];

    for (int i = 1; i < sz; i++) {
      arr[i] = in.readUTF();
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    FSDataInputStream in = fs.open(p);

    List<Integer> ids = null;
    int lastOffset = -1;

    int sz = in.readInt();
    docids = new int[105 * 50][];
    offsets = new int[105 * 50];

    for (int i = 0; i < 105 * 50; i++) {
      offsets[i] = -1;
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
   */
  static public int[] readDocnoMappingData(Path p, FileSystem fs) throws IOException {
    FSDataInputStream in = fs.open(p);

    // docnos start at one, so we need an array that's one larger than number of docs.
    int sz = in.readInt() + 1;
    int[] arr = new int[sz];

    for (int i = 1; i < sz; i++) {
      arr[i] = in.readInt();
    }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
    // docnos start at one, so we need an array that's one larger than number of docs.
    int sz = in.readInt() + 1;
    int[] arr = new int[sz];

    for (int i = 1; i < sz; i++) {
      arr[i] = in.readInt();
    }
    in.close();

    arr[0] = 0;
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    FSDataInputStream in = fs.open(p);

    List<Integer> ids = null;
    int lastOffset = -1;

    int sz = in.readInt();
    docids = new int[273 * 100][];
    offsets = new int[273 * 100];

    for (int i = 0; i < 273 * 100; i++) {
      offsets[i] = -1;
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    offsets = new int[blocks];
    fileno = new short[blocks];

    for (int i = 0; i < blocks; i++) {
      docnos[i] = in.readInt();
      offsets[i] = in.readInt();
      fileno[i] = in.readShort();

      if (i > 0 && i % 100000 == 0)
        LOG.info(i + " blocks read");
    }
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
    LOG.info("Reading docid mapping...");

    FSDataInputStream in = fs.open(p);
    // docnos start at one, so we need an array that's one larger than
    // number of docs
    int sz = in.readInt() + 1;
    int[] arr = new int[sz];
    int cnt = 0;
    for (int i = 1; i < sz; i++) {
      arr[i] = in.readInt();
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
    // number of docs
    int sz = in.readInt() + 1;
    int[] arr = new int[sz];
    int cnt = 0;
    for (int i = 1; i < sz; i++) {
      arr[i] = in.readInt();

      if (i % 500000 == 0) {
        LOG.info(i);
      }
      cnt++;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.