Examples of PTableType

Examples of org.apache.crunch.types.PTableType

885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
   * @param value the PType of the value in the table
   * @return PTableType for reading and writing avro tables
   */
  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here

Examples of org.apache.crunch.types.PTableType

140
141
142
143
144
145
146
147
148
149
150
  public <T> PType<T> as(PType<T> ptype) {
    if (ptype instanceof AvroType || ptype instanceof AvroGroupedTableType) {
      return ptype;
    }
    if (ptype instanceof PGroupedTableType) {
      PTableType ptt = ((PGroupedTableType) ptype).getTableType();
      return new AvroGroupedTableType((AvroTableType) as(ptt));
    }
    Class<T> typeClass = ptype.getTypeClass();
    PType<T> prim = Avros.getPrimitiveType(typeClass);
    if (prim != null) {
View Full Code Here

Examples of org.apache.crunch.types.PTableType

124
125
126
127
128
129
130
131
132
133
134
    if (ptype instanceof WritableType || ptype instanceof WritableTableType
        || ptype instanceof WritableGroupedTableType) {
      return ptype;
    }
    if (ptype instanceof PGroupedTableType) {
      PTableType ptt = ((PGroupedTableType) ptype).getTableType();
      return new WritableGroupedTableType((WritableTableType) as(ptt));
    }
    PType<T> prim = Writables.getPrimitiveType(ptype.getTypeClass());
    if (prim != null) {
      return prim;
View Full Code Here

Examples of org.apache.crunch.types.PTableType

51
52
53
54
55
56
57
58
59
60
61
62
63
  public SeqFileReaderFactory(PType<T> ptype) {
    this.converter = ptype.getConverter();
    this.mapFn = ptype.getInputMapFn();
    if (ptype instanceof PTableType) {
      PTableType ptt = (PTableType) ptype;
      this.key = SeqFileHelper.newInstance(ptt.getKeyType(), null);
      this.value = SeqFileHelper.newInstance(ptt.getValueType(), null);
    } else {
      this.key = NullWritable.get();
      this.value = SeqFileHelper.newInstance(ptype, null);
    }
  }
View Full Code Here

Examples of org.apache.crunch.types.PTableType

109
110
111
112
113
114
115
116
117
118
119
120
      if (at.getSchema().equals(Schema.create(Schema.Type.STRING))) {
        return true;
      }
    } else if (WritableTypeFamily.getInstance().equals(ptype.getFamily())) {
      if (ptype instanceof PTableType) {
        PTableType ptt = (PTableType) ptype;
        return isText(ptt.getKeyType()) && isText(ptt.getValueType());
      } else {
        return isText(ptype);
      }
    }
    return false;
View Full Code Here

Examples of org.apache.crunch.types.PTableType

260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeSequenceFileFromPTable(final FileSystem fs, final Path path, final PTable table)
      throws IOException {
    final PTableType pType = table.getPTableType();
    final Class<?> keyClass = pType.getConverter().getKeyClass();
    final Class<?> valueClass = pType.getConverter().getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path, keyClass,
        valueClass);

    for (final Object o : table.materialize()) {
      final Pair<?,?> p = (Pair) o;
      final Object key = pType.getKeyType().getOutputMapFn().map(p.first());
      final Object value = pType.getValueType().getOutputMapFn().map(p.second());
      writer.append(key, value);
    }

    writer.close();
  }
View Full Code Here

Examples of org.apache.crunch.types.PTableType

642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
    return PTypes.jsonString(clazz, AvroTypeFamily.getInstance());
  }

  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here

Examples of org.apache.crunch.types.PTableType

255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
  }
 
  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeSequenceFileFromPTable(final FileSystem fs, final Path path, final PTable table)
      throws IOException {
    final PTableType pType = table.getPTableType();
    final Class<?> keyClass = pType.getConverter().getKeyClass();
    final Class<?> valueClass = pType.getConverter().getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path, keyClass,
        valueClass);

    for (final Object o : table.materialize()) {
      final Pair<?,?> p = (Pair) o;
      final Object key = pType.getKeyType().getOutputMapFn().map(p.first());
      final Object value = pType.getValueType().getOutputMapFn().map(p.second());
      writer.append(key, value);
    }

    writer.close();
  }
View Full Code Here

Examples of org.apache.crunch.types.PTableType

595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
    return PTypes.jsonString(clazz, AvroTypeFamily.getInstance());
  }

  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here

Examples of org.apache.crunch.types.PTableType

630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
    return PTypes.jsonString(clazz, AvroTypeFamily.getInstance());
  }

  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.