Package org.apache.crunch.types

Examples of org.apache.crunch.types.PTableType


  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeSequenceFileFromPTable(final FileSystem fs, final Path path, final PTable table)
      throws IOException {
    final PTableType pType = table.getPTableType();
    final Class<?> keyClass = pType.getConverter().getKeyClass();
    final Class<?> valueClass = pType.getConverter().getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path, keyClass,
        valueClass);

    for (final Object o : table.materialize()) {
      final Pair<?,?> p = (Pair) o;
      final Object key = pType.getKeyType().getOutputMapFn().map(p.first());
      final Object value = pType.getValueType().getOutputMapFn().map(p.second());
      writer.append(key, value);
    }

    writer.close();
  }
View Full Code Here


    return PTypes.jsonString(clazz, AvroTypeFamily.getInstance());
  }

  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here

  }
 
  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeSequenceFileFromPTable(final FileSystem fs, final Path path, final PTable table)
      throws IOException {
    final PTableType pType = table.getPTableType();
    final Class<?> keyClass = pType.getConverter().getKeyClass();
    final Class<?> valueClass = pType.getConverter().getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path, keyClass,
        valueClass);

    for (final Object o : table.materialize()) {
      final Pair<?,?> p = (Pair) o;
      final Object key = pType.getKeyType().getOutputMapFn().map(p.first());
      final Object value = pType.getValueType().getOutputMapFn().map(p.second());
      writer.append(key, value);
    }

    writer.close();
  }
View Full Code Here

      if (at.getSchema().equals(Schema.create(Schema.Type.STRING))) {
        return true;
      }
    } else if (WritableTypeFamily.getInstance().equals(ptype.getFamily())) {
      if (ptype instanceof PTableType) {
        PTableType ptt = (PTableType) ptype;
        return isText(ptt.getKeyType()) && isText(ptt.getValueType());
      } else {
        return isText(ptype);
      }
    }
    return false;
View Full Code Here

    return PTypes.jsonString(clazz, AvroTypeFamily.getInstance());
  }

  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here

  }

  @SuppressWarnings({ "rawtypes", "unchecked" })
  private void writeSequenceFileFromPTable(final FileSystem fs, final Path path, final PTable table)
      throws IOException {
    final PTableType pType = table.getPTableType();
    final Class<?> keyClass = pType.getConverter().getKeyClass();
    final Class<?> valueClass = pType.getConverter().getValueClass();

    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path, keyClass,
        valueClass);

    for (final Object o : table.materialize()) {
      final Pair<?,?> p = (Pair) o;
      final Object key = pType.getKeyType().getOutputMapFn().map(p.first());
      final Object value = pType.getValueType().getOutputMapFn().map(p.second());
      writer.append(key, value);
    }

    writer.close();
  }
View Full Code Here

   * @param value the PType of the value in the table
   * @return PTableType for reading and writing avro tables
   */
  public static final <K, V> AvroTableType<K, V> tableOf(PType<K> key, PType<V> value) {
    if (key instanceof PTableType) {
      PTableType ptt = (PTableType) key;
      key = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    if (value instanceof PTableType) {
      PTableType ptt = (PTableType) value;
      value = Avros.pairs(ptt.getKeyType(), ptt.getValueType());
    }
    AvroType<K> avroKey = (AvroType<K>) key;
    AvroType<V> avroValue = (AvroType<V>) value;
    return new AvroTableType(avroKey, avroValue, Pair.class);
  }
View Full Code Here

  public <T> PType<T> as(PType<T> ptype) {
    if (ptype instanceof AvroType || ptype instanceof AvroGroupedTableType) {
      return ptype;
    }
    if (ptype instanceof PGroupedTableType) {
      PTableType ptt = ((PGroupedTableType) ptype).getTableType();
      return new AvroGroupedTableType((AvroTableType) as(ptt));
    }
    Class<T> typeClass = ptype.getTypeClass();
    PType<T> prim = Avros.getPrimitiveType(typeClass);
    if (prim != null) {
View Full Code Here

    if (ptype instanceof WritableType || ptype instanceof WritableTableType
        || ptype instanceof WritableGroupedTableType) {
      return ptype;
    }
    if (ptype instanceof PGroupedTableType) {
      PTableType ptt = ((PGroupedTableType) ptype).getTableType();
      return new WritableGroupedTableType((WritableTableType) as(ptt));
    }
    PType<T> prim = Writables.getPrimitiveType(ptype.getTypeClass());
    if (prim != null) {
      return prim;
View Full Code Here

  public SeqFileReaderFactory(PType<T> ptype) {
    this.converter = ptype.getConverter();
    this.mapFn = ptype.getInputMapFn();
    if (ptype instanceof PTableType) {
      PTableType ptt = (PTableType) ptype;
      this.key = SeqFileHelper.newInstance(ptt.getKeyType(), null);
      this.value = SeqFileHelper.newInstance(ptt.getValueType(), null);
    } else {
      this.key = NullWritable.get();
      this.value = SeqFileHelper.newInstance(ptype, null);
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.crunch.types.PTableType

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.