Package com.datasalt.pangool.io.Schema

Examples of com.datasalt.pangool.io.Schema.Field

282
283
284
285
286
287
288
289
290
291
292
293
294
        throw new TupleMRException("Can't sort by field '" + sortElement.getName()
            + "' since its type differs among sources");
      }
     
      if (sortElement.getCustomComparator() != null){
        Field field = firstSchema.getField(sortElement.getName());
        if (field.getType() != Type.OBJECT){
          throw new TupleMRException("Not allowed to specify custom comparator for type="+field.getType());
        }
      }
    }
    // group by fields need to be a prefix of sort by fields
    for(String groupField : groupByFields) {
View Full Code Here
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
      if(!Schema.containsFieldUsingAlias(schema,e.getName(), aliases)){
        throw new TupleMRException("Source '" + schemaName + "' doesn't contain field '"
            + e.getName());
      }
      if (e.getCustomComparator() != null){
        Field field = schema.getField(e.getName());
        if (field == null){
            field = schema.getField(aliases.get(e.getName()));
        }
        if (field.getType() != Type.OBJECT){
          throw new TupleMRException("Not allowed to set custom comparator for type="+field.getType());
        }
      }
    }

    for(SortElement e : ordering.getElements()) {
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

  public int compare(Schema schema, Criteria c, ITuple w1, int[] index1, ITuple w2,
      int[] index2,FieldSerializer[] serializers) {
    for(int i = 0; i < c.getElements().size(); i++) {
      Field field = schema.getField(i);
      SortElement e = c.getElements().get(i);
      Object o1 = w1.get(index1[i]);
      Object o2 = w2.get(index2[i]);
      FieldSerializer serializer = (serializers == null) ? null : serializers[i];
      int comparison = compareObjects(o1, o2, e.getCustomComparator(), field.getType(),serializer);
      if(comparison != 0) {
        return(e.getOrder() == Order.ASC ? comparison : -comparison);
      }
    }
    return 0;
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
264
  protected int compare(byte[] b1, int s1, byte[] b2, int s2, Schema schema,
      Criteria criteria, Offsets o) throws IOException {
    o.offset1 = s1;
    o.offset2 = s2;
    for(int depth = 0; depth < criteria.getElements().size(); depth++) {
      Field field = schema.getField(depth);
      Field.Type type = field.getType();
      SortElement sortElement = criteria.getElements().get(depth);
      Order sort = sortElement.getOrder();
      RawComparator comparator = sortElement.getCustomComparator();

      if(comparator != null) {
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
   */
  public static Schema toPangoolSchema(org.apache.avro.Schema avroSchema) {
    List<Field> fields = new ArrayList<Field>();
    for(org.apache.avro.Schema.Field avroField : avroSchema.getFields()) {
      org.apache.avro.Schema.Type type = avroField.schema().getType();
      Field pangoolField;
      switch(type){
      case INT: pangoolField = Field.create(avroField.name(),Type.INT); break;
      case LONG: pangoolField = Field.create(avroField.name(),Type.LONG); break;
      case FLOAT: pangoolField = Field.create(avroField.name(),Type.FLOAT); break;
      case DOUBLE: pangoolField = Field.create(avroField.name(),Type.DOUBLE); break;
      case BOOLEAN: pangoolField = Field.create(avroField.name(),Type.BOOLEAN); break;
      case STRING: pangoolField = Field.create(avroField.name(),Type.STRING); break;
      case BYTES:
        if (avroField.getProp(Field.METADATA_BYTES_AS_OBJECT) == null){
          pangoolField = Field.create(avroField.name(),Type.BYTES);
        } else {
          try {
          String objectClazz = avroField.getProp(Field.METADATA_OBJECT_CLASS);
          if (objectClazz != null){
            pangoolField = Field.createObject(avroField.name(),Class.forName(objectClazz));
          } else {
           
              String serializerString = avroField.getProp(Field.METADATA_OBJECT_SERIALIZER);
              String deserializerString = avroField.getProp(Field.METADATA_OBJECT_DESERIALIZER);
              Class<? extends FieldSerializer> ser=
                 (serializerString == null) ? null :
                  (Class<? extends FieldSerializer>)Class.forName(serializerString);
             
              Class<? extends FieldDeserializer> deser = (deserializerString == null) ? null:
                (Class<? extends FieldDeserializer>)Class.forName(deserializerString);
              pangoolField = Field.createObject(avroField.name(),ser,deser);
          }
          } catch(ClassNotFoundException e) {
            throw new PangoolRuntimeException(e);
          }
        }
        break;
      case ENUM:
        String objectClazz = avroField.getProp(Field.METADATA_OBJECT_CLASS);
        try{
          pangoolField = Field.createEnum(avroField.name(),Class.forName(objectClazz));
          } catch(ClassNotFoundException e){
            throw new PangoolRuntimeException(e);
          }
        break;
      default:
        throw new PangoolRuntimeException("Avro type:" + type + " can't be converted to Pangool Schema type");
      }
      //add properties
      for(Map.Entry<String,String> entry : avroField.props().entrySet()){
        if (!Field.RESERVED_KEYWORDS.contains(entry.getKey())){
          pangoolField.addProp(entry.getKey(),entry.getValue());
        }
      }
      fields.add(pangoolField);
     
    }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
    if (record == null){
      record = new Record(avroSchema);
    }
    for(int i = 0; i < pangoolSchema.getFields().size(); i++) {
      Object obj = tuple.get(i);
      Field field = pangoolSchema.getField(i);
      switch(field.getType()){
      case INT:
      case LONG:
      case FLOAT:
      case BOOLEAN:
      case DOUBLE:
      case BYTES:
        record.put(i, obj); //optimistic
        break;
      case OBJECT:
        FieldSerializer customSer = customSerializers[i];
        DataOutputBuffer buffer = buffers[i];
        buffer.reset();
        if (customSer != null){
          customSer.open(buffer);
          customSer.serialize(obj);
          customSer.close(); //TODO is this safe ?
        } else {
          hadoopSer.ser(obj, buffer);
        }
        //TODO this byteBuffer instances should be cached and reused
        ByteBuffer byteBuffer = ByteBuffer.wrap(buffer.getData(), 0,buffer.getLength());
        record.put(i, byteBuffer);
        break;
      case ENUM:
        record.put(i,obj.toString());
        break;
      case STRING:
        record.put(i,new Utf8(obj.toString())); //could be directly String ?
        break;
      default:
          throw
          new IOException("Not correspondence to Avro type from Pangool type " + field.getType());
      }
    }
    return record;
  }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
   * or deserialized
   * @return
   */
  public static Field createAvroField(String name,
       org.apache.avro.Schema avroSchema,  boolean isReflect){
    Field field = Field.createObject(name,Object.class);
    field.setObjectSerialization(AvroFieldSerialization.class);
    field.addProp("avro.schema",avroSchema.toString());
    field.addProp("avro.reflection",Boolean.toString(isReflect));
    return field;
  }
View Full Code Here
105
106
107
108
109
110
111
112
113
114
   * @param name Field's name
   * @param schema The schema of the field
   * @return
   */
  public static Field createTupleField(String name, Schema schema) {
    Field field = Field.createObject(name,Object.class);
    field.setObjectSerialization(TupleFieldSerialization.class);
    field.addProp("schema", schema.toString());
    return field;
  }
View Full Code Here
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
            }
            String currentValue = "";
            try {
              currentValue = readLine.get(index);
              if(currentValue != null) {
                Field field = schema.getFields().get(i);
                switch(field.getType()) {
                case DOUBLE:
                  tuple.set(i, Double.parseDouble(currentValue.trim()));
                  break;
                case FLOAT:
                  tuple.set(i, Float.parseFloat(currentValue.trim()));
                  break;
                case ENUM:
                  Class clazz = field.getObjectClass();
                  tuple.set(i, Enum.valueOf(clazz, currentValue.trim()));
                  break;
                case INT:
                  tuple.set(i, Integer.parseInt(currentValue.trim()));
                  break;
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
  public void readFields(ITuple tuple, Deserializer[] customDeserializers) throws IOException {
    Schema schema = tuple.getSchema();
    for(int index = 0; index < schema.getFields().size(); index++) {
      Deserializer customDeser = customDeserializers[index];
      Field field = schema.getField(index);
      switch(field.getType()) {
      case INT:
        tuple.set(index, WritableUtils.readVInt(input));
        break;
      case LONG:
        tuple.set(index, WritableUtils.readVLong(input));
        break;
      case DOUBLE:
        tuple.set(index, input.readDouble());
        break;
      case FLOAT:
        tuple.set(index, input.readFloat());
        break;
      case STRING:
        readUtf8(input, tuple, index);
        break;
      case BOOLEAN:
        byte b = input.readByte();
        tuple.set(index, (b != 0));
        break;
      case ENUM:
        readEnum(input, tuple, field.getObjectClass(), index);
        break;
      case BYTES:
        readBytes(input, tuple, index);
        break;
      case OBJECT:
        readCustomObject(input, tuple, field.getObjectClass(), index, customDeser);
        break;
      default:
        throw new IOException("Not supported type:" + field.getType());
      }
    }
  }
View Full Code Here
TOP

Related Classes of com.datasalt.pangool.io.Schema.Field

  • com.datasalt.pangool.BaseTest
  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin
  • com.datasalt.pangool.io.Fields
  • com.datasalt.pangool.io.Tuple
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TupleTextInputFormat$TupleTextInputReader
  • com.datasalt.pangool.tuplemr.mapred.SortComparator
  • com.datasalt.pangool.tuplemr.mapred.TestComparators
  • com.datasalt.pangool.tuplemr.serialization.SimpleTupleDeserializer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.