Examples of HowlFieldSchema

org.apache.howl.data.schema.HowlFieldSchema

Examples of org.apache.howl.data.schema.HowlFieldSchema

  private HowlSchema convertPigSchemaToHowlSchema(Schema pigSchema, HowlSchema tableSchema) throws FrontendException{


    List<HowlFieldSchema> fieldSchemas = new ArrayList<HowlFieldSchema>(pigSchema.size());
    for(FieldSchema fSchema : pigSchema.getFields()){
      byte type = fSchema.type;
      HowlFieldSchema howlFSchema;


      try {


        // Find out if we need to throw away the tuple or not.
        if(type == DataType.BAG && removeTupleFromBag(tableSchema, fSchema)){
          List<HowlFieldSchema> arrFields = new ArrayList<HowlFieldSchema>(1);
          arrFields.add(getHowlFSFromPigFS(fSchema.schema.getField(0).schema.getField(0)));
          howlFSchema = new HowlFieldSchema(fSchema.alias, Type.ARRAY, new HowlSchema(arrFields), null);
      }
      else{
          howlFSchema = getHowlFSFromPigFS(fSchema);
      }
      fieldSchemas.add(howlFSchema);

View Full Code Here

Examples of org.apache.howl.data.schema.HowlFieldSchema

    byte type = fSchema.type;
    switch(type){


    case DataType.CHARARRAY:
    case DataType.BIGCHARARRAY:
      return new HowlFieldSchema(fSchema.alias, Type.STRING, null);


    case DataType.INTEGER:
      return new HowlFieldSchema(fSchema.alias, Type.INT, null);


    case DataType.LONG:
      return new HowlFieldSchema(fSchema.alias, Type.BIGINT, null);


    case DataType.FLOAT:
      return new HowlFieldSchema(fSchema.alias, Type.FLOAT, null);


    case DataType.DOUBLE:
      return new HowlFieldSchema(fSchema.alias, Type.DOUBLE, null);


    case DataType.BAG:
      Schema bagSchema = fSchema.schema;
      List<HowlFieldSchema> arrFields = new ArrayList<HowlFieldSchema>(1);
      arrFields.add(getHowlFSFromPigFS(bagSchema.getField(0)));
      return new HowlFieldSchema(fSchema.alias, Type.ARRAY, new HowlSchema(arrFields), "");


    case DataType.TUPLE:
      List<String> fieldNames = new ArrayList<String>();
      List<HowlFieldSchema> howlFSs = new ArrayList<HowlFieldSchema>();
      for( FieldSchema fieldSchema : fSchema.schema.getFields()){
        fieldNames.add( fieldSchema.alias);
        howlFSs.add(getHowlFSFromPigFS(fieldSchema));
      }
      return new HowlFieldSchema(fSchema.alias, Type.STRUCT, new HowlSchema(howlFSs), "");


    case DataType.MAP:{
      // Pig's schema contain no type information about map's keys and
      // values. So, if its a new column assume <string,string> if its existing
      // return whatever is contained in the existing column.
      HowlFieldSchema mapField = getTableCol(fSchema.alias, howlTblSchema);
      HowlFieldSchema valFS;
      List<HowlFieldSchema> valFSList = new ArrayList<HowlFieldSchema>(1);


      if(mapField != null){
        Type mapValType = mapField.getMapValueSchema().get(0).getType();


        switch(mapValType){
        case STRING:
        case BIGINT:
        case INT:
        case FLOAT:
        case DOUBLE:
          valFS = new HowlFieldSchema(fSchema.alias, mapValType, null);
          break;
        default:
          throw new FrontendException("Only pig primitive types are supported as map value types.", PigHowlUtil.PIG_EXCEPTION_CODE);
        }
        valFSList.add(valFS);
        return new HowlFieldSchema(fSchema.alias,Type.MAP,Type.STRING, new HowlSchema(valFSList),"");
      }


      // Column not found in target table. Its a new column. Its schema is map<string,string>
      valFS = new HowlFieldSchema(fSchema.alias, Type.STRING, "");
      valFSList.add(valFS);
      return new HowlFieldSchema(fSchema.alias,Type.MAP,Type.STRING, new HowlSchema(valFSList),"");
     }


    default:
      throw new FrontendException("Unsupported type: "+type+"  in Pig's schema", PigHowlUtil.PIG_EXCEPTION_CODE);
    }

View Full Code Here

Examples of org.apache.howl.data.schema.HowlFieldSchema

      //      }
      //      return innerList;
    case ARRAY:
      // Unwrap the bag.
      DataBag pigBag = (DataBag)pigObj;
      HowlFieldSchema tupFS = howlFS.getArrayElementSchema().get(0);
      boolean needTuple = tupFS.getType() == Type.STRUCT;
      List<Object> bagContents = new ArrayList<Object>((int)pigBag.size());
      Iterator<Tuple> bagItr = pigBag.iterator();


      while(bagItr.hasNext()){
        // If there is only one element in tuple contained in bag, we throw away the tuple.

View Full Code Here

Examples of org.apache.howl.data.schema.HowlFieldSchema


    for(FieldSchema pigField : pigSchema.getFields()){
      byte type = pigField.type;
      String alias = pigField.alias;
      validateAlias(alias);
      HowlFieldSchema howlField = getTableCol(alias, tblSchema);


      if(DataType.isComplex(type)){
        switch(type){


        case DataType.MAP:
          if(howlField != null){
            if(howlField.getMapKeyType() != Type.STRING){
              throw new FrontendException("Key Type of map must be String "+howlField,  PigHowlUtil.PIG_EXCEPTION_CODE);
            }
            if(howlField.getMapValueSchema().get(0).isComplex()){
              throw new FrontendException("Value type of map cannot be complex" + howlField, PigHowlUtil.PIG_EXCEPTION_CODE);
            }
          }
          break;


        case DataType.BAG:
          // Only map is allowed as complex type in tuples inside bag.
          for(FieldSchema innerField : pigField.schema.getField(0).schema.getFields()){
            if(innerField.type == DataType.BAG || innerField.type == DataType.TUPLE) {
              throw new FrontendException("Complex types cannot be nested. "+innerField, PigHowlUtil.PIG_EXCEPTION_CODE);
            }
            validateAlias(innerField.alias);
          }
          if(howlField != null){
            // Do the same validation for HowlSchema.
            HowlFieldSchema arrayFieldScehma = howlField.getArrayElementSchema().get(0);
            Type hType = arrayFieldScehma.getType();
            if(hType == Type.STRUCT){
              for(HowlFieldSchema structFieldInBag : arrayFieldScehma.getStructSubSchema().getFields()){
                if(structFieldInBag.getType() == Type.STRUCT || structFieldInBag.getType() == Type.ARRAY){
                  throw new FrontendException("Nested Complex types not allowed "+ howlField, PigHowlUtil.PIG_EXCEPTION_CODE);
                }
              }
            }
            if(hType == Type.MAP){
              if(arrayFieldScehma.getMapKeyType() != Type.STRING){
                throw new FrontendException("Key Type of map must be String "+howlField, PigHowlUtil.PIG_EXCEPTION_CODE);
              }
              if(arrayFieldScehma.getMapValueSchema().get(0).isComplex()){
                throw new FrontendException("Value type of map cannot be complex "+howlField, PigHowlUtil.PIG_EXCEPTION_CODE);
              }
            }
            if(hType == Type.ARRAY) {
              throw new FrontendException("Arrays cannot contain array within it. "+howlField, PigHowlUtil.PIG_EXCEPTION_CODE);

View Full Code Here

Examples of org.apache.howl.data.schema.HowlFieldSchema

    // we simulate the tuple by putting the single field in a tuple
    ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1];
    bagSubFieldSchemas[0] = new ResourceFieldSchema().setName("innertuple")
      .setDescription("The tuple in the bag")
      .setType(DataType.TUPLE);
    HowlFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0);
    if(arrayElementFieldSchema.getType() == Type.STRUCT) {
      bagSubFieldSchemas[0].setSchema(getTupleSubSchema(arrayElementFieldSchema));
    } else {
      ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1];
      innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName("innerfield")
        .setDescription("The inner field in the tuple in the bag")

View Full Code Here

Examples of org.apache.howl.data.schema.HowlFieldSchema

  public static DataBag transformToBag(List<? extends Object> list, HowlFieldSchema hfs) throws Exception {
    if (list == null){
      return null;
    }


    HowlFieldSchema elementSubFieldSchema = hfs.getArrayElementSchema().getFields().get(0);
    if (elementSubFieldSchema.getType() == Type.STRUCT){
      DataBag db = new DefaultDataBag();
      for (Object o : list){
        db.add(transformToTuple((List<Object>)o,elementSubFieldSchema));
      }
      return db;

View Full Code Here

Examples of org.apache.howl.data.schema.HowlFieldSchema

      }
  }


  private static void validateIsPigCompatibleArrayWithPrimitivesOrSimpleComplexTypes(
          HowlFieldSchema hfs) throws IOException {
      HowlFieldSchema subFieldSchema = hfs.getArrayElementSchema().getFields().get(0);
      if (subFieldSchema.getType() == Type.STRUCT){
          validateIsPigCompatibleStructWithPrimitives(subFieldSchema);
      }else if (subFieldSchema.getType() == Type.MAP) {
          validateIsPigCompatiblePrimitive(subFieldSchema.getMapValueSchema().getFields().get(0));
      }else {
          validateIsPigCompatiblePrimitive(subFieldSchema);
      }
  }

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.