Package org.apache.hcatalog.data.schema

Examples of org.apache.hcatalog.data.schema.HCatFieldSchema


    private static DataBag transformToBag(List<? extends Object> list, HCatFieldSchema hfs) throws Exception {
        if (list == null) {
            return null;
        }

        HCatFieldSchema elementSubFieldSchema = hfs.getArrayElementSchema().getFields().get(0);
        DataBag db = new DefaultDataBag();
        for (Object o : list) {
            Tuple tuple;
            if (elementSubFieldSchema.getType() == Type.STRUCT) {
                tuple = transformToTuple((List<Object>) o, elementSubFieldSchema);
            } else {
                // bags always contain tuples
                tuple = tupFac.newTuple(extractPigObject(o, elementSubFieldSchema));
            }
View Full Code Here


     */
    protected HCatSchema convertPigSchemaToHCatSchema(Schema pigSchema, HCatSchema tableSchema) throws FrontendException {
        List<HCatFieldSchema> fieldSchemas = new ArrayList<HCatFieldSchema>(pigSchema.size());
        for (FieldSchema fSchema : pigSchema.getFields()) {
            try {
                HCatFieldSchema hcatFieldSchema = getColFromSchema(fSchema.alias, tableSchema);

                fieldSchemas.add(getHCatFSFromPigFS(fSchema, hcatFieldSchema));
            } catch (HCatException he) {
                throw new FrontendException(he.getMessage(), PigHCatUtil.PIG_EXCEPTION_CODE, he);
            }
View Full Code Here

        byte type = fSchema.type;
        switch (type) {

        case DataType.CHARARRAY:
        case DataType.BIGCHARARRAY:
            return new HCatFieldSchema(fSchema.alias, Type.STRING, null);

        case DataType.INTEGER:
            if (hcatFieldSchema != null) {
                if (!SUPPORTED_INTEGER_CONVERSIONS.contains(hcatFieldSchema.getType())) {
                    throw new FrontendException("Unsupported type: " + type + "  in Pig's schema",
                        PigHCatUtil.PIG_EXCEPTION_CODE);
                }
                return new HCatFieldSchema(fSchema.alias, hcatFieldSchema.getType(), null);
            } else {
                return new HCatFieldSchema(fSchema.alias, Type.INT, null);
            }

        case DataType.LONG:
            return new HCatFieldSchema(fSchema.alias, Type.BIGINT, null);

        case DataType.FLOAT:
            return new HCatFieldSchema(fSchema.alias, Type.FLOAT, null);

        case DataType.DOUBLE:
            return new HCatFieldSchema(fSchema.alias, Type.DOUBLE, null);

        case DataType.BYTEARRAY:
            return new HCatFieldSchema(fSchema.alias, Type.BINARY, null);

        case DataType.BAG:
            Schema bagSchema = fSchema.schema;
            List<HCatFieldSchema> arrFields = new ArrayList<HCatFieldSchema>(1);
            FieldSchema field;
            // Find out if we need to throw away the tuple or not.
            if (removeTupleFromBag(hcatFieldSchema, fSchema)) {
                field = bagSchema.getField(0).schema.getField(0);
            } else {
                field = bagSchema.getField(0);
            }
            arrFields.add(getHCatFSFromPigFS(field, hcatFieldSchema == null ? null : hcatFieldSchema.getArrayElementSchema().get(0)));
            return new HCatFieldSchema(fSchema.alias, Type.ARRAY, new HCatSchema(arrFields), "");

        case DataType.TUPLE:
            List<String> fieldNames = new ArrayList<String>();
            List<HCatFieldSchema> hcatFSs = new ArrayList<HCatFieldSchema>();
            HCatSchema structSubSchema = hcatFieldSchema == null ? null : hcatFieldSchema.getStructSubSchema();
            List<FieldSchema> fields = fSchema.schema.getFields();
            for (int i = 0; i < fields.size(); i++) {
                FieldSchema fieldSchema = fields.get(i);
                fieldNames.add(fieldSchema.alias);
                hcatFSs.add(getHCatFSFromPigFS(fieldSchema, structSubSchema == null ? null : structSubSchema.get(i)));
            }
            return new HCatFieldSchema(fSchema.alias, Type.STRUCT, new HCatSchema(hcatFSs), "");

        case DataType.MAP: {
            // Pig's schema contain no type information about map's keys and
            // values. So, if its a new column assume <string,string> if its existing
            // return whatever is contained in the existing column.

            HCatFieldSchema valFS;
            List<HCatFieldSchema> valFSList = new ArrayList<HCatFieldSchema>(1);

            if (hcatFieldSchema != null) {
                return new HCatFieldSchema(fSchema.alias, Type.MAP, Type.STRING, hcatFieldSchema.getMapValueSchema(), "");
            }

            // Column not found in target table. Its a new column. Its schema is map<string,string>
            valFS = new HCatFieldSchema(fSchema.alias, Type.STRING, "");
            valFSList.add(valFS);
            return new HCatFieldSchema(fSchema.alias, Type.MAP, Type.STRING, new HCatSchema(valFSList), "");
        }

        default:
            throw new FrontendException("Unsupported type: " + type + "  in Pig's schema", PigHCatUtil.PIG_EXCEPTION_CODE);
        }
View Full Code Here

                if (pigObj == null) {
                    return null;
                }
                // Unwrap the bag.
                DataBag pigBag = (DataBag) pigObj;
                HCatFieldSchema tupFS = hcatFS.getArrayElementSchema().get(0);
                boolean needTuple = tupFS.getType() == Type.STRUCT;
                List<Object> bagContents = new ArrayList<Object>((int) pigBag.size());
                Iterator<Tuple> bagItr = pigBag.iterator();

                while (bagItr.hasNext()) {
                    // If there is only one element in tuple contained in bag, we throw away the tuple.
View Full Code Here

        // Iterate through all the elements in Pig Schema and do validations as
        // dictated by semantics, consult HCatSchema of table when need be.

        for (FieldSchema pigField : pigSchema.getFields()) {
            HCatFieldSchema hcatField = getColFromSchema(pigField.alias, tblSchema);
            validateSchema(pigField, hcatField);
        }

        try {
            PigHCatUtil.validateHCatTableSchemaFollowsPigRules(tblSchema);
View Full Code Here

        bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas));
        ResourceSchema expected = new ResourceSchema().setFields(bagSubFieldSchemas);

        // Get the actual converted schema.
        HCatSchema hCatSchema = new HCatSchema(Lists.newArrayList(
            new HCatFieldSchema("innerLlama", HCatFieldSchema.Type.STRING, null)));
        HCatFieldSchema hCatFieldSchema =
            new HCatFieldSchema("llama", HCatFieldSchema.Type.ARRAY, hCatSchema, null);
        ResourceSchema actual = PigHCatUtil.getBagSubSchema(hCatFieldSchema);

        Assert.assertEquals(expected.toString(), actual.toString());
    }
View Full Code Here

            }
            if (fpos == -1) {
                return; // unknown field, we return.
            }
        }
        HCatFieldSchema hcatFieldSchema = s.getFields().get(fpos);
        Object currField = extractCurrentField(p, null, hcatFieldSchema, false);
        r.set(fpos, currField);
    }
View Full Code Here

            if (valueToken != JsonToken.START_OBJECT) {
                throw new IOException("Start of Object expected");
            }
            Map<Object, Object> map = new LinkedHashMap<Object, Object>();
            Type keyType = hcatFieldSchema.getMapKeyType();
            HCatFieldSchema valueSchema = hcatFieldSchema.getMapValueSchema().get(0);
            while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) {
                Object k = getObjectOfCorrespondingPrimitiveType(p.getCurrentName(), keyType);
                Object v;
                if (valueSchema.getType() == HCatFieldSchema.Type.STRUCT) {
                    v = extractCurrentField(p, null, valueSchema, false);
                } else {
                    v = extractCurrentField(p, null, valueSchema, true);
                }
View Full Code Here

    }

    private HCatSchema getProjectionSchema() throws HCatException {

        HCatSchema schema = new HCatSchema(new ArrayList<HCatFieldSchema>());
        schema.append(new HCatFieldSchema("key", HCatFieldSchema.Type.STRING,
            ""));
        schema.append(new HCatFieldSchema("testqualifier1",
            HCatFieldSchema.Type.STRING, ""));
        return schema;
    }
View Full Code Here

        if (writeToNonPartPigTable) {
            List<HCatFieldSchema> newHfsList = new ArrayList<HCatFieldSchema>();
            // change smallint and tinyint to int
            for (HCatFieldSchema hfs : s.getFields()) {
                if (hfs.getTypeString().equals("smallint")) {
                    newHfsList.add(new HCatFieldSchema(hfs.getName(),
                        HCatFieldSchema.Type.INT, hfs.getComment()));
                } else if (hfs.getTypeString().equals("tinyint")) {
                    newHfsList.add(new HCatFieldSchema(hfs.getName(),
                        HCatFieldSchema.Type.INT, hfs.getComment()));
                } else {
                    newHfsList.add(hfs);
                }
            }
View Full Code Here

TOP

Related Classes of org.apache.hcatalog.data.schema.HCatFieldSchema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.