Examples of HCatRecord

org.apache.hcatalog.data.HCatRecord
Abstract class exposing get and set semantics for basic record usage. Note : HCatRecord is designed only to be used as in-memory representation only. Don't use it to store data on the physical device.
org.apache.hive.hcatalog.data.HCatRecord
Abstract class exposing get and set semantics for basic record usage. Note : HCatRecord is designed only to be used as in-memory representation only. Don't use it to store data on the physical device.

Examples of org.apache.hcatalog.data.HCatRecord

        private static int count = 0;


        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
            HCatRecord record = new DefaultHCatRecord(3);
            HCatSchema schema = jobInfo.getOutputSchema();
            String vals[] = value.toString().split(",");
            record.setInteger("key", schema, Integer.parseInt(vals[0]));
            synchronized (MapWriteAbortTransaction.class) {
                if (count == 2) {
                    failedKey = vals[0];
                    throw new IOException("Failing map to test abort");
                }
                for (int i = 1; i < vals.length; i++) {
                    String pair[] = vals[i].split(":");
                    record.set(pair[0], schema, pair[1]);
                }
                context.write(null, record);
                count++;
            }

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord

        @Override
        public void map(LongWritable key, Text value, Context context
        ) throws IOException, InterruptedException {
            {
                try {
                    HCatRecord rec = writeRecords.get(writeCount);
                    context.write(null, rec);
                    writeCount++;


                } catch (Exception e) {

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord




    @Override
    public Tuple getNext() throws IOException {
        try {
            HCatRecord hr = (HCatRecord) (reader.nextKeyValue() ? reader.getCurrentValue() : null);
            Tuple t = PigHCatUtil.transformToTuple(hr, outputSchema);
            // TODO : we were discussing an iter interface, and also a LazyTuple
            // change this when plans for that solidifies.
            return t;
        } catch (ExecException e) {

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord

        try {
            committer = outFormat.getOutputCommitter(cntxt);
            committer.setupTask(cntxt);
            writer = outFormat.getRecordWriter(cntxt);
            while (recordItr.hasNext()) {
                HCatRecord rec = recordItr.next();
                writer.write(null, rec);
            }
            writer.close(cntxt);
            if (committer.needsTaskCommit(cntxt)) {
                committer.commitTask(cntxt);

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord

            currentKey = baseRecordReader.createKey();
            currentValue = baseRecordReader.createValue();
        }


        while (baseRecordReader.next(currentKey, currentValue)) {
            HCatRecord r = null;
            Throwable t = null;


            errorTracker.incRecords();


            try {
                Object o = deserializer.deserialize(currentValue);
                r = new LazyHCatRecord(o, deserializer.getObjectInspector());
            } catch (Throwable throwable) {
                t = throwable;
            }


            if (r == null) {
                errorTracker.incErrors(t);
                continue;
            }


            DefaultHCatRecord dr = new DefaultHCatRecord(outputSchema.size());
            int i = 0;
            for (String fieldName : outputSchema.getFieldNames()) {
                if (dataSchema.getPosition(fieldName) != null) {
                    dr.set(i, r.get(fieldName, dataSchema));
                } else {
                    dr.set(i, valuesNotInDataCols.get(fieldName));
                }
                i++;
            }

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord

        private int i = 0;


        @Override
        protected void map(LongWritable key, Text value, Context context)
            throws IOException, InterruptedException {
            HCatRecord record = null;
            String[] splits = value.toString().split(",");
            switch (i) {
            case 0:
                record = new DefaultHCatRecord(2);
                record.set(0, splits[0]);
                record.set(1, splits[1]);
                break;
            case 1:
                record = new DefaultHCatRecord(1);
                record.set(0, splits[0]);
                break;
            case 2:
                record = new DefaultHCatRecord(3);
                record.set(0, splits[0]);
                record.set(1, splits[1]);
                record.set(2, "extra");
                break;
            default:
                Assert.fail("This should not happen!!!!!");
            }
            MultiOutputFormat.write(tableNames[i], null, record, context);

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord


    public static class MapHCatWrite extends Mapper<LongWritable, Text, BytesWritable, HCatRecord> {
        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
            HCatRecord record = new DefaultHCatRecord(3);
            HCatSchema schema = jobInfo.getOutputSchema();
            String vals[] = value.toString().split(",");
            record.setInteger("key", schema, Integer.parseInt(vals[0]));
            for (int i = 1; i < vals.length; i++) {
                String pair[] = vals[i].split(":");
                record.set(pair[0], schema, pair[1]);
            }
            context.write(null, record);
        }

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord

            intnum100 = (((Integer) value.get(3)));
            intnum = ((Integer) value.get(4));
            longnum = ((Long) value.get(5));
            floatnum = ((Float) value.get(6));
            doublenum = ((Double) value.get(7));
            HCatRecord output = new DefaultHCatRecord(8);
            output.set(0, intnum1000 + 10);
            if (isnoPartPig) {
                output.set(1, ((int) (id + 10)));
            } else {
                output.set(1, ((short) (id + 10)));
            }
            if (isnoPartPig) {
                output.set(2, (int) (intnum5 + 10));
            } else {
                output.set(2, (byte) (intnum5 + 10));
            }


            output.set(3, intnum100 + 10);
            output.set(4, intnum + 10);
            output.set(5, (long) (longnum + 10));
            output.set(6, (float) (floatnum + 10));
            output.set(7, (double) (doublenum + 10));
            for (int i = 0; i < 8; i++) {
                System.err.println("XXX: class:" + output.get(i).getClass());
            }
            context.write(new IntWritable(0), output);


        }

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord

            b = (Long) value.get(3);
            f = (Float) value.get(4);
            d = (Double) value.get(5);
            s = (String) value.get(6);


            HCatRecord record = new DefaultHCatRecord(7);
            record.set(0, t);
            record.set(1, si);
            record.set(2, i);
            record.set(3, b);
            record.set(4, f);
            record.set(5, d);
            record.set(6, s);


            context.write(null, record);


        }

View Full Code Here

Examples of org.apache.hcatalog.data.HCatRecord


                count++;
                sum += Double.parseDouble(gpa.toString());
            }


            HCatRecord record = new DefaultHCatRecord(2);
            record.set(0, name);
            record.set(1, Double.toString(sum));


            context.write(null, record);
        }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.