Package org.apache.hadoop.hive.serde2

Examples of org.apache.hadoop.hive.serde2.CustomTextSerDe


  /**
   * Tests a writing a stripe with a stride dictionary, followed by a stripe without
   * followed by a stripe with.
   */
  public void testEmptyInIntDictionaryStream() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here


   *
   * @throws Exception
   */
  @Test
  public void testSkipWithEmptyArrayInEnd() throws Exception {
    ObjectInspector inspector;
    List<String> emptyList = Collections.emptyList();
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringListWithId.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here

  /**
   * Tests a writing a stripe with an integer column, which enters low memory mode before the first
   * index stride is complete.
   */
  public void testIntEnterLowMemoryModeInFirstStride() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here

  /**
   * Tests a writing a stripe with a string column, which enters low memory mode before the first
   * index stride is complete.
   */
  public void testStringEnterLowMemoryModeInFirstStride() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here

    public void close(boolean b) throws IOException {
      // if we haven't written any rows, we need to create a file with a
      // generic schema.
      if (writer == null) {
        // a row with no columns
        ObjectInspector inspector = ObjectInspectorFactory.
            getStandardStructObjectInspector(new ArrayList<String>(),
                new ArrayList<ObjectInspector>());
        writer = OrcFile.createWriter(fs, path, this.conf, inspector,
            stripeSize, compress, compressionSize, rowIndexStride);
      }
View Full Code Here

    checkOutput(expectedFileUrl.getPath(), outputFilename);
  }

  @Test
  public void testDump() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
View Full Code Here

    writer.close();
    checkOutput("orc-file-dump.out");
  }

  private void testDictionary(Configuration conf, String expectedOutputFilename) throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    // Turn off using the approximate entropy heuristic to turn off dictionary encoding
View Full Code Here

  //Test that if the number of distinct characters in distinct strings is less than the configured
  // threshold dictionary encoding is turned off.  If dictionary encoding is turned off the length
  // of the dictionary stream for the column will be 0 in the ORC file dump.
  @Test
  public void testEntropyThreshold() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    OrcConf.setFloatVar(conf, OrcConf.ConfVars.HIVE_ORC_ENTROPY_KEY_STRING_SIZE_THRESHOLD, 1);
View Full Code Here

        return struct;
      }
    };

    // Test control case (cases match)
    StructField field = STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
  }
View Full Code Here

  public void testCaseInsensitiveFieldsStruct() throws Exception {
    OrcStruct struct = new OrcStruct(Lists.newArrayList(FIELD_0));
    struct.setFieldValue(0, new Text("a"));

    // Test control case (cases match)
    StructField field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.serde2.CustomTextSerDe

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.