Examples of org.kitesdk.morphline.api.Record

org.kitesdk.morphline.api.Record
A record is a set of named fields where each field has a list of one or more values. A value can be of any type, i.e. any Java Object. That is, a record is a {@link ListMultimap} asin Guava’s {@link ArrayListMultimap}. Note that a field can be multi-valued and that any two records need not use common field names. This flexible data model corresponds exactly to the characteristics of the Solr/Lucene data model (i.e. a record is a SolrInputDocument). A field with zero values is removed from the record - it does not exist as such.

  @Test
  public void testReadJson() throws Exception {
    morphline = createMorphline("test-morphlines/readJson");    
    for (int j = 0; j < 3; j++) { // also test reuse of objects and low level avro buffers
      InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/stream.json"));
      Record record = new Record();
      record.put(Fields.ATTACHMENT_BODY, in);
      
      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));

View Full Code Here

  @Test
  public void testReadJsonWithMap() throws Exception {
    morphline = createMorphline("test-morphlines/readJsonWithMap");    
    for (int j = 0; j < 3; j++) { // also test reuse of objects and low level avro buffers
      InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/stream.json"));
      Record record = new Record();
      record.put(Fields.ATTACHMENT_BODY, in);
      
      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));

View Full Code Here

  @Test
  public void testExtractJsonPaths() throws Exception {
    morphline = createMorphline("test-morphlines/extractJsonPaths");    
    File file = new File(RESOURCES_DIR + "/test-documents/arrays.json");
    InputStream in = new FileInputStream(file);
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));

View Full Code Here

  @Test
  public void testExtractJsonPathsFlattened() throws Exception {
    morphline = createMorphline("test-morphlines/extractJsonPathsFlattened");    
    File file = new File(RESOURCES_DIR + "/test-documents/arrays.json");
    InputStream in = new FileInputStream(file);
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));

View Full Code Here

  
  @Test
  public void testIPv6() throws Exception {
    morphline = createMorphline("test-morphlines/geoIP");    
    
    Record record = new Record();
    String ip = "2001:620::1";
    record.put("ip", ip);
    Record expected = new Record();
    expected.put("ip", ip);
    expected.put("/country/iso_code", "CH");
    expected.put("/country/names/en", "Switzerland");
    expected.put("/country/names/zh-CN", "瑞士");
    expected.put("/location/latitude", 47.00016);
    expected.put("/location/longitude", 8.01427);
    expected.put("/location/latitude_longitude", "47.00016,8.01427");
    expected.put("/location/longitude_latitude", "8.01427,47.00016");
    
    processAndVerifySuccess(record, expected, false);
    Notifications.notifyShutdown(morphline);
    Notifications.notifyShutdown(morphline);
  }

View Full Code Here

  @Test
  public void testComplexDocuments() throws Exception {
    morphline = createMorphline("test-morphlines/extractJsonPaths");    
    File file = new File(RESOURCES_DIR + "/test-documents/complex.json");
    InputStream in = new FileInputStream(file);
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));

View Full Code Here

  @SuppressWarnings("unchecked")
  public void testExtractJsonPathsComplexFlattened() throws Exception {
    morphline = createMorphline("test-morphlines/extractJsonPathsFlattened");    
    File file = new File(RESOURCES_DIR + "/test-documents/complex.json");
    InputStream in = new FileInputStream(file);
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, in);
    
    startSession();
    assertEquals(1, collector.getNumStartEvents());
    assertTrue(morphline.process(record));        
    assertEquals(1, collector.getRecords().size());

View Full Code Here

    byte[] bytes = Files.toByteArray(file);
    long start = System.currentTimeMillis();
    long duration = durationSecs * 1000;
    int iters = 0; 
    while (System.currentTimeMillis() < start + duration) {
      Record record = new Record();
      record.put(Fields.ATTACHMENT_BODY, bytes);      
      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));    
      iters++;

View Full Code Here

    
    long start = System.currentTimeMillis();
    long duration = durationSecs * 1000;
    int iters = 0; 
    while (System.currentTimeMillis() < start + duration) {
      Record record = new Record();
      String msg = "128.101.101.101";
      record.put("ip", msg);
      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));    
      iters++;

View Full Code Here

  @Test
  public void testToAvroMap() throws Exception {
    morphline = createMorphline("test-morphlines/toAvroMap");
    
    byte[] bytes = new byte[] {47, 13};
    Record jdoc1 = new Record();     


    jdoc1.put("intField", 20);
    jdoc1.put("longField", 200L);
    jdoc1.put("stringField", "200");
    jdoc1.put("boolField", Boolean.TRUE);
    jdoc1.put("floatField", 200.0f);
    jdoc1.put("doubleField", 200.0);
    jdoc1.put("bytesField", bytes);
    jdoc1.put("nullField", null);
    jdoc1.getFields().putAll("arrayField", Arrays.asList(10.0, 20.0));
    collector.reset();
    assertTrue(morphline.process(jdoc1));
    
    GenericData.Record actual = (GenericData.Record) collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_BODY);
    Map map = (Map) actual.get("content");

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.kitesdk.morphline.api.Record

com.ngdata.hbaseindexer.morphline.LocalMorphlineResultToSolrMapper

com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapperTest

org.apache.flume.sink.solr.morphline.MorphlineHandlerImpl

org.apache.flume.sink.solr.morphline.TestMorphlineSolrSink

org.apache.solr.crunch.MorphlineFn

org.apache.solr.hadoop.morphline.MorphlineMapRunner

org.kitesdk.examples.morphlines.ExampleMorphlineTest

org.kitesdk.morphline.avro.AvroMorphlineTest

org.kitesdk.morphline.avro.ExtractAvroPathsBuilder$ExtractAvroPaths

org.kitesdk.morphline.avro.ExtractAvroTreeBuilder$ExtractAvroTree

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.