Examples of org.kitesdk.morphline.api.Record

org.kitesdk.morphline.api.Record
A record is a set of named fields where each field has a list of one or more values. A value can be of any type, i.e. any Java Object. That is, a record is a {@link ListMultimap} asin Guava’s {@link ArrayListMultimap}. Note that a field can be multi-valued and that any two records need not use common field names. This flexible data model corresponds exactly to the characteristics of the Solr/Lucene data model (i.e. a record is a SolrInputDocument). A field with zero values is removed from the record - it does not exist as such.

  public void testXQueryJoin() throws Exception {
    File table = new File("target/test-table.xml");
    generateTestTable(table, 3);
    morphline = createMorphline("test-morphlines/xquery-join");    
    InputStream in = new FileInputStream(new File(RESOURCES_DIR + "/test-documents/helloworld.xml"));
    Record record = new Record();
    record.put("id", "123");
    record.put(Fields.ATTACHMENT_BODY, in);
    processAndVerifySuccess(record, 
        ImmutableMultimap.of("id", "123", "outputId", "2", "outputText", "Hello, World!")
        );    
    in.close();
  }

View Full Code Here

      List attachments = record.get(Fields.ATTACHMENT_BODY);
      List mimeTypes = record.get(Fields.ATTACHMENT_MIME_TYPE);
      List charsets = record.get(Fields.ATTACHMENT_CHARSET);
      List names = record.get(Fields.ATTACHMENT_NAME);
      for (int i = 0; i < attachments.size(); i++) {
        Record outputRecord = record.copy();        
        outputRecord.getFields().replaceValues(Fields.ATTACHMENT_BODY, Collections.singletonList(attachments.get(i)));
        
        List<Object> replacement;
        replacement = i < mimeTypes.size() ? Collections.singletonList(mimeTypes.get(i)) : Collections.emptyList();
        outputRecord.getFields().replaceValues(Fields.ATTACHMENT_MIME_TYPE, replacement);
        
        replacement = i < charsets.size() ? Collections.singletonList(charsets.get(i)) : Collections.emptyList();
        outputRecord.getFields().replaceValues(Fields.ATTACHMENT_CHARSET, replacement);
        
        replacement = i < names.size() ? Collections.singletonList(names.get(i)) : Collections.emptyList();
        outputRecord.getFields().replaceValues(Fields.ATTACHMENT_NAME, replacement);
        
        // pass record to next command in chain:
        if (!super.doProcess(outputRecord)) {
          return false;
        }

View Full Code Here

    byte[] bytes = Files.toByteArray(file);
    long start = System.currentTimeMillis();
    long duration = durationSecs * 1000;
    int iters = 0; 
    while (System.currentTimeMillis() < start + duration) {
      Record record = new Record();
      record.put(Fields.ATTACHMENT_BODY, bytes);      
      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));    
      iters++;

View Full Code Here

        path = (Path) obj;
      } else {
        path = new Path(obj.toString());
      }
      
      Record template = inputRecord.copy();
      AbstractParser.removeAttachments(template);
      template.put(Fields.ATTACHMENT_MIME_TYPE, AVRO_MEMORY_MIME_TYPE);


      AvroParquetReader<IndexedRecord> reader = null;
      try {
        reader = new AvroParquetReader(conf, path);
        while (true) {

View Full Code Here

    }
  
    @Override
    protected boolean doProcess(Record record) {
      for (Command childRule : childRules) {
        Record copy = copyRecords ? record.copy() : record;
        if (!catchExceptions) {
          if (childRule.process(copy)) {
            return true; // rule was executed successfully; no need to try the other remaining rules
          }          
        } else {

View Full Code Here

      }
    }
        
    private boolean extract(GenericContainer datum, Record inputRecord) {
      incrementNumRecords();
      Record outputRecord = inputRecord.copy();
      outputRecord.put(Fields.ATTACHMENT_BODY, datum);
        
      // pass record to next command in chain:
      return getChild().process(outputRecord);
    }

View Full Code Here


  @Test
  public void testBasic() throws Exception {
    morphline = createMorphline("test-morphlines/startReportingMetricsToHTTP");    
    
    Record record = new Record();
    String msg = "foo";
    record.put(Fields.MESSAGE, msg);
    Record expected = new Record();
    expected.put(Fields.MESSAGE, msg);
    processAndVerifySuccess(record, expected);


    if ("true".equals(System.getProperty("HttpMetricsMorphlineTest.isDemo"))) {
      // wait forever so user can browse to http://localhost:8080/ and interactively explore the features
      Thread.sleep(Long.MAX_VALUE);

View Full Code Here

      this.renderedConfig = config.root().render();
    }
    
    @Override
    protected boolean doProcess(Record inputRecord) {
      Record outputRecord;
      outputRecord = ((extractInPlace || !extract) ? inputRecord : inputRecord.copy());
      if (extractInPlace) {
        // Ensure that we mutate the record inplace only if *all* expressions match.
        // To ensure this we potentially run doMatch() twice: the first time to check, the second
        // time to mutate

View Full Code Here

  }
  
  private void testDetectMimeTypesInternal(String configFile) throws Exception {
    // verify that Avro is classified as Avro  
    morphline = createMorphline(configFile);    
    Record record = new Record();    
    record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(AVRO_FILE));
    startSession();
    morphline.process(record);
    assertEquals(AVRO_MIME_TYPE, collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE));


    // verify that JPG isnt' classified as JPG because this morphline uses includeDefaultMimeTypes : false 
    collector.reset();
    record = new Record();    
    record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE));
    startSession();
    morphline.process(record);
    assertEquals("application/octet-stream", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE));
  }

View Full Code Here

  }
  
  @Test
  public void testDetectMimeTypesWithDefaultMimeTypes() throws Exception {
    morphline = createMorphline("test-morphlines/detectMimeTypesWithDefaultMimeTypes");    
    Record record = new Record();    
    record.put(Fields.ATTACHMENT_BODY, Files.toByteArray(JPG_FILE));
    startSession();
    morphline.process(record);
    assertEquals("image/jpeg", collector.getFirstRecord().getFirstValue(Fields.ATTACHMENT_MIME_TYPE));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.kitesdk.morphline.api.Record

com.ngdata.hbaseindexer.morphline.LocalMorphlineResultToSolrMapper

com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapperTest

org.apache.flume.sink.solr.morphline.MorphlineHandlerImpl

org.apache.flume.sink.solr.morphline.TestMorphlineSolrSink

org.apache.solr.crunch.MorphlineFn

org.apache.solr.hadoop.morphline.MorphlineMapRunner

org.kitesdk.examples.morphlines.ExampleMorphlineTest

org.kitesdk.morphline.avro.AvroMorphlineTest

org.kitesdk.morphline.avro.ExtractAvroPathsBuilder$ExtractAvroPaths

org.kitesdk.morphline.avro.ExtractAvroTreeBuilder$ExtractAvroTree

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.