Examples of Each

cascading.pipe.Each
The Each operator applies either a {@link Function} or a {@link Filter} to each entry in the {@link Tuple}stream. Any number of Each operators can follow an Each, {@link Splice}, or {@link Every}operator.
cn.bran.japid.tags.Each
a built-in tag for looping, with helping attributes such as, index odd/even line, isFirst, isLast, etc. Since we know this tag does not have a layout, the super's layout->doLayout pathway is short-cut in the render(). @author Bing Ran
org.thymeleaf.standard.expression.Each
@author Daniel Fernández @since 1.1
org.waveprotocol.wave.client.gadget.StateMap.Each

Examples of cascading.pipe.Each

    // declares: "time", "method", "event", "status", "size"
    Fields apacheFields = new Fields( "ip", "time", "method", "event", "status", "size" );
    String apacheRegex = "^([^ ]*) +[^ ]* +[^ ]* +\\[([^]]*)\\] +\\\"([^ ]*) ([^ ]*) [^ ]*\\\" ([^ ]*) ([^ ]*).*$";
    int[] apacheGroups = {1, 2, 3, 4, 5, 6};
    RegexParser parser = new RegexParser( apacheFields, apacheRegex, apacheGroups );
    Pipe importPipe = new Each( "import", new Fields( "line" ), parser );


    // create tap to read a resource from the local file system, if not an url for an external resource
    // Lfs allows for relative paths
    Tap logTap =
      inputPath.matches( "^[^:]+://.*" ) ? new Hfs( new TextLine(), inputPath ) : new Lfs( new TextLine(), inputPath );
    // create a tap to read/write from the default filesystem
    Tap parsedLogTap = new Hfs( apacheFields, logsPath );


    // connect the assembly to source and sink taps
    Flow importLogFlow = flowConnector.connect( logTap, parsedLogTap, importPipe );


    // create an assembly to parse out the time field into a timestamp
    // then count the number of requests per second and per minute


    // apply a text parser to create a timestamp with 'second' granularity
    // declares field "ts"
    DateParser dateParser = new DateParser( new Fields( "ts" ), "dd/MMM/yyyy:HH:mm:ss Z" );
    Pipe tsPipe = new Each( "arrival rate", new Fields( "time" ), dateParser, Fields.RESULTS );


    // name the per second assembly and split on tsPipe
    Pipe tsCountPipe = new Pipe( "tsCount", tsPipe );
    tsCountPipe = new GroupBy( tsCountPipe, new Fields( "ts" ) );
    tsCountPipe = new Every( tsCountPipe, Fields.GROUP, new Count() );


    // apply expression to create a timestamp with 'minute' granularity
    // declares field "tm"
    Pipe tmPipe = new Each( tsPipe, new ExpressionFunction( new Fields( "tm" ), "ts - (ts % (60 * 1000))", long.class ) );


    // name the per minute assembly and split on tmPipe
    Pipe tmCountPipe = new Pipe( "tmCount", tmPipe );
    tmCountPipe = new GroupBy( tmCountPipe, new Fields( "tm" ) );
    tmCountPipe = new Every( tmCountPipe, Fields.GROUP, new Count() );

View Full Code Here

Examples of cascading.pipe.Each


    // wordCountPipe will be recognized as an assembly and handled appropriately
    Flow count = flowConnector.connect( importedPages, sinks, wordCountPipe );


    // create an assembly to export the Hadoop sequence file to local text files
    Pipe exportPipe = new Each( "export pipe", new Identity() );


    Tap localSinkUrl = new Lfs( new TextLine(), localUrlsPath );
    Tap localSinkWord = new Lfs( new TextLine(), localWordsPath );


    // connect up both sinks using the same exportPipe assembly

View Full Code Here

Examples of cascading.pipe.Each

    {
    public ImportCrawlDataAssembly( String name )
      {
      // split the text line into "url" and "raw" with the default delimiter of tab
      RegexSplitter regexSplitter = new RegexSplitter( new Fields( "url", "raw" ) );
      Pipe importPipe = new Each( name, new Fields( "line" ), regexSplitter );
      // remove all pdf documents from the stream
      importPipe = new Each( importPipe, new Fields( "url" ), new RegexFilter( ".*\\.pdf$", true ) );
      // replace ":nl" with a new line, return the fields "url" and "page" to the stream.
      // discared the other fields in the stream
      RegexReplace regexReplace = new RegexReplace( new Fields( "page" ), ":nl:", "\n" );
      importPipe = new Each( importPipe, new Fields( "raw" ), regexReplace, new Fields( "url", "page" ) );


      setTails( importPipe );
      }

View Full Code Here

Examples of cascading.pipe.Each

      {
      // create a new pipe assembly to create the word count across all the pages, and the word count in a single page
      Pipe pipe = new Pipe( sourceName );


      // convert the html to xhtml using the TagSouParser. return only the fields "url" and "xml", discard the rest
      pipe = new Each( pipe, new Fields( "page" ), new TagSoupParser( new Fields( "xml" ) ), new Fields( "url", "xml" ) );
      // apply the given XPath expression to the xml in the "xml" field. this expression extracts the 'body' element.
      XPathGenerator bodyExtractor = new XPathGenerator( new Fields( "body" ), XPathOperation.NAMESPACE_XHTML, "//xhtml:body" );
      pipe = new Each( pipe, new Fields( "xml" ), bodyExtractor, new Fields( "url", "body" ) );
      // apply another XPath expression. this expression removes all elements from the xml, leaving only text nodes.
      // text nodes in a 'script' element are removed.
      String elementXPath = "//text()[ name(parent::node()) != 'script']";
      XPathGenerator elementRemover = new XPathGenerator( new Fields( "words" ), XPathOperation.NAMESPACE_XHTML, elementXPath );
      pipe = new Each( pipe, new Fields( "body" ), elementRemover, new Fields( "url", "words" ) );
      // apply the regex to break the document into individual words and stuff each word at a new tuple into the current
      // stream with field names "url" and "word"
      RegexGenerator wordGenerator = new RegexGenerator( new Fields( "word" ), "(?<!\\pL)(?=\\pL)[^ ]*(?<=\\pL)(?!\\pL)" );
      pipe = new Each( pipe, new Fields( "words" ), wordGenerator, new Fields( "url", "word" ) );


      // group on "url"
      Pipe urlCountPipe = new GroupBy( sinkUrlName, pipe, new Fields( "url", "word" ) );
      urlCountPipe = new Every( urlCountPipe, new Fields( "url", "word" ), new Count(), new Fields( "url", "word", "count" ) );

View Full Code Here

Examples of cascading.pipe.Each

    RegexParser parser = new RegexParser( apacheFields, apacheRegex, allGroups );


    // create the import pipe element, with the name 'import', and with the input argument named "line"
    // replace the incoming tuple with the parser results
    // "line" -> parser -> "ts"
    Pipe importPipe = new Each( "import", new Fields( "line" ), parser, Fields.RESULTS );


    // create a SINK tap to write to the default filesystem
    // by default, TextLine writes all fields out
    Tap remoteLogTap = new Hfs( new TextLine(), outputPath, SinkMode.REPLACE );

View Full Code Here

Examples of cascading.pipe.Each


  protected void init(Pipe[] pipes, Fields[] groupFields, int pipeFieldsSum, Fields groupingRename,
      MultiBuffer operation) {
    for (int i = 0; i < pipes.length; i++) {
      pipes[i] = new Pipe(UUID.randomUUID().toString(), pipes[i]);
      pipes[i] = new Each(pipes[i], Fields.ALL, new Identity(), Fields.RESULTS);
    }
    Fields resultFields =
        Fields.join(groupingRename, ((BaseOperation) operation).getFieldDeclaration());
    if (resultFields.size() > pipeFieldsSum) {
      throw new IllegalArgumentException("Can't have output more than sum of input pipes since this is a hack!");
    }
    // unfortunately, need to hack around CoGroup validation stuff since cascading assumes it will return #fields=sum of input pipes
    Fields fake = new Fields();
    fake = fake.append(resultFields);
    int i = 0;
    while (fake.size() < pipeFieldsSum) {
      fake = fake.append(new Fields("__" + i));
      i++;
    }
    Pipe result =
        new CoGroup(pipes, groupFields, fake, new MultiGroupJoiner(pipeFieldsSum, operation));
    result = new Each(result, resultFields, new Identity());
    setTails(result);
  }

View Full Code Here

Examples of cascading.pipe.Each


        DomainSpec spec = outTap.getSpec();
        LOG.info("Instantiating spec: " + spec);


        // Add the shard index as field #2.
        Pipe out = new Each(keyValuePairs, new Fields(0), new Shardize(shardField, spec), Fields.ALL);


        // Add the serialized key itself as field #3 for sorting.
        // TODO: Make secondary sorting optional, and come up with a function to generate
        // a sortable key (vs just using the same serialization as for sharding).
        out = new Each(out, new Fields(0), new MakeSortableKey(keySortField, spec), Fields.ALL);


        //put in order of shard, key, value, sortablekey
        out = new Each(out, new Fields(2, 0, 1, 3), new Identity(), Fields.RESULTS);
        out = new GroupBy(out, new Fields(0), new Fields(3)); // group by shard


        // emit shard, key, value
        out = new Each(out, new Fields(0, 1, 2), new Identity());
        setTails(out);
    }

View Full Code Here

Examples of cn.bran.japid.tags.Each

    p("        <h3>\n" + 
"          Routes derived from JAX-RS annotations:\n" + 
"          </h3>\n" + 
"            <div>\n" + 
"          ");// line 122
    final Each _Each0 = new Each(getOut()); _Each0.setOut(getOut()); _Each0.render(// line 127
jaxRoutes, new Each.DoBody<RouteEntry>(){ // line 127
public void render(final RouteEntry r, final int _size, final int _index, final boolean _isOdd, final String _parity, final boolean _isFirst, final boolean _isLast) { // line 127
// line 127
    p("            <pre><span class=\"line\">");// line 127
    p(_index);// line 128
    p("</span><span class=\"route\"><span class=\"verb\">");// line 128
    p(r.verb);// line 128
    p("</span><span class=\"path\">");// line 128
    p(r.path);// line 128
    p("</span><span class=\"call\">");// line 128
    p(r.action);// line 128
    p("</span></span></pre>\n" + 
"          ");// line 128
    
}


StringBuilder oriBuffer;
@Override
public void setBuffer(StringBuilder sb) {
  oriBuffer = getOut();
  setOut(sb);
}


@Override
public void resetBuffer() {
  setOut(oriBuffer);
}


}
);// line 127
    p("      </div>\n" + 
"    ");// line 129
    } else {// line 131
    p("      <h3>\n" + 
"          No routes derived from JAX-RS annotations found.\n" + 
"        </h3>\n" + 
"        ");// line 131
    }// line 135
    if(asBoolean(routes)) {// line 137
    p("        <h3>\n" + 
"          Routes defined in routes file:\n" + 
"          </h3>\n" + 
"            <div>\n" + 
"          ");// line 137
    final Each _Each1 = new Each(getOut()); _Each1.setOut(getOut()); _Each1.render(// line 142
routes, new Each.DoBody<scala.Tuple3>(){ // line 142
public void render(final scala.Tuple3 r, final int _size, final int _index, final boolean _isOdd, final String _parity, final boolean _isFirst, final boolean _isLast) { // line 142
// line 142
    p("            <pre><span class=\"line\">");// line 142
    p(_index);// line 143

View Full Code Here

Examples of org.thymeleaf.standard.expression.Each


        final String attributeValue = element.getAttributeValue(attributeName);


        final Configuration configuration = arguments.getConfiguration();


        final Each each = EachUtils.parseEach(configuration, arguments, attributeValue);


        final IStandardExpression iterVarExpr = each.getIterVar();
        final Object iterVarValue = iterVarExpr.execute(configuration, arguments);


        final IStandardExpression statusVarExpr = each.getStatusVar();
        final Object statusVarValue;
        if (statusVarExpr != null) {
            statusVarValue = statusVarExpr.execute(configuration, arguments);
        } else {
            statusVarValue = null;
        }


        final IStandardExpression iterableExpr = each.getIterable();
        final Object iteratedValue = iterableExpr.execute(configuration, arguments);


        final String iterVarName = (iterVarValue == null? null : iterVarValue.toString());
        if (StringUtils.isEmptyOrWhitespace(iterVarName)) {
            throw new TemplateProcessingException(

View Full Code Here

Examples of org.waveprotocol.wave.client.gadget.StateMap.Each

    // callback.
    // TODO(user): Remove this workaround once this is fixed in GGS.
    ScheduleCommand.addCommand(new Task() {
      @Override
      public void execute() {
        deltaState.each(new Each() {
          @Override
          public void apply(final String key, final String value) {
            if (value != null) {
              modifyState(key, value);
            } else {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.