Examples of cascading.operation.regex.RegexFilter

cascading.operation.regex.RegexFilter
Class RegexFilter will apply the regex patternString against every input Tuple value and filter the Tuple stream accordingly.
By default, Tuples that match the given pattern are kept, and Tuples that do not match are filtered out. This can be changed by setting removeMatch to true.
Also, by default, the whole Tuple is matched against the given patternString (tab delimited). If matchEachElement is set to true, the pattern is applied to each Tuple value individually.
This operation uses {@link java.util.regex.Matcher} internally, specifically the method {@link java.util.regex.Matcher#find()}. @see java.util.regex.Matcher @see java.util.regex.Pattern

    Pipe pipe = new Pipe( "split" );


    // run job on non-safe operation, forces 3 mr jobs.
    pipe = new Each( pipe, new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );


    pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) );


    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*102.*" ) );


    Map sources = new HashMap();
    sources.put( "split", source );


    Map sinks = new HashMap();

View Full Code Here

    Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache );
    Tap sink = getPlatform().getTextFile( getOutputPath( "splitsourcemerged" ), SinkMode.REPLACE );


    Pipe pipe = new Pipe( "split" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) );


    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*102.*" ) );


    Pipe merged = new GroupBy( "merged", Pipe.pipes( left, right ), new Fields( "line" ) );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, merged );

View Full Code Here


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Fields( "ip" ), new Count(), new Fields( "ip", "count" ) );


    pipe = new Each( pipe, new Fields( "ip" ), new RegexFilter( "^68.*" ) );


    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "ip" ), new RegexFilter( ".*46.*" ) );


    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "ip" ), new RegexFilter( ".*102.*" ) );


    Map sources = Cascades.tapsMap( "split", source );
    Map sinks = Cascades.tapsMap( Pipe.pipes( left, right ), Tap.taps( sink1, sink2 ) );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sinks, left, right );

View Full Code Here

    Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache );
    Tap sink = getPlatform().getTextFile( getOutputPath( "splitsourcemergedsamename" ), SinkMode.REPLACE );


    Pipe pipe = new Pipe( "split" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) );


    Pipe left = new Each( pipe, new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( pipe, new Fields( "line" ), new RegexFilter( ".*102.*" ) );


    Pipe merged = new GroupBy( "merged", Pipe.pipes( left, right ), new Fields( "line" ) );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, merged );

View Full Code Here


    // remove useless fields
    pipe = new Each( pipe, new Fields( "l", "n1", "v1", "n2", "v2" ), new Identity() );


    // remove lines if the names are the same
    pipe = new Each( pipe, new RegexFilter( "^[^\\t]*\\t([^\\t]*)\\t[^\\t]*\\t\\1\\t.*", true ) );


    // transpose values in fields by natural sort order
    pipe = new Each( pipe, new SortElements( new Fields( "n1", "v1" ), new Fields( "n2", "v2" ) ) );


    // unique the pipe

View Full Code Here

    Tap source = getPlatform().getTextFile( new Fields( "offset", "line" ), inputFileApache );


    Pipe pipe = new Each( new Pipe( "first" ), new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );
    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Each( new Pipe( "second", pipe ), new Fields( "ip" ), new RegexFilter( "7" ) );
    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Each( new Pipe( "third", pipe ), new Fields( "ip" ), new RegexFilter( "6" ) );
    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    Tap sinkFirst = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( "chainedtaps/first" ), SinkMode.REPLACE );
    Tap sinkSecond = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( "chainedtaps/second" ), SinkMode.REPLACE );
    Tap sinkThird = getPlatform().getTabDelimitedFile( new Fields( "ip" ), getOutputPath( "chainedtaps/third" ), SinkMode.REPLACE );

View Full Code Here


    Function splitter = new RegexSplitter( new Fields( "num", "char" ), " " );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitter );
    Pipe pipeUpper = new Each( new Pipe( "upper" ), new Fields( "line" ), splitter );
    pipeUpper = new Each( pipeUpper, new Fields( "num" ), new RegexFilter( "^fobar" ) ); // intentionally filtering all
    pipeUpper = new GroupBy( pipeUpper, new Fields( "num" ) );


    Pipe splice = new HashJoin( pipeLower, new Fields( "num" ), pipeUpper, new Fields( "num" ), Fields.size( 4 ), new OuterJoin() );


    Flow flow = getPlatform().getFlowConnector().connect( sources, sink, splice );

View Full Code Here

    Pipe pipe = new Pipe( "split" );


    // this operation is not safe
    pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );


    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );


    Map sources = new HashMap();
    sources.put( "split", source );


    Map sinks = new HashMap();

View Full Code Here

    Pipe pipe = new Pipe( "split" );


    // this operation is not safe
    pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );


    pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) );


    pipe = new Pipe( "middle", pipe );


    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );


    Map sources = new HashMap();
    sources.put( "split", source );


    Map sinks = new HashMap();

View Full Code Here


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Fields( "ip" ), new Count(), new Fields( "ip", "count" ) );


    pipe = new Each( pipe, new Fields( "ip" ), new RegexFilter( "^68.*" ) );


    Pipe left = new Each( new Pipe( "left", pipe ), new Fields( "ip" ), new RegexFilter( ".*46.*" ) );


    Pipe right = new Each( new Pipe( "right", pipe ), new Fields( "ip" ), new RegexFilter( ".*192.*" ) );


    Map sources = new HashMap();
    sources.put( "split", source );


    Map sinks = new HashMap();

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of cascading.operation.regex.RegexFilter

cascading.assembly.CrossTab

cascading.BasicPipesPlatformTest

cascading.CoGroupFieldedPipesPlatformTest

cascading.DistanceUseCasePlatformTest

cascading.FieldedPipesPlatformTest

cascading.flow.hadoop.BuildJobsHadoopPlatformTest

cascading.flow.iso.graph.LoneGroupAssertionGraph

cascading.function.FunctionPlatformTest

cascading.JoinFieldedPipesPlatformTest

cascading.MergePipesPlatformTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.