Examples of cascading.pipe.Checkpoint

cascading.pipe.Checkpoint
The Checkpoint pipe, if supported by the current planner, will force data to be persisted at the point in the tuple stream an instance of Checkpoint is inserted into the pipe assembly.
If a checkpoint {@link cascading.tap.Tap} is added to the {@link cascading.flow.FlowDef} via the{@link cascading.flow.FlowDef#addCheckpoint(Checkpoint,cascading.tap.Tap)} method, that Tap instancewill be used to capture the intermediate result sets.
It is required that any Scheme used as a checkpoint must source {@link cascading.tuple.Fields#UNKNOWN} andsink {@link cascading.tuple.Fields#ALL}.
If used with a {@link cascading.scheme.hadoop.TextDelimited} {@link cascading.scheme.Scheme} class andthe {@code hasHeader} value is {@code true}, a header with the resolved field names will be written to the file.
This is especially useful for debugging complex flows.
For the {@link cascading.flow.hadoop.HadoopFlowConnector} and Hadoop platform, a Checkpoint will force a newMapReduce job ( {@link cascading.flow.hadoop.HadoopFlowStep} into the {@link cascading.flow.Flow} plan.
This can be important when used in conjunction with a {@link HashJoin} where all the operations upstreamfrom the HashJoin significantly filter out data allowing it to fit in memory.

    int[] gisGroups = { 1, 2, 3, 4 };
    RegexParser parser = new RegexParser( fieldDeclaration, regex, gisGroups );
    Pipe gisPipe = new Each( new Pipe( "gis" ), new Fields( "line" ), parser );


    // checkpoint the cleaned-up GIS data
    Checkpoint tsvCheck = new Checkpoint( "tsv", gisPipe );


    // parse the "park" output
    Pipe parkPipe = new Pipe( "park", tsvCheck );
    regex = "^\\s+Community Type\\:\\s+Park.*$";
    parkPipe = new Each( parkPipe, new Fields( "misc" ), new RegexFilter( regex ) );


    // parse the "tree" output
    Pipe treePipe = new Pipe( "tree", tsvCheck );
    regex = "^\\s+Private\\:\\s+(\\S+)\\s+Tree ID\\:\\s+(\\d+)\\s+.*Situs Number\\:\\s+(\\d+)\\s+Tree Site\\:\\s+(\\d+)\\s+Species\\:\\s+(\\S.*\\S)\\s+Source.*$";
    treePipe = new Each( treePipe, new Fields( "misc" ), new RegexFilter( regex ) );


    Fields treeFields = new Fields( "priv", "tree_id", "situs", "tree_site", "raw_species" );
    int[] treeGroups = { 1, 2, 3, 4, 5 };
    parser = new RegexParser( treeFields, regex, treeGroups );
    treePipe = new Each( treePipe, new Fields( "misc" ), parser, Fields.ALL );


    // scrub "species" as a primary key
    regex = "^([\\w\\s]+).*$";
    int[] speciesGroups = { 1 };
    parser = new RegexParser( new Fields( "scrub_species" ), regex, speciesGroups );
    treePipe = new Each( treePipe, new Fields( "raw_species" ), parser, Fields.ALL );
    String expression = "scrub_species.trim().toLowerCase()";
    ExpressionFunction exprFunc = new ExpressionFunction( new Fields( "tree_species" ), expression, String.class );
    treePipe = new Each( treePipe, new Fields( "scrub_species" ), exprFunc, Fields.ALL );


    // join with tree metadata
    Pipe metaTreePipe = new Pipe( "meta_tree" );
    treePipe = new HashJoin( treePipe, new Fields( "tree_species" ), metaTreePipe, new Fields( "species" ), new InnerJoin() );
    treePipe = new Rename( treePipe, new Fields( "blurb" ), new Fields( "tree_name" ) );


    regex = "^(\\S+),(\\S+),(\\S+)\\s*$";
    int[] gpsGroups = { 1, 2, 3 };
    parser = new RegexParser( new Fields( "tree_lat", "tree_lng", "tree_alt" ), regex, gpsGroups );
    treePipe = new Each( treePipe, new Fields( "geo" ), parser, Fields.ALL );


    // determine a tree geohash
    Fields geohashArguments = new Fields( "tree_lat", "tree_lng" );
    treePipe = new Each( treePipe, geohashArguments, new GeoHashFunction( new Fields( "tree_geohash" ), 6 ), Fields.ALL );


    Fields fieldSelector = new Fields( "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_geohash" );
    treePipe = new Retain( treePipe, fieldSelector );


    // parse the "road" output
    Pipe roadPipe = new Pipe( "road", tsvCheck );
    regex = "^\\s+Sequence\\:.*\\s+Year Constructed\\:\\s+(\\d+)\\s+Traffic Count\\:\\s+(\\d+)\\s+Traffic Index\\:\\s+(\\w.*\\w)\\s+Traffic Class\\:\\s+(\\w.*\\w)\\s+Traffic Date.*\\s+Paving Length\\:\\s+(\\d+)\\s+Paving Width\\:\\s+(\\d+)\\s+Paving Area\\:\\s+(\\d+)\\s+Surface Type\\:\\s+(\\w.*\\w)\\s+Surface Thickness.*\\s+Bike Lane\\:\\s+(\\w+)\\s+Bus Route\\:\\s+(\\w+)\\s+Truck Route\\:\\s+(\\w+)\\s+Remediation.*$";
    roadPipe = new Each( roadPipe, new Fields( "misc" ), new RegexFilter( regex ) );
    Fields roadFields = new Fields( "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route" );
    int[] roadGroups = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
    parser = new RegexParser( roadFields, regex, roadGroups );
    roadPipe = new Each( roadPipe, new Fields( "misc" ), parser, Fields.ALL );


    // join with road metadata
    Pipe metaRoadPipe = new Pipe( "meta_road" );
    roadPipe = new HashJoin( roadPipe, new Fields( "surface_type" ), metaRoadPipe, new Fields( "pavement_type" ), new InnerJoin() );
    roadPipe = new Rename( roadPipe, new Fields( "blurb" ), new Fields( "road_name" ) );


    // estimate albedo based on the road surface age and pavement type
    Fields albedoArguments = new Fields( "year_construct", "albedo_new", "albedo_worn" );
    roadPipe = new Each( roadPipe, albedoArguments, new AlbedoFunction( new Fields( "albedo" ), 2002 ), Fields.ALL );


    // generate road segments, with midpoint, y=mx+b, and road_geohash for each
    Fields segmentArguments = new Fields( "geo" );
    Fields segmentResults = new Fields( "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "lat_mid", "lng_mid" );
    roadPipe = new Each( roadPipe, segmentArguments, new RoadSegmentFunction( segmentResults ), Fields.ALL );


    geohashArguments = new Fields( "lat_mid", "lng_mid" );
    roadPipe = new Each( roadPipe, geohashArguments, new GeoHashFunction( new Fields( "road_geohash" ), 6 ), Fields.ALL );


    fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "road_geohash" );
    roadPipe = new Retain( roadPipe, fieldSelector );


    // join the tree and road pipes to estimate shade
    Pipe shadePipe = new Pipe( "shade", roadPipe );
    shadePipe = new CoGroup( shadePipe, new Fields( "road_geohash" ), treePipe, new Fields( "tree_geohash" ), new InnerJoin() );


    // calculate a rough estimate for distance from tree to road, then filter for "< ~1 block"
    Fields treeDistArguments = new Fields( "tree_lat", "tree_lng", "lat0", "lng0", "lat1", "lng1" );
    Fields tree_dist = new Fields( "tree_dist" );
    shadePipe = new Each( shadePipe, treeDistArguments, new TreeDistanceFunction( tree_dist ), Fields.ALL );


    ExpressionFilter distFilter = new ExpressionFilter( "tree_dist > 25.0", Double.class );
    shadePipe = new Each( shadePipe, tree_dist, distFilter );


    // checkpoint this (big) calculation too
    fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "lat1", "lng1", "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_dist", "tree_geohash" );
    shadePipe = new Retain( shadePipe, fieldSelector );
    shadePipe = new GroupBy( shadePipe, new Fields( "tree_name" ), new Fields( "tree_dist" ) );


    Checkpoint shadeCheck = new Checkpoint( "shade", shadePipe );


    // determine the geohash for GPS tracks log events
    Pipe logsPipe = new Pipe( "logs" );
    geohashArguments = new Fields( "lat", "lng" );
    logsPipe = new Each( logsPipe, geohashArguments, new GeoHashFunction( new Fields( "gps_geohash" ), 6 ), Fields.ALL );

View Full Code Here

    {
    Pipe pipe = new Pipe( "third" );


    pipe = new Each( pipe, new FieldJoiner( new Fields( "mangled" ), "-" ) );


    pipe = new Checkpoint( "checkpoint", pipe );


    pipe = new Each( pipe, new Identity() );


    Tap sink = getPlatform().getTabDelimitedFile( new Fields( "mangled" ), getOutputPath( "unusedpath" ), SinkMode.REPLACE );
    Tap checkpoint = getPlatform().getTabDelimitedFile( Fields.ALL, getOutputPath( path ), SinkMode.REPLACE );

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new Checkpoint( pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );

View Full Code Here

    Pipe pipe = new Pipe( "test" );


    { // job 1
    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new Checkpoint( pipe );
    }


    { // job 2
    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    pipe = new Checkpoint( pipe );


    new FlowConnectorProps()
      .setCheckpointTapDecoratorClassName( DecoratorTap.class )
      .setProperties( pipe.getConfigDef() );
    }


    { // job 3
    pipe = new Each( pipe, new Identity() );


    pipe = new Checkpoint( pipe ); // this should be collapsed into the sink tap, not be a fourth job
    }


    Tap sink = getPlatform().getTextFile( getOutputPath( "manycheckpoint" ), SinkMode.REPLACE );


    Flow flow = getPlatform().getFlowConnector().connect( source, sink, pipe );

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new Checkpoint( "checkpoint", pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new Checkpoint( "checkpoint", pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );

View Full Code Here


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Checkpoint( "checkpoint", pipe );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    Tap sink = getPlatform().getTextFile( getOutputPath( "failcheckpointevery/sink" ), SinkMode.REPLACE );

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new Checkpoint( "checkpoint", pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new Checkpoint( "checkpoint", pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );


    pipe = new Checkpoint( "checkpoint", pipe );


    Tap sink = getPlatform().getTextFile( getOutputPath( "duplicatecheckpoint" ), SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()
      .setName( "restartable" )

View Full Code Here


    Pipe pipe = new Pipe( "test" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );


    pipe = new Checkpoint( pipe );


    pipe = new GroupBy( pipe, new Fields( "ip" ) );


    pipe = new Every( pipe, new Count(), new Fields( "ip", "count" ) );

View Full Code Here

0 1

TOP

Related Classes of cascading.pipe.Checkpoint

cascading.cascade.CascadePlatformTest

cascading.flow.iso.graph.MulitStepFlowGraph

cascading.pipe.checkpoint.CheckpointPlatformTest

cascading.tap.hadoop.DistCacheTapPlatformTest

copa.Main

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.