// specify a regex to split the GIS dump into known fields
Fields fieldDeclaration = new Fields( "blurb", "misc", "geo", "kind" );
String regex = "^\"(.*)\",\"(.*)\",\"(.*)\",\"(.*)\"$";
int[] gisGroups = { 1, 2, 3, 4 };
RegexParser parser = new RegexParser( fieldDeclaration, regex, gisGroups );
Pipe gisPipe = new Each( new Pipe( "gis" ), new Fields( "line" ), parser );
// checkpoint the cleaned-up GIS data
Checkpoint tsvCheck = new Checkpoint( "tsv", gisPipe );
// parse the "park" output
Pipe parkPipe = new Pipe( "park", tsvCheck );
regex = "^\\s+Community Type\\:\\s+Park.*$";
parkPipe = new Each( parkPipe, new Fields( "misc" ), new RegexFilter( regex ) );
// parse the "tree" output
Pipe treePipe = new Pipe( "tree", tsvCheck );
regex = "^\\s+Private\\:\\s+(\\S+)\\s+Tree ID\\:\\s+(\\d+)\\s+.*Situs Number\\:\\s+(\\d+)\\s+Tree Site\\:\\s+(\\d+)\\s+Species\\:\\s+(\\S.*\\S)\\s+Source.*$";
treePipe = new Each( treePipe, new Fields( "misc" ), new RegexFilter( regex ) );
Fields treeFields = new Fields( "priv", "tree_id", "situs", "tree_site", "raw_species" );
int[] treeGroups = { 1, 2, 3, 4, 5 };
parser = new RegexParser( treeFields, regex, treeGroups );
treePipe = new Each( treePipe, new Fields( "misc" ), parser, Fields.ALL );
// scrub "species" as a primary key
regex = "^([\\w\\s]+).*$";
int[] speciesGroups = { 1 };
parser = new RegexParser( new Fields( "scrub_species" ), regex, speciesGroups );
treePipe = new Each( treePipe, new Fields( "raw_species" ), parser, Fields.ALL );
String expression = "scrub_species.trim().toLowerCase()";
ExpressionFunction exprFunc = new ExpressionFunction( new Fields( "tree_species" ), expression, String.class );
treePipe = new Each( treePipe, new Fields( "scrub_species" ), exprFunc, Fields.ALL );
// join with tree metadata
Pipe metaTreePipe = new Pipe( "meta_tree" );
treePipe = new HashJoin( treePipe, new Fields( "tree_species" ), metaTreePipe, new Fields( "species" ), new InnerJoin() );
treePipe = new Rename( treePipe, new Fields( "blurb" ), new Fields( "tree_name" ) );
regex = "^(\\S+),(\\S+),(\\S+)\\s*$";
int[] gpsGroups = { 1, 2, 3 };
parser = new RegexParser( new Fields( "tree_lat", "tree_lng", "tree_alt" ), regex, gpsGroups );
treePipe = new Each( treePipe, new Fields( "geo" ), parser, Fields.ALL );
// determine a tree geohash
Fields geohashArguments = new Fields( "tree_lat", "tree_lng" );
treePipe = new Each( treePipe, geohashArguments, new GeoHashFunction( new Fields( "tree_geohash" ), 6 ), Fields.ALL );
Fields fieldSelector = new Fields( "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_geohash" );
treePipe = new Retain( treePipe, fieldSelector );
// parse the "road" output
Pipe roadPipe = new Pipe( "road", tsvCheck );
regex = "^\\s+Sequence\\:.*\\s+Year Constructed\\:\\s+(\\d+)\\s+Traffic Count\\:\\s+(\\d+)\\s+Traffic Index\\:\\s+(\\w.*\\w)\\s+Traffic Class\\:\\s+(\\w.*\\w)\\s+Traffic Date.*\\s+Paving Length\\:\\s+(\\d+)\\s+Paving Width\\:\\s+(\\d+)\\s+Paving Area\\:\\s+(\\d+)\\s+Surface Type\\:\\s+(\\w.*\\w)\\s+Surface Thickness.*\\s+Bike Lane\\:\\s+(\\w+)\\s+Bus Route\\:\\s+(\\w+)\\s+Truck Route\\:\\s+(\\w+)\\s+Remediation.*$";
roadPipe = new Each( roadPipe, new Fields( "misc" ), new RegexFilter( regex ) );
Fields roadFields = new Fields( "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route" );
int[] roadGroups = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
parser = new RegexParser( roadFields, regex, roadGroups );
roadPipe = new Each( roadPipe, new Fields( "misc" ), parser, Fields.ALL );
// join with road metadata
Pipe metaRoadPipe = new Pipe( "meta_road" );
roadPipe = new HashJoin( roadPipe, new Fields( "surface_type" ), metaRoadPipe, new Fields( "pavement_type" ), new InnerJoin() );