Examples of Hfs

cascading.tap.Hfs
cascading.tap.hadoop.Hfs
ode> will denote Dfs, and file://... will denote Lfs.
Call {@link #setTemporaryDirectory(java.util.Map,String)} to use a different temporary file directory pathother than the current Hadoop default path.
By default Cascading on Hadoop will assume any source or sink Tap using the {@code file://} URI schemeintends to read files from the local client filesystem (for example when using the {@code Lfs} Tap) where the Hadoopjob jar is started, Tap so will force any MapReduce jobs reading or writing to {@code file://} resources to run inHadoop "standalone mode" so that the file can be read.
To change this behavior, {@link HfsProps#setLocalModeScheme(java.util.Map,String)} to set a different scheme value,or to "none" to disable entirely for the case the file to be read is available on every Hadoop processing node in the exact same path.
Hfs can optionally combine multiple small files (or a series of small "blocks") into larger "splits". This reduces the number of resulting map tasks created by Hadoop and can improve application performance.
This is enabled by calling {@link HfsProps#setUseCombinedInput(boolean)} to {@code true}. By default, merging or combining splits into large ones is disabled.

Examples of cascading.tap.hadoop.Hfs

   * We previously would push the each to the next step, but if there is already data being written, save the cpu.
   */
  @Test
  public void testSplitComplex()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );


    Pipe pipe = new Pipe( "split" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs


  /** same as splitComplex, except pipe/branch naming is after the Each, not before */
  @Test
  public void testSplitComplex2()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );


    Pipe pipe = new Pipe( "split" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexParser( new Fields( "ip" ), "^[^ ]*" ), new Fields( "ip" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

    }


  @Test
  public void testMerge()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge1" );
    Tap source2 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge2" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe left = new Each( new Pipe( "left" ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );


    Pipe merge = new GroupBy( "merge", Pipe.pipes( left, right ), new Fields( "offset" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

    }


  @Test
  public void testDupeSource()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );
    Tap source2 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe left = new Each( new Pipe( "left" ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );
    right = new Each( right, new Fields( "line" ), new RegexFilter( ".*192.*" ) );
    right = new Each( right, new Fields( "line" ), new RegexFilter( ".*192.*" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

    }


  @Test
  public void testDupeSourceRepeat()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe pipe = new Pipe( "pipe" );


    Pipe merge = new CoGroup( "cogroup", pipe, new Fields( "offset" ), 1, Fields.size( 4 ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

    }


  @Test
  public void testDupeSource2()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe left = new Pipe( "left" );
    Pipe right = new Pipe( "right" );


    Pipe merge = new CoGroup( "cogroup", left, new Fields( "offset" ), right, new Fields( "offset" ), Fields.size( 4 ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

    }


  @Test
  public void testDupeSource3()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge" );
    Tap source2 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "bar/merge" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe left = new Pipe( "left" );
    Pipe middle = new Pipe( "middle" );
    Pipe right = new Pipe( "right" );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

//    }


  @Test
  public void testMerge2()
    {
    Tap source1 = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge1" );
    Tap source2 = new Hfs( new SequenceFile( new Fields( "offset", "line" ) ), "foo/merge2" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe left = new Each( new Pipe( "left" ), new Fields( "line" ), new RegexFilter( ".*46.*" ) );
    Pipe right = new Each( new Pipe( "right" ), new Fields( "line" ), new RegexFilter( ".*192.*" ) );


    Pipe merge = new GroupBy( "merge", Pipe.pipes( left, right ), new Fields( "offset" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

   * Tests the case where the same source is split, then re-merged
   */
  @Test
  public void testMergeSameSourceSplit()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo/merge1" );


    Tap sink = new Hfs( new TextLine(), "foo" );


    Pipe head = new Pipe( "source" );


    head = new Each( head, new Fields( "line" ), new ExpressionFilter( "line.length() != 0", String.class ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

    }


  @Test
  public void testCoGroupAroundCoGroup() throws Exception
    {
    Tap source10 = new Hfs( new TextLine( new Fields( "num" ) ), "foo" );
    Tap source20 = new Hfs( new TextLine( new Fields( "num" ) ), "bar" );


    Map sources = new HashMap();


    sources.put( "source20", source20 );
    sources.put( "source101", source10 );
    sources.put( "source102", source10 );


    // using null pos so all fields are written
    Tap sink = new Hfs( new TextLine(), "baz", true );


    Pipe pipeNum20 = new Pipe( "source20" );
    Pipe pipeNum101 = new Pipe( "source101" );
    Pipe pipeNum102 = new Pipe( "source102" );

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.