Examples of Hfs

cascading.tap.Hfs
cascading.tap.hadoop.Hfs
ode> will denote Dfs, and file://... will denote Lfs.
Call {@link #setTemporaryDirectory(java.util.Map,String)} to use a different temporary file directory pathother than the current Hadoop default path.
By default Cascading on Hadoop will assume any source or sink Tap using the {@code file://} URI schemeintends to read files from the local client filesystem (for example when using the {@code Lfs} Tap) where the Hadoopjob jar is started, Tap so will force any MapReduce jobs reading or writing to {@code file://} resources to run inHadoop "standalone mode" so that the file can be read.
To change this behavior, {@link HfsProps#setLocalModeScheme(java.util.Map,String)} to set a different scheme value,or to "none" to disable entirely for the case the file to be read is available on every Hadoop processing node in the exact same path.
Hfs can optionally combine multiple small files (or a series of small "blocks") into larger "splits". This reduces the number of resulting map tasks created by Hadoop and can improve application performance.
This is enabled by calling {@link HfsProps#setUseCombinedInput(boolean)} to {@code true}. By default, merging or combining splits into large ones is disabled.

Examples of cascading.tap.hadoop.Hfs

  public void testOneJob2() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new Each( pipe, new Fields( 1 ), new Identity(), new Fields( 2 ) ); // in:second out:all
    pipe = new Each( pipe, new Fields( 0 ), new Identity( new Fields( "_all" ) ), new Fields( 1 ) ); // in:all out:_all
    pipe = new GroupBy( pipe, new Fields( 0 ) ); // in:_all out:_all

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

  public void testOneJob3() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "a", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" ) );
    sources.put( "b", new Hfs( new TextLine( new Fields( "third", "fourth" ) ), "input/path/b" ) );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe splice = new CoGroup( pipeA, new Fields( 1 ), pipeB, new Fields( 1 ) );


    sinks.put( splice.getName(), new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, splice ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

  public void testOneJob4() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "a", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" ) );
    sources.put( "b", new Hfs( new TextLine( new Fields( "third", "fourth" ) ), "input/path/b" ) );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe cogroup = new CoGroup( pipeA, new Fields( 1 ), pipeB, new Fields( 1 ) );


    cogroup = new Each( cogroup, new Identity() );


    sinks.put( cogroup.getName(), new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, cogroup ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

  public void testOneJob5() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "a", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path/a" ) );
    sources.put( "b", new Hfs( new TextLine( new Fields( "third", "fourth" ) ), "input/path/b" ) );


    Pipe pipeA = new Pipe( "a" );
    Pipe pipeB = new Pipe( "b" );


    Pipe splice = new CoGroup( pipeA, pipeB );


    splice = new Each( splice, new Identity() );


    sinks.put( splice.getName(), new Hfs( new TextLine(), "output/path" ) );


    List steps = getPlatform().getFlowConnector().connect( sources, sinks, splice ).getFlowSteps();


    assertEquals( "wrong size", 1, steps.size() );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

  public void testNoGroup() throws IOException
    {
    Map sources = new HashMap();
    Map sinks = new HashMap();


    sources.put( "count", new Hfs( new TextLine( new Fields( "first", "second" ) ), "input/path" ) );
    sinks.put( "count", new Hfs( new TextLine( new Fields( 0, 1 ) ), "output/path" ) );


    Pipe pipe = new Pipe( "count" );
    pipe = new Each( pipe, new Identity() );
    pipe = new Every( pipe, new Fields( 1 ), new Count(), new Fields( 0, 1 ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs


  /** This should result in only two steps, one for each side */
  @Test
  public void testSplit()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );


    Pipe pipe = new Pipe( "split" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs


  /** this test verifies that the planner recognizes there are fewer tails than sinks. */
  @Test
  public void testSplitHangingTails()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );


    Pipe pipe = new Pipe( "split" );


    pipe = new Each( pipe, new Fields( "line" ), new RegexFilter( "^68.*" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

    }


  @Test
  public void testSplitOnNonSafeOperations()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );


    Pipe pipe = new Pipe( "split" );


    // this operation is not safe
    pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs

    }


  @Test
  public void testSplitOnNonSafeOperationsSimple()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );


    Pipe pipe = new Pipe( "split" );


    // this operation is not safe
    pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );

View Full Code Here

Examples of cascading.tap.hadoop.Hfs


  // verify unsafe splits happen when splitting on a pipe
  @Test
  public void testSplitOnNonSafeOperations2()
    {
    Tap source = new Hfs( new TextLine( new Fields( "offset", "line" ) ), "foo" );
    Tap sink1 = new Hfs( new TextLine(), "foo/split1", true );
    Tap sink2 = new Hfs( new TextLine(), "foo/split2", true );
    Tap sink3 = new Hfs( new TextLine(), "foo/split3", true );


    Pipe pipe = new Pipe( "split" );


    // this operation is not safe
    pipe = new Each( pipe, new Fields( "line" ), new TestFunction( new Fields( "ignore" ), new Tuple( 1 ), false ), new Fields( "line" ) );

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.