Package com.m6d.filecrush.crush

Examples of com.m6d.filecrush.crush.Crush

706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
  @Test
  public void executeBackwardsCompatibleText() throws Exception {
    writeFiles(false, true, false);

    Crush crush = new Crush();

    ToolRunner.run(job, crush, new String [] {
      "in", "out", "2", "TEXT"
    });


    /*
     * Crushed files.
     */
    verifyOutput(homeDir + "/in/dir",  "crushed_file-*-*-*", Format.TEXT, Format.TEXT, defaultCodec, "file10", "file11", "file12", "file13");
    verifyOutput(homeDir + "/in/text", "crushed_file-*-*-*", Format.TEXT, Format.TEXT, defaultCodec, "file60", "file61", "file62", "file63");


    /*
     * Crush input files should be moved to the clone dir.
     */
    for (String file : new String[] { "file10", "file11", "file12", "file13" }) {
      verifyOutput(homeDir + "/out" + homeDir + "/in/dir", file, Format.TEXT, Format.TEXT, null, file);
    }

    for (String file : new String[] { "file60", "file61", "file62", "file63" }) {
      verifyOutput(homeDir + "/out" + homeDir + "/in/text", file, Format.TEXT, Format.TEXT, null, file);
    }


    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND),      equalTo(3L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE),    equalTo(2L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED),    equalTo(1L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND),      equalTo(8L));
View Full Code Here
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
  @Test
  public void executeBackwardsCompatibleSequence() throws Exception {
    writeFiles(true, false, false);

    Crush crush = new Crush();

    ToolRunner.run(job, crush, new String [] {
      "in", "out", "2"
    });


    /*
     * Crushed files.
     */
    verifyOutput(homeDir + "/in/dir/subdir",            "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE, defaultCodec, "file20", "file21", "file22", "file23", "file24");
    verifyOutput(homeDir + "/in/dir/subdir/subsubdir",  "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE, defaultCodec, "file30", "file31", "file32", "file33", "file34");
    verifyOutput(homeDir + "/in/dir/subdir/other",      "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE, defaultCodec, "file40", "file41", "file42", "file43");
    verifyOutput(homeDir + "/in/dir/other",              "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE, defaultCodec, "file50", "file51", "file52", "file53", "file54", "file55");
    verifyOutput(homeDir + "/in",                        "crushed_file-*-*-*", Format.SEQUENCE, Format.SEQUENCE, defaultCodec, "file70", "file71", "file72");


    /*
     * Skipped files should be copied to the output.
     */
    verifyOutput(homeDir + "/in/dir/skipped", "file80", Format.SEQUENCE, Format.SEQUENCE, defaultCodec, "file80");


    /*
     * Crush input files should be moved to the clone dir.
     */
    for (String file : new String[] { "file20", "file21", "file22", "file23", "file24" }) {
      verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file30", "file31", "file32", "file33", "file34" }) {
      verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir/subsubdir", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file40", "file41", "file42", "file43" }) {
      verifyOutput(homeDir + "/out" + homeDir + "/in/dir/subdir/other", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file50", "file51", "file52", "file53", "file54", "file55" }) {
      verifyOutput(homeDir + "/out" + homeDir + "/in/dir/other", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec, file);
    }

    for (String file : new String[] { "file70", "file71", "file72" }) {
      verifyOutput(homeDir + "/out" + homeDir + "/in", file, Format.SEQUENCE, Format.SEQUENCE, defaultCodec, file);
    }

    Counters jobCounters = crush.getJobCounters();

    assertThat(jobCounters.getCounter(MapperCounter.DIRS_FOUND),      equalTo( 7L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_ELIGIBLE),    equalTo( 5L));
    assertThat(jobCounters.getCounter(MapperCounter.DIRS_SKIPPED),    equalTo( 2L));
    assertThat(jobCounters.getCounter(MapperCounter.FILES_FOUND),      equalTo(24L));
View Full Code Here
821
822
823
824
825
826
827
828
829
830
831
    // Ones to skip
    writeFile("in_skip_test/file90", Format.TEXT);
    writeFile("in_skip_test/file91", Format.TEXT);
    writeFile("in_skip_test/file92", Format.TEXT);

    Crush crush = new Crush();

    ToolRunner.run(job, crush, new String [] {
      "--threshold=0.015",
      "--max-file-blocks=1",
      "--verbose",
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
  private Crush crush;

  @Before
  public void before() throws IOException {
    crush = new Crush();

    JobConf job = new JobConf(false);
    crush.setConf(job);

    job.set("fs.default.name", "file:///");
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
    createFile(subsubdir, "skipped-4", 4, 25);
    createFile(subsubdir, "skipped-5", 5, 25);

    File out = new File(tmp.getRoot(), "out");

    ToolRunner.run(job, new Crush(), new String[] {
        "--input-format=text",
        "--output-format=text",
        "--compress=none",

        subdir.getAbsolutePath(), out.getAbsolutePath()
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
  public void noFiles() throws Exception {
    File in = tmp.newFolder("in");

    File out = new File(tmp.getRoot(), "out");

    ToolRunner.run(job, new Crush(), new String[] {
        in.getAbsolutePath(), out.getAbsolutePath()
    });

    assertThat(out.exists(), is(false));
  }
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
    createFile(in, "big-2.index", 2, 50);
    createFile(in, "big-3.index", 3, 50);

    File out = new File(tmp.getRoot(), "skip_test_out");

    ToolRunner.run(job, new Crush(), new String[] {
        "--input-format=text",
        "--output-format=text",
        "--ignore-regex=.*\\.index",
        "--compress=none",

 
View Full Code Here
TOP

Related Classes of com.m6d.filecrush.crush.Crush

  • com.m6d.filecrush.crush.CrushOptionParsingTest
  • com.m6d.filecrush.crush.CrushStandAloneSequenceFileTest
  • com.m6d.filecrush.crush.CrushStandAloneTextTest
  • com.m6d.filecrush.crush.CrushTest
  • com.m6d.filecrush.crush.integration.CrushMapReduceTest
  • org.apache.commons.cli.CommandLine
  • org.apache.commons.cli.GnuParser
  • org.apache.commons.cli.Option
  • org.apache.commons.cli.Options
  • org.apache.hadoop.fs.FileStatus
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.