Package com.scaleunlimited.cascading

Examples of com.scaleunlimited.cascading.BasePath

54
55
56
57
58
59
60
61
62
63
64
65
66
        BixoPlatform platform = new BixoPlatform(ParsePipeTest.class, Platform.Local);
       

        Pipe pipe = new Pipe("parse_source");
        ParsePipe parserPipe = new ParsePipe(pipe, new SimpleParser());
        BasePath inputPath = platform.makePath("build/test/ParserPipeTest/in");
        Tap in = platform.makeTap(platform.makeBinaryScheme(FetchedDatum.FIELDS), inputPath);
        BasePath outputPath = platform.makePath("build/test/ParserPipeTest/out");
        Tap out = platform.makeTap(platform.makeBinaryScheme(ParsedDatum.FIELDS), outputPath, SinkMode.REPLACE);

        TupleEntryCollector write = in.openForWrite(platform.makeFlowProcess());

        ArchiveReader archiveReader = ArchiveReaderFactory.get("src/test/resources/someHtml.arc");
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
    }

    static void setupWorkingDir(BasePlatform platform, BasePath workingDirPath, String seedUrlsfileName) throws Exception {
       
        // Check if we already have a crawldb
        BasePath crawlDbPath = null;
        BasePath loopDirPath = CrawlDirUtils.findLatestLoopDir(platform, workingDirPath);
        if (loopDirPath != null) {
            // Clear out any previous loop directory, so we're always starting from scratch
            LOGGER.info("deleting existing working dir");
            while (loopDirPath != null) {
                loopDirPath.delete(true);
                loopDirPath = CrawlDirUtils.findLatestLoopDir(platform, workingDirPath);
            }
        }

        // Create a "0-<timestamp>" loop sub-directory and import the seed urls
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
        // Build and run the flow.
       
        try {
            BixoPlatform platform = new BixoPlatform(DemoWebMiningTool.class, options.getPlatformMode());
            BasePath workingDirPath = platform.makePath(options.getWorkingDir());

            setupWorkingDir(platform, workingDirPath, CrawlConfig.SEED_URLS_FILENAME);
            BasePath latestDirPath = CrawlDirUtils.findLatestLoopDir(platform, workingDirPath);
            if (latestDirPath == null) {
                error("No previous cycle output dirs exist in " + workingDirPath, parser);
            }
           
            BasePath crawlDbPath = platform.makePath(latestDirPath, CrawlConfig.CRAWLDB_SUBDIR_NAME);
           
            UserAgent userAgent = new UserAgent(options.getAgentName(), CrawlConfig.EMAIL_ADDRESS, CrawlConfig.WEB_ADDRESS);
           
            FetcherPolicy fetcherPolicy = new FetcherPolicy();
            fetcherPolicy.setCrawlDelay(CrawlConfig.DEFAULT_CRAWL_DELAY);
            fetcherPolicy.setMaxContentSize(CrawlConfig.MAX_CONTENT_SIZE);
            fetcherPolicy.setFetcherMode(FetcherMode.EFFICIENT);
           
            // We only care about mime types that the Tika HTML parser can handle,
            // so restrict it to the same.
            Set<String> validMimeTypes = new HashSet<String>();
            Set<MediaType> supportedTypes = new HtmlParser().getSupportedTypes(new ParseContext());
            for (MediaType supportedType : supportedTypes) {
                validMimeTypes.add(String.format("%s/%s", supportedType.getType(), supportedType.getSubtype()));
            }
            fetcherPolicy.setValidMimeTypes(validMimeTypes);

            // Let's limit our crawl to two loops
            for (int curLoop = 1; curLoop <= 2; curLoop++) {
                BasePath curLoopDirPath = CrawlDirUtils.makeLoopDir(platform, workingDirPath, curLoop);
                Flow flow = DemoWebMiningWorkflow.createWebMiningWorkflow(platform, crawlDbPath, curLoopDirPath, fetcherPolicy, userAgent, options);
                flow.complete();

                // Update crawlDbPath to point to the latest crawl db
                crawlDbPath = platform.makePath(curLoopDirPath, CrawlConfig.CRAWLDB_SUBDIR_NAME);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
       
        try {
            BixoPlatform platform = new BixoPlatform(AnalyzeMbox.class, options.getPlatformMode());
            // Create the input (source tap), which is just a sequence file reader. We assume
          // that the file already has the results of splitting the mbox file into emails.
            BasePath inputPath = platform.makePath(inputFileName);
            platform.assertPathExists(inputPath, "input file");
            Tap sourceTap = platform.makeTap(platform.makeBinaryScheme(FetchedDatum.FIELDS), inputPath);
           
            Pipe pipe = new Pipe("Email Analyzer");
            pipe = new Each(pipe, new ParseEmailFunction());
View Full Code Here
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
                                        new Fields(AnalyzedDatum.URL_FIELD), new Fields(LinkDatum.URL_FN)), null, new OuterJoin());
        updatePipe = new Every(updatePipe, new UpdateCrawlDbBuffer(), Fields.RESULTS);

       
        // output : loop dir specific crawldb
        BasePath outCrawlDbPath = platform.makePath(curLoopDirPath, CrawlConfig.CRAWLDB_SUBDIR_NAME);
        Tap crawlDbSink = platform.makeTap(platform.makeTextScheme(), outCrawlDbPath, SinkMode.REPLACE);
        // Status,
        BasePath statusDirPath = platform.makePath(curLoopDirPath, CrawlConfig.STATUS_SUBDIR_NAME);
        Tap statusSink = platform.makeTap(platform.makeTextScheme(), statusDirPath);
        // Content
        BasePath contentDirPath = platform.makePath(curLoopDirPath, CrawlConfig.CONTENT_SUBDIR_NAME);
        Tap contentSink = platform.makeTap(platform.makeBinaryScheme(FetchedDatum.FIELDS), contentDirPath);
       
        // PageResults
        BasePath resultsDirPath = platform.makePath(curLoopDirPath, CrawlConfig.RESULTS_SUBDIR_NAME);
        Tap resultsSink = platform.makeTap(platform.makeTextScheme(), resultsDirPath);

        // Create the output map that connects each tail pipe to the appropriate sink.
        Map<String, Tap> sinkMap = new HashMap<String, Tap>();
        sinkMap.put(updatePipe.getName(), crawlDbSink);
View Full Code Here
49
50
51
52
53
54
55
56
57
   */
  public static BasePath makeLoopDir(BasePlatform platform, BasePath outputDir, int loopNumber)
      throws Exception {
    String timestamp = new SimpleDateFormat("yyyyMMdd'T'HHmmss")
        .format(new Date());
    BasePath loopDir = platform.makePath(outputDir, "" + loopNumber + "-" + timestamp);
    loopDir.mkdirs();
    return loopDir;
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
   * @throws Exception
   */
  public static BasePath findLatestLoopDir(BasePlatform platform, BasePath outputPath)
      throws Exception {
    int bestLoop = -1;
    BasePath result = null;

    BasePath[] paths = outputPath.list();
    for (BasePath path : paths) {
      if (!path.isDirectory()) {
        continue;
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
   * @throws Exception
   */
  public static BasePath findNextLoopDir(BasePlatform platform, BasePath outputPath,
      int loopNumber) throws Exception {
    int bestLoop = Integer.MAX_VALUE;
    BasePath result = null;

    BasePath[] paths = outputPath.list();
    for (BasePath path : paths) {
      if (!path.isDirectory()) {
        continue;
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
169
            try {
                // Verify crawl dir name is valid.
                extractLoopNumber(path);

                BasePath subdirPath = platform.makePath(path, subdirName);
                if (subdirPath.exists() && subdirPath.isDirectory()) {
                    result.add(subdirPath);
                }
            } catch (InvalidParameterException e) {
                LOGGER.debug("Ignoring directory :" + path.getName());
            }
View Full Code Here
TOP

Related Classes of com.scaleunlimited.cascading.BasePath

  • bixo.datum.ScoredUrlDatumTest
  • bixo.examples.crawl.DemoCrawlTool
  • bixo.examples.crawl.DemoCrawlWorkflow
  • bixo.examples.crawl.DemoCrawlWorkflowLRTest
  • bixo.examples.crawl.DemoStatusTool
  • bixo.examples.crawl.LatestUrlDatumBufferTest
  • bixo.examples.webmining.DemoWebMiningTool
  • bixo.examples.webmining.DemoWebMiningWorkflow
  • bixo.examples.webmining.DemoWebMiningWorkflowTest
  • bixo.fetcher.FetcherTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.