Examples of makePath()

Examples of bixo.config.BixoPlatform.makePath()

121
122
123
124
125
126
127
128
129
130
131
        String crawlDirName = options.getWorkingDir();

        try {
            BixoPlatform platform = new BixoPlatform(DemoStatusTool.class, options.getPlatformMode());
          BasePath crawlDirPath = platform.makePath(crawlDirName);

          platform.assertPathExists(crawlDirPath, "Prior crawl output directory does not exist");
         
          // Skip Hadoop/Cascading DEBUG messages.
            org.apache.log4j.Logger.getRootLogger().setLevel(Level.INFO);
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

149
150
151
152
153
154
155
156
157
158
159
            logsDir = logsDir + "/";
        }
       
        try {
            BixoPlatform platform = new BixoPlatform(DemoCrawlTool.class, options.getPlatformMode());
            BasePath outputPath = platform.makePath(outputDirName);

            // First check if the user want to clean
            if (options.isCleanOutputDir()) {
                if (outputPath.exists()) {
                    outputPath.delete(true);
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

169
170
171
172
173
174
175
176
177
178
179
                // In the /crawldb dir the input file will have a single URL for the target domain.

                BasePath curLoopDir = CrawlDirUtils.makeLoopDir(platform, outputPath, 0);
                String curLoopDirName = curLoopDir.getName();
                setLoopLoggerFile(logsDir + curLoopDirName, 0);
                BasePath crawlDbPath = platform.makePath(curLoopDir, CrawlConfig.CRAWLDB_SUBDIR_NAME);
               
                if (domain != null) {
                    importOneDomain(platform, domain, crawlDbPath);
                } else {
                    BasePath urlsPath = platform.makePath(urlsFile);
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

174
175
176
177
178
179
180
181
182
183
184
                BasePath crawlDbPath = platform.makePath(curLoopDir, CrawlConfig.CRAWLDB_SUBDIR_NAME);
               
                if (domain != null) {
                    importOneDomain(platform, domain, crawlDbPath);
                } else {
                    BasePath urlsPath = platform.makePath(urlsFile);
                    UrlImporter urlImporter = new UrlImporter(platform, urlsPath, crawlDbPath);
                    urlImporter.importUrls(false);                }
            }
           
            BasePath latestDirPath = CrawlDirUtils.findLatestLoopDir(platform, outputPath);
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

186
187
188
189
190
191
192
193
194
195
196
            if (latestDirPath == null) {
                System.err.println("No previous cycle output dirs exist in " + outputDirName);
                printUsageAndExit(parser);
            }

            BasePath crawlDbPath = platform.makePath(latestDirPath, CrawlConfig.CRAWLDB_SUBDIR_NAME);

            // Set up the start and end loop counts.
            int startLoop = CrawlDirUtils.extractLoopNumber(latestDirPath);
            int endLoop = startLoop + options.getNumLoops();
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

252
253
254
255
256
257
258
259
260
261
262
               
                // Writing out .dot files is a good way to verify your flows.
//              flow.writeDOT("build/valid-flow.dot");

                // Update crawlDbPath to point to the latest crawl db
                crawlDbPath = platform.makePath(curLoopDirPath, CrawlConfig.CRAWLDB_SUBDIR_NAME);
            }
        } catch (Throwable t) {
            System.err.println("Exception running tool: " + t.getMessage());
            t.printStackTrace(System.err);
            System.exit(-1);
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

69
70
71
72
73
74
75
76
77
78
79
    private String makeCrawlDb(String workingFolder, String input) throws Exception {

        BixoPlatform platform = new BixoPlatform(FetcherTest.class, Platform.Local);
       
        // We don't want to regenerate this DB all the time.
        BasePath workingPath = platform.makePath(workingFolder);
        BasePath crawlDBPath = platform.makePath(workingPath, URL_DB_NAME);
        if (!crawlDBPath.exists()) {
            Pipe importPipe = new Pipe("import URLs");
            importPipe = new Each(importPipe, new LoadUrlsFunction());
           
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

70
71
72
73
74
75
76
77
78
79
80
        BixoPlatform platform = new BixoPlatform(FetcherTest.class, Platform.Local);
       
        // We don't want to regenerate this DB all the time.
        BasePath workingPath = platform.makePath(workingFolder);
        BasePath crawlDBPath = platform.makePath(workingPath, URL_DB_NAME);
        if (!crawlDBPath.exists()) {
            Pipe importPipe = new Pipe("import URLs");
            importPipe = new Each(importPipe, new LoadUrlsFunction());
           
            BasePath inputPath = platform.makePath(input);
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

75
76
77
78
79
80
81
82
83
84
85
        BasePath crawlDBPath = platform.makePath(workingPath, URL_DB_NAME);
        if (!crawlDBPath.exists()) {
            Pipe importPipe = new Pipe("import URLs");
            importPipe = new Each(importPipe, new LoadUrlsFunction());
           
            BasePath inputPath = platform.makePath(input);
            Tap sourceTap = platform.makeTap(platform.makeTextScheme(), inputPath);
            Tap sinkTap = platform.makeTap(platform.makeBinaryScheme(UrlDatum.FIELDS), crawlDBPath, SinkMode.REPLACE);
           
            FlowConnector flowConnector = platform.makeFlowConnector();
            Flow flow = flowConnector.connect(sourceTap, sinkTap, importPipe);
View Full Code Here

Examples of bixo.config.BixoPlatform.makePath()

96
97
98
99
100
101
102
103
104
105
106
        String workingFolder = "build/it/FetcherTest/testStaleConnection/working";
        String input = makeCrawlDb(workingFolder, "src/it/resources/apple-pages.txt");
       
        BixoPlatform platform = new BixoPlatform(FetcherTest.class, Platform.Local);
        BasePath inputPath = platform.makePath(input);

        Tap in = platform.makeTap(platform.makeBinaryScheme(UrlDatum.FIELDS), inputPath);
        String outputDir = "build/it/FetcherTest/testStaleConnection/out";
        BasePath outputPath = platform.makePath(outputDir);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.