Package com.crawljax.core.configuration.CrawljaxConfiguration

Examples of com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
  private static final Logger LOG = LoggerFactory.getLogger(PluginExample.class);

  public static void main(String[] args) {

    CrawljaxConfigurationBuilder builder =
            CrawljaxConfiguration.builderFor("http://demo.crawljax.com/");
    builder.addPlugin(new OnNewStatePlugin() {

      @Override
      public void onNewState(CrawlerContext context, StateVertex newState) {
        // This will print the DOM when a new state is detected. You should see it in your
        // console.
        LOG.info("Found a new dom! Here it is:\n{}", context.getBrowser().getStrippedDom());
      }

      @Override
      public String toString() {
        return "Our example plugin";
      }
    });
    CrawljaxRunner crawljax = new CrawljaxRunner(builder.build());
    crawljax.call();
  }
View Full Code Here
21
22
23
24
25
26
27
28
29
30
31
32
33
  @Test
  public void testHiddenElementsSiteCrawl() throws Exception {
    CrawlSession crawl = new BaseCrawler("hidden-elements-site") {
      @Override
      public CrawljaxConfigurationBuilder newCrawlConfigurationBuilder() {
        CrawljaxConfigurationBuilder builder =
                super.newCrawlConfigurationBuilder();
        builder.crawlRules().crawlHiddenAnchors(true);
        return builder;
      }
    }.crawl();

    StateFlowGraph stateFlowGraph = crawl.getStateFlowGraph();
View Full Code Here
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
  @ClassRule
  public static final RunWithWebServer WEB_SERVER = new RunWithWebServer("/site");

  protected CrawljaxConfigurationBuilder setupConfig() {
    CrawljaxConfigurationBuilder builder =
            CrawljaxConfiguration.builderFor(WEB_SERVER.getSiteUrl().resolve("iframe"));
    builder.crawlRules().waitAfterEvent(100, TimeUnit.MILLISECONDS);
    builder.crawlRules().waitAfterReloadUrl(100, TimeUnit.MILLISECONDS);
    builder.setMaximumDepth(3);
    builder.crawlRules().click("a");
    builder.crawlRules().click("input");

    return builder;
  }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
    assertThat(session.getStateFlowGraph(), hasStates(13));
  }

  @Test
  public void testIframeExclusions() throws CrawljaxException {
    CrawljaxConfigurationBuilder builder = setupConfig();
    builder.crawlRules().dontCrawlFrame("frame1");
    builder.crawlRules().dontCrawlFrame("sub");
    builder.crawlRules().dontCrawlFrame("frame0");
    CrawljaxConfiguration config = builder.build();
    crawljax = new CrawljaxRunner(config);
    CrawlSession session = crawljax.call();
    assertThat(session.getStateFlowGraph(), hasEdges(3));
    assertThat(session.getStateFlowGraph(), hasStates(4));
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
    assertThat(session.getStateFlowGraph(), hasStates(4));
  }

  @Test
  public void testIFramesNotCrawled() throws CrawljaxException {
    CrawljaxConfigurationBuilder builder = setupConfig();
    builder.crawlRules().crawlFrames(false);
    crawljax = new CrawljaxRunner(builder.build());
    CrawlSession session = crawljax.call();
    assertThat(session.getStateFlowGraph(), hasEdges(3));
    assertThat(session.getStateFlowGraph(), hasStates(4));
  }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
    assertThat(session.getStateFlowGraph(), hasStates(4));
  }

  @Test
  public void testIFramesWildcardsNotCrawled() throws CrawljaxException {
    CrawljaxConfigurationBuilder builder = setupConfig();

    builder.crawlRules().dontCrawlFrame("frame%");
    builder.crawlRules().dontCrawlFrame("sub");
    crawljax = new CrawljaxRunner(builder.build());
    CrawlSession session = crawljax.call();
    assertThat(session.getStateFlowGraph(), hasEdges(3));
    assertThat(session.getStateFlowGraph(), hasStates(4));
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
    assertThat(session.getStateFlowGraph(), hasStates(4));
  }

  @Test
  public void testCrawlingOnlySubFrames() throws CrawljaxException {
    CrawljaxConfigurationBuilder builder = setupConfig();
    builder.crawlRules().dontCrawlFrame("frame1.frame10");
    crawljax = new CrawljaxRunner(builder.build());
    CrawlSession session = crawljax.call();
    assertEquals("Clickables", 12, session.getStateFlowGraph()
            .getAllEdges().size());
    assertEquals("States", 12, session.getStateFlowGraph().getAllStates()
            .size());
View Full Code Here
26
27
28
29
30
31
32
33
34
35
36
37
38
    CrawlSession crawl =
            new BaseCrawler(Resource.newClassPathResource("/site"),
                    "simplelink/simplelink.html") {
              @Override
              public CrawljaxConfigurationBuilder newCrawlConfigurationBuilder() {
                CrawljaxConfigurationBuilder builder =
                        super.newCrawlConfigurationBuilder();
                builder.setBrowserConfig(new BrowserConfiguration(BrowserType.CHROME));
                return builder;
              }
            }.crawl();
    assertThat(crawl.getStateFlowGraph(), hasStates(2));
  }
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
109
   * Override this method to specify a different configuration.
   *
   * @return a new {@link CrawljaxConfiguration} to crawl with.
   */
  protected CrawljaxConfigurationBuilder newCrawlConfigurationBuilder() {
    CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(getUrl());
    builder.crawlRules().clickDefaultElements();
    builder.setUnlimitedRuntime();
    builder.setUnlimitedCrawlDepth();
    builder.addPlugin(new PostCrawlStateGraphChecker());
    builder.setBrowserConfig(new BrowserConfiguration(BrowserProvider.getBrowserType()));
    return builder;
  }
View Full Code Here
17
18
19
20
21
22
23
24
25
26
    super(Resource.newClassPathResource("sites"), "simple-xpath-site");
  }

  @Override
  protected CrawljaxConfigurationBuilder newCrawlConfigurationBuilder() {
    CrawljaxConfigurationBuilder builder = super.newCrawlConfigurationBuilder();
    builder.crawlRules().click("a").underXPath("//A[@class='click']");
    builder.crawlRules().dontClickChildrenOf("div").withId("dontClick");
    return builder;
  }
View Full Code Here
TOP

Related Classes of com.crawljax.core.configuration.CrawljaxConfiguration.CrawljaxConfigurationBuilder

  • $.Runner
  • com.crawljax.browser.ChromeProxyConfig
  • com.crawljax.cli.JarRunner
  • com.crawljax.core.CandidateElementExtractorTest
  • com.crawljax.core.configuration.UnderXPathTest
  • com.crawljax.core.CrawlerStopTest
  • com.crawljax.core.IFrameTest
  • com.crawljax.core.largetests.LargeTestBase
  • com.crawljax.core.PassBasicHttpAuthTest
  • com.crawljax.core.plugin.OnFireEventFailedPluginTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.