Package org.apache.droids

Examples of org.apache.droids.LinkTask

28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
  @Test
  public void testOffer() throws Exception
  {
    LinkedList<LinkTask> queue;
    URI  uri;
    LinkTask task;

    queue = new SimpleTaskQueueWithHistory<LinkTask>();
    Assert.assertEquals(0, queue.size());
    uri = new URI("http://www.example.com");
    Assert.assertNotNull(uri);
    task = new LinkTask(null, uri, 1);
    Assert.assertNotNull(task);
    queue.offer(task);
    Assert.assertEquals(1, queue.size());
    queue.offer(task);
    Assert.assertEquals(1, queue.size());
View Full Code Here
161
162
163
164
165
166
167
168
169
170
   * Setting Anchor text of last added anchor
   * @param anchorText Text to be added
   */
  private void addAnchorText(String anchorText) {
    if(links.size() > 0) {
      LinkTask l = (LinkTask) links.get(links.size() - 1);
      l.setAnchorText(anchorText.replaceAll("\\s+", " ").trim());
      log.debug("Adding anchor: " + l.getAnchorText() + " on link: " + l);
    }
  }
View Full Code Here
178
179
180
181
182
183
184
185
186
    if (history == null)
      history = new HashSet<String>();
    if (links == null)
      links = new ArrayList<Link>();
    if (history.add(link.toString())) {
      links.add(new LinkTask(base, link, base.getDepth() + 1));
      log.debug("Added outlink: " + link + " with depth: " + base.getDepth() + 1);
    }
  }
View Full Code Here
63
64
65
66
67
68
69
70
      try {
        uri = new URI(location);
      } catch (URISyntaxException ex) {
        throw new InvalidTaskException("Invalid lication: " + location);
      }
      queue.offer(new LinkTask(null, uri, 0));
    }
  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
 
  // execute
  @Test
  public void execute_linkIsParsed() throws DroidsException, IOException, URISyntaxException {
    // Arrange
    final Link link = new LinkTask(null, new URI("http://www.google.com"), 1);
   
    // Act
    this.instance.execute(link);
   
    // Assert
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
 
  // execute
  @Test
  public void execute_linkIsParsed() throws DroidsException, IOException, URISyntaxException {
    // Arrange
    final Link link = new LinkTask(null, new URI("http://www.google.com"), 1);
   
    // Act
    this.instance.execute(link);
   
    // Assert
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
  @Test
  public void testOffer() throws Exception
  {
    Queue<LinkTask> queue;
    URI  uri;
    LinkTask task;

    queue = new SimpleTaskQueueWithHistory<LinkTask>();
    Assert.assertEquals(0, queue.size());
    uri = new URI("http://www.example.com");
    Assert.assertNotNull(uri);
    task = new LinkTask(null, uri, 1);
    Assert.assertNotNull(task);
    queue.offer(task);
    Assert.assertEquals(1, queue.size());
    queue.offer(task);
    Assert.assertEquals(1, queue.size());
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
  @Test
  public void testAddAll() throws URISyntaxException
  {
    Collection<Link> links = new LinkedList<Link>();
    links.add(new LinkTask(null, new URI("http://www.example.com"), 0));
    links.add(new LinkTask(null, new URI("http://www.example.com/1"), 1));
    links.add(new LinkTask(null, new URI("http://www.example.com/2"), 1));
    links.add(new LinkTask(null, new URI("http://www.example.com/3"), 1));
    links.add(new LinkTask(null, new URI("http://www.example.com/4"), 1));
   
    Queue<Link> queue = new SimpleTaskQueueWithHistory<Link>();
    assertEquals(0, queue.size());
    queue.addAll(links);
    assertEquals(5, queue.size());
   
    links.add(new LinkTask(null, new URI("http://www.example.com/1"), 1));
    links.add(new LinkTask(null, new URI("http://www.example.com/5"), 1));
    links.add(new LinkTask(null, new URI("http://www.example.com/2"), 1));

    queue.addAll(links);
    assertEquals(6, queue.size());
   
    queue.poll();
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
            // Test to see if the scheme is empty
            // This would indicate a relative URL, so resolve it against the task URI
            if(uri.getScheme() == null) {
              uri = ((Link) task).getURI().resolve(uri);
            }
            extractedTasks.add(new LinkTask((Link)task, uri, depth, tikaLink.getText()));
          } catch (URISyntaxException e) {
            if(LOG.isWarnEnabled()) {
              LOG.warn("URI not valid: "+ tikaLink.getUri());
            }
          }
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
            // Test to see if the scheme is empty
            // This would indicate a relative URL, so resolve it against the task URI
            if(uri.getScheme() == null) {
              uri = ((Link) task).getURI().resolve(uri);
            }
            extractedTasks.add(new LinkTask((Link)task, uri, depth, tikaLink.getText()));
          } catch (URISyntaxException e) {
            if(log.isWarnEnabled()) {
              log.warn("URI not valid: "+ tikaLink.getUri());
            }
          }
View Full Code Here
TOP

Related Classes of org.apache.droids.LinkTask

  • org.apache.droids.impl.TestSimpleTaskQueueWithHistory
  • org.apache.droids.parse.html.LinkExtractor
  • org.apache.droids.robot.crawler.CrawlingDroid
  • org.apache.droids.robot.crawler.TestCrawlingWorker
  • org.apache.droids.tika.TikaDocumentParser
  • org.apache.droids.tika.TikaHtmlParser
  • java.util.Date
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.