Package fr.eolya.crawler.queue

Examples of fr.eolya.crawler.queue.ISourceItemsQueue.push()

880
881
882
883
884
885
886
887
888
889
890
      }
      for (int i=0; i<startingUrls.size(); i++) {
        if (!startingUrls.get(i).onlyFirstCrawl || !src.isFirstCrawlCompleted()) {
          if (haveSiteMode && !src.isFirstCrawlCompleted() && !"s".equals(startingUrls.get(i).mode)) continue;
          try {
            sourceQueue.push(startingUrls.get(i).getMap(src.getId()));
          } catch (Exception e) {
            e.printStackTrace();
          }
        }
      }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
      if (queue!=null) {
        queue.reset();
        queue.start();
        assertEquals(0, queue.size());
       
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/2.html", 1).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/3.html", 2).getMap()));

        assertTrue(queue.contains("http://a.a.a/"));
        assertEquals(3, queue.size());
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
        queue.reset();
        queue.start();
        assertEquals(0, queue.size());
       
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/2.html", 1).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/3.html", 2).getMap()));

        assertTrue(queue.contains("http://a.a.a/"));
        assertEquals(3, queue.size());
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
        queue.start();
        assertEquals(0, queue.size());
       
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/2.html", 1).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/3.html", 2).getMap()));

        assertTrue(queue.contains("http://a.a.a/"));
        assertEquals(3, queue.size());

        assertEquals(0, queue.getDoneQueueSize());
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
        assertTrue(queue.isDone("http://a.a.a/"));

        assertFalse(queue.contains("http://a.a.a/"));
        assertEquals(0, queue.size());

        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/4.html", 2).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/3.html", 1).getMap()));
        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/2.html", 2).getMap()));
       
        assertEquals(2, queue.size());
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
        assertFalse(queue.contains("http://a.a.a/"));
        assertEquals(0, queue.size());

        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/4.html", 2).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/3.html", 1).getMap()));
        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/2.html", 2).getMap()));
       
        assertEquals(2, queue.size());
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
        assertFalse(queue.contains("http://a.a.a/"));
        assertEquals(0, queue.size());

        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/4.html", 2).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/3.html", 1).getMap()));
        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/2.html", 2).getMap()));
       
        assertEquals(2, queue.size());

        assertEquals("http://a.a.a/3.html", queue.pop("depth").get("url"));
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
        assertEquals(0, queue.size());

        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/4.html", 2).getMap()));
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/3.html", 1).getMap()));
        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/2.html", 2).getMap()));
       
        assertEquals(2, queue.size());

        assertEquals("http://a.a.a/3.html", queue.pop("depth").get("url"));
        assertEquals("http://a.a.a/4.html", queue.pop("depth").get("url"));
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
     
      queue = QueueFactory.getSourceItemsQueueInstance("mongodb", 1, dbConnection, "testFifoQueue", "TestMongoDBWebSiteUrlFifoQueue"); 
      if (queue!=null) {
        queue.start();
        assertEquals(0, queue.size());
        assertFalse(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
        queue.stop();
      }

      queue = QueueFactory.getSourceItemsQueueInstance("mongodb", 1, dbConnection, "testFifoQueue", "TestMongoDBWebSiteUrlFifoQueue"); 
      if (queue!=null) {
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
      queue = QueueFactory.getSourceItemsQueueInstance("mongodb", 1, dbConnection, "testFifoQueue", "TestMongoDBWebSiteUrlFifoQueue"); 
      if (queue!=null) {
        queue.start();
        assertEquals(0, queue.size());
        assertTrue(queue.push(getItemWeb(1, "http://a.a.a/", 0).getMap()));
      }
    }
    catch (Exception e) {
      e.printStackTrace();
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.