Package com.scaleunlimited.cascading

Examples of com.scaleunlimited.cascading.PartitioningKey

56
57
58
59
60
61
62
63
64
65
66
67
       
    @Test
    public void testRoundTrip() {
        List<ScoredUrlDatum> urls = makeUrls(2, "meta1");
        long fetchTime = System.currentTimeMillis();
        PartitioningKey groupingKey = new PartitioningKey("key", 1);
        FetchSetDatum pfd1 = new FetchSetDatum(urls, fetchTime, 0, groupingKey.getValue(), groupingKey.getRef());
        pfd1.setLastList(true);
       
        Tuple t = pfd1.getTuple();
       
        FetchSetDatum pfd2 = new FetchSetDatum(t);
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
   
    @Test
    public void testSerializable() throws Exception {
        List<ScoredUrlDatum> urls = makeUrls(2, "meta1");
        long fetchTime = System.currentTimeMillis();
        PartitioningKey groupingKey = new PartitioningKey("key", 1);
        FetchSetDatum pfd1 = new FetchSetDatum(urls, fetchTime, 1000, groupingKey.getValue(), groupingKey.getRef());

        ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
        ObjectOutputStream oos = new ObjectOutputStream(byteArray);
        oos.writeObject(pfd1);
       
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    @Test
    public void testGroupBy() throws Exception {
        List<ScoredUrlDatum> urls = makeUrls(2, null);
        long fetchTime = System.currentTimeMillis();
       
        PartitioningKey groupingKey1 = new PartitioningKey("key1", 1);
        FetchSetDatum pfd1 = new FetchSetDatum(urls, fetchTime, 0, groupingKey1.getValue(), groupingKey1.getRef());
        pfd1.setLastList(true);
        Comparable c1 = pfd1.getGroupingKey();
       
        PartitioningKey groupingKey2 = new PartitioningKey("key2", 1);
        FetchSetDatum pfd2 = new FetchSetDatum(urls, fetchTime, 0, groupingKey2.getValue(), groupingKey2.getRef());
        pfd2.setLastList(true);
        Comparable c2 = pfd2.getGroupingKey();
       
        Assert.assertEquals(0, c1.compareTo(c2));
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
        List<ScoredUrlDatum> urls = new LinkedList<ScoredUrlDatum>();
        ScoredUrlDatum url = new ScoredUrlDatum("http://domain.com/page-1", "key", UrlStatus.UNFETCHED, 1.0);
        urls.add(url);
       
        long fetchTime = System.currentTimeMillis();
        PartitioningKey groupingKey = new PartitioningKey("key", 1);
        FetchSetDatum pfd = new FetchSetDatum(urls, fetchTime, 1000, groupingKey.getValue(), groupingKey.getRef());
       
        BixoPlatform platform = new BixoPlatform(ScoredUrlDatumTest.class, platformMode);
        BasePath path = platform.makePath("build/test/ScoredUrlDatumTest/testCascadingSerialization/in");
        Tap in = platform.makeTap(platform.makeBinaryScheme(FetchSetDatum.FIELDS), path, SinkMode.REPLACE);
        TupleEntryCollector write = in.openForWrite(platform.makeFlowProcess());
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
       
        _policy.startFetchSet(key, crawlDelay);
       
        TupleEntryCollector collector = buffCall.getOutputCollector();

        PartitioningKey newKey = new PartitioningKey(key, _numReduceTasks);
       
        while (safeHasNext()) {
            ScoredUrlDatum scoredDatum = new ScoredUrlDatum(new TupleEntry(values.next()));
            FetchSetInfo setInfo = _policy.nextFetchSet(scoredDatum);
            if (setInfo != null) {
View Full Code Here
TOP

Related Classes of com.scaleunlimited.cascading.PartitioningKey

  • bixo.datum.FetchSetDatumTest
  • bixo.datum.ScoredUrlDatumTest
  • bixo.operations.MakeFetchSetsBuffer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.