Package net.yacy.kelondro.data.meta

Examples of net.yacy.kelondro.data.meta.URIMetadataRow.metadata()

109
110
111
112
113
114
115
116
117
118
119
        prop.put("geo_long", metadata.lon());

        prop.put("yacy_urlhash", metadata.url().hash());
        prop.putXML("yacy_loaddate", entry.loaddate().toString());
        prop.putXML("yacy_referrer_hash", (le == null) ? "" : ASCII.String(le.hash()));
        prop.putXML("yacy_referrer_url", (le == null) ? "" : le.metadata().url().toNormalform(false, true));
        prop.put("yacy_size", entry.size());
        prop.put("yacy_words",entry.wordCount());
       
        // return rewrite properties
        return prop;
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
                        urlstr = null;
                        urltxt = null;
                        metadata = null;
                        continue;
                    }
                    metadata = urle.metadata();
                    urlstr = metadata.url().toNormalform(false, true);
                    urltxt = nxTools.shortenURLString(urlstr, 72); // shorten the string text like a URL
                   
                    initiatorSeed = entry.getValue() == null || entry.getValue().initiatorHash == null ? null : sb.peers.getConnected(ASCII.String(entry.getValue().initiatorHash));
                    executorSeed = entry.getValue() == null || entry.getValue().executorHash == null ? null : sb.peers.getConnected(ASCII.String(entry.getValue().executorHash));
View Full Code Here
344
345
346
347
348
349
350
351
352
353
354
                                Log.logException(e);
                            }
                            final URIMetadataRow e = segment.urlMetadata().load(b);
                            segment.urlMetadata().remove(b);
                            if (e != null) {
                                url = e.metadata().url();
                                pw.println(url.getHost() + "/" + url.getFile());
                                for (final String supportedBlacklistType : supportedBlacklistTypes) {
                                    if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklist)) {
                                        Switchboard.urlBlacklist.add(
                                                supportedBlacklistType,
View Full Code Here
377
378
379
380
381
382
383
384
385
386
387
                                Log.logException(e);
                            }
                            final URIMetadataRow e = segment.urlMetadata().load(b);
                            segment.urlMetadata().remove(b);
                            if (e != null) {
                                url = e.metadata().url();
                                pw.println(url.getHost() + "/.*");
                                for (final String supportedBlacklistType : supportedBlacklistTypes) {
                                    if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklist)) {
                                        Switchboard.urlBlacklist.add(
                                                supportedBlacklistType,
View Full Code Here
429
430
431
432
433
434
435
436
437
438
439
            DigestURI url;
            URIMetadataRow entry;
            String us;
            long rn = -1;
            while (!ranked.isEmpty() && (entry = ranked.takeURL(false, 1000)) != null) {
                if ((entry == null) || (entry.metadata() == null)) continue;
                url = entry.metadata().url();
                if (url == null) continue;
                us = url.toNormalform(false, false);
                if (rn == -1) rn = entry.ranking();
                prop.put("genUrlList_urlList_"+i+"_urlExists", "1");
View Full Code Here
430
431
432
433
434
435
436
437
438
439
440
            URIMetadataRow entry;
            String us;
            long rn = -1;
            while (!ranked.isEmpty() && (entry = ranked.takeURL(false, 1000)) != null) {
                if ((entry == null) || (entry.metadata() == null)) continue;
                url = entry.metadata().url();
                if (url == null) continue;
                us = url.toNormalform(false, false);
                if (rn == -1) rn = entry.ranking();
                prop.put("genUrlList_urlList_"+i+"_urlExists", "1");
                prop.put("genUrlList_urlList_"+i+"_urlExists_urlhxCount", i);
View Full Code Here
518
519
520
521
522
523
524
525
526
527
528
                    URIMetadataRow.Components metadata;
                    String url;
                    while (i.hasNext()) {
                        entry = i.next();
                        if (this.set != null && !set.has(entry.hash())) continue;
                        metadata = entry.metadata();
                        url = metadata.url().toNormalform(true, false);
                        if (!url.matches(filter)) continue;
                        if (format == 0) {
                            pw.println(url);
                        }
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
        // search is running; retrieve results
        URIMetadataRow row;
        ArrayList<DigestURI> files = new ArrayList<DigestURI>();
        Components metadata;
        while ((row = rankedCache.takeURL(false, 1000)) != null) {
            metadata = row.metadata();
            if (metadata == null) continue;
            files.add(metadata.url());
            count--;
            if (count == 0) break;
        }
View Full Code Here
421
422
423
424
425
426
427
428
429
430
431
              this.misses.add(obrwi.getElement().urlhash());
              continue;
            }

            // prepare values for constraint check
            final URIMetadataRow.Components metadata = page.metadata();

            // check errors
            if (metadata == null) {
                this.sortout++;
                continue; // rare case where the url is corrupted
View Full Code Here
627
628
629
630
631
632
633
634
635
636
637
        if (this.hostResolver != null) while (domhashs.hasNext() && result.sizeSmaller(30)) {
            hosthash = domhashs.next();
            if (hosthash == null) continue;
            urlhash = this.hostResolver.get(hosthash);
            row = urlhash == null ? null : this.query.getSegment().urlMetadata().load(urlhash);
            hostname = row == null ? null : row.metadata().url().getHost();
            if (hostname != null) {
                result.set(hostname, this.hostNavigator.get(hosthash));
            }
        }
        if (result.sizeSmaller(2)) result.clear(); // navigators with one entry are not useful
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.