Examples of urlhash()

Examples of net.yacy.kelondro.data.word.WordReferenceRow.urlhash()

167
168
169
170
171
172
173
174
175
176
177
                    continue;
                }
                wordHash = estring.substring(0, p);
                wordhashes.add(wordHash);
                iEntry = new WordReferenceRow(estring.substring(p));
                urlHash = iEntry.urlhash();

                // block blacklisted entries
                if ((blockBlacklist) && (Switchboard.urlBlacklist.hashInBlacklistedCache(Blacklist.BLACKLIST_DHT, urlHash))) {
                    if (Network.log.isFine()) Network.log.logFine("transferRWI: blocked blacklisted URLHash '" + ASCII.String(urlHash) + "' from peer " + otherPeerName);
                    blocked++;
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

201
202
203
204
205
206
207
208
209
210
211
            final boolean httpPattern = pattern.equals("http://.*");
            final boolean noHttpButProtocolPattern = pattern.equals("https://.*") || pattern.equals("ftp://.*") || pattern.equals("smb://.*") || pattern.equals("file://.*");
            pollloop: while (true) {
                iEntry = decodedEntries.poll(1, TimeUnit.SECONDS);
                if (iEntry == null || iEntry == WordReferenceVars.poison) break pollloop;
                assert (iEntry.urlhash().length == index.row().primaryKeyLength);
                //if (iEntry.urlHash().length() != index.row().primaryKeyLength) continue;

                // increase flag counts
                for (int j = 0; j < 32; j++) {
                    if (iEntry.flags().get(j)) {this.flagcount[j]++;}
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

240
241
242
243
244
245
246
247
248
249
250
                final String hosthash = iEntry.hosthash();
                if (this.query.sitehash == null) {
                    // no site constraint there; maybe collect host navigation information
                    if (nav_hosts && this.query.urlMask_isCatchall) {
                        this.hostNavigator.inc(hosthash);
                        this.hostResolver.put(hosthash, iEntry.urlhash());
                    }
                } else {
                    if (!hosthash.equals(this.query.sitehash)) {
                        // filter out all domains that do not match with the site constraint
                        continue pollloop;
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

257
258
259
260
261
262
263
264
265
266
267
                    if (httpPattern && !httpFlagSet) continue pollloop;
                    if (noHttpButProtocolPattern && httpFlagSet) continue pollloop;
                }

                // finally make a double-check and insert result to stack
                if (this.urlhashes.add(iEntry.urlhash())) {
                    rankingtryloop: while (true) {
                        try {
                            this.stack.put(new ReverseElement<WordReferenceVars>(iEntry, this.order.cardinal(iEntry))); // inserts the element and removes the worst (which is smallest)
                            break rankingtryloop;
                        } catch (final ArithmeticException e) {
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

479
480
481
482
483
484
485
486
487
488
489
                    while (containerIterator.hasNext() && this.run) {
                        waiter();
                        entry = new WordReferenceVars(containerIterator.next());
                        // System.out.println("Wordhash: "+wordHash+" UrlHash:
                        // "+entry.getUrlHash());
                        final URIMetadataRow ue = Segment.this.urlMetadata.load(entry.urlhash());
                        if (ue == null) {
                            urlHashs.put(entry.urlhash());
                        } else {
                            url = ue.metadata().url();
                            if (url == null || Switchboard.urlBlacklist.isListed(Blacklist.BLACKLIST_CRAWLER, url)) {
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

481
482
483
484
485
486
487
488
489
490
491
                        entry = new WordReferenceVars(containerIterator.next());
                        // System.out.println("Wordhash: "+wordHash+" UrlHash:
                        // "+entry.getUrlHash());
                        final URIMetadataRow ue = Segment.this.urlMetadata.load(entry.urlhash());
                        if (ue == null) {
                            urlHashs.put(entry.urlhash());
                        } else {
                            url = ue.metadata().url();
                            if (url == null || Switchboard.urlBlacklist.isListed(Blacklist.BLACKLIST_CRAWLER, url)) {
                                urlHashs.put(entry.urlhash());
                            }
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

485
486
487
488
489
490
491
492
493
494
495
                        if (ue == null) {
                            urlHashs.put(entry.urlhash());
                        } else {
                            url = ue.metadata().url();
                            if (url == null || Switchboard.urlBlacklist.isListed(Blacklist.BLACKLIST_CRAWLER, url)) {
                                urlHashs.put(entry.urlhash());
                            }
                        }
                    }
                    if (!urlHashs.isEmpty()) try {
                        final int removed = Segment.this.termIndex.remove(container.getTermHash(), urlHashs);
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

201
202
203
204
205
206
207
208
209
210
211
            final boolean httpPattern = pattern.equals("http://.*");
            final boolean noHttpButProtocolPattern = pattern.equals("https://.*") || pattern.equals("ftp://.*") || pattern.equals("smb://.*") || pattern.equals("file://.*");
            pollloop: while (true) {
                iEntry = decodedEntries.poll(1, TimeUnit.SECONDS);
                if (iEntry == null || iEntry == WordReferenceVars.poison) break pollloop;
                assert (iEntry.urlhash().length == index.row().primaryKeyLength);
                //if (iEntry.urlHash().length() != index.row().primaryKeyLength) continue;

                // increase flag counts
                for (int j = 0; j < 32; j++) {
                    if (iEntry.flags().get(j)) {this.flagcount[j]++;}
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

240
241
242
243
244
245
246
247
248
249
250
                final String hosthash = iEntry.hosthash();
                if (this.query.sitehash == null) {
                    // no site constraint there; maybe collect host navigation information
                    if (nav_hosts && this.query.urlMask_isCatchall) {
                        this.hostNavigator.inc(hosthash);
                        this.hostResolver.put(hosthash, iEntry.urlhash());
                    }
                } else {
                    if (!hosthash.equals(this.query.sitehash)) {
                        // filter out all domains that do not match with the site constraint
                        continue pollloop;
View Full Code Here

Examples of net.yacy.kelondro.data.word.WordReferenceVars.urlhash()

259
260
261
262
263
264
265
266
267
268
269
                }

                // finally make a double-check and insert result to stack
                // the url hashes should be unique, no reason to check that
                //if (!this.urlhashes.has(iEntry.urlhash())) {
                    this.urlhashes.putUnique(iEntry.urlhash());
                    rankingtryloop: while (true) {
                        try {
                            this.stack.put(new ReverseElement<WordReferenceVars>(iEntry, this.order.cardinal(iEntry))); // inserts the element and removes the worst (which is smallest)
                            break rankingtryloop;
                        } catch (final ArithmeticException e) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.