Examples of HandleSet

net.yacy.kelondro.index.HandleSet

Examples of net.yacy.kelondro.index.HandleSet

    public boolean isLocal() {
        return this.domType == SEARCHDOM_LOCAL;
    }


    public static HandleSet hashes2Set(final String query) {
        final HandleSet keyhashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
        if (query != null) {
            for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try {
                keyhashes.put(ASCII.getBytes(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength)));
            } catch (final RowSpaceExceededException e) {
                Log.logException(e);
            }
        }
        return keyhashes;

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet

        }
        return keyhashes;
    }


    public static HandleSet hashes2Handles(final String query) {
        final HandleSet keyhashes = new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
        if (query != null) {
            for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try {
                keyhashes.put(ASCII.getBytes(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength)));
            } catch (final RowSpaceExceededException e) {
                Log.logException(e);
            }
        }
        return keyhashes;

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet

     * @param text
     * @return true if the query matches with the given text
     */
    public final boolean matchesText(final String text) {
        boolean ret = false;
        final HandleSet wordhashes = Word.words2hashesHandles(Condenser.getWords(text, null).keySet());
        if (!SetTools.anymatch(wordhashes, this.excludeHashes)) {
            ret = SetTools.totalInclusion(this.queryHashes, wordhashes);
        }
        return ret;
    }

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet

    }


    protected static final boolean anymatch(final String text, final HandleSet keyhashes) {
      // returns true if any of the word hashes in keyhashes appear in the String text
      // to do this, all words in the string must be recognized and transcoded to word hashes
      final HandleSet wordhashes = Word.words2hashesHandles(Condenser.getWords(text, null).keySet());
      return SetTools.anymatch(wordhashes, keyhashes);
    }

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet

    }


    public void filterOut(final SortedSet<String> blueList) {
        // filter out words that appear in this set
      // this is applied to the queryHashes
      final HandleSet blues = Word.words2hashesHandles(blueList);
      for (final byte[] b: blues) this.queryHashes.remove(b);
    }

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet

            String estring;
            int p;
            String wordHash;
            byte[] urlHash;
            WordReferenceRow iEntry;
            final HandleSet unknownURL = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0);
            final HandleSet knownURL = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0);
            final ArrayList<String> wordhashes = new ArrayList<String>();
            int received = 0;
            int blocked = 0;
            int receivedURL = 0;
            final IndexCell<WordReference> cell = sb.indexSegments.termIndex(Segments.Process.DHTIN);
            int count = 0;
            while (it.hasNext()) {
                serverCore.checkInterruption();
                estring = it.next();
                count++;
                if (count > 1000) break; // protection against flooding


                // check if RWI entry is well-formed
                p = estring.indexOf('{',0);
                if (p < 0 || estring.indexOf("x=",0) < 0 || !(estring.indexOf("[B@",0) < 0)) {
                    blocked++;
                    continue;
                }
                wordHash = estring.substring(0, p);
                wordhashes.add(wordHash);
                iEntry = new WordReferenceRow(estring.substring(p));
                urlHash = iEntry.urlhash();


                // block blacklisted entries
                if ((blockBlacklist) && (Switchboard.urlBlacklist.hashInBlacklistedCache(Blacklist.BLACKLIST_DHT, urlHash))) {
                    if (Network.log.isFine()) Network.log.logFine("transferRWI: blocked blacklisted URLHash '" + ASCII.String(urlHash) + "' from peer " + otherPeerName);
                    blocked++;
                    continue;
                }


                // check if the entry is in our network domain
                final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomainHash(urlHash);
                if (urlRejectReason != null) {
                    Network.log.logWarning("transferRWI: blocked URL hash '" + ASCII.String(urlHash) + "' (" + urlRejectReason + ") from peer " + otherPeerName + "; peer is suspected to be a spam-peer (or something is wrong)");
                    //if (yacyCore.log.isFine()) yacyCore.log.logFine("transferRWI: blocked URL hash '" + urlHash + "' (" + urlRejectReason + ") from peer " + otherPeerName);
                    blocked++;
                    continue;
                }


                // learn entry
                try {
                    cell.add(wordHash.getBytes(), iEntry);
                } catch (final Exception e) {
                    Log.logException(e);
                }
                serverCore.checkInterruption();


                // check if we need to ask for the corresponding URL
                if (!(knownURL.has(urlHash) || unknownURL.has(urlHash)))  try {
                    if (sb.indexSegments.urlMetadata(Segments.Process.DHTIN).exists(urlHash)) {
                        knownURL.put(urlHash);
                    } else {
                        unknownURL.put(urlHash);
                    }
                    receivedURL++;
                } catch (final Exception ex) {

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet


        // tell all threads to do nothing for a specific time
        sb.intermissionAllThreads(100);


        EventTracker.delete(EventTracker.EClass.SEARCH);
        final HandleSet abstractSet = (abstracts.length() == 0 || abstracts.equals("auto")) ? null : QueryParams.hashes2Set(abstracts);


        // store accessing peer
        Seed remoteSeed;
        try {
            remoteSeed = Seed.genRemoteSeed(oseed, key, false, client);
        } catch (final IOException e) {
            Network.log.logInfo("yacy.search: access with bad seed: " + e.getMessage());
            remoteSeed = null;
        }
        if (sb.peers == null) {
            Network.log.logSevere("yacy.search: seed cache not initialized");
        } else {
            sb.peers.peerActions.peerArrival(remoteSeed, true);
        }


        // prepare search
        final HandleSet queryhashes = QueryParams.hashes2Set(query);
        final HandleSet excludehashes = (exclude.length() == 0) ? new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0) : QueryParams.hashes2Set(exclude);
        final long timestamp = System.currentTimeMillis();


      // prepare a search profile
        final RankingProfile rankingProfile = (profile.length() == 0) ? new RankingProfile(ContentDomain.contentdomParser(contentdom)) : new RankingProfile("", profile);


        // prepare an abstract result
        final StringBuilder indexabstract = new StringBuilder(6000);
        int indexabstractContainercount = 0;
        int joincount = 0;
        QueryParams theQuery = null;
        SearchEvent theSearch = null;
        ArrayList<WeakPriorityBlockingQueue.Element<ResultEntry>> accu = null;
        if (query.length() == 0 && abstractSet != null) {
            // this is _not_ a normal search, only a request for index abstracts
            final Segment indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
            theQuery = new QueryParams(
                    null,
                    abstractSet,
                    new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0),
                    null,
                    snippetPattern,
                    null,
                    modifier,
                    maxdist,

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet

            if (agent == null) agent = System.getProperty("user.language");
            language = (agent == null) ? "en" : ISO639.userAgentLanguageDetection(agent);
            if (language == null) language = "en";
        }
        final TreeSet<String>[] query = QueryParams.cleanQuery(querystring); // converts also umlaute
        HandleSet q = Word.words2hashesHandles(query[0]);
        
        // tell all threads to do nothing for a specific time
        sb.intermissionAllThreads(3000);


        // prepare search

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet

     * @param urlhash
     * @return true, if the entry was removed; false if not
     */
    public boolean removeByURLHash(final byte[] urlhashBytes) {
        try {
            final HandleSet urlHashes = new HandleSet(12, Base64Order.enhancedCoder, 1);
            urlHashes.put(urlhashBytes);
            boolean ret = false;
            try {ret |= this.noloadStack.remove(urlHashes) > 0;} catch (final IOException e) {}
            try {ret |= this.coreStack.remove(urlHashes) > 0;} catch (final IOException e) {}
            try {ret |= this.limitStack.remove(urlHashes) > 0;} catch (final IOException e) {}
            try {ret |= this.remoteStack.remove(urlHashes) > 0;} catch (final IOException e) {}

View Full Code Here

Examples of net.yacy.kelondro.index.HandleSet

                    }
                }
            }


            // do the search
            final HandleSet queryHashes = Word.words2hashesHandles(query[0]);
            final Pattern snippetPattern = QueryParams.stringSearchPattern(originalquerystring);


            // check filters
            try {
                Pattern.compile(urlmask);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.