Examples of yacySeed


Examples of de.anomic.yacy.yacySeed

        // response values
        String result = "";
        String doublevalues = "0";

        final yacySeed otherPeer = sb.peers.get(iam);
        final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));

        if ((youare == null) || (!youare.equals(sb.peers.mySeed().hash))) {
            yacyCore.log.logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.peers.mySeed().hash);
            result = "wrong_target";
        } else if ((!granted) || (sb.isRobinsonMode())) {
View Full Code Here

Examples of de.anomic.yacy.yacySeed

       
        // create other peer crawl table using YaCyNews
        Iterator<yacyNewsDB.Record> recordIterator = sb.peers.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
        int showedCrawl = 0;
        yacyNewsDB.Record record;
        yacySeed peer;
        String peername;
        while (recordIterator.hasNext()) {
            record = recordIterator.next();
            if (record == null) {
                continue;
            }
            if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
                peer = sb.peers.get(record.originator());
                peername = (peer == null) ? record.originator() : peer.getName();

                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_dark", dark ? "1" : "0");
                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_cre", record.created().toString());
                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_peername", peername);
                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_startURL", record.attributes().get("startURL").toString());
                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_intention", record.attributes().get("intention").toString());
                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_generalDepth", record.attributes().get("generalDepth"));
                prop.put("otherCrawlStartInProgress_" + showedCrawl + "_crawlingQ", ("true".equals(record.attributes().get("crawlingQ"))) ? "1" : "0");
                showedCrawl++;
                if (showedCrawl > 20) {
                    break;
                }
            }
        }
        prop.put("otherCrawlStartInProgress", showedCrawl);
       
        // finished remote crawls
        recordIterator = sb.peers.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true);
        showedCrawl = 0;
        while (recordIterator.hasNext()) {
            record = recordIterator.next();
            if (record == null) {
                continue;
            }
            if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
                peer = sb.peers.get(record.originator());
                peername = (peer == null) ? record.originator() : peer.getName();

                prop.put("otherCrawlStartFinished_" + showedCrawl + "_dark", dark ? "1" : "0");
                prop.put("otherCrawlStartFinished_" + showedCrawl + "_cre", record.created().toString());
                prop.putHTML("otherCrawlStartFinished_" + showedCrawl + "_peername", peername);
                prop.putHTML("otherCrawlStartFinished_" + showedCrawl + "_startURL", record.attributes().get("startURL").toString());
View Full Code Here

Examples of de.anomic.yacy.yacySeed

        final String col = post.get("col", "");
        final File listsPath = env.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT);
       
        String otherPeerName = null;
        if (post.containsKey("iam")) {
            final yacySeed bla = sb.peers.get(post.get("iam", ""));
            if (bla != null) otherPeerName = bla.getName();
        }
        if (otherPeerName == null) otherPeerName = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP);
       
        if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(otherPeerName))) {
            // if we are a robinson cluster, answer only if this client is known by our network definition
View Full Code Here

Examples of de.anomic.yacy.yacySeed

            }
           
            String prot = "http"; // only httpd sessions listed
           
            // determining if the source is a yacy host
            yacySeed seed = null;
            if (doNameLookup) {
                seed = sb.peers.lookupByIP(userAddress,true,false,false);
                if (seed != null && (seed.hash.equals(sb.peers.mySeed().hash)) &&
                        (!seed.get(yacySeed.PORT,"").equals(Integer.toString(userPort)))) {
                    seed = null;
                }
            }
           
            prop.put("list_" + idx + "_dark", dark ? "1" : "0");
            dark = !dark;
            try {
                prop.put("list_" + idx + "_serverSessionID",URLEncoder.encode(s.getName(),"UTF8"));
            } catch (final UnsupportedEncodingException e) {
                // TODO Auto-generated catch block
                Log.logException(e);
            }
            prop.putHTML("list_" + idx + "_sessionName", s.getName());
            prop.put("list_" + idx + "_proto", prot);
            if (sessionTime > 1000*60) {
                prop.put("list_" + idx + "_ms", "0");
                prop.put("list_" + idx + "_ms_duration",yacyPeerActions.formatInterval(sessionTime));
            } else {
                prop.put("list_" + idx + "_ms", "1");
                prop.putNum("list_" + idx + "_ms_duration", sessionTime);
            }
            prop.putHTML("list_" + idx + "_source",(seed!=null)?seed.getName()+".yacy":userAddress.getHostAddress()+":"+userPort);
            prop.putHTML("list_" + idx + "_dest", "-");
            if (blockingRequest) {
                prop.put("list_" + idx + "_running", "0");
                prop.putNum("list_" + idx + "_running_reqNr", commandCount+1);
                numActivePending++;
View Full Code Here

Examples of de.anomic.yacy.yacySeed

            if (otherSeedString.length() == 0) {
                prop.put("response", "-1"); // request rejected
                return prop;
            }
            //Date remoteTime = yacyCore.parseUniversalDate((String) post.get(yacySeed.MYTIME)); // read remote time
            yacySeed otherSeed;
            try {
                otherSeed = yacySeed.genRemoteSeed(otherSeedString, key, false, ias.getHostAddress());
            } catch (IOException e) {
                prop.put("response", "-1"); // don't accept messages for bad seeds
                return prop;
            }

            String subject = crypt.simpleDecode(post.get("subject", ""), key); // message's subject
            String message = crypt.simpleDecode(post.get("message", ""), key); // message body
            if (subject == null || message == null) {
                prop.put("response", "-1"); // don't accept empty messages
                return prop;
            }
            message = message.trim();
            subject = subject.trim();
            if (subject.length() == 0 || message.length() == 0) {
                prop.put("response", "-1"); // don't accept empty messages
                return prop;
            }
           
            prop.put("response", "Thank you!");

            // save message
            MessageBoard.entry msgEntry = null;
            byte[] mb;
            mb = UTF8.getBytes(message);
            sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
                    "remote",
                    otherSeed.get(yacySeed.NAME, "anonymous"), otherSeed.hash,
                    sb.peers.mySeed().getName(), sb.peers.mySeed().hash,
                    subject, mb));

            messageForwardingViaEmail(sb, msgEntry);
View Full Code Here

Examples of de.anomic.yacy.yacySeed

        final serverObjects prop = new serverObjects();

        if (post != null) {
            if (post.containsKey("retrieve")) {
                final String peerhash = post.get("peer", null);
                final yacySeed seed = (peerhash == null) ? null : sb.peers.getConnected(peerhash);
                final RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.peers, seed, 20, 60000);
                if (feed != null) {
                    for (final Hit item: feed) {
                        //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate());
                       
View Full Code Here

Examples of de.anomic.yacy.yacySeed

        return (url == null ? "null" : url.toNormalform(true, false));
    }
   
    private static void listHosts(final Switchboard sb, final serverObjects prop) {
        // list known hosts
        yacySeed seed;
        int hc = 0;
        if (sb.peers != null && sb.peers.sizeConnected() > 0) {
            final Iterator<yacySeed> e = PeerSelection.getProvidesRemoteCrawlURLs(sb.peers);
            while (e.hasNext()) {
                seed = e.next();
                if (seed != null) {
                    prop.put("hosts_" + hc + "_hosthash", seed.hash);
                    prop.putHTML("hosts_" + hc + "_hostname", seed.hash + " " + seed.get(yacySeed.NAME, "nameless") + " (" + seed.getLong(yacySeed.RCOUNT, 0) + ")");
                    hc++;
                }
            }
            prop.put("hosts", hc);
        } else {
View Full Code Here

Examples of de.anomic.yacy.yacySeed

        EventTracker.delete(EventTracker.EClass.SEARCH);
        final HandleSet abstractSet = (abstracts.length() == 0 || abstracts.equals("auto")) ? null : QueryParams.hashes2Set(abstracts);

        // store accessing peer
        yacySeed remoteSeed;
        try {
            remoteSeed = yacySeed.genRemoteSeed(oseed, key, false, client);
        } catch (final IOException e) {
            yacyCore.log.logInfo("yacy.search: access with bad seed: " + e.getMessage());
            remoteSeed = null;
        }
        if (sb.peers == null) {
            yacyCore.log.logSevere("yacy.search: seed cache not initialized");
        } else {
            sb.peers.peerActions.peerArrival(remoteSeed, true);
        }

        // prepare search
        final HandleSet queryhashes = QueryParams.hashes2Set(query);
        final HandleSet excludehashes = (exclude.length() == 0) ? new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0) : QueryParams.hashes2Set(exclude);
        final long timestamp = System.currentTimeMillis();

      // prepare a search profile
        final RankingProfile rankingProfile = (profile.length() == 0) ? new RankingProfile(ContentDomain.contentdomParser(contentdom)) : new RankingProfile("", profile);

        // prepare an abstract result
        final StringBuilder indexabstract = new StringBuilder(6000);
        int indexabstractContainercount = 0;
        int joincount = 0;
        QueryParams theQuery = null;
        SearchEvent theSearch = null;
        ArrayList<WeakPriorityBlockingQueue.Element<ResultEntry>> accu = null;
        if (query.length() == 0 && abstractSet != null) {
            // this is _not_ a normal search, only a request for index abstracts
            final Segment indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
            theQuery = new QueryParams(
                    null,
                    abstractSet,
                    new HandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0),
                    null,
                    snippetPattern,
                    null,
                    maxdist,
                    prefer,
                    ContentDomain.contentdomParser(contentdom),
                    language,
                    "", // no navigation
                    CacheStrategy.CACHEONLY,
                    count,
                    0,
                    filter,
                    QueryParams.SEARCHDOM_LOCAL,
                    -1,
                    null,
                    false,
                    sitehash,
                    authorhash,
                    DigestURI.TLD_any_zone_filter,
                    client,
                    false,
                    indexSegment,
                    rankingProfile,
                    header.get(RequestHeader.USER_AGENT, ""),
                    false
                    );
            yacyCore.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");

            final long timer = System.currentTimeMillis();
            //final Map<byte[], ReferenceContainer<WordReference>>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls));
            final TreeMap<byte[], ReferenceContainer<WordReference>> incc = indexSegment.termIndex().searchConjunction(theQuery.queryHashes, QueryParams.hashes2Handles(urls));

            EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.COLLECTION, "", incc.size(), System.currentTimeMillis() - timer), false);
            if (incc != null) {
                final Iterator<Map.Entry<byte[], ReferenceContainer<WordReference>>> ci = incc.entrySet().iterator();
                Map.Entry<byte[], ReferenceContainer<WordReference>> entry;
                byte[] wordhash;
                while (ci.hasNext()) {
                    entry = ci.next();
                    wordhash = entry.getKey();
                    final ReferenceContainer<WordReference> container = entry.getValue();
                    indexabstractContainercount += container.size();
                    indexabstract.append("indexabstract.");
                    indexabstract.append(ASCII.String(wordhash));
                    indexabstract.append("=");
                    indexabstract.append(WordReferenceFactory.compressIndex(container, null, 1000).toString());
                    indexabstract.append(serverCore.CRLF_STRING);
                }
            }

            prop.put("indexcount", "");
            prop.put("joincount", "0");
            prop.put("references", "");

        } else {
            // retrieve index containers from search request
            theQuery = new QueryParams(
                    null,
                    queryhashes,
                    excludehashes,
                    null,
                    snippetPattern,
                    null,
                    maxdist,
                    prefer,
                    ContentDomain.contentdomParser(contentdom),
                    language,
                    "", // no navigation
                    CacheStrategy.CACHEONLY,
                    count,
                    0,
                    filter,
                    QueryParams.SEARCHDOM_LOCAL,
                    -1,
                    constraint,
                    false,
                    sitehash,
                    authorhash,
                    DigestURI.TLD_any_zone_filter,
                    client,
                    false,
                    sb.indexSegments.segment(Segments.Process.PUBLIC),
                    rankingProfile,
                    header.get(RequestHeader.USER_AGENT, ""),
                    false
                    );
            yacyCore.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
            yacyChannel.channels(yacyChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));

            // make event
            theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));

            // set statistic details of search result and find best result index set
View Full Code Here

Examples of de.anomic.yacy.yacySeed

            // open an editor page for the message
            // first ask if the other peer is online, and also what kind of document it accepts
            final Map<String, String> result = yacyClient.permissionMessage(sb.peers, hash);
            //System.out.println("DEBUG: permission request result = " + result.toString());
            String peerName;
            yacySeed targetPeer = null;
            if (hash.equals(sb.peers.mySeed().hash)) {
                peerName = sb.peers.mySeed().get(yacySeed.NAME,"nameless");
            } else {
                targetPeer = sb.peers.getConnected(hash);
                if (targetPeer == null)
                    peerName = "nameless";
                else
                    peerName = targetPeer.get(yacySeed.NAME,"nameless");
            }

            prop.putXML("mode_permission_peerName", peerName);
            final String response = (result == null) ? null : result.get("response");
            if (response == null || result == null) {
View Full Code Here

Examples of de.anomic.yacy.yacySeed

                prop.put("page_list_" + m + "_qph", qph);
               
                prop.put("page_list_" + m + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
                prop.putHTML("page_list_" + m + "_host", host);
                if (page == 5) {
                    final yacySeed remotepeer = sb.peers.lookupByIP(Domains.dnsResolve(host), true, true, true);
                    prop.putHTML("page_list_" + m + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName());
                }
                prop.putNum("page_list_" + m + "_count", handles.size());

                // next
                m++;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.