Examples of RSSMessage

net.yacy.cora.document.RSSMessage

Examples of net.yacy.cora.document.RSSMessage

        this.sb = sb;
        sb.setConfig("yacyStatus", "");
        
        // create a peer news channel
        final RSSFeed peernews = yacyChannel.channels(yacyChannel.PEERNEWS);
        peernews.addMessage(new RSSMessage("YaCy started", "", ""));
        
        // ensure that correct IP is used
        final String staticIP = sb.getConfig("staticIP", "");
        if (staticIP.length() != 0 && yacySeed.isProperIP(staticIP) == null) {
            serverCore.useStaticIP = true;

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

        // in now contains a list of unknown hashes
        String uhss = in.get("unknownURL");
        if (uhss == null) {
            return "no unknownURL tag in response";
        }
        yacyChannel.channels(yacyChannel.DHTSEND).addMessage(new RSSMessage("Sent " + indexes.size() + " RWIs to " + targetSeed.getName(), "", targetSeed.hash));


        uhss = uhss.trim();
        if (uhss.length() == 0 || uhss.equals(",")) { return null; } // all url's known, we are ready here


        final String[] uhs = uhss.split(",");
        if (uhs.length == 0) { return null; } // all url's known


        // extract the urlCache from the result
        final URIMetadataRow[] urls = new URIMetadataRow[uhs.length];
        for (int i = 0; i < uhs.length; i++) {
            urls[i] = urlCache.get(ASCII.getBytes(uhs[i]));
            if (urls[i] == null) {
                if (yacyCore.log.isFine()) yacyCore.log.logFine("DEBUG transferIndex: requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'");
            }
        }


        in = transferURL(targetSeed, urls, gzipBody, timeout);


        if (in == null) {
            return "no connection from transferURL";
        }


        result = in.get("result");
        if (result == null) {
            return "no result from transferURL";
        }


        if (!result.equals("ok")) {
            return result;
        }
        yacyChannel.channels(yacyChannel.DHTSEND).addMessage(new RSSMessage("Sent " + uhs.length + " URLs to peer " + targetSeed.getName(), "", targetSeed.hash));


        return null;
    }

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

                    document,
                    condenser,
                    searchEvent,
                    sourceName);
            final RSSFeed feed = yacyChannel.channels(queueEntry.initiator() == null ? yacyChannel.PROXY : Base64Order.enhancedCoder.equal(queueEntry.initiator(), ASCII.getBytes(this.peers.mySeed().hash)) ? yacyChannel.LOCALINDEXING : yacyChannel.REMOTEINDEXING);
            feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url().toNormalform(true, false)));
        } catch (final IOException e) {
            //if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase);
            addURLtoErrorDB(queueEntry.url(), (referrerURL == null) ? null : referrerURL.hash(), queueEntry.initiator(), dc_title, FailCategory.FINAL_LOAD_CONTEXT, "error storing url: " + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase + ", error = " + e.getMessage());
            return;
        }

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage


    @Override
    public void run() {
        searchSRURSS(this.results, this.urlBase, this.query, this.timeoutInit, this.maximumRecordsInit, this.verify, this.global, this.userAgent);
        int p = 1;
        RSSMessage message;
        try {
            while ((message = this.results.poll(this.timeoutInit, TimeUnit.MILLISECONDS)) != RSSMessage.POISON) {
                if (message == null) break;
                List<Integer> m = this.result.get(message.getLink());
                if (m == null) m = new ArrayList<Integer>();
                m.add(new Integer(p++));
                this.result.put(message, m);
            }
        } catch (final InterruptedException e) {

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

            final String userAgent) {
        final Thread job = new Thread() {
            @Override
            public void run() {
                int startRecord = 0;
                RSSMessage message;
                int maximumRecords = maximumRecordsInit;
                long timeout = timeoutInit;
                mainloop: while (timeout > 0 && maximumRecords > 0) {
                    final long st = System.currentTimeMillis();
                    RSSFeed feed;

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

            } else {
                final String firstHash = wordhashes.get(0);
                final String lastHash = wordhashes.get(wordhashes.size() - 1);
                final long avdist = (FlatWordPartitionScheme.std.dhtDistance(firstHash.getBytes(), null, sb.peers.mySeed()) + FlatWordPartitionScheme.std.dhtDistance(lastHash.getBytes(), null, sb.peers.mySeed())) / 2;
                sb.getLog().logInfo("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + receivedURL + " URLs from " + otherPeerName);
                EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + receivedURL + " URLs from " + otherPeerName, "", otherPeer.hash));
            }
            result = "ok";


            pause = (int) (sb.indexSegments.termIndex(Segments.Process.DHTIN).getBufferSize() * 20000 / sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 100000)); // estimation of necessary pause time
        }

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage


            // read the channel
            final RSSFeed feed = EventChannel.channels(channel);
            if (feed == null || feed.isEmpty()) continue channelIteration;


            RSSMessage message = feed.getChannel();
            if (message != null) {
                prop.putXML("channel_title", message.getTitle());
                prop.putXML("channel_description", message.getDescription());
                prop.put("channel_pubDate", message.getPubDate());
            }
            while (messageMaxCount > 0 && !feed.isEmpty()) {
                message = feed.pollMessage();
                if (message == null) continue;


                // create RSS entry
                prop.putXML("item_" + messageCount + "_title", channelName + ": " + message.getTitle());
                prop.putXML("item_" + messageCount + "_description", message.getDescription());
                prop.putXML("item_" + messageCount + "_link", message.getLink());
                prop.put("item_" + messageCount + "_pubDate", message.getPubDate());
                prop.put("item_" + messageCount + "_guid", message.getGuid());
                messageCount++;
                messageMaxCount--;
            }
            if (messageMaxCount == 0) break channelIteration;
        }

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

                final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getConfig("port", "8090") + "/yacysearch.rss";
                final BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>();
                SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, null, false, null);


                // take the results and compute some locations
                RSSMessage message;
                loop: while ((message = results.poll(maximumTime, TimeUnit.MILLISECONDS)) != RSSMessage.POISON) {


                    // find all associated locations
                    final Set<Location> locations = new HashSet<Location>();
                    final StringBuilder words = new StringBuilder(120);
                    if (search_title) words.append(message.getTitle().trim()).append(space);
                    if (search_publisher) words.append(message.getCopyright().trim()).append(space);
                    if (search_creator) words.append(message.getAuthor().trim()).append(space);
                    String subject = "";
                    assert message != null;
                    assert message.getSubject() != null;
                    for (final String s: message.getSubject()) subject += s.trim() + space;
                    if (search_subject) words.append(subject).append(space);
                    final String[] wordlist = words.toString().trim().split(space);
                    for (final String word: wordlist) if (word.length() >= 3) locations.addAll(LibraryProvider.geoLoc.find(word, true));
                    for (int i = 0; i < wordlist.length - 1; i++) locations.addAll(LibraryProvider.geoLoc.find(wordlist[i] + space + wordlist[i + 1], true));
                    for (int i = 0; i < wordlist.length - 2; i++) locations.addAll(LibraryProvider.geoLoc.find(wordlist[i] + space + wordlist[i + 1] + space + wordlist[i + 2], true));


                    // add locations from metatag
                    if (metatag) {
                        if (message.getLat() != 0.0f && message.getLon() != 0.0f) {
                            locations.add(new Location(message.getLon(), message.getLat(), message.getTitle().trim()));
                        }
                    }


                    for (final Location location: locations) {
                        // write for all locations a point to this message
                        prop.put("kml_placemark_" + placemarkCounter + "_location", location.getName());
                        prop.put("kml_placemark_" + placemarkCounter + "_name", message.getTitle());
                        prop.put("kml_placemark_" + placemarkCounter + "_author", message.getAuthor());
                        prop.put("kml_placemark_" + placemarkCounter + "_copyright", message.getCopyright());
                        prop.put("kml_placemark_" + placemarkCounter + "_subject", subject.trim());
                        prop.put("kml_placemark_" + placemarkCounter + "_description", message.getDescription());
                        prop.put("kml_placemark_" + placemarkCounter + "_date", message.getPubDate());
                        prop.putXML("kml_placemark_" + placemarkCounter + "_url", message.getLink());
                        prop.put("kml_placemark_" + placemarkCounter + "_pointname", location.getName());
                        prop.put("kml_placemark_" + placemarkCounter + "_lon", location.lon());
                        prop.put("kml_placemark_" + placemarkCounter + "_lat", location.lat());
                        placemarkCounter++;
                        if (placemarkCounter >= maximumRecords) break loop;

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage


            // return rewrite properties
            final int more = sb.indexSegments.urlMetadata(Segments.Process.DHTIN).size() - sizeBefore;
            doublevalues = Integer.toString(received - more);
            Network.log.logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");
            EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash));
            if ((received - more) > 0) Network.log.logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
            result = "ok";
        }


        prop.put("double", doublevalues);

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

                    rankingProfile,
                    header.get(RequestHeader.USER_AGENT, ""),
                    false
                    );
            Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
            EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));


            // make event
            theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));


            // set statistic details of search result and find best result index set

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.