Examples of RSSMessage

net.yacy.cora.document.RSSMessage

Examples of net.yacy.cora.document.RSSMessage

        // index all selected items: description only
        if (rss != null && post.containsKey("indexSelectedItemContent")) {
            final RSSFeed feed = rss.getFeed();
            loop: for (final Map.Entry<String, String> entry: post.entrySet()) {
                if (entry.getValue().startsWith("mark_")) try {
                    final RSSMessage message = feed.getMessage(entry.getValue().substring(5));
                    final DigestURI messageurl = new DigestURI(message.getLink());
                    if (RSSLoader.indexTriggered.containsKey(messageurl.hash())) continue loop;
                    if (sb.urlExists(Segments.Process.LOCALCRAWLING, messageurl.hash()) != null) continue loop;
                    sb.addToIndex(messageurl, null, null);
                    RSSLoader.indexTriggered.insertIfAbsent(messageurl.hash(), new Date());
                } catch (final IOException e) {
                    Log.logException(e);
                } catch (final Failure e) {
                    Log.logException(e);
                }
            }
        }


        if (rss != null && post.containsKey("indexAllItemContent")) {
            record_api = true;
            final RSSFeed feed = rss.getFeed();
            RSSLoader.indexAllRssFeed(sb, url, feed);
        }


        if (record_api && rss != null && rss.getFeed() != null && rss.getFeed().getChannel() != null) {
            // record API action
            RSSLoader.recordAPI(sb, post.get(WorkTables.TABLE_API_COL_APICALL_PK, null), url, rss.getFeed(), repeat_time, repeat_unit);
        }


        // show items from rss
        if (rss != null) {
            prop.put("showitems", 1);
            final RSSFeed feed = rss.getFeed();
            final RSSMessage channel = feed.getChannel();
            prop.putHTML("showitems_title", channel == null ? "" : channel.getTitle());
            String author = channel == null ? "" : channel.getAuthor();
            if (author == null || author.length() == 0) author = channel == null ? "" : channel.getCopyright();
            Date pubDate = channel == null ? null : channel.getPubDate();
            prop.putHTML("showitems_author", author == null ? "" : author);
            prop.putHTML("showitems_description", channel == null ? "" : channel.getDescription());
            prop.putHTML("showitems_language", channel == null ? "" : channel.getLanguage());
            prop.putHTML("showitems_date", (pubDate == null) ? "" : DateFormat.getDateTimeInstance().format(pubDate));
            prop.putHTML("showitems_ttl", channel == null ? "" : channel.getTTL());
            prop.putHTML("showitems_docs", channel == null ? "" : channel.getDocs());


            int i = 0;
            for (final Hit item: feed) {
                try {
                    final DigestURI messageurl = new DigestURI(item.getLink());

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

            // filter out words that appear in bluelist
            theQuery.filterOut(Switchboard.blueList);


            // log
            Log.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + QueryParams.hashSet2hashString(theQuery.queryHashes) + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed");
            EventChannel.channels(EventChannel.LOCALSEARCH).addMessage(new RSSMessage("Local Search Request", theQuery.queryString, ""));
            final long timestamp = System.currentTimeMillis();


            // create a new search event
            if (SearchEventCache.getEvent(theQuery.id(false)) == null) {
                theQuery.setOffset(0); // in case that this is a new search, always start without a offset

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage


    @Override
    public void run() {
        searchSRURSS(this.results, this.urlBase, this.query, this.timeoutInit, this.maximumRecordsInit, this.verify, this.global, this.userAgent);
        int p = 1;
        RSSMessage message;
        try {
            while ((message = this.results.poll(this.timeoutInit, TimeUnit.MILLISECONDS)) != RSSMessage.POISON) {
                if (message == null) break;
                List<Integer> m = this.result.get(message.getLink());
                if (m == null) m = new ArrayList<Integer>();
                m.add(new Integer(p++));
                this.result.put(message, m);
            }
        } catch (final InterruptedException e) {

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

            final String userAgent) {
        final Thread job = new Thread() {
            @Override
            public void run() {
                int startRecord = 0;
                RSSMessage message;
                int maximumRecords = maximumRecordsInit;
                long timeout = timeoutInit;
                mainloop: while (timeout > 0 && maximumRecords > 0) {
                    final long st = System.currentTimeMillis();
                    RSSFeed feed;

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

        this.sb = sb;
        sb.setConfig("yacyStatus", "");


        // create a peer news channel
        final RSSFeed peernews = EventChannel.channels(EventChannel.PEERNEWS);
        peernews.addMessage(new RSSMessage("YaCy started", "", ""));


        // ensure that correct IP is used
        final String staticIP = sb.getConfig("staticIP", "");
        if (staticIP.length() != 0 && Seed.isProperIP(staticIP) == null) {
            serverCore.useStaticIP = true;

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

        if (peer == null) return false;
        final boolean res = connectPeer(peer, direct);
        if (res) {
            // perform all actions if peer is effective new
            processPeerArrival(peer);
            EventChannel.channels(EventChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " joined the network", "", ""));
        }
        return res;
    }

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

        synchronized (this.seedDB) {
            if (!this.seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { this.disconnects++; }
            peer.put("dct", Long.toString(System.currentTimeMillis()));
            this.seedDB.addDisconnected(peer); // update info
        }
        EventChannel.channels(EventChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " left the network", "", ""));
    }

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

        if (peer == null) return;
        // this is called only if the peer has junior status
        this.seedDB.addPotential(peer);
        // perform all actions
        processPeerArrival(peer);
        EventChannel.channels(EventChannel.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " sent me a ping", "", ""));
    }

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

                    document,
                    condenser,
                    searchEvent,
                    sourceName);
            final RSSFeed feed = EventChannel.channels(queueEntry.initiator() == null ? EventChannel.PROXY : Base64Order.enhancedCoder.equal(queueEntry.initiator(), ASCII.getBytes(this.peers.mySeed().hash)) ? EventChannel.LOCALINDEXING : EventChannel.REMOTEINDEXING);
            feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url().toNormalform(true, false)));
        } catch (final IOException e) {
            //if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase);
            addURLtoErrorDB(url, (referrerURL == null) ? null : referrerURL.hash(), queueEntry.initiator(), dc_title, FailCategory.FINAL_LOAD_CONTEXT, "error storing url: " + url.toNormalform(false, true) + "': process case=" + processCase + ", error = " + e.getMessage());
            return;
        }

View Full Code Here

Examples of net.yacy.cora.document.RSSMessage

        // in now contains a list of unknown hashes
        String uhss = in.get("unknownURL");
        if (uhss == null) {
            return "no unknownURL tag in response";
        }
        EventChannel.channels(EventChannel.DHTSEND).addMessage(new RSSMessage("Sent " + indexes.size() + " RWIs to " + targetSeed.getName(), "", targetSeed.hash));


        uhss = uhss.trim();
        if (uhss.length() == 0 || uhss.equals(",")) { return null; } // all url's known, we are ready here


        final String[] uhs = uhss.split(",");
        if (uhs.length == 0) { return null; } // all url's known


        // extract the urlCache from the result
        final URIMetadataRow[] urls = new URIMetadataRow[uhs.length];
        for (int i = 0; i < uhs.length; i++) {
            urls[i] = urlCache.get(ASCII.getBytes(uhs[i]));
            if (urls[i] == null) {
                if (Network.log.isFine()) Network.log.logFine("DEBUG transferIndex: requested url hash '" + uhs[i] + "', unknownURL='" + uhss + "'");
            }
        }


        in = transferURL(targetSeed, urls, gzipBody, timeout);


        if (in == null) {
            return "no connection from transferURL";
        }


        result = in.get("result");
        if (result == null) {
            return "no result from transferURL";
        }


        if (!result.equals("ok")) {
            return result;
        }
        EventChannel.channels(EventChannel.DHTSEND).addMessage(new RSSMessage("Sent " + uhs.length + " URLs to peer " + targetSeed.getName(), "", targetSeed.hash));


        return null;
    }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.