Package de.jetwick.es

Examples of de.jetwick.es.JetwickQuery


                logger.warn("Couldn't parse hits per page:" + hitsStr + " " + ex.getMessage());
            }
        }

        String idStr = parameters.getString("id");
        JetwickQuery q = null;

        if (idStr != null) {
            try {
                int index = idStr.lastIndexOf("/");
                if (index > 0 && index + 1 < idStr.length())
                    idStr = idStr.substring(index + 1);

                q = new TweetQuery(true).addFilterQuery(ElasticTweetSearch._ID + "tweet", Long.parseLong(idStr));
            } catch (Exception ignore) {
            }
        }

        if (q == null) {
            String originStr = parameters.getString("findOrigin");
            if (originStr != null) {
                logger.info("[stats] findOrigin from lastQuery:" + lastQuery);
                q = getTweetSearch().createFindOriginQuery(lastQuery, originStr, 3);
            }
        }

        String queryStr = parameters.getString("q");       
        if(Helper.isEmpty(queryStr))
            queryStr = parameters.getString("query");
        if (queryStr == null)
            queryStr = "";
       
        if(queryStr.contains("*")) {
            warn("Cannot process query with asterisk");           
            queryStr = "";
        }

        userName = "";
        if (q == null) {
            userName = parameters.getString("u");
            if (userName == null)
                userName = parameters.getString("user");

            if (userName == null)
                userName = "";
            q = new TweetQuery(queryStr);
            if (!Helper.isEmpty(userName))
                q.addUserFilter(userName);
        }

        String fromDateStr = parameters.getString("until");
        if (fromDateStr != null) {
            if (!fromDateStr.contains("T"))
                fromDateStr += "T00:00:00Z";

            q.addFilterQuery(ElasticTweetSearch.DATE, "[" + fromDateStr + " TO *]");
        }

        // front page/empty => sort against relevance
        // user search    => sort against latest date
        // other        => sort against retweets if no sort specified

        String sort = parameters.getString("sort");
        if ("retweets".equals(sort))
            q.setSort(ElasticTweetSearch.RT_COUNT, "desc");
        else if ("latest".equals(sort))
            q.setSort(ElasticTweetSearch.DATE, "desc");
        else if ("oldest".equals(sort))
            q.setSort(ElasticTweetSearch.DATE, "asc");
        else if ("relevance".equals(sort))
            q.setSort(ElasticTweetSearch.RELEVANCE, "desc");
        else {
            q.setSort(ElasticTweetSearch.RT_COUNT, "desc");

            if (!Helper.isEmpty(userName))
                q.setSort(ElasticTweetSearch.DATE, "desc");
        }

        // front page: avoid slow queries for matchall query and filter against latest tweets only
        if (queryStr.isEmpty() && q.getFilterQueries().isEmpty() && fromDateStr == null) {
            logger.info(addIP("[stats] q=''"));
            q.addLatestDateFilter(8);
            if (Helper.isEmpty(sort))
                q.setSort(ElasticTweetSearch.RELEVANCE, "desc");
        }

        String filter = parameters.getString("filter");
        if (Helper.isEmpty(userName) && !"none".equals(filter)) {
            q.addNoSpamFilter().addNoDupsFilter().addIsOriginalTweetFilter().addLatestDateFilter(24 * 7);
        }
        return q;
    }
View Full Code Here


        urlTrends = new UrlTrendPanel("urltrends") {

            @Override
            protected void onUrlClick(AjaxRequestTarget target, String name) {
                JetwickQuery q;
                if (lastQuery != null)
                    q = lastQuery;
                else
                    q = new TweetQuery(true);

                if (name == null) {
                    q.removeFilterQueries(ElasticTweetSearch.FIRST_URL_TITLE);
                } else
                    q.addFilterQuery(ElasticTweetSearch.FIRST_URL_TITLE, name);

                doSearch(q, 0, true);
                updateAfterAjax(target, false);
            }

            @Override
            protected void onDirectUrlClick(AjaxRequestTarget target, String name) {
                if (lastQuery == null || name == null || name.isEmpty())
                    return;

                TweetQuery q = new TweetQuery(true);
                q.addFilterQuery(ElasticTweetSearch.FIRST_URL_TITLE, name);
                try {
                    List<JTweet> tweets = getTweetSearch().collectObjects(getTweetSearch().query(q.setSize(1)));
                    if (tweets.size() > 0 && tweets.get(0).getUrlEntries().size() > 0) {
                        // TODO there could be more than 1 url!
                        UrlEntry entry = tweets.get(0).getUrlEntries().iterator().next();
                        getRequestCycle().setRequestTarget(new RedirectRequestTarget(entry.getResolvedUrl()));
                    }
                } catch (Exception ex) {
                    logger.error("Error while executing onDirectUrlClick", ex);
                }
            }
        };
        add(urlTrends.setOutputMarkupId(true));

        ssPanel = new SavedSearchPanel("savedSearches") {

            @Override
            public void onClick(AjaxRequestTarget target, long ssId) {
                String searchType = parameters.getString("search");
                if (searchType != null && !searchType.isEmpty() && !SearchBox.ALL.equals(searchType)) {
                    warn("Removed user filter when executing your saved search");
                    searchBox.setSearchType(SearchBox.ALL);
                }
                JUser user = getMySession().getUser();
                SavedSearch ss = user.getSavedSearch(ssId);
                if (ss != null) {
                    doSearch(ss.getQuery(), 0, true);
                    uindexProvider.get().save(user, true);
                }
                updateSSCounts(target);
                updateAfterAjax(target, true);
            }

            @Override
            public void onRemove(AjaxRequestTarget target, long ssId) {
                JUser user = getMySession().getUser();
                user.removeSavedSearch(ssId);
                uindexProvider.get().save(user, true);
                updateSSCounts(target);
            }

            @Override
            public void onSave(AjaxRequestTarget target, long ssId) {
                if (lastQuery == null)
                    return;

                SavedSearch ss = new SavedSearch(ssId, lastQuery);
                JUser user = getMySession().getUser();
                user.addSavedSearch(ss);
                uindexProvider.get().save(user, true);
                updateSSCounts(target);
            }

            @Override
            public void updateSSCounts(AjaxRequestTarget target) {
                try {
                    JUser user = getMySession().getUser();
                    if (user != null) {
                        StopWatch sw = new StopWatch().start();
                        update(getTweetSearch().updateSavedSearches(user.getSavedSearches()));
                        if (target != null)
                            target.addComponent(ssPanel);
                        logger.info("Updated saved search counts for " + user.getScreenName() + " " + sw.stop().getSeconds());
                    }
                } catch (Exception ex) {
                    logger.error("Error while searching in savedSearches", ex);
                }
            }

            @Override
            public String translate(long id) {
                SavedSearch ss = getMySession().getUser().getSavedSearch(id);
                return ss.getName();
            }
        };

        add(ssPanel.setOutputMarkupId(true));

        add(new UserPanel("userPanel", this) {

            @Override
            public void onLogout() {
                getMySession().logout(uindexProvider.get(), (WebResponse) getResponse(), true);
                setResponsePage(TweetSearchPage.class, parameters);
            }

            @Override
            public void updateAfterAjax(AjaxRequestTarget target) {
                TweetSearchPage.this.updateAfterAjax(target, false);
            }

            @Override
            public void onHomeline(AjaxRequestTarget target, String user) {
                searchBox.setSearchType(SearchBox.FRIENDS);
                doSearch(createFriendQuery(""), 0, false);
                TweetSearchPage.this.updateAfterAjax(target, true);
            }

            @Override
            public void onShowTweets(AjaxRequestTarget target, String userName) {
                searchBox.setSearchType(SearchBox.USER);
                doSearch(new TweetQuery(true).addUserFilter(userName).setSort(ElasticTweetSearch.DATE, "desc"), 0, false);
                TweetSearchPage.this.updateAfterAjax(target, true);
            }

            @Override
            protected Collection<String> getUserChoices(String input) {
                return getTweetSearch().getUserChoices(lastQuery, input);
            }
        });

        tagCloudPanel = new TagCloudPanel("tagcloud") {

            @Override
            protected void onTagClick(String name) {
                if (lastQuery != null) {
                    lastQuery.setQuery((lastQuery.getQuery() + " " + name).trim());
                    doSearch(lastQuery, 0, true);
                } else {
                    // never happens?
                    PageParameters pp = new PageParameters();
                    pp.add("q", name);
                    setResponsePage(TweetSearchPage.class, pp);
                }
            }

            @Override
            protected void onFindOriginClick(String tag) {
                PageParameters pp = new PageParameters();
                pp.add("findOrigin", tag);

                doSearch(createQuery(pp), 0, true);
                // this preserves parameters but cannot be context sensitive!
//                setResponsePage(TweetSearchPage.class, pp);
            }
        };
        add(tagCloudPanel.setOutputMarkupId(true));

        navigationPanel = new NavigationPanel("navigation", hitsPerPage) {

            @Override
            public void onPageChange(AjaxRequestTarget target, int page) {
                // this does not scroll to top:
//                doOldSearch(page);
//                updateAfterAjax(target);

                doOldSearch(page);
            }
        };
        add(navigationPanel.setOutputMarkupId(true));

        facetPanel = new FacetPanel("filterPanel") {

            @Override
            public void onRemoveAllFilter(AjaxRequestTarget target, String key) {
                if (lastQuery != null)
                    lastQuery.removeFilterQueries(key);
                else {
                    logger.error("last query cannot be null but was! ... when clicking on facets!?");
                    return;
                }

                doOldSearch(0);
                updateAfterAjax(target, false);
            }

            @Override
            public void onFilterChange(AjaxRequestTarget target, String key, Object val, Boolean selected) {
                if (lastQuery != null) {
                    if (selected == null)
                        lastQuery.removeFilterQuery(key, val);
                    else if (selected)
                        lastQuery.addFilterQuery("-" + key, val);
                    else
                        lastQuery.addFilterQuery(key, val);
                } else {
                    logger.error("last query cannot be null but was! ... when clicking on facets!?");
                    return;
                }

                doOldSearch(0);
                updateAfterAjax(target, false);
            }
        };
        add(facetPanel.setOutputMarkupId(true));

        dateFilterPanel = new JSDateFilter("dateFilter") {

            @Override
            protected void onFilterChange(AjaxRequestTarget target, String filter, Boolean selected) {
                if (lastQuery != null) {
                    if (selected == null) {
                        lastQuery.removeFilterQueries(filter);
                    } else if (selected) {
                        lastQuery.replaceFilterQuery(filter);
                    } else
                        lastQuery.reduceFilterQuery(filter);
                } else {
                    logger.error("last query cannot be null but was! ... when clicking on facets!?");
                    return;
                }

                doOldSearch(0);
                updateAfterAjax(target, false);
            }

            @Override
            protected boolean isAlreadyFiltered(String key, Object val) {
                if (lastQuery != null)
                    return lastQuery.containsFilter(key, val);

                return false;
            }

            @Override
            public String getFilterName(String key) {
                return facetPanel.getFilterName(key);
            }
        };
        add(dateFilterPanel.setOutputMarkupId(true));

        // TODO M2.1
        language = getWebRequestCycle().getWebRequest().getHttpServletRequest().getLocale().getLanguage();
        remoteHost = getWebRequestCycle().getWebRequest().getHttpServletRequest().getRemoteHost();
        resultsPanel = new ResultsPanel("results", language) {

            @Override
            public void onSortClicked(AjaxRequestTarget target, String sortKey, String sortVal) {
                if (lastQuery != null) {
                    lastQuery.setSort(sortKey, sortVal);
                    doSearch(lastQuery, 0, false);
                    updateAfterAjax(target, false);
                }
            }

            @Override
            public void onUserClick(String userName, String queryStr) {
                PageParameters p = new PageParameters();
                if (queryStr != null && !queryStr.isEmpty())
                    p.add("q", queryStr);
                if (userName != null) {
                    p.add("user", userName.trim());
                    searchBox.setSearchType(SearchBox.USER);
                }

                doSearch(createQuery(p), 0, true);
            }

            @Override
            public Collection<JTweet> onTweetClick(long id, boolean retweet) {
                logger.info("[stats] search replies of:" + id + " retweet:" + retweet);
                return getTweetSearch().searchReplies(id, retweet);
            }

            @Override
            protected void onRetweet(JTweet tweet, AjaxRequestTarget target) {
                if (getMySession().hasLoggedIn())
                    try {
                        getTwitterSearch().doRetweet(tweet.getTwitterId());
                        info("Retweeted " + tweet.getFromUser().getScreenName() + " by " + getTwitterSearch().getUser());
                    } catch (Exception ex) {
                        error("Cannot retweet " + tweet.getFromUser().getScreenName() + ". "
                                + "Problems with twitter. Please try again.");
                    }
                else
                    error("Please login.");
                updateAfterAjax(target, false);
            }

            @Override
            public void onFindSimilar(JTweet tweet, AjaxRequestTarget target) {
                JetwickQuery query = new SimilarTweetQuery(tweet, true);
                if (tweet.getTextTerms().size() == 0) {
                    warn("Try a different tweet. This tweet is too short.");
                    return;
                }

                logger.info("[stats] similar search:" + query.toString());
                doSearch(query, 0, false);
                updateAfterAjax(target, false);
            }

            @Override
            public Collection<JTweet> onInReplyOfClick(long id) {
                JTweet tw = getTweetSearch().findByTwitterId(id);
                logger.info("[stats] search tweet:" + id + " " + tw);
                if (tw != null)
                    return Arrays.asList(tw);
                else
                    return new ArrayList();
            }

            @Override
            public String getTweetsAsString() {
                if (lastQuery != null)
                    return twindexProvider.get().getTweetsAsString(lastQuery, "\t");

                return "";
            }

            @Override
            public void onHtmlExport() {
                if (lastQuery != null) {
                    PrinterPage printerPage = new PrinterPage();
                    List<JTweet> tweets = twindexProvider.get().searchTweets(lastQuery);
                    printerPage.setResults(tweets);
                    setResponsePage(printerPage);
                }
            }
        };
        add(resultsPanel.setOutputMarkupId(true));
//        add(wikiPanel = new WikipediaLazyLoadPanel("wikipanel"));

        String searchType = parameters.getString("search");
        String tmpUserName = null;
        boolean showSpacer = true;
        if (getMySession().hasLoggedIn()) {
            if (query.getQuery().isEmpty() && userName.isEmpty() && (searchType == null || searchType.isEmpty()))
                searchType = SearchBox.FRIENDS;
            tmpUserName = getMySession().getUser().getScreenName();
//            showSpacer = false;
        } else {
            ssPanel.setVisible(false);
            facetPanel.setVisible(false);
//            dateFilterPanel.setVisible(false);
            // so that the jetwick link on my twitter account works ;)           
            if (userName.isEmpty()) {
                tagCloudPanel.setVisible(false);

//                if (query.getQuery().isEmpty()) {
//                    resultsPanel.setVisible(false);
//                    navigationPanel.setVisible(false);
//                    showSpacer = false;
//                }
            }
        }
        dateFilterPanel.setVisible(false);

        searchBox = new SearchBox("searchbox", tmpUserName, searchType, showSpacer) {

            @Override
            protected Collection<String> getQueryChoices(String input) {
                return getTweetSearch().getQueryChoices(lastQuery, input);
            }

            @Override
            protected void onSelectionChange(AjaxRequestTarget target, String newValue) {
                if (lastQuery == null)
                    return;

                JetwickQuery tmpQ = lastQuery.getCopy().setQuery(newValue);
                tmpQ.removeFilterQueries(ElasticTweetSearch.DATE);
                doSearch(tmpQ, 0, false, true);
                updateAfterAjax(target, false);
            }

            @Override
View Full Code Here

        doSearch();
    }

    private void doSearch() {
        Collection<JUser> users = new LinkedHashSet<JUser>();       
        JetwickQuery query = new TweetQuery(queryString).
                setEscape(true).setSort(ElasticTweetSearch.RT_COUNT, "desc").
                addLatestDateFilter(24).addNoDupsFilter().addNoSpamFilter().
                addIsOriginalTweetFilter().setSize(10);
       
        long start = System.currentTimeMillis();
        long totalHits = 0;
        SearchResponse rsp = null;
        try {
            rsp = getTweetSearch().query(users, query);
            totalHits = rsp.getHits().getTotalHits();
            logger.info(addIP("[mstats] " + totalHits + " hits for: " + query.toString()));
        } catch (Exception ex) {
            logger.error("Error while searching " + query.toString(), ex);
        }

        String msg = "";
        if (totalHits > 0) {
            float time = (System.currentTimeMillis() - start) / 100.0f;
View Full Code Here

        Configuration cfg = injector.getInstance(Configuration.class);
        TwitterSearch twSearch = injector.getInstance(TwitterSearch.class);
        twSearch.initTwitter4JInstance(cfg.getTwitterSearchCredits().getToken(),
                cfg.getTwitterSearchCredits().getTokenSecret(), true);
        ElasticTweetSearch fromUserSearch = new ElasticTweetSearch(injector.getInstance(Configuration.class));
        JetwickQuery query = new TweetQuery().addFilterQuery(ElasticTweetSearch.UPDATE_DT, "[* TO *]");
        // TODO set facetlimit to 2000
        query.addFacetField("user").setSize(0);
        SearchResponse rsp = fromUserSearch.query(query);

        TermsFacet tf = (TermsFacet) rsp.getFacets().facet("user");
        logger.info("found: " + tf.entries().size() + " users with the specified criteria");
        int SLEEP = 30;
View Full Code Here

        // TODO send via RMI
    }

    public void fillFrom(final String fromUrl) {
        ElasticTweetSearch fromTweetSearch = new ElasticTweetSearch(fromUrl);
        JetwickQuery query = new TweetQuery();
        long maxPage = 1;
        int hitsPerPage = 300;
        Set<JUser> users = new LinkedHashSet<JUser>();
        Runnable optimizeOnExit = new Runnable() {

            @Override
            public void run() {
                userSearch.refresh();
                logger.info(userCounter + " users pushed to default tweet search from " + fromUrl);
            }
        };
        Runtime.getRuntime().addShutdownHook(new Thread(optimizeOnExit));

        for (int page = 0; page < maxPage; page++) {
            query.attachPagability(page, hitsPerPage);
            users.clear();

            SearchResponse rsp;
            try {
                rsp = fromTweetSearch.query(users, query);
            } catch (Exception ex) {
                logger.warn("Error while searching!", ex);
                continue;
            }
            if (maxPage == 1) {
                maxPage = rsp.getHits().getTotalHits() / hitsPerPage + 1;
                logger.info("Paging though query:" + query.toString());
                logger.info("Set numFound to " + rsp.getHits().getTotalHits());
            }

            for (JUser user : users) {
                userSearch.save(user, false);
View Full Code Here

    @Test
    public void testNormalSearch() throws Exception {
//        setUp();
        ElasticTweetSearch search = getInstance(ElasticTweetSearch.class);
        JetwickQuery query = new TweetQuery("timetabling");
        tester.startPage(new TweetSearchPage(query));
        tester.assertNoErrorMessage();

        verify(search).query(new LinkedHashSet<JUser>(), query);
    }
View Full Code Here

    @Test
    public void testWithDate() throws InterruptedException {
        TweetSearchPage page = getInstance(TweetSearchPage.class);
        PageParameters pp = new PageParameters();
        pp.put("until", "2011-02-01");
        JetwickQuery q = page.createQuery(pp);
        assertEquals("[2011-02-01T00:00:00Z TO *]", q.getFirstFilterQuery("dt"));

        pp = new PageParameters();
        pp.put("until", "2011-02-01T00:00:00Z");
        q = page.createQuery(pp);
        assertEquals("[2011-02-01T00:00:00Z TO *]", q.getFirstFilterQuery("dt"));
    }
View Full Code Here

            }
        }
    }

    public Collection<JTweet> search() {
        JetwickQuery query = new TweetQuery(). // should be not too old
                addFilterQuery(DATE, "[" + new MyDate().minusDays(1).toLocalString() + " TO *]").
                // should be high quality
                addFilterQuery(QUALITY, "[90 TO *]").
                // should be the first tweet with this content
                addFilterQuery(DUP_COUNT, 0).
                // only tweets which were already tweeted minRT-times
                addFilterQuery(RT_COUNT, "[" + minRT + " TO *]").
                // only original tweets
                addFilterQuery(IS_RT, false).
                // for english our spam + dup detection works ok
                addFilterQuery(ElasticTweetSearch.LANG, "(en OR de OR sp)").
                setSort(RT_COUNT, "desc").
                setSize(50);

        logger.info(query.toString());
        int TRIALS = 2;
        for (int trial = 0; trial < TRIALS; trial++) {
            try {
                return tweetSearch.collectObjects(tweetSearch.query(query));
            } catch (Exception ex) {
View Full Code Here

TOP

Related Classes of de.jetwick.es.JetwickQuery

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.