Package com.gentics.cr.lucene.indexaccessor

Examples of com.gentics.cr.lucene.indexaccessor.IndexAccessor

96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
  private synchronized void reIndex() throws IOException {
    UseCase ucReIndex = MonitorFactory.startUseCase("reIndex()");
   
    // build a dictionary (from the spell package)
    log.debug("Starting to reindex SYN index.");
    IndexAccessor synonymAccessor = synonym.getSynonymLocation().getAccessor();

    IndexWriter synonymWriter = synonymAccessor.getWriter();
    Collection<CRResolvableBean> objectsToIndex = null;
    try {
      if (rp == null) {
        throw new CRException("FATAL ERROR", "RequestProcessor not available");
      }

      // and get the current rule
      String rule = (String) config.get(RULE_KEY);
      if (rule == null) {
        rule = "";
      }
      if (rule.length() == 0 || rule == null) {
        rule = "1 == 1";
      }


      try {
        CRRequest req = new CRRequest();
        req.setRequestFilter(rule);
        status.setCurrentStatusString("SYN Get objects to update " + "in the index ...");
        objectsToIndex = getObjectsToUpdate(req, rp, true, null);
      } catch (Exception e) {
        log.error("ERROR while cleaning SYN index", e);
      }


      if (objectsToIndex == null) {
        log.debug("SYN Rule returned no objects to index. Skipping...");
        return;
      }

      status.setObjectCount(objectsToIndex.size());
      log.debug("SYN index job with " + objectsToIndex.size() + " objects to index.");

      String descriptorName = (String) config.get(DESCRIPTOR_NAME_KEY);
      String synonymName = (String) config.get(SYNONYM_NAME_KEY);


      status.setCurrentStatusString("Starting to index slices.");
      int objCount = 0;
      try {
        for (Iterator<CRResolvableBean> iterator = objectsToIndex.iterator(); iterator.hasNext();) {
          CRResolvableBean bean = iterator.next();
          iterator.remove();
          objCount++;
          String descriptorValue = bean.getString(descriptorName);
          String synonymValue = bean.getString(synonymName);
          if (descriptorValue != null && synonymValue != null) {
            descriptorValue = descriptorValue.toLowerCase();
            if (synonymValue != null) {
              synonymValue = synonymValue.toLowerCase();
            }
            Document doc = new Document();
            doc.add(new Field("Deskriptor", descriptorValue, Field.Store.YES, Field.Index.NOT_ANALYZED));
            doc.add(new Field("Synonym", synonymValue, Field.Store.YES, Field.Index.NOT_ANALYZED));
            synonymWriter.addDocument(doc);
            log.debug("WRITE SYN " + objCount + " " + descriptorValue + " " + synonymValue);
            synonymWriter.commit();
            log.debug("Number of actual Synonym: " + synonymWriter.numDocs());
          }
        }
      } finally {
        // if documents where added to the index create a reopen file and
        // optimize the writer
        log.debug("Number of indexed Synonyms finished: " + synonymWriter.numDocs());
        synonymAccessor.release(synonymWriter);
      }
 
      log.debug("Finished reindexing synonym index.");
      ucReIndex.stop();
    } catch (Exception e) {
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
    GenericConfiguration autoConf = (GenericConfiguration) config.get("synonymlocation");
    LuceneIndexLocation synonymLocation = LuceneIndexLocation
        .getIndexLocation(new CRConfigUtil(autoConf, "synonymlocation"));
   
   
    IndexAccessor ia = synonymLocation.getAccessor();
    Searcher synonymSearcher = ia.getPrioritizedSearcher();
    IndexReader synonymReader = ia.getReader(false);
   
   
    try {
      HashSet<String> searchedTerms = new HashSet<String>();
     
      //get all searched Terms out of query
      for (int i = 0; i < searchedAttributes.length; i++) {
        String subquery = query;
        while (subquery.indexOf(searchedAttributes[i] + ":") > 0) {
          subquery = subquery.substring(subquery.indexOf(searchedAttributes[i] + ":") + searchedAttributes[i].length() + 1);
          int substringUntil = -1;
          int pos1 = subquery.indexOf(")");
          int pos2 = subquery.indexOf(" ");
          if (pos1 != -1) {
            substringUntil = pos1;
          }
          if (pos2 != -1) {
            substringUntil = pos2;
          }
          if (pos1 != -1 && pos2 != -1) {
            if (pos1 <= pos2) {
              substringUntil = pos1;
            }
            else {
              substringUntil = pos2;
            }
          }
          if (substringUntil == -1) {
            substringUntil = subquery.length();
          }
          String addtoSet = subquery.substring(0, substringUntil).replaceAll("\\*", "").replaceAll("\\(", "").replaceAll("\\)", "");
          searchedTerms.add(addtoSet);
          subquery = subquery.substring(substringUntil);
        }
       
      }
     
      //create the query-String for synonym-Index with all searchedTerms
      Iterator<String> it = searchedTerms.iterator();
      String queryString = "";
      while (it.hasNext()) {
        queryString = queryString + "Deskriptor:" + it.next() + " ";
      }
      Query querySynonym;
      try {
        querySynonym = super.parse(queryString);
      } catch (ParseException e) {
        e.printStackTrace();
        log.debug("Error while parsing query for accessing the synonym Index.", e);
        return query;
      }
     
      //get all Synonyms from SynonymIndex and add them to searchQuery
      log.debug("Synonym Query String: " + querySynonym.toString());
      TopDocs docs = synonymSearcher.search(querySynonym, MAX_SYNONYMS);
      log.debug("total found synonyms: " + docs.totalHits); 
      for (ScoreDoc doc : docs.scoreDocs) {
        Document d = synonymReader.document(doc.doc);
        for (int i = 0; i < searchedAttributes.length; i++) {
          query = query + " OR " + searchedAttributes[i] + ":" + d.get("Synonym");
        }
      }
    } finally {
      ia.release(synonymSearcher);
      ia.release(synonymReader, false);
    }

    return query;
  }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
  @Override
  protected final void indexCR(final IndexLocation indexLocation, final CRConfigUtil config) throws CRException {

    log.debug("Starting to clear index.");
    LuceneIndexLocation autocompleteLocation = autocompleter.getAutocompleteLocation();
    IndexAccessor ia = autocompleteLocation.getAccessor();
    IndexWriter writer = null;
    try {
      writer = ia.getWriter();
      writer.deleteAll();
      autocompleteLocation.resetIndexJobCreationTimes();
      autocompleteLocation.createReopenFile();
    } catch (IOException e) {
      log.error("Could not clear index", e);
    } finally {
      ia.release(writer);
    }
    log.debug("Finished clearing index.");

  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
    LuceneIndexLocation source = this.autocompleter.getSource();
    LuceneIndexLocation autocompleteLocation = this.autocompleter.getAutocompleteLocation();
    String autocompletefield = this.autocompleter.getAutocompletefield();

    IndexAccessor sia = source.getAccessor();
    IndexReader sourceReader = sia.getReader(false);
    LuceneDictionary dict = new LuceneDictionary(sourceReader, autocompletefield);
    IndexAccessor aia = autocompleteLocation.getAccessor();
    // IndexReader reader = aia.getReader(false);
    IndexWriter writer = aia.getWriter();

    try {
      writer.setMergeFactor(300);
      writer.setMaxBufferedDocs(150);
      // go through every word, storing the original word (incl. n-grams)
      // and the number of times it occurs
      // CREATE WORD LIST FROM SOURCE INDEX
      Map<String, Integer> wordsMap = new HashMap<String, Integer>();
      Iterator<String> iter = (Iterator<String>) dict.getWordsIterator();
      while (iter.hasNext()) {
        String word = iter.next();
        int len = word.length();
        if (len < 3) {
          continue; // too short we bail but "too long" is fine...
        }
        if (wordsMap.containsKey(word)) {
          throw new IllegalStateException("Lucene returned a bad word list");
        } else {
          // use the number of documents this word appears in
          wordsMap.put(word, sourceReader.docFreq(new Term(autocompletefield, word)));
        }
      }
      // DELETE OLD OBJECTS FROM INDEX
      writer.deleteAll();

      // UPDATE DOCUMENTS IN AUTOCOMPLETE INDEX
      for (String word : wordsMap.keySet()) {
        // ok index the word
        Document doc = new Document();
        doc.add(new Field(SOURCE_WORD_FIELD, word, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); // orig term
        doc.add(new Field(GRAMMED_WORDS_FIELD, word, Field.Store.YES, Field.Index.ANALYZED)); // grammed
        doc.add(new Field(COUNT_FIELD, Integer.toString(wordsMap.get(word)), Field.Store.YES,
            Field.Index.NOT_ANALYZED_NO_NORMS)); // count
        writer.addDocument(doc);
      }
      writer.optimize();
      autocompleteLocation.createReopenFile();
    } finally {

      sia.release(sourceReader, false);
      // close writer

      aia.release(writer);
      // aia.release(reader,false);
    }
    log.debug("Finished reindexing autocomplete index.");
    ucReIndex.stop();
  }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
    singleConfig3 = new CRConfigUtil(sc3, "sc3");
  }

  public void testLuceneSingleIndexLocation() {
    LuceneIndexLocation singleLoc1 = LuceneIndexLocation.getIndexLocation(singleConfig1);
    IndexAccessor ia1 = singleLoc1.getAccessor();

    LuceneIndexLocation singleLoc2 = LuceneIndexLocation.getIndexLocation(singleConfig2);
    IndexAccessor ia2 = singleLoc2.getAccessor();

    LuceneIndexLocation singleLoc3 = LuceneIndexLocation.getIndexLocation(singleConfig3);
    IndexAccessor ia3 = singleLoc3.getAccessor();

    LuceneIndexLocation singleLoc4 = LuceneIndexLocation.getIndexLocation(singleConfig3);

    assertEquals("The accessors are not the same.", ia1, ia2);
    assertEquals("The accessors of different locations are the same.", ia1 == ia3, false);
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
    if (autocompletereopenupdate || useAutocompleteIndexExtension) {
      checkForUpdate();
    }

    IndexAccessor ia = autocompleteLocation.getAccessor();
    Searcher autoCompleteSearcher = ia.getPrioritizedSearcher();
    IndexReader autoCompleteReader = ia.getReader(false);
    try {
      Query query = new TermQuery(new Term(GRAMMED_WORDS_FIELD, term));
      Sort sort = new Sort(new SortField(COUNT_FIELD, SortField.LONG, true));
      TopDocs docs = autoCompleteSearcher.search(query, null, 5, sort);
      int id = 1;
      for (ScoreDoc doc : docs.scoreDocs) {
        CRResolvableBean bean = new CRResolvableBean(id++);
        Document d = autoCompleteReader.document(doc.doc);
        bean.set(SOURCE_WORD_FIELD, d.get(SOURCE_WORD_FIELD));
        bean.set(COUNT_FIELD, d.get(COUNT_FIELD));
        result.add(bean);
      }
    } finally {
      ia.release(autoCompleteSearcher);
      ia.release(autoCompleteReader, false);
    }

    return result;
  }
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
  private void checkForUpdate() {

    // Use the old checkForUpdate logic for backward compatibility if the
    // AutocompleteIndexExtension is not used
    if (!useAutocompleteIndexExtension) {
      IndexAccessor ia = source.getAccessor();
      boolean reopened = false;
      try {
        IndexReader reader = ia.getReader(false);
        Directory dir = reader.directory();
        try {
          if (dir.fileExists("reopen")) {
            long lastupdate = dir.fileModified("reopen");
            if (lastupdate != lastupdatestored) {
              reopened = true;
              lastupdatestored = lastupdate;
            }
          }
        } finally {
          ia.release(reader, false);
        }
        if (reopened) {

          reIndex();

        }
      } catch (IOException e) {
        log.debug("Could not reIndex autocomplete index.", e);
      }
    } else {
      // the new checkForUpdate Logic only calls reopenCheck on the
      // IndexLocation
      IndexAccessor ia = autocompleteLocation.getAccessor();
      autocompleteLocation.reopenCheck(ia, null);
    }
  }
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
  @Deprecated
  private synchronized void reIndex() throws IOException {
    UseCase ucReIndex = MonitorFactory.startUseCase("reIndex()");
    // build a dictionary (from the spell package)
    log.debug("Starting to reindex autocomplete index.");
    IndexAccessor sia = this.source.getAccessor();
    IndexReader sourceReader = sia.getReader(false);
    LuceneDictionary dict = new LuceneDictionary(sourceReader, this.autocompletefield);
    IndexAccessor aia = this.autocompleteLocation.getAccessor();
    // IndexReader reader = aia.getReader(false);
    IndexWriter writer = aia.getWriter();

    try {
      writer.setMergeFactor(300);
      writer.setMaxBufferedDocs(150);
      // go through every word, storing the original word (incl. n-grams)
      // and the number of times it occurs
      // CREATE WORD LIST FROM SOURCE INDEX
      Map<String, Integer> wordsMap = new HashMap<String, Integer>();
      Iterator<String> iter = (Iterator<String>) dict.getWordsIterator();
      while (iter.hasNext()) {
        String word = iter.next();
        int len = word.length();
        if (len < 3) {
          continue; // too short we bail but "too long" is fine...
        }
        if (wordsMap.containsKey(word)) {
          throw new IllegalStateException("Lucene returned a bad word list");
        } else {
          // use the number of documents this word appears in
          wordsMap.put(word, sourceReader.docFreq(new Term(autocompletefield, word)));
        }
      }
      // DELETE OLD OBJECTS FROM INDEX
      writer.deleteAll();

      // UPDATE DOCUMENTS IN AUTOCOMPLETE INDEX
      for (String word : wordsMap.keySet()) {
        // ok index the word
        Document doc = new Document();
        doc.add(new Field(SOURCE_WORD_FIELD, word, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); // orig term
        doc.add(new Field(GRAMMED_WORDS_FIELD, word, Field.Store.YES, Field.Index.ANALYZED)); // grammed
        doc.add(new Field(COUNT_FIELD, Integer.toString(wordsMap.get(word)), Field.Store.YES,
            Field.Index.NOT_ANALYZED_NO_NORMS)); // count
        writer.addDocument(doc);
      }
      writer.optimize();
    } finally {

      sia.release(sourceReader, false);
      // close writer

      aia.release(writer);
      // aia.release(reader,false);
    }
    autocompleteLocation.createReopenFile();
    log.debug("Finished reindexing autocomplete index.");
    ucReIndex.stop();
View Full Code Here
TOP

Related Classes of com.gentics.cr.lucene.indexaccessor.IndexAccessor

  • com.gentics.cr.lucene.autocomplete.AutocompleteIndexDeleteJob
  • com.gentics.cr.lucene.autocomplete.AutocompleteIndexJob
  • com.gentics.cr.lucene.autocomplete.Autocompleter
  • com.gentics.cr.lucene.didyoumean.DidyoumeanIndexJob
  • com.gentics.cr.lucene.indexer.index.CRLuceneDeleteJob
  • com.gentics.cr.lucene.indexer.index.CRLuceneIndexJob
  • com.gentics.cr.lucene.indexer.index.CRLuceneOptimizeJob
  • com.gentics.cr.lucene.indexer.index.LuceneIndexLocation
  • com.gentics.cr.lucene.indexer.index.LuceneIndexLocationTest
  • com.gentics.cr.lucene.indexer.index.LuceneMultiIndexLocation
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.