Package org.apache.lucene.document

Examples of org.apache.lucene.document.Fieldable


      // Also absorb any changes to fields we had already
      // seen before (eg suddenly turning on norms or
      // vectors, etc.):

      for(int i=0;i<numDocFields;i++) {
        Fieldable field = (Fieldable) docFields.get(i);

        FieldInfo fi = fieldInfos.add(field.name(), field.isIndexed(), field.isTermVectorStored(),
                                      field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
                                      field.getOmitNorms(), false);
        if (fi.isIndexed && !fi.omitNorms) {
          // Maybe grow our buffered norms
          if (norms.length <= fi.number) {
            int newSize = (int) ((1+fi.number)*1.25);
            BufferedNorms[] newNorms = new BufferedNorms[newSize];
            System.arraycopy(norms, 0, newNorms, 0, norms.length);
            norms = newNorms;
          }
         
          if (norms[fi.number] == null)
            norms[fi.number] = new BufferedNorms();

          hasNorms = true;
        }

        // Make sure we have a FieldData allocated
        int hashPos = fi.name.hashCode() & fieldDataHashMask;
        FieldData fp = fieldDataHash[hashPos];
        while(fp != null && !fp.fieldInfo.name.equals(fi.name))
          fp = fp.next;

        if (fp == null) {

          fp = new FieldData(fi);
          fp.next = fieldDataHash[hashPos];
          fieldDataHash[hashPos] = fp;

          if (numAllFieldData == allFieldDataArray.length) {
            int newSize = (int) (allFieldDataArray.length*1.5);
            int newHashSize = fieldDataHash.length*2;

            FieldData newArray[] = new FieldData[newSize];
            FieldData newHashArray[] = new FieldData[newHashSize];
            System.arraycopy(allFieldDataArray, 0, newArray, 0, numAllFieldData);

            // Rehash
            fieldDataHashMask = newSize-1;
            for(int j=0;j<fieldDataHash.length;j++) {
              FieldData fp0 = fieldDataHash[j];
              while(fp0 != null) {
                hashPos = fp0.fieldInfo.name.hashCode() & fieldDataHashMask;
                FieldData nextFP0 = fp0.next;
                fp0.next = newHashArray[hashPos];
                newHashArray[hashPos] = fp0;
                fp0 = nextFP0;
              }
            }

            allFieldDataArray = newArray;
            fieldDataHash = newHashArray;
          }
          allFieldDataArray[numAllFieldData++] = fp;
        } else {
          assert fp.fieldInfo == fi;
        }

        if (thisFieldGen != fp.lastGen) {

          // First time we're seeing this field for this doc
          fp.lastGen = thisFieldGen;
          fp.fieldCount = 0;
          fp.doVectors = fp.doVectorPositions = fp.doVectorOffsets = false;
          fp.doNorms = fi.isIndexed && !fi.omitNorms;

          if (numFieldData == fieldDataArray.length) {
            int newSize = fieldDataArray.length*2;
            FieldData newArray[] = new FieldData[newSize];
            System.arraycopy(fieldDataArray, 0, newArray, 0, numFieldData);
            fieldDataArray = newArray;

          }
          fieldDataArray[numFieldData++] = fp;
        }

        if (field.isTermVectorStored()) {
          if (!fp.doVectors && numVectorFields++ == vectorFieldPointers.length) {
            final int newSize = (int) (numVectorFields*1.5);
            vectorFieldPointers = new long[newSize];
            vectorFieldNumbers = new int[newSize];
          }
          fp.doVectors = true;
          docHasVectors = true;

          fp.doVectorPositions |= field.isStorePositionWithTermVector();
          fp.doVectorOffsets |= field.isStoreOffsetWithTermVector();
        }

        if (fp.fieldCount == fp.docFields.length) {
          Fieldable[] newArray = new Fieldable[fp.docFields.length*2];
          System.arraycopy(fp.docFields, 0, newArray, 0, fp.docFields.length);
          fp.docFields = newArray;
        }

        // Lazily allocate arrays for postings:
        if (field.isIndexed() && fp.postingsHash == null)
          fp.initPostingArrays();

        fp.docFields[fp.fieldCount++] = field;
      }

View Full Code Here


        // Walk through all occurrences in this doc for this
        // field:
        try {
          for(int j=0;j<limit;j++) {
            Fieldable field = docFieldsFinal[j];

            if (field.isIndexed())
              invertField(field, analyzer, maxFieldLength);

            if (field.isStored()) {
              numStoredFields++;
              boolean success = false;
              try {
                localFieldsWriter.writeField(fieldInfo, field);
                success = true;
View Full Code Here

    List<Fieldable> docFields = doc.getFields();
    String[] fields = new String[docFields.size()];
    String[] values = new String[docFields.size()];
    for (int i = 0; i < docFields.size(); i++) {
      Fieldable field = docFields.get(i);
      fields[i] = field.name();
      values[i] = field.stringValue();
    }

    return new HitDetails(fields, values);
  }
View Full Code Here

            Document copy = new Document();
            // mark the document that reindexing is required
            copy.add(new Field(FieldNames.REINDEXING_REQUIRED, "",
                    Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
            for (Fieldable f : (List<Fieldable>) doc.getFields()) {
                Fieldable field = null;
                Field.TermVector tv = getTermVectorParameter(f);
                Field.Store stored = getStoreParameter(f);
                Field.Index indexed = getIndexParameter(f);
                if (f instanceof LazyTextExtractorField || f.readerValue() != null) {
                    // replace all readers with empty string reader
                    field = new Field(f.name(), new StringReader(""), tv);
                } else if (f.stringValue() != null) {
                    field = new Field(f.name(), f.stringValue(),
                            stored, indexed, tv);
                } else if (f.isBinary()) {
                    field = new Field(f.name(), f.binaryValue(), stored);
                }
                if (field != null) {
                    field.setOmitNorms(f.getOmitNorms());
                    copy.add(field);
                }
            }
            // schedule the original document for later indexing
            Document existing = indexingQueue.addDocument(doc);
View Full Code Here

     *
     * @param old the document to dispose.
     */
    public static void disposeDocument(Document old) {
        for (Object o : old.getFields()) {
            Fieldable f = (Fieldable) o;
            try {
                if (f.readerValue() != null) {
                    f.readerValue().close();
                } else if (f instanceof LazyTextExtractorField) {
                    LazyTextExtractorField field = (LazyTextExtractorField) f;
                    field.dispose();
                }
            } catch (IOException ex) {
View Full Code Here

     * @return <code>true</code> if the document is ready; <code>false</code>
     *         otherwise.
     */
    public static boolean isDocumentReady(Document doc) {
        for (Object o : doc.getFields()) {
            Fieldable f = (Fieldable) o;
            if (f instanceof LazyTextExtractorField) {
                LazyTextExtractorField field = (LazyTextExtractorField) f;
                if (!field.isExtractorFinished()) {
                    return false;
                }
View Full Code Here

        indexStream.writeLong(fieldsStream.getFilePointer());

        int storedCount = 0;
        Iterator fieldIterator = doc.getFields().iterator();
        while (fieldIterator.hasNext()) {
            Fieldable field = (Fieldable) fieldIterator.next();
            if (field.isStored())
                storedCount++;
        }
        fieldsStream.writeVInt(storedCount);

        fieldIterator = doc.getFields().iterator();
        while (fieldIterator.hasNext()) {
            Fieldable field = (Fieldable) fieldIterator.next();
            if (field.isStored())
              writeField(fieldInfos.fieldInfo(field.name()), field);
        }
    }
View Full Code Here

        FieldSelector sel = new SetBasedFieldSelector(new HashSet(), lazyFields);
        doc = reader.document(reader.maxDoc() - 1, sel);
        Fieldable[] fieldables = doc.getFieldables("bin1");
        assertNotNull(fieldables);
        assertEquals(1, fieldables.length);
        Fieldable fb1 = fieldables[0];
        assertTrue(fb1.isBinary());
        assertEquals(bin.length, fb1.getBinaryLength());
        data1 = fb1.getBinaryValue();
        assertEquals(bin.length, fb1.getBinaryLength());
        for (int i = 0; i < bin.length; i++) {
          assertEquals(bin[i], data1[i + fb1.getBinaryOffset()]);
        }
        reader.close();
        // force optimize

View Full Code Here

    doVectors = false;
    doVectorPositions = false;
    doVectorOffsets = false;

    for(int i=0;i<count;i++) {
      Fieldable field = fields[i];
      if (field.isIndexed() && field.isTermVectorStored()) {
        doVectors = true;
        doVectorPositions |= field.isStorePositionWithTermVector();
        doVectorOffsets |= field.isStoreOffsetWithTermVector();
      }
    }

    if (doVectors) {
      if (perThread.doc == null) {
View Full Code Here

  // Tokenizes the fields of a document into Postings.
  private final void invertDocument(Document doc)
          throws IOException {
    Iterator fieldIterator = doc.getFields().iterator();
    while (fieldIterator.hasNext()) {
      Fieldable field = (Fieldable) fieldIterator.next();
      String fieldName = field.name();
      int fieldNumber = fieldInfos.fieldNumber(fieldName);

      int length = fieldLengths[fieldNumber];     // length of field
      int position = fieldPositions[fieldNumber]; // position in field
      if (length>0) position+=analyzer.getPositionIncrementGap(fieldName);
      int offset = fieldOffsets[fieldNumber];       // offset field

      if (field.isIndexed()) {
        if (!field.isTokenized()) {      // un-tokenized field
          String stringValue = field.stringValue();
          if(field.isStoreOffsetWithTermVector())
            addPosition(fieldName, stringValue, position++, new TermVectorOffsetInfo(offset, offset + stringValue.length()));
          else
            addPosition(fieldName, stringValue, position++, null);
          offset += stringValue.length();
          length++;
        } else
        {
          Reader reader;        // find or make Reader
          if (field.readerValue() != null)
            reader = field.readerValue();
          else if (field.stringValue() != null)
            reader = new StringReader(field.stringValue());
          else
            throw new IllegalArgumentException
                    ("field must have either String or Reader value");

          // Tokenize field and add to postingTable
          TokenStream stream = analyzer.tokenStream(fieldName, reader);
          try {
            Token lastToken = null;
            for (Token t = stream.next(); t != null; t = stream.next()) {
              position += (t.getPositionIncrement() - 1);
             
              if(field.isStoreOffsetWithTermVector())
                addPosition(fieldName, t.termText(), position++, new TermVectorOffsetInfo(offset + t.startOffset(), offset + t.endOffset()));
              else
                addPosition(fieldName, t.termText(), position++, null);
             
              lastToken = t;
              if (++length >= maxFieldLength) {
                if (infoStream != null)
                  infoStream.println("maxFieldLength " +maxFieldLength+ " reached, ignoring following tokens");
                break;
              }
            }
           
            if(lastToken != null)
              offset += lastToken.endOffset() + 1;
           
          } finally {
            stream.close();
          }
        }

        fieldLengths[fieldNumber] = length;    // save field length
        fieldPositions[fieldNumber] = position;    // save field position
        fieldBoosts[fieldNumber] *= field.getBoost();
        fieldOffsets[fieldNumber] = offset;
      }
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.lucene.document.Fieldable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.