Examples of DocumentList

Boundary cases are important for {@link #checkpoint()}: The typical pattern for consuming an object that implements this interface is as follows (disregarding exception handling):
 DocumentList docList = ... Document doc; while (doc = docList.nextDocument()) { handleDoc(doc); if (whatever reason) break; } String check = doclist.checkpoint(); 
Note: because of the restriction that the next call to {@link #nextDocument()} invalidates the previous Document, and thereare similar restrictions in the {@link Document} interface, it is possibleto provide a single stateful object that implements {@link DocumentList}, {@link Document} and {@link Property}, by returning {@code this}(or {@code null}) to all calls to {@link #nextDocument()} and{@link Document#findProperty(String)}. However, if preferred, the implementor may also use separate objects for each of those interfaces. @since 1.0
  • edu.harvard.wcfia.yoshikoder.document.DocumentList
    @author will
  • org.openntf.domino.impl.DocumentList

  • Examples of com.alimama.mdrill.index.utils.DocumentList

          TermInfosWriter.setSkipInterVal(16);
        }

        heartBeater = new HeartBeater(context);
        heartBeater.needHeartBeat();
        this.doclistcache=new DocumentList();
        this.ramMerger = new RamWriter();

        String[] fieldslist = fieldStrs.split(",");
        this.documentConverter = new DocumentConverter(fieldslist,"solrconfig.xml", "schema.xml");
        shardWriter = this.initShardWriter(context);
    View Full Code Here

    Examples of com.alimama.mdrill.index.utils.DocumentList

          RamWriter ram = doclistcache.toRamWriter(documentConverter, analyzer,context);
          ramMerger.process(ram);
          if (this.maybeFlush(ramMerger,context,true)) {
            ramMerger = null;
          }
          doclistcache=new DocumentList();
          shardWriter.optimize();
          shardWriter.close();
          Configuration conf = context.getConfiguration();
          FileSystem fs = FileSystem.get(conf);
          fs.copyFromLocalFile(new Path(localtmpath), new Path(tmpath));
    View Full Code Here

    Examples of com.alimama.mdrill.index.utils.DocumentList

          if(!doclistcache.isoversize())
          {
            continue;
          }
          RamWriter ram = doclistcache.toRamWriter(documentConverter, analyzer,context);
          doclistcache=new DocumentList();

         
          ramMerger.process(ram);
          if (this.maybeFlush(ramMerger,context,false)) {
            ramMerger =  new RamWriter();;
    View Full Code Here

    Examples of com.google.enterprise.connector.spi.DocumentList

        ArrayList<Document> documents = new ArrayList<Document>();
        TraversalManager tm = ConnectorFixture.getTraversalManager(
            connectorSession);
        // Get the first set of documents.
        tm.setBatchHint(20);
        DocumentList docList = tm.startTraversal();
        assertNotNull("startTraversal returned a null document list", docList);
        while (docList != null) {
          Document doc;
          while (null != (doc = docList.nextDocument())) {
            documents.add((NotesConnectorDocument) doc);
          }
          String checkpoint = docList.checkpoint();
          // Resume traversal.
          tm.setBatchHint(20);
          docList = tm.resumeTraversal(checkpoint);
        }
        return documents;
    View Full Code Here

    Examples of com.google.enterprise.connector.spi.DocumentList

        assertNotNull(tm);
        assertTrue(tm instanceof NotesTraversalManager);

        // Get the first set of documents.
        tm.setBatchHint(5);
        DocumentList docList = tm.startTraversal();
        Document doc;
        String checkpoint = null;
        List<String> docIdList = new ArrayList<String>(10);
        assertNotNull("startTraversal returned a null document list", docList);
        while (null != (doc = docList.nextDocument())) {
          String docId = doc.findProperty(SpiConstants.PROPNAME_DOCID)
              .nextValue().toString();
          assertNotNull("Missing doc id", docId);
          docIdList.add(docId);
        }
        checkpoint = docList.checkpoint();
        assertNotNull("Checkpoint was null", checkpoint);
        assertTrue("No docs found", docIdList.size() > 0);

        // Resume traversal.
        tm.setBatchHint(5);
        docList = tm.resumeTraversal(checkpoint);
        while (null != (doc = docList.nextDocument())) {
          String docId = doc.findProperty(SpiConstants.PROPNAME_DOCID).
              nextValue().toString();
          assertNotNull("Missing doc id", docId);
          assertFalse("Found same docid in new doc list: " + docId,
              docIdList.contains(docId));
    View Full Code Here

    Examples of com.google.enterprise.connector.spi.DocumentList

        Set<String> docIdListFirstTraversal = new HashSet<String>(100);
        List<String> duplicatesFirstTraversal = new ArrayList<String>(100);
        // Get the first set of documents.
        tm.setBatchHint(20);
        DocumentList docList = tm.startTraversal();
        while (docList != null) {
          Document doc = null;
          while (null != (doc = docList.nextDocument())) {
            String docId = doc.findProperty(SpiConstants.PROPNAME_DOCID).
                nextValue().toString();
            if (!docIdListFirstTraversal.add(docId)) {
              duplicatesFirstTraversal.add(docId);
            }
          }
          String checkpoint = docList.checkpoint();
          assertNotNull("Checkpoint was null", checkpoint);

          // Resume traversal.
          tm.setBatchHint(20);
          docList = tm.resumeTraversal(checkpoint);
        }
        assertTrue("No documents traversed", docIdListFirstTraversal.size() > 0);
        // TODO: do we want to investigate the presence of duplicates?
        //if (duplicatesFirstTraversal.size() > 0) {
        //  System.out.println("Found duplicates during first traversal: " +
        //      duplicatesFirstTraversal.size());
        //}

        Set<String> docIdListSecondTraversal = new HashSet<String>(100);
        List<String> duplicatesSecondTraversal = new ArrayList<String>(100);
        // Get the second set of documents.
        tm.setBatchHint(20);
        docList = tm.startTraversal();
        while (docList != null) {
          Document doc = null;
          while (null != (doc = docList.nextDocument())) {
            String docId = doc.findProperty(SpiConstants.PROPNAME_DOCID).
                nextValue().toString();
            if (!docIdListSecondTraversal.add(docId)) {
              duplicatesSecondTraversal.add(docId);
            }
          }
          String checkpoint = docList.checkpoint();
          assertNotNull("Checkpoint was null", checkpoint);

          // Resume traversal.
          tm.setBatchHint(20);
          docList = tm.resumeTraversal(checkpoint);
    View Full Code Here

    Examples of com.google.enterprise.connector.spi.DocumentList

        TraversalManager tm = ConnectorFixture.getTraversalManager(
            (NotesConnectorSession) session);

        // Get the first set of documents.
        tm.setBatchHint(25);
        DocumentList docList = tm.startTraversal();
        assertNotNull("startTraversal returned a null document list", docList);
        Document doc;
        while (null != (doc = docList.nextDocument())) {
          String docId = doc.findProperty(SpiConstants.PROPNAME_DOCID).
              nextValue().toString();
          if (docId.contains(("/$File/"))) {
            assertNotNull("Missing attachment filename " + docId,
                doc.findProperty(NCCONST.PROPNAME_NCATTACHMENTFILENAME));
    View Full Code Here

    Examples of com.google.enterprise.connector.spi.DocumentList

          int batchHint) throws RepositoryException, PushException, FeedException {
        DctmTraversalManager dctmTM = (DctmTraversalManager) queryTraversalManager;
        dctmTM.setBatchHint(batchHint);
        System.out.println(batchHint);

        DocumentList documentList = dctmTM.startTraversal();
        // int nb = resultSet.size();
        // System.out.println("nb vaut " + nb);
        // The real connector manager will not always start from the beginning.
        // It will start from the beginning if it receives an explicit admin
        // command to do so, or if it thinks that it has never run this
        // connector
        // before. It decides whether it has run a connector before by storing
        // every checkpoint it receives from
        // the connector. If it can find no stored checkpoint, it assumes that
        // it has never run this connector before and starts from the beginning,
        // as here.
        if (documentList == null) {
          // in this test program, we will stop in this situation. The real
          // connector manager might wait for a while, then try again
          return;
        }

        DocPusher push = null;
        try {
          push = new DocPusher(new GsaFeedConnection(null, "gogol", 19900, -1),
              "dctm", new FileSizeLimitInfo(), new DocumentFilterChain());

        } catch (MalformedURLException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }

        int counter = 0;

        while (true) {
          counter = 0;
          Document pm = null;
          while ((pm = documentList.nextDocument()) != null) {
            System.out.println("pm change");
            counter++;

            if (counter == batchHint) {
              System.out.println("counter == batchhint !!!!");
              // this test program only takes batchHint results from each
              // resultSet. The real connector manager may take fewer -
              // for example, if it receives a shutdown request
              break;
            }
            int k = pm.getPropertyNames().size();
            System.out.println("counter " + counter + ",  num properties " + k);
            System.out.println(pm.findProperty(
                SpiConstants.PROPNAME_DISPLAYURL).nextValue());
            push.take(pm);
          }
          String checkpoint = "";
          if (counter != 0) {
            System.out.println("appel checkpoint");
            checkpoint = documentList.checkpoint();
            System.out.println("appel checkpoint " + checkpoint);
          }
          dctmTM.resumeTraversal(checkpoint);
        }
      }
    View Full Code Here

    Examples of com.google.enterprise.connector.spi.DocumentList

        }
        assertEquals(DmInitialize.DM_RETURN_TOP_BOUNDED, counter);
      }

      public void testResumeTraversalWithSimilarDate() throws RepositoryException {
        DocumentList documentList = null;

        String checkPoint = "{\"uuid\":\"090000018000015d\",\"lastModified\":\"2006-12-14 20:09:13\"}";

        qtm.setBatchHint(1);
        documentList = qtm.resumeTraversal(checkPoint);

        DctmSysobjectDocument map = null;
        while ((map = (DctmSysobjectDocument) documentList.nextDocument()) != null) {
          String docId = map.findProperty(SpiConstants.PROPNAME_DOCID)
              .nextValue().toString();
          String expectedid = "090000018000015e";
          assertEquals(expectedid, docId);
          Value date =
    View Full Code Here

    Examples of com.google.enterprise.connector.spi.DocumentList

        // until one returns results, all of them have been tried,
        // or time expires.
        TraversalTimer timer = new TraversalTimer(traversalContext);
        boolean isMore;
        do {
          DocumentList documentList = execQuery(checkpoint);
          if (documentList != null)
            return documentList;
          isMore = checkpoint.advance();
        } while (isMore && timer.isTicking());
        return isMore ? new EmptyDocumentList(checkpoint.asString()) : null;
    View Full Code Here
    TOP
    Copyright © 2018 www.massapi.com. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.