Examples of com.linkedin.databus.core.ConcurrentAppendableCompositeFileInputStream

com.linkedin.databus.core.ConcurrentAppendableCompositeFileInputStream

A composite InputStream that sequentially reads from an ordered and filtered list of files found inside the input directory and takes care of reading files that are concurrently appended in a defined order.

 __________________________________________________ 1 |                                                  | 1 ______________             _____________           ______ ______           _______________ |              | 1       1 |             | 1     1 |             |1      *  |              | | CompositeFIS | --------->| SequenceFIS |-------->| Enumerator  |--------> |  SingleFIS   | |______________|           |_____________|         |_____________|          |______________| |1 | | | | |1 _________________ |               | |    TrailFile  | |     Notifier  | |_______________| CompositeFIS : ConcurrentAppendableCompositeFileInputStream SingleFIS : ConcurrentAppendableSingleFileInputStream

   * @throws DatabusException
   */
  private ConcurrentAppendableCompositeFileInputStream locateScnInTrailFile(String xmlDir, String xmlPrefix)
      throws Exception
  {
    ConcurrentAppendableCompositeFileInputStream compositeInputStream = null;
    TrailFilePositionSetter.FilePositionResult filePositionResult = null;
    TrailFilePositionSetter trailFilePositionSetter = null;


    while(compositeInputStream == null)
    {


      _log.info("Requesting trail file position setter for scn: " + _scn.get());
      trailFilePositionSetter = new TrailFilePositionSetter(xmlDir,xmlPrefix, getName());
      filePositionResult = trailFilePositionSetter.locateFilePosition(_scn.get(), new GGXMLTrailTransactionFinder());
      _log.info("File position at : "+ filePositionResult);
      switch(filePositionResult.getStatus())
      {
        case ERROR:
          _log.fatal("Unable to locate the scn in the trail file.");
          throw new DatabusException("Unable to find the given scn " + _scn.get() + " in the trail files");
        case NO_TXNS_FOUND:


          //If the latest scn is not found in the trail files, then use the earliest scn.
          if(_scn.get() == TrailFilePositionSetter.USE_LATEST_SCN)
          {
            _log.info("Switching from USE_LATEST_SCN to USE_EARLIEST_SCN because no trail files were not found");
            _scn.set(TrailFilePositionSetter.USE_EARLIEST_SCN);
          }


          long noTxnsFoundSleepTime = 500;                      //TODO sleep get configuration for sleep time
          _log.info("NO_TXNS_FOUND, sleeping for "+ noTxnsFoundSleepTime + " ms before retrying");
          Thread.sleep(noTxnsFoundSleepTime);
          break;
        case EXACT_SCN_NOT_FOUND:
        {
          _log.info("Exact SCN was not found, the closest scn found was: " + filePositionResult.getTxnPos().getMinScn());
          compositeInputStream = new ConcurrentAppendableCompositeFileInputStream(xmlDir,
                                                                                  filePositionResult.getTxnPos().getFile(),
                                                                                  filePositionResult.getTxnPos().getFileOffset(),
                                                                                  new TrailFilePositionSetter.FileFilter(new File(xmlDir), xmlPrefix),
                                                                                  false);
          long foundScn = filePositionResult.getTxnPos().getMaxScn();
          /**
           * If exact scn is not found, the trail file position setter returns the next immediate available scn, i.e., the contract guarantees
           * a scn always greater than the given scn (foundscn > _scn). We use the _scn (requested scn to be found) as the prevScn to start the event buffer.
           * And the scn found as the current scn(first event in the relay).
           */
          if(foundScn <= _scn.get())
            throw new DatabusException("EXACT_SCN_NOT_FOUND, but foundScn is <= _scn ");


          _startPrevScn.set(_scn.get());
          _log.info("Changing current scn from " + _scn.get() + " to " + foundScn);
          _log.info("Planning to use prevScn " + _startPrevScn);
          _scn.set(foundScn);
          break;
        }
        case FOUND:
        {
          _log.info("Exact SCN was  found" + filePositionResult.getTxnPos().getMaxScn());
          compositeInputStream = new ConcurrentAppendableCompositeFileInputStream(xmlDir,
                                                                                  filePositionResult.getTxnPos().getFile(),
                                                                                  filePositionResult.getTxnPos().getFileOffset(),
                                                                                  new TrailFilePositionSetter.FileFilter(new File(xmlDir), xmlPrefix),
                                                                                  false);
          /**

View Full Code Here


    @Override
    public void run()
    {


      ConcurrentAppendableCompositeFileInputStream compositeInputStream = null;
      try
      {
        if(_xmlCallback == null)
          _xmlCallback = new HandleXmlCallback();


        String xmlDir = GGEventGenerationFactory.uriToGGDir(_pConfig.getUri());
        String xmlPrefix = GGEventGenerationFactory.uriToXmlPrefix(_pConfig.getUri());
        File file = new File(xmlDir);
        if(!file.exists() || !file.isDirectory())
        {
          _log.fatal("Unable to load the directory: "+ xmlDir + " it doesn't seem to be a valid directory");
          throw new DatabusException("Invalid trail file directory");
        }


        boolean parseError = false;
        do
        {
          try{
             _log.info("Using xml directory : "+ xmlDir + " and using the xml Prefix : " + xmlPrefix);
            compositeInputStream = locateScnInTrailFile(xmlDir,xmlPrefix);
            compositeInputStream.setGGParserStats(_ggParserStats);
            _log.info("Attempting to start the parser...");




            //Not a retry, first time the producer is started, in which case, start the eventBuffer with the appropriate scn
            if(!parseError)
            {
              _log.info("Starting dbusEventBuffer with _scn : " + _startPrevScn.get());
              getEventBuffer().start(_startPrevScn.get());
            }
            else
            {
                _log.warn("Umm, looks like the parser had failed, this is an retry attempt using _scn: " + _scn.get());
                _log.info("CompositeInputStream used:" + compositeInputStream);
            }


           StaxBuilder builder = new StaxBuilder(_schemaRegistryService, wrapStreamWithXmlTags(compositeInputStream), _pConfig, _xmlCallback);


            if(_log.isDebugEnabled())
              _log.debug("CompositeInputStream used:" + compositeInputStream);


            _parser = builder.getParser();
            builder.processXml(); // --> The call doesn't return after this (it starts processing the xml trail files), unless a shutdown is requested or an exception is thrown.
            parseError = false;  //--> If this code path is executed, then the shutdown has been requested
          }
          catch (XMLStreamException e)
          {
            _ggParserStats.addParsingError();


            //If the parser was in the middle of execution and an shutdown was issued, then an xmlstream exception is expected.
            if(_shutdownRequested )
            {
              parseError = false;
            }
            else
            {
              _log.error("Error while parsing the xml, will retry loading the parser", e);
              _log.info("Last scn seen before the crash: " + _scn.get());
              _log.info("CompositeInputStream used:" + compositeInputStream);
              parseError = true;
            }
          }
            finally
          {
            if(compositeInputStream != null)
              compositeInputStream.close();
          }
        }while(parseError);  //TODO && retry count (add config to control number of retires)


      }
      catch (RuntimeException e) {

View Full Code Here


    LOG.info("starting from position " + _filePositionResult);


    //set up the parser
    _filter = new TrailFilePositionSetter.FileFilter(new File(xmlDir), xmlPrefix);
    _compositeInputStream = new ConcurrentAppendableCompositeFileInputStream(
        xmlDir,
        _filePositionResult.getTxnPos().getFile(),
        _filePositionResult.getTxnPos().getFileOffset(),
        _filter,
        !continuous);

View Full Code Here

TOP

Related Classes of com.linkedin.databus.core.ConcurrentAppendableCompositeFileInputStream

com.linkedin.databus.core.ConcurrentAppendableCompositeFileInputStream.InputStreamEnumerator.ParserLag

com.linkedin.databus.core.util.NamedThreadFactory

com.linkedin.databus2.ggParser.staxparser.validator.XmlFormatTrailValidator

com.linkedin.databus2.relay.GoldenGateEventProducer

com.linkedin.databus2.relay.GoldenGateEventProducer$WorkerThread

java.io.InputStream

java.util.concurrent.ScheduledThreadPoolExecutor

java.io.IOException

java.io.SequenceInputStream

com.linkedin.databus2.core.DatabusException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.