Package com.linkedin.databus2.producers.db

Examples of com.linkedin.databus2.producers.db.EventReaderSummary

182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
              .getStatsCollector(statsCollectorName),
          maxScnReaderWriters);
    } else if (uri.startsWith("mock")) {
      // Get all relevant pConfig attributes
      //TODO add real instantiation
      EventProducerServiceProvider mockProvider = _producersRegistry.getEventProducerServiceProvider("mock");
      if (null == mockProvider)
      {
        throw new DatabusRuntimeException("relay event producer not available: " + "mock");
      }
      producer = mockProvider.createProducer(pConfig, schemaRegistryService,
                                             dbusEventBuffer,
                                             _inBoundStatsCollectors
                                                     .getStatsCollector(statsCollectorName),
                                             maxScnReaderWriters);
    } else if (uri.startsWith("gg:")){
      producer = new GoldenGateEventProducer(pConfig,
                                             schemaRegistryService,
                                             dbusEventBuffer,
                                             _inBoundStatsCollectors
                                                 .getStatsCollector(statsCollectorName),
                                             maxScnReaderWriters);

    } else if (uri.startsWith("mysql:")){
       LOG.info("Adding OpenReplicatorEventProducer for uri :" + uri);
       final String serviceName = "or";
       EventProducerServiceProvider orProvider = _producersRegistry.getEventProducerServiceProvider(serviceName);
       if (null == orProvider)
       {
         throw new DatabusRuntimeException("relay event producer not available: " + serviceName);
       }
       producer = orProvider.createProducer(pConfig, schemaRegistryService,
                                            dbusEventBuffer,
                                            _inBoundStatsCollectors.getStatsCollector(statsCollectorName),
                                            maxScnReaderWriters);
    } else
     {
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
     {
      // Get all relevant pConfig attributes and initialize the nettyThreadPool objects
      RelayEventProducer.DatabusClientNettyThreadPools nettyThreadPools =
            new RelayEventProducer.DatabusClientNettyThreadPools(0,getNetworkTimeoutTimer(),getBossExecutorService(),
                                        getIoExecutorService(), getHttpChannelGroup());
      producer = new RelayEventProducer(pConfig, dbusEventBuffer,
          _inBoundStatsCollectors
              .getStatsCollector(statsCollectorName),
          maxScnReaderWriters,nettyThreadPools);
    }
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
        for ( OracleTriggerMonitoredSourceInfo sourceInfo : _sources)
        {
          LOG.info("Bootstrapping " + sourceInfo.getEventView());
          _bootstrapSeedWriter.start(maxScn);
          EventReaderSummary summary = readEventsForSource(sourceInfo, maxScn);
          // Script assumes seeding is done for one schema at a time
          // just use one source to get the schema name for sy$txlog
          endScn = getMaxScn(_sources.get(0));
          _bootstrapSeedWriter.endEvents(BootstrapEventBuffer.END_OF_SOURCE, endScn, null);
          summaries.add(summary);
View Full Code Here
585
586
587
588
589
590
591
592
      }
    }
    long timeEnd = System.currentTimeMillis();
    long elapsedMin = (timeEnd - timeStart)/(MILLISEC_TO_MIN);
    LOG.info("Processed " + numRowsFetched + " rows of Source: " +  sourceInfo.getSourceName() + " in " + elapsedMin + " minutes" );
    return new EventReaderSummary(sourceInfo.getSourceId(), sourceInfo.getSourceName(), -1,
                                  numRowsFetched, totalEventSize, (timeEnd - timeStart),totProcessTime,0,0,0);
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
       
        String dir = _config.getAvroSeedInputDirMap().get(sourceInfo.getEventView());
       
        File d = new File(dir);

        EventReaderSummary summary = readEventsFromHadoopFiles(sourceInfo, d, endScn);
        // Script assumes seeding is done for one schema at a time
        _bootstrapEventBuffer.endEvents(BootstrapEventBuffer.END_OF_SOURCE, endScn, null);
        summaries.add(summary);
      }
      } catch (Exception ex) {
View Full Code Here
252
253
254
255
256
257
258
259
      }
   
    long timeEnd = System.currentTimeMillis();
    long elapsedMin = (timeEnd - timeStart)/(MILLISEC_TO_MIN);
    LOG.info("Processed " + numRead + " rows of Source: " +  sourceInfo.getSourceName() + " in " + elapsedMin + " minutes" );
    return new EventReaderSummary(sourceInfo.getSourceId(), sourceInfo.getSourceName(), -1,
                    (int)numRead, numBytes, (timeEnd - timeStart),(timeEnd-timeStart)/numRead,0,0,0);
  }
View Full Code Here
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
      }
      long endDbUpdatesMs = System.currentTimeMillis();
      long dbUpdatesElapsedTimeMs = endDbUpdatesMs - startDbUpdatesMs;

      // Log Event Summary at logical source level
      EventReaderSummary summary = new EventReaderSummary(sourceId, _monitoredSources.get(sourceId).getSourceName(),
                                                          scn, eventsInDbUpdate, dbUpdatesEventsSize,-1L /* Not supported */,
                                                          dbUpdatesElapsedTimeMs,  timestamp, timestamp, -1L /* Not supported */);
      if (_eventsLog.isInfoEnabled())
      {
        _eventsLog.info(summary.toString());
      }
      summaries.add(summary);

      if(_log.isDebugEnabled())
        _log.debug("There are "+ eventsInDbUpdate + " events seen in the current dbUpdate");
    }

    // Log Event Summary at Physical source level
    ReadEventCycleSummary summary = new ReadEventCycleSummary(_pConfig.getName(),
                                                              summaries,
                                                              scn,
                                                              -1 /* Overall time including query time not calculated */);

    if (_eventsLog.isInfoEnabled())
    {
      _eventsLog.info(summary.toString());
    }

    _log.info("Writing "+ eventsInTransactionCount + " events from transaction with scn: " + scn);
    if(scn <= 0)
      throw new DatabusException("Unable to write events to buffer because of negative/zero scn: " + scn);
View Full Code Here
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
        long endDbUpdatesMs = System.currentTimeMillis();
        long dbUpdatesElapsedTimeMs = endDbUpdatesMs - startDbUpdatesMs;

        // Log Event Summary at logical source level
        EventReaderSummary summary = new EventReaderSummary(sourceId, _monitoredSources.get(sourceId).getSourceName(),
                                                            scn, t.getDbChangeEntrySet().size(), dbUpdatesEventsSize,-1L /* Not supported */,
                                                            dbUpdatesElapsedTimeMs,  timestamp, timestamp, -1L /* Not supported */);
        if (_eventsLog.isInfoEnabled())
        {
          _eventsLog.info(summary.toString());
        }
        summaries.add(summary);

        long tsEnd = System.currentTimeMillis();
        perSourceStats.addTimeOfLastDBAccess(tsEnd);
        globalStats.addTimeOfLastDBAccess(tsEnd);
      }
      _eventBuffer.endEvents(scn, _relayInboundStatsCollector);
      // Log Event Summary at Physical source level
      ReadEventCycleSummary summary = new ReadEventCycleSummary(_physicalSourceStaticConfig.getName(),
                                                                summaries,
                                                                scn,
                                                                -1 /* Overall time including query time not calculated */);

      if (_eventsLog.isInfoEnabled())
      {
        _eventsLog.info(summary.toString());
      }
    }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
    protected EventReaderSummary getSummary(int sourceId)
    {
      DbusEventsTotalStats stats = _stats.getSourceStats(sourceId);
      if (stats != null)
      {
        EventReaderSummary summary = new EventReaderSummary( (short) sourceId, stats.getDimension(),
            stats.getMaxScn(), (int) stats.getNumDataEvents(),
            stats.getSizeDataEvents()*(int) (stats.getNumDataEvents()),
            System.currentTimeMillis(), 0, _stats.getTotalStats().getTimestampMinScnEvent(),
            stats.getTimestampMaxScnEvent(),0L);
        return summary;
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
    /**
     * produce diff of eventreader summary s1-s2
     */
    protected EventReaderSummary getDiff(EventReaderSummary s1, EventReaderSummary s2)
    {
      EventReaderSummary diff = new EventReaderSummary(s1.getSourceId(), s1.getSourceName(), s1.getEndOfPeriodSCN(),
          s1.getNumberOfEvents()-s2.getNumberOfEvents(), s1.getSizeOfSerializedEvents()-s2.getSizeOfSerializedEvents(), s1.getReadMillis()-s2.getReadMillis(),
          s1.getEventMillis(),
          s2.getTimeProdEnd(), s1.getTimeProdEnd(), s1.getQueryExecTime());
      return diff;
    }
View Full Code Here
TOP

Related Classes of com.linkedin.databus2.producers.db.EventReaderSummary

  • com.google.code.or.common.glossary.Row
  • com.linkedin.databus.bootstrap.server.BootstrapHttpServer
  • com.linkedin.databus.bootstrap.server.BootstrapRequestProcessor
  • com.linkedin.databus.bootstrap.server.BootstrapRequestProcessorBase
  • com.linkedin.databus.bootstrap.utils.BootstrapAvroFileEventReader
  • com.linkedin.databus.bootstrap.utils.BootstrapAvroFileSeederMain
  • com.linkedin.databus.bootstrap.utils.BootstrapSrcDBEventReader
  • com.linkedin.databus.bootstrap.utils.BootstrapTableReader
  • com.linkedin.databus.client.BootstrapPullThread
  • com.linkedin.databus.client.consumer.TestInternalMetadata
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.