Examples of PhysicalSourceConfig


Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

    String partitionFunction = "constant:1";

    String pPartName = "part1";
    short pPartId = 0;

    PhysicalSourceConfig pc = new PhysicalSourceConfig(pPartName, uri, pPartId);
    for(int i=0; i<sourceIds.length; i++) {
      LogicalSourceConfig lc = new LogicalSourceConfig();
      lc.setId(sourceIds[i]);
      lc.setName(sourceNames[i]);
      lc.setPartitionFunction(partitionFunction);
      lc.setUri(pPartName + "." + sourceNames[i]); // this format is expected by GG
      pc.addSource(lc);
    }


    return pc.build();
  }
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

                                   String outputDir,
                                   List<String> srcNames,
                                   SchemaMetaDataManager manager)
       throws Exception
  {
    PhysicalSourceConfig config = new PhysicalSourceConfig();
    FileSystemSchemaRegistryService s = FileSystemSchemaRegistryService.build(
                                             new FileSystemSchemaRegistryService.StaticConfig(new File(schemaRegistryLocation), 0, false, false));
    dbName = dbName.trim().toLowerCase();
    config.setName(dbName);
    config.setUri(uri);
    for (String srcName : srcNames)
    {
      VersionedSchema schema = null;
      schema = s.fetchLatestVersionedSchemaBySourceName(srcName);

      String dbObjectName = SchemaHelper.getMetaField(schema.getSchema(), "dbFieldName");

      LogicalSourceConfig c = new LogicalSourceConfig();
      c.setId(manager.getSrcId(srcName));
      c.setName(srcName);
      c.setUri(dbName + "." + dbObjectName);
      c.setPartitionFunction("constant:1");
      config.addSource(c);
    }

    DatabusRelaySourcesInFiles relaySourcesInFiles = new DatabusRelaySourcesInFiles(outputDir);
    relaySourcesInFiles.add(dbName, config);
    boolean success = relaySourcesInFiles.save();
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

    parseArgs(args);

      File sourcesJson = new File(_sSourcesConfigFile);

      ObjectMapper mapper = new ObjectMapper();
      PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
      physicalSourceConfig.checkForNulls();

      Config config = new Config();

      ConfigLoader<StaticConfig> configLoader =
                new ConfigLoader<StaticConfig>("databus.seed.", config);
      _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);

      // Make sure the URI from the configuration file identifies an Oracle JDBC source.
      String uri = physicalSourceConfig.getUri();
      if(!uri.startsWith("jdbc:oracle"))
      {
        throw new InvalidConfigException("Invalid source URI (" +
            physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
      }

      OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());

      // Parse each one of the logical sources
      _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
      FileSystemSchemaRegistryService schemaRegistryService =
            FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());

      for(LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources())
      {
        OracleTriggerMonitoredSourceInfo source =
            factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
      }
      _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(),_sources);

      _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
 
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

      // Load the source configuration JSON file
      //File sourcesJson = new File("integration-test/config/sources-member2.json");
      File sourcesJson = new File(_sSourcesConfigFile);

      ObjectMapper mapper = new ObjectMapper();
      PhysicalSourceConfig physicalSourceConfig = mapper.readValue(sourcesJson, PhysicalSourceConfig.class);
      physicalSourceConfig.checkForNulls();

      Config config = new Config();

      ConfigLoader<StaticConfig> configLoader =
                new ConfigLoader<StaticConfig>("databus.seed.", config);
      _sStaticConfig = configLoader.loadConfig(_sBootstrapConfigProps);

      // Make sure the URI from the configuration file identifies an Oracle JDBC source.
      String uri = physicalSourceConfig.getUri();
      if(!uri.startsWith("jdbc:oracle"))
      {
        throw new InvalidConfigException("Invalid source URI (" +
            physicalSourceConfig.getUri() + "). Only jdbc:oracle: URIs are supported.");
      }

      String sourceTypeStr = physicalSourceConfig.getReplBitSetter().getSourceType();
        if (SourceType.TOKEN.toString().equalsIgnoreCase(sourceTypeStr))
          throw new InvalidConfigException("Token Source-type for Replication bit setter config cannot be set for trigger-based Databus relay !!");

      // Create the OracleDataSource used to get DB connection(s)
      try
      {
        Class oracleDataSourceClass = OracleJarUtils.loadClass("oracle.jdbc.pool.OracleDataSource");
        Object ods = oracleDataSourceClass.newInstance();
        Method setURLMethod = oracleDataSourceClass.getMethod("setURL", String.class);
        setURLMethod.invoke(ods, uri);
        _sDataStore = (DataSource) ods;
      } catch (Exception e)
      {
        String errMsg = "Error creating a data source object ";
        LOG.error(errMsg, e);
        throw e;
      }

      //TODO: Need a better way than relaying on RelayFactory for generating MonitoredSourceInfo
      OracleEventProducerFactory factory = new BootstrapSeederOracleEventProducerFactory(_sStaticConfig.getController().getPKeyNameMap());

      // Parse each one of the logical sources
      _sources = new ArrayList<OracleTriggerMonitoredSourceInfo>();
      FileSystemSchemaRegistryService schemaRegistryService =
            FileSystemSchemaRegistryService.build(_sStaticConfig.getSchemaRegistry().getFileSystem());

      Set<String> seenUris = new HashSet<String>();
      for(LogicalSourceConfig sourceConfig : physicalSourceConfig.getSources())
      {
        String srcUri  = sourceConfig.getUri();
        if ( seenUris.contains(srcUri))
        {
          String msg = "Uri (" + srcUri + ") is used for more than one sources. Currently Bootstrap Seeder cannot support seeding sources with the same URI together. Please have them run seperately !!";
          LOG.fatal(msg);
          throw new InvalidConfigException(msg);
        }
        seenUris.add(srcUri);
        OracleTriggerMonitoredSourceInfo source =
            factory.buildOracleMonitoredSourceInfo(sourceConfig.build(), physicalSourceConfig.build(), schemaRegistryService);
        _sources.add(source);
      }
      _sSeeder = new BootstrapDBSeeder(_sStaticConfig.getBootstrap(),_sources);

      _sBootstrapBuffer = new BootstrapEventBuffer(_sStaticConfig.getController().getCommitInterval() * 2);
 
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

    Assert.assertTrue( f.exists(),"Physical Src Config file present?");

    BufferedReader r = new BufferedReader(new FileReader(f));

    String json = r.readLine();
    PhysicalSourceConfig config = PhysicalSourceConfig.fromString(json);
    config.checkForNulls();

    Assert.assertEquals(config.getUri(), uri);
    Assert.assertEquals(config.getName(), "example");

    List<LogicalSourceConfig> srcs = config.getSources();
    Assert.assertEquals(srcs.size(),2 );
    Assert.assertEquals(srcs.get(0).getId(), 1);
    Assert.assertEquals(srcs.get(0).getName(), "com.linkedin.events.example.person");
    Assert.assertEquals(srcs.get(0).getUri(), "example.sy$person");
    Assert.assertEquals(srcs.get(1).getId(), 2);
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

  }

  @Override
  public ConfigBuilder<? extends PhysicalSourceStaticConfig> createConfigBuilder(String propPrefix)
  {
    return new PhysicalSourceConfig();
  }
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

  }

  @Override
  public ConfigBuilder<? extends PhysicalSourceStaticConfig> createConfigBuilder(String propPrefix)
  {
    return new PhysicalSourceConfig();
  }
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

    // create main relay with random generator
    PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcs.length];

    String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);

    PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder(
                                                                                 (short) pId, pSourceName, "mock",
                                                                                 500, 0, srcs);

    srcConfigs[0] = src1;
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

        new PhysicalSourceStaticConfig[null == _fileNames ? 0 : _fileNames.length];
    if(_fileNames == null) return list;

    for(int i = 0; i < _fileNames.length; ++i) {
      File sourceJson = _fileNames[i];
      PhysicalSourceConfig pConfig = null;
      Exception e = null;
      try
      {
        pConfig = mapper.readValue(sourceJson, PhysicalSourceConfig.class);
      }
      catch (JsonParseException jpe) {
        e = jpe;
      } catch (JsonMappingException jme) {
        e = jme;
      } catch (IOException ioe) {
        e = ioe;
      }
      if(e != null || pConfig == null) {
        throw new InvalidConfigException(e);
      }
      pConfig.checkForNulls();
      LOG.info("Generated Physical source config: name= " + pConfig.getId());

      list[i] = pConfig.build();
    }
    /*
    for(PhysicalSourceStaticConfig pCfg : pConfigs) {
      for(LogicalSourceStaticConfig lSC : pCfg.getSources()) {
        config.setSourceName("" + lSC.getId(), lSC.getName());
View Full Code Here

Examples of com.linkedin.databus2.relay.config.PhysicalSourceConfig

      int i = 0;
      int eventRatePerSec = 20;
      for (String[] srcs : srcNames)
      {

        PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder(
            (short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock",
            500, eventRatePerSec, srcs);
        srcConfigs[i++] = src1;
      }
      int relayPort = Utils.getAvailablePort(11993);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.