Examples of SqoopConnector


Examples of org.apache.sqoop.connector.spi.SqoopConnector

     * 9. Otherwise, Insert the connection inputs followed by job inputs (using
     *    updateJob and updateConnection)
     */
    RepositoryTransaction tx = null;
    try {
      SqoopConnector connector =
        ConnectorManager.getInstance().getConnector(newConnector
          .getUniqueName());

      Validator validator = connector.getValidator();

      boolean upgradeSuccessful = true;

      MetadataUpgrader upgrader = connector.getMetadataUpgrader();
      List<MConnection> connections = findConnectionsForConnector(
        connectorID);
      List<MJob> jobs = findJobsForConnector(connectorID);
      // -- BEGIN TXN --
      tx = getTransaction();
      tx.begin();
      deleteConnectionsAndJobs(connections, jobs, tx);
      updateConnector(newConnector, tx);
      for (MConnection connection : connections) {
        long connectionID = connection.getPersistenceId();
        // Make a new copy of the forms from the connector,
        // else the values will get set in the forms in the connector for
        // each connection.
        List<MForm> forms = newConnector.getConnectionForms().clone(false).getForms();
        MConnectionForms newConnectionForms = new MConnectionForms(forms);
        upgrader.upgrade(connection.getConnectorPart(), newConnectionForms);
        MConnection newConnection = new MConnection(connectorID,
          newConnectionForms, connection.getFrameworkPart());
        newConnection.setPersistenceId(connectionID);

        // Transform form structures to objects for validations
        Object newConfigurationObject = ClassUtils.instantiate(connector.getConnectionConfigurationClass());
        FormUtils.fromForms(newConnection.getConnectorPart().getForms(), newConfigurationObject);

        Validation validation = validator.validateConnection(newConfigurationObject);
        if (validation.getStatus().canProceed()) {
          updateConnection(newConnection, tx);
        } else {
          logInvalidModelObject("connection", newConnection, validation);
          upgradeSuccessful = false;
        }
      }
      for (MJob job : jobs) {
        // Make a new copy of the forms from the connector,
        // else the values will get set in the forms in the connector for
        // each connection.
        List<MForm> forms = newConnector.getJobForms(job.getType()).clone(false).getForms();
        MJobForms newJobForms = new MJobForms(job.getType(), forms);
        upgrader.upgrade(job.getConnectorPart(), newJobForms);
        MJob newJob = new MJob(connectorID, job.getConnectionId(),
          job.getType(), newJobForms, job.getFrameworkPart());
        newJob.setPersistenceId(job.getPersistenceId());

        // Transform form structures to objects for validations
        Object newConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(job.getType()));
        FormUtils.fromForms(newJob.getConnectorPart().getForms(), newConfigurationObject);

        Validation validation = validator.validateJob(newJob.getType(), newConfigurationObject);
        if (validation.getStatus().canProceed()) {
          updateJob(newJob, tx);
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

  public void testConnectorUpgradeWithValidConnectionsAndJobs() {
    MConnector newConnector = connector(1, "1.1");
    MConnector oldConnector = connector(1);

    // prepare the sqoop connector
    SqoopConnector sqconnector = mock(SqoopConnector.class);
    when(validator.validateConnection(any(MConnection.class))).thenReturn(valid);
    when(validator.validateJob(any(MJob.Type.class), any(MJob.class))).thenReturn(valid);
    when(sqconnector.getValidator()).thenReturn(validator);
    when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
    when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
    when(sqconnector.getJobConfigurationClass(any(MJob.Type.class))).thenReturn(ImportJobConfiguration.class);
    when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);

    // prepare the connections and jobs
    List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
    List<MJob> jobList = jobs(job(1,1,1), job(2,1,2));
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

  @Test
  public void testConnectorUpgradeWithInvalidConnectionsAndJobs() {
    MConnector newConnector = connector(1, "1.1");
    MConnector oldConnector = connector(1);

    SqoopConnector sqconnector = mock(SqoopConnector.class);
    when(validator.validateConnection(any(MConnection.class))).thenReturn(invalid);
    when(validator.validateJob(any(MJob.Type.class), any(MJob.class))).thenReturn(invalid);
    when(sqconnector.getValidator()).thenReturn(validator);
    when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
    when(sqconnector.getConnectionConfigurationClass()).thenReturn(EmptyConfigurationClass.class);
    when(sqconnector.getJobConfigurationClass(any(MJob.Type.class))).thenReturn(ImportJobConfiguration.class);
    when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);

    List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
    List<MJob> jobList = jobs(job(1,1,1), job(2,1,2));
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

  @Test
  public void testConnectorUpgradeHandlerFindConnectionsForConnectorError() {
    MConnector newConnector = connector(1, "1.1");
    MConnector oldConnector = connector(1);

    SqoopConnector sqconnector = mock(SqoopConnector.class);
    when(sqconnector.getValidator()).thenReturn(validator);
    when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
    when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);

    SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
        "find connections for connector error.");
    doThrow(exception).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

  @Test
  public void testConnectorUpgradeHandlerFindJobsForConnectorError() {
    MConnector newConnector = connector(1, "1.1");
    MConnector oldConnector = connector(1);

    SqoopConnector sqconnector = mock(SqoopConnector.class);
    when(sqconnector.getValidator()).thenReturn(validator);
    when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
    when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);

    List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
    doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

  @Test
  public void testConnectorUpgradeHandlerDeleteJobInputsError() {
    MConnector newConnector = connector(1, "1.1");
    MConnector oldConnector = connector(1);

    SqoopConnector sqconnector = mock(SqoopConnector.class);
    when(sqconnector.getValidator()).thenReturn(validator);
    when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
    when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);

    List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
    List<MJob> jobList = jobs(job(1,1,1), job(2,1,2));
    doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

  @Test
  public void testConnectorUpgradeHandlerDeleteConnectionInputsError() {
    MConnector newConnector = connector(1, "1.1");
    MConnector oldConnector = connector(1);

    SqoopConnector sqconnector = mock(SqoopConnector.class);
    when(sqconnector.getValidator()).thenReturn(validator);
    when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
    when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);

    List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
    List<MJob> jobList = jobs(job(1,1,1), job(2,1,2));
    doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

      throw new SqoopException(ServerError.SERVER_0003,
        "Detected incorrect form structure");
    }

    // Responsible connector for this session
    SqoopConnector connector =
      ConnectorManager.getInstance().getConnector(job.getConnectorId());

    // Get validator objects
    Validator connectorValidator = connector.getValidator();
    Validator frameworkValidator = FrameworkManager.getInstance().getValidator();

    // We need translate forms to configuration objects
    Object connectorConfig = ClassUtils.instantiate(
      connector.getJobConfigurationClass(job.getType()));
    Object frameworkConfig = ClassUtils.instantiate(
      FrameworkManager.getInstance().getJobConfigurationClass(job.getType()));

    FormUtils.fromForms(job.getConnectorPart().getForms(), connectorConfig);
    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkConfig);
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

    if (!connection.getEnabled()) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0010,
        "Connection id: " + connection.getPersistenceId());
    }

    SqoopConnector connector =
      ConnectorManager.getInstance().getConnector(job.getConnectorId());

    // Transform forms to connector specific classes
    Object connectorConnection = ClassUtils.instantiate(
      connector.getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getConnectorPart().getForms(),
      connectorConnection);

    Object connectorJob = ClassUtils.instantiate(
      connector.getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getConnectorPart().getForms(), connectorJob);

    // Transform framework specific forms
    Object frameworkConnection = ClassUtils.instantiate(
      FrameworkManager.getInstance().getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getFrameworkPart().getForms(),
      frameworkConnection);

    Object frameworkJob = ClassUtils.instantiate(
      FrameworkManager.getInstance().getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);

    // Create request object
    MSubmission summary = new MSubmission(jobId);
    SubmissionRequest request = executionEngine.createSubmissionRequest();

    summary.setCreationUser(username);
    summary.setLastUpdateUser(username);

    // Save important variables to the submission request
    request.setSummary(summary);
    request.setConnector(connector);
    request.setConfigConnectorConnection(connectorConnection);
    request.setConfigConnectorJob(connectorJob);
    request.setConfigFrameworkConnection(frameworkConnection);
    request.setConfigFrameworkJob(frameworkJob);
    request.setJobType(job.getType());
    request.setJobName(job.getName());
    request.setJobId(job.getPersistenceId());
    request.setNotificationUrl(notificationBaseUrl + jobId);

    // Let's register all important jars
    // sqoop-common
    request.addJarForClass(MapContext.class);
    // sqoop-core
    request.addJarForClass(FrameworkManager.class);
    // sqoop-spi
    request.addJarForClass(SqoopConnector.class);
    // Execution engine jar
    request.addJarForClass(executionEngine.getClass());
    // Connector in use
    request.addJarForClass(connector.getClass());

    // Extra libraries that Sqoop code requires
    request.addJarForClass(JSONValue.class);

    // Get connector callbacks
    switch (job.getType()) {
      case IMPORT:
        request.setConnectorCallbacks(connector.getImporter());
        break;
      case EXPORT:
        request.setConnectorCallbacks(connector.getExporter());
        break;
      default:
        throw new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }
View Full Code Here

Examples of org.apache.sqoop.connector.spi.SqoopConnector

  @Test
  public void testConnectorUpgradeHandlerUpdateConnectorError() {
    MConnector newConnector = connector(1, "1.1");
    MConnector oldConnector = connector(1);

    SqoopConnector sqconnector = mock(SqoopConnector.class);
    when(sqconnector.getValidator()).thenReturn(validator);
    when(sqconnector.getMetadataUpgrader()).thenReturn(upgrader);
    when(connectorMgr.getConnector(anyString())).thenReturn(sqconnector);

    List<MConnection> connectionList = connections(connection(1,1), connection(2,1));
    List<MJob> jobList = jobs(job(1,1,1), job(2,1,2));
    doReturn(connectionList).when(repoHandler).findConnectionsForConnector(anyLong(), any(Connection.class));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.