Examples of MJob


Examples of org.apache.sqoop.model.MJob

  private void cloneJob(Long jobId) throws IOException {
    printlnResource(Constants.RES_CLONE_CLONING_JOB, jobId);

    ConsoleReader reader = new ConsoleReader();

    MJob job = client.getJob(jobId);
    job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);

    ResourceBundle connectorBundle = client.getResourceBundle(job.getConnectorId());
    ResourceBundle frameworkBundle = client.getFrameworkResourceBundle();

    Status status = Status.FINE;

    // Remove persistent id as we're making a clone
    job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);

    printlnResource(Constants.RES_PROMPT_UPDATE_JOB_METADATA);
    do {
      // Print error introduction if needed
      if( !status.canProceed() ) {
        errorIntroduction();
      }

      // Fill in data from user
      if(!fillJob(reader, job, connectorBundle, frameworkBundle)) {
        return;
      }

      // Try to create
      status = client.createJob(job);
    } while(!status.canProceed());

    printlnResource(Constants.RES_CLONE_JOB_SUCCESSFUL, status.name(), job.getPersistenceId());
  }
View Full Code Here

Examples of org.apache.sqoop.model.MJob

public class TestJobBean {
  @Test
  public void testSerialization() throws ParseException {
    Date created = new Date();
    Date updated = new Date();
    MJob job = getJob("ahoj", MJob.Type.IMPORT);
    job.setName("The big job");
    job.setPersistenceId(666);
    job.setCreationDate(created);
    job.setLastUpdateDate(updated);

    // Fill some data at the beginning
    MStringInput input = (MStringInput) job.getConnectorPart().getForms()
      .get(0).getInputs().get(0);
    input.setValue("Hi there!");

    // Serialize it to JSON object
    JobBean bean = new JobBean(job);
    JSONObject json = bean.extract(false);

    // "Move" it across network in text form
    String string = json.toJSONString();

    // Retrieved transferred object
    JSONObject retrievedJson = (JSONObject)JSONValue.parseWithException(string);
    JobBean retrievedBean = new JobBean();
    retrievedBean.restore(retrievedJson);
    MJob target = retrievedBean.getJobs().get(0);

    // Check id and name
    assertEquals(666, target.getPersistenceId());
    assertEquals(MJob.Type.IMPORT, target.getType());
    assertEquals("The big job", target.getName());
    assertEquals(created, target.getCreationDate());
    assertEquals(updated, target.getLastUpdateDate());

    // Test that value was correctly moved
    MStringInput targetInput = (MStringInput) target.getConnectorPart()
      .getForms().get(0).getInputs().get(0);
    assertEquals("Hi there!", targetInput.getValue());
  }
View Full Code Here

Examples of org.apache.sqoop.model.MJob

                           getFramework().getConnectionForms()
    );
  }

  public static MJob getJob(String name, MJob.Type type) {
    return new MJob(1, 1,
                    type,
                    getConnector(name).getJobForms(type),
                    getFramework().getJobForms(type)
    );
  }
View Full Code Here

Examples of org.apache.sqoop.model.MJob

    MConnection connection = getClient().newConnection(1L);
    fillConnectionForm(connection);
    createConnection(connection);

    // Job creation
    MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT);

    // Connector values
    MFormList forms = job.getConnectorPart();
    forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName()));
    forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName("id"));
    // Framework values
    fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE);
    createJob(job);

    MSubmission submission = getClient().startSubmission(job.getPersistenceId());
    assertTrue(submission.getStatus().isRunning());

    // Wait until the job finish - this active waiting will be removed once
    // Sqoop client API will get blocking support.
    do {
      Thread.sleep(5000);
      submission = getClient().getSubmissionStatus(job.getPersistenceId());
    } while(submission.getStatus().isRunning());

    // Assert correct output
    assertMapreduceOutput(
      "1,'USA','San Francisco'",
View Full Code Here

Examples of org.apache.sqoop.model.MJob

    fillConnectionForm(connection);

    createConnection(connection);

    // Job creation
    MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT);

    // Connector values
    MFormList forms = job.getConnectorPart();
    forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName()));
    forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName("id"));
    forms.getStringInput("table.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country"));
    // Framework values
    fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE);
    createJob(job);

    MSubmission submission = getClient().startSubmission(job.getPersistenceId());
    assertTrue(submission.getStatus().isRunning());

    // Wait until the job finish - this active waiting will be removed once
    // Sqoop client API will get blocking support.
    do {
      Thread.sleep(5000);
      submission = getClient().getSubmissionStatus(job.getPersistenceId());
    } while(submission.getStatus().isRunning());

    // Assert correct output
    assertMapreduceOutput(
      "1,'USA'",
View Full Code Here

Examples of org.apache.sqoop.model.MJob

      MJob.Type type = MJob.Type.valueOf(stringType);

      List<MForm> connectorForms = restoreForms(connectorPart);
      List<MForm> frameworkForms = restoreForms(frameworkPart);

      MJob job = new MJob(
        connectorId,
        connectionId,
        type,
        new MJobForms(type, connectorForms),
        new MJobForms(type, frameworkForms)
      );

      job.setPersistenceId((Long) object.get(ID));
      job.setName((String) object.get(NAME));
      job.setCreationDate(new Date((Long) object.get(CREATED)));
      job.setLastUpdateDate(new Date((Long) object.get(UPDATED)));

      jobs.add(job);
    }

    if(jsonObject.containsKey(CONNECTOR_RESOURCES)) {
View Full Code Here

Examples of org.apache.sqoop.model.MJob

  }

  public MSubmission submit(long jobId) {
    Repository repository = RepositoryManager.getInstance().getRepository();

    MJob job = repository.findJob(jobId);
    if(job == null) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0004,
        "Unknown job id " + jobId);
    }
    MConnection connection = repository.findConnection(job.getConnectionId());
    SqoopConnector connector =
      ConnectorManager.getInstance().getConnector(job.getConnectorId());

    // Transform forms to connector specific classes
    Object connectorConnection = ClassUtils.instantiate(
      connector.getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getConnectorPart().getForms(),
      connectorConnection);

    Object connectorJob = ClassUtils.instantiate(
      connector.getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getConnectorPart().getForms(), connectorJob);

    // Transform framework specific forms
    Object frameworkConnection = ClassUtils.instantiate(
      getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getFrameworkPart().getForms(),
      frameworkConnection);

    Object frameworkJob = ClassUtils.instantiate(
      getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);

    // Create request object
    MSubmission summary = new MSubmission(jobId);
    SubmissionRequest request = executionEngine.createSubmissionRequest();

    // Save important variables to the submission request
    request.setSummary(summary);
    request.setConnector(connector);
    request.setConfigConnectorConnection(connectorConnection);
    request.setConfigConnectorJob(connectorJob);
    request.setConfigFrameworkConnection(frameworkConnection);
    request.setConfigFrameworkJob(frameworkJob);
    request.setJobType(job.getType());
    request.setJobName(job.getName());
    request.setJobId(job.getPersistenceId());
    request.setNotificationUrl(notificationBaseUrl + jobId);

    // Let's register all important jars
    // sqoop-common
    request.addJarForClass(MapContext.class);
    // sqoop-core
    request.addJarForClass(FrameworkManager.class);
    // sqoop-spi
    request.addJarForClass(SqoopConnector.class);
    // Execution engine jar
    request.addJarForClass(executionEngine.getClass());
    // Connector in use
    request.addJarForClass(connector.getClass());

    // Extra libraries that Sqoop code requires
    request.addJarForClass(JSONValue.class);

    // Get connector callbacks
    switch (job.getType()) {
      case IMPORT:
        request.setConnectorCallbacks(connector.getImporter());
        break;
      case EXPORT:
        request.setConnectorCallbacks(connector.getExporter());
        break;
      default:
        throw  new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }
    LOG.debug("Using callbacks: " + request.getConnectorCallbacks());

    // Initialize submission from connector perspective
    CallbackBase baseCallbacks = request.getConnectorCallbacks();

    Class<? extends Initializer> initializerClass = baseCallbacks.getInitializer();
    Initializer initializer = (Initializer) ClassUtils.instantiate(initializerClass);

    if(initializer == null) {
      throw  new SqoopException(FrameworkError.FRAMEWORK_0006,
        "Can't create initializer instance: " + initializerClass.getName());
    }

    // Initializer context
    InitializerContext initializerContext = new InitializerContext(request.getConnectorContext());

    // Initialize submission from connector perspective
    initializer.initialize(initializerContext,
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob());

    // Add job specific jars to
    request.addJars(initializer.getJars(initializerContext,
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob()));

    // Bootstrap job from framework perspective
    switch (job.getType()) {
      case IMPORT:
        prepareImportSubmission(request);
        break;
      case EXPORT:
        prepareExportSubmission(request);
        break;
      default:
        throw  new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }

    // Make sure that this job id is not currently running and submit the job
    // only if it's not.
    synchronized (submissionMutex) {
View Full Code Here

Examples of org.apache.sqoop.model.MJob

      throw new SqoopException(ServerError.SERVER_0003,
        "Expected one job metadata but got " + jobs.size());
    }

    // Job object
    MJob job = jobs.get(0);

    // Verify that user is not trying to spoof us
    MJobForms connectorForms
      = ConnectorManager.getInstance().getConnectorMetadata(job.getConnectorId())
      .getJobForms(job.getType());
    MJobForms frameworkForms = FrameworkManager.getInstance().getFramework()
      .getJobForms(job.getType());

    if(!connectorForms.equals(job.getConnectorPart())
      || !frameworkForms.equals(job.getFrameworkPart())) {
      throw new SqoopException(ServerError.SERVER_0003,
        "Detected incorrect form structure");
    }

    // Responsible connector for this session
    SqoopConnector connector =
      ConnectorManager.getInstance().getConnector(job.getConnectorId());

    // Get validator objects
    Validator connectorValidator = connector.getValidator();
    Validator frameworkValidator = FrameworkManager.getInstance().getValidator();

    // We need translate forms to configuration objects
    Object connectorConfig = ClassUtils.instantiate(
      connector.getJobConfigurationClass(job.getType()));
    Object frameworkConfig = ClassUtils.instantiate(
      FrameworkManager.getInstance().getJobConfigurationClass(job.getType()));

    FormUtils.fromForms(job.getConnectorPart().getForms(), connectorConfig);
    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkConfig);

    // Validate both parts
    Validation connectorValidation =
      connectorValidator.validateJob(job.getType(), connectorConfig);
    Validation frameworkValidation =
      frameworkValidator.validateJob(job.getType(), frameworkConfig);

    Status finalStatus = Status.getWorstStatus(connectorValidation.getStatus(),
      frameworkValidation.getStatus());

    // Return back validations in all cases
    ValidationBean outputBean =
      new ValidationBean(connectorValidation, frameworkValidation);

    // If we're good enough let's perform the action
    if(finalStatus.canProceed()) {
      if(update) {
        RepositoryManager.getInstance().getRepository().updateJob(job);
      } else {
        RepositoryManager.getInstance().getRepository().createJob(job);
        outputBean.setId(job.getPersistenceId());
      }

    }

    return outputBean;
View Full Code Here

Examples of org.apache.sqoop.model.MJob

        }
      }
    } else {
      long jid = Long.valueOf(sjid);

      MJob job = repository.findJob(jid);
      long connectorId = job.getConnectorId();

      bean = new JobBean(job);

      bean.addConnectorBundle(connectorId,
        ConnectorManager.getInstance().getResourceBundle(connectorId, locale));
View Full Code Here

Examples of org.apache.sqoop.model.MJob

    }

    // Load prepared connections into database
    loadJobs();

    MJob jobImport = handler.findJob(1, getDerbyConnection());
    assertNotNull(jobImport);
    assertEquals(1, jobImport.getPersistenceId());
    assertEquals("JA", jobImport.getName());
    assertEquals(MJob.Type.IMPORT, jobImport.getType());

    List<MForm> forms;

    // Check connector part
    forms = jobImport.getConnectorPart().getForms();
    assertEquals("Value5", forms.get(0).getInputs().get(0).getValue());
    assertNull(forms.get(0).getInputs().get(1).getValue());
    assertEquals("Value7", forms.get(1).getInputs().get(0).getValue());
    assertNull(forms.get(1).getInputs().get(1).getValue());

    // Check framework part
    forms = jobImport.getFrameworkPart().getForms();
    assertEquals("Value17", forms.get(0).getInputs().get(0).getValue());
    assertNull(forms.get(0).getInputs().get(1).getValue());
    assertEquals("Value19", forms.get(1).getInputs().get(0).getValue());
    assertNull(forms.get(1).getInputs().get(1).getValue());
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.