Examples of org.apache.hadoop.hive.common.ValidTxnList

org.apache.hadoop.hive.common.ValidTxnList
Models the list of transactions that should be included in a snapshot. It is modelled as a high water mark, which is the largest transaction id that has been committed and a list of transactions that are not included.


  // Write the current set of valid transactions into the conf file so that it can be read by
  // the input format.
  private int recordValidTxns() {
    try {
      ValidTxnList txns = txnMgr.getValidTxns();
      conf.set(ValidTxnList.VALID_TXNS_KEY, txns.toString());
      return 0;
    } catch (LockException e) {
      errorMessage = "FAILED: Error in determing valid transactions: " + e.getMessage();
      SQLState = ErrorMsg.findSQLState(e.getMessage());
      downstreamError = e;

View Full Code Here

      // the input split.
      // Based on the split we're passed we go instantiate the real reader and then iterate on it
      // until it finishes.
      AcidInputFormat aif =
          instantiate(AcidInputFormat.class, jobConf.get(INPUT_FORMAT_CLASS_NAME));
      ValidTxnList txnList =
          new ValidTxnListImpl(jobConf.get(ValidTxnList.VALID_TXNS_KEY));


      AcidInputFormat.RawReader<V> reader =
          aif.getRawReader(jobConf, jobConf.getBoolean(IS_MAJOR, false), split.getBucket(),
              txnList, split.getBaseDir(), split.getDeltaDirs());

View Full Code Here

    AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
        .inspector(inspector).bucket(BUCKET).writingBase(true)
        .maximumTransactionId(100);
    of.getRecordUpdater(root, options).close(false);


    ValidTxnList txnList = new ValidTxnListImpl("200:");
    AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, txnList);


    Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(),
        BUCKET);
    Reader baseReader = OrcFile.createReader(basePath,

View Full Code Here

    ru.update(200, 0, 3, new MyRow("update 3"));
    ru.delete(200, 0, 7);
    ru.delete(200, 0, 8);
    ru.close(false);


    ValidTxnList txnList = new ValidTxnListImpl("200:");
    AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, txnList);


    assertEquals(new Path(root, "base_0000100"), directory.getBaseDirectory());
    assertEquals(new Path(root, "delta_0000200_0000200"),
        directory.getCurrentDirectories().get(0).getPath());


    Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(),
        BUCKET);
    Reader baseReader = OrcFile.createReader(basePath,
        OrcFile.readerOptions(conf));
    OrcRawRecordMerger merger =
        new OrcRawRecordMerger(conf, true, baseReader, false, BUCKET,
            createMaximalTxnList(), new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));
    assertEquals(null, merger.getMinKey());
    assertEquals(null, merger.getMaxKey());
    RecordIdentifier id = merger.createKey();
    OrcStruct event = merger.createValue();


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 200), id);
    assertEquals("update 1", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 1, 0), id);
    assertEquals("second", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 200), id);
    assertEquals("update 2", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 200), id);
    assertEquals("update 3", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 4, 0), id);
    assertEquals("fifth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 5, 0), id);
    assertEquals("sixth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 6, 0), id);
    assertEquals("seventh", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 9, 0), id);
    assertEquals("tenth", getValue(event));


    assertEquals(false, merger.next(id, event));
    merger.close();


    // make a merger that doesn't collapse events
    merger = new OrcRawRecordMerger(conf, false, baseReader, false, BUCKET,
            createMaximalTxnList(), new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 200), id);
    assertEquals("update 1", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 0), id);
    assertEquals("first", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 1, 0), id);
    assertEquals("second", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 200), id);
    assertEquals("update 2", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 0), id);
    assertEquals("third", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 200), id);
    assertEquals("update 3", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 0), id);
    assertEquals("fourth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 4, 0), id);
    assertEquals("fifth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 5, 0), id);
    assertEquals("sixth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 6, 0), id);
    assertEquals("seventh", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 0), id);
    assertEquals("eighth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 0), id);
    assertEquals("ninth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 9, 0), id);
    assertEquals("tenth", getValue(event));


    assertEquals(false, merger.next(id, event));
    merger.close();


    // try ignoring the 200 transaction and make sure it works still
    ValidTxnList txns = new ValidTxnListImpl("2000:200");
    merger =
        new OrcRawRecordMerger(conf, true, baseReader, false, BUCKET,
            txns, new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));
    for(int i=0; i < values.length; ++i) {

View Full Code Here

    return fields;
  }


  private void checkDataWritten(long minTxn, long maxTxn, int buckets, int numExpectedFiles,
                                String... records) throws Exception {
    ValidTxnList txns = msClient.getValidTxns();
    AcidUtils.Directory dir = AcidUtils.getAcidState(new Path(partLocation), conf, txns);
    Assert.assertEquals(0, dir.getObsolete().size());
    Assert.assertEquals(0, dir.getOriginalFiles().size());
    List<AcidUtils.ParsedDelta> current = dir.getCurrentDirectories();
    System.out.println("Files found: ");
    for (AcidUtils.ParsedDelta pd : current) System.out.println(pd.getPath().toString());
    Assert.assertEquals(numExpectedFiles, current.size());


    // find the absolute mininum transaction
    long min = Long.MAX_VALUE;
    long max = Long.MIN_VALUE;
    for (AcidUtils.ParsedDelta pd : current) {
      if (pd.getMaxTransaction() > max) max = pd.getMaxTransaction();
      if (pd.getMinTransaction() < min) min = pd.getMinTransaction();
    }
    Assert.assertEquals(minTxn, min);
    Assert.assertEquals(maxTxn, max);


    InputFormat inf = new OrcInputFormat();
    JobConf job = new JobConf();
    job.set("mapred.input.dir", partLocation.toString());
    job.set("bucket_count", Integer.toString(buckets));
    job.set(ValidTxnList.VALID_TXNS_KEY, txns.toString());
    InputSplit[] splits = inf.getSplits(job, 1);
    Assert.assertEquals(1, splits.length);
    org.apache.hadoop.mapred.RecordReader<NullWritable, OrcStruct> rr =
            inf.getRecordReader(splits[0], job, Reporter.NULL);

View Full Code Here

    }
    Assert.assertEquals(false, rr.next(key, value));
  }


  private void checkNothingWritten() throws Exception {
    ValidTxnList txns = msClient.getValidTxns();
    AcidUtils.Directory dir = AcidUtils.getAcidState(new Path(partLocation), conf, txns);
    Assert.assertEquals(0, dir.getObsolete().size());
    Assert.assertEquals(0, dir.getOriginalFiles().size());
    List<AcidUtils.ParsedDelta> current = dir.getCurrentDirectories();
    Assert.assertEquals(0, current.size());

View Full Code Here

    Assert.assertEquals(1L, (long) tids.get(0));
    Assert.assertEquals(2L, (long) tids.get(1));
    Assert.assertEquals(3L, (long) tids.get(2));
    client.rollbackTxn(1);
    client.commitTxn(2);
    ValidTxnList validTxns = client.getValidTxns();
    Assert.assertFalse(validTxns.isTxnCommitted(1));
    Assert.assertTrue(validTxns.isTxnCommitted(2));
    Assert.assertFalse(validTxns.isTxnCommitted(3));
    Assert.assertFalse(validTxns.isTxnCommitted(4));
  }

View Full Code Here

    Assert.assertFalse(validTxns.isTxnCommitted(4));
  }


  @Test
  public void testTxnRange() throws Exception {
    ValidTxnList validTxns = client.getValidTxns();
    Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
        validTxns.isTxnRangeCommitted(1L, 3L));
    List<Long> tids = client.openTxns("me", 5).getTxn_ids();


    HeartbeatTxnRangeResponse rsp = client.heartbeatTxnRange(1, 5);
    Assert.assertEquals(0, rsp.getNosuch().size());
    Assert.assertEquals(0, rsp.getAborted().size());


    client.rollbackTxn(1L);
    client.commitTxn(2L);
    client.commitTxn(3L);
    client.commitTxn(4L);
    validTxns = client.getValidTxns();
    System.out.println("validTxns = " + validTxns);
    Assert.assertEquals(ValidTxnList.RangeResponse.ALL,
        validTxns.isTxnRangeCommitted(2L, 2L));
    Assert.assertEquals(ValidTxnList.RangeResponse.ALL,
        validTxns.isTxnRangeCommitted(2L, 3L));
    Assert.assertEquals(ValidTxnList.RangeResponse.ALL,
        validTxns.isTxnRangeCommitted(2L, 4L));
    Assert.assertEquals(ValidTxnList.RangeResponse.ALL,
        validTxns.isTxnRangeCommitted(3L, 4L));


    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,
        validTxns.isTxnRangeCommitted(1L, 4L));
    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,
        validTxns.isTxnRangeCommitted(2L, 5L));
    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,
        validTxns.isTxnRangeCommitted(1L, 2L));
    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,
        validTxns.isTxnRangeCommitted(4L, 5L));


    Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
        validTxns.isTxnRangeCommitted(1L, 1L));
    Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
        validTxns.isTxnRangeCommitted(5L, 10L));


    validTxns = new ValidTxnListImpl("10:4:5:6");
    Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
        validTxns.isTxnRangeCommitted(4,6));
    Assert.assertEquals(ValidTxnList.RangeResponse.ALL,
        validTxns.isTxnRangeCommitted(7, 10));
    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,
        validTxns.isTxnRangeCommitted(7, 11));
    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,
        validTxns.isTxnRangeCommitted(3, 6));
    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,
        validTxns.isTxnRangeCommitted(4, 7));
    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,
        validTxns.isTxnRangeCommitted(1, 12));
    Assert.assertEquals(ValidTxnList.RangeResponse.ALL,
        validTxns.isTxnRangeCommitted(1, 3));
  }

View Full Code Here

  }


  @Test
  public void stringifyValidTxns() throws Exception {
    // Test with just high water mark
    ValidTxnList validTxns = new ValidTxnListImpl("1:");
    String asString = validTxns.toString();
    Assert.assertEquals("1:", asString);
    validTxns = new ValidTxnListImpl(asString);
    Assert.assertEquals(1, validTxns.getHighWatermark());
    Assert.assertNotNull(validTxns.getOpenTransactions());
    Assert.assertEquals(0, validTxns.getOpenTransactions().length);
    asString = validTxns.toString();
    Assert.assertEquals("1:", asString);
    validTxns = new ValidTxnListImpl(asString);
    Assert.assertEquals(1, validTxns.getHighWatermark());
    Assert.assertNotNull(validTxns.getOpenTransactions());
    Assert.assertEquals(0, validTxns.getOpenTransactions().length);


    // Test with open transactions
    validTxns = new ValidTxnListImpl("10:5:3");
    asString = validTxns.toString();
    if (!asString.equals("10:3:5") && !asString.equals("10:5:3")) {
      Assert.fail("Unexpected string value " + asString);
    }
    validTxns = new ValidTxnListImpl(asString);
    Assert.assertEquals(10, validTxns.getHighWatermark());
    Assert.assertNotNull(validTxns.getOpenTransactions());
    Assert.assertEquals(2, validTxns.getOpenTransactions().length);
    boolean sawThree = false, sawFive = false;
    for (long tid : validTxns.getOpenTransactions()) {
      if (tid == 3)  sawThree = true;
      else if (tid == 5) sawFive = true;
      else  Assert.fail("Unexpected value " + tid);
    }
    Assert.assertTrue(sawThree);

View Full Code Here


  // Write the current set of valid transactions into the conf file so that it can be read by
  // the input format.
  private int recordValidTxns() {
    try {
      ValidTxnList txns = txnMgr.getValidTxns();
      conf.set(ValidTxnList.VALID_TXNS_KEY, txns.toString());
      return 0;
    } catch (LockException e) {
      errorMessage = "FAILED: Error in determing valid transactions: " + e.getMessage();
      SQLState = ErrorMsg.findSQLState(e.getMessage());
      downstreamError = e;

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.hive.common.ValidTxnList

org.apache.hadoop.hive.metastore.TestHiveMetaStoreTxns

org.apache.hadoop.hive.ql.Driver

org.apache.hadoop.hive.ql.io.orc.OrcInputFormat

org.apache.hadoop.hive.ql.io.orc.TestOrcRawRecordMerger

org.apache.hadoop.hive.ql.txn.compactor.Cleaner

org.apache.hadoop.hive.ql.txn.compactor.CompactorMR$CompactorMap

org.apache.hadoop.hive.ql.txn.compactor.Initiator

org.apache.hadoop.hive.ql.txn.compactor.Worker

org.apache.hive.hcatalog.streaming.TestStreaming

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.