Examples of RecordScanner

co.cask.cdap.api.data.batch.RecordScanner
Defines a reader of a dataset {@link co.cask.cdap.api.data.batch.Split}. @param < RECORD> the type of objects that represents a single record
org.lilyproject.repository.api.RecordScanner
A RecordScanner allows to sequentially run over all or part of the records stored in the Lily repository.
A scanner can be obtained from {@link Repository#getScanner(RecordScan)}
.
RecordScanner's implement Iterable, so you can use a for-each loop directly on them.

When done with a scanner, be sure to call {@link #close()} on them to releasethe server-side resources.

Examples of co.cask.cdap.api.data.batch.RecordScanner

    if (!(split instanceof DatasetInputSplit)) {
      throw new IOException("Invalid type for InputSplit: " + split.getClass().getName());
    }
    final DatasetInputSplit datasetInputSplit = (DatasetInputSplit) split;


    final RecordScanner recordScanner = recordScannable.createSplitRecordScanner(
        new Split() {
          @Override
          public long getLength() {
            try {
              return split.getLength();
            } catch (IOException e) {
              throw new RuntimeException(e);
            }
          }
        }
    );


    return new RecordReader<Void, ObjectWritable>() {
      private final AtomicBoolean initialized = new AtomicBoolean(false);


      private void initialize() throws IOException {
        try {
          recordScanner.initialize(datasetInputSplit.getDataSetSplit());
        } catch (InterruptedException ie) {
          Thread.currentThread().interrupt();
          throw new IOException("Interrupted while initializing reader", ie);
        }
        initialized.set(true);
      }


      @Override
      public boolean next(Void key, ObjectWritable value) throws IOException {
        if (!initialized.get()) {
          initialize();
        }


        try {
          boolean retVal = recordScanner.nextRecord();
          if (retVal) {
            value.set(recordScanner.getCurrentRecord());
          }
          return retVal;
        } catch (InterruptedException e) {
          Thread.currentThread().interrupt();
          throw new IOException(e);
        }
      }


      @Override
      public Void createKey() {
        return null;
      }


      @Override
      public ObjectWritable createValue() {
        return new ObjectWritable();
      }


      @Override
      public long getPos() throws IOException {
        // Not required.
        return 0;
      }


      @Override
      public void close() throws IOException {
        try {
          recordScanner.close();
        } finally {
          recordScannable.close();
        }
      }


      @Override
      public float getProgress() throws IOException {
        try {
          return recordScanner.getProgress();
        } catch (InterruptedException e) {
          Thread.currentThread().interrupt();
          throw new IOException(e);
        }
      }

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner

    private Cache<String, RecordScanner> recordScannerMap;


    @GET
    @Produces("application/json")
    public EntityList<Record> get(@PathParam("id") String scanId, @DefaultValue("1") @QueryParam("batch") Long batch, @Context UriInfo uriInfo) {
        RecordScanner scanner = recordScannerMap.getIfPresent(scanId);
        if (scanner != null) {
            List<Record> records = new ArrayList<Record>();


            try {
                Record record;
                while(records.size() < batch && (record = scanner.next()) != null) {
                    records.add(record);
                }
            } catch (RepositoryException e) {
                throw new ResourceException(e, INTERNAL_SERVER_ERROR.getStatusCode());
            } catch (InterruptedException e) {

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner

        }
    }


    @DELETE
    public Response delete(@PathParam("id") String scanId) {
        RecordScanner scanner = this.recordScannerMap.getIfPresent(scanId);
        if (scanner != null) {
            scanner.close();
            this.recordScannerMap.invalidate(scanId);
            return Response.ok().build();
        } else {
            throw new ResourceException("No scan with ID " + scanId + " found", NOT_FOUND.getStatusCode());
        }

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner

        // Change the start/stop record IDs on the scan to the current split
        TableSplit split = (TableSplit)inputSplit;
        scan.setRawStartRecordId(split.getStartRow());
        scan.setRawStopRecordId(split.getEndRow());


        RecordScanner scanner = null;
        try {
            String hbaseTableName = Bytes.toString(split.getTableName());
            String repositoryTableName = RepoAndTableUtil.extractLilyTableName(repositoryName, hbaseTableName);
            scanner = lilyClient.getRepository(repositoryName).getTable(repositoryTableName).getScanner(scan);
        } catch (RepositoryException e) {

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner

        cache = RecordScannerMapBuilder.createRecordScannerMap(50, TimeUnit.MILLISECONDS);
    }


    @Test
    public void testRecordScannerMapExpiration() throws Exception{
        RecordScanner scanner = new DummyRecordScanner();
        String id = "TTT";
        cache.put(id, scanner);
        Thread.sleep(100);
        RecordScanner returnScanner = cache.getIfPresent(id);
        Assert.assertNull(returnScanner);


    }

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner


    }


    @Test
    public void testRecordScannerMapAccessExpiration() throws Exception{
        RecordScanner scanner = new DummyRecordScanner();
        String id = "QQQ";
        cache.put(id, scanner);
        Thread.sleep(10);
        Assert.assertNotNull(cache.getIfPresent(id));
        Thread.sleep(100);

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner

        // Do a scan
        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("A"));
        scan.setStopRecordId(idGenerator.newRecordId("B"));


        RecordScanner scanner = repository.getScanner(scan);
        int i = 0;
        while (scanner.next() != null) {
            i++;
        }


        assertEquals("Number of scanned records", 10, i);
    }

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner


        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("ZA"));
        scan.setStopRecordId(idGenerator.newRecordId("ZZ")); // stop row is exclusive


        RecordScanner scanner = repository.getScanner(scan);


        int i = 0;
        Record record;
        while ((record = scanner.next()) != null) {
            assertEquals(record.getField(fieldType1.getName()), fieldValues.get(i));
            i++;
        }
        scanner.close();
        assertEquals("Found 25 records", 25, i);


        // Same using for-each loop
        scanner = repository.getScanner(scan);
        i = 0;
        for (Record result : scanner) {
            assertEquals(result.getField(fieldType1.getName()), fieldValues.get(i));
            i++;
        }
        scanner.close();
        assertEquals("Found 25 records", 25, i);


        // Scan all records, this should give at least 26 results
        scan = new RecordScan();
        scanner = repository.getScanner(scan);
        i = 0;
        while (scanner.next() != null) {
            i++;
        }


        assertTrue("Found at least 26 records", i >= 26);
    }

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner

        repository.create(record);


        RecordScan scan = new RecordScan();


        // normal Record scanner!
        RecordScanner scanner = repository.getScanner(scan);
        final Record next = scanner.next();
        assertNotNull(next);


        // this cast will fail
        final IdRecord casted = (IdRecord) next;
    }

View Full Code Here

Examples of org.lilyproject.repository.api.RecordScanner

                .create();


        RecordScan scan = new RecordScan();
        scan.setStartRecordId(idGenerator.newRecordId("PrefixScanTest"));


        RecordScanner scanner = repository.getScanner(scan);
        assertEquals(idGenerator.newRecordId("PrefixScanTest"), scanner.next().getId());
        assertEquals(idGenerator.newRecordId("PrefixScanTest-suffix1"), scanner.next().getId());
        assertEquals(idGenerator.newRecordId("PrefixScanTest-suffix2"), scanner.next().getId());
        // the scanner would run till the end of the table
        assertNotNull(scanner.next());
        scanner.close();


        scan.setRecordFilter(new RecordIdPrefixFilter(idGenerator.newRecordId("PrefixScanTest")));
        scanner = repository.getScanner(scan);
        assertEquals(idGenerator.newRecordId("PrefixScanTest"), scanner.next().getId());
        assertEquals(idGenerator.newRecordId("PrefixScanTest-suffix1"), scanner.next().getId());
        assertEquals(idGenerator.newRecordId("PrefixScanTest-suffix2"), scanner.next().getId());
        // due to the prefix filter, the scanner stops once there are no records left with the same prefix
        assertNull(scanner.next());
        scanner.close();


        //
        // When using UUID record ID's, prefix scans make less sense, except for retrieving
        // variants
        //
        RecordId uuid = idGenerator.newRecordId();
        RecordId varid1 = idGenerator.newRecordId(uuid, ImmutableMap.of("lang", "en", "year", "1999"));
        RecordId varid2 = idGenerator.newRecordId(uuid, ImmutableMap.of("lang", "fr"));


        repository.recordBuilder()
                .id(uuid)
                .recordType(recordType1.getName())
                .field(fieldType1.getName(), "foo")
                .create();


        repository.recordBuilder()
                .id(varid1)
                .recordType(recordType1.getName())
                .field(fieldType1.getName(), "foo")
                .create();


        repository.recordBuilder()
                .id(varid2)
                .recordType(recordType1.getName())
                .field(fieldType1.getName(), "foo")
                .create();


        scan = new RecordScan();
        scan.setStartRecordId(uuid);
        scan.setRecordFilter(new RecordIdPrefixFilter(uuid));
        scanner = repository.getScanner(scan);
        assertEquals(uuid, scanner.next().getId());
        assertEquals(varid1, scanner.next().getId());
        assertEquals(varid2, scanner.next().getId());
        assertNull(scanner.next());
        scanner.close();
    }

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.