Examples of com.netflix.zeno.fastblob.record.ByteDataBuffer

Package com.netflix.zeno.fastblob.record

Examples of com.netflix.zeno.fastblob.record.ByteDataBuffer

com.netflix.zeno.fastblob.record.ByteDataBuffer
Writes data to a SegmentedByteArray, tracking the index to which it writes. @author dkoszewnik


    @Test
    public void evictorCountsReferencesAndDoesNotEvictObjectsTooEarly() {
        FlatBlobEvictor evictor = new FlatBlobEvictor(typeDSerializerFactory, flatBlobFramework);


        ByteDataBuffer d1Buf = new ByteDataBuffer();
        ByteDataBuffer d2Buf = new ByteDataBuffer();
        ByteDataBuffer d3Buf = new ByteDataBuffer();


        flatBlobFramework.serialize("TypeD", d1, d1Buf);
        flatBlobFramework.serialize("TypeD", d2, d2Buf);
        flatBlobFramework.serialize("TypeD", d3, d3Buf);

View Full Code Here

    private <T> void readTypeStateObjectsDoubleSnapshotRefresh(StreamingByteData byteData, FastBlobSchema schema, ByteArrayOrdinalMap map) throws IOException{
        FastBlobHeapFriendlyClientFrameworkSerializer frameworkSerializer = (FastBlobHeapFriendlyClientFrameworkSerializer)stateEngine.getFrameworkSerializer();
        FastBlobDeserializationRecord rec = new FastBlobDeserializationRecord(schema, byteData);
        FastBlobTypeDeserializationState<T> typeDeserializationState = stateEngine.getTypeDeserializationState(schema.getName());
        FastBlobSerializationRecord serializationRecord = null;
        ByteDataBuffer deserializedRecordBuffer = null;


        int numObjects = VarInt.readVInt(byteData);
        int numObjectsReused = 0;
        int numFlawedSerializationIntegrity = 0;


        if(numObjects != 0 && eventHandler != null) {
            eventHandler.addedObjects(schema.getName(), numObjects);
        }


        if(typeDeserializationState != null) {
            serializationRecord = new FastBlobSerializationRecord(typeDeserializationState.getSchema());


            frameworkSerializer.setCheckSerializationIntegrity(false);


            deserializedRecordBuffer = new ByteDataBuffer();
            typeDeserializationState.populateByteArrayOrdinalMap(map);


            frameworkSerializer.setCheckSerializationIntegrity(true);
        }


        int currentOrdinal = 0;


        for(int j=0;j<numObjects;j++) {
            int currentOrdinalDelta = VarInt.readVInt(byteData);


            currentOrdinal += currentOrdinalDelta;


            int recordSize = rec.position(byteData.currentStreamPosition());


            if(typeDeserializationState != null) {
                NFTypeSerializer<T> serializer = typeDeserializationState.getSerializer();
                T deserializedObject = serializer.deserialize(rec);
                serializer.serialize(deserializedObject, serializationRecord);
                serializationRecord.writeDataTo(deserializedRecordBuffer);


                int previousOrdinal = map.get(deserializedRecordBuffer);


                serializationRecord.reset();
                deserializedRecordBuffer.reset();


                if(previousOrdinal != -1 && !frameworkSerializer.isSerializationIntegrityFlawed()) {
                    typeDeserializationState.copyPrevious(currentOrdinal, previousOrdinal);
                    numObjectsReused++;
                } else {

View Full Code Here

     */
    public FlatBlobSerializationRecord(FastBlobSchema schema) {
        this.fieldData = new ByteDataBuffer[schema.numFields()];
        this.isNonNull = new boolean[schema.numFields()];
        for (int i = 0; i < fieldData.length; i++) {
            fieldData[i] = new ByteDataBuffer(32);
        }
        setSchema(schema);
    }

View Full Code Here

     *
     * This method is only intended to be used during heap-friendly double snapshot refresh.
     */
    public void populateByteArrayOrdinalMap(ByteArrayOrdinalMap ordinalMap) {
        FastBlobSerializationRecord rec = new FastBlobSerializationRecord(serializer.getFastBlobSchema());
        ByteDataBuffer scratch = new ByteDataBuffer();


        for(int i=0;i<objects.size();i++) {
            T obj = objects.get(i);
            if(obj != null) {
                serializer.serialize(obj, rec);
                rec.writeDataTo(scratch);
                ordinalMap.put(scratch, i);
                scratch.reset();
                rec.reset();
            }
        }


        previousObjects = objects;

View Full Code Here

     * Fill this state from the serialized data which exists in this ByteArrayOrdinalMap
     *
     * @param ordinalMap
     */
    public void populateFromByteOrdinalMap(final ByteArrayOrdinalMap ordinalMap) {
        ByteDataBuffer byteData = ordinalMap.getByteData();
        AtomicLongArray pointersAndOrdinals = ordinalMap.getPointersAndOrdinals();
        FastBlobDeserializationRecord rec = new FastBlobDeserializationRecord(getSchema(), byteData.getUnderlyingArray());
        for (int i = 0; i < pointersAndOrdinals.length(); i++) {
            long pointerAndOrdinal = pointersAndOrdinals.get(i);
            if(!ByteArrayOrdinalMap.isPointerAndOrdinalEmpty(pointerAndOrdinal)) {
                long pointer = ByteArrayOrdinalMap.getPointer(pointerAndOrdinal);
                int ordinal = ByteArrayOrdinalMap.getOrdinal(pointerAndOrdinal);


                int sizeOfData = VarInt.readVInt(byteData.getUnderlyingArray(), pointer);
                pointer += VarInt.sizeOfVInt(sizeOfData);


                rec.position(pointer);


                add(ordinal, rec);

View Full Code Here


        for(int i=0;i<1000;i++) {


            int numElements = rand.nextInt(1000);


            ByteDataBuffer buf = new ByteDataBuffer();


            OrdinalMapping ordinalMapping = new OrdinalMapping();
            StateOrdinalMapping stateOrdinalMap = ordinalMapping.createStateOrdinalMapping("ElementType", numElements);
            for(int j=0;j<numElements;j++) {
                stateOrdinalMap.setMappedOrdinal(j, rand.nextInt(10000));
                VarInt.writeVInt(buf, j);
            }


            ByteDataBuffer toDataBuffer = copyToBufferAndRemapOrdinals(schema, buf, ordinalMapping);


            Assert.assertEquals(100, VarInt.readVInt(toDataBuffer.getUnderlyingArray(), 0));


            int listDataSize = VarInt.readVInt(toDataBuffer.getUnderlyingArray(), 1);


            long position = VarInt.sizeOfVInt(listDataSize) + 1;
            long endPosition = position + listDataSize;


            int counter = 0;


            while(position < endPosition) {
                int encoded = VarInt.readVInt(toDataBuffer.getUnderlyingArray(), position);
                position += VarInt.sizeOfVInt(encoded);
                Assert.assertEquals(encoded, stateOrdinalMap.getMappedOrdinal(counter));
                counter++;
            }
        }

View Full Code Here


        for(int i=0;i<1000;i++) {


            int numElements = rand.nextInt(1000);


            ByteDataBuffer buf = new ByteDataBuffer();


            BitSet usedMappings = new BitSet();


            Map<Integer, Integer> expectedMap = new HashMap<Integer, Integer>();


            OrdinalMapping ordinalMapping = new OrdinalMapping();
            StateOrdinalMapping stateOrdinalMap = ordinalMapping.createStateOrdinalMapping("ElementType", numElements);
            for(int j=0;j<numElements;j++) {


                int mapping = getRandomMapping(rand, usedMappings);


                stateOrdinalMap.setMappedOrdinal(j, mapping);
                VarInt.writeVInt(buf, j);
                VarInt.writeVInt(buf, j == 0 ? 0 : 1);


                expectedMap.put(mapping, mapping);
            }


            ByteDataBuffer toDataBuffer = copyToBufferAndRemapOrdinals(schema, buf, ordinalMapping);


            Assert.assertEquals(100, VarInt.readVInt(toDataBuffer.getUnderlyingArray(), 0));


            Map<Integer, Integer> actualMap = new HashMap<Integer, Integer>();


            int listDataSize = VarInt.readVInt(toDataBuffer.getUnderlyingArray(), 1);


            long position = VarInt.sizeOfVInt(listDataSize) + 1;
            long endPosition = position + listDataSize;


            int prevValue = 0;
            while(position < endPosition) {
                int key = VarInt.readVInt(toDataBuffer.getUnderlyingArray(), position);
                position += VarInt.sizeOfVInt(key);
                int delta = VarInt.readVInt(toDataBuffer.getUnderlyingArray(), position);
                int value = prevValue + delta;
                prevValue = value;
                position += VarInt.sizeOfVInt(delta);


                actualMap.put(key, value);

View Full Code Here


        for(int i=0;i<1000;i++) {


            int numElements = rand.nextInt(1000);


            ByteDataBuffer buf = new ByteDataBuffer();


            Set<Integer> expectedSet = new HashSet<Integer>();
            BitSet usedMappings = new BitSet();


            OrdinalMapping ordinalMapping = new OrdinalMapping();
            StateOrdinalMapping stateOrdinalMap = ordinalMapping.createStateOrdinalMapping("ElementType", numElements);
            for(int j=0;j<numElements;j++) {
                int mapping = getRandomMapping(rand, usedMappings);
                stateOrdinalMap.setMappedOrdinal(j, mapping);
                VarInt.writeVInt(buf, j == 0 ? 0 : 1);
                expectedSet.add(mapping);
            }


            ByteDataBuffer toDataBuffer = copyToBufferAndRemapOrdinals(schema, buf, ordinalMapping);


            Assert.assertEquals(100, VarInt.readVInt(toDataBuffer.getUnderlyingArray(), 0));


            int listDataSize = VarInt.readVInt(toDataBuffer.getUnderlyingArray(), 1);


            long position = VarInt.sizeOfVInt(listDataSize) + 1;
            long endPosition = position + listDataSize;


            Set<Integer> actualSet = new HashSet<Integer>();
            int prevOrdinal = 0;
            while(position < endPosition) {
                int deltaOrdinal = VarInt.readVInt(toDataBuffer.getUnderlyingArray(), position);
                position += VarInt.sizeOfVInt(deltaOrdinal);
                int currentOrdinal = prevOrdinal + deltaOrdinal;
                prevOrdinal = currentOrdinal;
                actualSet.add(currentOrdinal);
            }

View Full Code Here

    /**
     * Serialize an integer, use zig-zag encoding to (probably) get a small positive value, then encode the result as a variable-byte integer.
     */
    @Override
    public void serializePrimitive(FlatBlobSerializationRecord rec, String fieldName, int value) {
        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(fieldName);


        // zig zag encoding
        VarInt.writeVInt(fieldBuffer, (value << 1) ^ (value >> 31));
    }

View Full Code Here




    private ByteDataBuffer copyToBufferAndRemapOrdinals(FastBlobSchema schema,
            ByteDataBuffer buf, OrdinalMapping ordinalMapping) {


        ByteDataBuffer fromDataBuffer = new ByteDataBuffer();
        VarInt.writeVInt(fromDataBuffer, 100);
        VarInt.writeVInt(fromDataBuffer, (int)buf.length());


        fromDataBuffer.copyFrom(buf);


        FastBlobDeserializationRecord rec = new FastBlobDeserializationRecord(schema, fromDataBuffer.getUnderlyingArray());
        rec.position(0);


        ByteDataBuffer toDataBuffer = new ByteDataBuffer();
        new OrdinalRemapper(ordinalMapping).remapOrdinals(rec, toDataBuffer);
        return toDataBuffer;
    }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of com.netflix.zeno.fastblob.record.ByteDataBuffer

com.netflix.zeno.fastblob.CompatibilitySerializerTest

com.netflix.zeno.fastblob.FastBlobFrameworkSerializer

com.netflix.zeno.fastblob.FastBlobHeapFriendlyClientFrameworkSerializer

com.netflix.zeno.fastblob.io.FastBlobReader

com.netflix.zeno.fastblob.NFTypeSerializerTest

com.netflix.zeno.fastblob.OrdinalRemapperTest

com.netflix.zeno.fastblob.state.ByteArrayOrdinalMap

com.netflix.zeno.fastblob.state.ByteArrayOrdinalMapTest

com.netflix.zeno.fastblob.state.ByteDataBufferTest

com.netflix.zeno.fastblob.state.compressed.ByteSequenceRetainer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.