Package com.netflix.zeno.fastblob.record

Examples of com.netflix.zeno.fastblob.record.ByteDataBuffer

48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
        TestSerializer testSerializer = new TestSerializer();
        testSerializer.setSerializationFramework(new FastBlobStateEngine(serializerFactory));
        FastBlobSerializationRecord rec = new FastBlobSerializationRecord(testSerializer.getFastBlobSchema());
        testSerializer.serialize(null, rec);

        ByteDataBuffer buf = new ByteDataBuffer();

        rec.writeDataTo(buf);

        FastBlobDeserializationRecord deserializeRec = new FastBlobDeserializationRecord(testSerializer.getFastBlobSchema(), buf.getUnderlyingArray());
        deserializeRec.position(0);

        Boolean deserialized = testSerializer.deserialize(deserializeRec);

        Assert.assertEquals(Boolean.TRUE, deserialized);
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
    public void testBytes() throws Exception {
        NFTypeSerializer<Object[]> testSerializer = framework.getSerializer(TestSerializerBytes.NAME);
        FastBlobSerializationRecord rec = new FastBlobSerializationRecord(testSerializer.getFastBlobSchema());
        testSerializer.serialize(new Object[]{bytes, bytes2}, rec);

        ByteDataBuffer buf = new ByteDataBuffer();

        rec.writeDataTo(buf);

        FastBlobDeserializationRecord deserializeRec = new FastBlobDeserializationRecord(testSerializer.getFastBlobSchema(), buf.getUnderlyingArray());
        deserializeRec.position(0);

        Object[] deserialized = testSerializer.deserialize(deserializeRec);

        Assert.assertArrayEquals(bytes, (byte[])deserialized[0]);
View Full Code Here
67
68
69
70
71
72
73
74
75
76
        this(262144);
    }

    public ByteArrayOrdinalMap(int bufferSize) {
        this.freeOrdinalTracker = new FreeOrdinalTracker();
        this.byteData = new ByteDataBuffer(bufferSize);
        this.pointersAndOrdinals = emptyKeyArray(256);
        this.sizeBeforeGrow = 179; /// 70% load factor
        this.size = 0;
    }
View Full Code Here
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
        for(int i=0;i<numThreads;i++) {
            final int threadNumber = i;
            executor.submit( new Runnable() {
                @Override
                public void run() {
                    final ByteDataBuffer mappedBuffer = new ByteDataBuffer();
                    final FastBlobDeserializationRecord rec = new FastBlobDeserializationRecord(destState.getSchema(), byteData.getUnderlyingArray());
                    final boolean imageMembershipsFlags[] = new boolean[imageMemberships.length];
                    final OrdinalRemapper remapper = new OrdinalRemapper(ordinalMapping);

                    for(int j = threadNumber;j < pointersAndOrdinals.length();j += numThreads) {
                        long pointerAndOrdinal = pointersAndOrdinals.get(j);
                        if(pointerAndOrdinal != EMPTY_BUCKET_VALUE) {
                            long pointer = pointerAndOrdinal & 0xFFFFFFFFFL;
                            int ordinal = (int)(pointerAndOrdinal >> 36);

                            for(int imageIndex=0;imageIndex<imageMemberships.length;imageIndex++) {
                                imageMembershipsFlags[imageIndex] = imageMemberships[imageIndex].get(ordinal);
                            }

                            int sizeOfData = VarInt.readVInt(byteData.getUnderlyingArray(), pointer);
                            pointer += VarInt.sizeOfVInt(sizeOfData);

                            rec.position(pointer);
                            remapper.remapOrdinals(rec, mappedBuffer);

                            int newOrdinal = destState.addData(mappedBuffer, FastBlobImageUtils.toLong(imageMembershipsFlags));
                            stateOrdinalMapping.setMappedOrdinal(ordinal, newOrdinal);

                            mappedBuffer.reset();
                        }
                    }
                }
            });
        }
View Full Code Here
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
        for(int i=0;i<keys.length;i++) {
            keys[i] = (VarInt.readVLong(is) << 36) | VarInt.readVLong(is);
        }

        ByteDataBuffer byteData = new ByteDataBuffer(262144);

        long byteDataSize = VarInt.readVLong(is);

        for(long i=0;i<byteDataSize;i++) {
            byteData.write((byte)is.read());
        }

        FreeOrdinalTracker freeOrdinalTracker = FreeOrdinalTracker.deserializeFrom(is);

        ByteArrayOrdinalMap deserializedMap = new ByteArrayOrdinalMap(keys, byteData, freeOrdinalTracker, hashedKeyArraySize);
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
        Assert.assertEquals(null, deserialized[1]);
    }

    private FastBlobDeserializationRecord serializeDeserialize(NFTypeSerializer<String[]> testSerializer, FastBlobSerializationRecord rec) throws IOException {

        ByteDataBuffer buf = new ByteDataBuffer();

        rec.writeDataTo(buf);

        FastBlobDeserializationRecord deserializeRec = new FastBlobDeserializationRecord(testSerializer.getFastBlobSchema(), buf.getUnderlyingArray());

        deserializeRec.position(0);

        return deserializeRec;
    }
View Full Code Here
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
public class ByteDataBufferTest {

    @Test
    public void recordsData() {
        ByteDataBuffer buf = new ByteDataBuffer(256);

        for(int i=0;i<1000;i++) {
            buf.write((byte)i);
        }

        for(int i=0;i<1000;i++) {
            assertEquals((byte)i, buf.get(i));
        }
    }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
        }
    }

    @Test
    public void canBeReset() {
        ByteDataBuffer buf = new ByteDataBuffer(256);

        for(int i=0;i<1000;i++) {
            buf.write((byte)10);
        }

        buf.reset();

        for(int i=0;i<1000;i++) {
            buf.write((byte)i);
        }

        for(int i=0;i<1000;i++) {
            assertEquals((byte)i, buf.get(i));
        }
    }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
    @Test
    public void compacts() {
        ByteArrayOrdinalMap map = new ByteArrayOrdinalMap();

        ByteDataBuffer buf = new ByteDataBuffer();

        /// add 1000 entries
        for(int i=0;i<1000;i++) {
            VarInt.writeVInt(buf, 10000 + i);
            map.getOrAssignOrdinal(buf);
            buf.reset();
        }


        /// mark half of the entries used
        ThreadSafeBitSet bitSet = new ThreadSafeBitSet(10);
        for(int i=0;i<1000;i+=2) {
            bitSet.set(i);
        }

        /// compact away the unused entries
        map.compact(bitSet);

        /// ensure that the used entries are still available
        for(int i=0;i<1000;i+=2) {
            VarInt.writeVInt(buf, 10000 + i);
            Assert.assertEquals(i, map.getOrAssignOrdinal(buf));
            buf.reset();
        }


        /// track the ordinals which are assigned to new values
        Set<Integer> newlyAssignedOrdinals = new HashSet<Integer>();
        for(int i=1;i<1000;i+=2) {
            VarInt.writeVInt(buf, 50230532 + i);
            int newOrdinal = map.getOrAssignOrdinal(buf);
            newlyAssignedOrdinals.add(newOrdinal);
            buf.reset();
        }

        /// those ordinals should be the recycled ones after the compact.
        for(int i=1;i<1000;i+=2) {
            Assert.assertTrue(newlyAssignedOrdinals.contains(i));
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    @Test
    public void clientSideHeapSafeUsageTest() {
        ByteArrayOrdinalMap map = new ByteArrayOrdinalMap();

        ByteDataBuffer buf = new ByteDataBuffer();

        for(int i=0;i<1000;i++) {
            VarInt.writeVInt(buf, 10000 + i);
            map.put(buf, i + 50);
            buf.reset();
        }

        for(int i=0;i<1000;i++) {
            VarInt.writeVInt(buf, 10000 + i);
            Assert.assertEquals(i + 50, map.get(buf));
            buf.reset();
        }

        for(int i=0;i<1000;i++) {
            VarInt.writeVInt(buf, 20000 + i);
            Assert.assertEquals(-1, map.get(buf));
            buf.reset();
        }

        map.clear();

        Assert.assertEquals(0, map.getDataSize());

        for(int i=0;i<5000;i++) {
            VarInt.writeVInt(buf, 20000 + i);
            Assert.assertEquals(-1, map.get(buf));
            buf.reset();
        }

        for(int i=0;i<5000;i++) {
            VarInt.writeVInt(buf, 20000 + i);
            map.put(buf, i + 50);
            buf.reset();
        }

    }
View Full Code Here
TOP

Related Classes of com.netflix.zeno.fastblob.record.ByteDataBuffer

  • com.netflix.zeno.fastblob.CompatibilitySerializerTest
  • com.netflix.zeno.fastblob.FastBlobFrameworkSerializer
  • com.netflix.zeno.fastblob.FastBlobHeapFriendlyClientFrameworkSerializer
  • com.netflix.zeno.fastblob.io.FastBlobReader
  • com.netflix.zeno.fastblob.NFTypeSerializerTest
  • com.netflix.zeno.fastblob.OrdinalRemapperTest
  • com.netflix.zeno.fastblob.state.ByteArrayOrdinalMap
  • com.netflix.zeno.fastblob.state.ByteArrayOrdinalMapTest
  • com.netflix.zeno.fastblob.state.ByteDataBufferTest
  • com.netflix.zeno.fastblob.state.compressed.ByteSequenceRetainer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.