Examples of SSTableWriter


Examples of org.apache.cassandra.io.sstable.SSTableWriter

    @Test
    public void testEnumeratekeys() throws IOException
    {
        File tempSS = tempSSTableFile("Keyspace1", "Standard1");
        ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
        SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
       
        // Add rowA
        cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
        writer.append(Util.dk("rowA"), cfamily);
        cfamily.clear();
       
        // Add rowB
        cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
        writer.append(Util.dk("rowB"), cfamily);
        cfamily.clear();
    
        writer.closeAndOpenReader();
       
        // Enumerate and verify
        File temp = File.createTempFile("Standard1", ".txt");
        SSTableExport.enumeratekeys(writer.getFilename(), new PrintStream(temp.getPath()));

       
        FileReader file = new FileReader(temp);
        char[] buf = new char[(int) temp.length()];
        file.read(buf);
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

    @Test
    public void testExportSimpleCf() throws IOException
    {
        File tempSS = tempSSTableFile("Keyspace1", "Standard1");
        ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
        SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
       
        int nowInSec = (int)(System.currentTimeMillis() / 1000) + 42; //live for 42 seconds
        // Add rowA
        cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
        cfamily.addColumn(null, new ExpiringColumn(ByteBufferUtil.bytes("colExp"), ByteBufferUtil.bytes("valExp"), System.currentTimeMillis(), 42, nowInSec));
        writer.append(Util.dk("rowA"), cfamily);
        cfamily.clear();
       
        // Add rowB
        cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
        writer.append(Util.dk("rowB"), cfamily);
        cfamily.clear();

        // Add rowExclude
        cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), System.currentTimeMillis());
        writer.append(Util.dk("rowExclude"), cfamily);
        cfamily.clear();

        SSTableReader reader = writer.closeAndOpenReader();
       
        // Export to JSON and verify
        File tempJson = File.createTempFile("Standard1", ".json");
        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
       
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

    @Test
    public void testExportSuperCf() throws IOException
    {
        File tempSS = tempSSTableFile("Keyspace1", "Super4");
        ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Super4");
        SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
       
        // Add rowA
        cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superA"), ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
        writer.append(Util.dk("rowA"), cfamily);
        cfamily.clear();
       
        // Add rowB
        cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superB"), ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
        writer.append(Util.dk("rowB"), cfamily);
        cfamily.clear();

        // Add rowExclude
        cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superX"), ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), System.currentTimeMillis());
        writer.append(Util.dk("rowExclude"), cfamily);
        cfamily.clear();

        SSTableReader reader = writer.closeAndOpenReader();
       
        // Export to JSON and verify
        File tempJson = File.createTempFile("Super4", ".json");
        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
       
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

    @Test
    public void testRoundTripStandardCf() throws IOException, ParseException
    {
        File tempSS = tempSSTableFile("Keyspace1", "Standard1");
        ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
        SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
       
        // Add rowA
        cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
        writer.append(Util.dk("rowA"), cfamily);
        cfamily.clear();

        // Add rowExclude
        cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
        writer.append(Util.dk("rowExclude"), cfamily);
        cfamily.clear();

        SSTableReader reader = writer.closeAndOpenReader();
       
        // Export to JSON and verify
        File tempJson = File.createTempFile("Standard1", ".json");
        SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
       
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

                // a crash could cause data loss.
                cfs.markCompacted(toCompact, compactionType);
                return;
            }

            SSTableWriter writer = cfs.createCompactionWriter(keysPerSSTable, cfs.directories.getLocationForDisk(dataDirectory), toCompact);
            writers.add(writer);
            while (nni.hasNext())
            {
                if (ci.isStopRequested())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());

                AbstractCompactedRow row = nni.next();
                if (row.isEmpty())
                    continue;

                RowIndexEntry indexEntry = writer.append(row);
                totalkeysWritten++;

                if (DatabaseDescriptor.getPreheatKeyCache())
                {
                    for (SSTableReader sstable : toCompact)
                    {
                        if (sstable.getCachedPosition(row.key, false) != null)
                        {
                            cachedKeys.put(row.key, indexEntry);
                            break;
                        }
                    }
                }
                if (!nni.hasNext() || newSSTableSegmentThresholdReached(writer))
                {
                    SSTableReader toIndex = writer.closeAndOpenReader(getMaxDataAge(toCompact));
                    cachedKeyMap.put(toIndex, cachedKeys);
                    sstables.add(toIndex);
                    if (nni.hasNext())
                    {
                        writer = cfs.createCompactionWriter(keysPerSSTable, cfs.directories.getLocationForDisk(dataDirectory), toCompact);
                        writers.add(writer);
                        cachedKeys = new HashMap<DecoratedKey, RowIndexEntry>();
                    }
                }
            }
        }
        catch (Throwable t)
        {
            for (SSTableWriter writer : writers)
                writer.abort();
            throw Throwables.propagate(t);
        }
        finally
        {
            try
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

        int importedKeys = 0;
        long start = System.currentTimeMillis();
        Map<?, ?> data = parser.readValueAs(new TypeReference<Map<?, ?>>() {});

        keyCountToImport = (keyCountToImport == null) ? data.size() : keyCountToImport;
        SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);

        System.out.printf("Importing %s keys...%n", keyCountToImport);

        // sort by dk representation, but hold onto the hex version
        SortedMap<DecoratedKey,String> decoratedKeys = new TreeMap<DecoratedKey,String>();

        for (Object keyObject : data.keySet())
        {
            String key = (String) keyObject;
            decoratedKeys.put(partitioner.decorateKey(hexToBytes(key)), key);
        }

        for (Map.Entry<DecoratedKey, String> rowKey : decoratedKeys.entrySet())
        {
            if (columnFamily.getColumnFamilyType() == ColumnFamilyType.Super)
            {
                addToSuperCF((Map<?, ?>) data.get(rowKey.getValue()), columnFamily);
            }
            else
            {
                addToStandardCF((List<?>) data.get(rowKey.getValue()), columnFamily);
            }

            writer.append(rowKey.getKey(), columnFamily);
            columnFamily.clear();

            importedKeys++;

            long current = System.currentTimeMillis();

            if (current - start >= 5000) // 5 secs.
            {
                System.out.printf("Currently imported %d keys.%n", importedKeys);
                start = current;
            }

            if (keyCountToImport == importedKeys)
                break;
        }

        writer.closeAndOpenReader();

        return importedKeys;
    }
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

        }

        System.out.printf("Importing %s keys...%n", keyCountToImport);

        parser = getParser(jsonFile); // renewing parser
        SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);

        int lineNumber = 1;
        DecoratedKey prevStoredKey = null;

        while (parser.nextToken() != null)
        {
            String key = parser.getCurrentName();

            if (key != null)
            {
                String tokenName = parser.nextToken().name();

                if (tokenName.equals("START_ARRAY"))
                {
                    if (columnFamily.getColumnFamilyType() == ColumnFamilyType.Super)
                    {
                        throw new RuntimeException("Can't write Standard columns to the Super Column Family.");
                    }

                    List<?> columns = parser.readValueAs(new TypeReference<List<?>>() {});
                    addToStandardCF(columns, columnFamily);
                }
                else if (tokenName.equals("START_OBJECT"))
                {
                    if (columnFamily.getColumnFamilyType() == ColumnFamilyType.Standard)
                    {
                        throw new RuntimeException("Can't write Super columns to the Standard Column Family.");
                    }

                    Map<?, ?> columns = parser.readValueAs(new TypeReference<Map<?, ?>>() {});
                    addToSuperCF(columns, columnFamily);
                }
                else
                {
                    throw new UnsupportedOperationException("Only Array or Hash allowed as row content.");
                }

                DecoratedKey currentKey = partitioner.decorateKey(hexToBytes(key));

                if (prevStoredKey != null && prevStoredKey.compareTo(currentKey) != -1)
                {
                    System.err.printf("Line %d: Key %s is greater than previous, collection is not sorted properly. Aborting import. You might need to delete SSTables manually.%n", lineNumber, key);
                    return -1;
                }

                // saving decorated key
                writer.append(currentKey, columnFamily);
                columnFamily.clear();

                prevStoredKey = currentKey;
                importedKeys++;
                lineNumber++;

                long current = System.currentTimeMillis();

                if (current - start >= 5000) // 5 secs.
                {
                    System.out.printf("Currently imported %d keys.%n", importedKeys);
                    start = current;
                }

                if (keyCountToImport == importedKeys)
                    break;
            }
        }

        writer.closeAndOpenReader();

        return importedKeys;
    }
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

        int importedKeys = 0;
        long start = System.currentTimeMillis();
        Map<?, ?> data = parser.readValueAs(new TypeReference<Map<?, ?>>() {});

        keyCountToImport = (keyCountToImport == null) ? data.size() : keyCountToImport;
        SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);

        System.out.printf("Importing %s keys...%n", keyCountToImport);

        // sort by dk representation, but hold onto the hex version
        SortedMap<DecoratedKey,String> decoratedKeys = new TreeMap<DecoratedKey,String>();

        for (Object keyObject : data.keySet())
        {
            String key = (String) keyObject;
            decoratedKeys.put(partitioner.decorateKey(hexToBytes(key)), key);
        }

        for (Map.Entry<DecoratedKey, String> rowKey : decoratedKeys.entrySet())
        {
            if (columnFamily.getType() == ColumnFamilyType.Super)
            {
                addToSuperCF((Map<?, ?>) data.get(rowKey.getValue()), columnFamily);
            }
            else
            {
                addToStandardCF((List<?>) data.get(rowKey.getValue()), columnFamily);
            }

            writer.append(rowKey.getKey(), columnFamily);
            columnFamily.clear();

            importedKeys++;

            long current = System.currentTimeMillis();

            if (current - start >= 5000) // 5 secs.
            {
                System.out.printf("Currently imported %d keys.%n", importedKeys);
                start = current;
            }

            if (keyCountToImport == importedKeys)
                break;
        }

        writer.closeAndOpenReader();

        return importedKeys;
    }
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

        }

        System.out.printf("Importing %s keys...%n", keyCountToImport);

        parser = getParser(jsonFile); // renewing parser
        SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);

        int lineNumber = 1;
        DecoratedKey prevStoredKey = null;

        while (parser.nextToken() != null)
        {
            String key = parser.getCurrentName();

            if (key != null)
            {
                String tokenName = parser.nextToken().name();

                if (tokenName.equals("START_ARRAY"))
                {
                    if (columnFamily.getType() == ColumnFamilyType.Super)
                    {
                        throw new RuntimeException("Can't write Standard columns to the Super Column Family.");
                    }

                    List<?> columns = parser.readValueAs(new TypeReference<List<?>>() {});
                    addToStandardCF(columns, columnFamily);
                }
                else if (tokenName.equals("START_OBJECT"))
                {
                    if (columnFamily.getType() == ColumnFamilyType.Standard)
                    {
                        throw new RuntimeException("Can't write Super columns to the Standard Column Family.");
                    }

                    Map<?, ?> columns = parser.readValueAs(new TypeReference<Map<?, ?>>() {});
                    addToSuperCF(columns, columnFamily);
                }
                else
                {
                    throw new UnsupportedOperationException("Only Array or Hash allowed as row content.");
                }

                DecoratedKey currentKey = partitioner.decorateKey(hexToBytes(key));

                if (prevStoredKey != null && prevStoredKey.compareTo(currentKey) != -1)
                {
                    System.err.printf("Line %d: Key %s is greater than previous, collection is not sorted properly. Aborting import. You might need to delete SSTables manually.%n", lineNumber, key);
                    return -1;
                }

                // saving decorated key
                writer.append(currentKey, columnFamily);
                columnFamily.clear();

                prevStoredKey = currentKey;
                importedKeys++;
                lineNumber++;

                long current = System.currentTimeMillis();

                if (current - start >= 5000) // 5 secs.
                {
                    System.out.printf("Currently imported %d keys.%n", importedKeys);
                    start = current;
                }

                if (keyCountToImport == importedKeys)
                    break;
            }
        }

        writer.closeAndOpenReader();

        return importedKeys;
    }
View Full Code Here

Examples of org.apache.cassandra.io.sstable.SSTableWriter

                                      + keySize // keys in data file
                                      + currentThroughput.get()) // data
                                     * 1.2); // bloom filter and row index overhead
        SSTableReader ssTable;
        // errors when creating the writer that may leave empty temp files.
        SSTableWriter writer = cfs.createFlushWriter(columnFamilies.size(), estimatedSize, context);
        try
        {
            // (we can't clear out the map as-we-go to free up memory,
            //  since the memtable is being used for queries in the "pending flush" category)
            for (Map.Entry<RowPosition, ColumnFamily> entry : columnFamilies.entrySet())
            {
                ColumnFamily cf = entry.getValue();
                if (cf.isMarkedForDelete())
                {
                    // Pedantically, you could purge column level tombstones that are past GcGRace when writing to the SSTable.
                    // But it can result in unexpected behaviour where deletes never make it to disk,
                    // as they are lost and so cannot override existing column values. So we only remove deleted columns if there
                    // is a CF level tombstone to ensure the delete makes it into an SSTable.
                    ColumnFamilyStore.removeDeletedColumnsOnly(cf, Integer.MIN_VALUE);
                }
                writer.append((DecoratedKey)entry.getKey(), cf);
            }

            ssTable = writer.closeAndOpenReader();
        }
        catch (Exception e)
        {
            writer.abort();
            throw FBUtilities.unchecked(e);
        }
        logger.info(String.format("Completed flushing %s (%d bytes)",
                                  ssTable.getFilename(), new File(ssTable.getFilename()).length()));
        return ssTable;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.