Package org.apache.cassandra.io.sstable

Examples of org.apache.cassandra.io.sstable.SSTableRewriter


        long minRepairedAt = getMinRepairedAt(actuallyCompact);
        // we only need the age of the data that we're actually retaining
        long maxAge = getMaxDataAge(actuallyCompact);
        if (collector != null)
            collector.beginCompaction(ci);
        SSTableRewriter writer = new SSTableRewriter(cfs, sstables, maxAge, compactionType, offline);
        try
        {
            if (!iter.hasNext())
            {
                // don't mark compacted in the finally block, since if there _is_ nondeleted data,
                // we need to sync it (via closeAndOpen) first, so there is no period during which
                // a crash could cause data loss.
                cfs.markObsolete(sstables, compactionType);
                return;
            }

            writer.switchWriter(createCompactionWriter(sstableDirectory, keysPerSSTable, minRepairedAt));
            while (iter.hasNext())
            {
                if (ci.isStopRequested())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());

                AbstractCompactedRow row = iter.next();
                if (writer.append(row) != null)
                {
                    totalKeysWritten++;
                    if (newSSTableSegmentThresholdReached(writer.currentWriter()))
                    {
                        writer.switchWriter(createCompactionWriter(sstableDirectory, keysPerSSTable, minRepairedAt));
                    }
                }
            }

            // don't replace old sstables yet, as we need to mark the compaction finished in the system table
            writer.finish(false);
        }
        catch (Throwable t)
        {
            writer.abort();
            throw t;
        }
        finally
        {
            controller.close();

            // point of no return -- the new sstables are live on disk; next we'll start deleting the old ones
            // (in replaceCompactedSSTables)
            if (taskId != null)
                SystemKeyspace.finishCompaction(taskId);

            if (collector != null)
                collector.finishCompaction(ci);

            try
            {
                // We don't expect this to throw, but just in case, we do it after the cleanup above, to make sure
                // we don't end up with compaction information hanging around indefinitely in limbo.
                iter.close();
            }
            catch (IOException e)
            {
                throw new RuntimeException(e);
            }
        }

        Collection<SSTableReader> oldSStables = this.sstables;
        List<SSTableReader> newSStables = writer.finished();
        if (!offline)
            cfs.getDataTracker().markCompactedSSTablesReplaced(oldSStables, newSStables, compactionType);

        // log a bunch of statistics about the result and save to system table compaction_history
        long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
View Full Code Here


        ICompactionScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
        CleanupInfo ci = new CleanupInfo(sstable, scanner);

        metrics.beginCompaction(ci);
        SSTableRewriter writer = new SSTableRewriter(cfs, new HashSet<>(ImmutableSet.of(sstable)), sstable.maxDataAge, OperationType.CLEANUP, false);

        try (CompactionController controller = new CompactionController(cfs, Collections.singleton(sstable), getDefaultGcBefore(cfs)))
        {
            writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));

            while (scanner.hasNext())
            {
                if (ci.isStopRequested())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());

                SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
                row = cleanupStrategy.cleanup(row);
                if (row == null)
                    continue;
                AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(row));
                if (writer.append(compactedRow) != null)
                    totalkeysWritten++;
            }

            // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
            cfs.indexManager.flushIndexesBlocking();

            writer.finish();
        }
        catch (Throwable e)
        {
            writer.abort();
            throw Throwables.propagate(e);
        }
        finally
        {
            scanner.close();
            metrics.finishCompaction(ci);
        }

        List<SSTableReader> results = writer.finished();
        if (!results.isEmpty())
        {
            String format = "Cleaned up to %s.  %,d to %,d (~%d%% of original) bytes for %,d keys.  Time: %,dms.";
            long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
            long startsize = sstable.onDiskLength();
View Full Code Here

            logger.info("Anticompacting {}", sstable);
            Set<SSTableReader> sstableAsSet = new HashSet<>();
            sstableAsSet.add(sstable);

            File destination = cfs.directories.getDirectoryForNewSSTables();
            SSTableRewriter repairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);
            SSTableRewriter unRepairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);

            AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();
            List<ICompactionScanner> scanners = strategy.getScanners(Arrays.asList(sstable));

            try (CompactionController controller = new CompactionController(cfs, new HashSet<>(Collections.singleton(sstable)), CFMetaData.DEFAULT_GC_GRACE_SECONDS))
            {
                repairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable));
                unRepairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstable));

                CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners, controller);

                try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator())
                {
                    while(iter.hasNext())
                    {
                        AbstractCompactedRow row = iter.next();
                        // if current range from sstable is repaired, save it into the new repaired sstable
                        if (Range.isInRanges(row.key.getToken(), ranges))
                        {
                            repairedSSTableWriter.append(row);
                            repairedKeyCount++;
                        }
                        // otherwise save into the new 'non-repaired' table
                        else
                        {
                            unRepairedSSTableWriter.append(row);
                            unrepairedKeyCount++;
                        }
                    }
                }
                // we have the same readers being rewritten by both writers, so we ask the first one NOT to close them
                // so that the second one can do so safely, without leaving us with references < 0 or any other ugliness
                repairedSSTableWriter.finish(false, repairedAt);
                unRepairedSSTableWriter.finish(ActiveRepairService.UNREPAIRED_SSTABLE);
                // add repaired table with a non-null timestamp field to be saved in SSTableMetadata#repairedAt
                anticompactedSSTables.addAll(repairedSSTableWriter.finished());
                anticompactedSSTables.addAll(unRepairedSSTableWriter.finished());
            }
            catch (Throwable e)
            {
                logger.error("Error anticompacting " + sstable, e);
                repairedSSTableWriter.abort();
                unRepairedSSTableWriter.abort();
            }
        }
        String format = "Repaired {} keys of {} for {}/{}";
        logger.debug(format, repairedKeyCount, (repairedKeyCount + unrepairedKeyCount), cfs.keyspace, cfs.getColumnFamilyName());
        String format2 = "Anticompaction completed successfully, anticompacted from {} to {} sstable(s).";
View Full Code Here

        ICompactionScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
        CleanupInfo ci = new CleanupInfo(sstable, scanner);

        metrics.beginCompaction(ci);
        SSTableRewriter writer = new SSTableRewriter(cfs, new HashSet<>(ImmutableSet.of(sstable)), sstable.maxDataAge, OperationType.CLEANUP, false);

        try (CompactionController controller = new CompactionController(cfs, Collections.singleton(sstable), getDefaultGcBefore(cfs)))
        {
            writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));

            while (scanner.hasNext())
            {
                if (ci.isStopRequested())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());

                SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
                row = cleanupStrategy.cleanup(row);
                if (row == null)
                    continue;
                AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(row));
                if (writer.append(compactedRow) != null)
                    totalkeysWritten++;
            }

            // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
            cfs.indexManager.flushIndexesBlocking();

            writer.finish();
        }
        catch (Throwable e)
        {
            writer.abort();
            throw Throwables.propagate(e);
        }
        finally
        {
            scanner.close();
            metrics.finishCompaction(ci);
        }

        List<SSTableReader> results = writer.finished();
        if (!results.isEmpty())
        {
            String format = "Cleaned up to %s.  %,d to %,d (~%d%% of original) bytes for %,d keys.  Time: %,dms.";
            long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
            long startsize = sstable.onDiskLength();
View Full Code Here

            logger.info("Anticompacting {}", sstable);
            Set<SSTableReader> sstableAsSet = new HashSet<>();
            sstableAsSet.add(sstable);

            File destination = cfs.directories.getDirectoryForCompactedSSTables();
            SSTableRewriter repairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);
            SSTableRewriter unRepairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);

            AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();
            List<ICompactionScanner> scanners = strategy.getScanners(Arrays.asList(sstable));

            try (CompactionController controller = new CompactionController(cfs, new HashSet<>(Collections.singleton(sstable)), CFMetaData.DEFAULT_GC_GRACE_SECONDS))
            {
                repairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable));
                unRepairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstable));

                CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners, controller);

                try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator())
                {
                    while(iter.hasNext())
                    {
                        AbstractCompactedRow row = iter.next();
                        // if current range from sstable is repaired, save it into the new repaired sstable
                        if (Range.isInRanges(row.key.getToken(), ranges))
                        {
                            repairedSSTableWriter.append(row);
                            repairedKeyCount++;
                        }
                        // otherwise save into the new 'non-repaired' table
                        else
                        {
                            unRepairedSSTableWriter.append(row);
                            unrepairedKeyCount++;
                        }
                    }
                }
                // we have the same readers being rewritten by both writers, so we ask the first one NOT to close them
                // so that the second one can do so safely, without leaving us with references < 0 or any other ugliness
                repairedSSTableWriter.finish(false, repairedAt);
                unRepairedSSTableWriter.finish(ActiveRepairService.UNREPAIRED_SSTABLE);
                // add repaired table with a non-null timestamp field to be saved in SSTableMetadata#repairedAt
                anticompactedSSTables.addAll(repairedSSTableWriter.finished());
                anticompactedSSTables.addAll(unRepairedSSTableWriter.finished());
            }
            catch (Throwable e)
            {
                logger.error("Error anticompacting " + sstable, e);
                repairedSSTableWriter.abort();
                unRepairedSSTableWriter.abort();
            }
        }
        String format = "Repaired {} keys of {} for {}/{}";
        logger.debug(format, repairedKeyCount, (repairedKeyCount + unrepairedKeyCount), cfs.keyspace, cfs.getColumnFamilyName());
        String format2 = "Anticompaction completed successfully, anticompacted from {} to {} sstable(s).";
View Full Code Here

        ICompactionScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
        CleanupInfo ci = new CleanupInfo(sstable, scanner);

        metrics.beginCompaction(ci);
        SSTableRewriter writer = new SSTableRewriter(cfs, new HashSet<>(ImmutableSet.of(sstable)), sstable.maxDataAge, OperationType.CLEANUP, false);

        try (CompactionController controller = new CompactionController(cfs, Collections.singleton(sstable), getDefaultGcBefore(cfs)))
        {
            writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));

            while (scanner.hasNext())
            {
                if (ci.isStopRequested())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());

                SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
                row = cleanupStrategy.cleanup(row);
                if (row == null)
                    continue;
                AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(row));
                if (writer.append(compactedRow) != null)
                    totalkeysWritten++;
            }

            // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
            cfs.indexManager.flushIndexesBlocking();

            writer.finish();
        }
        catch (Throwable e)
        {
            writer.abort();
            throw Throwables.propagate(e);
        }
        finally
        {
            scanner.close();
            metrics.finishCompaction(ci);
        }

        List<SSTableReader> results = writer.finished();
        if (!results.isEmpty())
        {
            String format = "Cleaned up to %s.  %,d to %,d (~%d%% of original) bytes for %,d keys.  Time: %,dms.";
            long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
            long startsize = sstable.onDiskLength();
View Full Code Here

            logger.info("Anticompacting {}", sstable);
            Set<SSTableReader> sstableAsSet = new HashSet<>();
            sstableAsSet.add(sstable);

            File destination = cfs.directories.getDirectoryForNewSSTables();
            SSTableRewriter repairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);
            SSTableRewriter unRepairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);

            AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();
            List<ICompactionScanner> scanners = strategy.getScanners(Arrays.asList(sstable));

            try (CompactionController controller = new CompactionController(cfs, new HashSet<>(Collections.singleton(sstable)), CFMetaData.DEFAULT_GC_GRACE_SECONDS))
            {
                repairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable));
                unRepairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstable));

                CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners, controller);

                try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator())
                {
                    while(iter.hasNext())
                    {
                        AbstractCompactedRow row = iter.next();
                        // if current range from sstable is repaired, save it into the new repaired sstable
                        if (Range.isInRanges(row.key.getToken(), ranges))
                        {
                            repairedSSTableWriter.append(row);
                            repairedKeyCount++;
                        }
                        // otherwise save into the new 'non-repaired' table
                        else
                        {
                            unRepairedSSTableWriter.append(row);
                            unrepairedKeyCount++;
                        }
                    }
                }
                // we have the same readers being rewritten by both writers, so we ask the first one NOT to close them
                // so that the second one can do so safely, without leaving us with references < 0 or any other ugliness
                repairedSSTableWriter.finish(false, repairedAt);
                unRepairedSSTableWriter.finish(ActiveRepairService.UNREPAIRED_SSTABLE);
                // add repaired table with a non-null timestamp field to be saved in SSTableMetadata#repairedAt
                anticompactedSSTables.addAll(repairedSSTableWriter.finished());
                anticompactedSSTables.addAll(unRepairedSSTableWriter.finished());
            }
            catch (Throwable e)
            {
                logger.error("Error anticompacting " + sstable, e);
                repairedSSTableWriter.abort();
                unRepairedSSTableWriter.abort();
            }
        }
        String format = "Repaired {} keys of {} for {}/{}";
        logger.debug(format, repairedKeyCount, (repairedKeyCount + unrepairedKeyCount), cfs.keyspace, cfs.getColumnFamilyName());
        String format2 = "Anticompaction completed successfully, anticompacted from {} to {} sstable(s).";
View Full Code Here

        ICompactionScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
        CleanupInfo ci = new CleanupInfo(sstable, scanner);

        metrics.beginCompaction(ci);
        SSTableRewriter writer = new SSTableRewriter(cfs, new HashSet<>(ImmutableSet.of(sstable)), sstable.maxDataAge, OperationType.CLEANUP, false);

        try (CompactionController controller = new CompactionController(cfs, Collections.singleton(sstable), getDefaultGcBefore(cfs)))
        {
            writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));

            while (scanner.hasNext())
            {
                if (ci.isStopRequested())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());

                SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
                row = cleanupStrategy.cleanup(row);
                if (row == null)
                    continue;
                AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(row));
                if (writer.append(compactedRow) != null)
                    totalkeysWritten++;
            }

            // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
            cfs.indexManager.flushIndexesBlocking();

            writer.finish();
        }
        catch (Throwable e)
        {
            writer.abort();
            throw Throwables.propagate(e);
        }
        finally
        {
            scanner.close();
            metrics.finishCompaction(ci);
        }

        List<SSTableReader> results = writer.finished();
        if (!results.isEmpty())
        {
            String format = "Cleaned up to %s.  %,d to %,d (~%d%% of original) bytes for %,d keys.  Time: %,dms.";
            long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
            long startsize = sstable.onDiskLength();
View Full Code Here

            logger.info("Anticompacting {}", sstable);
            Set<SSTableReader> sstableAsSet = new HashSet<>();
            sstableAsSet.add(sstable);

            File destination = cfs.directories.getDirectoryForNewSSTables();
            SSTableRewriter repairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);
            SSTableRewriter unRepairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);

            AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();
            List<ICompactionScanner> scanners = strategy.getScanners(Arrays.asList(sstable));

            try (CompactionController controller = new CompactionController(cfs, new HashSet<>(Collections.singleton(sstable)), CFMetaData.DEFAULT_GC_GRACE_SECONDS))
            {
                repairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, sstable));
                unRepairedSSTableWriter.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstable));

                CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners, controller);

                try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator())
                {
                    while(iter.hasNext())
                    {
                        AbstractCompactedRow row = iter.next();
                        // if current range from sstable is repaired, save it into the new repaired sstable
                        if (Range.isInRanges(row.key.getToken(), ranges))
                        {
                            repairedSSTableWriter.append(row);
                            repairedKeyCount++;
                        }
                        // otherwise save into the new 'non-repaired' table
                        else
                        {
                            unRepairedSSTableWriter.append(row);
                            unrepairedKeyCount++;
                        }
                    }
                }
                // we have the same readers being rewritten by both writers, so we ask the first one NOT to close them
                // so that the second one can do so safely, without leaving us with references < 0 or any other ugliness
                repairedSSTableWriter.finish(false, repairedAt);
                unRepairedSSTableWriter.finish(ActiveRepairService.UNREPAIRED_SSTABLE);
                // add repaired table with a non-null timestamp field to be saved in SSTableMetadata#repairedAt
                anticompactedSSTables.addAll(repairedSSTableWriter.finished());
                anticompactedSSTables.addAll(unRepairedSSTableWriter.finished());
            }
            catch (Throwable e)
            {
                logger.error("Error anticompacting " + sstable, e);
                repairedSSTableWriter.abort();
                unRepairedSSTableWriter.abort();
            }
        }
        String format = "Repaired {} keys of {} for {}/{}";
        logger.debug(format, repairedKeyCount, (repairedKeyCount + unrepairedKeyCount), cfs.keyspace, cfs.getColumnFamilyName());
        String format2 = "Anticompaction completed successfully, anticompacted from {} to {} sstable(s).";
View Full Code Here

        ICompactionScanner scanner = cleanupStrategy.getScanner(sstable, getRateLimiter());
        CleanupInfo ci = new CleanupInfo(sstable, scanner);

        metrics.beginCompaction(ci);
        Set<SSTableReader> oldSSTable = Sets.newHashSet(sstable);
        SSTableRewriter writer = new SSTableRewriter(cfs, oldSSTable, sstable.maxDataAge, false);
        List<SSTableReader> finished;
        try (CompactionController controller = new CompactionController(cfs, Collections.singleton(sstable), getDefaultGcBefore(cfs)))
        {
            writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, sstable.getSSTableMetadata().repairedAt, sstable));

            while (scanner.hasNext())
            {
                if (ci.isStopRequested())
                    throw new CompactionInterruptedException(ci.getCompactionInfo());

                SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
                row = cleanupStrategy.cleanup(row);
                if (row == null)
                    continue;
                AbstractCompactedRow compactedRow = new LazilyCompactedRow(controller, Collections.singletonList(row));
                if (writer.append(compactedRow) != null)
                    totalkeysWritten++;
            }

            // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
            cfs.indexManager.flushIndexesBlocking();

            finished = writer.finish();
            cfs.getDataTracker().markCompactedSSTablesReplaced(oldSSTable, finished, OperationType.CLEANUP);
        }
        catch (Throwable e)
        {
            writer.abort();
            throw Throwables.propagate(e);
        }
        finally
        {
            scanner.close();
View Full Code Here

TOP

Related Classes of org.apache.cassandra.io.sstable.SSTableRewriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.