Package net.yacy.kelondro.index

Examples of net.yacy.kelondro.index.Index


        return null;
    }

    public void addUnique(final Row.Entry row) throws IOException, RowSpaceExceededException {
        assert row.objectsize() <= this.rowdef.objectsize;
        Index table = (this.current == null) ? null : this.tables.get(this.current);
        synchronized (this.tables) {
            assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
            if (table == null) table = newTable(); else table = checkTable(table);
        }
        table.addUnique(row);
    }
View Full Code Here


        }
        return report;
    }

    public boolean delete(final byte[] key) throws IOException {
        final Index table = keeperOf(key);
        if (table == null) return false;
        return table.delete(key);
    }
View Full Code Here

        if (table == null) return false;
        return table.delete(key);
    }

    public Row.Entry remove(final byte[] key) throws IOException {
        final Index table = keeperOf(key);
        if (table == null) return null;
        return table.remove(key);
    }
View Full Code Here

        return table.remove(key);
    }

    public Row.Entry removeOne() throws IOException {
        final Iterator<Index> i = this.tables.values().iterator();
        Index table, maxtable = null;
        int maxcount = -1;
        while (i.hasNext()) {
            table = i.next();
            if (table.size() > maxcount) {
                maxtable = table;
                maxcount = table.size();
            }
        }
        if (maxtable == null) {
            return null;
        }
View Full Code Here

        return maxtable.removeOne();
    }

    public List<Row.Entry> top(final int count) throws IOException {
        final Iterator<Index> i = this.tables.values().iterator();
        Index table, maxtable = null;
        int maxcount = -1;
        while (i.hasNext()) {
            table = i.next();
            if (table.size() > maxcount) {
                maxtable = table;
                maxcount = table.size();
            }
        }
        if (maxtable == null) {
            return null;
        }
View Full Code Here

            final memprofiler profiler = new memprofiler(1024, 320, 120, new File(tablename_test + ".profile.png"));
            profiler.start();

            // create the database access
            final Row testRow = new Row("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, Base64Order.enhancedCoder);
            final Index table_test = selectTableType(dbe_test, tablename_test, testRow);
            final Index table_reference = (dbe_reference == null) ? null : selectTableType(dbe_reference, tablename_reference, testRow);

            final long afterinit = System.currentTimeMillis();
            System.out.println("Test for db-engine " + dbe_test +  " started to create file " + tablename_test + " with test " + command);

            // execute command
            if (command.equals("create")) {
                // do nothing, since opening of the database access must handle this
                System.out.println("Database created");
            }

            if (command.equals("fill")) {
                // fill database with random entries;
                // args: <number-of-entries> <random-startpoint>
                // example: java -ea -Xmx200m fill kelondroEcoTable kelondroFlexTable filldbtest 50000 0
                final long count = Long.parseLong(args[4]);
                final long randomstart = Long.parseLong(args[5]);
                final Random random = new Random(randomstart);
                byte[] key;
                for (int i = 0; i < count; i++) {
                    key = randomHash(random);
                    table_test.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
                    if (table_reference != null) table_reference.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
                    if (i % 1000 == 0) {
                        System.out.println(i + " entries. ");
                    }
                }
            }

            if (command.equals("read")) {
                // read the database and compare with random entries;
                // args: <number-of-entries> <random-startpoint>
                final long count = Long.parseLong(args[4]);
                final long randomstart = Long.parseLong(args[5]);
                final Random random = new Random(randomstart);
                Row.Entry entry;
                byte[] key;
                for (int i = 0; i < count; i++) {
                    key = randomHash(random);
                    entry = table_test.get(key, false);
                    if (entry == null)
                        System.out.println("missing value for entry " + UTF8.String(key) + " in test table");
                    else
                        if (!ByteBuffer.equals(entry.getColBytes(1, false), key)) System.out.println("wrong value for entry " + UTF8.String(key) + ": " + UTF8.String(entry.getColBytes(1, false)) + " in test table");
                    if (table_reference != null) {
                        entry = table_reference.get(key, false);
                        if (entry == null)
                            System.out.println("missing value for entry " + UTF8.String(key) + " in reference table");
                        else
                            if (!ByteBuffer.equals(entry.getColBytes(1, false), key)) System.out.println("wrong value for entry " + UTF8.String(key) + ": " + UTF8.String(entry.getColBytes(1, false)) + " in reference table");
                    }

                    if (i % 1000 == 0) {
                        System.out.println(i + " entries processed so far.");
                    }
                }
            }

            if (command.equals("ramtest")) {
                // fill database with random entries and delete them again;
                // this is repeated without termination; after each loop
                // the current ram is printed out
                // args: <number-of-entries> <random-startpoint>
                final long count = Long.parseLong(args[4]);
                final long randomstart = Long.parseLong(args[5]);
                byte[] key;
                Random random;
                long start, write, remove;
                int loop = 0;
                while (true) {
                    // write
                    random = new Random(randomstart);
                    start = System.currentTimeMillis();
                    for (int i = 0; i < count; i++) {
                        key = randomHash(random);
                        table_test.put(table_test.row().newEntry(new byte[][]{key, key, dummyvalue2}));
                    }
                    write = System.currentTimeMillis() - start;

                    // delete
                    random = new Random(randomstart);
                    start = System.currentTimeMillis();
                    for (int i = 0; i < count; i++) {
                        key = randomHash(random);
                        table_test.delete(key);
                    }
                    remove = System.currentTimeMillis() - start;

                    System.out.println("Loop " + loop + ": Write = " + write + ", Remove = " + remove);
                    System.out.println(" bevore GC: " +
                              "free = " + MemoryControl.free() +
                            ", max = " + MemoryControl.maxMemory() +
                            ", total = " + MemoryControl.total());
                    System.gc();
                    System.out.println(" after  GC: " +
                            "free = " + MemoryControl.free() +
                          ", max = " + MemoryControl.maxMemory() +
                          ", total = " + MemoryControl.total());
                  loop++;
                }
            }

            if (command.equals("list")) {
                CloneableIterator<Row.Entry> i = null;
                if (table_test instanceof SQLTable) i = ((SQLTable) table_test).rows();
                if(i != null) {
                    Row.Entry row;
                    while (i.hasNext()) {
                        row = i.next();
                        for (int j = 0; j < row.columns(); j++) System.out.print(row.getColUTF8(j) + ",");
                        System.out.println();
                    }
                }
            }

            final long aftercommand = System.currentTimeMillis();
            checkEquivalence(table_test, table_reference);
            final long afterequiv = System.currentTimeMillis();
            // final report
            System.out.println("Database size = " + table_test.size() + " unique entries.");

            // finally close the database/table
            table_test.close();
            if (table_reference != null) table_reference.close();

            final long afterclose = System.currentTimeMillis();

            System.out.println("Execution time: open=" + (afterinit - startup) +
                    ", command=" + (aftercommand - afterinit) +
View Full Code Here

            final String tablename,
            final boolean useTailCache,
            final boolean exceed134217727) {
        this.location = path;
        this.tablename = tablename;
        Index backupIndex = null;
        backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727);
        this.urlIndexFile = backupIndex; //new Cache(backupIndex, 20000000, 20000000);
        this.exportthread = null; // will have a export thread assigned if exporter is running
        this.statsDump = null;
        this.solr = null;
View Full Code Here

        return tablename + "-" + keysize + "-" + payloadsize + ".eco";
    }

    public void declareRelation(final String name, final int keysize, final int payloadsize) throws RowSpaceExceededException {
        // try to get the relation from the relation-cache
        final Index relation = this.relations.get(name);
        if (relation != null) return;
        // try to find the relation as stored on file
        final String[] list = this.baseDir.list();
        final String targetfilename = filename(name, keysize, payloadsize);
        for (int i = 0; i < list.length; i++) {
            if (list[i].startsWith(name)) {
                if (!list[i].equals(targetfilename)) continue;
                final Row row = rowdef(list[i]);
                if (row.primaryKeyLength != keysize || row.column(1).cellwidth != payloadsize) continue; // a wrong table
                Index table;
                try {
                    table = new Table(new File(this.baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727, true);
                } catch (final RowSpaceExceededException e) {
                    table = new Table(new File(this.baseDir, list[i]), row, 0, 0, false, this.exceed134217727, true);
                }
                this.relations.put(name, table);
                return;
            }
        }
        // the relation does not exist, create it
        final Row row = rowdef(keysize, payloadsize);
        Index table;
        try {
            table = new Table(new File(this.baseDir, targetfilename), row, 1024*1024, 0, this.useTailCache, this.exceed134217727, true);
        } catch (final RowSpaceExceededException e) {
            table = new Table(new File(this.baseDir, targetfilename), row, 0, 0, false, this.exceed134217727, true);
        }
View Full Code Here

        this.relations.put(name, table);
    }

    public Index getRelation(final String name) throws RowSpaceExceededException {
        // try to get the relation from the relation-cache
        final Index relation = this.relations.get(name);
        if (relation != null) return relation;
        // try to find the relation as stored on file
        final String[] list = this.baseDir.list();
        for (final String element : list) {
            if (element.startsWith(name)) {
                final Row row = rowdef(element);
                Index table;
                try {
                    table = new Table(new File(this.baseDir, element), row, 1024*1024, 0, this.useTailCache, this.exceed134217727, true);
                } catch (final RowSpaceExceededException e) {
                    table = new Table(new File(this.baseDir, element), row, 0, 0, false, this.exceed134217727, true);
                }
View Full Code Here

        if (r == null) return null;
        return UTF8.String(r);
    }

    public byte[] putRelation(final String name, final byte[] key, final byte[] value) throws IOException, RowSpaceExceededException {
        final Index table = getRelation(name);
        if (table == null) return null;
        final Row.Entry entry = table.row().newEntry();
        entry.setCol(0, key);
        entry.setCol(1, System.currentTimeMillis());
        entry.setCol(2, 1000000);
        entry.setCol(3, value);
        final Row.Entry oldentry = table.replace(entry);
        if (oldentry == null) return null;
        return oldentry.getColBytes(3, true);
    }
View Full Code Here

TOP

Related Classes of net.yacy.kelondro.index.Index

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.