Package net.yacy.kelondro.index

Examples of net.yacy.kelondro.index.Index


        return tablename + "-" + keysize + "-" + payloadsize + ".eco";
    }
   
    public void declareRelation(final String name, final int keysize, final int payloadsize) throws RowSpaceExceededException {
        // try to get the relation from the relation-cache
        final Index relation = relations.get(name);
        if (relation != null) return;
        // try to find the relation as stored on file
        final String[] list = baseDir.list();
        final String targetfilename = filename(name, keysize, payloadsize);
        for (int i = 0; i < list.length; i++) {
            if (list[i].startsWith(name)) {
                if (!list[i].equals(targetfilename)) continue;
                final Row row = rowdef(list[i]);
                if (row.primaryKeyLength != keysize || row.column(1).cellwidth != payloadsize) continue; // a wrong table
                Index table;
                try {
                    table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727);
                } catch (RowSpaceExceededException e) {
                    table = new Table(new File(baseDir, list[i]), row, 0, 0, false, this.exceed134217727);
                }
                relations.put(name, table);
                return;
            }
        }
        // the relation does not exist, create it
        final Row row = rowdef(keysize, payloadsize);
        Index table;
        try {
            table = new Table(new File(baseDir, targetfilename), row, 1024*1024, 0, this.useTailCache, this.exceed134217727);
        } catch (RowSpaceExceededException e) {
            table = new Table(new File(baseDir, targetfilename), row, 0, 0, false, this.exceed134217727);
        }
View Full Code Here


        relations.put(name, table);
    }
   
    public Index getRelation(final String name) throws RowSpaceExceededException {
        // try to get the relation from the relation-cache
        final Index relation = relations.get(name);
        if (relation != null) return relation;
        // try to find the relation as stored on file
        final String[] list = baseDir.list();
        for (int i = 0; i < list.length; i++) {
            if (list[i].startsWith(name)) {
                final Row row = rowdef(list[i]);
                Index table;
                try {
                    table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727);
                } catch (RowSpaceExceededException e) {
                    table = new Table(new File(baseDir, list[i]), row, 0, 0, false, this.exceed134217727);
                }
View Full Code Here

        if (r == null) return null;
        return UTF8.String(r);
    }
   
    public byte[] putRelation(final String name, final byte[] key, final byte[] value) throws IOException, RowSpaceExceededException {
        final Index table = getRelation(name);
        if (table == null) return null;
        final Row.Entry entry = table.row().newEntry();
        entry.setCol(0, key);
        entry.setCol(1, System.currentTimeMillis());
        entry.setCol(2, 1000000);
        entry.setCol(3, value);
        final Row.Entry oldentry = table.replace(entry);
        if (oldentry == null) return null;
        return oldentry.getColBytes(3, true);
    }
View Full Code Here

        if (r == null) return null;
        return UTF8.String(r);
    }
   
    public byte[] getRelation(final String name, final byte[] key) throws IOException, RowSpaceExceededException {
        final Index table = getRelation(name);
        if (table == null) return null;
        final Row.Entry entry = table.get(key);
        if (entry == null) return null;
        return entry.getColBytes(3, true);
    }
View Full Code Here

        if (entry == null) return null;
        return entry.getColBytes(3, true);
    }
   
    public boolean hasRelation(final String name, final byte[] key) throws RowSpaceExceededException {
        final Index table = getRelation(name);
        if (table == null) return false;
        return table.has(key);
    }
View Full Code Here

        if (table == null) return false;
        return table.has(key);
    }
   
    public byte[] removeRelation(final String name, final byte[] key) throws IOException, RowSpaceExceededException {
        final Index table = getRelation(name);
        if (table == null) return null;
        final Row.Entry entry = table.remove(key);
        if (entry == null) return null;
        return entry.getColBytes(3, true);
    }
View Full Code Here

            final memprofiler profiler = new memprofiler(1024, 320, 120, new File(tablename_test + ".profile.png"));
            profiler.start();
           
            // create the database access
            final Row testRow = new Row("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, Base64Order.enhancedCoder);
            final Index table_test = selectTableType(dbe_test, tablename_test, testRow);
            final Index table_reference = (dbe_reference == null) ? null : selectTableType(dbe_reference, tablename_reference, testRow);
           
            final long afterinit = System.currentTimeMillis();
            System.out.println("Test for db-engine " + dbe_test +  " started to create file " + tablename_test + " with test " + command);
           
            // execute command
            if (command.equals("create")) {
                // do nothing, since opening of the database access must handle this
                System.out.println("Database created");
            }
           
            if (command.equals("fill")) {
                // fill database with random entries;
                // args: <number-of-entries> <random-startpoint>
                // example: java -ea -Xmx200m fill kelondroEcoTable kelondroFlexTable filldbtest 50000 0
                final long count = Long.parseLong(args[4]);
                final long randomstart = Long.parseLong(args[5]);
                final Random random = new Random(randomstart);
                byte[] key;
                for (int i = 0; i < count; i++) {
                    key = randomHash(random);
                    table_test.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
                    if (table_reference != null) table_reference.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
                    if (i % 1000 == 0) {
                        System.out.println(i + " entries. ");
                    }
                }
            }
           
            if (command.equals("read")) {
                // read the database and compare with random entries;
                // args: <number-of-entries> <random-startpoint>
                final long count = Long.parseLong(args[4]);
                final long randomstart = Long.parseLong(args[5]);
                final Random random = new Random(randomstart);
                Row.Entry entry;
                byte[] key;
                for (int i = 0; i < count; i++) {
                    key = randomHash(random);
                    entry = table_test.get(key);
                    if (entry == null)
                        System.out.println("missing value for entry " + UTF8.String(key) + " in test table");
                    else
                        if (!ByteBuffer.equals(entry.getColBytes(1, false), key)) System.out.println("wrong value for entry " + UTF8.String(key) + ": " + UTF8.String(entry.getColBytes(1, false)) + " in test table");
                    if (table_reference != null) {
                        entry = table_reference.get(key);
                        if (entry == null)
                            System.out.println("missing value for entry " + UTF8.String(key) + " in reference table");
                        else
                            if (!ByteBuffer.equals(entry.getColBytes(1, false), key)) System.out.println("wrong value for entry " + UTF8.String(key) + ": " + UTF8.String(entry.getColBytes(1, false)) + " in reference table");
                    }
                   
                    if (i % 1000 == 0) {
                        System.out.println(i + " entries processed so far.");
                    }
                }
            }
           
            if (command.equals("ramtest")) {
                // fill database with random entries and delete them again;
                // this is repeated without termination; after each loop
                // the current ram is printed out
                // args: <number-of-entries> <random-startpoint>
                final long count = Long.parseLong(args[4]);
                final long randomstart = Long.parseLong(args[5]);
                byte[] key;
                Random random;
                long start, write, remove;
                int loop = 0;
                while (true) {
                    // write
                    random = new Random(randomstart);
                    start = System.currentTimeMillis();
                    for (int i = 0; i < count; i++) {
                        key = randomHash(random);
                        table_test.put(table_test.row().newEntry(new byte[][]{key, key, dummyvalue2}));
                    }
                    write = System.currentTimeMillis() - start;

                    // delete
                    random = new Random(randomstart);
                    start = System.currentTimeMillis();
                    for (int i = 0; i < count; i++) {
                        key = randomHash(random);
                        table_test.delete(key);
                    }
                    remove = System.currentTimeMillis() - start;
                   
                    System.out.println("Loop " + loop + ": Write = " + write + ", Remove = " + remove);
                    System.out.println(" bevore GC: " +
                              "free = " + MemoryControl.free() +
                            ", max = " + MemoryControl.maxMemory +
                            ", total = " + MemoryControl.total());
                    System.gc();
                    System.out.println(" after  GC: " +
                            "free = " + MemoryControl.free() +
                          ", max = " + MemoryControl.maxMemory +
                          ", total = " + MemoryControl.total());
                  loop++;
                }
            }
           
            if (command.equals("list")) {
                CloneableIterator<Row.Entry> i = null;
                if (table_test instanceof SQLTable) i = ((SQLTable) table_test).rows();
                if(i != null) {
                    Row.Entry row;
                    while (i.hasNext()) {
                        row = i.next();
                        for (int j = 0; j < row.columns(); j++) System.out.print(row.getColString(j) + ",");
                        System.out.println();
                    }
                }
            }
           
            final long aftercommand = System.currentTimeMillis();
            checkEquivalence(table_test, table_reference);
            final long afterequiv = System.currentTimeMillis();
            // final report
            System.out.println("Database size = " + table_test.size() + " unique entries.");
           
            // finally close the database/table
            table_test.close();
            if (table_reference != null) table_reference.close();
           
            final long afterclose = System.currentTimeMillis();
           
            System.out.println("Execution time: open=" + (afterinit - startup) +
                    ", command=" + (aftercommand - afterinit) +
View Full Code Here

    public boolean has(final byte[] key) {
        return keeperOf(key) != null;
    }

    public Row.Entry get(final byte[] key, final boolean forcecopy) throws IOException {
        final Index keeper = keeperOf(key);
        if (keeper == null) return null;
        return keeper.get(key, forcecopy);
    }
View Full Code Here

        return table;
    }

    public Row.Entry replace(final Row.Entry row) throws IOException, RowSpaceExceededException {
        assert row.objectsize() <= this.rowdef.objectsize;
        Index keeper = keeperOf(row.getPrimaryKeyBytes());
        if (keeper != null) return keeper.replace(row);
        synchronized (this.tables) {
            assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
            keeper = (this.current == null) ? newTable() : checkTable(this.tables.get(this.current));
        }
        keeper.put(row);
        return null;
    }
View Full Code Here

     */
    public boolean put(final Row.Entry row) throws IOException, RowSpaceExceededException {
        assert row.objectsize() <= this.rowdef.objectsize;
        final byte[] key = row.getPrimaryKeyBytes();
        if (this.tables == null) return true;
        Index keeper = null;
        synchronized (this.tables) {
            keeper = keeperOf(key);
        }
        if (keeper != null) return keeper.put(row);
        synchronized (this.tables) {
            keeper = keeperOf(key); // we must check that again because it could have changed in between
            if (keeper != null) return keeper.put(row);
            assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
            keeper = (this.current == null) ? newTable() : checkTable(this.tables.get(this.current));
            final boolean b = keeper.put(row);
            assert b;
            return b;
        }
    }
View Full Code Here

TOP

Related Classes of net.yacy.kelondro.index.Index

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.