Examples of BatchInserter


Examples of org.neo4j.kernel.impl.batchinsert.BatchInserter

    @Ignore
    @Test
    public void testInsertionSpeed()
    {
        BatchInserter inserter = new BatchInserterImpl( new File( PATH, "3" ).getAbsolutePath() );
        BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider( inserter );
        BatchInserterIndex index = provider.nodeIndex( "yeah", EXACT_CONFIG );
        index.setCacheCapacity( "key", 1000000 );
        long t = currentTimeMillis();
        for ( int i = 0; i < 1000000; i++ )
        {
            Map<String, Object> properties = map( "key", "value" + i );
            long id = inserter.createNode( properties );
            index.add( id, properties );
        }
        System.out.println( "insert:" + ( currentTimeMillis() - t ) );
        index.flush();
View Full Code Here

Examples of org.neo4j.kernel.impl.batchinsert.BatchInserter

    @Test
    public void testFindCreatedIndex()
    {
        String indexName = "persons";
        String path = new File( PATH, "4" ).getAbsolutePath();
        BatchInserter inserter = new BatchInserterImpl( path );
        LuceneBatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(
                inserter );
        BatchInserterIndex persons = indexProvider.nodeIndex( "persons",
                stringMap( "type", "exact" ) );
        Map<String, Object> properties = map( "name", "test" );
        long node = inserter.createNode( properties );
        persons.add( node, properties );
        indexProvider.shutdown();
        inserter.shutdown();
        GraphDatabaseService graphDb = new EmbeddedGraphDatabase( path );
        Transaction tx = graphDb.beginTx();
        try
        {
            IndexManager indexManager = graphDb.index();
View Full Code Here

Examples of org.neo4j.kernel.impl.batchinsert.BatchInserter

    }

    @Test
    public void testCanIndexRelationships()
    {
        BatchInserter inserter = new BatchInserterImpl( new File( PATH, "5" ).getAbsolutePath() );
        BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(
                inserter );
        BatchInserterIndex edgesIndex = indexProvider.relationshipIndex(
                "edgeIndex", stringMap( "provider", "lucene", "type", "exact" ) );

        long nodeId1 = inserter.createNode( map( "ID", "1" ) );
        long nodeId2 = inserter.createNode( map( "ID", "2" ) );
        long relationshipId = inserter.createRelationship( nodeId1, nodeId2,
                EdgeType.KNOWS, null );

        edgesIndex.add( relationshipId, map( "EDGE_TYPE", EdgeType.KNOWS.name() ) );
        edgesIndex.flush();

        assertEquals(
                String.format( "Should return relationship id" ),
                new Long( relationshipId ),
                edgesIndex.query( "EDGE_TYPE", EdgeType.KNOWS.name() ).getSingle() );

        indexProvider.shutdown();
        inserter.shutdown();
    }
View Full Code Here

Examples of org.neo4j.kernel.impl.batchinsert.BatchInserter

    }
   
    @Test
    public void triggerNPEAfterFlush()
    {
        BatchInserter inserter = new BatchInserterImpl( new File( PATH, "6" ).getAbsolutePath() );
        BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider( inserter );
        BatchInserterIndex index = provider.nodeIndex( "Node-exact", EXACT_CONFIG );
       
        Map<String, Object> map = map( "name", "Something" );
        long node = inserter.createNode( map );
        index.add( node, map );
        index.flush();
        assertContains( index.get( "name", "Something" ), node );
       
        provider.shutdown();
        inserter.shutdown();
    }
View Full Code Here

Examples of org.neo4j.kernel.impl.batchinsert.BatchInserter

   
    @Test
    public void testNumericValues()
    {
        String path = new File( PATH, "7" ).getAbsolutePath();
        BatchInserter inserter = new BatchInserterImpl( path );
        BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider(
                inserter );
        BatchInserterIndex index = provider.nodeIndex( "mine", EXACT_CONFIG );
       
        long node1 = inserter.createNode( null );
        index.add( node1, map( "number", numeric( 45 ) ) );
        long node2 = inserter.createNode( null );
        index.add( node2, map( "number", numeric( 21 ) ) );
        assertContains( index.query( "number",
                newIntRange( "number", 21, 50, true, true ) ), node1, node2 );
       
        provider.shutdown();
        inserter.shutdown();
       
        GraphDatabaseService db = new EmbeddedGraphDatabase( path );
        Node n1 = db.getNodeById( node1 );
        Node n2 = db.getNodeById( node2 );
        Index<Node> idx = db.index().forNodes( "mine" );
View Full Code Here

Examples of org.neo4j.kernel.impl.batchinsert.BatchInserter

    }
   
    @Test
    public void indexNumbers() throws Exception
    {
        BatchInserter inserter = new BatchInserterImpl( new File( PATH, "8" ).getAbsolutePath() );
        BatchInserterIndexProvider provider = new LuceneBatchInserterIndexProvider(
                inserter );
        BatchInserterIndex index = provider.nodeIndex( "mine", EXACT_CONFIG );
       
        long id = inserter.createNode( null );
        Map<String, Object> props = new HashMap<String, Object>();
        props.put( "key", 123L );
        index.add( id, props );
        index.flush();
       
        assertEquals( 1, index.get( "key", 123L ).size() );
        assertEquals( 1, index.get( "key", "123" ).size() );
       
        provider.shutdown();
        inserter.shutdown();
    }
View Full Code Here

Examples of org.neo4j.unsafe.batchinsert.BatchInserter

  private void importMapOSM() throws Exception {
    // START SNIPPET: importOsm
    OSMImporter importer = new OSMImporter("map.osm");
    importer.setCharset(Charset.forName("UTF-8"));
    BatchInserter batchInserter = BatchInserters.inserter(databasePath);
    importer.importFile(batchInserter, "map.osm", false);
    batchInserter.shutdown();

    GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabase(databasePath);
    importer.reIndex(db);
    db.shutdown();
    // END SNIPPET: importOsm
View Full Code Here

Examples of org.neo4j.unsafe.batchinsert.BatchInserter

  }
 
  private void importOSMDatabase(String osmPath, String dbPath, String layerName) throws ParseException, IOException, XMLStreamException, InterruptedException {
    OSMImporter importer = new OSMImporter(layerName, new ConsoleListener(), makeFilterEnvelope());
    importer.setCharset(Charset.forName("UTF-8"));
        BatchInserter batchInserter = BatchInserters.inserter(dbPath, Neo4jTestCase.LARGE_CONFIG);
        //GraphDatabaseService graphDb = batchInserter.getGraphDbService();
    //importer.importFile(graphDb, osmPath, false, 10000, true);
    importer.importFile(batchInserter, osmPath, false);
    batchInserter.shutdown();
    //graphDb.shutdown();
    // Weird hack to force GC on large loads
    long start = System.currentTimeMillis();
    if (System.currentTimeMillis() - start > 300000) {
      for (int i = 0; i < 3; i++) {
View Full Code Here

Examples of org.neo4j.unsafe.batchinsert.BatchInserter

        {
            boolean nodeAutoIndexingEnabled = indexer.isNodeAutoIndexingEnabled(factory.getConnection());
            boolean relationshipAutoIndexingEnabled = indexer
                    .isRelationshipAutoIndexingEnabled(factory.getConnection());

            BatchInserter inserter = getBatchInserter();
            BatchInserterIndexProvider indexProvider = new LuceneBatchInserterIndexProvider(inserter);

            if (inserter == null)
            {
                log.error("Unable to create instance of BatchInserter. Opertion will fail");
                throw new PersistenceException("Unable to create instance of BatchInserter. Opertion will fail");
            }

            if (resource != null && resource.isActive())
            {
                log.error("Batch Insertion MUST not be executed in a transaction");
                throw new PersistenceException("Batch Insertion MUST not be executed in a transaction");
            }

            Map<Object, Long> pkToNodeIdMap = new HashMap<Object, Long>();
            for (com.impetus.kundera.graph.Node graphNode : nodes)
            {
                if (graphNode.isDirty())
                {
                    graphNode.handlePreEvent();
                    // Delete can not be executed in batch, deleting normally
                    if (graphNode.isInState(RemovedState.class))
                    {
                        delete(graphNode.getData(), graphNode.getEntityId());
                    }
                    else if (graphNode.isUpdate())
                    {
                        // Neo4J allows only batch insertion, follow usual path
                        // for normal updates
                        persist(graphNode);
                    }
                    else
                    {
                        // Insert node
                        Object entity = graphNode.getData();
                        EntityMetadata m = KunderaMetadataManager.getEntityMetadata(kunderaMetadata, entity.getClass());
                        Object pk = PropertyAccessorHelper.getId(entity, m);
                        Map<String, Object> nodeProperties = mapper.createNodeProperties(entity, m);
                        long nodeId = inserter.createNode(nodeProperties);
                        pkToNodeIdMap.put(pk, nodeId);

                        // Index Node
                        indexer.indexNodeUsingBatchIndexer(indexProvider, m, nodeId, nodeProperties,
                                nodeAutoIndexingEnabled);

                        // Insert relationships for this particular node
                        if (!getRelationHolders(graphNode).isEmpty())
                        {
                            for (RelationHolder rh : getRelationHolders(graphNode))
                            {
                                // Search Node (to be connected to ) in Neo4J
                                // graph
                                EntityMetadata targetNodeMetadata = KunderaMetadataManager.getEntityMetadata(
                                        kunderaMetadata, rh.getRelationValue().getClass());
                                Object targetNodeKey = PropertyAccessorHelper.getId(rh.getRelationValue(),
                                        targetNodeMetadata);
                                Long targetNodeId = pkToNodeIdMap.get(targetNodeKey);

                                if (targetNodeId != null)
                                {
                                    /**
                                     * Join this node (source node) to target
                                     * node via relationship
                                     */
                                    // Relationship Type
                                    DynamicRelationshipType relType = DynamicRelationshipType.withName(rh
                                            .getRelationName());

                                    // Relationship Properties
                                    Map<String, Object> relationshipProperties = null;
                                    Object relationshipObj = rh.getRelationVia();
                                    if (relationshipObj != null)
                                    {
                                        EntityMetadata relationMetadata = KunderaMetadataManager.getEntityMetadata(
                                                kunderaMetadata, relationshipObj.getClass());

                                        relationshipProperties = mapper.createRelationshipProperties(m,
                                                targetNodeMetadata, relationshipObj);

                                        // Finally insert relationship
                                        long relationshipId = inserter.createRelationship(nodeId, targetNodeId,
                                                relType, relationshipProperties);

                                        // Index this relationship
                                        indexer.indexRelationshipUsingBatchIndexer(indexProvider, relationMetadata,
                                                relationshipId, relationshipProperties, relationshipAutoIndexingEnabled);
                                    }

                                }
                            }
                        }
                    }
                    graphNode.handlePostEvent();
                }
            }

            // Shutdown Batch inserter
            indexProvider.shutdown();
            inserter.shutdown();

            // Restore Graph Database service
            factory.setConnection((GraphDatabaseService) factory.createPoolOrConnection());

            return pkToNodeIdMap.size();
View Full Code Here

Examples of org.neo4j.unsafe.batchinsert.BatchInserter

        if (factory.getConnection() != null)
        {
            factory.getConnection().shutdown();
        }

        BatchInserter inserter = null;

        // Create Batch inserter with configuration if specified
        Neo4JSchemaMetadata nsmd = Neo4JPropertyReader.nsmd;
        ClientProperties cp = nsmd != null ? nsmd.getClientProperties() : null;
        if (cp != null)
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.