Examples of DNode


Examples of com.splout.db.dnode.DNode

   
    SploutConfiguration config1 = SploutConfiguration.getTestConfig();
    SploutConfiguration config2 = SploutConfiguration.getTestConfig();
    config2.setProperty(DNodeProperties.DATA_FOLDER, config1.getString(DNodeProperties.DATA_FOLDER) + "-" + 1);
   
    DNode dnode1 = TestUtils.getTestDNode(config1, failingDHandler, "dnode-" + this.getClass().getName() + "-1");
    DNode dnode2 = TestUtils.getTestDNode(config2, dHandler, "dnode-" + this.getClass().getName() + "-2");

    try {
      ReplicationEntry repEntry = new ReplicationEntry(0, dnode1.getAddress(), dnode2.getAddress());
      Tablespace tablespace1 = new Tablespace(PartitionMap.oneShardOpenedMap(), new ReplicationMap(Arrays.asList(repEntry)), 1l, 0l);
      handler.getContext().getTablespaceVersionsMap().put(new TablespaceVersion("tablespace1", 1l), tablespace1);
      handler.getContext().getCurrentVersionsMap().put("tablespace1", 1l);

      QueryStatus qStatus = handler.query("tablespace1", "2", "SELECT 1;", null);
      Assert.assertEquals(new Integer(0), qStatus.getShard());
      Assert.assertEquals("[1]", qStatus.getResult().toString());
    } finally {
      handler.close();
      dnode1.stop();
      dnode2.stop();
      Hazelcast.shutdownAll();
    }
  }
View Full Code Here

Examples of com.splout.db.dnode.DNode

          return true;
        }
      }
    }.waitAtMost(5000);

    final DNode dnode1 = getdNodes().get(1);

    final Set<Integer> partitionsByNode1 = new HashSet<Integer>();
    partitionsByNode1.add(0);
    partitionsByNode1.add(1);

    // shutdown DNode1 and see what happens with auto-rebalancing
    // the "partitionsByNode1" will become under-replicated and after a short period of time should be balanced
    dnode1.testCommand(TestCommands.SHUTDOWN.toString());

    // waiting until the system becomes under-replicated
    new TestUtils.NotWaitingForeverCondition() {

      @Override
      public boolean endCondition() {
        try {
          boolean dnode1NotPresent = true;
          for(int i = 0; i < N_QNODES; i++) {
            Map.Entry<String, Tablespace> tEntry = clients[i].overview().getTablespaceMap().entrySet()
                .iterator().next();
            ReplicationMap currentReplicationMap = tEntry.getValue().getReplicationMap();
            for(ReplicationEntry entry : currentReplicationMap.getReplicationEntries()) {
              if(entry.getNodes().contains(dnode1.getAddress())) {
                dnode1NotPresent = false;
              }
            }
          }
          return dnode1NotPresent;
        } catch(IOException e) {
          // test failed
          e.printStackTrace();
          return true;
        }
      }
    }.waitAtMost(5000);

    // waiting now until the system recovers itself without dnode1
    new TestUtils.NotWaitingForeverCondition() {

      @Override
      public boolean endCondition() {
        try {
          boolean balanced = true;
          for(int i = 0; i < N_QNODES; i++) {
            Map.Entry<String, Tablespace> tEntry = clients[i].overview().getTablespaceMap().entrySet()
                .iterator().next();
            ReplicationMap currentReplicationMap = tEntry.getValue().getReplicationMap();
            for(ReplicationEntry entry : currentReplicationMap.getReplicationEntries()) {
              if(entry.getNodes().size() < entry.getExpectedReplicationFactor()) {
                balanced = false;
              }
            }
          }
          return balanced;
        } catch(IOException e) {
          // test failed
          e.printStackTrace();
          return true;
        }
      }
    }.waitAtMost(5000);

    // now we bring back dnode1 to life
    // what will happen now is that the partitions it seves will be over-replicated
    dnode1.testCommand(TestCommands.RESTART.toString());

    // waiting now until the system is over-replicated
    new TestUtils.NotWaitingForeverCondition() {

      @Override
View Full Code Here

Examples of com.splout.db.dnode.DNode

    SploutConfiguration config2 = SploutConfiguration.getTestConfig();
    // since #12 and #7 we must disable replica balancing which is enabled for testing to fail the deploy
    config.setProperty(QNodeProperties.REPLICA_BALANCE_ENABLE, false);
    config.setProperty(QNodeProperties.DEPLOY_SECONDS_TO_CHECK_ERROR, 1); // this has to be quick for testing
    DNodeHandler dHandler = new DNodeHandler();
    DNode dnode1 = TestUtils
        .getTestDNode(config1, dHandler, "dnode-" + this.getClass().getName() + "-1");

    StuckInDeployHandler stuck = new StuckInDeployHandler();
    DNode dnode2 = TestUtils.getTestDNode(config2, stuck, "dnode-" + this.getClass().getName() + "-2");

    try {
      handler.init(config);

      ReplicationEntry repEntry1 = new ReplicationEntry(0, dnode1.getAddress(), dnode2.getAddress());

      DeployRequest deployRequest1 = new DeployRequest();
      deployRequest1.setTablespace("partition1");
      deployRequest1.setPartitionMap(PartitionMap.oneShardOpenedMap().getPartitionEntries());
      deployRequest1.setReplicationMap(Arrays.asList(repEntry1));

      File fakeDeployFolder = new File(DEPLOY_FOLDER_THAT_DOESNT_EXIST);
      // Remember we don't create the folder on purpose for making the DNode fail
      deployRequest1.setData_uri(fakeDeployFolder.toURI().toString());

      List<DeployRequest> l = new ArrayList<DeployRequest>();
      l.add(deployRequest1);

      handler.deploy(l);

      Thread.sleep(3000);

      assertEquals(0, handler.getContext().getTablespaceVersionsMap().size());

      // the other DNode should have been interrupted
      assertEquals(true, stuck.interrupted);

      // everything OK
    } finally {
      handler.close();
      dnode1.stop();
      dnode2.stop();
      Hazelcast.shutdownAll();
    }
  }
View Full Code Here

Examples of com.splout.db.dnode.DNode

  public void testDeployEnding() throws Throwable {
    // Test what happens when DNodes complete the deploy process
    final QNodeHandler handler = new QNodeHandler();
    SploutConfiguration config = SploutConfiguration.getTestConfig();
    DNodeHandler dHandler = new DNodeHandler();
    DNode dnode = TestUtils.getTestDNode(config, dHandler, "dnode-" + this.getClass().getName() + "-1");

    try {
      handler.init(config);

      DeployRequest deployRequest1 = new DeployRequest();
      deployRequest1.setTablespace("partition1");
      deployRequest1.setPartitionMap(PartitionMap.oneShardOpenedMap().getPartitionEntries());
      deployRequest1.setReplicationMap(ReplicationMap.oneToOneMap(dnode.getAddress())
          .getReplicationEntries());

      File fakeDeployFolder = new File(FAKE_DEPLOY_FOLDER);
      fakeDeployFolder.mkdir();
      File deployData = new File(fakeDeployFolder, "0.db");
      deployData.createNewFile();
      deployRequest1.setData_uri(fakeDeployFolder.toURI().toString());

      List<DeployRequest> l = new ArrayList<DeployRequest>();
      l.add(deployRequest1);

      handler.deploy(l);

      new TestUtils.NotWaitingForeverCondition() {
        @Override
        public boolean endCondition() {
          boolean cond1 = handler.getContext().getTablespaceVersionsMap().values().size() == 1;
          boolean cond2 = handler.getContext().getCurrentVersionsMap().get("partition1") != null;
          return cond1 && cond2;
        }
      }.waitAtMost(5000);

      assertEquals((long) handler.getContext().getTablespaceVersionsMap().keySet().iterator().next()
          .getVersion(), (long) handler.getContext().getCurrentVersionsMap().values().iterator().next());
      // everything OK
    } finally {
      handler.close();
      dnode.stop();
      Hazelcast.shutdownAll();
    }
  }
View Full Code Here

Examples of com.splout.db.dnode.DNode

  public void testQuery() throws Throwable {
    QNodeHandler handler = new QNodeHandler();
    handler.init(SploutConfiguration.getTestConfig());

    SploutConfiguration config = SploutConfiguration.getTestConfig();
    DNode dnode = TestUtils.getTestDNode(config, dHandler, "dnode-" + this.getClass().getName() + "-2");
    try {
      ReplicationEntry repEntry = new ReplicationEntry(0, dnode.getAddress());
      Tablespace tablespace1 = new Tablespace(PartitionMap.oneShardOpenedMap(), new ReplicationMap(
          Arrays.asList(repEntry)), 0l, 0l);
      handler.getContext().getTablespaceVersionsMap()
          .put(new TablespaceVersion("tablespace1", 0l), tablespace1);
      handler.getContext().getCurrentVersionsMap().put("tablespace1", 0l);

      // Query key 2 (> 1 < 10)
      QueryStatus qStatus = handler.query("tablespace1", "2", "SELECT 1;", null);
      Assert.assertEquals(new Integer(0), qStatus.getShard());
      Assert.assertEquals("[1]", qStatus.getResult().toString());
    } finally {
      handler.close();
      dnode.stop();
      Hazelcast.shutdownAll();
    }
  }
View Full Code Here

Examples of com.splout.db.dnode.DNode

    // same as testQuery but using manual partition instead of "key"
    QNodeHandler handler = new QNodeHandler();
    handler.init(SploutConfiguration.getTestConfig());

    SploutConfiguration config = SploutConfiguration.getTestConfig();
    DNode dnode = TestUtils.getTestDNode(config, dHandler, "dnode-" + this.getClass().getName() + "-6");
    try {
      ReplicationEntry repEntry = new ReplicationEntry(0, dnode.getAddress());
      Tablespace tablespace1 = new Tablespace(PartitionMap.oneShardOpenedMap(), new ReplicationMap(
          Arrays.asList(repEntry)), 0l, 0l);
      handler.getContext().getTablespaceVersionsMap()
          .put(new TablespaceVersion("tablespace1", 0l), tablespace1);
      handler.getContext().getCurrentVersionsMap().put("tablespace1", 0l);

      // Query shard 0
      QueryStatus qStatus = handler.query("tablespace1", null, "SELECT 1;", "0");
      Assert.assertEquals(new Integer(0), qStatus.getShard());
      Assert.assertEquals("[1]", qStatus.getResult().toString());

      // Query random partition
      qStatus = handler.query("tablespace1", null, "SELECT 1;", Querier.PARTITION_RANDOM);
      Assert.assertEquals(new Integer(0), qStatus.getShard());
      Assert.assertEquals("[1]", qStatus.getResult().toString());

    } finally {
      handler.close();
      dnode.stop();
      Hazelcast.shutdownAll();
    }
  }
View Full Code Here

Examples of com.splout.db.dnode.DNode

      config = SploutConfiguration.get();
      // we need to change some props for avoiding conflicts, ports, etc
      config.setProperty(DNodeProperties.PORT, config.getInt(DNodeProperties.PORT) + i);
      config.setProperty(DNodeProperties.DATA_FOLDER, config.getString(DNodeProperties.DATA_FOLDER) + "-" + i);
      config.setProperty(FetcherProperties.TEMP_DIR, config.getString(FetcherProperties.TEMP_DIR) + "-" + i);
      DNode dnode = new DNode(config, new DNodeHandler());
      dnode.init();
    }
    QNode qnode = new QNode();
    qnode.start(config, new QNodeHandler());
  }
View Full Code Here

Examples of com.splout.db.dnode.DNode

  @Test
  public void testMultiDeployFiring() throws Throwable {
    // Same as test deploy firing, but with more than one DNode and different deploy actions
    SploutConfiguration config1 = SploutConfiguration.getTestConfig();

    DNode dnode1 = TestUtils.getTestDNode(config1, new IDNodeHandler() {
      @Override
      public void init(SploutConfiguration config) throws Exception {
      }

      @Override
      public String sqlQuery(String tablespace, long version, int partition, String query)
          throws DNodeException {
        return null;
      }

      @Override
      public String deploy(List<DeployAction> deployActions, long distributedBarrier)
          throws DNodeException {
        /*
         * DNode1 asserts
         */
        Assert.assertEquals(2, deployActions.size());
        Assert.assertEquals("hdfs://foo/bar1/0.db", deployActions.get(0).getDataURI());
        Assert.assertEquals("hdfs://foo/bar2/0.db", deployActions.get(1).getDataURI());
        Assert.assertEquals("partition1", deployActions.get(0).getTablespace());
        Assert.assertEquals("partition2", deployActions.get(1).getTablespace());
        return "FOO";
      }

      @Override
      public String rollback(List<RollbackAction> rollbackActions, String ignoreMe)
          throws DNodeException {
        return null;
      }

      @Override
      public String status() throws DNodeException {
        return null;
      }

      @Override
      public void stop() throws Exception {
      }

      @Override
      public void giveGreenLigth() {
      }

      @Override
      public String abortDeploy(long version) throws DNodeException {
        return null;
      }

      @Override
      public String deleteOldVersions(List<com.splout.db.thrift.TablespaceVersion> versions)
          throws DNodeException {
        return null;
      }

      @Override
      public String testCommand(String command) throws DNodeException {
        // TODO Auto-generated method stub
        return null;
      }
    }, "dnode-" + this.getClass().getName() + "-3");

    SploutConfiguration config2 = SploutConfiguration.getTestConfig();
    DNode dnode2 = TestUtils.getTestDNode(config2, new IDNodeHandler() {
      @Override
      public void init(SploutConfiguration config) throws Exception {
      }

      @Override
      public String sqlQuery(String tablespace, long version, int partition, String query)
          throws DNodeException {
        return null;
      }

      @Override
      public String deploy(List<DeployAction> deployActions, long distributedBarrier)
          throws DNodeException {
        /*
         * DNode2 asserts
         */
        Assert.assertEquals(1, deployActions.size());
        Assert.assertEquals("hdfs://foo/bar1/0.db", deployActions.get(0).getDataURI());
        Assert.assertEquals("partition1", deployActions.get(0).getTablespace());
        return "FOO";
      }

      @Override
      public String rollback(List<RollbackAction> rollbackActions, String ignoreMe)
          throws DNodeException {
        return null;
      }

      @Override
      public String status() throws DNodeException {
        return null;
      }

      @Override
      public void stop() throws Exception {
      }

      @Override
      public void giveGreenLigth() {
      }

      @Override
      public String abortDeploy(long version) throws DNodeException {
        return null;
      }

      @Override
      public String deleteOldVersions(List<com.splout.db.thrift.TablespaceVersion> versions)
          throws DNodeException {
        return null;
      }

      @Override
      public String testCommand(String command) throws DNodeException {
        // TODO Auto-generated method stub
        return null;
      }
    }, "dnode-" + this.getClass().getName() + "-4");

    QNodeHandler handler = new QNodeHandler();
    try {
      handler.init(config1);
      ReplicationEntry repEntry1 = new ReplicationEntry(0, dnode1.getAddress(), dnode2.getAddress());
      ReplicationEntry repEntry2 = new ReplicationEntry(0, dnode1.getAddress());

      DeployRequest deployRequest1 = new DeployRequest();
      deployRequest1.setTablespace("partition1");
      deployRequest1.setPartitionMap(PartitionMap.oneShardOpenedMap().getPartitionEntries());
      deployRequest1.setReplicationMap(Arrays.asList(repEntry1));
      deployRequest1.setData_uri("hdfs://foo/bar1");

      DeployRequest deployRequest2 = new DeployRequest();
      deployRequest2.setTablespace("partition2");
      deployRequest2.setPartitionMap(PartitionMap.oneShardOpenedMap().getPartitionEntries());
      deployRequest2.setReplicationMap(Arrays.asList(repEntry2));
      deployRequest2.setData_uri("hdfs://foo/bar2");

      List<DeployRequest> l = new ArrayList<DeployRequest>();
      l.add(deployRequest1);
      l.add(deployRequest2);

      handler.deploy(l);
    } finally {
      handler.close();
      dnode1.stop();
      dnode2.stop();
      Hazelcast.shutdownAll();
    }
  }
View Full Code Here

Examples of com.splout.db.dnode.DNode

    // Test the business logic that produces the firing of the deployment (not the continuation of it) For that, we will
    // use dummy DNodeHandlers
    QNodeHandler handler = new QNodeHandler();
    SploutConfiguration config = SploutConfiguration.getTestConfig();

    DNode dnode = TestUtils.getTestDNode(config, new IDNodeHandler() {
      @Override
      public void init(SploutConfiguration config) throws Exception {
      }

      @Override
      public String sqlQuery(String tablespace, long version, int partition, String query)
          throws DNodeException {
        return null;
      }

      @Override
      public String deploy(List<DeployAction> deployActions, long version) throws DNodeException {
        Assert.assertEquals(1, deployActions.size());
        Assert.assertEquals("hdfs://foo/bar/0.db", deployActions.get(0).getDataURI());
        Assert.assertEquals("partition1", deployActions.get(0).getTablespace());
        Assert.assertTrue(version >= 0); // TODO Is this the right checking here?
        return "FOO";
      }

      @Override
      public String rollback(List<RollbackAction> rollbackActions, String ignoreMe)
          throws DNodeException {
        return null;
      }

      @Override
      public String status() throws DNodeException {
        return null;
      }

      @Override
      public void stop() throws Exception {
      }

      @Override
      public void giveGreenLigth() {
      }

      @Override
      public String abortDeploy(long version) throws DNodeException {
        return null;
      }

      @Override
      public String deleteOldVersions(List<com.splout.db.thrift.TablespaceVersion> versions)
          throws DNodeException {
        return null;
      }

      @Override
      public String testCommand(String command) throws DNodeException {
        // TODO Auto-generated method stub
        return null;
      }
    }, "dnode-" + this.getClass().getName() + "-5");

    try {
      handler.init(config);

      ReplicationEntry repEntry = new ReplicationEntry(0, dnode.getAddress());

      DeployRequest deployRequest = new DeployRequest();
      deployRequest.setTablespace("partition1");
      deployRequest.setPartitionMap(PartitionMap.oneShardOpenedMap().getPartitionEntries());
      deployRequest.setReplicationMap(Arrays.asList(repEntry));
      deployRequest.setData_uri("hdfs://foo/bar");

      List<DeployRequest> l = new ArrayList<DeployRequest>();
      l.add(deployRequest);
      handler.deploy(l);
    } finally {
      handler.close();
      dnode.stop();
      Hazelcast.shutdownAll();
    }
  }
View Full Code Here

Examples of com.splout.db.dnode.DNode

    DNodeHandler dNodeHandler1 = new DNodeHandler();

    final SploutConfiguration config1 = SploutConfiguration.getTestConfig();

    DNode dnode1 = TestUtils.getTestDNode(config1, dNodeHandler1, "dnode-" + this.getClass().getName()
        + "-1");
    final String dnode1Address = dnode1.getAddress();

    try {

      assertEquals(handler.getDNodeList().size(), 1);
      // wait until connection pool has been generated
      new TestUtils.NotWaitingForeverCondition() {

        @Override
        public boolean endCondition() {
          return handler.getContext().getThriftClientCache().get(dnode1Address) != null
              && handler.getContext().getThriftClientCache().get(dnode1Address).size() == 40;
        }
      }.waitAtMost(5000);

      dnode1.stop();

      assertEquals(handler.getDNodeList().size(), 0);
      assertNull(handler.getContext().getThriftClientCache().get(dnode1.getAddress()));

      dnode1 = TestUtils.getTestDNode(config, dNodeHandler1, "dnode-" + this.getClass().getName() + "-2");

      assertEquals(handler.getDNodeList().size(), 1);

      // wait until connection pool has been regenerated
      new TestUtils.NotWaitingForeverCondition() {

        @Override
        public boolean endCondition() {
          return handler.getContext().getThriftClientCache().get(dnode1Address) != null
              && handler.getContext().getThriftClientCache().get(dnode1Address).size() == 40;
        }
      }.waitAtMost(5000);

    } finally {
      dnode1.stop();
      qnode.close();
    }
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.