Examples of org.apache.helix.ZNRecord

org.apache.helix.ZNRecord
Generic Record Format to store data at a Node This can be used to store simpleFields mapFields listFields

      LOG.error("Input object must be of type ZNRecord but it is " + data + ". Will not write to zk");
      throw new HelixException("Input object is not of type ZNRecord (was " + data + ")");
    }


    // apply retention policy on list field
    ZNRecord record = (ZNRecord) data;
    int max = getListFieldBound(record);
    if (max < Integer.MAX_VALUE)
    {
      Map<String, List<String>> listMap = record.getListFields();
      for (String key : listMap.keySet())
      {
        List<String> list = listMap.get(key);
        if (list.size() > max)
        {
          listMap.put(key, list.subList(0, max));
        }
      }
    }
    
    StringWriter sw = new StringWriter();
    try
    {
      JsonFactory f = new JsonFactory();
      JsonGenerator g = f.createJsonGenerator(sw);


      g.writeStartObject();


      // write id field
      g.writeRaw("\n  ");
      g.writeStringField("id", record.getId());


      // write simepleFields
      g.writeRaw("\n  ");
      g.writeObjectFieldStart("simpleFields");
      for (String key : record.getSimpleFields().keySet())
      {
        g.writeRaw("\n    ");
        g.writeStringField(key, record.getSimpleField(key));
      }
      g.writeRaw("\n  ");
      g.writeEndObject(); // for simpleFields


      // write listFields
      g.writeRaw("\n  ");
      g.writeObjectFieldStart("listFields");
      for (String key : record.getListFields().keySet())
      {
        // g.writeStringField(key, record.getListField(key).toString());


        // g.writeObjectFieldStart(key);
        g.writeRaw("\n    ");
        g.writeArrayFieldStart(key);
        List<String> list = record.getListField(key);
        for (String listValue : list)
        {
          g.writeString(listValue);
        }
        // g.writeEndObject();
        g.writeEndArray();


      }
      g.writeRaw("\n  ");
      g.writeEndObject(); // for listFields


      // write mapFields
      g.writeRaw("\n  ");
      g.writeObjectFieldStart("mapFields");
      for (String key : record.getMapFields().keySet())
      {
        // g.writeStringField(key, record.getMapField(key).toString());
        g.writeRaw("\n    ");
        g.writeObjectFieldStart(key);
        Map<String, String> map = record.getMapField(key);
        for (String mapKey : map.keySet())
        {
          g.writeRaw("\n      ");
          g.writeStringField(mapKey, map.get(mapKey));
        }
        g.writeRaw("\n    ");
        g.writeEndObject();


      }
      g.writeRaw("\n  ");
      g.writeEndObject(); // for mapFields


      g.writeRaw("\n");
      g.writeEndObject(); // for whole znrecord


      // important: will force flushing of output, close underlying output
      // stream
      g.close();
    }
    catch (Exception e)
    {
      LOG.error("Exception during data serialization. Will not write to zk. Data (first 1k): "
          + sw.toString().substring(0, 1024), e);
      throw new HelixException(e);
    }
    
    // check size
    if (sw.toString().getBytes().length > ZNRecord.SIZE_LIMIT)
    {
      LOG.error("Data size larger than 1M, ZNRecord.id: " + record.getId() 
          + ". Will not write to zk. Data (first 1k): " + sw.toString().substring(0, 1024));
      throw new HelixException("Data size larger than 1M, ZNRecord.id: " + record.getId());
    }
    
    return sw.toString().getBytes();
  }

View Full Code Here

      LOG.error("ZNode is empty.");
      return null;
    }
    
    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
    ZNRecord record = null;


    try
    {
      JsonFactory f = new JsonFactory();
      JsonParser jp = f.createJsonParser(bais);


      jp.nextToken(); // will return JsonToken.START_OBJECT (verify?)
      while (jp.nextToken() != JsonToken.END_OBJECT)
      {
        String fieldname = jp.getCurrentName();
        jp.nextToken(); // move to value, or START_OBJECT/START_ARRAY
        if ("id".equals(fieldname))
        { 
          // contains an object
          record = new ZNRecord(jp.getText());
        }
        else if ("simpleFields".equals(fieldname))
        {
          while (jp.nextToken() != JsonToken.END_OBJECT)
          {
            String key = jp.getCurrentName();
            jp.nextToken(); // move to value
            record.setSimpleField(key, jp.getText());
          }
        }
        else if ("mapFields".equals(fieldname))
        {
          // user.setVerified(jp.getCurrentToken() == JsonToken.VALUE_TRUE);
          while (jp.nextToken() != JsonToken.END_OBJECT)
          {
            String key = jp.getCurrentName();
            record.setMapField(key, new TreeMap<String, String>());
            jp.nextToken(); // move to value


            while (jp.nextToken() != JsonToken.END_OBJECT)
            {
              String mapKey = jp.getCurrentName();
              jp.nextToken(); // move to value
              record.getMapField(key).put(mapKey, jp.getText());
            }
          }


        }
        else if ("listFields".equals(fieldname))
        {
          // user.setUserImage(jp.getBinaryValue());
          while (jp.nextToken() != JsonToken.END_OBJECT)
          {
            String key = jp.getCurrentName();
            record.setListField(key, new ArrayList<String>());
            jp.nextToken(); // move to value
            while (jp.nextToken() != JsonToken.END_ARRAY)
            {
              record.getListField(key).add(jp.getText());
            }


          }


        }

View Full Code Here

    return record;
  }


  public static void main(String[] args)
  {
    ZNRecord record = new ZNRecord("record");
    final int recordSize = 10;
    for (int i = 0; i < recordSize; i++)
    {
      record.setSimpleField("" + i, "" + i);
      record.setListField("" + i, new ArrayList<String>());
      for (int j = 0; j < recordSize; j++)
      {
        record.getListField("" + i).add("" + j);
      }


      record.setMapField("" + i, new TreeMap<String, String>());
      for (int j = 0; j < recordSize; j++)
      {
        record.getMapField("" + i).put("" + j, "" + j);
      }
    }


    ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer();
    byte[] bytes = serializer.serialize(record);
    System.out.println(new String(bytes));
    ZNRecord record2 = (ZNRecord) serializer.deserialize(bytes);
    System.out.println(record2);


    long start = System.currentTimeMillis();
    for (int i = 0; i < 100; i++)
    {

View Full Code Here

  @Test()
  public void testSimple()
  {
    List<InstanceConfig> instances;
    RoutingTableProvider routingTable = new RoutingTableProvider();
    ZNRecord record = new ZNRecord("TESTDB");


    // one master
    add(record, "TESTDB_0", "localhost_8900", "MASTER");
    List<ExternalView> externalViewList = new ArrayList<ExternalView>();
    externalViewList.add(new ExternalView(record));

View Full Code Here

  {
    List<InstanceConfig> instances;
    RoutingTableProvider routingTable = new RoutingTableProvider();


    List<ExternalView> externalViewList = new ArrayList<ExternalView>();
    ZNRecord record = new ZNRecord("TESTDB");


    // one master
    add(record, "TESTDB_0", "localhost_8900", "MASTER");
    externalViewList.add(new ExternalView(record));
    routingTable.onExternalViewChange(externalViewList, changeContext);

View Full Code Here

    List<InstanceConfig> instancesList;
    Set<InstanceConfig> instancesSet;
    InstanceConfig instancesArray[];
    RoutingTableProvider routingTable = new RoutingTableProvider();
    List<ExternalView> externalViewList = new ArrayList<ExternalView>();
    ZNRecord record = new ZNRecord("TESTDB");


    // one master
    add(record, "TESTDB_0", "localhost_8900", "MASTER");
    add(record, "TESTDB_1", "localhost_8900", "MASTER");
    add(record, "TESTDB_2", "localhost_8900", "MASTER");

View Full Code Here

  @Test()
  public void testMultiThread() throws Exception
  {
    final RoutingTableProvider routingTable = new RoutingTableProvider();
    List<ExternalView> externalViewList = new ArrayList<ExternalView>();
    ZNRecord record = new ZNRecord("TESTDB");
    for (int i = 0; i < 1000; i++)
    {
      add(record, "TESTDB_" + i, "localhost_8900", "MASTER");
    }
    externalViewList.add(new ExternalView(record));

View Full Code Here

          setupTool.getClusterManagementTool().getResourceExternalView(clusterName,
                                                                       resourceName);


      if (idealState != null)
      {
        ZNRecord partInfo = new ZNRecord(resourceName + "/" + partitionName);
        ZNRecord idealStateRec = idealState.getRecord();
        partInfo.setSimpleFields(idealStateRec.getSimpleFields());
        if (idealStateRec.getMapField(partitionName) != null)
        {
          partInfo.setMapField(partitionName, idealStateRec.getMapField(partitionName));
        }
        if (idealStateRec.getListField(partitionName) != null)
        {
          partInfo.setListField(partitionName, idealStateRec.getListField(partitionName));
        }
        System.out.println("IdealState for " + resourceName + "/" + partitionName + ":");
        System.out.println(new String(new ZNRecordSerializer().serialize(partInfo)));
      }
      else
      {
        System.out.println("No idealState for " + resourceName + "/" + partitionName);
      }


      System.out.println();


      if (externalView != null)
      {
        ZNRecord partInfo = new ZNRecord(resourceName + "/" + partitionName);
        ZNRecord extViewRec = externalView.getRecord();
        partInfo.setSimpleFields(extViewRec.getSimpleFields());
        if (extViewRec.getMapField(partitionName) != null)
        {
          partInfo.setMapField(partitionName, extViewRec.getMapField(partitionName));
        }
        if (extViewRec.getListField(partitionName) != null)
        {
          partInfo.setListField(partitionName, extViewRec.getListField(partitionName));
        }


        System.out.println("ExternalView for " + resourceName + "/" + partitionName + ":");
        System.out.println(new String(new ZNRecordSerializer().serialize(partInfo)));
      }
      else
      {
        System.out.println("No externalView for " + resourceName + "/" + partitionName);
      }
      return 0;


    }
    else if (cmd.hasOption(enableInstance))
    {
      String clusterName = cmd.getOptionValues(enableInstance)[0];
      String instanceName = cmd.getOptionValues(enableInstance)[1];
      if (instanceName.contains(":"))
      {
        instanceName = instanceName.replaceAll(":", "_");
      }
      boolean enabled =
          Boolean.parseBoolean(cmd.getOptionValues(enableInstance)[2].toLowerCase());


      setupTool.getClusterManagementTool().enableInstance(clusterName,
                                                          instanceName,
                                                          enabled);
      return 0;
    }
    else if (cmd.hasOption(enablePartition))
    {
      String[] args = cmd.getOptionValues(enablePartition);


      boolean enabled = Boolean.parseBoolean(args[0].toLowerCase());
      String clusterName = args[1];
      String instanceName = args[2];
      String resourceName = args[3];


      List<String> partitionNames =
          Arrays.asList(Arrays.copyOfRange(args, 4, args.length));
      setupTool.getClusterManagementTool().enablePartition(enabled,
                                                           clusterName,
                                                           instanceName,
                                                           resourceName,
                                                           partitionNames);
      return 0;
    }
    else if (cmd.hasOption(resetPartition))
    {
      String[] args = cmd.getOptionValues(resetPartition);


      String clusterName = args[0];
      String instanceName = args[1];
      String resourceName = args[2];
      List<String> partitionNames =
          Arrays.asList(Arrays.copyOfRange(args, 3, args.length));


      setupTool.getClusterManagementTool().resetPartition(clusterName,
                                                          instanceName,
                                                          resourceName,
                                                          partitionNames);
      return 0;
    }
    else if (cmd.hasOption(resetInstance))
    {
      String[] args = cmd.getOptionValues(resetInstance);


      String clusterName = args[0];
      List<String> instanceNames =
          Arrays.asList(Arrays.copyOfRange(args, 1, args.length));


      setupTool.getClusterManagementTool().resetInstance(clusterName, instanceNames);
      return 0;
    }
    else if (cmd.hasOption(resetResource))
    {
      String[] args = cmd.getOptionValues(resetResource);


      String clusterName = args[0];
      List<String> resourceNames =
          Arrays.asList(Arrays.copyOfRange(args, 1, args.length));


      setupTool.getClusterManagementTool().resetResource(clusterName, resourceNames);
      return 0;
    }
    else if (cmd.hasOption(enableCluster))
    {
      String[] params = cmd.getOptionValues(enableCluster);
      String clusterName = params[0];
      boolean enabled = Boolean.parseBoolean(params[1].toLowerCase());
      setupTool.getClusterManagementTool().enableCluster(clusterName, enabled);


      return 0;
    }
    else if (cmd.hasOption(listStateModels))
    {
      String clusterName = cmd.getOptionValues(listStateModels)[0];


      List<String> stateModels =
          setupTool.getClusterManagementTool().getStateModelDefs(clusterName);


      System.out.println("Existing state models:");
      for (String stateModel : stateModels)
      {
        System.out.println(stateModel);
      }
      return 0;
    }
    else if (cmd.hasOption(listStateModel))
    {
      String clusterName = cmd.getOptionValues(listStateModel)[0];
      String stateModel = cmd.getOptionValues(listStateModel)[1];
      StateModelDefinition stateModelDef =
          setupTool.getClusterManagementTool().getStateModelDef(clusterName, stateModel);
      String result =
          new String(new ZNRecordSerializer().serialize(stateModelDef.getRecord()));
      System.out.println("StateModelDefinition: " + result);
      return 0;
    }
    else if (cmd.hasOption(addStateModelDef))
    {
      String clusterName = cmd.getOptionValues(addStateModelDef)[0];
      String stateModelFile = cmd.getOptionValues(addStateModelDef)[1];


      ZNRecord stateModelRecord =
          (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(stateModelFile)));
      if (stateModelRecord.getId() == null || stateModelRecord.getId().length() == 0)
      {
        throw new IllegalArgumentException("ZNRecord for state model definition must have an id");
      }
      setupTool.getClusterManagementTool()
               .addStateModelDef(clusterName,
                                 stateModelRecord.getId(),
                                 new StateModelDefinition(stateModelRecord));
      return 0;
    }
    else if (cmd.hasOption(addIdealState))
    {

View Full Code Here

  }


  // TODO: remove this. has moved to ZkHelixAdmin
  public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException
  {
    ZNRecord idealStateRecord =
        (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile)));
    if (idealStateRecord.getId() == null
        || !idealStateRecord.getId().equals(resourceName))
    {
      throw new IllegalArgumentException("ideal state must have same id as resource name");
    }
    _admin.setResourceIdealState(clusterName,
                                 resourceName,

View Full Code Here

    // create 10 current states
    for (int i = 0; i < 10; i++)
    {
      String path = curStatePath + "/session_0/TestDB" + i;
      boolean success =
          accessor.create(path, new ZNRecord("TestDB" + i), AccessOption.PERSISTENT);
      Assert.assertTrue(success, "Should succeed in create: " + path);
    }


    // verify wtCache
    // TestHelper.printCache(accessor._wtCache);
    ret = TestHelper.verifyZkCache(cachePaths, accessor._wtCache._cache, _gZkClient, false);
    Assert.assertTrue(ret, "wtCache doesn't match data on Zk");


    // update each current state 10 times, single thread
    for (int i = 0; i < 10; i++)
    {
      String path = curStatePath + "/session_0/TestDB" + i;
      for (int j = 0; j < 10; j++)
      {
        ZNRecord newRecord = new ZNRecord("TestDB" + i);
        newRecord.setSimpleField("" + j, "" + j);
        boolean success =
            accessor.update(path, new ZNRecordUpdater(newRecord), AccessOption.PERSISTENT);
        Assert.assertTrue(success, "Should succeed in update: " + path);


      }
    }


    // verify cache
//    TestHelper.printCache(accessor._wtCache._cache);
    ret = TestHelper.verifyZkCache(cachePaths, accessor._wtCache._cache, _gZkClient, false);
    Assert.assertTrue(ret, "wtCache doesn't match data on Zk");


    // set 10 external views
    for (int i = 0; i < 10; i++)
    {
      String path =
          PropertyPathConfig.getPath(PropertyType.EXTERNALVIEW, clusterName, "TestDB" + i);
      boolean success = accessor.set(path, new ZNRecord("TestDB" + i), AccessOption.PERSISTENT);
      Assert.assertTrue(success, "Should succeed in set: " + path);
    }


    // verify wtCache
    // accessor.printWtCache();
    ret = TestHelper.verifyZkCache(cachePaths, accessor._wtCache._cache, _gZkClient, false);
    Assert.assertTrue(ret, "wtCache doesn't match data on Zk");




    // get 10 external views
    for (int i = 0; i < 10; i++)
    {
      String path =
          PropertyPathConfig.getPath(PropertyType.EXTERNALVIEW, clusterName, "TestDB" + i);
      ZNRecord record = accessor.get(path, null, 0);
      Assert.assertEquals(record.getId(), "TestDB" + i);
    }


    // getChildNames
    List<String> childNames = accessor.getChildNames(extViewPath, 0);
    // System.out.println(childNames);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.helix.ZNRecord

com.linkedin.databus.client.pub.ClusterCheckpointPersistenceProvider

org.apache.helix.alerts.TestAddAlerts

org.apache.helix.alerts.TestAddPersistentStats

org.apache.helix.alerts.TestArrivingParticipantStats

org.apache.helix.api.accessor.ClusterAccessor

org.apache.helix.controller.ExternalViewGenerator

org.apache.helix.controller.rebalancer.AutoRebalancer

org.apache.helix.controller.stages.BaseStageTest

org.apache.helix.controller.stages.ExternalViewComputeStage

org.apache.helix.controller.stages.StatsAggregationStage

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.