Examples of TextArrayWritable


Examples of com.manning.hip.ch3.TextArrayWritable

    }

    private void loadCSV() throws IOException {            //<co id="ch02_comment_csv_inputformat9"/>
      String line = reader.getCurrentValue().toString();
      String[] tokens = parser.parseLine(line);            //<co id="ch02_comment_csv_inputformat10"/>
      value = new TextArrayWritable(convert(tokens));
    }
View Full Code Here

Examples of hitune.analysis.mapreduce.TextArrayWritable

            newRecord.put("phase_start", ""+start);
            newRecord.put("phase_end", ""+end);
            newRecord.put("phase_count", ""+phaseCount);

            if(!initialized){
                TextArrayWritable newValue = new TextArrayWritable(newRecord.keySet().toArray(new String[0]));
                output.collect(null, newValue);
                initialized = true;
            }

            if(!funcCount.equals("")){
                newRecord.put("breakdown_type", "function");
                log.debug("funcList: " + funcList);
                log.debug("funcCount: " + funcCount);
                List <String> tmp = String2List(funcList,SEPERATOR_COMMA );
                List <String> counts = String2List(funcCount,SEPERATOR_COMMA );
                for(int i = 0 ; i < tmp.size(); i++ ){
                    log.debug("function:" +tmp.get(i) + " count:" + counts.get(i));
                    newRecord.put("breakdown_name", tmp.get(i));
                    newRecord.put("breakdown_count", ""+counts.get(i));
                    String [] contents = new String[newRecord.keySet().size()];
                    int j = 0;
                    for (String index:  newRecord.keySet() ){
                        contents[j] = newRecord.get(index);
                        log.debug("content: " + index + "," +contents[j] );
                        j++;

                    }
                    TextArrayWritable newValue = new TextArrayWritable(contents);
                    output.collect(null, newValue);
                    contents=null;
                }
            }
            if(!statusCount.equals("")){
                newRecord.put("breakdown_type", "state");
                log.debug("statusList: " + statusList);
                log.debug("statusCount: " + statusCount);
                List <String> tmp = String2List(statusList,SEPERATOR_COMMA );
                List <String> counts = String2List(statusCount,SEPERATOR_COMMA );
                for(int i = 0 ; i < tmp.size(); i++ ){
                    log.debug("function:" +tmp.get(i) + " count:" + counts.get(i));
                    newRecord.put("breakdown_name", tmp.get(i));
                    newRecord.put("breakdown_count", ""+counts.get(i));
                    String [] contents = new String[newRecord.size()];
                    int j = 0;
                    for (String index:  newRecord.keySet() ){
                        contents[j] = newRecord.get(index);
                        j++;
                        log.debug("content: " + index + "," +contents[i] );
                    }
                    TextArrayWritable newValue = new TextArrayWritable(contents);
                    output.collect(null, newValue);
                    contents=null;
                }
            }
        }
View Full Code Here

Examples of hitune.analysis.mapreduce.TextArrayWritable

            while(values.hasNext()){
                V value = (V) values.next();
                HiTuneRecord valproxy = new HiTuneRecord(value);
               
                Text newKey = new Text();
                TextArrayWritable newValue = null;
                String timestamp = "" + valproxy.getTime()/1000;

                String [] fields = valproxy.getFields();
                if(!initialized){
                    newKey.set("Timestamp");
                    newValue = new TextArrayWritable(fields);
                    output.collect(newKey, newValue);
                    initialized = true;
                }
                newKey.set(timestamp);
                String [] contents = new String[fields.length];
                for (int i=0; i<fields.length; i++  ){
                    contents[i] = valproxy.getValue(fields[i]);
                }
                newValue = new TextArrayWritable(contents);
                output.collect(newKey, newValue);
            }
        }
View Full Code Here

Examples of hitune.analysis.mapreduce.TextArrayWritable

                    "last_level_callee_num", "host", "phase_stack", "func", "phase_count"};
            for(String head:headers){
                newRecord.put(head, "");
            }
            if(!initialized){
                TextArrayWritable newValue = new TextArrayWritable(newRecord.keySet().toArray(new String[0]));
                output.collect(null, newValue);
                initialized = true;
            }


            TreeSet <HiTuneRecord> arrays = new TreeSet<HiTuneRecord>(new RecordComparator());

            TreeSet <HiTuneRecord> stackarrays = new TreeSet<HiTuneRecord>(new RecordComparator());
            HiTuneRecord phase = null;
            //log.debug("key: " + key.toString());
            while(values.hasNext()){
                try{
                    HiTuneRecord temp_proxyval = new HiTuneRecord(values.next());

                    V newvalue = (V) temp_proxyval.getObject().getClass().getConstructor().newInstance();
                    HiTuneRecord proxyval = new HiTuneRecord(newvalue);

                    for (String field: temp_proxyval.getFields()){
                        proxyval.add(field, temp_proxyval.getValue(field));
                    }

                    String function = proxyval.getValue("func");
                    log.debug(" val: " + proxyval.toString());

                    if (function.equals("_PHASE_")){
                        phase = proxyval;
                        continue;
                    }else{
                        if(function.indexOf("#")!=-1 ){
                            if(funcInStackFormat){
                                stackarrays.add(proxyval);
                                if(stackarrays.size()> limitNum){
                                    stackarrays.remove(stackarrays.last());
                                }
                            }
                        }else{
                            //log.debug("add new val: " + val);
                            arrays.add(proxyval);
                            if(arrays.size()> limitNum){
                                arrays.remove(arrays.last());
                            }
                        }

                    }

                } catch (IllegalArgumentException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (SecurityException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (InstantiationException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (IllegalAccessException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (InvocationTargetException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (NoSuchMethodException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }

            }

            if(funcInStackFormat){
                int len = limitNum<stackarrays.size()?limitNum:stackarrays.size();
                HiTuneRecord[] candidates = stackarrays.toArray(new HiTuneRecord[0]);
                for (int i = 0; i <stackarrays.size(); i++ ){
                    HiTuneRecord val = candidates[i];
                    if(val!=null){
                        newRecord.clear();
                        for(String head : headers){
                            if(head.equals("phase_count")){
                                newRecord.put(head,  phase.getValue("callee_num"));
                            }else{
                                newRecord.put(head, val.getValue(head));
                            }

                        }
                        String [] contents = new String[newRecord.keySet().size()];
                        int j = 0;
                        for (String index:  newRecord.keySet() ){
                            contents[j] = newRecord.get(index);
                            log.debug("content: " + index + "," +contents[j] );
                            j++;

                        }
                        TextArrayWritable newValue = new TextArrayWritable(contents);
                        output.collect(null, newValue);
                    }


                }
            }else {
                int len = limitNum<arrays.size()?limitNum:arrays.size();
                HiTuneRecord[] candidates = arrays.toArray(new HiTuneRecord[0]);

                for (int i = 0; i <len; i++ ){

                    HiTuneRecord val = candidates[i];
                    log.debug("dump val: " + val);
                    if(val!=null){
                        newRecord.clear();
                        for(String head : headers){
                            if(head.equals("phase_count")){
                                newRecord.put(head,  phase.getValue("callee_num"));
                            }else{
                                newRecord.put(head, val.getValue(head));
                            }
                        }
                        String [] contents = new String[newRecord.keySet().size()];
                        int j = 0;
                        for (String index:  newRecord.keySet() ){
                            contents[j] = newRecord.get(index);
                            log.debug("content: " + index + "," +contents[j] );
                            j++;

                        }
                        TextArrayWritable newValue = new TextArrayWritable(contents);
                        output.collect(null, newValue);
                    }


                }
View Full Code Here

Examples of hitune.analysis.mapreduce.TextArrayWritable

                OutputCollector<Text, TextArrayWritable> output, Reporter reporter)
        throws IOException {
            // TODO Auto-generated method stub
            //organizing into csv format
            Text newKey = null;
            TextArrayWritable newValue = null;

            Map<String, String> map = new HashMap<String, String>();
            Map<String, String> filter = new HashMap<String, String>();
            HiTuneRecord valproxy = null;
            while(values.hasNext()){
                valproxy= new HiTuneRecord(values.next());
                //copyGernetic(record, newrecord);
                for(String metric : valproxy.getFields()){
                    log.debug("copy metric:" + metric);
                    String val = valproxy.getValue(metric);
                    if(val == null){
                        val = "";
                    }
                    map.put(metric,val);
                }
            }
            if(valproxy!=null){
                //do filtering
                filter.putAll(valproxy.getCommonFields());

                for(String metric : this.metrics){
                    log.debug("collect metric:" + metric);
                    String val = map.get(metric);
                    if(val == null){
                        val = "";
                    }
                    filter.put(metric,val);
                }
                filter.put("id", new HiTuneKey(key).getKey());


                String [] fields = filter.keySet().toArray(new String[0]);
                if(!initialized){
                    newValue = new TextArrayWritable(fields);
                    output.collect(newKey, newValue);
                    initialized = true;
                    newValue = null;
                }
                String [] contents = new String[fields.length];
                for (int i=0; i<fields.length; i++  ){
                    contents[i] = filter.get(fields[i]);
                }
                newValue = new TextArrayWritable(contents);
                output.collect(newKey, newValue);
            }
        }
View Full Code Here

Examples of org.apache.hama.bsp.TextArrayWritable

    FileStatus[] globStatus = fs.globStatus(new Path(TEST_OUTPUT + "/part-*"));
    for (FileStatus fts : globStatus) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, fts.getPath(),
          conf);
      Text key = new Text();
      TextArrayWritable value = new TextArrayWritable();

      while (reader.next(key, value)) {
        Writable[] writables = value.get();
        assertTrue(writables.length <= 10);
        for (Writable t : writables) {
          int outlinkId = Integer.parseInt(t.toString());
          assertTrue(outlinkId <= 20);
          assertTrue(outlinkId >= 0);
View Full Code Here

Examples of org.apache.hama.bsp.TextArrayWritable

      for (int i = 0; i < input.length; i++) {
        String[] x = input[i].split("\t");

        Text vertex = new Text(x[0]);
        TextArrayWritable arr = new TextArrayWritable();
        Writable[] values = new Writable[x.length - 1];
        for (int j = 1; j < x.length; j++) {
          values[j - 1] = new Text(x[j]);
        }
        arr.set(values);
        writer1.append(vertex, arr);
      }

      writer1.close();
View Full Code Here

Examples of org.apache.hama.bsp.TextArrayWritable

    FileStatus[] globStatus = fs.globStatus(new Path(TEST_OUTPUT + "/part-*"));
    for (FileStatus fts : globStatus) {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, fts.getPath(),
          conf);
      Text key = new Text();
      TextArrayWritable value = new TextArrayWritable();

      while (reader.next(key, value)) {
        String values = "";
        for (Writable v : value.get()) {
          values += v.toString() + " ";
        }
        LOG.info(fts.getPath() + ": " + key.toString() + " | " + values);
      }
      reader.close();
View Full Code Here

Examples of org.apache.hama.bsp.TextArrayWritable

          for (Integer v : e.getValue()) {
            values[i] = new Text(String.valueOf(v));
            i++;
          }

          TextArrayWritable value = new TextArrayWritable();
          value.set(values);
          peer.write(new Text(String.valueOf(e.getKey())), value);
        }
      }
    }
View Full Code Here

Examples of org.apache.hama.bsp.TextArrayWritable

      for (int i = startID; i < endID; i++) {
        HashSet<Integer> set = Sets.newHashSet();
        for (int j = 0; j < maxOutEdges; j++) {
          set.add(r.nextInt(sizeN));
        }
        TextArrayWritable textArrayWritable = new TextArrayWritable();
        Text[] arr = new Text[set.size()];
        int index = 0;
        for (int x : set) {
          arr[index++] = new Text(x + "");
        }
        textArrayWritable.set(arr);
        peer.write(new Text(i + ""), textArrayWritable);
      }

    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.