Examples of DISCARD

cascading.pipe.assembly.Discard
Class Discard is a {@link cascading.pipe.SubAssembly} that will retain all incoming fields except those given on the constructor.
Use this pipe to narrow a stream of tuples, removing unused data. @see Retain
org.jgroups.protocols.DISCARD
Discards up or down messages based on a percentage; e.g., setting property 'up' to 0.1 causes 10% of all up messages to be discarded. Setting 'down' or 'up' to 0 causes no loss, whereas 1 discards all messages (not very useful).

Examples of cascading.pipe.assembly.Discard


    Fields filterArguments = new Fields( "uid1", "uid2" );
    String uidFilter = "uid1.compareToIgnoreCase( uid2 ) >= 0";
    invertPipe = new Each( invertPipe, filterArguments, new ExpressionFilter( uidFilter, String.class ) );
    Fields ignore = new Fields( "ignore" );
    invertPipe = new Discard( invertPipe, ignore );


    /*
    flow part #4
    count the number of tokens in common for each uid pair and apply a threshold
    */


    Pipe commonPipe = new GroupBy( new Pipe( "uid common", invertPipe ), new Fields( "uid1", "uid2" ) );
    commonPipe = new Every( commonPipe, Fields.ALL, new Count( new Fields( "common" ) ), Fields.ALL );


    String commonFilter = String.format( "common < %d", MIN_COMMON_TOKENS );
    commonPipe = new Each( commonPipe, new Fields( "common" ), new ExpressionFilter( commonFilter, Integer.TYPE ) );


    /*
    flow part #5
    count the number of tokens overall for each uid, then join to calculate
    the vector length for uid1
    */


    Fields tokenCount = new Fields( "token_count" );
    Pipe countPipe = new GroupBy( "count", joinPipe, new Fields( "uid" ) );
    countPipe = new Every( countPipe, Fields.ALL, new Count( tokenCount ), Fields.ALL );


    joinPipe = new CoGroup( countPipe, new Fields( "uid" ), commonPipe, new Fields( "uid1" ) );
    joinPipe = new Pipe( "common", joinPipe );
    joinPipe = new Discard( joinPipe, new Fields( "uid" ) );


    joinPipe = new Rename( joinPipe, tokenCount, new Fields( "token_count1" ) );


    /*
    flow part #6 join to be able to calculate the vector length for

View Full Code Here

Examples of cascading.pipe.assembly.Discard

    Pipe pipe = new Pipe( "head" );


    Fields discardFields = trainingFields.appendSelector( predictorFields );


    if( !discardFields.isNone() )
      pipe = new Discard( pipe, discardFields );


    Tap source = getPlatform().getDelimitedFile( "\t", "\"", planner.getFieldTypeResolver(), DATA_PATH + testModel + ".tsv", SinkMode.KEEP );
    Tap sink = getPlatform().getDelimitedFile( "\t", "\"", null, getResultPath(), SinkMode.REPLACE );


    FlowDef flowDef = FlowDef.flowDef()

View Full Code Here

Examples of cascading.pipe.assembly.Discard

      buffer = new PredictionSelectionBuffer( ensembleSpec );


    pipe = new Every( pipe, predictedFields, buffer, Fields.SWAP );


    if( modelSchema.getKeyFields().isNone() )
      pipe = new Discard( pipe, keyFields );


    setTails( pipe );
    }

View Full Code Here

Examples of cascading.pipe.assembly.Discard

    }


  protected void performTest( String inputData, Fields predictedFields, Fields expectedFields, EnsembleSpec<TreeSpec> ensembleSpec ) throws IOException
    {
    Pipe pipe = new Pipe( "head" );
    pipe = new Discard( pipe, predictedFields );
    pipe = new ParallelEnsembleAssembly( pipe, ensembleSpec );
    pipe = new Pipe( "tail", pipe );


    Tap source = getPlatform().getDelimitedFile( expectedFields.append( predictedFields ), true, ",", "\"", DATA_PATH + inputData, SinkMode.KEEP );
    Tap sink = getPlatform().getDelimitedFile( Fields.ALL, true, ",", "\"", getResultPath(), SinkMode.REPLACE );

View Full Code Here

Examples of cascading.pipe.assembly.Discard


    Function splitterLower = new RegexSplitter( new Fields( "numA", "left" ), " " );
    Function splitterUpper = new RegexSplitter( new Fields( "numB", "right" ), " " );


    Pipe offsetLower = new Pipe( "offsetLower" );
    offsetLower = new Discard( offsetLower, new Fields( "offset" ) );
    offsetLower = new Each( offsetLower, new Fields( "line" ), splitterLower );


    Pipe pipeLower = new Each( new Pipe( "lower" ), new Fields( "line" ), splitterUpper );


    Pipe cogroup = new CoGroup( offsetLower, new Fields( "numA" ), pipeLower, new Fields( "numB" ) );

View Full Code Here

Examples of org.jgroups.protocols.DISCARD




    @Test(dataProvider="configProvider")
    public void testReceptionOfAllMessagesWithDISCARD(Protocol prot) throws Throwable {
        System.out.println("prot=" + prot.getClass().getSimpleName());
        DISCARD discard=new DISCARD();
        discard.setDownDiscardRate(0.1); // discard all down message with 10% probability


        Protocol[] stack=new Protocol[]{prot, discard};
        createStack(stack);
        _testReceptionOfAllMessages();
    }

View Full Code Here

Examples of org.jgroups.protocols.DISCARD

            System.out.println("[" + channel.getAddress() + "] " + channel.downcall(Event.GET_DIGEST_EVT).toString());
        }
    }


    private static void insertDISCARD(JChannel ch, Address exclude) throws Exception {
        DISCARD discard=new DISCARD();
        discard.setExcludeItself(true);
        discard.addIgnoreMember(exclude); // ignore messages from this member
        ch.getProtocolStack().insertProtocol(discard, ProtocolStack.BELOW, "NAKACK");
    }

View Full Code Here

Examples of org.jgroups.protocols.DISCARD

    void cleanup() {
        Util.close(c2, c1);
    }


    public void testLastMessageDropped() throws Exception {
        DISCARD discard=new DISCARD();
        ProtocolStack stack=c1.getProtocolStack();
        stack.insertProtocol(discard, ProtocolStack.BELOW, NAKACK.class);
        c1.setOpt(Channel.LOCAL, false);


        Message m1=new Message(null, null, 1);
        Message m2=new Message(null, null, 2);
        Message m3=new Message(null, null, 3);


        MyReceiver receiver=new MyReceiver();
        c2.setReceiver(receiver);
        c1.send(m1);
        c1.send(m2);
        discard.setDropDownMulticasts(1); // drop the next multicast
        c1.send(m3);
        Util.sleep(100);


        List<Integer> list=receiver.getMsgs();
        for(int i=0; i < 10; i++)  {

View Full Code Here

Examples of org.jgroups.protocols.DISCARD

    /**
     * Tests sending 1, 2 (OOB) and 3, where they are received in the order 1, 3, 2. Message 3 should not get delivered
     * until message 4 is received (http://jira.jboss.com/jira/browse/JGRP-780)
     */
    public void testRegularAndOOBUnicasts() throws Exception {
        DISCARD discard=new DISCARD();
        ProtocolStack stack=c1.getProtocolStack();
        stack.insertProtocol(discard, ProtocolStack.BELOW, UNICAST.class, UNICAST2.class);


        Address dest=c2.getAddress();
        Message m1=new Message(dest, null, 1);
        Message m2=new Message(dest, null, 2);
        m2.setFlag(Message.OOB);
        Message m3=new Message(dest, null, 3);


        MyReceiver receiver=new MyReceiver("C2");
        c2.setReceiver(receiver);
        c1.send(m1);
        discard.setDropDownUnicasts(1);
        c1.send(m2);
        c1.send(m3);


        sendStableMessages(c1,c2);
        Util.sleep(1000); // time for potential retransmission

View Full Code Here

Examples of org.jgroups.protocols.DISCARD

        assert list.size() == 3 : "list is " + list;
        assert list.contains(1) && list.contains(2) && list.contains(3);
    }


    public void testRegularAndOOBUnicasts2() throws Exception {
        DISCARD discard=new DISCARD();
        ProtocolStack stack=c1.getProtocolStack();
        stack.insertProtocol(discard, ProtocolStack.BELOW, UNICAST.class, UNICAST2.class);


        Address dest=c2.getAddress();
        Message m1=new Message(dest, null, 1);
        Message m2=new Message(dest, null, 2);
        m2.setFlag(Message.OOB);
        Message m3=new Message(dest, null, 3);
        m3.setFlag(Message.OOB);
        Message m4=new Message(dest, null, 4);


        MyReceiver receiver=new MyReceiver("C2");
        c2.setReceiver(receiver);
        c1.send(m1);


        discard.setDropDownUnicasts(1);
        c1.send(m3);


        discard.setDropDownUnicasts(1);
        c1.send(m2);
        
        c1.send(m4);
        Util.sleep(1000); // sleep some time to receive all messages

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.