Examples of Tokenize

net.sf.saxon.functions.Tokenize
This class implements the tokenize() function for regular expression matching. This returns a sequence of strings representing the unmatched substrings: the separators which match the regular expression are not returned.
org.apache.pig.builtin.TOKENIZE
Given a chararray as an argument, this method will split the chararray and return a bag with a tuple for each chararray that results from the split. The string is split on space, double quote, comma, open parend, close parend, and asterisk (star).
org.pdf4j.saxon.functions.Tokenize
This class implements the tokenize() function for regular expression matching. This returns a sequence of strings representing the unmatched substrings: the separators which match the regular expression are not returned.

Examples of net.sf.saxon.functions.Tokenize

    private void registerIdrefKey(Configuration config) {
        PatternFinder idref = IdrefTest.getInstance();
        StringFn sf = (StringFn)SystemFunction.makeSystemFunction(
                "string", new Expression[]{new ContextItemExpression()});
        StringLiteral regex = new StringLiteral("\\s+");
        Tokenize use = (Tokenize)SystemFunction.makeSystemFunction("tokenize", new Expression[]{sf, regex});
        KeyDefinition key = new KeyDefinition(idref, use, null, null);
        key.setIndexedItemType(BuiltInAtomicType.STRING);
        try {
            addKeyDefinition(StandardNames.getStructuredQName(StandardNames.XS_IDREFS), key, config);
        } catch (XPathException err) {

View Full Code Here

Examples of net.sf.saxon.functions.Tokenize

    private void registerIdrefKey(Configuration config) {
        PatternFinder idref = IdrefTest.getInstance();
        Expression eval = new Atomizer(new ContextItemExpression(), config);


        StringLiteral regex = new StringLiteral("\\s");
        Tokenize use = (Tokenize)SystemFunction.makeSystemFunction("tokenize", new Expression[]{eval, regex});
        KeyDefinition key = new KeyDefinition(idref, use, null, null);
        key.setIndexedItemType(BuiltInAtomicType.STRING);
        try {
            addKeyDefinition(StandardNames.getStructuredQName(StandardNames.XS_IDREFS), key, config);
        } catch (XPathException err) {

View Full Code Here

Examples of org.apache.pig.builtin.TOKENIZE

        t1.set(0, "123 456\"789");
        Tuple t2 = tf.newTuple(1);
        t2.set(0, null);
        Tuple t3 = tf.newTuple(0);
        
        TOKENIZE f = new TOKENIZE();
        DataBag b = f.exec(t1);
        assertTrue(b.size()==3);
        Iterator<Tuple> i = b.iterator();
        Tuple rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("789"));
        
        b = f.exec(t2);
        assertTrue(b==null);
        
        b = f.exec(t3);
        assertTrue(b==null);
    }

View Full Code Here

Examples of org.apache.pig.builtin.TOKENIZE

        t1.set(0, "123 456\"789");
        Tuple t2 = tf.newTuple(1);
        t2.set(0, null);
        Tuple t3 = tf.newTuple(0);
        
        TOKENIZE f = new TOKENIZE();
        DataBag b = f.exec(t1);
        assertTrue(b.size()==3);
        Iterator<Tuple> i = b.iterator();
        Tuple rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("789"));
        
        b = f.exec(t2);
        assertTrue(b==null);
        
        b = f.exec(t3);
        assertTrue(b==null);
    }

View Full Code Here

Examples of org.apache.pig.builtin.TOKENIZE

        t1.set(0, "123 456\"789");
        Tuple t2 = tf.newTuple(1);
        t2.set(0, null);
        Tuple t3 = tf.newTuple(0);
        
        TOKENIZE f = new TOKENIZE();
        DataBag b = f.exec(t1);
        assertTrue(b.size()==3);
        Iterator<Tuple> i = b.iterator();
        Tuple rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("789"));
        
        b = f.exec(t2);
        assertTrue(b==null);
        
        b = f.exec(t3);
        assertTrue(b==null);
    }

View Full Code Here

Examples of org.apache.pig.builtin.TOKENIZE

        t1.set(0, "123 456\"789");
        Tuple t2 = tf.newTuple(1);
        t2.set(0, null);
        Tuple t3 = tf.newTuple(0);
        
        TOKENIZE f = new TOKENIZE();
        DataBag b = f.exec(t1);
        assertTrue(b.size()==3);
        Iterator<Tuple> i = b.iterator();
        Tuple rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("789"));
        
        b = f.exec(t2);
        assertTrue(b==null);
        
        b = f.exec(t3);
        assertTrue(b==null);
    }

View Full Code Here

Examples of org.apache.pig.builtin.TOKENIZE

        t1.set(0, "123 456\"789");
        Tuple t2 = tf.newTuple(1);
        t2.set(0, null);
        Tuple t3 = tf.newTuple(0);


        TOKENIZE f = new TOKENIZE();
        DataBag b = f.exec(t1);
        assertTrue(b.size()==3);
        Iterator<Tuple> i = b.iterator();
        Tuple rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("789"));
        
        // Check when delim specified
        Tuple t4 = tf.newTuple(2);
        t4.set(0, "123|456|78\"9");
        t4.set(1, "|");        
        b = f.exec(t4);
        assertTrue(b.size()==3);
        i = b.iterator();
        rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("78\"9"));


        b = f.exec(t2);
        assertTrue(b==null);
        
        b = f.exec(t3);
        assertTrue(b==null);
    }

View Full Code Here

Examples of org.apache.pig.builtin.TOKENIZE

        t1.set(0, "123 456\"789");
        Tuple t2 = tf.newTuple(1);
        t2.set(0, null);
        Tuple t3 = tf.newTuple(0);


        TOKENIZE f = new TOKENIZE();
        DataBag b = f.exec(t1);
        assertTrue(b.size()==3);
        Iterator<Tuple> i = b.iterator();
        Tuple rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("789"));
        
        // Check when delim specified
        Tuple t4 = tf.newTuple(2);
        t4.set(0, "123|456|78\"9");
        t4.set(1, "|");        
        b = f.exec(t4);
        assertTrue(b.size()==3);
        i = b.iterator();
        rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("78\"9"));


        b = f.exec(t2);
        assertTrue(b==null);
        
        b = f.exec(t3);
        assertTrue(b==null);
    }

View Full Code Here

Examples of org.apache.pig.builtin.TOKENIZE

        t1.set(0, "123 456\"789");
        Tuple t2 = tf.newTuple(1);
        t2.set(0, null);
        Tuple t3 = tf.newTuple(0);


        TOKENIZE f = new TOKENIZE();
        DataBag b = f.exec(t1);
        assertTrue(b.size()==3);
        Iterator<Tuple> i = b.iterator();
        Tuple rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("789"));
        
        // Check when delim specified
        Tuple t4 = tf.newTuple(2);
        t4.set(0, "123|456|78\"9");
        t4.set(1, "|");        
        b = f.exec(t4);
        assertTrue(b.size()==3);
        i = b.iterator();
        rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("78\"9"));


        b = f.exec(t2);
        assertTrue(b==null);
        
        b = f.exec(t3);
        assertTrue(b==null);
    }

View Full Code Here

Examples of org.apache.pig.builtin.TOKENIZE

        t1.set(0, "123 456\"789");
        Tuple t2 = tf.newTuple(1);
        t2.set(0, null);
        Tuple t3 = tf.newTuple(0);


        TOKENIZE f = new TOKENIZE();
        DataBag b = f.exec(t1);
        assertTrue(b.size()==3);
        Iterator<Tuple> i = b.iterator();
        Tuple rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("789"));
        
        // Check when delim specified
        Tuple t4 = tf.newTuple(2);
        t4.set(0, "123|456|78\"9");
        t4.set(1, "|");        
        b = f.exec(t4);
        assertTrue(b.size()==3);
        i = b.iterator();
        rt = i.next();
        assertTrue(rt.get(0).equals("123"));
        rt = i.next();
        assertTrue(rt.get(0).equals("456"));
        rt = i.next();
        assertTrue(rt.get(0).equals("78\"9"));


        b = f.exec(t2);
        assertTrue(b==null);
        
        b = f.exec(t3);
        assertTrue(b==null);
    }

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.