Examples of LineageTracer


Examples of org.apache.pig.pen.util.LineageTracer

        if (illustrator != null) {
            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            illustrator.addData(tOut);
            illustrator.getEquivalenceClasses().get(eqClassIndex).add(
                    (Tuple) out);
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            for (int i = 0; i < data.length; i++) {
                lineageTracer.union(tOut, data[i]);
            }
            return tOut;
        } else {
            return (Tuple) out;
        }
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

   
    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        if(illustrator != null) {
            ExampleTuple tIn = (ExampleTuple) in;
            LineageTracer lineage = illustrator.getLineage();
            lineage.insert(tIn);
            if (!isTmpStore)
                illustrator.getEquivalenceClasses().get(eqClassIndex).add(tIn);
            illustrator.addData((Tuple) out);
        }
        return (Tuple) out;
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            illustrator.addData(tOut);
            if (!illustrator.getEqClassesShared()) {
                illustrator.getEquivalenceClasses().get(0).add(tOut);
            }
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            tOut.synthetic = ((ExampleTuple) in).synthetic;
            lineageTracer.union((ExampleTuple) in , tOut);
            return tOut;
        } else {
            return (Tuple) out;
        }
    }
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

      
       @Override
       public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
           if(illustrator != null) {
               ExampleTuple tOut = new ExampleTuple((Tuple) out);
               LineageTracer lineageTracer = illustrator.getLineage();
               lineageTracer.insert(tOut);
               Tuple tmp;
               boolean synthetic = false;
               if (illustrator.getEquivalenceClasses() == null) {
                   LinkedList<IdentityHashSet<Tuple>> equivalenceClasses = new LinkedList<IdentityHashSet<Tuple>>();
                   for (int i = 0; i < numInputs; ++i) {
                       IdentityHashSet<Tuple> equivalenceClass = new IdentityHashSet<Tuple>();
                       equivalenceClasses.add(equivalenceClass);
                   }
                   illustrator.setEquivalenceClasses(equivalenceClasses, this);
               }

               if (distinct) {
                   int count;
                   for (count = 0; tupIter.hasNext(); ++count) {
                       NullableTuple ntp = tupIter.next();
                       tmp = (Tuple) ntp.getValueAsPigType();
                       if (!tmp.equals(tOut))
                           lineageTracer.union(tOut, tmp);
                   }
                   if (count > 1) // only non-distinct tuples are inserted into the equivalence class
                       illustrator.getEquivalenceClasses().get(eqClassIndex).add(tOut);
                   illustrator.addData((Tuple) tOut);
                   return (Tuple) tOut;
               }
               boolean outInEqClass = true;
               try {
                   for (int i = 1; i < numInputs+1; i++)
                   {
                       DataBag dbs = (DataBag) ((Tuple) out).get(i);
                       Iterator<Tuple> iter = dbs.iterator();
                       if (dbs.size() <= 1 && outInEqClass) // all inputs have >= 2 records
                           outInEqClass = false;
                       while (iter.hasNext()) {
                           tmp = iter.next();
                           // any of synthetic data in bags causes the output tuple to be synthetic
                           if (!synthetic && ((ExampleTuple)tmp).synthetic)
                               synthetic = true;
                           lineageTracer.union(tOut, tmp);
                       }
                   }
               } catch (ExecException e) {
                 // TODO better exception handling
                 throw new RuntimeException("Illustrator exception :"+e.getMessage());
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

   
    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        if(illustrator != null) {
            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            if (illustrator.getEquivalenceClasses() == null) {
                LinkedList<IdentityHashSet<Tuple>> equivalenceClasses = new LinkedList<IdentityHashSet<Tuple>>();
                for (int i = 0; i < numInputs; ++i) {
                    IdentityHashSet<Tuple> equivalenceClass = new IdentityHashSet<Tuple>();
                    equivalenceClasses.add(equivalenceClass);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

    }
   
    private Map<Operator, DataBag> getData(PhysicalPlan plan) throws PigException, IOException, InterruptedException
    {
        // get data on a physical plan possibly trimmed of one branch
        lineage = new LineageTracer();
        IllustratorAttacher attacher = new IllustratorAttacher(plan, lineage, MAX_RECORDS, poLoadToSchemaMap, pigContext);
        attacher.visit();
        if (oriLimitMap != null) {
            for (Map.Entry<LOLimit, Long> entry : oriLimitMap.entrySet()) {
                logToPhyMap.get(entry.getKey()).getIllustrator().setOriginalLimit(entry.getValue());
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

    public void init() throws IOException, InterruptedException {

        Map<Operator, DataBag> data = eg.getData();

        LineageTracer lineage = eg.getLineage();
        Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = eg.getLoToEqClassMap();
        for (Operator leaf : plan.getSinks()) {
            Lineage.put(leaf, lineage);
            AffinityGroups.put(leaf, eg.getEqClasses());
        }
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

        if (continueTrimming) {
            try {

                continueTrimming = checkCompleteness(cg);
               
                LineageTracer lineage = null;
                // create affinity groups
                if (cg.getInputs(plan).size() == 1) {
                    lineage = eg.getLineage();
                    AffinityGroups.put(cg.getInputs(plan).get(0), eg.getEqClasses());
                    Lineage.put(cg.getInputs(plan).get(0), lineage);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

          throw new FrontendException("Exception: "+e.getMessage());
        }
    }

    private boolean checkCompleteness(LogicalRelationalOperator op) throws Exception {
        LineageTracer lineage = Lineage.get(op);
        Lineage.remove(op);

        Collection<IdentityHashSet<Tuple>> affinityGroups = AffinityGroups
                .get(op);
        AffinityGroups.remove(op);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

   
    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        if(illustrator != null) {
            ExampleTuple tIn = (ExampleTuple) in;
            LineageTracer lineage = illustrator.getLineage();
            lineage.insert(tIn);
            if (!isTmpStore)
                illustrator.getEquivalenceClasses().get(eqClassIndex).add(tIn);
            illustrator.addData((Tuple) out);
        }
        return (Tuple) out;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.