Examples of LineageTracer


Examples of org.apache.pig.pen.util.LineageTracer

            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            illustrator.addData(tOut);
            if (!illustrator.getEqClassesShared()) {
                illustrator.getEquivalenceClasses().get(0).add(tOut);
            }
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            tOut.synthetic = ((ExampleTuple) in).synthetic;
            lineageTracer.union((ExampleTuple) in , tOut);
            return tOut;
        } else {
            return (Tuple) out;
        }
    }
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        if(illustrator != null) {
            ExampleTuple tIn = (ExampleTuple) in;
            LineageTracer lineage = illustrator.getLineage();
            lineage.insert(tIn);
            if (!isTmpStore)
                illustrator.getEquivalenceClasses().get(eqClassIndex).add(tIn);
            illustrator.addData((Tuple) out);
        }
        return (Tuple) out;
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

       @Override
       public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
           if(illustrator != null) {
               ExampleTuple tOut = new ExampleTuple((Tuple) out);
               LineageTracer lineageTracer = illustrator.getLineage();
               lineageTracer.insert(tOut);
               Tuple tmp;
               boolean synthetic = false;
               if (illustrator.getEquivalenceClasses() == null) {
                   LinkedList<IdentityHashSet<Tuple>> equivalenceClasses = new LinkedList<IdentityHashSet<Tuple>>();
                   for (int i = 0; i < numInputs; ++i) {
                       IdentityHashSet<Tuple> equivalenceClass = new IdentityHashSet<Tuple>();
                       equivalenceClasses.add(equivalenceClass);
                   }
                   illustrator.setEquivalenceClasses(equivalenceClasses, this);
               }

               if (distinct) {
                   int count;
                   for (count = 0; tupIter.hasNext(); ++count) {
                       NullableTuple ntp = tupIter.next();
                       tmp = (Tuple) ntp.getValueAsPigType();
                       if (!tmp.equals(tOut))
                           lineageTracer.union(tOut, tmp);
                   }
                   if (count > 1) // only non-distinct tuples are inserted into the equivalence class
                       illustrator.getEquivalenceClasses().get(eqClassIndex).add(tOut);
                   illustrator.addData((Tuple) tOut);
                   return (Tuple) tOut;
               }
               boolean outInEqClass = true;
               try {
                   for (int i = 1; i < numInputs+1; i++)
                   {
                       DataBag dbs = (DataBag) ((Tuple) out).get(i);
                       Iterator<Tuple> iter = dbs.iterator();
                       if (dbs.size() <= 1 && outInEqClass) // all inputs have >= 2 records
                           outInEqClass = false;
                       while (iter.hasNext()) {
                           tmp = iter.next();
                           // any of synthetic data in bags causes the output tuple to be synthetic
                           if (!synthetic && ((ExampleTuple)tmp).synthetic)
                               synthetic = true;
                           lineageTracer.union(tOut, tmp);
                       }
                   }
               } catch (ExecException e) {
                 // TODO better exception handling
                 throw new RuntimeException("Illustrator exception :"+e.getMessage());
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

   
    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        if(illustrator != null) {
            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            if (illustrator.getEquivalenceClasses() == null) {
                LinkedList<IdentityHashSet<Tuple>> equivalenceClasses = new LinkedList<IdentityHashSet<Tuple>>();
                for (int i = 0; i < numInputs; ++i) {
                    IdentityHashSet<Tuple> equivalenceClass = new IdentityHashSet<Tuple>();
                    equivalenceClasses.add(equivalenceClass);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

   
    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        if(illustrator != null) {
            ExampleTuple tIn = (ExampleTuple) in;
            LineageTracer lineage = illustrator.getLineage();
            lineage.insert(tIn);
            if (!isTmpStore)
                illustrator.getEquivalenceClasses().get(eqClassIndex).add(tIn);
            illustrator.addData((Tuple) out);
        }
        return (Tuple) out;
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

        for (LogicalOperator in : op.getPlan().getPredecessors(op)) {
            if (derivedData.get(in) == null)
                return null;
        }

        LineageTracer oldLineage = this.lineage;
        this.lineage = new LineageTracer();

        PhysicalOperator physOp = LogToPhyMap.get(op);
        Random r = new Random();
        // get the list of original inputs
        // List<PhysicalOperator> inputs = physOp.getInputs();
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

            visitor.visit();
        } catch (VisitorException e) {
            log.error(e.getMessage());
        }

        LineageTracer lineage = visitor.lineage;
        Lineage.put(plan.getLeaves().get(0), lineage);
        Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = visitor.OpToEqClasses;
        Collection<IdentityHashSet<Tuple>> EqClasses = visitor.EqClasses;
        Map<IdentityHashSet<Tuple>, Integer> affinityGroup = new HashMap<IdentityHashSet<Tuple>, Integer>();
        for (IdentityHashSet<Tuple> set : EqClasses) {
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

            Map<IdentityHashSet<Tuple>, Integer> affinityGroups = null;

            continueTrimming = checkCompleteness(cg);
           
            DerivedDataVisitor visitor = null;
            LineageTracer lineage = null;
            // create affinity groups
            if (cg.getInputs().size() == 1) {
                affinityGroups = new HashMap<IdentityHashSet<Tuple>, Integer>();
                LogicalOperator childOp = cg.getInputs().get(0);
                visitor = new DerivedDataVisitor(childOp, null, baseData,
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

        Lineage.put(childOp, visitor.lineage);

    }

    private boolean checkCompleteness(LogicalOperator op) {
        LineageTracer lineage = Lineage.get(op);
        Lineage.remove(op);

        Map<IdentityHashSet<Tuple>, Integer> affinityGroups = AffinityGroups
                .get(op);
        AffinityGroups.remove(op);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

            POLoad poLoad = (POLoad) LogToPhyMap.get(op);
//            PigContext oldPC = poLoad.getPc();
//            poLoad.setPc(pigContext);

            poLoad.setLineageTracer(new LineageTracer());

            Tuple t = null;
            int count = 0;
            for (Result res = poLoad.getNext(t); res.returnStatus != POStatus.STATUS_EOP
                    && count < MAX_RECORDS; res = poLoad.getNext(t)) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.