Package eu.stratosphere.api.java.record.operators

Examples of eu.stratosphere.api.java.record.operators.FileDataSource

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
      this.outputPath = args[7];
    }
   
    /* Create the 6 data sources: */
    /* part: (partkey | name, mfgr, brand, type, size, container, retailprice, comment) */
    FileDataSource partInput = new FileDataSource(
      new IntTupleDataInFormat(), this.partInputPath, "\"part\" source");
    //partInput.setOutputContract(UniqueKey.class);
//    partInput.getCompilerHints().setAvgNumValuesPerKey(1);

    /* partsupp: (partkey | suppkey, availqty, supplycost, comment) */
    FileDataSource partSuppInput = new FileDataSource(
      new IntTupleDataInFormat(), this.partSuppInputPath, "\"partsupp\" source");

    /* orders: (orderkey | custkey, orderstatus, totalprice, orderdate, orderpriority, clerk, shippriority, comment) */
    FileDataSource ordersInput = new FileDataSource(
      new IntTupleDataInFormat(), this.ordersInputPath, "\"orders\" source");
    //ordersInput.setOutputContract(UniqueKey.class);
//    ordersInput.getCompilerHints().setAvgNumValuesPerKey(1);

    /* lineitem: (orderkey | partkey, suppkey, linenumber, quantity, extendedprice, discount, tax, ...) */
    FileDataSource lineItemInput = new FileDataSource(
      new IntTupleDataInFormat(), this.lineItemInputPath, "\"lineitem\" source");

    /* supplier: (suppkey | name, address, nationkey, phone, acctbal, comment) */
    FileDataSource supplierInput = new FileDataSource(
      new IntTupleDataInFormat(), this.supplierInputPath, "\"supplier\" source");
    //supplierInput.setOutputContract(UniqueKey.class);
//    supplierInput.getCompilerHints().setAvgNumValuesPerKey(1);

    /* nation: (nationkey | name, regionkey, comment) */
    FileDataSource nationInput = new FileDataSource(
      new IntTupleDataInFormat(), this.nationInputPath, "\"nation\" source");
    //nationInput.setOutputContract(UniqueKey.class);
//    nationInput.getCompilerHints().setAvgNumValuesPerKey(1);

    /* Filter on part's name, project values to NULL: */
 
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
    // parse job parameters
    int numSubTasks   = (args.length > 0 ? Integer.parseInt(args[0]) : 1);
    String dataInput = (args.length > 1 ? args[1] : "");
    String output    = (args.length > 2 ? args[2] : "");

    FileDataSource source = new FileDataSource(new TextInputFormat(), dataInput, "Input Lines");
    MapOperator mapper = MapOperator.builder(new TokenizeLine())
      .input(source)
      .name("Tokenize Lines")
      .build();
    ReduceOperator reducer = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
203
204
205
   
    new NepheleJobGraphGenerator().compileJobGraph(oPlan);
  }
 
  private Plan getRecordTestPlan(boolean joinPreservesSolutionSet, boolean mapBeforeSolutionDelta) {
    FileDataSource solutionSetInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Solution Set");
    FileDataSource worksetInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Workset");
   
    FileDataSource invariantInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Invariant Input");
   
    DeltaIteration iteration = new DeltaIteration(0, ITERATION_NAME);
    iteration.setInitialSolutionSet(solutionSetInput);
    iteration.setInitialWorkset(worksetInput);
    iteration.setMaximumNumberOfIterations(100);
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
public class AdditionalOperatorsTest extends CompilerTestBase {

  @Test
  public void testCrossWithSmall() {
    // construct the plan
    FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
    FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
   
    CrossOperator cross = CrossWithSmallOperator.builder(new DummyCrossStub())
        .input1(source1).input2(source2)
        .name("Cross").build();
 
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
  }
 
  @Test
  public void testCrossWithLarge() {
    // construct the plan
    FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
    FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
   
    CrossOperator cross= CrossWithLargeOperator.builder(new DummyCrossStub())
        .input1(source1).input2(source2)
        .name("Cross").build();
 
View Full Code Here
TOP

Related Classes of eu.stratosphere.api.java.record.operators.FileDataSource

  • eu.stratosphere.pact.compiler.AdditionalOperatorsTest
  • eu.stratosphere.pact.compiler.BranchingPlansCompilerTest
  • eu.stratosphere.pact.compiler.DOPChangeTest
  • eu.stratosphere.pact.compiler.GroupOrderTest
  • eu.stratosphere.pact.compiler.HardPlansCompilationTest
  • eu.stratosphere.pact.compiler.ReduceAllTest
  • eu.stratosphere.pact.compiler.UnionPropertyPropagationTest
  • eu.stratosphere.pact.compiler.WorksetIterationsRecordApiCompilerTest
  • eu.stratosphere.test.accumulators.AccumulatorITCase
  • eu.stratosphere.test.accumulators.AccumulatorIterativeITCase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.