Package org.apache.crunch.impl.spark.collect

Examples of org.apache.crunch.impl.spark.collect.SparkCollectFactory

39
40
41
42
43
44
45
46
  private final String sparkConnect;
  private JavaSparkContext sparkContext;
  private final Map<PCollection<?>, StorageLevel> cachedCollections = Maps.newHashMap();

  public SparkPipeline(String sparkConnect, String appName) {
    super(appName, new Configuration(), new SparkCollectFactory());
    this.sparkConnect = Preconditions.checkNotNull(sparkConnect);
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
    super(appName, new Configuration(), new SparkCollectFactory());
    this.sparkConnect = Preconditions.checkNotNull(sparkConnect);
  }

  public SparkPipeline(JavaSparkContext sparkContext, String appName) {
    super(appName, new Configuration(), new SparkCollectFactory());
    this.sparkContext = Preconditions.checkNotNull(sparkContext);
    this.sparkConnect = sparkContext.getSparkHome().orNull();
  }
View Full Code Here
53
54
55
56
57
58
59
60
61
  public SparkPipeline(String sparkConnect, String appName) {
    this(sparkConnect, appName, null);
  }

  public SparkPipeline(String sparkConnect, String appName, Class<?> jarClass) {
    super(appName, new Configuration(), new SparkCollectFactory());
    this.sparkConnect = Preconditions.checkNotNull(sparkConnect);
    this.jarClass = jarClass;
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
    this.sparkConnect = Preconditions.checkNotNull(sparkConnect);
    this.jarClass = jarClass;
  }

  public SparkPipeline(JavaSparkContext sparkContext, String appName) {
    super(appName, new Configuration(), new SparkCollectFactory());
    this.sparkContext = Preconditions.checkNotNull(sparkContext);
    this.sparkConnect = sparkContext.getSparkHome().orNull();
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
  public SparkPipeline(String sparkConnect, String appName, Class<?> jarClass) {
    this(sparkConnect, appName, jarClass, new Configuration());
  }

  public SparkPipeline(String sparkConnect, String appName, Class<?> jarClass, Configuration conf) {
    super(appName, conf, new SparkCollectFactory());
    this.sparkConnect = Preconditions.checkNotNull(sparkConnect);
    this.jarClass = jarClass;
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
    this.sparkConnect = Preconditions.checkNotNull(sparkConnect);
    this.jarClass = jarClass;
  }

  public SparkPipeline(JavaSparkContext sparkContext, String appName) {
    super(appName, sparkContext.hadoopConfiguration(), new SparkCollectFactory());
    this.sparkContext = Preconditions.checkNotNull(sparkContext);
    this.sparkConnect = sparkContext.getSparkHome().orNull();
  }
View Full Code Here
TOP

Related Classes of org.apache.crunch.impl.spark.collect.SparkCollectFactory

  • org.apache.crunch.impl.spark.SparkPipeline
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.