Examples of com.datasalt.pangool.io.Schema

com.datasalt.pangool.io.Schema
A list of {@link Field} elements that a {@link ITuple} instance contains.

    return field;
  }


  private Field checkFieldInSchema(String fieldName, int schemaId)
      throws TupleMRException {
    Schema schema = mrConfig.getIntermediateSchema(schemaId);
    Field field = schema.getField(fieldName);
    if(field == null) {
      throw new TupleMRException("Field '" + fieldName + "' not present in source '"
          + schema.getName() + "' " + schema);
    }
    return field;
  }

View Full Code Here

    return groupSchema;
  }


  private void calculateIndexTranslations() {
    for(int schemaId = 0; schemaId < mrConfig.getIntermediateSchemas().size(); schemaId++) {
      Schema sourceSchema = mrConfig.getIntermediateSchema(schemaId);
      commonToIntermediateIndexes.add(getIndexTranslation(commonSchema, sourceSchema));
      groupToIntermediateIndexes.add(getIndexTranslation(groupSchema, sourceSchema));
      if(specificSchemas != null && !specificSchemas.isEmpty()) {
        Schema particularSchema = specificSchemas.get(schemaId);
        specificToIntermediateIndexes.add(getIndexTranslation(particularSchema,
            sourceSchema));
      }
    }
    commonToIntermediateIndexes = Collections

View Full Code Here

          "Need to specify source order in common OrderBy when using specific OrderBy");
    }
    if(ordering.getSchemaOrderIndex() != null) {
      throw new TupleMRException("Not allowed to set source order in specific order");
    }
    Schema schema = getSchemaByName(schemaName);
    for(SortElement e : ordering.getElements()) {
      if(!schema.containsField(e.getName())) {
        throw new TupleMRException("Source '" + schemaName + "' doesn't contain field '"
            + e.getName());
      }
    }

View Full Code Here

  }


  private static Schema getPangoolTweetSchema() {
    Field tweetIdField = Field.create("tweet_id", Schema.Field.Type.INT);
    Field tweetHashTags = Fields.createAvroField("tweet_hashtags", getAvroStringArraySchema(), false);
    return new Schema("tweet", Arrays.asList(tweetIdField, tweetHashTags));
  }

View Full Code Here

  }


  private static Schema getPangoolRetweetSchema() {
    Field userId = Field.create("username", Schema.Field.Type.STRING);
    Field tweetId = Field.create("tweet_id", Schema.Field.Type.INT);
    return new Schema("retweet", Arrays.asList(userId, tweetId));
  }

View Full Code Here

    super("Usage: AvroTopicalWordCount [input_path] [output_path]");
  }


  static Schema getSchema() {
    Field avroField = Fields.createAvroField("my_avro", getAvroSchema(), false);
    return new Schema("schema", Arrays.asList(avroField));
  }

View Full Code Here

  static Schema getURLRegisterSchema() {
    List<Field> urlRegisterFields = new ArrayList<Field>();
    urlRegisterFields.add(Field.create("url", Type.STRING));
    urlRegisterFields.add(Field.create("timestamp", Type.LONG));
    urlRegisterFields.add(Field.create("ip", Type.STRING));
    return new Schema("urlRegister", urlRegisterFields);
  }

View Full Code Here


  static Schema getURLMapSchema() {
    List<Field> urlMapFields = new ArrayList<Field>();
    urlMapFields.add(Field.create("nonCanonicalUrl", Type.STRING));
    urlMapFields.add(Field.create("canonicalUrl", Type.STRING));
    return new Schema("urlMap", urlMapFields);
  }

View Full Code Here

  static Schema getURLRegisterSchema() {
    List<Field> urlRegisterFields = new ArrayList<Field>();
    urlRegisterFields.add(Field.create("url",Type.STRING));
    urlRegisterFields.add(Field.create("timestamp",Type.LONG));
    urlRegisterFields.add(Field.create("ip",Type.STRING));
    return new Schema("urlRegister", urlRegisterFields);    
  }

View Full Code Here


  static Schema getURLMapSchema() {
    List<Field> urlMapFields = new ArrayList<Field>();
    urlMapFields.add(Field.create("nonCanonicalUrl",Type.STRING));
    urlMapFields.add(Field.create("canonicalUrl",Type.STRING));
    return new Schema("urlMap", urlMapFields);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.datasalt.pangool.io.Schema

com.datasalt.pangool.examples.avro.AvroCustomSerializationJob

com.datasalt.pangool.examples.avro.AvroTopicalWordCount

com.datasalt.pangool.examples.avro.AvroTweetsJoin

com.datasalt.pangool.examples.gameoflife.GameOfLifeJob

com.datasalt.pangool.examples.movingaverage.MovingAverage

com.datasalt.pangool.examples.secondarysort.SecondarySort

com.datasalt.pangool.examples.simplesecondarysort.SimpleSecondarySort

com.datasalt.pangool.examples.topicalwordcount.TopicalWordCount

com.datasalt.pangool.examples.topicalwordcount.TopicFingerprint

com.datasalt.pangool.examples.topnhashtags.TopNHashTags

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.