Examples of CSVStrategy


Examples of com.googlecode.jcsv.CSVStrategy

      FileSplit split = (FileSplit) genericSplit;
      Configuration conf = context.getConfiguration();
      this.maxLineLength = conf.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
     
      boolean skipFirstLine = false;
      csvStrategy = new CSVStrategy(separator, quote, '#', hasHeader && start == 0, true);

      // Skip txt header
      skipFirstLine = (hasHeader && start == 0);
     
      start = split.getStart();
View Full Code Here

Examples of com.googlecode.jcsv.CSVStrategy

      FileSplit split = (FileSplit) genericSplit;
      Configuration conf = context.getConfiguration();
      this.maxLineLength = conf.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);

      boolean skipFirstLine = false;
      csvStrategy = new CSVStrategy(separator, quote, '#', hasHeader && start == 0, true);

      // Skip txt header
      skipFirstLine = (hasHeader && start == 0);

      start = split.getStart();
View Full Code Here

Examples of com.googlecode.jcsv.CSVStrategy

      FileSplit split = (FileSplit) genericSplit;
      Configuration conf = context.getConfiguration();
      this.maxLineLength = conf.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
     
      boolean skipFirstLine = false;
      csvStrategy = new CSVStrategy(separator, quote, '#', hasHeader && start == 0, true);

      // Skip txt header
      skipFirstLine = (hasHeader && start == 0);
     
      start = split.getStart();
View Full Code Here

Examples of com.googlecode.jcsv.CSVStrategy

      FileSplit split = (FileSplit) genericSplit;
      Configuration conf = context.getConfiguration();
      this.maxLineLength = conf.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
     
      boolean skipFirstLine = false;
      csvStrategy = new CSVStrategy(separator, quote, '#', hasHeader && start == 0, true);

      // Skip txt header
      skipFirstLine = (hasHeader && start == 0);
     
      start = split.getStart();
View Full Code Here

Examples of com.googlecode.jcsv.CSVStrategy

  public void testMasRecordSize() throws IOException {
    NullableCSVTokenizer tok = new NullableCSVTokenizer('\\', false, null);
    tok.setMaxFieldSize(10);
    CSVReader<String[]> cvs = new CSVReaderBuilder<String[]>(new StringReader(
        "hola,que,\"tal va la vida en este mundo tan cruel\",te,va,yobien"))
        .strategy(new CSVStrategy(',', '"', '\\', false, true)).tokenizer(tok)
        .entryParser(new DefaultCSVEntryParser()).build();

    cvs.readNext();
  }
View Full Code Here

Examples of com.googlecode.jcsv.CSVStrategy

  }

  public CSVReader<String[]> getCSVParser(String line, char separator, char quote, char escape,
      boolean strictQuotes, String nullString) {
    return new CSVReaderBuilder<String[]>(new StringReader(line))
        .strategy(new CSVStrategy(separator, quote, '#', false, true))
        .tokenizer(new NullableCSVTokenizer(escape, strictQuotes, nullString))
        .entryParser(new DefaultCSVEntryParser()).build();
  }
View Full Code Here

Examples of com.googlecode.jcsv.CSVStrategy

      FileSplit split = (FileSplit) genericSplit;
      Configuration conf = context.getConfiguration();
      this.maxLineLength = conf.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
     
      boolean skipFirstLine = false;
      csvStrategy = new CSVStrategy(separator, quote, '#', hasHeader && start == 0, true);

      // Skip txt header
      skipFirstLine = (hasHeader && start == 0);
     
      start = split.getStart();
View Full Code Here

Examples of org.apache.commons.csv.CSVStrategy

      templateAdd.overwritePending=false;
    }

    templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
   
    strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
    String sep = params.get(SEPARATOR);
    if (sep!=null) {
      if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
      strategy.setDelimiter(sep.charAt(0));
    }
View Full Code Here

Examples of org.apache.commons.csv.CSVStrategy

        String encStr = params.getFieldParam(fname,ENCAPSULATOR);
        char fenc = encStr==null || encStr.length()==0 ? (char)-2 : encStr.charAt(0);
        String escStr = params.getFieldParam(fname,ESCAPE);
        char fesc = escStr==null || escStr.length()==0 ? CSVStrategy.ESCAPE_DISABLED : escStr.charAt(0);

        CSVStrategy fstrat = new CSVStrategy(fsep,fenc,CSVStrategy.COMMENTS_DISABLED,fesc, false, false, false, false);
        adders[i] = new CSVLoader.FieldSplitter(fstrat, adders[i]);
      }
    }
  }
View Full Code Here

Examples of org.apache.commons.csv.CSVStrategy

  }

  public void writeResponse() throws IOException {
    SolrParams params = req.getParams();

    strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
    CSVStrategy strat = strategy;

    String sep = params.get(CSV_SEPARATOR);
    if (sep!=null) {
      if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
      strat.setDelimiter(sep.charAt(0));
    }

    String nl = params.get(CSV_NEWLINE);
    if (nl!=null) {
      if (nl.length()==0) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid newline:'"+nl+"'");
      strat.setPrinterNewline(nl);
    }

    String encapsulator = params.get(CSV_ENCAPSULATOR);
    String escape = params.get(CSV_ESCAPE);
    if (encapsulator!=null) {
      if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'");
      strat.setEncapsulator(encapsulator.charAt(0));
    }

    if (escape!=null) {
      if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'");
      strat.setEscape(escape.charAt(0));
      if (encapsulator == null) {
        strat.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);
      }
    }

    if (strat.getEscape() == '\\') {
      // If the escape is the standard backslash, then also enable
      // unicode escapes (it's harmless since 'u' would not otherwise
      // be escaped.
      strat.setUnicodeEscapeInterpretation(true);
    }

    printer = new CSVPrinter(writer, strategy);
   

    CSVStrategy mvStrategy = new CSVStrategy(strategy.getDelimiter(), CSVStrategy.ENCAPSULATOR_DISABLED, CSVStrategy.COMMENTS_DISABLED, '\\', false, false, false, false);
    strat = mvStrategy;

    sep = params.get(MV_SEPARATOR);
    if (sep!=null) {
      if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv separator:'"+sep+"'");
      strat.setDelimiter(sep.charAt(0));
    }

    encapsulator = params.get(MV_ENCAPSULATOR);
    escape = params.get(MV_ESCAPE);

    if (encapsulator!=null) {
      if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv encapsulator:'"+encapsulator+"'");
      strat.setEncapsulator(encapsulator.charAt(0));
      if (escape == null) {
        strat.setEscape(CSVStrategy.ESCAPE_DISABLED);
      }
    }

    escape = params.get(MV_ESCAPE);
    if (escape!=null) {
      if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv escape:'"+escape+"'");
      strat.setEscape(escape.charAt(0));
      // encapsulator will already be disabled if it wasn't specified
    }

    returnScore = returnFields != null && returnFields.contains("score");
    boolean needListOfFields = returnFields==null || returnFields.size()==0 || (returnFields.size()==1 && returnScore) || returnFields.contains("*");
    Collection<String> fields = returnFields;

    Object responseObj = rsp.getValues().get("response");
    if (needListOfFields) {
      if (responseObj instanceof SolrDocumentList) {
        // get the list of fields from the SolrDocumentList
        fields = new LinkedHashSet<String>();
        for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
          fields.addAll(sdoc.getFieldNames());
        }
      } else {
        // get the list of fields from the index
        fields = req.getSearcher().getFieldNames();
      }
      if (returnScore) {
        fields.add("score");
      } else {
        fields.remove("score");
      }
    }

    CSVSharedBufPrinter csvPrinterMV = new CSVSharedBufPrinter(mvWriter, mvStrategy);

    for (String field : fields) {
      if (field.equals("score")) {
        CSVField csvField = new CSVField();
        csvField.name = "score";
        csvFields.put("score", csvField);
        continue;
      }

      SchemaField sf = schema.getFieldOrNull(field);
      if (sf == null) {
        FieldType ft = new StrField();
        sf = new SchemaField(field, ft);
      }

      // if we got the list of fields from the index, only list stored fields
      if (returnFields==null && sf != null && !sf.stored()) {
        continue;
      }

      // check for per-field overrides
      sep = params.get("f." + field + '.' + CSV_SEPARATOR);
      encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR);
      escape = params.get("f." + field + '.' + CSV_ESCAPE);

      CSVSharedBufPrinter csvPrinter = csvPrinterMV;
      if (sep != null || encapsulator != null || escape != null) {
        // create a new strategy + printer if there were any per-field overrides
        strat = (CSVStrategy)mvStrategy.clone();
        if (sep!=null) {
          if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv separator:'"+sep+"'");
          strat.setDelimiter(sep.charAt(0));
        }
        if (encapsulator!=null) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.