Examples of Chunker


Examples of opennlp.tools.chunker.Chunker

    }
    @Test
    public void testLoadEnChunker() throws IOException{
        ChunkerModel model = openNLP.getChunkerModel("en");
        Assert.assertNotNull(model);
        Chunker chunker = openNLP.getChunker("en");
        Assert.assertNotNull(chunker);
    }
View Full Code Here

Examples of opennlp.tools.chunker.Chunker

    }
    @Test
    public void testLoadMissingChunker() throws IOException{
        ChunkerModel model = openNLP.getChunkerModel("ru");
        Assert.assertNull(model);
        Chunker chunker = openNLP.getChunker("ru");
        Assert.assertNull(chunker);
    }   
View Full Code Here

Examples of opennlp.tools.chunker.Chunker

      File d = new File(dataDir);
     
      MaxentModel buildModel = null;
      MaxentModel checkModel = null;
      POSTagger posTagger = null;
      Chunker chunker = null;
      HeadRules headRules = null;

      if (!d.isDirectory()) {
        FileInputStream fis = new FileInputStream(d);
        ParserModel model = new ParserModel(fis);
View Full Code Here

Examples of org.neo4j.batchimport.utils.Chunker

        for (int i = 0; i < relTypesCount; i++) relTypeIds[i] = inserter.getRelTypeId(relTypes[i]);
    }

    private void initReader() throws IOException {
        nodesReader = new BufferedReader(readerFor(nodesFile), MEGABYTE);
        nodeChunker = new Chunker(nodesReader, delim);

        relsReader = new BufferedReader(readerFor(relationshipsFile), MEGABYTE);
        relChunker = new Chunker(relsReader, delim);
    }
View Full Code Here

Examples of org.neo4j.batchimport.utils.Chunker

public class ChunkerLineData extends AbstractLineData {
    private final Chunker chunker;

    public ChunkerLineData(Reader reader, char delim, int offset) {
        super(offset);
        chunker = new Chunker(reader, delim);
        initHeaders(createHeaders(readRawRow()));
        createMapData(lineSize, offset);
    }
View Full Code Here

Examples of org.neo4j.batchimport.utils.Chunker

    @Test
    public void testReadHeader() throws Exception {
        final BufferedReader reader = new BufferedReader(new StringReader(file));
        final String[] header = reader.readLine().split("\t");
        final Chunker chunker = new Chunker(reader, '\t');
        readLine(header, chunker, "FOO", "42");
        readLine(header, chunker, "", "42");
        assertEquals(Chunker.EOF,chunker.nextWord());
    }
View Full Code Here

Examples of org.neo4j.batchimport.utils.Chunker

    }

    @Test
    public void testPerformance() throws Exception {
        final BufferedReader reader = new BufferedReader(new FileReader(PerformanceTestFile.TEST_CSV));
        final Chunker chunker = new Chunker(reader, '\t');

        int res = 0;
        long time = System.currentTimeMillis();
        String token;
        while ( (token = chunker.nextWord()) != Chunker.EOF)  {
            if (token!=Chunker.NO_VALUE && token != Chunker.EOL) res++;
        }
        time = System.currentTimeMillis() - time;
        System.out.println("time = " + time + " ms.");
        Assert.assertEquals((PerformanceTestFile.ROWS) * PerformanceTestFile.COLS, res);
View Full Code Here

Examples of org.simpleframework.http.core.Chunker

  }
 
  public void testChunks(int chunkSize, int dribble, int entitySize) throws Exception {    
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      ByteArrayOutputStream plain = new ByteArrayOutputStream();
      Chunker encode = new Chunker(out);
      StringBuffer buf = new StringBuffer();
      int fill = 0;
     
      for(int i = 0, line = 0; i < entitySize; i++) {
         String text = "["+String.valueOf(i)+"]";
        
         if(fill >= chunkSize) {
            encode.write(buf.toString().getBytes("UTF-8"));
            plain.write(buf.toString().getBytes("UTF-8"));
            buf.setLength(0);
            fill = 0;           
            line = 0;
         }        
         line += text.length();
         fill += text.length();
         buf.append(text);
        
         if(line >= 48) {
            buf.append("\n");
            fill++;
            line = 0;
         }

      }
      if(buf.length() > 0) {
         encode.write(buf.toString().getBytes("UTF-8"));
         plain.write(buf.toString().getBytes("UTF-8"));
      }
      buffer = new ArrayAllocator().allocate(); // N.B clear previous buffer
      encode.close();
      byte[] data = out.toByteArray();
      byte[] plainText = plain.toByteArray();
      //System.out.println(">>"+new String(data, 0, data.length, "UTF-8")+"<<");
      //System.out.println("}}"+new String(plainText, 0, plainText.length,"UTF-8")+"{{");
      DribbleCursor cursor = new DribbleCursor(new StreamCursor(new ByteArrayInputStream(data)), dribble);
View Full Code Here

Examples of org.syncany.chunk.Chunker

  @Test
  public void testStringSerialization() {
    final int CHUNK_SIZE = 512*1024;
   
    Chunker chunker = new FixedChunker(CHUNK_SIZE);
    assertEquals("Other toString() result expected.", "Fixed-"+CHUNK_SIZE+"-"+FixedChunker.DEFAULT_DIGEST_ALG, chunker.toString());
  }
View Full Code Here

Examples of org.syncany.chunk.Chunker

    File inputRandom5MBFile = TestFileUtil.createRandomFileInDirectory(tempDir, TOTAL_FILE_SIZE);
   
    File outputCopyOfRandom5MBFile = TestFileUtil.getRandomFilenameInDirectory(tempDir);
    FileOutputStream outputCopyOfRandom5MBFileOutputStream = new FileOutputStream(outputCopyOfRandom5MBFile);
   
    Chunker chunker = new FixedChunker(EXACT_CHUNK_SIZE, FixedChunker.DEFAULT_DIGEST_ALG);
   
    // Create chunks
    int actualChunkCount = 0;
    Enumeration<Chunk> chunkEnumeration = chunker.createChunks(inputRandom5MBFile);
    Chunk lastChunk = null;
   
    while (chunkEnumeration.hasMoreElements()) {
      actualChunkCount++;
      lastChunk = chunkEnumeration.nextElement();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.