Package org.apache.gora.examples.generated

Examples of org.apache.gora.examples.generated.WebPage

76
77
78
79
80
81
82
83
84
85
86
87
    filter.setFieldName(WebPage.Field.URL.toString());
    filter.setFilterOp(FilterOp.NOT_EQUALS);
    filter.setFilterIfMissing(true);
    filter.getOperands().add(new Utf8("example.org"));
   
    WebPage page = WebPage.newBuilder().build();
    page.setUrl(new Utf8("example.org"));
    assertTrue(filter.filter("irrelevant", page));
   
    page.setUrl(new Utf8("something.else"));
    assertFalse(filter.filter("irrelevant", page));
  }
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
  }
 
  public static void createWebPageData(DataStore<String, WebPage> dataStore)
      throws IOException {
    try{
      WebPage page;
      log.info("creating web page data");
     
      for(int i=0; i<URLS.length; i++) {
        page = WebPage.newBuilder().build();
        page.setUrl(new Utf8(URLS[i]));
        page.setParsedContent(new ArrayList<CharSequence>());
        if (CONTENTS[i]!=null){
          page.setContent(ByteBuffer.wrap(CONTENTS[i].getBytes()));
          for(String token : CONTENTS[i].split(" ")) {
            page.getParsedContent().add(new Utf8(token)); 
          }
        }
        for(int j=0; j<LINKS[i].length; j++) {
          page.getOutlinks().put(new Utf8(URLS[LINKS[i][j]]), new Utf8(ANCHORS[i][j]));
        }
       
        Metadata metadata = Metadata.newBuilder().build();
        metadata.setVersion(1);
        metadata.getData().put(new Utf8("metakey"), new Utf8("metavalue"));
        page.setMetadata(metadata);

        dataStore.put(URLS[i], page);
      }
      dataStore.flush();
      log.info("finished creating web page data");
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
    Result<String, WebPage> result = store.newQuery().execute();

    int i = 0;
    while (result.next()) {
      WebPage page = result.get();
      TestIOUtils.testSerializeDeserialize(page);
      i++;
    }
    assertEquals(WebPageDataCreator.URLS.length, i);
  }
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
   * what we get 'before' and 'after' (de)serialization processes.
   * @throws Exception
   */
  @Test
  public void testSerdeMultipleWebPages() throws Exception {
    WebPage page1 = WebPage.newBuilder().build();
    WebPage page2 = WebPage.newBuilder().build();
    WebPage page3 = WebPage.newBuilder().build();

    page1.setUrl(new Utf8("foo"));
    page2.setUrl(new Utf8("baz"));
    page3.setUrl(new Utf8("bar"));
    page1.setParsedContent(new ArrayList<CharSequence>());
    page1.getParsedContent().add(new Utf8("coo"));
    page2.setOutlinks(new HashMap<CharSequence, CharSequence>());
    page2.getOutlinks().put(new Utf8("a"), new Utf8("b"));

View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
    // Since "content" is an optional field, we are forced to reopen the DataStore
    // to retrieve the union correctly
   
    // Test writing+reading a null value. FIELD in HBASE MUST become DELETED
    WebPage page = webPageStore.get("com.example/http") ;
    page.setContent(null) ;
    webPageStore.put("com.example/http", page) ;
    webPageStore.close() ;
    webPageStore = testDriver.createDataStore(String.class, WebPage.class);
    page = webPageStore.get("com.example/http") ;
    assertNull(page.getContent()) ;
    // Check directly with HBase
    table = new HTable(conf,"WebPage");
    get = new Get(Bytes.toBytes("com.example/http"));
    result = table.get(get);
    actualBytes = result.getValue(Bytes.toBytes("content"), null);
    assertNull(actualBytes);
    table.close();
   
    // Test writing+reading an empty bytes field. FIELD in HBASE MUST
    // become EMPTY (byte[0])
    page = webPageStore.get("com.example/http") ;
    page.setContent(ByteBuffer.wrap("".getBytes())) ;
    webPageStore.put("com.example/http", page) ;
    webPageStore.close() ;
    webPageStore = testDriver.createDataStore(String.class, WebPage.class);
    page = webPageStore.get("com.example/http") ;
    assertTrue(Arrays.equals("".getBytes(),page.getContent().array())) ;
    // Check directly with HBase
    table = new HTable(conf,"WebPage");
    get = new Get(Bytes.toBytes("com.example/http"));
    result = table.get(get);
    actualBytes = result.getValue(Bytes.toBytes("content"), null);
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
   * the value is written in raw format
   * @throws Exception
   */
  @Test
  public void assertTopLevelUnions() throws Exception {
    WebPage page = webPageStore.newPersistent();
   
    // Write webpage data
    page.setUrl((CharSequence) new Utf8("http://example.com"));
    byte[] contentBytes = "example content in example.com".getBytes();
    ByteBuffer buff = ByteBuffer.wrap(contentBytes);
    page.setContent(buff);
    webPageStore.put("com.example/http", page);
    webPageStore.flush() ;
   
    // Read directly from HBase
    HTable table = new HTable(conf,"WebPage");
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
   * with Avro.
   * @throws Exception
   */
  @Test
  public void assertTopLevelUnionsNull() throws Exception {
    WebPage page = webPageStore.newPersistent();
   
    // Write webpage data
    page.setUrl((CharSequence) new Utf8("http://example.com"));
    page.setContent(null);     // This won't change internal field status to dirty, so
    page.setDirty("content") ; // need to change it manually
    webPageStore.put("com.example/http", page);
    webPageStore.flush() ;
   
    // Read directly from HBase
    HTable table = new HTable(conf,"WebPage");
View Full Code Here
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
    String header = "header";
    String[] headers = { "firstHeader", "secondHeader", "thirdHeader",
        "fourthHeader", "fifthHeader", "sixthHeader" };

    for (int i = 0; i < urls.length; i++) {
      WebPage webPage = WebPage.newBuilder().build();
      webPage.setUrl(new Utf8(urls[i]));
      webPage.setHeaders(new HashMap<CharSequence, CharSequence>());
      for (int j = 0; j < headers.length; j++) {
        webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j]));
      }
      dataStore.put(webPage.getUrl().toString(), webPage);
    }

    dataStore.flush();

    // nullable map field removal test
    for (int i = 0; i < urls.length; i++) {
      WebPage webPage = dataStore.get(urls[i]);
      webPage.setHeaders(null);
      dataStore.put(webPage.getUrl().toString(), webPage);
    }

    dataStore.flush();

    for (int i = 0; i < urls.length; i++) {
      WebPage webPage = dataStore.get(urls[i]);
      assertNull(webPage.getHeaders());
    }
  }
View Full Code Here
745
746
747
748
749
750
751
752
753
  private static void testGetWebPage(DataStore<String, WebPage> store, String[] fields)
    throws IOException, Exception {
    createWebPageData(store);

    for(int i=0; i<URLS.length; i++) {
      WebPage page = store.get(URLS[i], fields);
      assertWebPage(page, i);
    }
  }
View Full Code Here
770
771
772
773
774
775
776
777
778
779
      Query<String, WebPage> query = store.newQuery();
      query.setFields(fields);
      query.setKey(URLS[i]);
      Result<String, WebPage> result = query.execute();
      assertTrue(result.next());
      WebPage page = result.get();
      assertWebPage(page, i);
      assertFalse(result.next());
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.gora.examples.generated.WebPage

  • org.apache.gora.avro.store.TestAvroStore
  • org.apache.gora.avro.TestPersistentDatumReader
  • org.apache.gora.examples.WebPageDataCreator
  • org.apache.gora.filter.TestMapFieldValueFilter
  • org.apache.gora.filter.TestSingleFieldValueFilter
  • org.apache.gora.hbase.store.TestHBaseStore
  • org.apache.gora.mapreduce.TestPersistentSerialization
  • org.apache.gora.memory.store.MemStoreTest
  • org.apache.gora.persistency.impl.StateManagerImpl
  • org.apache.gora.persistency.impl.TestPersistentBase
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.