Examples of DatasetRepository

com.cloudera.cdk.data.DatasetRepository

A logical repository (storage system) of {@link Dataset}s.

Implementations of {@code DatasetRepository} are storage systems that containzero or more {@link Dataset}s. A repository acts as a factory, as well as a registry, of datasets. Users can {@link #create(String,DatasetDescriptor)} anew {@link Dataset} with a name and schema, or retrieve a handle to anexisting dataset, by name, by way of the {@link #load(String)} method. Whilenot expressly forbidden, most repositories are expected to support only a single concrete {@link Dataset} implementation.

No guarantees are made as to the durability, reliability, or availability of the underlying storage. That is, a {@code DatasetRepository} could be ondisk, in memory, or some combination. See the implementation class for details about the guarantees it provides.

Implementations of {@link DatasetRepository} are immutable.
@see Dataset @see DatasetDescriptor
org.kitesdk.data.DatasetRepository
org.kitesdk.data.spi.DatasetRepository

A logical repository (storage system) of {@link Dataset}s.

Implementations of {@code DatasetRepository} are storage systems that containzero or more {@link Dataset}s. A repository acts as a factory, as well as a registry, of datasets. You can {@link #create(String,String,DatasetDescriptor)} anew {@link Dataset} with a name and schema, or retrieve a handle to anexisting dataset, by name, by way of the {@link #load(String,String)} method. Whilenot expressly forbidden, most repositories are expected to support only a single concrete {@link Dataset} implementation.

No guarantees are made as to the durability, reliability, or availability of the underlying storage. That is, a {@code DatasetRepository} could be ondisk, in memory, or some combination. See the implementation class for details about the guarantees it provides.

Implementations of {@link DatasetRepository} are immutable.
@see Dataset @see DatasetDescriptor
org.openrdf.repository.dataset.DatasetRepository

Examples of org.kitesdk.data.spi.DatasetRepository

    super(console);
  }


  @Override
  public int run() throws IOException {
    DatasetRepository repo = getDatasetRepository();


    if (datasets == null || datasets.isEmpty()) {
      throw new IllegalArgumentException("No dataset names were specified.");
    }


    for (String datasetUriOrName : datasets) {
      if (isDataUri(datasets.get(0))) {
        Datasets.delete(datasetUriOrName);
      } else {
        repo.delete(namespace, datasetUriOrName);
      }
      console.debug("Deleted dataset {}", datasetUriOrName);
    }


    return 0;

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

    } else {
      LOG.warn(
          "kite.datasetName is deprecated, instead use kite.uri=<dataset-uri>");
      Preconditions.checkArgument(datasetName != null,
          "kite.datasetName is required if kite.uri is not used");
      DatasetRepository repo = getDatasetRepository();
      repo.delete(datasetNamespace, datasetName);
    }
  }

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

    } else {
      LOG.warn(
          "kite.datasetName is deprecated, instead use kite.uri=<dataset-uri>");
      Preconditions.checkArgument(datasetName != null,
          "kite.datasetName is required if kite.uri is not used");
      DatasetRepository repo = getDatasetRepository();
      descriptor = repo.load(datasetNamespace, datasetName).getDescriptor();
    }


    DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(
        descriptor);
    configureSchema(descriptorBuilder, avroSchemaFile, avroSchemaReflectClass);


    if (columnDescriptorFile != null) {
      File columnDescriptor = new File(columnDescriptorFile);
      try {
        if (columnDescriptor.exists()) {
          descriptorBuilder.columnMapping(columnDescriptor);
        } else {
          descriptorBuilder.columnMapping(Resources.getResource(
              columnDescriptorFile).openStream());
        }
      } catch (IOException e) {
        throw new MojoExecutionException("Problem while reading file "
            + columnDescriptorFile, e);
      }
    }


    if (uri != null) {
      Datasets.update(uri, descriptorBuilder.build());
    } else {
      // datasetName is checked above
      DatasetRepository repo = getDatasetRepository();
      repo.update(datasetNamespace, datasetName, descriptorBuilder.build());
    }
  }

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

    } else {
      LOG.warn(
          "kite.datasetName is deprecated, instead use kite.uri=<dataset-uri>");
      Preconditions.checkArgument(datasetName != null,
          "kite.datasetName is required if kite.uri is not used");
      DatasetRepository repo = getDatasetRepository();
      repo.create(datasetNamespace, datasetName, descriptorBuilder.build());
    }
  }

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

  @Test
  public void testExternalURI() {
    URI hdfsUri = getDFS().getUri();
    URI repoUri = URI.create("repo:hive:/tmp/hive-repo?hdfs:host=" +
        hdfsUri.getHost() + "&hdfs:port=" + hdfsUri.getPort());
    DatasetRepository repo = DatasetRepositories.repositoryFor(repoUri);


    Assert.assertNotNull("Received a repository", repo);
    org.junit.Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());


    // verify location
    DatasetDescriptor created = repo.create("tmp", "test",
        new DatasetDescriptor.Builder()
        .schemaLiteral("\"string\"")
        .build()).getDescriptor();
    Assert.assertEquals("Location should be in HDFS",
        "hdfs", created.getLocation().getScheme());

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

  @Test
  public void testExternalURIWithHostAndPort() {
    URI hdfsUri = getDFS().getUri();
    URI repoUri = URI.create("repo:hive://meta-host:1234/tmp/data?hdfs:host=" +
        hdfsUri.getHost() + "&hdfs:port=" + hdfsUri.getPort());
    DatasetRepository repo = DatasetRepositories.repositoryFor(repoUri);


    Assert.assertNotNull("Received a repository", repo);
    org.junit.Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());
  }

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

  @Test
  public void testExternalURIWithRootPath() {
    URI hdfsUri = getDFS().getUri();
    URI repoUri = URI.create("repo:hive:/?hdfs:host=" + hdfsUri.getHost() +
        "&hdfs:port=" + hdfsUri.getPort());
    DatasetRepository repo = DatasetRepositories.repositoryFor(repoUri);
    Assert.assertNotNull("Received a repository", repo);
    Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    MetadataProvider provider = ((HiveExternalDatasetRepository) repo)
        .getMetadataProvider();
    Assert.assertTrue("Repo should be using a HCatalogExternalMetadataProvider",
        provider instanceof HiveExternalMetadataProvider);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());
  }

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

  @Test
  public void testExternalURIWithHostAndPortAndRootPath() {
    URI hdfsUri = getDFS().getUri();
    URI repoUri = URI.create("repo:hive://meta-host:1234/?hdfs:host=" +
        hdfsUri.getHost() + "&hdfs:port=" + hdfsUri.getPort());
    DatasetRepository repo = DatasetRepositories.repositoryFor(repoUri);
    Assert.assertNotNull("Received a repository", repo);
    Assert.assertTrue("Repo should be a HCatalogExternalDatasetRepository",
        repo instanceof HiveExternalDatasetRepository);
    MetadataProvider provider = ((HiveExternalDatasetRepository) repo)
        .getMetadataProvider();
    Assert.assertTrue("Repo should be using a HCatalogManagedMetadataProvider",
        provider instanceof HiveExternalMetadataProvider);
    Assert.assertEquals("Repository URI", repoUri, repo.getUri());
  }

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

    test.hcatalog = false;
    test.rootDirectory = "/tmp/datasets";
    test.hadoopConfiguration = new Properties();
    test.hadoopConfiguration.setProperty("fs.default.name", fsUri.toString());


    DatasetRepository repo = test.getDatasetRepository();
    Assert.assertNotNull("Should create repo successfully", repo);
    Assert.assertEquals("Should be a HDFS repo",
        "repo:hdfs://" + fsUri.getAuthority() + "/tmp/datasets",
        repo.getUri().toString());
  }

View Full Code Here

Examples of org.kitesdk.data.spi.DatasetRepository

    test.hcatalog = false;
    test.rootDirectory = "/tmp/datasets";
    test.hadoopConfiguration = new Properties();
    test.hadoopConfiguration.setProperty("fs.defaultFS", fsUri.toString());


    DatasetRepository repo = test.getDatasetRepository();
    Assert.assertNotNull("Should create repo successfully", repo);
    Assert.assertEquals("Should be a HDFS repo",
        "repo:hdfs://" + fsUri.getAuthority() + "/tmp/datasets",
        repo.getUri().toString());
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.