Examples of org.apache.hadoop.fs.FileSystem.copyFromLocalFile()

org.apache.hadoop.fs.FileSystem.copyFromLocalFile()
The src file is on the local disk. Add it to FS at the given dst name and the source is kept intact afterwards

          Path pathDst = new Path(hdfs.getWorkingDirectory() + "/temp",
              pathSrc.getName());


          // LOG.info("WorkingDirectory: " + hdfs.getWorkingDirectory());
          LOG.debug("copyToHDFSFile: " + pathDst);
          hdfs.copyFromLocalFile(pathSrc, pathDst);
          hdfs.deleteOnExit(pathDst);


          finalPath = pathDst.makeQualified(hdfs).toString();
          finalArr[i] = finalPath;
        }

View Full Code Here

                // DistributedCache
                if (shipToCluster) {
                    Path dst = 
                        new Path(FileLocalizer.getTemporaryPath(pigContext).toString());
                    FileSystem fs = dst.getFileSystem(conf);
                    fs.copyFromLocalFile(src, dst);
                    
                    // Construct the dst#srcName uri for DistributedCache
                    URI dstURI = null;
                    try {
                        dstURI = new URI(dst.toString() + "#" + src.getName());

View Full Code Here

      // use jar name if job is not named. 
      if ("".equals(job.getJobName())){
        job.setJobName(new Path(originalJarPath).getName());
      }
      job.setJar(submitJarFile.toString());
      fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile);
      fs.setReplication(submitJarFile, replication);
      fs.setPermission(submitJarFile, new FsPermission(JOB_FILE_PERMISSION));
    } else {
      LOG.warn("No job jar file set.  User classes may not be found. "+
               "See JobConf(Class) or JobConf#setJar(String).");

View Full Code Here

    for (int i = 0; i < tablepaths.length; i++) {
      Path localPath = new Path(tablepaths[i]);
      Path tablePath = new Path(rootDir, names[i]);
      fs.mkdirs(tablePath);
      Path dfsPath = new Path(tablePath, localPath.getName());
      fs.copyFromLocalFile(localPath, dfsPath);
      TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV, option);


      // Add fake table statistic data to tables.
      // It gives more various situations to unit tests.
      TableStats stats = new TableStats();

View Full Code Here

    } else if (action.equalsIgnoreCase("client")) {
      InetAddress addr = InetAddress.getLocalHost();
      String host = addr.getHostName();
      fs.createNewFile(new Path("/accumulo-scale/clients/" + host));
      test.client();
      fs.copyFromLocalFile(new Path("/tmp/scale.out"), new Path("/accumulo-scale/results/" + host));
    } else if (action.equalsIgnoreCase("teardown")) {
      test.teardown();
    }
  }

View Full Code Here

      report.setStatus("Closing done!");
      FileSystem fileSystem = FileSystem.get(_conf);


      report.setStatus("Starting copy to final destination...");
      Path destination = new Path(_conf.get("finalDestination"));
      fileSystem.copyFromLocalFile(new Path(file.getAbsolutePath()), destination);
      report.setStatus("Copy to final destination done!");
      report.setStatus("Deleting tmp files...");
      FileUtil.fullyDelete(file);
      report.setStatus("Deleting tmp files done!");
      t.interrupt();

View Full Code Here

    File sentrySite = File.createTempFile("sentry-site", "xml");
    sentrySite.deleteOnExit();
    File authProviderDir = new File(RESOURCES_DIR, "sentry");
    String authProviderName = "test-authz-provider.ini";
    FileSystem clusterFs = dfsCluster.getFileSystem();
    clusterFs.copyFromLocalFile(false,
      new Path(authProviderDir.toString(), authProviderName),
      new Path(authProviderName));


    // need to write sentry-site at execution time because we don't know
    // the location of sentry.solr.provider.resource beforehand

View Full Code Here

    writer.close();


    Path input = getTestTempFilePath("bayesinput");
    Configuration conf = new Configuration();
    FileSystem fs = input.getFileSystem(conf);
    fs.copyFromLocalFile(new Path(tempInputFile.getAbsolutePath()), input);
  }


  @Test
  public void testSelfTestBayes() throws Exception {
    BayesParameters params = new BayesParameters(1);

View Full Code Here

        lps2.println("ten");
        lps2.println("eleven");
        lps2.close();
        
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(pigServer.getPigContext().getProperties()));
        fs.copyFromLocalFile(new Path(lookupFile1.toString()), new Path("lookup1"));
        fs.copyFromLocalFile(new Path(lookupFile1.toString()), new Path("lookup2"));
        pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "' AS (key:chararray);");
        pigServer.registerQuery("B = FOREACH A GENERATE org.apache.pig.piggybank.evaluation.string.LookupInFiles(key, 'lookup1', 'lookup2');");
        Iterator<Tuple> iter = pigServer.openIterator("B");

View Full Code Here

        lps2.println("eleven");
        lps2.close();
        
        FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(pigServer.getPigContext().getProperties()));
        fs.copyFromLocalFile(new Path(lookupFile1.toString()), new Path("lookup1"));
        fs.copyFromLocalFile(new Path(lookupFile1.toString()), new Path("lookup2"));
        pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "' AS (key:chararray);");
        pigServer.registerQuery("B = FOREACH A GENERATE org.apache.pig.piggybank.evaluation.string.LookupInFiles(key, 'lookup1', 'lookup2');");
        Iterator<Tuple> iter = pigServer.openIterator("B");
        
        int r = (Integer)iter.next().get(0);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.