Examples of FSAnalyzer

com.cloudera.recordbreaker.analyzer.FSAnalyzer
FSAnalyzer crawls a filesystem and figures out its schema contents. We place the results of that analysis into a store for future analytics @author "Michael Cafarella"

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

public class JoinPage extends WebPage {
  final class JoinPageDisplay extends WebMarkupContainer {
    public JoinPageDisplay(String name, String fid1Str, String fid2Str) {
      super(name);
      FishEye fe = FishEye.getInstance();
      FSAnalyzer fsa = fe.getAnalyzer();
    
      if (fe.hasFSAndCrawl()) {
        if (fid1Str != null && fid2Str != null) {
          try {
            long fid1 = Long.parseLong(fid1Str);
            long fid2 = Long.parseLong(fid2Str);
            FileSummary fs1 = new FileSummary(fsa, fid1);
            FileSummary fs2 = new FileSummary(fsa, fid2);
            FileSummaryData fsd1 = fsa.getFileSummaryData(fid1);
            FileSummaryData fsd2 = fsa.getFileSummaryData(fid2);
            DataDescriptor dd1 = fsd1.getDataDescriptor();
            DataDescriptor dd2 = fsd2.getDataDescriptor();
            List<SchemaDescriptor> sds1 = dd1.getSchemaDescriptor();
            List<SchemaDescriptor> sds2 = dd2.getSchemaDescriptor();                        
            SchemaDescriptor sd1 = sds1.get(0);

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

      add(new AjaxButton("submitjoinquery") {
          protected void onSubmit(final AjaxRequestTarget target, final Form form) {
            FishEye fe = FishEye.getInstance();
            DataQuery dq = DataQuery.getInstance();
            ValueMap vals = (ValueMap) form.getModelObject();
            FSAnalyzer fsa = fe.getAnalyzer();
            FileSummaryData fsd1 = fsa.getFileSummaryData(finalFid1);
            String path1 = fsd1.path + fsd1.fname;
            DataDescriptor dd1 = fsd1.getDataDescriptor();
            FileSummaryData fsd2 = fsa.getFileSummaryData(finalFid2);
            String path2 = fsd2.path + fsd2.fname;
            DataDescriptor dd2 = fsd2.getDataDescriptor();
            
            List<List<String>> results = null;
            if (dq != null) {

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

  }


  public void onConfigure() {
    FishEye fe = FishEye.getInstance();
    AccessController accessCtrl = fe.getAccessController();
    FSAnalyzer fsAnalyzer = fe.getAnalyzer();
    
    FileSummary fileSummary = null;
    if (targetFid >= 0) {
      fileSummary = new FileSummary(fsAnalyzer, targetFid);
    } else if (targetName != null) {
      fileSummary = fsAnalyzer.getSingleFileSummary(targetName);
    }


    if (fileSummary != null) {
      setVisibilityAllowed(fe.hasFSAndCrawl() && !accessCtrl.hasReadAccess(fileSummary));
    } else {

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

            this.fid = Long.parseLong(fidStr);
            final FileSummary fs = new FileSummary(fe.getAnalyzer(), fid);
            final long fsFid = fid;
            final String fsPath = fs.getPath().toString();
            
            FSAnalyzer fsa = fe.getAnalyzer();
            FileSummaryData fsd = fsa.getFileSummaryData(fid);
            DataDescriptor dd = fsd.getDataDescriptor();
            List<TypeGuessSummary> tgses = fs.getTypeGuesses();
            
            add(new Label("filetitle", fs.getFname()));
            add(new ExternalLink("filesubtitlelink", urlFor(FilesPage.class, new PageParameters("targetdir=" + fs.getPath().getParent().toString())).toString(), fs.getPath().getParent().toString()));

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

            //loginErrorMsgDisplay.setVisibilityAllowed(false);            
            //target.add(loginErrorMsgDisplay);
            FishEye fe = FishEye.getInstance();
            DataQuery dq = DataQuery.getInstance();
            ValueMap vals = (ValueMap) form.getModelObject();
            FSAnalyzer fsa = fe.getAnalyzer();
            FileSummaryData fsd = fsa.getFileSummaryData(finalFid);
            String path = fsd.path + fsd.fname;
            DataDescriptor dd = fsd.getDataDescriptor();
            List<List<String>> results = null;
            if (dq != null) {
              // Open a new window for the query results

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

        if (fidStr != null) {
          try {
            fid = Long.parseLong(fidStr);
            FileSummary fs = new FileSummary(fe.getAnalyzer(), fid);
            DataQuery dq = DataQuery.getInstance();
            FSAnalyzer fsa = fe.getAnalyzer();
            FileSummaryData fsd = fsa.getFileSummaryData(fid);
            DataDescriptor dd = fsd.getDataDescriptor();


            if (dq != null) {
              queryResults = dq.query(dd, null, projClauseStr, selClauseStr);
            }
          } catch (Exception ex) {
            ex.printStackTrace();
          }
        }
        ///
        // Multi table query!
        ///
        if (fidStr1 != null) {
          try {
            fid1 = Long.parseLong(fidStr1);
            FileSummary fs1 = new FileSummary(fe.getAnalyzer(), fid1);
            fid2 = Long.parseLong(fidStr2);
            FileSummary fs2 = new FileSummary(fe.getAnalyzer(), fid2);
            DataQuery dq = DataQuery.getInstance();
            FSAnalyzer fsa = fe.getAnalyzer();
            FileSummaryData fsd1 = fsa.getFileSummaryData(fid1);
            DataDescriptor dd1 = fsd1.getDataDescriptor();
            FileSummaryData fsd2 = fsa.getFileSummaryData(fid2);
            DataDescriptor dd2 = fsd2.getDataDescriptor();


            if (dq != null) {
              queryResults = dq.query(dd1, dd2, projClauseStr, selClauseStr);
            }

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

    this.fisheyeDir = fisheyeDir;
    this.fisheyePort = port;


    File fisheyeStore = new File(fisheyeDir, FISHEYE_METADATA_STORE);
    File fisheyeSchemas = new File(fisheyeDir, FISHEYE_SCHEMA_REPO);
    this.analyzer = new FSAnalyzer(fisheyeStore, fisheyeSchemas);
    this.crawler = new FSCrawler(analyzer);
    this.accessCtrl = new AccessController();
    FishEye.fisheyeInstance = this;
    restartIncompleteCrawl();
  }

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

      }
      if (! targetDir.mkdirs()) {
        throw new IOException("Cannot create: " + targetDir);
      }
      File metadataStore = new File(targetDir, "metadata");
      FSAnalyzer fsa = new FSAnalyzer(metadataStore, schemaDbDir);
      fsa.close();
    }
  }

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

  
  public FileContentsTable(long fid) {
    super("filecontentstable");
    this.fid = fid;
    FishEye fe = FishEye.getInstance();
    FSAnalyzer fsa = fe.getAnalyzer();
    FileSummaryData fsd = fsa.getFileSummaryData(fid);
    String path = fsd.path + fsd.fname;    
    DataDescriptor dd = fsd.getDataDescriptor();
    final boolean querySupported = dd.isHiveSupported() && fe.isQueryServerAvailable(false);    
    List<SchemaDescriptor> sds = dd.getSchemaDescriptor();

View Full Code Here

Examples of com.cloudera.recordbreaker.analyzer.FSAnalyzer

    if (fid < 0) {
      setVisibilityAllowed(false);
    } else {
      FishEye fe = FishEye.getInstance();
      AccessController accessCtrl = fe.getAccessController();    
      FSAnalyzer fsAnalyzer = fe.getAnalyzer();
      FileSummary fileSummary = new FileSummary(fsAnalyzer, fid);
      try {
        setVisibilityAllowed(fe.hasFSAndCrawl() && accessCtrl.hasReadAccess(fileSummary) && fileSummary.isDir() && fileSummary.getDataDescriptor().getSchemaDescriptor().size() > 0);
      } catch (IOException iex) {
        setVisibilityAllowed(false);

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.