Examples of HCatURI

org.apache.oozie.util.HCatURI
Utility class to parse HCatalog URI

Examples of org.apache.oozie.util.HCatURI

                Map<String, Collection<String>> partitionMap = actionPartitionMap.get(coordActionId);
                if (partitionMap != null) {
                    Iterator<String> tableItr = partitionMap.keySet().iterator();
                    while (tableItr.hasNext()) {
                        String tableKey = tableItr.next();
                        HCatURI hcatUri = null;
                        Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
                        if (partKeyPatterns != null) {
                            synchronized (partKeyPatterns) {
                                Collection<String> partKeys = partitionMap.get(tableKey);
                                if (partKeys != null) {

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

    }


    @Override
    public void registerForNotification(URI uri, Configuration conf, String user, String actionID)
            throws URIHandlerException {
        HCatURI hcatURI;
        try {
            hcatURI = new HCatURI(uri);
        }
        catch (URISyntaxException e) {
            throw new URIHandlerException(ErrorCode.E0906, uri, e);
        }
        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        if (!hcatService.isRegisteredForNotification(hcatURI)) {
            HCatClient client = getHCatClient(uri, conf, user);
            try {
                String topic = client.getMessageBusTopicName(hcatURI.getDb(), hcatURI.getTable());
                if (topic == null) {
                    return;
                }
                hcatService.registerForNotification(hcatURI, topic, new HCatMessageHandler(uri.getAuthority()));
            }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        pdmService.addMissingDependency(hcatURI, actionID);
    }


    @Override
    public boolean unregisterFromNotification(URI uri, String actionID) {
        HCatURI hcatURI;
        try {
            hcatURI = new HCatURI(uri);
        }
        catch (URISyntaxException e) {
            throw new RuntimeException(e); // Unexpected at this point
        }
        PartitionDependencyManagerService pdmService = Services.get().get(PartitionDependencyManagerService.class);

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

    }


    @Override
    public void validate(String uri) throws URIHandlerException {
        try {
            new HCatURI(uri); // will fail if uri syntax is incorrect
        }
        catch (URISyntaxException e) {
            throw new URIHandlerException(ErrorCode.E0906, uri, e);
        }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        return metastoreURI;
    }


    private boolean exists(URI uri, HCatClient client, boolean closeClient) throws HCatAccessorException {
        try {
            HCatURI hcatURI = new HCatURI(uri.toString());
            List<HCatPartition> partitions = client.getPartitions(hcatURI.getDb(), hcatURI.getTable(),
                    hcatURI.getPartitionMap());
            return (partitions != null && !partitions.isEmpty());
        }
        catch (ConnectionFailureException e) {
            throw new HCatAccessorException(ErrorCode.E1501, e);
        }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

                        continue;
                    }
                    Collection<WaitingAction> waitingActions = ((WaitingActions) element.getObjectValue())
                            .getWaitingActions();
                    Iterator<WaitingAction> wactionItr = waitingActions.iterator();
                    HCatURI hcatURI = null;
                    while(wactionItr.hasNext()) {
                        WaitingAction waction = wactionItr.next();
                        if(staleActions.contains(waction.getActionID())) {
                            try {
                                hcatURI = new HCatURI(waction.getDependencyURI());
                                wactionItr.remove();
                            }
                            catch (URISyntaxException e) {
                                continue;
                            }
                        }
                    }
                    if (waitingActions.isEmpty() && hcatURI != null) {
                        missingCache.remove(key);
                        // Decrement partition key pattern count if the cache entry is removed
                        SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
                        String partKeys = sortedPKV.getPartKeys();
                        String tableKey = hcatURI.getServer() + TABLE_DELIMITER + hcatURI.getDb() + TABLE_DELIMITER
                                + hcatURI.getTable();
                        String hcatURIStr = hcatURI.toURIString();
                        decrementPartKeyPatternCount(tableKey, partKeys, hcatURIStr);
                    }
                }
            }
        }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        part2 = "dt=20120102;country=us";
        part3 = "dt=20120103;country=us";
        String newHCatDependency1 = "hcat://" + server + "/" + db + "/" + table1 + "/" + part1;
        String newHCatDependency2 = "hcat://" + server + "/" + db + "/" + table1 + "/" + part2;
        String newHCatDependency3 = "hcat://" + server + "/" + db + "/" + table2 + "/" + part3;
        HCatURI dep1 = new HCatURI(newHCatDependency1);
        HCatURI dep2 = new HCatURI(newHCatDependency2);
        HCatURI dep3 = new HCatURI(newHCatDependency3);
        // create db, table and partitions
        populateTable();


        String actionId1 = addInitRecords(newHCatDependency1);
        String actionId2 = addInitRecords(newHCatDependency2);
        String actionId3 = addInitRecords(newHCatDependency3);


        // Assume dependency cache on dummy server with missing push dependencies registered
        PartitionDependencyManagerService dummyPdms = new PartitionDependencyManagerService();
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        dummyPdms.init(Services.get());
        dummyPdms.addMissingDependency(dep1, actionId1);
        dummyPdms.addMissingDependency(dep2, actionId2);
        dummyPdms.addMissingDependency(dep3, actionId3);


        Collection<String> waitingActions = (Collection<String>)dummyPdms.getWaitingActions(dep1);
        assertEquals(1, waitingActions.size());
        waitingActions = (Collection<String>)dummyPdms.getWaitingActions(dep2);
        assertEquals(1, waitingActions.size());
        waitingActions = (Collection<String>)dummyPdms.getWaitingActions(dep3);
        assertEquals(1, waitingActions.size());


        //Dependency cache on living server doesn't have these partitions registered at this point
        waitingActions = (Collection<String>)pdms.getWaitingActions(dep1);
        assertNull(waitingActions);
        waitingActions = (Collection<String>)pdms.getWaitingActions(dep2);
        assertNull(waitingActions);
        waitingActions = (Collection<String>)pdms.getWaitingActions(dep3);
        assertNull(waitingActions);


        //Assume dummy server is down, and recovery service on living server pick up these jobs
        dummyPdms.destroy();
        Runnable recoveryRunnable = new RecoveryRunnable(60, 0, 60);
        recoveryRunnable.run();
        waitFor(30 * 1000, new Predicate() {
            public boolean evaluate() throws Exception {
                Collection<String> waitingActions;
                PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
                HCatURI dep1 = new HCatURI("hcat://"+ server + "/" + db + "/" + table1 + "/" + part1);
                HCatURI dep2 = new HCatURI("hcat://"+ server + "/" + db + "/" + table1 + "/" + part2);
                HCatURI dep3 = new HCatURI("hcat://"+ server + "/" + db + "/" + table2 + "/" + part3);
                waitingActions = pdms.getWaitingActions(dep1);
                if(waitingActions == null) {
                    return false;
                }
                waitingActions = pdms.getWaitingActions(dep2);

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        String actionId1 = "1234465451";
        String actionId2 = "1234465452";
        String actionId3 = "1234465453";


        // add partitions as missing
        HCatURI dep1 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/dt=20120101;country=us");
        HCatURI dep2 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/country=us;dt=20120101");
        HCatURI dep3 = new HCatURI("hcat://hcat-server2.domain.com:5080/mydb/mytbl2/dt=20120102;country=us");


        // actionId1-->(dep1,2), actionId2-->(dep2), actionId3-->(dep2,3)
        addMissingDependencyAndRegister(dep1, actionId1, pdms);
        addMissingDependencyAndRegister(dep2, actionId1, pdms);
        addMissingDependencyAndRegister(dep2, actionId2, pdms);

View Full Code Here

Examples of org.apache.oozie.util.HCatURI


    @Override
    public boolean delete(URI uri, Configuration conf) throws LauncherException {
        HCatClient client = getHCatClient(uri, conf);
        try {
            HCatURI hcatURI = new HCatURI(uri.toString());
            client.dropPartitions(hcatURI.getDb(), hcatURI.getTable(), hcatURI.getPartitionMap(), true);
            System.out.println("Dropped partitions for " + uri);
            return true;
        }
        catch (ConnectionFailureException e) {
            throw new LauncherException("Error trying to drop " + uri, e);

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        setupServices("testevictionontimetoidle");
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        int numItems = 50;
        long startTime = System.currentTimeMillis();
        for (int i = 0; i < numItems; i++) {
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
            pdms.addMissingDependency(dep, "" + i);
        }
        verifyWaitingAction(pdms, numItems);
        LOG.info("Time taken to insert and retrive " + numItems + " items is "
                + (System.currentTimeMillis() - startTime));
        // timeToIdleSeconds is 1
        Thread.sleep(1100);
        for (int i = 0; i < numItems; i++) {
            assertNull(pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)));
        }
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.