Examples of HCatURI

org.apache.oozie.util.HCatURI
Utility class to parse HCatalog URI

Examples of org.apache.oozie.util.HCatURI

        if (uris != null) {
            String[] uriList = uris.split(CoordELFunctions.DIR_SEPARATOR);
            // get the partition values list and find minimum
            try {
                // initialize minValue with first partition value
                minPartition = new HCatURI(uriList[0]).getPartitionValue(partitionName);
                if (minPartition == null || minPartition.isEmpty()) {
                    throw new RuntimeException("No value in data-in uri for partition key: " + partitionName);
                }
                for (int i = 1; i < uriList.length; i++) {
                        String value = new HCatURI(uriList[i]).getPartitionValue(partitionName);
                        if(value.compareTo(minPartition) < 0) { //sticking to string comparison since some numerical date
                                                                //values can also contain letters e.g. 20120101T0300Z (UTC)
                            minPartition = value;
                        }
                }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        if (uris != null) {
            String[] uriList = uris.split(CoordELFunctions.DIR_SEPARATOR);
            // get the partition values list and find minimum
            try {
                // initialize minValue with first partition value
                maxPartition = new HCatURI(uriList[0]).getPartitionValue(partitionName);
                if (maxPartition == null || maxPartition.isEmpty()) {
                    throw new RuntimeException("No value in data-in uri for partition key: " + partitionName);
                }
                for(int i = 1; i < uriList.length; i++) {
                        String value = new HCatURI(uriList[i]).getPartitionValue(partitionName);
                        if(value.compareTo(maxPartition) > 0) {
                            maxPartition = value;
                        }
                }
            }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

            for (String uri : uriList) {
                if (filter.length() > 0) {
                    filter.append(" OR ");
                }
                try {
                    filter.append(new HCatURI(uri).toPartitionFilter(type));
                }
                catch (URISyntaxException e) {
                    throw new RuntimeException("Parsing exception for HCatURI " + uri + ". details: " + e);
                }
            }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        else {
            LOG.warn("URI is NULL");
            return null;
        }
        LOG.info("uriTemplate [{0}] ", uriTemplate);
        HCatURI hcatURI;
        try {
            hcatURI = new HCatURI(uriTemplate.toString());
        }
        catch (URISyntaxException e) {
            LOG.info("uriTemplate [{0}]. Reason [{1}]: ", uriTemplate, e);
            throw new RuntimeException("HCat URI can't be parsed " + e);
        }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        String server2 = "hcat-server2.domain.com:5080";
        String db = "mydb";
        String table1 = "mytbl1";
        String table2 = "mytbl2";
        // add partition as missing
        HCatURI dep1 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/dt=20120101;country=us");
        HCatURI dep2 = new HCatURI("hcat://hcat-server1.domain.com:5080/mydb/mytbl1/country=us;dt=20120101");
        HCatURI dep3 = new HCatURI("hcat://hcat-server2.domain.com:5080/mydb/mytbl2/dt=20120102;country=us");
        HCatURI dep4 = new HCatURI("hcat://hcat-server2.domain.com:5080/mydb/mytbl2/dt=20120102;country=us;state=CA");


        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        addMissingDependencyAndRegister(dep1, actionId1, pdms);
        addMissingDependencyAndRegister(dep2, actionId1, pdms);
        addMissingDependencyAndRegister(dep2, actionId2, pdms);
        addMissingDependencyAndRegister(dep2, actionId3, pdms);
        addMissingDependencyAndRegister(dep3, actionId3, pdms);
        addMissingDependencyAndRegister(dep4, actionId4, pdms);
        // Add duplicates. RecoveryService will add duplicates
        addMissingDependencyAndRegister(dep4, actionId4, pdms);
        addMissingDependencyAndRegister(dep4, actionId4, pdms);


        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        JMSAccessorService jmsService = Services.get().get(JMSAccessorService.class);
        JMSConnectionInfo connInfo = hcatService.getJMSConnectionInfo(dep1.getURI());
        assertTrue(hcatService.isRegisteredForNotification(dep1)); //server1,db,table1
        assertTrue(hcatService.isRegisteredForNotification(dep3)); //server2,db,table2
        assertTrue(jmsService.isListeningToTopic(connInfo, dep1.getDb() + "." + dep1.getTable()));
        assertTrue(jmsService.isListeningToTopic(connInfo, dep3.getDb() + "." + dep3.getTable()));


        assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
        assertTrue(pdms.getWaitingActions(dep2).contains(actionId1));
        assertTrue(pdms.getWaitingActions(dep2).contains(actionId2));
        assertTrue(pdms.getWaitingActions(dep2).contains(actionId2));
        assertTrue(pdms.getWaitingActions(dep3).contains(actionId3));
        assertTrue(pdms.getWaitingActions(dep4).contains(actionId4));
        // Should not contain duplicates
        assertEquals(1, pdms.getWaitingActions(dep4).size());


        pdms.removeMissingDependency(dep2, actionId1);
        assertTrue(pdms.getWaitingActions(dep1).contains(actionId1));
        assertEquals(2, pdms.getWaitingActions(dep2).size());
        assertTrue(!pdms.getWaitingActions(dep2).contains(actionId1));
        assertNull(pdms.getAvailableDependencyURIs(actionId1));


        pdms.partitionAvailable(server2, db, table2, getPartitionMap("dt=20120102;country=us;state=NY"));
        assertNull(pdms.getWaitingActions(dep3));
        assertTrue(pdms.getAvailableDependencyURIs(actionId3).contains(dep3.getURI().toString()));


        pdms.partitionAvailable(server2, db, table2, getPartitionMap("dt=20120102;country=us;state=CA"));
        assertNull(pdms.getWaitingActions(dep4));
        assertTrue(pdms.getAvailableDependencyURIs(actionId4).contains(dep4.getURI().toString()));


        pdms.partitionAvailable(server1, db, table1, getPartitionMap("dt=20120101;country=us"));
        assertNull(pdms.getWaitingActions(dep1));
        assertNull(pdms.getWaitingActions(dep2));
        assertTrue(pdms.getAvailableDependencyURIs(actionId2).contains(dep2.getURI().toString()));

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        System.gc();
        MemoryMXBean mb = ManagementFactory.getMemoryMXBean();
        long usedMemBeforeInsert = mb.getHeapMemoryUsage().getUsed();
        long startTime = System.currentTimeMillis();
        for (int i = 0; i < numItems; i++) {
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
            pdms.addMissingDependency(dep, "" + i);
        }
        long usedMemAfterInsert = mb.getHeapMemoryUsage().getUsed();
        long endTime = System.currentTimeMillis();
        LOG.info("Time taken to insert " + numItems + " items is " + (endTime - startTime));

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

    }


    protected void verifyWaitingAction(PartitionDependencyManagerService pdms, int numItems) throws URISyntaxException {
        for (int i = 0; i < numItems; i++) {
            String actionID = "" + i;
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + actionID);
            Collection<String> waitingActions = pdms.getWaitingActions(dep);
            assertNotNull(dep.toURIString() + " is missing in cache", waitingActions);
            assertTrue(dep.toURIString() + " is missing in cache", waitingActions.contains(actionID));
        }
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        // Checks for all missing dependencies
        new CoordPushDependencyCheckXCommand(actionId, true).call();
        checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
        assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency2)));


        // Make first dependency available
        addPartition(db, table, "dt=20120430;country=brazil");
        new CoordPushDependencyCheckXCommand(actionId).call();
        checkCoordAction(actionId, "", CoordinatorAction.Status.READY);
        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
        assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
        new CoordPushDependencyCheckXCommand(actionId, true).call();
        checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
        assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));


        // Timeout is 10 mins. Change action created time to before 12 min to make the action
        // timeout.
        long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
        setCoordActionCreationTime(actionId, timeOutCreationTime);
        new CoordPushDependencyCheckXCommand(actionId).call();
        Thread.sleep(100);
        // Check for timeout status and unregistered missing dependencies
        checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.TIMEDOUT);
        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
        assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));


    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
        new CoordPushDependencyCheckXCommand(actionId, true).call();
        checkCoordAction(actionId, newHCatDependency1, CoordinatorAction.Status.WAITING);
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
        assertTrue(pdms.getWaitingActions(new HCatURI(newHCatDependency1)).contains(actionId));
        assertTrue(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));


        // Timeout is 10 mins. Change action created time to before 12 min to make the action
        // timeout.
        long timeOutCreationTime = System.currentTimeMillis() - (12 * 60 * 1000);
        setCoordActionCreationTime(actionId, timeOutCreationTime);
        // Set some missing dependency. Instead of latest or future just setting a current one for testing as
        // we are only interested in ensuring CoordActionInputCheckXCommand is run
        setMissingDependencies(actionId, newHCatDependency + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency3);
        addPartition(db, table, "dt=20120430;country=brazil");
        checkDependencies(actionId, newHCatDependency + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency3,
                newHCatDependency1);
        new CoordPushDependencyCheckXCommand(actionId).call();
        // Somehow with hive 0.10 it takes 1 second more.
        Thread.sleep(1300);


        checkDependencies(actionId, newHCatDependency3, "");
        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency1)));
        assertFalse(hcatService.isRegisteredForNotification(new HCatURI(newHCatDependency1)));
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.