Examples of HCatURI

org.apache.oozie.util.HCatURI
Utility class to parse HCatalog URI

Examples of org.apache.oozie.util.HCatURI

        setupServices("testevictionontimetolive");
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        int numItems = 50;
        long startTime = System.currentTimeMillis();
        for (int i = 0; i < numItems; i++) {
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
            pdms.addMissingDependency(dep, "" + i);
        }
        verifyWaitingAction(pdms, numItems);
        LOG.info("Time taken to insert and retrive " + numItems + " items is "
                + (System.currentTimeMillis() - startTime));
        // timeToLiveSeconds is 1
        Thread.sleep(1100);
        for (int i = 0; i < numItems; i++) {
            assertNull(pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)));
        }
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

    public void testMaxElementsInMemory() throws Exception {
        setupServices("testmaxelementsinmemory"); // maxElementsInMemory="500" overflowToDisk="false"
        PartitionDependencyManagerService pdms = Services.get().get(PartitionDependencyManagerService.class);
        int numItems = 1000;
        for (int i = 0; i < numItems; i++) {
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + i);
            pdms.addMissingDependency(dep, "" + i);
        }
        // First 500 should have been evicted. But it is LRU and the last 350 removed is between 250 and 750.
        for (int i = 0; i < 150; i++) {
            assertNull(pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)));
        }
        int evicted = 0;
        for (int i = 150; i < 750; i++) {
            if (pdms.getWaitingActions(new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + "" + i)) == null) {
                evicted++;
            }
        }
        assertEquals(350, evicted);
        for (int i = 750; i < 1000; i++) {
            String actionID = "" + i;
            HCatURI dep = new HCatURI("hcat://hcat.server.com:5080/mydb/mytbl/id=" + actionID);
            Collection<String> waitingActions = pdms.getWaitingActions(dep);
            assertNotNull(dep.toURIString() + " is missing in cache", waitingActions);
            assertTrue(dep.toURIString() + " is missing in cache", waitingActions.contains(actionID));
        }
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        populateTable(db, table);
        String actionId = addInitRecords(newHCatDependency);
        CoordinatorAction ca = checkCoordActionDependencies(actionId, newHCatDependency);
        assertEquals(CoordinatorAction.Status.WAITING, ca.getStatus());
        // Register the missing dependencies to PDMS assuming CoordPushDependencyCheckCommand did this.
        pdms.addMissingDependency(new HCatURI(newHCatDependency1), actionId);
        pdms.addMissingDependency(new HCatURI(newHCatDependency2), actionId);


        sleep(2000);
        Runnable recoveryRunnable = new RecoveryRunnable(0, 1, 1);
        recoveryRunnable.run();
        sleep(2000);


        // Recovery service should have discovered newHCatDependency2 and JMS Connection should exist
        // and newHCatDependency1 should be in PDMS waiting list
        assertTrue(jmsService.isListeningToTopic(hcatService.getJMSConnectionInfo(new URI(newHCatDependency2)), "hcat."
                + db + "." + table));
        checkCoordActionDependencies(actionId, newHCatDependency1);


        assertNull(pdms.getWaitingActions(new HCatURI(newHCatDependency2)));
        Collection<String> waitingActions = pdms.getWaitingActions(new HCatURI(newHCatDependency1));
        assertEquals(1, waitingActions.size());
        assertTrue(waitingActions.contains(actionId));
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

     *
     * @param dataInName
     * @return DB name
     */
    public static String ph3_coord_databaseIn(String dataName) {
        HCatURI hcatURI = getURIFromResolved(dataName, EventType.input);
        if (hcatURI != null) {
            return hcatURI.getDb();
        }
        else {
            return "";
        }
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

     *
     * @param dataOutName
     * @return DB name
     */
    public static String ph3_coord_databaseOut(String dataName) {
        HCatURI hcatURI = getURIFromResolved(dataName, EventType.output);
        if (hcatURI != null) {
            return hcatURI.getDb();
        }
        else {
            return "";
        }
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

     *
     * @param dataInName
     * @return Table name
     */
    public static String ph3_coord_tableIn(String dataName) {
        HCatURI hcatURI = getURIFromResolved(dataName, EventType.input);
        if (hcatURI != null) {
            return hcatURI.getTable();
        }
        else {
            return "";
        }
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

     *
     * @param dataOutName
     * @return Table name
     */
    public static String ph3_coord_tableOut(String dataName) {
        HCatURI hcatURI = getURIFromResolved(dataName, EventType.output);
        if (hcatURI != null) {
            return hcatURI.getTable();
        }
        else {
            return "";
        }
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
        if (unresolved != null && unresolved.booleanValue() == true) {
            return "${coord:dataOutPartitionValue('" + dataOutName + "', '" + partitionName + "')}";
        }
        try {
            HCatURI hcatUri = new HCatURI(uri);
            return hcatUri.getPartitionValue(partitionName);
        }
        catch(URISyntaxException urie) {
            XLog.getLog(HCatELFunctions.class).warn("Exception with uriTemplate [{0}]. Reason [{1}]: ", uri, urie);
            throw new RuntimeException("HCat URI can't be parsed " + urie);
        }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

        Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
        if (unresolved != null && unresolved.booleanValue() == true) {
            return "${coord:dataOutPartitions('" + dataOutName + "')}";
        }
        try {
            return new HCatURI(uri).toPartitionString();
        }
        catch (URISyntaxException e) {
            throw new RuntimeException("Parsing exception for HCatURI " + uri + ". details: " + e);
        }
    }

View Full Code Here

Examples of org.apache.oozie.util.HCatURI

                if (uriList.length > 1) {
                    throw new RuntimeException("Multiple partitions not supported for hive-export type. Dataset name: "
                        + dataInName + " URI: " + uri);
                }
                try {
                    partitionValue = new HCatURI(uri).toPartitionValueString(type);
                }
                catch (URISyntaxException e) {
                    throw new RuntimeException("Parsing exception for HCatURI " + uri, e);
                }
            } else {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.