Examples of SLACalculatorMemory

org.apache.oozie.sla.SLACalculatorMemory
Implementation class for SLACalculator that calculates SLA related to start/end/duration of jobs using a memory-based map

Examples of org.apache.oozie.sla.SLACalculatorMemory

    }


    public void testSLAFailOverWithHA() throws Exception {


        SLAService slas = Services.get().get(SLAService.class);
        SLACalculatorMemory slaCalcMem = (SLACalculatorMemory) slas.getSLACalculator();
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);


        // start another dummy oozie instance (dummy sla and eventhandler
        // services)
        DummyZKOozie dummyOozie_1 = null;
        try {
            dummyOozie_1 = new DummyZKOozie("a", "http://blah");
            DummySLACalculatorMemory dummyCalc = new DummySLACalculatorMemory();
            EventHandlerService dummyEhs = new EventHandlerService();
            dummyCalc.setEventHandlerService(dummyEhs);
            dummyEhs.init(Services.get());
            dummyCalc.init(Services.get().getConf());


            // Case 1 workflow job submitted to dummy server,
            // but before start running, the dummy server is down
            createWorkflow("job-1");
            SLARegistrationBean sla1 = TestSLAService._createSLARegistration("job-1", AppType.WORKFLOW_JOB);
            sla1.setExpectedStart(new Date(System.currentTimeMillis() - 2 * 3600 * 1000)); // 2 hr before
            sla1.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 3600 * 1000)); // 1 hr before
            sla1.setExpectedDuration(10 * 60 * 1000); // 10 mins
            dummyCalc.addRegistration(sla1.getId(), sla1);


            // Case 2. workflow job submitted to dummy server, start running,
            // then the dummy server is down
            createWorkflow("job-2");
            SLARegistrationBean sla2 = TestSLAService._createSLARegistration("job-2", AppType.WORKFLOW_JOB);
            sla2.setExpectedStart(new Date(System.currentTimeMillis() - 2 * 3600 * 1000)); // 2hr before
            sla2.setExpectedEnd(new Date(System.currentTimeMillis() + 1 * 3600 * 1000)); // 1hr ahead
            sla2.setExpectedDuration(10 * 60 * 1000); // 10 mins
            dummyCalc.addRegistration(sla2.getId(), sla2);
            dummyCalc.addJobStatus(sla2.getId(), WorkflowJob.Status.RUNNING.name(), EventStatus.STARTED, new Date(),
                    new Date());


            dummyCalc.updateAllSlaStatus();
            dummyEhs.new EventWorker().run();
            assertTrue(output.toString().contains(sla2.getId() + " Sla START - MISS!!!"));


            // suppose dummy Server is down
            dummyCalc.clear();
            dummyCalc = null;
            dummyOozie_1.teardown();


            slaCalcMem.updateAllSlaStatus();


            // Job 1 started running on the living server --> start miss
            slaCalcMem.addJobStatus(sla1.getId(), WorkflowJob.Status.RUNNING.name(), EventStatus.STARTED, new Date(),
                    new Date());
            // job 1 is added to slamap of living oozie server
            assertNotNull(slaCalcMem.get(sla1.getId()));
            ehs.new EventWorker().run();
            assertTrue(output.toString().contains(sla1.getId() + " Sla START - MISS!!!"));


            // Job 1 succeeded on the living server --> duration met and end miss
            slaCalcMem.addJobStatus(sla1.getId(), WorkflowJob.Status.SUCCEEDED.name(), EventStatus.SUCCESS, new Date(),
                    new Date());
            ehs.new EventWorker().run();
            assertTrue(output.toString().contains(sla1.getId() + " Sla DURATION - MET!!!"));
            assertTrue(output.toString().contains(sla1.getId() + " Sla END - MISS!!!"));


            // Job 2 succeeded on the living server --> duration met and end met
            slaCalcMem.addJobStatus(sla2.getId(), WorkflowJob.Status.SUCCEEDED.name(), EventStatus.SUCCESS, new Date(),
                    new Date());
            // eventProc >= 7(already processed duration/end met), should be removed from slaMap
            assertNull(slaCalcMem.get(sla2.getId()));
            ehs.new EventWorker().run();
            assertTrue(output.toString().contains(sla2.getId() + " Sla DURATION - MET!!!"));
            assertTrue(output.toString().contains(sla2.getId() + " Sla END - MET!!!"));
        }
        finally {

View Full Code Here

Examples of org.apache.oozie.sla.SLACalculatorMemory

        // Coord Action of jobs 5-6 already started and currently running (to test history set)
        createDBEntryForStarted(id5, expectedStartTS, expectedEndTS2);
        createDBEntryForStarted(id6, expectedStartTS, expectedEndTS2);


        SLAService slas = Services.get().get(SLAService.class);
        SLACalculatorMemory slaCalcMem = (SLACalculatorMemory) slas.getSLACalculator();
        slaCalcMem.init(Services.get().getConf());
        List<String> slaMapKeys = new ArrayList<String>();
        Iterator<String> itr = slaCalcMem.iterator();
        while (itr.hasNext()) {
            slaMapKeys.add(itr.next());
        }
        assertEquals(6, slaMapKeys.size());


        DummyZKOozie dummyOozie_1 = null;
        try {
            // start another dummy oozie instance (dummy sla and event handler services)
            dummyOozie_1 = new DummyZKOozie("a", "http://blah");
            DummySLACalculatorMemory dummySlaCalcMem = new DummySLACalculatorMemory();
            EventHandlerService dummyEhs = new EventHandlerService();
            dummySlaCalcMem.setEventHandlerService(dummyEhs);
            dummyEhs.init(Services.get());
            dummySlaCalcMem.init(Services.get().getConf());
            slaMapKeys = new ArrayList<String>();
            itr = dummySlaCalcMem.iterator();
            while (itr.hasNext()) {
                slaMapKeys.add(itr.next());
            }
            assertEquals(6, slaMapKeys.size());


            // Coord Action 1,3 run and update status on *non-dummy* server
            updateCoordAction(id1, "RUNNING");
            slaCalcMem
                    .addJobStatus(id1, CoordinatorAction.Status.RUNNING.name(), EventStatus.STARTED, new Date(), null);
            updateCoordAction(id3, "FAILED");
            slaCalcMem.addJobStatus(id3, CoordinatorAction.Status.FAILED.name(), EventStatus.FAILURE, null, new Date());


            // Coord Action 2,4 run and update status on *dummy* server
            updateCoordAction(id2, "RUNNING");
            dummySlaCalcMem.addJobStatus(id2, CoordinatorAction.Status.RUNNING.name(), EventStatus.STARTED, new Date(),
                    null);
            updateCoordAction(id4, "FAILED");
            dummySlaCalcMem.addJobStatus(id4, CoordinatorAction.Status.FAILED.name(), EventStatus.FAILURE, null,
                    new Date());


            // Both servers iterate SlaMap (updateAllSlaStatus)
            slaCalcMem.updateAllSlaStatus();
            dummySlaCalcMem.updateAllSlaStatus();


            // SlaMap on both Servers synced
            SLACalcStatus sla1_nodummy = slaCalcMem.get(id1);
            SLACalcStatus sla1_dummy = dummySlaCalcMem.get(id1);
            SLACalcStatus sla2_nodummy = slaCalcMem.get(id2);
            SLACalcStatus sla2_dummy = dummySlaCalcMem.get(id2);
            assertEquals(1, sla1_nodummy.getEventProcessed());
            assertEquals(1, sla1_dummy.getEventProcessed());
            assertEquals(1, sla2_dummy.getEventProcessed());
            assertEquals(1, sla2_nodummy.getEventProcessed());
            assertFalse(slaCalcMem.isJobIdInSLAMap(id3));
            assertFalse(dummySlaCalcMem.isJobIdInSLAMap(id3));
            assertFalse(slaCalcMem.isJobIdInSLAMap(id4));
            assertFalse(dummySlaCalcMem.isJobIdInSLAMap(id4));


            Byte eventProc = (Byte) SLASummaryQueryExecutor.getInstance().getSingleValue(
                    SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED, id3);
            assertEquals(8, eventProc.byteValue());
            eventProc = (Byte) SLASummaryQueryExecutor.getInstance().getSingleValue(
                    SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED, id4);
            assertEquals(8, eventProc.byteValue());


            // Action 5 was processed as END_MISS in updateAllSlaStatus, put into history set
            assertTrue(slaCalcMem.isJobIdInHistorySet(id5));
            assertTrue(dummySlaCalcMem.isJobIdInHistorySet(id6));
            // Action 6 was processed as END_MISS in updateAllSlaStatus, put into history set
            assertTrue(slaCalcMem.isJobIdInHistorySet(id5));
            assertTrue(dummySlaCalcMem.isJobIdInHistorySet(id6));


            eventProc = (Byte) SLASummaryQueryExecutor.getInstance().getSingleValue(
                    SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED, id5);
            assertEquals(7, eventProc.byteValue());
            eventProc = (Byte) SLASummaryQueryExecutor.getInstance().getSingleValue(
                    SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED, id6);
            assertEquals(7, eventProc.byteValue());


            // Action 1 Succeeded on non-dummy server
            updateCoordAction(id1, "SUCCEEDED");
            slaCalcMem.addJobStatus(id1, CoordinatorAction.Status.SUCCEEDED.name(), EventStatus.SUCCESS, new Date(
                    System.currentTimeMillis() - 1800 * 1000), new Date());


            // Action 2 Succeeded on dummy server
            updateCoordAction(id2, "SUCCEEDED");
            dummySlaCalcMem.addJobStatus(id2, CoordinatorAction.Status.SUCCEEDED.name(), EventStatus.SUCCESS, new Date(
                    System.currentTimeMillis() - 1800 * 1000), new Date());


            // Both servers iterate SlaMap (updateAllSlaStatus)
            slaCalcMem.updateAllSlaStatus();
            dummySlaCalcMem.updateAllSlaStatus();


            // Action 1, 2 are removed from both servers
            assertNull(slaCalcMem.get(id1));
            assertNull(dummySlaCalcMem.get(id1));
            assertNull(slaCalcMem.get(id2));
            assertNull(dummySlaCalcMem.get(id2));
            eventProc = (Byte) SLASummaryQueryExecutor.getInstance().getSingleValue(
                    SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED, id1);
            assertEquals(8, eventProc.byteValue());
            eventProc = (Byte) SLASummaryQueryExecutor.getInstance().getSingleValue(
                    SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED, id2);
            assertEquals(8, eventProc.byteValue());


            // Test HistoryPurgeWorker purges Action 5,6 from history set
            updateCoordAction(id5, "SUCCEEDED");
            slaCalcMem.new HistoryPurgeWorker().run();
            assertFalse(slaCalcMem.isJobIdInHistorySet(id5));
            updateCoordAction(id6, "SUCCEEDED");
            dummySlaCalcMem.new HistoryPurgeWorker().run();
            assertFalse(dummySlaCalcMem.isJobIdInHistorySet(id6));


        }

View Full Code Here

Examples of org.apache.oozie.sla.SLACalculatorMemory

        Date expectedStartTS = new Date(System.currentTimeMillis() - 2 * 3600 * 1000); // get MISS
        Date expectedEndTS = new Date(System.currentTimeMillis() - 1 * 3600 * 1000); // get MISS
        createDBEntry(id1, expectedStartTS, expectedEndTS);


        SLAService slas = Services.get().get(SLAService.class);
        SLACalculatorMemory slaCalcMem = (SLACalculatorMemory) slas.getSLACalculator();
        slaCalcMem.init(Services.get().getConf()); // loads the job in sla map


        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        EventQueue ehs_q = ehs.getEventQueue();


        DummyZKOozie dummyOozie_1 = null;
        try {
            // start another dummy oozie instance (dummy sla and event handler services)
            dummyOozie_1 = new DummyZKOozie("a", "http://blah");
            DummySLACalculatorMemory dummySlaCalcMem = new DummySLACalculatorMemory();
            dummySlaCalcMem.init(Services.get().getConf());
            EventHandlerService dummyEhs = new EventHandlerService();
            dummySlaCalcMem.setEventHandlerService(dummyEhs);
            dummyEhs.init(Services.get());
            EventQueue dummyEhs_q = dummyEhs.getEventQueue();


            // Action started on Server 1
            updateCoordAction(id1, "RUNNING");
            slaCalcMem
                    .addJobStatus(id1, CoordinatorAction.Status.RUNNING.name(), EventStatus.STARTED, new Date(), null);
            SLACalcStatus s1 = (SLACalcStatus) ehs_q.poll();
            assertEquals(SLAStatus.IN_PROCESS, s1.getSLAStatus());


            // Action ended on Server 2
            updateCoordAction(id1, "FAILED");
            dummySlaCalcMem.addJobStatus(id1, CoordinatorAction.Status.FAILED.name(), EventStatus.FAILURE, new Date(
                    System.currentTimeMillis() - 1800 * 1000),
                    new Date());
            dummyEhs_q.poll(); // getting rid of the duration_miss event
            SLACalcStatus s2 = (SLACalcStatus) dummyEhs_q.poll();
            assertEquals(SLAStatus.MISS, s2.getSLAStatus());


            slaCalcMem.updateAllSlaStatus();
            dummySlaCalcMem.updateAllSlaStatus();
            assertEquals(0, ehs_q.size()); // no dupe event should be created again by Server 1
        }
        finally {
            if (dummyOozie_1 != null) {

View Full Code Here

Examples of org.apache.oozie.sla.SLACalculatorMemory

    public void init(Services services) throws ServiceException {
        try {
            Configuration conf = services.getConf();
            Class<? extends SLACalculator> calcClazz = (Class<? extends SLACalculator>) ConfigurationService.getClass(
                    conf, CONF_CALCULATOR_IMPL);
            calcImpl = calcClazz == null ? new SLACalculatorMemory() : (SLACalculator) calcClazz.newInstance();
            calcImpl.init(conf);
            eventHandler = Services.get().get(EventHandlerService.class);
            if (eventHandler == null) {
                throw new ServiceException(ErrorCode.E0103, "EventHandlerService", "Add it under config "
                        + Services.CONF_SERVICE_EXT_CLASSES + " or declare it BEFORE SLAService");

View Full Code Here

Examples of org.apache.oozie.sla.SLACalculatorMemory

    public void init(Services services) throws ServiceException {
        try {
            Configuration conf = services.getConf();
            Class<? extends SLACalculator> calcClazz = (Class<? extends SLACalculator>) conf.getClass(
                    CONF_CALCULATOR_IMPL, null);
            calcImpl = calcClazz == null ? new SLACalculatorMemory() : (SLACalculator) calcClazz.newInstance();
            calcImpl.init(conf);
            eventHandler = Services.get().get(EventHandlerService.class);
            if (eventHandler == null) {
                throw new ServiceException(ErrorCode.E0103, "EventHandlerService", "Add it under config "
                        + Services.CONF_SERVICE_EXT_CLASSES + " or declare it BEFORE SLAService");

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.