Package com.socrata.datasync.job

Examples of com.socrata.datasync.job.JobStatus

67
68
69
70
71
72
73
74
75
76
        try {
            InputStream inputControlFile = new FileInputStream(controlFile);
            return publishViaFTPDropboxV2(userPrefs, datasetId, csvOrTsvFile, inputControlFile);
        } catch (Exception e) {
            e.printStackTrace();
            JobStatus status = JobStatus.PUBLISH_ERROR;
            status.setMessage("Error uploading control file: " + e.getMessage());
            return status;
        }
    }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
        try {
            InputStream inputControlFile = new ByteArrayInputStream(controlFileContent.getBytes("UTF-8"));
            return publishViaFTPDropboxV2(userPrefs, datasetId, csvOrTsvFile, inputControlFile);
        } catch (Exception e) {
            e.printStackTrace();
            JobStatus status = JobStatus.PUBLISH_ERROR;
            status.setMessage("Error uploading control file content: " + e.getMessage());
            return status;
        }
    }
View Full Code Here
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
                job = new MetadataJob(jobFileToRun);
              }
              else {
                job = new IntegrationJob(jobFileToRun);
              }
                JobStatus status = job.run();
                if(status.isError()) {
                    System.err.print("Job completed with errors: ");
                    System.err.println(status.getMessage());
                    System.exit(1);
                } else {
                    // job ran successfully!
                    System.out.println("Job completed successfully");
                    System.out.println(status.getMessage());
                }
            } catch (IOException e) {
                System.err.println("Error running " + jobFileToRun + ": " + e.toString());
                System.exit(1);
            }
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
            System.exit(1);
        }
  }

    public SimpleIntegrationRunner(Job job) {
        JobStatus status;
        try {
            status = job.run();
            if(status.isError()) {
                System.err.print("Job completed with errors: ");
                System.err.println(status.getMessage());
                System.exit(1);
            } else {
                System.out.println("Job completed successfully");
                if(job.getClass() == PortJob.class) {
                    System.out.println(status.getMessage() + ". " +
                        "Your newly created dataset is at:\n" +
                        ((PortJob)job).getSinkSiteDomain() + "/d/" + ((PortJob)job).getSinkSetID());
                    }
                System.out.println(status.getMessage());
            }
        } catch (IOException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
52
        // Uncommend to do replace via FTP (SmartUpdate)
        //jobToRun.setPublishViaFTP(true);
        //jobToRun.setPathToFTPControlFile("control.json");

        JobStatus status = jobToRun.run();
        if(status.isError()) {
            System.err.println("Job failed: " + status.getMessage());
        } else {
            System.out.println("Job ran successfully!");
        }
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
        if(fileToPublish.equals(""))
            return JobStatus.MISSING_FILE_TO_PUBLISH;

        File publishFile = new File(fileToPublish);
        if(!publishFile.exists() || publishFile.isDirectory()) {
            JobStatus errorStatus = JobStatus.FILE_TO_PUBLISH_DOESNT_EXIST;
            errorStatus.setMessage(fileToPublish + ": File to publish does not exist");
            return errorStatus;
        }

        String fileExtension = Utils.getFileExtension(fileToPublish);
        if(!allowedFileToPublishExtensions.contains(fileExtension))
            return JobStatus.FILE_TO_PUBLISH_INVALID_FORMAT;

        Dataset schema;
        try {
            schema = DatasetUtils.getDatasetInfo(connectionInfo.getUrl(), job.getDatasetID());

            if(job.getPublishViaDi2Http() || job.getPublishViaFTP()) {

                ControlFile control = job.getControlFile();
                FileTypeControl fileControl = null;
                switch (fileExtension) {
                    case "csv": fileControl = control.csv; break;
                    case "tsv": fileControl = control.tsv; break;
                }

                JobStatus actionOkay = checkAction(control.action, job, schema);
                if (actionOkay.isError())
                    return actionOkay;

                if (fileControl == null && !control.action.equalsIgnoreCase(PublishMethod.delete.name())) {
                    JobStatus noFileTypeContent = JobStatus.PUBLISH_ERROR;
                    noFileTypeContent.setMessage("The control file for '" + publishFile.getName() +
                            "' requires that the '" + fileExtension + "' option be filled in");
                    return noFileTypeContent;
                }

                String[] headers = getHeaders(fileControl, publishFile);
                if (headers == null) {
                    JobStatus noHeaders = JobStatus.PUBLISH_ERROR;
                    noHeaders.setMessage("Headers must be specified in one of " + publishFile.getName() + " or the control file using 'columns'");
                    return noHeaders;
                }

                Set<String> synthetics = fileControl.syntheticLocations.keySet();
                JobStatus csvDatasetAgreement = checkColumnAgreement(schema, headers, synthetics, publishFile.getName());
                if (csvDatasetAgreement.isError())
                    return csvDatasetAgreement;

                JobStatus controlHeaderAgreement = checkControlAgreement(fileControl, schema, headers, publishFile.getName());
                if (controlHeaderAgreement.isError())
                    return controlHeaderAgreement;

                JobStatus controlSensibility = validateControlFile(fileControl, connectionInfo.getUrl());
                if (controlSensibility.isError())
                    return controlSensibility;
            }
        } catch (Exception e) {
            // Not going to fail jobs on the validation check
        }
View Full Code Here
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
    private static JobStatus validateControlFile(FileTypeControl fileControl, String urlBase) {

        if (fileControl == null) return JobStatus.VALID;

        JobStatus goodTimestampFormats = checkTimeFormattingValidity(fileControl);
        if (goodTimestampFormats.isError())
            return goodTimestampFormats;

        JobStatus goodEncoding = checkEncodingValidity(fileControl, urlBase);
        if (goodEncoding.isError())
            return goodEncoding;

        return JobStatus.VALID;
    }
View Full Code Here
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
            methods.append("\t" + m.name() + "\n");
            if (m.name().equalsIgnoreCase(action))
                okAction = true;
        }
        if (!okAction) {
            JobStatus status = JobStatus.PUBLISH_ERROR;
            status.setMessage("Unknown Publish Method: " +
                    "The control file must specify the publishing method via the 'action' option as one of: \n" +
                    methods.toString());
            return status;
        }
        if (!PublishMethod.replace.name().equalsIgnoreCase(action) && job.getPublishViaFTP()) {
            JobStatus status = JobStatus.PUBLISH_ERROR;
            status.setMessage("FTP does not currently support upsert, append or delete");
            return status;
        }
        PublishMethod publishMethod = job.getPublishMethod();
        if (publishMethod != null && !action.equalsIgnoreCase(publishMethod.name())) {
            JobStatus status = JobStatus.PUBLISH_ERROR;
            status.setMessage("Conflicting Publish Methods: " +
                    "The publish method selected was '" + publishMethod.name() +
                    "', but the 'action' option in the control file specifies the publish method as '" + action + ".");
            return status;
        }
        String rowIdentifier = DatasetUtils.getRowIdentifierName(schema);
        if (rowIdentifier == null && PublishMethod.delete.name().equalsIgnoreCase(action)) {
            JobStatus status = JobStatus.PUBLISH_ERROR;
            status.setMessage("Dataset Requirement Unfulfilled: " +
                    "To delete from a dataset, a row identifier must be set. Dataset '" + schema.getId() +
                    "' does not have a row identifier set");
            return status;
        }
        return JobStatus.VALID;
View Full Code Here
388
389
390
391
392
393
394
395
396
397
398
399
        for (String format : timeFormats) {
            if (format.equalsIgnoreCase(supportedTimeFormat))
                continue;
            try { formatter = DateTimeFormat.forPattern(format); }
            catch (IllegalArgumentException e) {
                JobStatus status = JobStatus.PUBLISH_ERROR;
                status.setMessage("Unsupported Date Time Format: The time format '" + format +
                        "' specified in the control file is not a valid pattern." +
                        "\nPlease consult " + jodaLink + " for more information");
                return status;
            }
        }
View Full Code Here
421
422
423
424
425
426
427
428
429
430
431
432
                    encodingFound = true;
                    break;
                }
            }
            if (!encodingFound) {
                JobStatus status = JobStatus.PUBLISH_ERROR;
                status.setMessage("Unsupported Encoding: The encoding '" + encoding + "' in the control file is not supported." +
                        "\nPlease consult " + charsetUri + " for a listing of supported encodings");
                return status;
            }
        } catch (Exception e) {
            // no reason to fail jobs because of encoding check
View Full Code Here
TOP

Related Classes of com.socrata.datasync.job.JobStatus

  • com.socrata.datasync.LibraryExample
  • com.socrata.datasync.publishers.DeltaImporter2Publisher
  • com.socrata.datasync.publishers.FTPDropbox2Publisher
  • com.socrata.datasync.SimpleIntegrationRunner
  • com.socrata.datasync.ui.PortJobTab
  • com.socrata.datasync.utilities.FTPUtilityTest
  • com.socrata.datasync.validation.IntegrationJobValidity
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.