// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.api.adwords.lib.utils.v201309;
import com.google.api.adwords.lib.AdWordsUser;
import com.google.api.adwords.lib.AuthToken;
import com.google.api.adwords.lib.AuthTokenException;
import com.google.api.adwords.lib.utils.JaxBSerializer;
import com.google.api.adwords.v201309.jaxb.cm.DownloadFormat;
import com.google.api.adwords.v201309.jaxb.cm.ReportDefinition;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.namespace.QName;
/**
* Utility class for downloading reports like in the following code:
*
* <pre>
* <code>ReportUtils.downloadReport(adWordsUser, reportDefinition, outputStream);
* </code>
* </pre>
*
* or if you are using AWQL, like in the following code:
*
* <pre>
* <code>ReportUtils.downloadReport(adWordsUser, query, DownloadFormat, outputStream);
* </code>
* </pre>
*
* The {@code adWordsUser} is used to authenticate the request against the {@code reportDownloadUrl}
* .
*
* @author api.arogal@gmail.com (Adam Rogal)
* @author api.kwinter@gmail.com (Kevin Winter)
* @author api.thagikura@gmail.com (Takeshi Hagikura)
*/
public class ReportUtils {
/** The URI of the download server. */
private static final String DOWNLOAD_SERVER_URI = "/api/adwords/reportdownload";
/** The hostname of the download server. */
private static final String DOWNLOAD_SERVER = "https://adwords.google.com";
/** The version to append to url for Ad Hoc report downloads. */
private static final String VERSION = "v201309";
/** Regular expression used to match attributes in xml tags. */
private static final String REMOVE_ATTRIBUTES_REGEX =
"( )?(xmlns|xsi):(\\w)+=\".*?\"|ns\\d:|<\\?xml.*?>";
/** Regular expression used to match the type attribute in report download error response. */
private static final String ERROR_TYPE_REGEX = "^.*<type>(.*?)</type>.*$";
/** Regular expression used to match the trigger attribute in report download error response. */
private static final String ERROR_TRIGGER_REGEX = "^.*<trigger>(.*?)</trigger>.*$";
/**
* Regular expression used to match the field path attribute in report download error response.
*/
private static final String ERROR_FIELD_PATH_REGEX = "^.*<fieldpath>(.*?)</fieldpath>.*$";
/** Regular expression used to remove self closing tags. */
private static final String REMOVE_SELF_CLOSING_TAG = "<\\w+( )?/>";
/** Regex group number that report exception message is stored in. */
private static final int ERROR_MESSAGE_GROUP = 1;
// Static so we hold only a single reference of the JAXBContext
private static final JaxBSerializer<ReportDefinition> serializer =
new JaxBSerializer<ReportDefinition>(ReportDefinition.class, new QName("reportDefinition"));
/** Length of the report head. */
// Visible for testing.
static final int REPORT_HEAD_LENGTH = 1024;
/**
* {@code ReportUtils} is not meant to have any instances.
*/
private ReportUtils() {}
/**
* Extracts the error field from the report download error response.
*
* @param responseMessage The http response message from which extract the error field string.
* @param regex The regex pattern to extract.
* @return The extracted field string.
*/
private static String extractErrorField(String responseMessage, String regex) {
String result = "";
Matcher matcher = Pattern.compile(regex).matcher(responseMessage);
if (matcher.matches()) {
result = matcher.group(ERROR_MESSAGE_GROUP);
}
return result;
}
/**
* Downloads a report.
*
* @param outputStream the output stream to download to
* @param conn HttpURLConnection used to download the report.
* @return the report download response. The {@code outputStream} will be flushed and closed.
* @throws IOException if there was an exception while downloading the report
*/
private static ReportDownloadResponse reportDownloadExecute(
OutputStream outputStream, HttpURLConnection conn) throws IOException {
int status = conn.getResponseCode();
ReportDownloadResponse result = null;
if (status == HttpURLConnection.HTTP_OK) {
copyStreams(conn.getInputStream(), outputStream);
result = new ReportDownloadSuccessResponse(status, "SUCCESS");
} else {
// Anything other than success means the body has an error message.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
BufferedOutputStream output = new BufferedOutputStream(baos);
copyStreams(conn.getErrorStream(), output);
output.close();
String responseMessage = baos.toString();
result = new ReportDownloadErrorResponse(status,
extractErrorField(responseMessage, ERROR_TYPE_REGEX),
extractErrorField(responseMessage, ERROR_TRIGGER_REGEX),
extractErrorField(responseMessage, ERROR_FIELD_PATH_REGEX));
}
return result;
}
/**
* Make sure we have valid credentials and returns HttpURLConnection.
*
* @param adWordsUser the user to generate authentication
* @return HttpURLConnection used to download the report.
* @throws AuthTokenException if the credentials are invalid.
* @throws MalformedURLException if the report download URL could not be used
* @throws IOException if there was an exception while downloading the report
* @throws ProtocolException if there is an error in the underlying protocol.
*/
private static HttpURLConnection reportDownloadPreProcess(AdWordsUser adWordsUser)
throws AuthTokenException, MalformedURLException, IOException, ProtocolException {
String downloadUrl = generateAdHocReportUrl(adWordsUser);
// Make sure we have valid credentials.
reloadAuthToken(adWordsUser);
HttpURLConnection conn = getReportHttpUrlConnection(downloadUrl,
adWordsUser.getRegisteredAuthToken(), adWordsUser.isReportsReturnMoneyInMicros(),
adWordsUser.getClientCustomerId(), adWordsUser.getDeveloperToken());
conn.setRequestProperty("Content-type", "application/x-www-form-urlencoded");
conn.setRequestMethod("POST");
conn.setDoOutput(true);
return conn;
}
/**
* Downloads a report to the provided output stream. On success, the outputStream will be flushed
* and closed.
*
* @param adWordsUser the user to generate authentication
* @param reportDefinition the report definition to serialize
* @param outputStream the output stream to download to
* @return the report download response. For a successful requests, the {@code outputStream} will
* be flushed and closed.
* @throws ReportException If there is any issue making HTTP request with server.
*/
public static ReportDownloadResponse downloadReport(
AdWordsUser adWordsUser, ReportDefinition reportDefinition, OutputStream outputStream)
throws ReportException {
try {
HttpURLConnection conn = reportDownloadPreProcess(adWordsUser);
writeReportToStream(conn.getOutputStream(), reportDefinition);
return reportDownloadExecute(outputStream, conn);
} catch (AuthTokenException e) {
throw new ReportException("Could not obtain AuthToken", e);
} catch (MalformedURLException e) {
throw new ReportException("Created invalid report download URL.", e);
} catch (IOException e) {
throw new ReportException("Problem sending data to report download server.", e);
}
}
/**
* Downloads a report to the provided output stream with AWQL. On success, the outputStream will
* be flushed and closed.
*
* @param adWordsUser the user to generate authentication
* @param query AWQL query string.
* @param format the expected format type.
* @param outputStream the output stream to download to
* @return the report download response. For a successful requests, the {@code outputStream} will
* be flushed and closed.
* @throws ReportException If there is any issue making HTTP request with server.
*/
public static ReportDownloadResponse downloadReport(
AdWordsUser adWordsUser, String query, DownloadFormat format, OutputStream outputStream)
throws ReportException {
try {
HttpURLConnection conn = reportDownloadPreProcess(adWordsUser);
writeReportToStream(conn.getOutputStream(), query, format);
return reportDownloadExecute(outputStream, conn);
} catch (AuthTokenException e) {
throw new ReportException("Could not obtain AuthToken", e);
} catch (MalformedURLException e) {
throw new ReportException("Created invalid report download URL.", e);
} catch (IOException e) {
throw new ReportException("Problem sending data to report download server.", e);
}
}
/**
* Serializes the report to XML and writes it form url-encoded to the provided stream.
*
* @param outputStream the output stream to download to
* @param reportDefinition the report definition to serialize
* @throws IOException if there was an exception while downloading the report
* @throws UnsupportedEncodingException if the character encoding is not supported.
*/
private static void writeReportToStream(
OutputStream outputStream, ReportDefinition reportDefinition)
throws UnsupportedEncodingException, IOException {
String reportDefinitionXml = toXml(reportDefinition);
OutputStreamWriter writer = new OutputStreamWriter(outputStream);
writer.write("__rdxml=" + URLEncoder.encode(reportDefinitionXml, "UTF-8"));
writer.close();
}
/**
* Writes a query and format string to the provided stream.
*
* @param outputStream the output stream to download to
* @param query AWQL query string.
* @param format the expected format type.
* @throws UnsupportedEncodingException if the character encoding is not supported.
* @throws IOException if there was an exception while downloading the report
*/
private static void writeReportToStream(
OutputStream outputStream, String query, DownloadFormat format)
throws UnsupportedEncodingException, IOException {
OutputStreamWriter writer = new OutputStreamWriter(outputStream);
writer.write("__rdquery=" + URLEncoder.encode(query, "UTF-8"));
writer.write("&__fmt=" + URLEncoder.encode(format.value(), "UTF-8"));
writer.close();
}
/**
* Serialize the report definition and sanitize the results.
*
* @param reportDefinition the report definition to serialize
* @return Sanitized XML for the report download.
*/
public static String toXml(ReportDefinition reportDefinition) {
return sanitize(serializer.serialize(reportDefinition));
}
/**
* Santizes the xml by removing all attributes, all self-closed tags as well as renames the root
* element to <reportDefinition>
*
* @param string
* @return Sanitized xml string suitable to send to the report download server.
*/
private static String sanitize(String string) {
string = string.replaceAll(REMOVE_ATTRIBUTES_REGEX, "");
string = string.replaceAll(REMOVE_SELF_CLOSING_TAG, "");
string = string.replaceAll("ReportDefinition", "reportDefinition");
return string;
}
private static void reloadAuthToken(AdWordsUser adWordsUser) throws AuthTokenException {
if (adWordsUser.getRegisteredAuthToken() == null) {
adWordsUser.setAuthToken(
new AuthToken(adWordsUser.getEmail(), adWordsUser.getPassword()).getAuthToken());
}
}
/**
* Gets the endpoint server for the production environment.
*
* @param user
* @return the endpoint server.
*/
private static String getServer(AdWordsUser user) {
return DOWNLOAD_SERVER;
}
public static String generateAdHocReportUrl(AdWordsUser user) {
return getServer(user) + DOWNLOAD_SERVER_URI + '/' + VERSION;
}
/**
* Gets the report HTTP URL connection given report URL and proper information needed to
* authenticate the request.
*
* @param reportUrl the URL of the report response or download
* @param authToken the authentication token used for authentication
* @param returnMoneyInMicros {@code true} if money values should be returned in micros
* @param clientCustomerId the clientCustomerId HTTP header
* @param developerToken the Developer Token
* @return the report HTTP URL connection
* @throws MalformedURLException if the report URL could not be used
* @throws IOException if there was a problem connecting to the URL
*/
public static HttpURLConnection getReportHttpUrlConnection(String reportUrl, String authToken,
boolean returnMoneyInMicros, String clientCustomerId, String developerToken)
throws MalformedURLException, IOException {
HttpURLConnection httpUrlConnection = (HttpURLConnection) new URL(reportUrl).openConnection();
httpUrlConnection.setRequestMethod("GET");
httpUrlConnection.setRequestProperty("Authorization", "GoogleLogin auth=" + authToken);
httpUrlConnection.setRequestProperty("developerToken", developerToken);
httpUrlConnection.setRequestProperty("clientCustomerId", clientCustomerId);
httpUrlConnection.setRequestProperty(
"returnMoneyInMicros", Boolean.toString(returnMoneyInMicros));
// Required so that 301s to the final location don't trigger a redirect.
httpUrlConnection.setInstanceFollowRedirects(false);
return httpUrlConnection;
}
/**
* Copies the {@code inputStream} into the {@code outputSteam} and finally closes the both
* streams.
*/
private static void copyStreams(InputStream inputStream, OutputStream outputStream)
throws IOException {
BufferedInputStream bis = new BufferedInputStream(inputStream);
BufferedOutputStream bos = new BufferedOutputStream(outputStream);
try {
int i = 0;
byte[] buffer = new byte[REPORT_HEAD_LENGTH];
while ((i = bis.read(buffer)) != -1) {
bos.write(buffer, 0, i);
}
} finally {
if (bis != null) {
bis.close();
}
if (bos != null) {
bos.close();
}
}
}
}