Package org.owasp.webscarab.httpclient

Source Code of org.owasp.webscarab.httpclient.URLFetcher

/***********************************************************************
*
* $CVSHeader$
*
* This file is part of WebScarab, an Open Web Application Security
* Project utility. For details, please see http://www.owasp.org/
*
* Copyright (c) 2002 - 2004 Rogan Dawes
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*
* Getting Source
* ==============
*
* Source for this application is maintained at Sourceforge.net, a
* repository for free software projects.
*
* For details, please see http://www.sourceforge.net/projects/owasp
*
*/

/*
* URLFetcher.java
*
* Created on April 12, 2003, 1:31 AM
*/

package org.owasp.webscarab.httpclient;

import java.io.IOException;

import java.net.Socket;
import java.net.InetSocketAddress;

import java.io.InputStream;
import java.io.OutputStream;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.logging.Level;

import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.SSLContext;

import java.util.logging.Logger;
import javax.net.ssl.SSLPeerUnverifiedException;
import jcifs.ntlmssp.NtlmFlags;
import jcifs.ntlmssp.NtlmMessage;
import jcifs.ntlmssp.Type1Message;
import jcifs.ntlmssp.Type2Message;
import jcifs.ntlmssp.Type3Message;
import jcifs.util.Base64;

import org.owasp.webscarab.model.HttpUrl;
import org.owasp.webscarab.model.NamedValue;
import org.owasp.webscarab.model.Request;
import org.owasp.webscarab.model.Response;

import org.owasp.webscarab.util.Glob;

/** Creates a new instance of URLFetcher
* @author rdawes
*/
public class URLFetcher implements HTTPClient {

    // These represent the SSL classes required to connect to the server.
    private String _keyFingerprint = null;
    private SSLContextManager _sslContextManager = null;

    private Logger _logger = Logger.getLogger(getClass().getName());

    private String _httpProxy = "";
    private int _httpProxyPort = -1;
    private String _httpsProxy = "";
    private int _httpsProxyPort = -1;
    private String[] _noProxy = new String[0];

    private Socket _socket = null;
    private boolean _direct = false;
    private Response _response = null;

    // these represent an already connected socket, and the end point thereof.
    private InputStream _in = null;
    private OutputStream _out = null;
    private String _host = null;
    private int _port = 0;
    private long _lastRequestTime = 0;

    private int _timeout = 0;
    private int _connectTimeout = 10000;

    private Authenticator _authenticator = null;
    private String _authCreds = null;
    private String _proxyAuthCreds = null;

    /** Creates a new instance of URLFetcher
     */
    public URLFetcher() {
    }

    /** Tells URLFetcher which HTTP proxy to use, if any
     * @param proxy The address or name of the proxy server to use for HTTP requests
     * @param proxyport The port on the proxy server to connect to
     */
    public void setHttpProxy(String proxy, int proxyport) {
        _httpProxy = proxy;
        if (_httpProxy == null) _httpProxy = "";
        _httpProxyPort = proxyport;
    }

    /** Tells URLFetcher which HTTPS proxy to use, if any
     * @param proxy The address or name of the proxy server to use for HTTPS requests
     * @param proxyport The port on the proxy server to connect to
     */
    public void setHttpsProxy(String proxy, int proxyport) {
        _httpsProxy = proxy;
        if (_httpsProxy == null) _httpsProxy = "";
        _httpsProxyPort = proxyport;
    }

    /** Accepts an array of hostnames or domains for which no proxy should be used.
     * if the hostname begins with a period ("."), than all hosts in that domain will
     * ignore the configured proxies
     * @param noproxy An array of hosts or domains for which no proxy should be used.
     * Domains must start with a period (".")
     */
    public void setNoProxy(String[] noproxy) {
        if (noproxy == null) {
            _noProxy = new String[0];
        } else if (noproxy.length == 0) {
            _noProxy = noproxy;
        } else {
            _noProxy = new String[noproxy.length];
            System.arraycopy(noproxy, 0, _noProxy, 0, noproxy.length);
        }
    }

    public void setSSLContextManager(SSLContextManager sslContextManager) {
        _sslContextManager = sslContextManager;
    }

    public void setTimeouts(int connectTimeout, int readTimeout) {
        _connectTimeout = connectTimeout;
        _timeout = readTimeout;
    }

    public void setAuthenticator(Authenticator authenticator) {
        _authenticator = authenticator;
    }

    public Authenticator getAuthenticator() {
        return _authenticator;
    }

    /** Can be used by a calling class to fetch a request without spawning an additional
     * thread. This is appropriate when the calling class is already running in an
     * independant thread, and must wait for the response before continuing.
     * @return the retrieved response
     * @param request the request to retrieve.
     */
    public Response fetchResponse(Request request) throws IOException {
        if (_response != null) {
            _response.flushContentStream(); // flush the content stream, just in case it wasn't read
            _response = null;
        }
        if (request == null) {
            _logger.severe("Asked to fetch a null request");
            return null;
        }
        HttpUrl url = request.getURL();
        if (url == null) {
            _logger.severe("Asked to fetch a request with a null URL");
            return null;
        }

        // if the previous auth method was not "Basic", force a new connection
        if (_authCreds != null && !_authCreds.startsWith("Basic"))
            _lastRequestTime = 0;
        if (_proxyAuthCreds != null && !_proxyAuthCreds.startsWith("Basic"))
            _lastRequestTime = 0;

        // Get any provided credentials from the request
        _authCreds = request.getHeader("Authorization");
        _proxyAuthCreds = request.getHeader("Proxy-Authorization");
        String keyFingerprint = request.getHeader("X-SSLClientCertificate");
        request.deleteHeader("X-SSLClientCertificate");
        if (keyFingerprint == null && _keyFingerprint == null) {
            // no problem
        } else if (keyFingerprint != null && _keyFingerprint != null && keyFingerprint.equals(_keyFingerprint)) {
            // no problem
        } else {
            // force a new connection, and change the fingerprint
            _keyFingerprint = keyFingerprint;
            _lastRequestTime = 0;
        }

        String status;

        String oldProxyAuthHeader = null;
        if (_proxyAuthCreds == null && _authenticator!= null && useProxy(url))
            _proxyAuthCreds = _authenticator.getProxyCredentials(url.toString().startsWith("https") ? _httpsProxy : _httpProxy, null);
        String proxyAuthHeader = constructAuthenticationHeader(null, _proxyAuthCreds);

        String oldAuthHeader = null;
        if (_authCreds == null && _authenticator!= null)
            _authCreds = _authenticator.getCredentials(url, null);
        String authHeader = constructAuthenticationHeader(null, _authCreds);

        int tries = 0;
        do {
            // make sure that we have a "clean" request each time through
            request.deleteHeader("Authorization");
            request.deleteHeader("Proxy-Authorization");

            _response = null;
            connect(url, true);
            if (_response != null) { // there was an error opening the socket
                return _response;
            }

            if (authHeader != null) {
                request.setHeader("Authorization", authHeader);
                if (authHeader.startsWith("NTLM") || authHeader.startsWith("Negotiate")) {
                    if (request.getVersion().equals("HTTP/1.0")) {
                        // we have to explicitly tell the server to keep the connection alive for 1.0
                        request.setHeader("Connection", "Keep-Alive");
                    } else {
                        request.deleteHeader("Connection");
                    }
                }
            }
            // depending on whether we are connected directly to the server, or via a proxy
            if (_direct) {
                request.writeDirect(_out);
            } else {
                if (proxyAuthHeader != null) {
                    request.setHeader("Proxy-Authorization", proxyAuthHeader);
                    if (proxyAuthHeader.startsWith("NTLM") || proxyAuthHeader.startsWith("Negotiate")) {
                        if (request.getVersion().equals("HTTP/1.0")) {
                            // we have to explicitly tell the server to keep the connection alive for 1.0
                            request.setHeader("Connection", "Keep-Alive");
                        } else {
                            request.deleteHeader("Connection");
                        }
                    }
                }
                request.write(_out);
            }
            _out.flush();
            _logger.finest("Request : \n" + request.toString());

            _response = new Response();
            _response.setRequest(request);

            // test for spurious 100 header from IIS 4 and 5.
            // See http://mail.python.org/pipermail/python-list/2000-December/023204.html
            _logger.fine("Reading the response");
            do {
                _response.read(_in);
                status = _response.getStatus();
            } while (status.equals("100"));

            {
                StringBuffer buff = new StringBuffer();
                buff.append(_response.getStatusLine()).append("\n");
                NamedValue[] headers = _response.getHeaders();
                if (headers != null)
                    for (int i=0; i< headers.length; i++)
                        buff.append(headers[i].getName()).append(": ").append(headers[i].getValue()).append("\n");
                _logger.finest("Response:\n" + buff.toString());
            }

            if (status.equals("407")) {
                _response.flushContentStream();
                oldProxyAuthHeader = proxyAuthHeader;
                String[] challenges = _response.getHeaders("Proxy-Authenticate");
                if (_proxyAuthCreds == null && _authenticator != null) {
                    _proxyAuthCreds = _authenticator.getProxyCredentials(_httpProxy, challenges);
                }
                proxyAuthHeader = constructAuthenticationHeader(challenges, _proxyAuthCreds);
                if (proxyAuthHeader != null && oldProxyAuthHeader != null && oldProxyAuthHeader.equals(proxyAuthHeader)) {
                    _logger.info("No possible authentication");
                    proxyAuthHeader = null;
                }
            }

            if (status.equals("401")) {
                _response.flushContentStream();
                oldAuthHeader = authHeader;
                String[] challenges = _response.getHeaders("WWW-Authenticate");
                if (_authCreds == null && _authenticator != null)
                    _authCreds = _authenticator.getCredentials(url, challenges);
                _logger.finer("Auth creds are " + _authCreds);
                authHeader = constructAuthenticationHeader(challenges, _authCreds);
                _logger.finer("Auth header is " + authHeader);
                if (authHeader != null && oldAuthHeader != null && oldAuthHeader.equals(authHeader)) {
                    _logger.info("No possible authentication");
                    authHeader = null;
                }
            }

            // if the request method is HEAD, we get no contents, EVEN though there
            // may be a Content-Length header.
            if (request.getMethod().equals("HEAD")) _response.setNoBody();

            _logger.info(request.getURL() +" : " + _response.getStatusLine());

            String connection = _response.getHeader("Proxy-Connection");
            if (connection != null && "close".equalsIgnoreCase(connection)) {
                _in = null;
                _out = null;
                // do NOT close the socket itself, since the message body has not yet been read!
            } else {
                connection = _response.getHeader("Connection");
                String version = request.getVersion();
                if (version.equals("HTTP/1.0") && "Keep-alive".equalsIgnoreCase(connection)) {
                    _lastRequestTime = System.currentTimeMillis();
                } else if (version.equals("HTTP/1.1") && (connection == null || !connection.equalsIgnoreCase("Close"))) {
                    _lastRequestTime = System.currentTimeMillis();
                } else {
                    _logger.info("Closing connection!");
                    _in = null;
                    _out = null;
                    // do NOT close the socket itself, since the message body has not yet been read!
                }
            }
            tries ++;
        } while (tries < 3 && ((status.equals("401") && authHeader != null) || (status.equals("407") && proxyAuthHeader != null)));

        if (_authCreds != null)
            request.setHeader("Authorization", _authCreds);
        if (_proxyAuthCreds != null)
            request.setHeader("Proxy-Authorization", _proxyAuthCreds);
        if (_keyFingerprint != null)
            request.setHeader("X-SSLClientCertificate", _keyFingerprint);

        return _response;
    }

    public X509Certificate getCertificate() {
        if (_socket instanceof SSLSocket) {
            SSLSocket sslSock = (SSLSocket) _socket;
            try {
                Certificate[] peerCertificates;
                peerCertificates = sslSock.getSession().getPeerCertificates();
                if (peerCertificates[0] instanceof X509Certificate) {
                    return (X509Certificate) peerCertificates[0];
                }
                _logger.log(Level.WARNING, "Unexpected certificate type {0}",
                        peerCertificates[0].getType());
            } catch (SSLPeerUnverifiedException ex) {
                _logger.log(Level.WARNING, "No peer certificate available", ex);
            }
        }
        return null;
    }

    /**
     * Attempt to connect to the specified host:port combination. No data is
     * sent, if scheme is https, then a handshake will be performed.
     * @param url Host, port and scheme are used.
     */
    public void connect(HttpUrl url) throws IOException {
        connect(url, true);
    }

    private void connect(HttpUrl url, boolean enableSNI) throws IOException {
        if (! invalidSocket(url)) return;
        _logger.fine("Opening a new connection");
        _socket = new Socket();
        _socket.setSoTimeout(_timeout);
        _direct = true;

        // We record where we are connected to, in case we might reuse this socket later
        _host = url.getHost();
        _port = url.getPort();
        boolean ssl = url.getScheme().equalsIgnoreCase("https");

        if (useProxy(url)) {
            if (!ssl) {
                _logger.fine("Connect to " + _httpProxy + ":" + _httpProxyPort);
                _socket.connect(new InetSocketAddress(_httpProxy, _httpProxyPort), _connectTimeout);
                _in = _socket.getInputStream();
                _out = _socket.getOutputStream();
                _direct = false;
            } else {
                _socket.connect(new InetSocketAddress(_httpsProxy, _httpsProxyPort), _connectTimeout);
                _in = _socket.getInputStream();
                _out = _socket.getOutputStream();
                String oldAuthHeader = null;
                String authHeader = constructAuthenticationHeader(null, _proxyAuthCreds);
                String status;
                do {
                    _out.write(("CONNECT " + _host + ":" + _port + " HTTP/1.0\r\n").getBytes());
                    if (authHeader != null) {
                        _out.write(("Proxy-Authorization: " + authHeader + "\r\n").getBytes());
                    }
                    _out.write("\r\n".getBytes());
                    _out.flush();
                    _logger.fine("Sent CONNECT, reading Proxy response");
                    Response response = new Response();
                    response.read(_in);
                    _logger.fine("Got proxy response " + response.getStatusLine());
                    status = response.getStatus();
                    if (status.equals("407")) {
                        response.flushContentStream();
                        oldAuthHeader = authHeader;
                        String[] challenges = response.getHeaders("Proxy-Authenticate");
                        if (_proxyAuthCreds == null && _authenticator != null)
                            _proxyAuthCreds = _authenticator.getProxyCredentials(_httpsProxy, challenges);
                        if (_proxyAuthCreds == null) {
                            _response = response;
                            return;
                        }
                        authHeader = constructAuthenticationHeader(challenges, _proxyAuthCreds);
                        if (authHeader == null || oldAuthHeader != null && oldAuthHeader.equals(authHeader)) {
                            _response = response;
                            return;
                        }
                    }
                } while (status.equals("407") && authHeader != null);
                _logger.fine("HTTPS CONNECT successful");
            }
        } else {
            _logger.fine("Connect to " + _host + ":" + _port );
            _socket.connect(new InetSocketAddress(_host, _port), _connectTimeout);
        }

        if (ssl) {
            // if no fingerprint is specified, get the default one
            if (_keyFingerprint == null)
                _keyFingerprint = _sslContextManager.getDefaultKey();
            _logger.fine("Key fingerprint is " + _keyFingerprint);
            // get the associated context manager
            SSLContext sslContext = _sslContextManager.getSSLContext(_keyFingerprint);
            if (sslContext == null)
                throw new IOException("No SSL cert found matching fingerprint: " + _keyFingerprint);
            // Use the factory to create a secure socket connected to the
            // HTTPS port of the specified web server.
            try {
                SSLSocketFactory factory = sslContext.getSocketFactory();
                // Empty host name avoids the SNI extension from being set
                String hostname = "";
                if (enableSNI) {
                    hostname = _socket.getInetAddress().getHostName();
                }
                SSLSocket sslsocket = (SSLSocket) factory.createSocket(_socket, hostname, _socket.getPort(), true);
                sslsocket.setEnabledProtocols(new String[] {"TLSv1"});
                sslsocket.setUseClientMode(true);
                _socket = sslsocket;
                _socket.setSoTimeout(_timeout);
            } catch (IOException ioe) {
                _logger.severe("Error layering SSL over the existing socket: " + ioe);
                throw ioe;
            }
            try {
                ((SSLSocket) _socket).startHandshake();
            } catch (IOException ioe) {
                // Workaround for Java inability to continue on ignored SNI
                if (enableSNI && ioe.getMessage().equals("handshake alert:  unrecognized_name")) {
                    _logger.fine("Server received saw wrong SNI host, retrying without SNI");
                    connect(url, false);
                    return;
                }
                _logger.severe("Error during SSL handshake: " + ioe);
                throw ioe;
            }
            _logger.fine("Finished negotiating SSL");
        }
        _in = _socket.getInputStream();
        _out = _socket.getOutputStream();
        // reset timeout
        _lastRequestTime = System.currentTimeMillis();
    }

    private boolean useProxy(HttpUrl url) {
        String host = url.getHost();
        boolean ssl = url.getScheme().equalsIgnoreCase("https");

        if (ssl && "".equals(_httpsProxy)) {
            return false;
        } else if (!ssl && "".equals(_httpProxy)) {
            return false;
        } else {
            for (int i=0; i<_noProxy.length; i++) {
                if (_noProxy[i].startsWith(".") && host.endsWith(_noProxy[i])) {
                    return false;
                } else if (_noProxy[i].equals("<local>") && host.indexOf('.') < 0) {
                    return false;
                } else if (host.equals(_noProxy[i])) {
                    return false;
                } else {
                    try {
                        if (host.matches(Glob.globToRE(_noProxy[i]))) return false;
                    } catch (Exception e) {
                        // fail silently
                    }
                }
            }
        }
        return true;
    }

    private boolean invalidSocket(HttpUrl url) {
        if (_host == null || _in == null) return true; // _out may be null if we are testing
        // the right host
        if (url.getHost().equals(_host)) {
            int urlport = url.getPort();
            // and the right port
            if (urlport == _port) {
                // in the last 1 second, it could still be valid
                long now = System.currentTimeMillis();
                if (now - _lastRequestTime > 1000) {
                    _logger.fine("Socket has expired (" + (now - _lastRequestTime) + "), open a new one!");
                    return true;
                } else if (_socket.isOutputShutdown() || _socket.isClosed()) {
                    _logger.fine("Existing socket is closed");
                    return true;
                } else {
                    _logger.fine("Existing socket is valid, reusing it!");
                    return false;
                }
            } else {
                _logger.fine("Previous request was to a different port");
            }
        } else {
            _logger.fine("Previous request was to a different host");
        }
        return true;
    }

    private String constructAuthenticationHeader(String[] challenges, String credentials) {
        /* credentials string looks like:
         * Basic BASE64(username:password)
         * or
         * NTLM BASE64(domain\ username:password)
         */
        // _logger.info("Constructing auth header from " + credentials);
        if (credentials == null)
            return null;
        if (credentials.startsWith("Basic")) {
            return credentials;
        }
        if (challenges != null) {
            for (int i=0; i<challenges.length; i++) {
                _logger.fine("Challenge: " + challenges[i]);
                if (challenges[i].startsWith("NTLM") && credentials.startsWith("NTLM")) {
                    return attemptNegotiation(challenges[i], credentials);
                }
                if (challenges[i].startsWith("Negotiate") && credentials.startsWith("Negotiate")) {
                    _logger.fine("Attempting 'Negotiate' Authentication");
                    return attemptNegotiation(challenges[i], credentials);
                }
                _logger.info("Can't do auth for " + challenges[i]);
            }
      return null;
        }
  /* unknown header: "Authorization: some gibberish" */
        return credentials;
    }

    private String attemptNegotiation(String challenge, String credentials) {
        String authMethod = null;
        String authorization = null;
        if (challenge.startsWith("NTLM")) {
            if (challenge.length() == 4) {
                authMethod = "NTLM";
            }
            if (challenge.indexOf(' ') == 4) {
                authMethod = "NTLM";
                authorization = challenge.substring(5).trim();
            }
        } else if (challenge.startsWith("Negotiate")) {
            if (challenge.length() == 9) {
                authMethod = "Negotiate";
            }
            if (challenge.indexOf(' ') == 9) {
                authMethod = "Negotiate";
                authorization = challenge.substring(10).trim();
            }
        }
        if (authMethod == null) return null;
        NtlmMessage message = null;
        if (authorization != null) {
            try {
                message = new Type2Message(Base64.decode(authorization));
            } catch (IOException ioe) {
                ioe.printStackTrace();
                return null;
            }
        }
        // reconnect();
        int flags = NtlmFlags.NTLMSSP_NEGOTIATE_NTLM2 | NtlmFlags.NTLMSSP_NEGOTIATE_ALWAYS_SIGN | NtlmFlags.NTLMSSP_NEGOTIATE_NTLM | NtlmFlags.NTLMSSP_REQUEST_TARGET | NtlmFlags.NTLMSSP_NEGOTIATE_OEM | NtlmFlags.NTLMSSP_NEGOTIATE_UNICODE;
        if (message == null) {
            message = new Type1Message(flags, null, null);
        } else {
            credentials = credentials.substring(authMethod.length()+1); // strip off the "NTLM " or "Negotiate "
            credentials = new String(Base64.decode(credentials)); // decode the base64
            String domain = credentials.substring(0, credentials.indexOf("\\"));
            String user = credentials.substring(domain.length()+1, credentials.indexOf(":"));
            String password = credentials.substring(domain.length()+user.length()+2);
            Type2Message type2 = (Type2Message) message;
            flags ^= NtlmFlags.NTLMSSP_NEGOTIATE_OEM;
            message = new Type3Message(type2, password, domain, user, null, flags);
        }
        return authMethod + " " + Base64.encode(message.toByteArray());
    }

}
TOP

Related Classes of org.owasp.webscarab.httpclient.URLFetcher

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.