* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

  private static final Charset UTF8 = Charset.forName("UTF-8");

  // Uncomment to cause the test to hang after the gateway instance is setup.

  // This will allow the gateway instance to be hit directly via some external client.

//  @Test

//  public void hang() throws IOException {

//    System.out.println( "Server on port " + driver.gateway.getAddresses()[0].getPort() );

//    System.out.println();

//    System.in.read();

//  }

  private static Logger log = LoggerFactory.getLogger( GatewayBasicFuncTest.class );

  public static GatewayFuncTestDriver driver = new GatewayFuncTestDriver();

  // Controls the host name to which the gateway dispatch requests.  This may be the name of a sandbox VM

  // or an EC2 instance.  Currently only a single host is supported.

  private static final String TEST_HOST = "vm.local";

  // Specifies if the test requests should go through the gateway or directly to the services.

  // This is frequently used to verify the behavior of the test both with and without the gateway.

  private static final boolean USE_GATEWAY = true;

  // Specifies if the test requests should be sent to mock services or the real services.

  // This is frequently used to verify the behavior of the test both with and without mock services.

  private static final boolean USE_MOCK_SERVICES = true;

  // Specifies if the GATEWAY_HOME created for the test should be deleted when the test suite is complete.

  // This is frequently used during debugging to keep the GATEWAY_HOME around for inspection.

  private static final boolean CLEANUP_TEST = true;

//  private static final boolean USE_MOCK_SERVICES = false;

//  private static final boolean CLEANUP_TEST = false;

  private static int findFreePort() throws IOException {

    ServerSocket socket = new ServerSocket(0);

    int port = socket.getLocalPort();

    socket.close();

    return port;

  }

  /**

   * Creates a deployment of a gateway instance that all test methods will share.  This method also creates a

   * registry of sorts for all of the services that will be used by the test methods.

   * The createTopology method is used to create the topology file that would normally be read from disk.

   * The driver.setupGateway invocation is where the creation of GATEWAY_HOME occurs.

   * @throws Exception Thrown if any failure occurs.

   */

  @BeforeClass

  public static void setupSuite() throws Exception {

    Log.setLog( new NoOpLogger() );

    GatewayTestConfig config = new GatewayTestConfig();

    config.setGatewayPath( "gateway" );

    driver.setResourceBase( GatewayBasicFuncTest.class );

    driver.setupLdap( findFreePort() );

    driver.setupService( "WEBHDFS", "http://" + TEST_HOST + ":50070/webhdfs", "/cluster/webhdfs", USE_MOCK_SERVICES );

    driver.setupService( "DATANODE", "http://" + TEST_HOST + ":50075/webhdfs", "/cluster/webhdfs/data", USE_MOCK_SERVICES );

    driver.setupService( "WEBHCAT", "http://" + TEST_HOST + ":50111/templeton", "/cluster/templeton", USE_MOCK_SERVICES );

    driver.setupService( "OOZIE", "http://" + TEST_HOST + ":11000/oozie", "/cluster/oozie", USE_MOCK_SERVICES );

    driver.setupService( "HIVE", "http://" + TEST_HOST + ":10000", "/cluster/hive", USE_MOCK_SERVICES );

    driver.setupService( "WEBHBASE", "http://" + TEST_HOST + ":60080", "/cluster/hbase", USE_MOCK_SERVICES );

    driver.setupService( "NAMENODE", "hdfs://" + TEST_HOST + ":8020", null, USE_MOCK_SERVICES );

    driver.setupService( "JOBTRACKER", "thrift://" + TEST_HOST + ":8021", null, USE_MOCK_SERVICES );

    driver.setupService( "RESOURCEMANAGER", "http://" + TEST_HOST + ":8088/ws", "/cluster/resourcemanager", USE_MOCK_SERVICES );

    driver.setupGateway( config, "cluster", createTopology(), USE_GATEWAY );

  }

  @AfterClass

  public static void cleanupSuite() throws Exception {

    if( CLEANUP_TEST ) {

      driver.cleanup();

    }

  }

  @After

  public void cleanupTest() {

    driver.reset();

  }

  /**

   * Creates a topology that is deployed to the gateway instance for the test suite.

   * Note that this topology is shared by all of the test methods in this suite.

   * @return A populated XML structure for a topology file.

   */

  private static XMLTag createTopology() {

    XMLTag xml = XMLDoc.newDocument( true )

        .addRoot( "topology" )

          .addTag( "gateway" )

            .addTag( "provider" )

              .addTag( "role" ).addText( "webappsec" )

              .addTag( "name" ).addText( "WebAppSec" )

              .addTag( "enabled" ).addText( "true" )

              .addTag( "param" )

                .addTag( "name" ).addText( "csrf.enabled" )

                .addTag( "value" ).addText( "true" ).gotoParent().gotoParent()

            .addTag( "provider" )

              .addTag( "role" ).addText( "authentication" )

              .addTag( "name" ).addText( "ShiroProvider" )

              .addTag( "enabled" ).addText( "true" )

              .addTag( "param" )

                .addTag( "name" ).addText( "main.ldapRealm" )

                .addTag( "value" ).addText( "org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm" ).gotoParent()

              .addTag( "param" )

                .addTag( "name" ).addText( "main.ldapRealm.userDnTemplate" )

                .addTag( "value" ).addText( "uid={0},ou=people,dc=hadoop,dc=apache,dc=org" ).gotoParent()

              .addTag( "param" )

                .addTag( "name" ).addText( "main.ldapRealm.contextFactory.url" )

                .addTag( "value" ).addText( driver.getLdapUrl() ).gotoParent()

              .addTag( "param" )

                .addTag( "name" ).addText( "main.ldapRealm.contextFactory.authenticationMechanism" )

                .addTag( "value" ).addText( "simple" ).gotoParent()

              .addTag( "param" )

                .addTag( "name" ).addText( "urls./**" )

                .addTag( "value" ).addText( "authcBasic" ).gotoParent().gotoParent()

            .addTag( "provider" )

              .addTag( "role" ).addText( "identity-assertion" )

              .addTag( "enabled" ).addText( "true" )

              .addTag( "name" ).addText( "Default" ).gotoParent()

            .addTag( "provider" )

              .addTag( "role" ).addText( "authorization" )

              .addTag( "enabled" ).addText( "true" )

              .addTag( "name" ).addText( "AclsAuthz" ).gotoParent()

              .addTag( "param" )

                .addTag( "name" ).addText( "webhdfs-acl" )

                .addTag( "value" ).addText( "hdfs;*;*" ).gotoParent()

          .gotoRoot()

          .addTag( "service" )

            .addTag( "role" ).addText( "WEBHDFS" )

            .addTag( "url" ).addText( driver.getRealUrl( "WEBHDFS" ) ).gotoParent()

          .addTag( "service" )

            .addTag( "role" ).addText( "NAMENODE" )

            .addTag( "url" ).addText( driver.getRealUrl( "NAMENODE" ) ).gotoParent()

          .addTag( "service" )

            .addTag( "role" ).addText( "DATANODE" )

            .addTag( "url" ).addText( driver.getRealUrl( "DATANODE" ) ).gotoParent()

          .addTag( "service" )

            .addTag( "role" ).addText( "JOBTRACKER" )

            .addTag( "url" ).addText( driver.getRealUrl( "JOBTRACKER" ) ).gotoParent()

          .addTag( "service" )

            .addTag( "role" ).addText( "WEBHCAT" )

            .addTag( "url" ).addText( driver.getRealUrl( "WEBHCAT" ) ).gotoParent()

          .addTag( "service" )

            .addTag( "role" ).addText( "OOZIE" )

            .addTag( "url" ).addText( driver.getRealUrl( "OOZIE" ) ).gotoParent()

          .addTag( "service" )

            .addTag( "role" ).addText( "HIVE" )

            .addTag( "url" ).addText( driver.getRealUrl( "HIVE" ) ).gotoParent()

          .addTag( "service" )

            .addTag( "role" ).addText( "WEBHBASE" )

            .addTag( "url" ).addText( driver.getRealUrl( "WEBHBASE" ) ).gotoParent()

        .addTag( "service" )

            .addTag( "role" ).addText( "RESOURCEMANAGER" )

            .addTag( "url" ).addText( driver.getRealUrl( "RESOURCEMANAGER" ) ).gotoParent()

        .gotoRoot();

    return xml;

  }

  @Test

  public void testBasicJsonUseCase() throws IOException {

    String root = "/tmp/GatewayBasicFuncTest/testBasicJsonUseCase";

    String username = "hdfs";

    String password = "hdfs-password";

    /* Create a directory.

    curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]"

    The client receives a respond with a boolean JSON object:

    HTTP/1.1 HttpStatus.SC_OK OK

    Content-Type: application/json

    Transfer-Encoding: chunked

    {"boolean": true}

    */

    driver.getMock( "WEBHDFS" )

        .expect()

        .method( "PUT" )

        .pathInfo( "/v1" + root + "/dir" )

        .queryParam( "op", "MKDIRS" )

        .queryParam( "user.name", username )

        .respond()

        .status( HttpStatus.SC_OK )

        .content( driver.getResourceBytes( "webhdfs-success.json" ) )

        .contentType( "application/json" );

    Cookie cookie = given()

        //.log().all()

        .auth().preemptive().basic( username, password )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "MKDIRS" )

        .expect()

        //.log().all()

        .statusCode( HttpStatus.SC_OK )

        .header( "Set-Cookie", containsString( "JSESSIONID" ) )

        .header( "Set-Cookie", containsString( "HttpOnly" ) )

        .contentType( "application/json" )

        .content( "boolean", is( true ) )

        .when().put( driver.getUrl( "WEBHDFS" ) + "/v1" + root + "/dir" ).getDetailedCookie( "JSESSIONID" );

    assertThat( cookie.isSecured(), is( true ) );

    assertThat( cookie.getPath(), is( "/gateway/cluster" ) );

    assertThat( cookie.getValue().length(), greaterThan( 16 ) );

    driver.assertComplete();

  }

  @Test

  public void testBasicOutboundHeaderUseCase() throws IOException {

    String root = "/tmp/GatewayBasicFuncTest/testBasicOutboundHeaderUseCase";

    String username = "hdfs";

    String password = "hdfs-password";

    InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];

    String gatewayHostName = gatewayAddress.getHostName();

    String gatewayAddrName = InetAddress.getByName( gatewayHostName ).getHostAddress();

    driver.getMock( "WEBHDFS" )

        .expect()

        .method( "PUT" )

        .pathInfo( "/v1" + root + "/dir/file" )

        .header( "Host", driver.getRealAddr( "WEBHDFS" ) )

        .queryParam( "op", "CREATE" )

        .queryParam( "user.name", username )

        .respond()

        .status( HttpStatus.SC_TEMPORARY_REDIRECT )

        .header( "Location", driver.getRealUrl( "DATANODE" ) + "/v1" + root + "/dir/file?op=CREATE&user.name=hdfs" );

    Response response = given()

        //.log().all()

        .auth().preemptive().basic( username, password )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "CREATE" )

        .expect()

        //.log().ifError()

        .statusCode( HttpStatus.SC_TEMPORARY_REDIRECT )

        .when().put( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file" );

    String location = response.getHeader( "Location" );

    //System.out.println( location );

    log.debug( "Redirect location: " + response.getHeader( "Location" ) );

    if( driver.isUseGateway() ) {

      MatcherAssert.assertThat( location, anyOf(

          startsWith( "http://" + gatewayHostName + ":" + gatewayAddress.getPort() + "/" ),

          startsWith( "http://" + gatewayAddrName + ":" + gatewayAddress.getPort() + "/" ) ) );

      MatcherAssert.assertThat( location, containsString( "?_=" ) );

    }

    MatcherAssert.assertThat( location, not( containsString( "host=" ) ) );

    MatcherAssert.assertThat( location, not( containsString( "port=" ) ) );

  }

  @Test

  public void testHdfsTildeUseCase() throws IOException {

    String root = "/tmp/GatewayBasicFuncTest/testHdfsTildeUseCase";

    String username = "hdfs";

    String password = "hdfs-password";

    InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];

    // Attempt to delete the test directory in case a previous run failed.

    // Ignore any result.

    // Cleanup anything that might have been leftover because the test failed previously.

    driver.getMock( "WEBHDFS" )

        .expect()

        .method( "DELETE" )

        .from( "testHdfsTildeUseCase" )

        .pathInfo( "/v1/user/hdfs" + root )

        .queryParam( "op", "DELETE" )

        .queryParam( "user.name", username )

        .queryParam( "recursive", "true" )

        .respond()

        .status( HttpStatus.SC_OK );

    given()

        .auth().preemptive().basic( username, password )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "DELETE" )

        .queryParam( "recursive", "true" )

        .expect()

            //.log().all();

        .statusCode( HttpStatus.SC_OK )

        .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1/~" + root + ( driver.isUseGateway() ? "" : "?user.name=" + username ) );

    driver.assertComplete();

    driver.getMock( "WEBHDFS" )

        .expect()

        .method( "PUT" )

        .pathInfo( "/v1/user/hdfs/dir" )

        .queryParam( "op", "MKDIRS" )

        .queryParam( "user.name", username )

        .respond()

        .status( HttpStatus.SC_OK )

        .content( driver.getResourceBytes( "webhdfs-success.json" ) )

        .contentType( "application/json" );

    given()

        //.log().all()

        .auth().preemptive().basic( username, password )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "MKDIRS" )

        .expect()

            //.log().all();

        .statusCode( HttpStatus.SC_OK )

        .contentType( "application/json" )

        .content( "boolean", is( true ) )

        .when().put( driver.getUrl( "WEBHDFS" ) + "/v1/~/dir" );

    driver.assertComplete();

  }

  @Test

  public void testBasicHdfsUseCase() throws IOException {

    String root = "/tmp/GatewayBasicFuncTest/testBasicHdfsUseCase";

    String username = "hdfs";

    String password = "hdfs-password";

    InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];

    String gatewayHostName = gatewayAddress.getHostName();

    String gatewayAddrName = InetAddress.getByName( gatewayHostName ).getHostAddress();

    // Attempt to delete the test directory in case a previous run failed.

    // Ignore any result.

    // Cleanup anything that might have been leftover because the test failed previously.

    driver.getMock( "WEBHDFS" )

        .expect()

        .method( "DELETE" )

        .from( "testBasicHdfsUseCase-1" )

        .pathInfo( "/v1" + root )

        .queryParam( "op", "DELETE" )

        .queryParam( "user.name", username )

        .queryParam( "recursive", "true" )

        .respond()

        .status( HttpStatus.SC_OK );

    given()

        //.log().all()

        .auth().preemptive().basic( username, password )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "DELETE" )

        .queryParam( "recursive", "true" )

        .expect()

        .log().all()

        .statusCode( HttpStatus.SC_OK )

        .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1" + root + ( driver.isUseGateway() ? "" : "?user.name=" + username ) );

    driver.assertComplete();

    /* Create a directory.

    curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]"

    The client receives a respond with a boolean JSON object:

    HTTP/1.1 HttpStatus.SC_OK OK

    Content-Type: application/json

    Transfer-Encoding: chunked

    {"boolean": true}

    */

    driver.getMock( "WEBHDFS" )

        .expect()

        .method( "PUT" )

        .pathInfo( "/v1" + root + "/dir" )

        .queryParam( "op", "MKDIRS" )

        .queryParam( "user.name", username )

        .respond()

        .status( HttpStatus.SC_OK )

        .content( driver.getResourceBytes( "webhdfs-success.json" ) )

        .contentType( "application/json" );

    given()

        //.log().all()

        .auth().preemptive().basic( username, password )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "MKDIRS" )

        .expect()

        //.log().all();

        .statusCode( HttpStatus.SC_OK )

        .contentType( "application/json" )

        .content( "boolean", is( true ) )

        .when().put( driver.getUrl( "WEBHDFS" ) + "/v1" + root + "/dir" );

    driver.assertComplete();

    driver.getMock( "WEBHDFS" )

        .expect()

        .method( "GET" )

        .pathInfo( "/v1" + root )

        .queryParam( "op", "LISTSTATUS" )

        .queryParam( "user.name", username )

        .respond()

        .status( HttpStatus.SC_OK )

        .content( driver.getResourceBytes( "webhdfs-liststatus-test.json" ) )

        .contentType( "application/json" );

    given()

        //.log().all()

        .auth().preemptive().basic( username, password )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "LISTSTATUS" )

        .expect()

        //.log().ifError()

        .statusCode( HttpStatus.SC_OK )

        .content( "FileStatuses.FileStatus[0].pathSuffix", is( "dir" ) )

        .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root );

    driver.assertComplete();

    //NEGATIVE: Test a bad password.

    given()

        //.log().all()

        .auth().preemptive().basic( username, "invalid-password" )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "LISTSTATUS" )

        .expect()

        //.log().ifError()

        .statusCode( HttpStatus.SC_UNAUTHORIZED )

        .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root );

    driver.assertComplete();

    //NEGATIVE: Test a bad user.

    given()

        //.log().all()

        .auth().preemptive().basic( "hdfs-user", "hdfs-password" )

        .header("X-XSRF-Header", "jksdhfkhdsf")

        .queryParam( "op", "LISTSTATUS" )

        .expect()

        //.log().ifError()

        .statusCode( HttpStatus.SC_UNAUTHORIZED )

        .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root );

    driver.assertComplete();

    //NEGATIVE: Test a valid but unauthorized user.

    given()

      //.log().all()

      .auth().preemptive().basic( "mapred-user", "mapred-password" )

      .header("X-XSRF-Header", "jksdhfkhdsf")

      .queryParam( "op", "LISTSTATUS" )

      .expect()

      //.log().ifError()

      .statusCode( HttpStatus.SC_UNAUTHORIZED )

      .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root );

    /* Add a file.

    curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE

                       [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]

                     [&permission=<OCTAL>][&buffersize=<INT>]"

    The expect is redirected to a datanode where the file data is to be written:

    HTTP/1.1 307 TEMPORARY_REDIRECT

    Location: http://<DATANODE>:<PORT>/webhdfs/v1/<PATH>?op=CREATE...

    Content-Length: 0