diff --git a/plugins/rest/core/src/main/java/org/pentaho/di/trans/steps/rest/Rest.java b/plugins/rest/core/src/main/java/org/pentaho/di/trans/steps/rest/Rest.java index ff75c2ca36e3..09b8e6adaf0f 100644 --- a/plugins/rest/core/src/main/java/org/pentaho/di/trans/steps/rest/Rest.java +++ b/plugins/rest/core/src/main/java/org/pentaho/di/trans/steps/rest/Rest.java @@ -76,6 +76,22 @@ public Rest( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, } protected Object[] callRest( Object[] rowData ) throws KettleException { + + Client client = null; + try { + client = getClient( rowData ); + WebTarget webResource = buildRequest( client, rowData ); + return invokeRequest( webResource, rowData ); + } catch ( Exception e ) { + throw new KettleException( BaseMessages.getString( PKG, "Rest.Error.CanNotReadURL", data.realUrl ), e ); + } finally { + if ( client != null ) { + client.close(); + } + } + } + + protected Client getClient( Object[] rowData ) throws KettleException { // get dynamic url ? if ( meta.isUrlInField() ) { data.realUrl = data.inputRowMeta.getString( rowData, data.indexOfUrlField ); @@ -87,195 +103,196 @@ protected Object[] callRest( Object[] rowData ) throws KettleException { throw new KettleException( BaseMessages.getString( PKG, "Rest.Error.MethodMissing" ) ); } } - WebTarget webResource = null; Client client = null; - Object[] newRow = null; - if ( rowData != null ) { - newRow = rowData.clone(); + if ( isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "Rest.Log.ConnectingToURL", data.realUrl ) ); } - try { - if ( isDetailed() ) { - logDetailed( BaseMessages.getString( PKG, "Rest.Log.ConnectingToURL", data.realUrl ) ); - } - // // Register a custom StringMessageBodyWriter to solve PDI-17423 - ClientBuilder clientBuilder = ClientBuilder.newBuilder(); - clientBuilder - .withConfig( data.config ) - .property( HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true ); - if ( meta.isIgnoreSsl() || !Utils.isEmpty( data.trustStoreFile ) ) { - clientBuilder.sslContext( data.sslContext ); - clientBuilder.hostnameVerifier( ( s1, s2 ) -> true ); - } - client = clientBuilder.build(); - if ( data.basicAuthentication != null ) { - client.register( data.basicAuthentication ); - } - // create a WebResource object, which encapsulates a web resource for the client - webResource = client.target( data.realUrl ); - - // used for calculating the responseTime - long startTime = System.currentTimeMillis(); - - if ( data.useMatrixParams ) { - // Add matrix parameters - UriBuilder builder = webResource.getUriBuilder(); - for ( int i = 0; i < data.nrMatrixParams; i++ ) { - String value = data.inputRowMeta.getString( rowData, data.indexOfMatrixParamFields[ i ] ); - if ( isDebug() ) { - logDebug( + // // Register a custom StringMessageBodyWriter to solve PDI-17423 + ClientBuilder clientBuilder = ClientBuilder.newBuilder(); + clientBuilder + .withConfig( data.config ) + .property( HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true ); + if ( meta.isIgnoreSsl() || !Utils.isEmpty( data.trustStoreFile ) ) { + clientBuilder.sslContext( data.sslContext ); + clientBuilder.hostnameVerifier( ( s1, s2 ) -> true ); + } + client = clientBuilder.build(); + if ( data.basicAuthentication != null ) { + client.register( data.basicAuthentication ); + } + return client; + } + + protected WebTarget buildRequest( Client client, Object[] rowData ) throws KettleException { + WebTarget webResource = null; + // create a WebResource object, which encapsulates a web resource for the client + webResource = client.target( data.realUrl ); + + if ( data.useMatrixParams ) { + // Add matrix parameters + UriBuilder builder = webResource.getUriBuilder(); + for ( int i = 0; i < data.nrMatrixParams; i++ ) { + String value = data.inputRowMeta.getString( rowData, data.indexOfMatrixParamFields[ i ] ); + if ( isDebug() ) { + logDebug( BaseMessages.getString( PKG, "Rest.Log.matrixParameterValue", data.matrixParamNames[ i ], value ) ); - } - builder = builder.matrixParam( data.matrixParamNames[ i ], - UriComponent.encode( value, UriComponent.Type.QUERY_PARAM ) ); } - webResource = client.target( builder.build() ); + builder = builder.matrixParam( data.matrixParamNames[ i ], + UriComponent.encode( value, UriComponent.Type.QUERY_PARAM_SPACE_ENCODED ) ); } + webResource = client.target( builder.build() ); + } - if ( data.useParams ) { - // Add query parameters - for ( int i = 0; i < data.nrParams; i++ ) { - String value = data.inputRowMeta.getString( rowData, data.indexOfParamFields[ i ] ); - if ( isDebug() ) { - logDebug( BaseMessages.getString( PKG, "Rest.Log.queryParameterValue", data.paramNames[ i ], value ) ); - } - webResource = webResource.queryParam( data.paramNames[ i ], UriComponent.encode( value, UriComponent.Type.QUERY_PARAM ) ); - } - } - if ( isDebug() ) { - logDebug( BaseMessages.getString( PKG, "Rest.Log.ConnectingToURL", webResource.getUri() ) ); - } - Invocation.Builder invocationBuilder = webResource.request(); - String contentType = null; // media type override, if not null - if ( data.useHeaders ) { - // Add headers - for ( int i = 0; i < data.nrheader; i++ ) { - String value = data.inputRowMeta.getString( rowData, data.indexOfHeaderFields[ i ] ); - - // unsure if an already set header will be returned to builder - invocationBuilder.header( data.headerNames[ i ], value ); - if ( "Content-Type".equals( data.headerNames[ i ] ) ) { - contentType = value; - } - if ( isDebug() ) { - logDebug( BaseMessages.getString( PKG, "Rest.Log.HeaderValue", data.headerNames[ i ], value ) ); - } + if ( data.useParams ) { + // Add query parameters + for ( int i = 0; i < data.nrParams; i++ ) { + String value = data.inputRowMeta.getString( rowData, data.indexOfParamFields[ i ] ); + if ( isDebug() ) { + logDebug( BaseMessages.getString( PKG, "Rest.Log.queryParameterValue", data.paramNames[ i ], value ) ); } + webResource = webResource.queryParam( data.paramNames[ i ], + UriComponent.encode( value, UriComponent.Type.QUERY_PARAM_SPACE_ENCODED ) ); } + } + if ( isDebug() ) { + logDebug( BaseMessages.getString( PKG, "Rest.Log.ConnectingToURL", webResource.getUri() ) ); + } + return webResource; + } + + private Object[] invokeRequest( WebTarget webResource, Object[] rowData ) throws KettleException { + Object[] newRow = null; + if ( rowData != null ) { + newRow = rowData.clone(); + } + + // used for calculating the responseTime + long startTime = System.currentTimeMillis(); - Response response; - String entityString = ""; - if ( data.useBody ) { - // Set Http request entity - entityString = Const.NVL( data.inputRowMeta.getString( rowData, data.indexOfBodyField ), "" ); + Invocation.Builder invocationBuilder = webResource.request(); + + String contentType = null; // media type override, if not null + if ( data.useHeaders ) { + // Add headers + for ( int i = 0; i < data.nrheader; i++ ) { + String value = data.inputRowMeta.getString( rowData, data.indexOfHeaderFields[ i ] ); + + // unsure if an already set header will be returned to builder + invocationBuilder.header( data.headerNames[ i ], value ); + if ( "Content-Type".equals( data.headerNames[ i ] ) ) { + contentType = value; + } if ( isDebug() ) { - logDebug( BaseMessages.getString( PKG, "Rest.Log.BodyValue", entityString ) ); + logDebug( BaseMessages.getString( PKG, "Rest.Log.HeaderValue", data.headerNames[ i ], value ) ); } } - try { - if ( data.method.equals( RestMeta.HTTP_METHOD_GET ) ) { - response = invocationBuilder.get( Response.class ); - } else if ( data.method.equals( RestMeta.HTTP_METHOD_POST ) ) { - if ( null != contentType ) { - response = invocationBuilder.post( Entity.entity( entityString, contentType ) ); - } else { - // response = builder.type( data.mediaType ).post( ClientResponse.class, entityString ); - response = invocationBuilder.post( Entity.entity( entityString, data.mediaType ) ); - } - } else if ( data.method.equals( RestMeta.HTTP_METHOD_PUT ) ) { - if ( null != contentType ) { - response = invocationBuilder.put( Entity.entity( entityString, contentType ) ); - } else { - response = invocationBuilder.put( Entity.entity( entityString, data.mediaType ) ); - } - } else if ( data.method.equals( RestMeta.HTTP_METHOD_DELETE ) ) { - response = invocationBuilder.delete(); - } else if ( data.method.equals( RestMeta.HTTP_METHOD_HEAD ) ) { - response = invocationBuilder.head(); - } else if ( data.method.equals( RestMeta.HTTP_METHOD_OPTIONS ) ) { - response = invocationBuilder.options(); - } else if ( data.method.equals( RestMeta.HTTP_METHOD_PATCH ) ) { - if ( null != contentType ) { - response = - invocationBuilder.method( + } + + Response response; + String entityString = ""; + if ( data.useBody ) { + // Set Http request entity + entityString = Const.NVL( data.inputRowMeta.getString( rowData, data.indexOfBodyField ), "" ); + if ( isDebug() ) { + logDebug( BaseMessages.getString( PKG, "Rest.Log.BodyValue", entityString ) ); + } + } + boolean debug = true; + try { + if ( data.method.equals( RestMeta.HTTP_METHOD_GET ) ) { + response = invocationBuilder.get( Response.class ); + } else if ( data.method.equals( RestMeta.HTTP_METHOD_POST ) ) { + if ( null != contentType ) { + response = invocationBuilder.post( Entity.entity( entityString, contentType ) ); + } else { + // response = builder.type( data.mediaType ).post( ClientResponse.class, entityString ); + response = invocationBuilder.post( Entity.entity( entityString, data.mediaType ) ); + } + } else if ( data.method.equals( RestMeta.HTTP_METHOD_PUT ) ) { + if ( null != contentType ) { + response = invocationBuilder.put( Entity.entity( entityString, contentType ) ); + } else { + response = invocationBuilder.put( Entity.entity( entityString, data.mediaType ) ); + } + } else if ( data.method.equals( RestMeta.HTTP_METHOD_DELETE ) ) { + response = invocationBuilder.delete(); + } else if ( data.method.equals( RestMeta.HTTP_METHOD_HEAD ) ) { + response = invocationBuilder.head(); + } else if ( data.method.equals( RestMeta.HTTP_METHOD_OPTIONS ) ) { + response = invocationBuilder.options(); + } else if ( data.method.equals( RestMeta.HTTP_METHOD_PATCH ) ) { + if ( null != contentType ) { + response = + invocationBuilder.method( RestMeta.HTTP_METHOD_PATCH, Entity.entity( entityString, contentType ) ); - } else { - response = - invocationBuilder.method( - RestMeta.HTTP_METHOD_PATCH, Entity.entity( entityString, data.mediaType ) ); - } } else { - throw new KettleException( BaseMessages.getString( PKG, "Rest.Error.UnknownMethod", data.method ) ); + response = + invocationBuilder.method( + RestMeta.HTTP_METHOD_PATCH, Entity.entity( entityString, data.mediaType ) ); } - } catch ( Exception e ) { - throw new KettleException( "Request could not be processed", e ); + } else { + throw new KettleException( BaseMessages.getString( PKG, "Rest.Error.UnknownMethod", data.method ) ); } - // Get response time - long responseTime = System.currentTimeMillis() - startTime; - if ( isDetailed() ) { - logDetailed( + } catch ( Exception e ) { + throw new KettleException( "Request could not be processed", e ); + } + // Get response time + long responseTime = System.currentTimeMillis() - startTime; + if ( isDetailed() ) { + logDetailed( BaseMessages.getString( PKG, "Rest.Log.ResponseTime", String.valueOf( responseTime ), data.realUrl ) ); - } + } - // Get status - int status = response.getStatus(); - // Display status code - if ( isDebug() ) { - logDebug( BaseMessages.getString( PKG, "Rest.Log.ResponseCode", "" + status ) ); - } + // Get status + int status = response.getStatus(); + // Display status code + if ( isDebug() ) { + logDebug( BaseMessages.getString( PKG, "Rest.Log.ResponseCode", "" + status ) ); + } - // Get Response - String body; - String headerString = null; - try { - body = response.readEntity( String.class ); - } catch ( Exception ex ) { - body = ""; - } - // get Header - MultivaluedMap headers = searchForHeaders( response ); - JSONObject json = new JSONObject(); - for ( java.util.Map.Entry> entry : headers.entrySet() ) { - String name = entry.getKey(); - List value = entry.getValue(); - if ( value.size() > 1 ) { - json.put( name, value ); - } else { - json.put( name, value.get( 0 ) ); - } - } - headerString = json.toJSONString(); - // for output - int returnFieldsOffset = data.inputRowMeta.size(); - // add response to output - if ( !Utils.isEmpty( data.resultFieldName ) ) { - newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, body ); - returnFieldsOffset++; + // Get Response + String body; + String headerString = null; + try { + body = response.readEntity( String.class ); + } catch ( Exception ex ) { + body = ""; + } + // get Header + MultivaluedMap headers = searchForHeaders( response ); + JSONObject json = new JSONObject(); + for ( java.util.Map.Entry> entry : headers.entrySet() ) { + String name = entry.getKey(); + List value = entry.getValue(); + if ( value.size() > 1 ) { + json.put( name, value ); + } else { + json.put( name, value.get( 0 ) ); } + } + headerString = json.toJSONString(); + // for output + int returnFieldsOffset = data.inputRowMeta.size(); + // add response to output + if ( !Utils.isEmpty( data.resultFieldName ) ) { + newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, body ); + returnFieldsOffset++; + } - // add status to output - if ( !Utils.isEmpty( data.resultCodeFieldName ) ) { - newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, new Long( status ) ); - returnFieldsOffset++; - } + // add status to output + if ( !Utils.isEmpty( data.resultCodeFieldName ) ) { + newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, new Long( status ) ); + returnFieldsOffset++; + } - // add response time to output - if ( !Utils.isEmpty( data.resultResponseFieldName ) ) { - newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, new Long( responseTime ) ); - returnFieldsOffset++; - } - // add response header to output - if ( !Utils.isEmpty( data.resultHeaderFieldName ) ) { - newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString ); - } - } catch ( Exception e ) { - throw new KettleException( BaseMessages.getString( PKG, "Rest.Error.CanNotReadURL", data.realUrl ), e ); - } finally { - if ( webResource != null ) { - webResource = null; - } - if ( client != null ) { - client.close(); - } + // add response time to output + if ( !Utils.isEmpty( data.resultResponseFieldName ) ) { + newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, new Long( responseTime ) ); + returnFieldsOffset++; + } + // add response header to output + if ( !Utils.isEmpty( data.resultHeaderFieldName ) ) { + newRow = RowDataUtil.addValueData( newRow, returnFieldsOffset, headerString ); } return newRow; } diff --git a/plugins/rest/core/src/test/java/org/pentaho/di/trans/steps/rest/RestTest.java b/plugins/rest/core/src/test/java/org/pentaho/di/trans/steps/rest/RestTest.java index e9d88886d579..2dee38b38f1d 100644 --- a/plugins/rest/core/src/test/java/org/pentaho/di/trans/steps/rest/RestTest.java +++ b/plugins/rest/core/src/test/java/org/pentaho/di/trans/steps/rest/RestTest.java @@ -36,6 +36,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @RunWith( MockitoJUnitRunner.StrictStubs.class ) @@ -72,6 +73,8 @@ public void testCallEndpointWithGetVerb() throws KettleException { Rest rest = mock( Rest.class ); doCallRealMethod().when( rest ).callRest( any() ); + doCallRealMethod().when( rest ).getClient( any() ); + doCallRealMethod().when( rest ).buildRequest( any(), any() ); doCallRealMethod().when( rest ).searchForHeaders( any() ); ReflectionTestUtils.setField( rest, "meta", meta ); @@ -93,19 +96,9 @@ public void testCallEndpointWithGetVerb() throws KettleException { @Test public void testEncodingParams() throws KettleException { - Object[] params = new Object[1]; + Object[] params = new Object[2]; params[0] = "{a:{[val1]}}"; - - Invocation.Builder builder = mock( Invocation.Builder.class ); - - WebTarget resource = mock( WebTarget.class ); - lenient().doReturn( builder ).when( resource ).request(); - - Client client = mock( Client.class ); - lenient().doReturn( resource ).when( client ).target( anyString() ); - - ClientBuilder clientBuilder = mock( ClientBuilder.class ); - lenient().when( clientBuilder.build() ).thenReturn( client ); + params[1] = "string with spaces"; RestMeta meta = mock( RestMeta.class ); doReturn( false ).when( meta ).isUrlInField(); @@ -113,6 +106,7 @@ public void testEncodingParams() throws KettleException { RowMetaInterface rmi = mock( RowMetaInterface.class ); doReturn( params[0] ).when( rmi ).getString( params, 0 ); + doReturn( params[1] ).when( rmi ).getString( params, 1 ); RestData data = mock( RestData.class ); data.method = RestMeta.HTTP_METHOD_POST; @@ -123,28 +117,24 @@ public void testEncodingParams() throws KettleException { data.resultHeaderFieldName = "headers"; data.realUrl = "http://localhost:8080/pentaho"; data.useParams = true; - data.nrParams = 1; + data.nrParams = 2; data.mediaType = MediaType.APPLICATION_JSON_TYPE; // Add one index to this array - data.indexOfParamFields = new int[] {0}; - data.paramNames = new String[] {"param1"}; + data.indexOfParamFields = new int[] {0, 1}; + data.paramNames = new String[] {"param1", "param2"}; Rest rest = mock( Rest.class ); - doCallRealMethod().when( rest ).callRest( any() ); + doCallRealMethod().when( rest ).getClient( any() ); + doCallRealMethod().when( rest ).buildRequest( any(), any() ); ReflectionTestUtils.setField( rest, "meta", meta ); ReflectionTestUtils.setField( rest, "data", data ); - try { - Object[] output = rest.callRest( params ); - } catch ( Exception exception ) { - // Ignore the ConnectExcepion which is expected as rest call to localhost:8080 will fail in unit test - // IllegalStateException is throws when the parameters are not encoded - if ( exception.getCause() instanceof IllegalStateException ) { - Assert.fail(); - } - } + Client client = rest.getClient( params ); + WebTarget webResource = rest.buildRequest( client, params ); + String expected = "http://localhost:8080/pentaho?param1=%7Ba%3A%7B%5Bval1%5D%7D%7D¶m2=string%20with%20spaces"; + assertEquals( expected, webResource.getUri().toString() ); } /** @@ -175,20 +165,24 @@ public void testPutWithEmptyBody() throws KettleException { data.method = RestMeta.HTTP_METHOD_PUT; data.config = new ClientConfig(); data.inputRowMeta = rmi; - data.realUrl = "http://localhost:8080/pentaho"; + // should be non-routable so we can consistetly not connect + data.realUrl = "http://192.0.2.1:8080/pentaho"; data.mediaType = MediaType.TEXT_PLAIN_TYPE; data.useBody = true; // do not set data.indexOfBodyField Rest rest = mock( Rest.class ); doCallRealMethod().when( rest ).callRest( any() ); + doCallRealMethod().when( rest ).getClient( any() ); + doCallRealMethod().when( rest ).buildRequest( any(), any() ); ReflectionTestUtils.setField( rest, "meta", meta ); ReflectionTestUtils.setField( rest, "data", data ); try { rest.callRest( new Object[] { 0 } ); - } catch ( Exception exception ) { + Assert.fail( "Expected an exception" ); + } catch ( KettleException exception ) { // Ignore the ConnectException which is expected as rest call to localhost:8080 will fail in unit test // IllegalStateException is throws when the body is null if ( exception.getCause().getCause() instanceof IllegalStateException ) {