Merge branch 'feat-s3-multipart-upload' into dev

src/erlaws_s3.erl

@@ -12,6 +12,7 @@
 -export([list_buckets/0, create_bucket/1, create_bucket/2, delete_bucket/1]).
 -export([list_contents/1, list_contents/2, put_object/5, put_file/5, get_object/2]).
 -export([info_object/2, delete_object/2]).
+-export([initiate_mp_upload/4, complete_mp_upload/4, abort_mp_upload/3, upload_part/6]).
 
 %% include record definitions
 -include_lib("xmerl/include/xmerl.hrl").
@@ -195,10 +196,90 @@ put_object(Bucket, Key, Data, ContentType, Metadata) when is_integer(hd(ContentT
 put_object(Bucket, Key, Data, HTTPHeaders, Metadata) ->
     try genericRequest(put, Bucket, Key, [], Metadata, HTTPHeaders, Data) of
 	{ok, Headers, _Body} -> 
-	    RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
-			{value, {_, ReqId}, _} -> ReqId;
-			_ -> "" end,
-		{ok, #s3_object_info{key=Key, size=size(Data)}, {requestId, RequestId}}
+	    {ok,
+	     #s3_object_info{key=Key, size=iolist_size(Data),
+			     etag=proplists:get_value("etag", Headers)},
+	     {requestId, proplists:get_value(?S3_REQ_ID_HEADER, Headers, "")}}
+    catch
+	throw:{error, Descr} ->
+	    {error, Descr}
+    end.
+
+%% @doc
+%% Initiates multipart upload.
+%% @end
+-spec initiate_mp_upload(Bucket::string(), Key::string(),
+			 HTTPHeaders::[{string(), string()}],
+			 Metadata::[{string(), string()}]) ->
+			    {ok, UploadId::string(), ReqId::string()}.
+initiate_mp_upload(Bucket, Key, HTTPHeaders, Metadata) ->
+    try genericRequest(post, Bucket, Key, [{"uploads", ""}],
+		       Metadata, HTTPHeaders, <<>>) of
+	{ok, Headers, Body} ->
+	    {XmlDoc, _Rest} = xmerl_scan:string(binary_to_list(Body)),
+	    [_XBucket|_] = xmerl_xpath:string("/InitiateMultipartUploadResult/Bucket/text()", XmlDoc),
+	    [_XKey|_] = xmerl_xpath:string("/InitiateMultipartUploadResult/Key/text()", XmlDoc),
+	    [XUploadId|_] = xmerl_xpath:string("/InitiateMultipartUploadResult/UploadId/text()", XmlDoc),
+	    {ok, XUploadId#xmlText.value, proplists:get_value(?S3_REQ_ID_HEADER, Headers, "")}
+    catch
+	throw:{error, Descr} ->
+	    {error, Descr}
+    end.
+
+%% @doc
+%% Completes multipart upload.
+%% @end
+-spec complete_mp_upload(Bucket::string(), Key::string(), UploadId::string(),
+			 [{PartNum::integer(), ETag::string()}]) -> ok.
+complete_mp_upload(Bucket, Key, UploadId, Parts) ->
+    F = fun({PartNum, ETag}) ->
+		{'Part', [],
+		 [
+		  {'PartNumber', [integer_to_list(PartNum)]},
+		  {'ETag', [ETag]}
+		 ]
+		}
+	end,
+    Req = {'CompleteMultipartUpload', [], [F(X) || X <- Parts]},
+    XMLReqBody = xmerl:export_simple([Req], xmerl_xml),
+    try genericRequest(post, Bucket, Key,
+		       [{"uploadId", UploadId}], [], [], XMLReqBody) of
+	{ok, _Headers, _Body} -> ok
+    catch
+	throw:{error, Descr} ->
+	    {error, Descr}
+    end.
+
+%% @doc
+%% Aborts multipart upload.
+%% @end
+-spec abort_mp_upload(Bucket::string(), Key::string(), UploadId::string()) ->
+			      ok.
+abort_mp_upload(Bucket, Key, UploadId) ->
+    try genericRequest(delete, Bucket, Key,
+		       [{"uploadId", UploadId}], [], [], <<>>) of
+	{ok, _Headers, _Body} -> ok
+    catch
+	throw:{error, Descr} ->
+	    {error, Descr}
+    end.
+
+%% @doc
+%% Uploads a part in multipart upload.
+%% @end
+-spec upload_part(Bucket::string(), Key::string(), PartNum::integer(),
+		  UploadId::string(), Data::binary(),
+		  HTTPHeaders::[{string(), string()}]) -> ok.
+upload_part(Bucket, Key, PartNum, UploadId, Data, HTTPHeaders) ->
+    try genericRequest(put, Bucket, Key,
+		       [{"partNumber", integer_to_list(PartNum)},
+			{"uploadId", UploadId}],
+		       [], HTTPHeaders, Data) of
+	{ok, Headers, _Body} ->
+	    {ok,
+	     #s3_object_info{key=Key, size=iolist_size(Data),
+			     etag=proplists:get_value("etag", Headers)},
+	     {requestId, proplists:get_value(?S3_REQ_ID_HEADER, Headers, "")}}
     catch
 	throw:{error, Descr} ->
 	    {error, Descr}
@@ -354,10 +435,10 @@ buildContentHeaders(Contents) when is_integer(Contents) ->
     [{"Content-Length", integer_to_list(Contents)}];
 % Detect gzip header and put appropriate Content-Encoding. Questionable?..
 buildContentHeaders(<<16#1f, 16#8b, _/binary>> = Contents) -> 
-    [{"Content-Length", integer_to_list(size(Contents))},
+    [{"Content-Length", integer_to_list(iolist_size(Contents))},
      {"Content-Encoding", "gzip"}];
 buildContentHeaders(Contents) -> 
-    [{"Content-Length", integer_to_list(size(Contents))}].
+    [{"Content-Length", integer_to_list(iolist_size(Contents))}].
 
 buildMetadataHeaders(Metadata) ->
     lists:foldl(fun({Key, Value}, Acc) ->
@@ -391,8 +472,8 @@ genericRequest( Method, Bucket, Path, QueryParams, Metadata,
     MethodString = string:to_upper( atom_to_list(Method) ),
     Url = buildUrl(Bucket,Path,QueryParams),
 
-    ContentMD5 = case Body of
-		     <<>> -> "";
+    ContentMD5 = case iolist_size(Body) of
+		     0 -> "";
 		     _ -> binary_to_list(base64:encode(erlang:md5(Body)))
 		 end,
 
@@ -411,7 +492,9 @@ genericRequest( Method, Bucket, Path, QueryParams, Metadata,
 
     Signature = sign(SecretAccessKey,
 		     stringToSign(MethodString, ContentMD5, ContentType, Date,
-				   Bucket, Path, Headers )),
+				  Bucket,
+				  Path ++ erlaws_util:queryParams(QueryParams),
+				  Headers )),
 
     FinalHeaders = [ {"Authorization","AWS " ++ AccessKey ++ ":" ++ Signature },
 		     {"Host", buildHost(Bucket) },
@@ -423,6 +506,7 @@ genericRequest( Method, Bucket, Path, QueryParams, Metadata,
  		  get -> { Url, FinalHeaders };
 		  head -> { Url, FinalHeaders };
  		  put -> { Url, FinalHeaders, ContentType, Body };
+ 		  post -> { Url, FinalHeaders, ContentType, Body };
  		  delete -> { Url, FinalHeaders }
  	      end,
 

src/erlaws_util.erl

@@ -15,8 +15,17 @@ mkEnumeration(Values, Separator) ->
 
 queryParams( [] ) -> "";
 queryParams( ParamList ) -> 
-    "?" ++ mkEnumeration([[url_encode(Param), "=", url_encode(Value)]
-			  || {Param, Value} <- ParamList], "&" ).
+    L0 = lists:foldl(fun({P, V}, Acc) ->
+			     case V of
+				 "" ->
+				     [url_encode(P)|Acc];
+				 _ ->
+				     [url_encode(P) ++ "=" ++ url_encode(V)|Acc]
+			     end
+		     end,
+		     [],
+		     lists:keysort(1, ParamList)),
+    "?" ++ mkEnumeration(lists:reverse(L0), "&"). 
 
 %% The following code is taken from the ibrowse Http client
 %% library.