Amazon S3 (s3fs)

Installation

Install dependencies

# yum install wget gcc-c++ fuse fuse-devel openssl-devel libxml2-devel libcurl-devel

Install s3fs

# wget https://s3fs.googlecode.com/files/s3fs-1.74.tar.gz
# tar xf s3fs-1.74.tar.gz
# cd s3fs-1.74
# ./configure
# make
# make install

Set credentials and fstab

# echo 'AWS_ACCESS_KEY:AWS_SECRET_KEY' > /etc/passwd-s3fs
# echo '/usr/local/bin/s3fs#dfstest /mnt/s3 fuse rw,allow_other 0 0' >> /etc/fstab

Benchmarks

Write

[root@ip-172-31-17-241 s3]# for i in 1 8 64 512 4096; do dd if=/dev/zero of=/mnt/s3/$i bs=1024 count=$i; done
1+0 records in
1+0 records out
1024 bytes (1.0 kB) copied, 0.180836 s, 5.7 kB/s
8+0 records in
8+0 records out
8192 bytes (8.2 kB) copied, 0.222971 s, 36.7 kB/s
64+0 records in
64+0 records out
65536 bytes (66 kB) copied, 0.204588 s, 320 kB/s
512+0 records in
512+0 records out
524288 bytes (524 kB) copied, 0.239062 s, 2.2 MB/s
4096+0 records in
4096+0 records out
4194304 bytes (4.2 MB) copied, 0.451609 s, 9.3 MB/s
[root@ip-172-31-17-241 s3]#

Read

[root@ip-172-31-24-167 s3]# for i in 1 8 64 512 4096; do dd if=/mnt/s3/$i of=/dev/null bs=1024 count=$i; done
1+0 records in
1+0 records out
1024 bytes (1.0 kB) copied, 0.188405 s, 5.4 kB/s
8+0 records in
8+0 records out
8192 bytes (8.2 kB) copied, 0.177893 s, 46.1 kB/s
64+0 records in
64+0 records out
65536 bytes (66 kB) copied, 0.182741 s, 359 kB/s
512+0 records in
512+0 records out
524288 bytes (524 kB) copied, 0.184243 s, 2.8 MB/s
4096+0 records in
4096+0 records out
4194304 bytes (4.2 MB) copied, 0.269714 s, 15.6 MB/s
[root@ip-172-31-24-167 s3]#