mogadm

#!/usr/bin/perl
# vim:ts=4 sw=4 ft=perl et:

use strict;
use warnings;
use Getopt::Long;
use LWP::Simple;  # FIXME: use of this makes 'mog check' hang too long when multiple things down
use Socket;

my @topcmds = qw(check stats host device domain class slave fsck rebalance settings);
my $usage = {
    check => {
        des => "Check the state of the MogileFS world.",
    },
    stats => {
        des => "Show MogileFS system statistics.  (DEPRECATED: use mogstats instead)",
    },
    settings => {
        des => "Change/list server settings.",
        subcmd => {
            list => {
                des => "List all server settings",
            },
            set => {
                args => "<key> <value>",
                des => "Set server setting 'key' to 'value'.",
            },
        },
    },
    host => {
        des => "Add/modify hosts.",
        subcmd => {
            list => {
                des => "List all hosts.",
            },
            add => {
                des => "Add a host to MogileFS.",
                args => "<hostname> [opts]",
                opts => {
                    "<hostname>"  => "Hostname of machine",
                    "--status=s"  => "One of {alive,down}.  Default 'down'.",
                    "--ip=s"      => "IP address of machine.",
                    "--port=i"    => "HTTP port of mogstored",
                    "--getport=i" => "Alternate HTTP port serving readonly traffic",
                    "--altip=s"   => "Alternate IP that is machine is reachable from",
                    "--altmask=s" => "Netmask which, when matches client, uses alt IP",
                },
            },
            modify => {
                des => "Modify a host's properties.",
                args => "<hostname> [opts]",
                opts => {
                    "<hostname>" => "Host name.",
                    "--status=s" => "One of {alive,down}.",
                    "--ip=s"     => "IP address of machine.",
                    "--port=i"   => "HTTP port of mogstored",
                    "--getport=i" => "Alternate HTTP port serving readonly traffic",
                    "--altip=s"   => "Alternate IP that is machine is reachable from",
                    "--altmask=s" => "Netmask which, when matches client, uses alt IP",
                },
            },
            mark => {
                des => "Change the status of a host.  (equivalent to 'modify --status')",
                args => "<hostname> <status>",
                opts => {
                    "<hostname>" => "Host name to bring up or down.",
                    "<status>"   => "One of {alive,down}.",
                }
            },
            delete => {
                des => "Delete a host.",
                args => "<hostname>",
                opts => {
                    "<hostname>" => "Host name to delete.",
                },
            },
        },
    },
    device => {
        des => "Add/modify devices.",
        subcmd => {
            list => {
                des => "List all devices, for each host.",
                args => "[opts]",
                opts => {
                    "--all"  => "Include dead devices in list.",
                },
            },
            summary => {
                des => "List the summary of devices, for each host.",
                args => "[opts]",
                opts => {
                    "--status=s"  => "Devices of status A. Defaults to 'alive,readonly'",
                },
            },
            add => {
                des  => "Add a device to a host.",
                args => "<hostname> <devid> [opts]",
                opts => {
                    "<hostname>" => "Hostname to add a device",
                    "<devid>"    => "Numeric devid.  Never reuse these.",
                    "--status=s" => "One of 'alive' or 'down'.  Defaults to 'alive'.",
                },
            },
            mark => {
                des  => "Mark a device as {alive,dead,down,drain,readonly}",
                args => "<hostname> <devid> <status>",
                opts => {
                    "<hostname>" => "Hostname of device",
                    "<devid>"    => "Numeric devid to modify.",
                    "<status>"   => "One of {alive,dead,down,drain,readonly}",
                },
            },
            modify => {
                des => "Modify a device's properties.",
                args => "<hostname> <devid> [opts]",
                opts => {
                    "<hostname>" => "Hostname of device",
                    "<devid>"    => "Numeric devid to modify.",
                    "--status=s" => "One of {alive,dead,down,drain,readonly}",
                    "--weight=i" => "Positive numeric weight for device",
                },
            },
        },
    },
    domain => {
        des => "Add/modify domains (namespaces)",
        subcmd => {
            list => {
                des => "List all hosts.",
            },
            add => {
                des => "Add a domain (namespace)",
                args => "<domain>",
                opts => {
                    "<domain>" => "Domain (namespace) to add.",
                },
            },
            delete => {
                des => "Delete a domain.",
                args => "<domain>",
                opts => {
                    "<domain>" => "Domain (namespace) to add.",
                },
            },
        },
    },
    class => {
        des => "Add/modify file classes.",
        subcmd => {
            list => {
                des => "List all classes, for each domain.",
            },
            add => {
                des => "Add a file class to a domain.",
                args => "<domain> <class> [opts]",
                opts => {
                    "<domain>" => "Domain to add class to.",
                    "<class>"  => "Name of class to add.",
                    "--mindevcount=i" => "Minimum number of replicas.",
                    "--replpolicy=s" => "Replication policy string.",
                },
            },
            modify => {
                des => "Modify properties of a file class.",
                args => "<domain> <class> [opts]",
                opts => {
                    "<domain>" => "Domain to add class to.",
                    "<class>"  => "Name of class to add.",
                    "--mindevcount=i" => "Minimum number of replicas.",
                    "--replpolicy=s" => "Replication policy string.",
                },
            },
            delete => {
                des => "Delete a file class from a domain.",
                args => "<domain> <class>",
                opts => {
                    "<domain>" => "Domain of class to delete.",
                    "<class>"  => "Class to delete.",
                },

            },
        },
    },
    slave => {
        des => 'Manipulate slave database information in a running mogilefsd.',
        subcmd => {
            list => {
                des => 'List current store slave nodes.',
            },
            add => {
                des => 'Add a slave node for store usage',
                args => '<slave_key> [opts]',
                opts => {
                    '--dsn=s' => "DBI DSN specifying what database to connect to.",
                    '--username=s' => "DBI username for connecting.",
                    '--password=s' => "DBI password for connecting.",
                },
            },
            modify => {
                des => 'Modify a slave node for store usage',
                args => '<slave_key> [opts]',
                opts => {
                    '--dsn=s' => "DBI DSN specifying what database to connect to.",
                    '--username=s' => "DBI username for connecting.",
                    '--password=s' => "DBI password for connecting.",
                },
            },
            delete => {
                des => 'Delete a slave node for store usage',
                args => '<slave_key>',
            },
        },
    },
    rebalance => {
        des => "Control file rebalancing operations.",
        subcmd => {
            start => {
                des => 'Start a rebalance job',
            },
            stop => {
                des => 'Stop a rebalance job',
            },
            status => {
                des => 'Show status of current rebalance job',
            },
            settings => {
                des => 'Display rebalance settings',
            },
            test => {
                des => 'Show what devices the current policy would match',
            },
            policy => {
                des => 'Add or adjust the current policy',
                args => '[opts]',
                opts => {
                    '--options=s' => "Policy string (see docs/wiki for details)",
                },
            },
        },
    },
    fsck => {
        des => "Control a background filesystem check operation.",
        subcmd => {
            start => {
                des => 'Start (or resume) background fsck',
            },
            stop => {
                des => 'Stop (pause) background fsck',
            },
            status => {
                des => 'Show fsck status',
            },
            reset => {
                des => 'Reset fsck position back to the beginning',
                args => '[opts]',
                opts => {
                    '--policy-only' => "Check repl policy (assumed locations); don't stat storage nodes",
                    '--startpos=i'  => "FID to start at.",
                }
            },
            clearlog => {
                des => 'Clear the fsck log',
            },
            printlog => {
                des => 'Display the fsck log',
            },
            taillog => {
                des => 'Tail the fsck log',
            },

        },
    },
};

# load up our config files
my %opts;

Getopt::Long::Configure("require_order", "pass_through");
GetOptions(
        "trackers=s" => \$opts{trackers},
        "config=s"   => \$opts{config},
        "lib=s"      => \$opts{lib},
        "help"       => \$opts{help},
        "verbose"    => \$opts{verbose},
    ) or abortWithUsage();
Getopt::Long::Configure("require_order", "no_pass_through");

my @configs = ($opts{config}, "$ENV{HOME}/.mogilefs.conf", "/etc/mogilefs/mogilefs.conf");
foreach my $fn (reverse @configs) {
    next unless $fn && -e $fn;
    open FILE, "<$fn"
        or die "unable to open $fn: $!\n";
    while (<FILE>) {
        s/\#.*//;
        next unless m!^\s*(\w+)\s*=\s*(.+?)\s*$!;
        $opts{$1} = $2 unless ( defined $opts{$1} );
    }
    close FILE;
}

# bail for help
abortWithUsage() if $opts{help};

# make sure we have at least a topcmd
my $topcmd = shift(@ARGV);
abortWithUsage() unless $topcmd && $usage->{$topcmd};

# break up the trackers and ensure we got some
if ($opts{trackers}) {
    $opts{trackers} = [ split(/\s*,\s*/, $opts{trackers}) ];
}
fail_text('no_trackers')
    unless ($opts{trackers} && @{$opts{trackers}}) || detect_local_tracker();

# okay, load up the libraries that we need
if ($opts{lib}) {
    eval "use lib '$opts{lib}';";
}
eval "use MogileFS::Admin; use MogileFS::Client; 1;" or fail_text('cant_find_module');

# dispatch if it's special
if ($topcmd eq 'check') {
    die "Unknown options/arguments to 'check' command.\n" if @ARGV;
    cmd_check();
} elsif ($topcmd eq 'stats') {
    die "Unknown options/arguments to 'stats' command.\n" if @ARGV;
    cmd_stats();
}

# get the verb
my $verb = shift(@ARGV) or
    abort_with_topcmd_help($topcmd);
my $cmdinfo = $usage->{$topcmd}{subcmd}{$verb};
abort_with_topcmd_help($topcmd) unless $cmdinfo;

my $badargs = sub {
    my $msg = shift;
    abort_with_topcmd_help($topcmd, $verb, $msg);
};

# get the non-option (non --foo) arguments:
my %cmdargs;
if (my $args = $cmdinfo->{args}) {
    my @args = split(/ /, $args);
    foreach my $arg (@args) {
        # positional (but named) parameter
        if ($arg =~ /^<(.+)>$/) {
            my $argname = $1;
            my $val = shift @ARGV;
            # map e.g. "dev5" to 5
            if ($argname eq "devid" && $val && $val =~ /^dev(\d+)$/) {
                $val = $1;
            }
            $badargs->("Missing argument '$argname'") unless defined $val;
            $badargs->("Unexpected option.  Expected argument '$argname'") if $val =~ /^-/;
            $cmdargs{$argname} = $val;
        } elsif ($arg eq "[opts]") {
            # handled later.
        } else {
            die "INTERNAL ERROR.";
        }
    }
    $badargs->("Unexpected extra argument.") if @ARGV && $ARGV[0] !~ /^-/;
} else {
    $badargs->("Unexpected arguments when expecting none.") if @ARGV;
}

# parse the options
if (my $opts = $cmdinfo->{opts}) {
    my %getopts;
    foreach (keys %$opts) {
        my $k = $_;
        next if $k =~ /^</;  # don't care about these.
        die "BOGUS KEY: '$k'" unless $k =~ /^--([\w\-]+)(=.+)?$/;
        my ($oname, $type) = ($1, $2 || "");
        $getopts{"$oname$type"} = \$cmdargs{$1};
    }
    GetOptions(%getopts)
        or abort_with_topcmd_help($topcmd, $verb);
}

# see what we should do
my $cmdsub = do {
    no strict 'refs';
    *{"cmd_${topcmd}_${verb}"} or abortWithUsage();
};

# now call our lovely lovely sub
$cmdsub->(\%cmdargs);
exit 0;

sub detect_local_tracker {
    require IO::Socket::INET;
    my $loctrack = "127.0.0.1:7001";
    my $sock = IO::Socket::INET->new(PeerAddr => $loctrack, Timeout => 1);
    return 0 unless $sock;
    $opts{trackers} = [$loctrack];
    return 1;
}

###########################################################################
## command routines
###########################################################################

sub cmd_check {
    # step 1: we want to check each tracker for responsiveness
    my $now = time();
    my ($hosts, $devices);
    $| = 1;
    print "Checking trackers...\n";
    foreach my $t (@{$opts{trackers}}) {
        print "  $t ... ";
        my $mogadm = mogadm($t);
        if ($mogadm) {
            my $lhosts = hosts($mogadm);
            my $ldevs = devices($mogadm);
            if ($lhosts && $ldevs) {
                print "OK\n";
                $hosts = $lhosts;
                $devices = $ldevs;
            } else {
                print "REQUEST FAILURE (is the tracker up?)\n";
            }
        } else {
            print "INITIAL FAILURE\n";
        }
    }

    # we should have hosts if we get here
    fail_text('no_hosts') unless $hosts;
    print "\n";

    # step 2: now hit each of the hosts for responsiveness
    print "Checking hosts...\n";
    my @urls;
    foreach my $hostid (sort { $a <=> $b } keys %$hosts) {
        printf "  [%2d] %s ... ", $hostid, $hosts->{$hostid}->{hostname};
        if ($hosts->{$hostid}->{status} eq 'alive') {
            my $url = 'http://' . $hosts->{$hostid}->{hostip} . ':' . $hosts->{$hostid}->{http_port} . '/';
            my $file = get($url);
            if (defined $file) {
                print "OK\n";
                push @urls, [ $hostid, $url ];
            } else {
                print "REQUEST FAILURE\n";
            }
        } else {
            print "skipping; status = $hosts->{$hostid}->{status}\n";
        }
    }

    # everything should be chill
    fail_text('no_devices') unless @urls;
    print "\n";

    # step 3: check devices for each host
    print "Checking devices...\n";
    printf "  host device      %10s %10s %10s %7s  %7s   %4s\n", 'size(G)', 'used(G)', 'free(G)', 'use% ', 'ob state', 'I/O%';
    printf "  ---- ------------ ---------- ---------- ---------- ------ ---------- -----\n";
    my %total;
    # Initialize to zero so that the total outputs doesn't need to check for undefined.
    map { $total{$_} = 0; } qw(total used avail);

    foreach my $hosturl (@urls) {
        my ($hostid, $url) = @$hosturl;
        my $devs = $devices->{$hostid};
        foreach my $devid (sort { $a <=> $b } keys %$devs) {
            my $dev = $devs->{$devid};
            my $status = $dev->{status} || "??";
            next if $status eq "dead";

            printf "  [%2d] %-7s", $hostid, "dev$devid";

            my $usage = get($url . "/dev$devid/usage");
            if (defined $usage) {
                my %data = ( map { split(/:\s+/, $_) } split(/\r?\n/, $usage) );
                foreach (qw(time used total avail)) {
                    $data{$_} = 0 if (!$data{$_} || 
                                       $data{$_} !~ /\A\d+(\.\d+)?\Z/); 
                }
                $data{age} = $now - $data{time};
                $data{used} /= 1024**2;
                $data{total} /= 1024**2;
                $data{available} /= 1024**2;
                $data{avail} = $data{available};
                my $pct = 100 - $data{available}/$data{total}*100;
                $total{used} += $data{used};
                $total{avail} += $data{avail};
                $total{total} += $data{total};

                $dev->{utilization}  = 0 if (!defined($dev->{utilization}) || 
                                              $dev->{utilization} !~ /\A\d+(\.\d+)?\Z/);
                printf("     %10.3f %10.3f %10.3f %6.2f%%  %-7s %5.1f\n",
                       (map { $data{$_} } qw(total used avail)),
                       $pct, ($dev->{observed_state} || "?"),
                       $dev->{utilization});
            } else {
                print "REQUEST FAILURE\n";
            }
        }
    }
    my $pct = 0;
    # Avoid division by zero
    $pct = 100 - $total{avail}/$total{total}*100 if($total{total} > 0);

    printf "  ---- ------------ ---------- ---------- ---------- ------\n";
    printf "             total:%10.3f %10.3f %10.3f %6.2f%%\n", (map { $total{$_} } qw(total used avail)), $pct;

    # if we get here, all's well
    ok();
}

sub cmd_stats {
    fail("mogadm stats is deprecated by new 'mogstats' utility");
}

sub cmd_host_list {
    my $hosts = hosts();
    fail_text('no_hosts') unless $hosts;

    foreach my $hostid (sort keys %$hosts) {
        my $host = $hosts->{$hostid};
        print "$host->{hostname} [$hostid]: $host->{status}\n";
        my @data = (
            'IP', "$host->{hostip}:$host->{http_port}",
            'Alt IP', $host->{altip},
            'Alt Mask', $host->{altmask},
            'GET Port', $host->{http_get_port},
        );
        while (my ($k, $v) = splice(@data, 0, 2)) {
            next unless $v;
            printf "  %-10s\%s\n", "$k:", $v;
        }
        print "\n";
    }
    ok();
}

sub cmd_host_add {
    my $args = shift;

    my $hosts = hosts_byname();
    fail_text('no_hosts') unless $hosts;

    my $name = delete $args->{hostname};
    cmd_help_die("No hostname") unless $name;
    fail('Host already exists.') if $hosts->{$name};

    # make sure we have an ip
    unless ($args->{ip}) {
        my $addr = gethostbyname($name);
        fail_text('host_add_no_ip') unless $addr;
        $args->{ip} = inet_ntoa($addr);
    }

    # defaults
    $args->{port}   ||= 7500;
    $args->{status} ||= 'down';

    # FIXME: verify the status can't be 'alive' if we can't get to ip:port
    # OR BETTER: also make default status the reachability of that ip:port

    # now create the host
    my $mogadm = mogadm();
    $mogadm->create_host($name, $args);
    if ($mogadm->err) {
        fail("Failure creating host: " . $mogadm->errstr);
    }

    ok('Host has been created.');
}

sub cmd_host_modify {
    my $args = shift;
    my $name = delete $args->{hostname};

    # FIXME: verify the status can't be 'alive' if we can't get to ip:port

    # now modify the host
    my $mogadm = mogadm();
    $mogadm->update_host($name, $args);
    if ($mogadm->err) {
        fail("Failure modifying host: " . $mogadm->errstr);
    }

    ok('Host has been modified.');
}

sub cmd_host_delete {
    my $args = shift;
    my $name = delete $args->{hostname};

    # now modify the host
    my $mogadm = mogadm();
    $mogadm->delete_host($name);
    if ($mogadm->err) {
        fail("Failure deleting host: " . $mogadm->errstr);
    }

    ok('Host has been deleted.');
}

sub cmd_host_mark {
    my $args = shift;

    my $mogadm = mogadm();
    $mogadm->update_host($args->{hostname}, { status => $args->{status} });
    if ($mogadm->err) {
        fail("Failure updating host status: " . $mogadm->errstr);
    }

    ok('Host status updated.');
}

sub cmd_domain_list {
    # actually lists domains and classes
    my $domains = domains() or
        fail_text('no_domains');
    # now iterate
    printf " %-20s %-20s %-12s  %-12s\n", "domain", "class", "mindevcount", "replpolicy";
    printf "%-20s %-20s %-12s %-12s\n", '-' x 20, '-' x 20, '-' x 13, '-' x 12;
    foreach my $domain (sort keys %$domains) {
        foreach my $class (sort keys %{$domains->{$domain}}) {
            my $dom = $domains->{$domain}->{$class};
            printf " %-20s %-20s      %-8d %-13s\n", $domain, $class,
                $dom->{mindevcount} || 0, $dom->{replpolicy} || '';
        }
        print "\n";
    }

    ok();
}

sub cmd_domain_add {
    my $args = shift;

    my $domains = domains() or
        fail_text('no_domains');

    # make sure it doesn't exist
    my $domain = delete $args->{domain};
    fail('Domain already exists.') if $domains->{$domain};

    # create
    my $mogadm = mogadm();
    $mogadm->create_domain($domain);
    if ($mogadm->err) {
        fail('Error creating domain: ' . $mogadm->errstr);
    }

    ok('Domain created.');
}

sub cmd_domain_delete {
    my $args = shift;
    my $domains = domains() or
        fail_text('no_domains');

    # make sure it doesn't exist
    my $domain = $args->{domain};
    fail('Domain not found.') unless $domains->{$domain};

    # destroy
    my $mogadm = mogadm();
    $mogadm->delete_domain($domain);
    if ($mogadm->err) {
        fail('Error deleting domain: ' . $mogadm->errstr);
    }

    ok('Domain deleted.');
}

sub cmd_class_list {
    # same, pass it through
    cmd_domain_list();
}

sub cmd_class_add {
    my $args = shift;
    my $domains = domains() or
        fail_text('no_domains');

    my $domain = delete $args->{domain};
    my $class  = delete $args->{class};

    cmd_help_die() unless $domain && $class;
    fail('Domain not found.') unless $domains->{$domain};
    fail('Class already exists.') if $domains->{$domain}->{$class};

    $args->{mindevcount} ||= 2;
    $args->{replpolicy}  ||= '';

    my $mogadm = mogadm();
    $mogadm->create_class($domain, $class, $args);
    if ($mogadm->err) {
        fail('Error creating class: ' . $mogadm->errstr);
    }

    ok('Class created.');
}

sub cmd_class_modify {
    my $args = shift;
    my $domains = domains() or
        fail_text('no_domains');

    my $domain = delete $args->{domain};
    my $class  = delete $args->{class};

    cmd_help_die() unless $domain && $class;
    fail('Domain not found.') unless $domains->{$domain};
    fail('Class does not exist.') unless $domains->{$domain}->{$class};

    $args->{mindevcount} ||= 2;
    $args->{replpolicy}  ||= '';

    my $mogadm = mogadm();
    $mogadm->update_class($domain, $class, $args);
    if ($mogadm->err) {
        fail('Error updating class: ' . $mogadm->errstr);
    }

    ok('Class updated.');
}

sub cmd_class_delete {
    my $args = shift;

    my $domains = domains() or
        fail_text('no_domains');

    my $domain = $args->{domain};
    my $class  = $args->{class};

    cmd_help_die() unless $domain && $class;
    fail('Domain not found.') unless $domains->{$domain};
    fail('Class does not exist.') unless $domains->{$domain}->{$class};

    my $mogadm = mogadm();
    $mogadm->delete_class($domain, $class);
    if ($mogadm->err) {
        fail('Error deleting class: ' . $mogadm->errstr);
    }

    ok('Class deleted.');
}

sub cmd_device_add {
    my $args = shift;

    my $hosts = hosts() or
        fail_text('no_hosts');

    my $host  = $args->{hostname};
    my $devid = $args->{devid};
    my $state = $args->{status} || "alive";

    cmd_help_die("devid should be numeric") unless $devid =~ /^\d+$/;

    # FIXME: server should be fixed to verify via HTTP that the devid directory exists

    my $mogadm = mogadm();
    $mogadm->create_device(hostname => $host, devid => $devid, state => $state);

    if ($mogadm->err) {
        fail('Error adding device: ' . $mogadm->errstr);
    }

    ok('Device added.');
}

sub cmd_device_mark {
    my $args = shift;

    print "***NOTE***: As of server version 2.40 'drain' has changed. See docs/wiki\n";
    my $mogadm = mogadm();
    $mogadm->change_device_state($args->{hostname},
                                 $args->{devid},
                                 $args->{status});
    if ($mogadm->err) {
        fail('Error updating device: ' . $mogadm->errstr);
    }

    ok('Device updated.');
}

sub cmd_device_modify {
    my $args = shift;
    my $hostname = delete $args->{hostname};
    my $devid = delete $args->{devid};

    print "***NOTE***: As of server version 2.40 'drain' has changed. See docs/wiki\n";
    my $mogadm = mogadm();
    $mogadm->update_device($hostname, $devid, $args);

    if ($mogadm->err) {
        fail('Error updating device: ' . $mogadm->errstr);
    }

    ok('Device updated.');
}

sub cmd_device_list {
    my $args = shift;

    my $hosts = hosts();
    fail_text('no_hosts') unless $hosts;

    my $devs = devices();
    fail_text('no_devices') unless $devs;

    foreach my $hostid (sort keys %$hosts) {
        my $host = $hosts->{$hostid};
        print "$host->{hostname} [$hostid]: $host->{status}\n";

        printf "%6s  %-10s %7s %7s %7s\n", '', '', 'used(G)', 'free(G)', 'total(G)';
        foreach my $devid (sort keys %{$devs->{$hostid} || {}}) {
            my $dev = $devs->{$hostid}->{$devid};
            next if $dev->{status} eq "dead" && ! $args->{all};

            my $total = $dev->{mb_total} / 1024;
            my $used = $dev->{mb_used} / 1024;
            my $free = $total - $used;
            printf "%6s: %-10s %-7.3f %-7.3f %-7.3f\n", "dev$devid", $dev->{status}, $used, $free, $total;
        }

        print "\n";
    }

    ok();
}

sub cmd_device_summary {
    my $args = shift;
    my %show_state;
    $show_state{$_} = 1 foreach split(/,/, ($args->{status} || "alive,readonly"));

    my $hosts = hosts();
    fail_text('no_hosts') unless $hosts;

    my $devs = devices();
    fail_text('no_devices') unless $devs;

    printf "%-15s %6s %7s  %8s %8s %8s %8s\n", 'Hostname', 'HostID', 'Status', 'used(G)', 'free(G)', 'total(G)', '%Used';
    foreach my $hostid (sort keys %$hosts) {
        my $host = $hosts->{$hostid};
        my ($total,$used) = (0, 0);

        foreach my $devid (sort keys %{$devs->{$hostid} || {}}) {
            my $dev = $devs->{$hostid}->{$devid};
            next unless $show_state{$dev->{status}};

            my $devtotal = $dev->{mb_total} / 1024;
            my $devused  = $dev->{mb_used} / 1024;

            $total += $devtotal;
            $used  += $devused;
        }
        my $free = $total - $used;
        printf "%-15s [%4d]: %6s", $host->{hostname}, $hostid, $host->{status};
        printf "  %8.3f %8.3f %8.3f ", $used, $free, $total;
        printf "%8.2f", 100*$used/$total if $total;
        print "\n";
    }

    ok();

}

sub cmd_slave_list {
    my $mogadm = mogadm();

    my $slaves = $mogadm->slave_list();

    foreach my $key (sort keys %$slaves) {
        my $slavedata = $slaves->{$key};
        my ($dsn, $username, $password) = @$slavedata;
        print "$key --dsn=$dsn --username=$username --password=$password\n";
    }

    ok();
}

sub cmd_slave_add {
    my $mogadm = mogadm();
    my $args = shift;

    my $rc = $mogadm->slave_add($args->{slave_key}, $args->{dsn}, $args->{username}, $args->{password});

    if ($rc) {
        ok("Slave added");
    } else {
        fail("Slave failed to be added");
    }
}

sub cmd_slave_modify {
    my $mogadm = mogadm();
    my $args = shift;

    my $key = delete $args->{slave_key} or cmd_help_die("Key argument is required");

    my $rc = $mogadm->slave_modify($key, %$args);

    if ($rc) {
        ok("Slave modify success");
    } else {
        fail("Slave modify failure: " . $mogadm->errstr);
    }
}

sub cmd_slave_delete {
    my $mogadm = mogadm();

    my $args = shift;

    my $rc = $mogadm->slave_delete($args->{slave_key});

    if ($rc) {
        ok("Slave deleted");
    } else {
        fail("Slave delete failed");
    }
}

sub cmd_rebalance_start {
    my $mogadm = mogadm();
    my $res = $mogadm->rebalance_start || fail($mogadm->errstr);
    ok("rebalance started");
}

sub cmd_rebalance_stop {
    my $mogadm = mogadm();
    my $res = $mogadm->rebalance_stop || fail($mogadm->errstr);
    ok("rebalance stopped");
}

# TODO: Make output prettier? Put hostname next to device name, print device
# info?
sub cmd_rebalance_test {
    my $mogadm = mogadm();
    my $res = $mogadm->rebalance_test || fail($mogadm->errstr);
    print "Tested rebalance policy...\n";
    my $s = $mogadm->server_settings;
    print "Policy: ", $s->{rebal_policy}, "\n\n";
    print "Source devices:\n";
    for my $dev (sort split /,/, $res->{sdevs}) {
        print " - ", $dev, "\n";
    }
    print "Destination devices:\n";
    for my $dev (sort split /,/, $res->{ddevs}) {
        print " - ", $dev, "\n";
    }
}

sub cmd_rebalance_status {
    my $mogadm = mogadm();

    my $ss  = $mogadm->server_settings or fail ($mogadm->errstr);
    my $res = $mogadm->rebalance_status or fail ($mogadm->errstr);
    if ($ss->{rebal_host}) {
        print "Rebalance is running\n";
    } else {
        print "Rebalance is stopped\n";
    }
    print "Rebalance status:\n";
    for my $o (sort split /\s+/, $res->{state}) {
        my ($k, $v) = split /=/, $o;
        printf("%25s = %-s\n", $k, $v);
    }
}

sub cmd_rebalance_policy {
    my $mogadm = mogadm();
    my $args   = shift;

    my $res = $mogadm->rebalance_set_policy($args->{options})
        or fail($mogadm->errstr);

    ok("changed policy setting");
}

sub cmd_rebalance_settings {
    my $mogadm = mogadm();

    my $ss = $mogadm->server_settings
        or fail("can't get settings");
    foreach my $k (sort keys %$ss) {
        next unless ($k =~ '^rebal_');
        next if ($k eq 'rebal_state');
        printf("%25s = %-s\n", $k, $ss->{$k});
    }
}

sub cmd_fsck_start {
    my $mogadm = mogadm();
    my $res = $mogadm->fsck_start || fail($mogadm->errstr);
    ok("fsck started");
}

sub cmd_fsck_stop {
    my $mogadm = mogadm();
    my $res = $mogadm->fsck_stop || fail($mogadm->errstr);
    ok("fsck stopped");
}

sub cmd_fsck_reset {
    my $mogadm = mogadm();
    my $args = shift;
    my $res = $mogadm->fsck_reset(
                                  policy_only => $args->{"policy-only"},
                                  startpos => $args->{"startpos"},
                                  )
        or fail($mogadm->errstr);
    ok("fsck stopped");
}

sub cmd_fsck_clearlog {
    my $mogadm = mogadm();
    my $res = $mogadm->fsck_clearlog || fail($mogadm->errstr);
    ok("fsck log cleared");
}

sub _log_dump {
    my %opts   = @_;
    my $max    = $opts{start};
    my $mogadm = mogadm();

    my $fmt = "%-20s %5s %13s %10s\n";
    printf($fmt, "unixtime", "event", "fid", "devid");
    while (1) {
        my @rows = $mogadm->fsck_log_rows(after_logid => $max);
        unless (@rows) {
            $opts{on_stall}->();
            next;
        }
        foreach my $row (@rows) {
            printf($fmt,
                   $row->{utime},
                   $row->{evcode},
                   $row->{fid},
                   $row->{devid} || "-");
            $max = $row->{logid};
        }
    }
}

sub cmd_fsck_printlog {
    _log_dump(start     => 0,
              on_stall => sub { exit 0; });
}

sub cmd_fsck_taillog {
    my $mogadm = mogadm();
    my $status = $mogadm->fsck_status
        or fail("can't get fsck status");
    _log_dump(start     => $status->{max_logid} - 20,
              on_stall => sub { sleep 5; });
}

sub cmd_fsck_status {
    my $mogadm = mogadm();
    my $status = $mogadm->fsck_status
        or fail("can't get fsck status");

    my %known = map { $_ => 1 } qw(
                                   current_time
                                   max_logid
                                   );
    my $st = sub {
        my $k = shift;
        $known{$k} = 1;
        return $status->{$k};
    };

    my $line = sub {
        printf("%11s: %-s\n", @_);
    };
    print "\n";
    my $host = $st->('host');
    $line->("Running", $st->('running') ? "Yes (on $host)" : "No");

    my $ratio = $st->('end_fid') ? ($st->('max_fid_checked') / $st->('end_fid')) : 0;
    my $perc  = sprintf("%0.02f%%", 100 * $ratio);

    $line->("Status",
            $st->('max_fid_checked') . " / " . $st->('end_fid')
            . " ($perc)");
    my $elap = $st->('start_time') ?
        (($st->('stop_time') || $st->('current_time')) - $st->('start_time')) :
        0;
    my $as_time = sub {
        my $s = shift;
        return int($s) . "s" if $s < 60;
        return int($s/60) . "m";
    };
    my $per_sec = $elap ? ($st->('max_fid_checked') / $elap) : 0;
    $line->("Time",  sprintf("%s (%d fids/s; %s remain)",
                             $as_time->($elap),
                             sprintf("%0.1f", $per_sec),
                             $as_time->($per_sec ?
                                        (($st->('end_fid') - $st->('max_fid_checked'))
                                         / $per_sec) :
                                        0)));

    $line->("Check Type", ($st->('policy_only') ?
                           "Repl policy only (skip file checks)" :
                           "Normal (check policy + files)"));

    if (my @unk = grep { !$known{$_} } sort keys %$status) {
        print "\n";
        foreach (@unk) {
            $line->("[$_]", $status->{$_});
        }
    }
    print "\n";
}

sub cmd_settings_list {
    my $mogadm = mogadm();
    unless ($mogadm->can("server_settings")) {
        fail("settings commands require MogileFS::Client >= 1.07");
    }
    my $ss = $mogadm->server_settings
        or fail("can't get settings");
    foreach my $k (sort keys %$ss) {
        # Don't list noisy "setting"
        next if ($k =~ '^rebal');
        printf("%25s = %-s\n", $k, $ss->{$k});
    }
}

sub cmd_settings_set {
    my $mogadm = mogadm();
    unless ($mogadm->can("set_server_setting")) {
        fail("settings commands require MogileFS::Client >= 1.07");
    }
    my $args = shift;

    $mogadm->set_server_setting($args->{key}, $args->{value})
        or fail($mogadm->errstr);
    ok();
}

###########################################################################
## helper routines
###########################################################################

sub abortWithUsage {
    my $ret = "Usage:  (enter any command prefix, leaving off options, for further help)\n\n";
    foreach my $cmd (@topcmds) {
        my $sbc = $usage->{$cmd}->{subcmd};
        if ($sbc) {
            $ret .= "  mogadm $cmd ...\n";
        } else {
            $ret .= sprintf("  mogadm %-25s %-s\n",
                            "$cmd",
                            $usage->{$cmd}{des} || "");
            next;
        }
        foreach my $v (sort keys %$sbc) {
            my $scv = $usage->{$cmd}{subcmd}{$v};
            $ret .= "       ";
            my $dotdot = $scv->{args} ? "..." : "";
            $ret .= sprintf("  %-25s %-s\n",
                            "$cmd $v $dotdot",
                            $scv->{des} || "");

        }
    }
    print $ret, "\n";
    exit(1);
}

sub abort_with_topcmd_help {
    my ($cmd, $verb, $msg) = @_;
    if ($msg) {
        print "\nERROR: $msg\n\n";
    }
    my $cmdsfx = $verb ? "-$verb" : "";
    my $ret = "Help for '$cmd$cmdsfx' command:\n";
    unless ($verb) {
        $ret .= " (enter any command prefix, leaving off options, for further help)\n";
    }
    $ret .= "\n";
    foreach my $subcmdv (sort keys %{$usage->{$cmd}{subcmd}}) {
        next if $verb && $verb ne $subcmdv;
        my $scv = $usage->{$cmd}{subcmd}{$subcmdv};
        $ret .= sprintf("  %-50s %-s\n",
                        "mogadm $cmd $subcmdv " . ($scv->{args} || ""),
                        $scv->{des});
    }
    print $ret, "\n";
    if ($verb) {
        my $scv = $usage->{$cmd}{subcmd}{$verb};
        foreach my $opt (sort {
            (substr($b, 0, 1) cmp substr($a, 0, 1)) ||
                $a cmp $b
        } keys %{$scv->{opts} || {}})
        {
            printf("      %-20s %s\n", $opt, $scv->{opts}->{$opt});
        }
        print "\n";
    }
    exit 1;
}

sub cmd_help_die {
    my ($msg) = @_;
    abort_with_topcmd_help($topcmd, $verb, $msg);
}


sub text {
    return {

        ######################################################################
        cant_find_module => <<END,
Unable to find MogileFS::Admin module.  Please ensure that you have the
module installed in a location in your search path.  Or, add a search path
to mogadm:

    mogadm --lib=/path/to/lib

Or add it to the configuration file:

    lib = /path/to/lib
END

        ######################################################################
        no_mogadm => <<END,
Unable to access MogileFS tracker and/or instantiate a MogileFS::Admin object.
END

        ######################################################################
        no_domains => "Unable to retrieve domains from tracker(s).\n",

        ######################################################################
        no_devices => "No devices found on tracker(s).\n",

        ######################################################################
        host_add_no_ip => <<END,
Hostname does not resolve to an IP, and you didn\'t specify one on the options
list.  Please either verify the host resolves, or try again:

    mogadm host add <hostname> --ip=<ipaddr> [...]
END

        ######################################################################
        no_hosts => <<END,
Unable to retrieve host information from tracker(s).
END

        ######################################################################
        no_trackers => <<END,
In order to use the mogadm toolkit, you need to provide the information about
where your trackers are.

In your configuration file:

    trackers = 10.10.0.33:7001, 10.10.0.34:7001

Or on the command line

    mogadm --trackers=10.10.0.33:7001,10.10.0.34:7001
END

    }->{$_[0]} || "UNDEFINED [$_[0]]";
}

sub fail_text {
    print STDERR text($_[0]) . "\n";
    exit 1;
}

sub fail {
    print STDERR $_[0] . "\n";
    exit 1;
}

sub ok {
    if ($opts{verbose}) {
        print STDOUT $_[0] . "\n" if $_[0];
    }
    exit 0;
}

sub mogadm {
    my $host = shift();
    if ($host) {
        $host = [ $host ] unless ref $host;
    } else {
        $host = $opts{trackers};
    }
    # 10 seconds is the max time used for any of the admin locks (fsck status)
    # plus we leave a bit of time for work.
    my $timeout = 15; 
    $timeout = $opts{timeout} if $opts{timeout} && $opts{timeout} =~ /^[0-9]+$/;
#    $MogileFS::DEBUG = 2;
    my $mogadm = MogileFS::Admin->new( hosts => $host, timeout => $timeout );
    fail_text('no_mogadm') unless $mogadm;
    return $mogadm;
}

sub stats {
    my $mogadm = shift() || mogadm();
    my $res;
    eval {
        $res = $mogadm->get_stats();
    };
    return undef if $@;
    return $res;
}
    
sub hosts_byname {
    my $mogadm = shift() || mogadm();
    fail_text('no_mogadm') unless $mogadm;

    my $res;
    eval {
        $res = _array_to_hashref($mogadm->get_hosts(), 'hostname');
    };
    return undef if $@;
    return $res;
}

sub hosts {
    my $mogadm = shift() || mogadm();
    fail_text('no_mogadm') unless $mogadm;

    my $res;
    eval {
        $res = _array_to_hashref($mogadm->get_hosts(), 'hostid');
    };
    return undef if $@;
    return $res;
}

sub devices {
    my $mogadm = shift() || mogadm();
    fail_text('no_mogadm') unless $mogadm;

    my $res;
    eval {
        $res = _array_to_hashref($mogadm->get_devices(), [ 'hostid', 'devid' ]);
    };
    return undef if $@;
    return $res;
}

sub domains {
    my $mogadm = shift() || mogadm();
    fail_text('no_mogadm') unless $mogadm;

    my $res;
    eval {
        $res = $mogadm->get_domains();
    };
    return undef if $@;
    return $res;
}

sub _array_to_hashref {
    my ($array, $key) = @_;
    die "bad caller to _array_to_hashref\n"
        unless $array && $key;
    $key = [ $key ] unless ref $key eq 'ARRAY';
    my $kmax = scalar(@$key) - 1;

    # and a dose of handwavium...
    my %res;
    foreach my $row (@$array) {
        my $ref = \%res;
        for (my $i = 0; $i <= $kmax; $i++) {
            if ($i == $kmax) {
                # we're on the last key so just assign into $ref
                $ref->{$row->{$key->[$i]}} = $row;
            } else {
                # not on the last, so keep descending
                $ref->{$row->{$key->[$i]}} ||= {};
                $ref = $ref->{$row->{$key->[$i]}};
            }
        }
    }

    # return result.. duh
    return \%res;
}

__END__

=head1 NAME

mogadm - MogileFS admin tool

=head1 SYNOPSIS

 $ mogadm [config options] <argument(s)> [argument options]

 $ mogadm
   ....
   (prints contextual help, if missing command/arguments)
   ...

=head1 OPTIONS

=over 8

=item B<--lib=/path/to/lib>

Set this option to a path to include this directory in the module
search path.

=item B<--trackers=10.0.0.117:7001,10.0.0.118:7001,...>

Use these MogileFS trackers for status information.

=back

=head1 ARGUMENTS

=over 8

=item B<check>

Check to ensure that all of the MogileFS system components are functioning
and that we can contact everybody.  The quickest way of ensuring that the
entire MogileFS system is functional from the current machine's point of view.

=item B<stats>

Attempt to get statistics for the MogileFS world. This should provide
statistics on: the number of files present on each device; the number of files
in each (domain,class) tuple; statistics about present replication devcounts
and pending replications.

Due to design constraints, the stats command may presently fail if some
statistic takes too long to generate.

=item B<host add E<lt>hostE<gt> [host options]>

=item B<host modify E<lt>hostE<gt> [host options]>

=item B<host mark E<lt>hostE<gt> E<lt>statusE<gt>>

=item B<host delete E<lt>hostE<gt>>

=item B<host list>

Functions for manipulating hosts.  For add and modify, host options is in
the format of normal command line options and can include anything in the
L</"HOST OPTIONS"> section.

=item B<device add E<lt>hostE<gt> E<lt>device idE<gt>>

=item B<device mark E<lt>hostE<gt> E<lt>device idE<gt> E<lt>statusE<gt>>

=item B<device modify E<lt>hostE<gt> E<lt>deviceE<gt> [device options]>

=item B<device delete E<lt>hostE<gt> E<lt>deviceE<gt>>

=item B<device list>

Functions for manipulating devices. For add and modify, device options are in
the format of normal command line options and can include anything in the
L</"DEVICE OPTIONS"> section.

=item B<domain add E<lt>domainE<gt>>

=item B<domain delete E<lt>domainE<gt>>

=item B<domain list>

Simple commands for managing MogileFS domains.  Note that you cannot delete
a domain unless it has no classes and is devoid of files.

=item B<class add E<lt>domainE<gt> E<lt>classE<gt> [class options]>

=item B<class modify E<lt>domainE<gt> E<lt>classE<gt> [class options]>

=item B<class delete E<lt>domainE<gt> E<lt>classE<gt>>

=item B<class list>

Commands for working with classes.  Please see the L</"CLASS OPTIONS"> section
for the options to use with add/modify.  Also, delete requires that the class
have no files in it before it will allow the deletion.

=item B<slave ...>

Add/remove slaves replicating from MogileFS master database.

TODO: detail this

Run B<mogadm slave> by itself for contextual help.

=item B<fsck printlog>

=item B<fsck taillog>

=item B<fsck clearlog>

Display or clear the log of fsck events.

=item B<fsck reset [fsck options]>

Reset fsck position back to the beginning.  Please see the L</"FSCK OPTIONS">
section for options to use with fsck.

=item B<fsck start>

Start (or resume) background fsck from the last checked fid. If you want to
check every fid, you must call B<fsck reset> before calling start.

=item B<fsck status>

Show the status of the presently active (or last if none active) fsck. This
includes what FIDs are being checked, time statistics, check type as well as a
summary of problems encountered so far.

=item B<fsck stop>

Stop (pause) background fsck

=item B<settings list> 

Display all present MogileFS settings.

=item B<settings set E<lt>keyE<gt> E<lt>valueE<gt>>

Set the server setting for 'key' to 'value'.

The current settings are E<lt>enable_rebalanceE<gt> (set to 1 to start
rebalance mode to move files to under-used devices) and
E<lt>memcache_serversE<gt> (enable memcached caching in the tracker).

=back

=head1 HOST OPTIONS

=over 8

=item B<--ip=E<lt>ip of hostE<gt>>

=item B<--port=E<lt>port of mogstored on hostE<gt>>

Contact information for the host.  This is the minimum set of information needed
to setup a host.

=item B<--getport=E<lt>alternate retrieval part on hostE<gt>>

If provided, causes the tracker to use this port for retrieving files.  Uploads are
still processed at the standard port.

=item B<--altip=E<lt>alternate IPE<gt>>

=item B<--altmask=E<lt>mask to activate alternate IPE<gt>>

If a client request comes in from an IP that matches the alternate mask, then the
host IP is treated as the alternate IP instead of the standard IP.  This can be
used, for example, if you have two networks and you need to return one IP to
reach the node on one network, but a second IP to reach it on the alternate
network.

=item B<--status=E<lt>host statusE<gt>>

Valid host statuses are one of: alive, down, dead.

=back

=head1 DEVICE OPTIONS

=over 8

=item B<--status=E<lt>device statusE<gt>>

Valid device statuses are one of: alive, dead, down, drain, readonly.

=item B<--weight=E<lt>device weight<gt>>

The weight used in calculation of preferred paths. It must be a positive
integer.

=back

=head1 CLASS OPTIONS

=over 8

=item B<--mindevcount=E<lt>valueE<gt>>

Number of devices the files in this class should be replicated across.  Can be
set to anything >= 1.

=item B<--replpolicy=E<lt>valueE<gt>>

Stringified replication policy. ie "MultipleHosts(3)" is equivalent to a
--mindevcount=3. See documentation or plugins on alternative policies.

=back

=head1 FSCK OPTIONS

=over 8

=item B<--policy-only>

Check replication policy (assumed locations) only; don't stat storage nodes for
actual file presence.

=back

=head1 EXAMPLES

Host manipulation:

    $ mogadm host list
    $ mogadm host add foo.local
    $ mogadm host add foo.local --status=down --ip=10.0.0.34 --port=7900
    $ mogadm host mark foo.local down
    $ mogadm host modify foo.local --port=7500
    $ mogadm host delete foo.local

Device manipulation:

    $ mogadm device list
    $ mogadm device summary
    $ mogadm device summary --status=dead,down
    $ mogadm device add foo.local 16
    $ mogadm device add foo.local 17 --status=alive
    $ mogadm device mark foo.local 17 down
    $ mogadm device modify foo.local 17 --status=alive --weight=10
    $ mogadm device delete foo.local 17

Domain manipulation:

    $ mogadm domain list
    $ mogadm domain add first.domain
    $ mogadm domain delete first.domain

Class manipulation

    $ mogadm class list
    $ mogadm class add first.domain my.class
    $ mogadm class add first.domain my.class --mindevcount=3
    $ mogadm class add first.domain my.class --replpolicy="MultipleHosts(3)"
    $ mogadm class modify first.domain my.class --mindevcount=2
    $ mogadm class modify first.domain my.class --replpolicy="MultipleHosts(3)"
    $ mogadm class delete first.domain my.class

Check the status of your entire MogileFS system:

    $ mogadm check
    
Check MogileFS system statistics:

    $ mogadm stats

Check every file in the entire MogileFS system:

    $ mogadm fsck reset
    $ mogadm fsck start
    $ mogadm fsck status
    $ mogadm fsck printlog

See all the things mogadm can do:

    $ mogadm

Get help on a sub-command:

    $ mogadm device


=head1 CONFIGURATION

It is recommended that you create a configuration file such as C</etc/mogilefs/mogilefs.conf> (or at C<~/.mogilefs.conf>) to
be used for configuration information.  Basically all you need is something like:

    trackers = 10.0.0.23:7001, 10.0.0.15:7001

    # if MogileFS::Admin files aren't installed in standard places:
    lib = /home/mogilefs/cgi-bin

Note that these can also be specified on the command line, as per above.

=head1 AUTHOR

Brad Fitzpatrick E<lt>L<brad@danga.com>E<gt>

Mark Smith E<lt>L<junior@danga.com>E<gt>

Leon Brocard E<lt>L<acme@astray.com>E<gt>, open sourced permissions from Foxtons Ltd.

Robin H. Johnson E<lt>robbat2@orbis-terrarum.netE<gt>

=head1 BUGS

Please report any on the MogileFS mailing list: L<http://groups.google.com/group/mogile/>.

=head1 LICENSE

Licensed for use and redistribution under the same terms as Perl itself.

=cut