diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 930abf0b5d172..6d77c87fd7e2d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -1921,20 +1921,45 @@ public long getLength(Path f) throws IOException { * @return content summary. */ public ContentSummary getContentSummary(Path f) throws IOException { + return getContentSummary(f, null); + } + + /** Return the {@link ContentSummary} of a given {@link Path}. + * @param f path to use + * @param owner path owner + * @throws FileNotFoundException if the path does not resolve + * @throws IOException IO failure + * @return content summary. + */ + public ContentSummary getContentSummary(Path f, String owner) throws IOException { FileStatus status = getFileStatus(f); if (status.isFile()) { // f is a file long length = status.getLen(); - return new ContentSummary.Builder().length(length). - fileCount(1).directoryCount(0).spaceConsumed(length).build(); + // owner mismatch, output 0 0 0 + if (owner == null || owner.equals(status.getOwner())) { + return new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(length).build(); + } else { + return new ContentSummary.Builder().length(length). + fileCount(0).directoryCount(0).spaceConsumed(0).build(); + } } // f is a directory - long[] summary = {0, 0, 1}; + long i = (owner == null || owner.equals(status.getOwner())) ? 1 : 0; + long[] summary = {0, 0, i}; for(FileStatus s : listStatus(f)) { long length = s.getLen(); - ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : - new ContentSummary.Builder().length(length). - fileCount(1).directoryCount(0).spaceConsumed(length).build(); + ContentSummary c; + if (s.isDirectory()) { + c = getContentSummary(s.getPath(), owner); + } else if (owner == null || owner.equals(s.getOwner())) { + c = new ContentSummary.Builder().length(length). + fileCount(1).directoryCount(0).spaceConsumed(length).build(); + } else { + // skip owner mismatch + continue; + } summary[0] += c.getLength(); summary[1] += c.getFileCount(); summary[2] += c.getDirectoryCount(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index bd7be229725e7..179c69a4be963 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -57,6 +57,7 @@ public static void registerCommands(CommandFactory factory) { private static final String OPTION_QUOTA_AND_USAGE = "u"; private static final String OPTION_ECPOLICY = "e"; private static final String OPTION_SNAPSHOT_COUNT = "s"; + private static final String OPTION_OWNER = "o"; public static final String NAME = "count"; public static final String USAGE = @@ -64,7 +65,7 @@ public static void registerCommands(CommandFactory factory) { + "] [-" + OPTION_TYPE + " []] [-" + OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT + "] [-" + OPTION_ECPOLICY + "] [-" + OPTION_SNAPSHOT_COUNT - + "] ..."; + + "] [-" + OPTION_OWNER + " []] ..."; public static final String DESCRIPTION = "Count the number of directories, files and bytes under the paths\n" + "that match the specified file pattern. The output columns are:\n" + @@ -92,10 +93,11 @@ public static void registerCommands(CommandFactory factory) { "ram_disk, ssd, disk, archive and nvdimm.\n" + "It can also pass the value '', 'all' or 'ALL' to specify all " + "the storage types.\n" + - "The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" + - "the usage against the quota without the detailed content summary."+ - "The -" + OPTION_ECPOLICY + " option shows the erasure coding policy." - + "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts."; + "The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and " + + "the usage against the quota without the detailed content summary. \n" + + "The -" + OPTION_ECPOLICY + " option shows the erasure coding policy. \n" + + "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts. \n" + + "The -" + OPTION_OWNER + " option shows the owner's counts."; private boolean showQuotas; private boolean humanReadable; @@ -105,6 +107,7 @@ public static void registerCommands(CommandFactory factory) { private boolean excludeSnapshots; private boolean displayECPolicy; private boolean showSnapshot; + private String owner; /** Constructor */ public Count() {} @@ -128,6 +131,7 @@ protected void processOptions(LinkedList args) { OPTION_EXCLUDE_SNAPSHOT, OPTION_ECPOLICY, OPTION_SNAPSHOT_COUNT); cf.addOptionWithValue(OPTION_TYPE); + cf.addOptionWithValue(OPTION_OWNER); cf.parse(args); if (args.isEmpty()) { // default path is the current working directory args.add("."); @@ -138,6 +142,7 @@ protected void processOptions(LinkedList args) { excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT); displayECPolicy = cf.getOpt(OPTION_ECPOLICY); showSnapshot = cf.getOpt(OPTION_SNAPSHOT_COUNT); + owner = cf.getOptValue(OPTION_OWNER); if (showQuotas || showQuotasAndUsageOnly) { String types = cf.getOptValue(OPTION_TYPE); @@ -157,7 +162,10 @@ protected void processOptions(LinkedList args) { if (cf.getOpt(OPTION_HEADER)) { StringBuilder headString = new StringBuilder(); - if (showQuotabyType) { + if (owner != null && !owner.isEmpty()) { + headString.append("OWNER"); + headString.append(ContentSummary.getHeader(false)); + } else if (showQuotabyType) { headString.append(QuotaUsage.getStorageTypeHeader(storageTypes)); } else { if (showQuotasAndUsageOnly) { @@ -195,7 +203,12 @@ private List getAndCheckStorageTypes(String types) { @Override protected void processPath(PathData src) throws IOException { StringBuilder outputString = new StringBuilder(); - if (showQuotasAndUsageOnly || showQuotabyType) { + if (owner != null && !owner.isEmpty()) { + outputString.append(owner); + ContentSummary summary = src.fs.getContentSummary(src.path, owner); + outputString.append(summary.toString( + false, isHumanReadable())); + } else if (showQuotasAndUsageOnly || showQuotabyType) { QuotaUsage usage = src.fs.getQuotaUsage(src.path); outputString.append(usage.toString( isHumanReadable(), showQuotabyType, storageTypes)); diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 34d5d5371bcb0..fc481e13fa7cc 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -139,7 +139,7 @@ Identical to the -get command. count ----- -Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t []] [-u] [-e] [-s] ` +Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t []] [-u] [-e] [-s] [-o []] ` Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME @@ -165,6 +165,8 @@ The ERASURECODING\_POLICY is name of the policy for the file. If an erasure codi The -s option shows the snapshot counts for each directory. +The -o option shows the owner's counts. + Example: * `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2` @@ -176,6 +178,7 @@ Example: * `hadoop fs -count -u -h -v hdfs://nn1.example.com/file1` * `hadoop fs -count -e hdfs://nn1.example.com/file1` * `hadoop fs -count -s hdfs://nn1.example.com/file1` +* `hadoop fs -count -o user hdfs://nn1.example.com/file1` Exit Code: diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java index 6ce01fe7176e1..93174ebef122a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -141,6 +141,7 @@ public Token[] addDelegationTokens(String renewer, Credentials creds) public String getScheme(); public Path fixRelativePart(Path p); public ContentSummary getContentSummary(Path f); + public ContentSummary getContentSummary(Path f, String owner); public QuotaUsage getQuotaUsage(Path f); void setQuota(Path f, long namespaceQuota, long storagespaceQuota); void setQuotaByStorageType(Path f, StorageType type, long quota); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java index 612954de784db..695e4015a10bf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java @@ -116,6 +116,7 @@ public FSDataOutputStream create(Path f, FsPermission permission, public short getReplication(Path src); public void processDeleteOnExit(); public ContentSummary getContentSummary(Path f); + public ContentSummary getContentSummary(Path f, String owner); public QuotaUsage getQuotaUsage(Path f); void setQuota(Path f, long namespaceQuota, long storagespaceQuota); void setQuotaByStorageType(Path f, StorageType type, long quota); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index 0e5a104f14e25..7ed7b1734f2a3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -439,6 +439,27 @@ public void processPathWithSnapshotHeader() throws Exception { verifyNoMoreInteractions(out); } + @Test + public void processOptionsHeaderWithOwner() { + LinkedList options = new LinkedList(); + options.add("-v"); + options.add("-o"); + options.add("user"); + options.add("dummy"); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + count.processOptions(options); + + String withOwnerHeader = + "OWNER DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME"; + verify(out).println(withOwnerHeader); + verifyNoMoreInteractions(out); + } + @Test public void getCommandName() { Count count = new Count(); @@ -477,7 +498,7 @@ public void getUsage() { String actual = count.getUsage(); String expected = "-count [-q] [-h] [-v] [-t []]" - + " [-u] [-x] [-e] [-s] ..."; + + " [-u] [-x] [-e] [-s] [-o []] ..."; assertEquals(expected, actual, "Count.getUsage"); } @@ -507,10 +528,12 @@ public void getDescription() { + "ram_disk, ssd, disk, archive and nvdimm.\n" + "It can also pass the value '', 'all' or 'ALL' to specify all the " + "storage types.\n" - + "The -u option shows the quota and \n" - + "the usage against the quota without the detailed content summary." - + "The -e option shows the erasure coding policy." - + "The -s option shows snapshot counts."; + + "The -u option shows the quota and " + + "the usage against the quota without the detailed content summary. \n" + + "The -e option shows the erasure coding policy. \n" + + "The -s option shows snapshot counts. \n" + + "The -o option shows the owner's counts."; + assertEquals(expected, actual, "Count.getDescription"); } diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 574dfd2852277..3c8ae04a1f10f 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -250,7 +250,7 @@ RegexpComparator - ^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] \[-x\] \[-e\] \[-s\] <path> \.\.\. :( )* + ^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] \[-x\] \[-e\] \[-s\] \[-o \[<user>\]\] <path> \.\.\. :( )* RegexpComparator