From ea7f701931d197038b15725fb81b0a46f8369899 Mon Sep 17 00:00:00 2001 From: Bence Kosztolnik Date: Tue, 19 Nov 2024 13:27:33 +0100 Subject: [PATCH 1/4] YARN-11738. Modernize SecretManager config. Make hash algorithm at SecretManager configurable. - hadoop.security.hmac-algorithm: The name of the hashing algorithm. Default: HmacSHA1 - hadoop.security.hmac-length: The length of the random keys to use. Default: 64 Change-Id: I735573c1d7b9f256e05722c98cd550cd8dd4acf0 --- .../fs/CommonConfigurationKeysPublic.java | 16 +++++++ .../hadoop/security/token/SecretManager.java | 46 ++++++++++++------- .../src/main/resources/core-default.xml | 26 +++++++++++ .../TestNMTokenSecretManagerInNM.java | 2 + 4 files changed, 73 insertions(+), 17 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 0b36aec318dfb..9725bdbbeb14e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -1005,6 +1005,22 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY = "hadoop.security.credstore.java-keystore-provider.password-file"; + /** + * @see + * + * core-default.xml + */ + public static final String HMAC_ALGORITHM = "hadoop.security.hmac-algorithm"; + public static final String DEFAULT_HMAC_ALGORITHM = "HmacSHA1"; + + /** + * @see + * + * core-default.xml + */ + public static final String HMAC_LENGTH = "hadoop.security.hmac-length"; + public static final int DEFAULT_HMAC_LENGTH = 64; + /** * @see * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java index 514806ddec804..757d9fc811ac3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java @@ -27,8 +27,13 @@ import javax.crypto.SecretKey; import javax.crypto.spec.SecretKeySpec; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.ipc.StandbyException; @@ -40,6 +45,8 @@ @InterfaceAudience.Public @InterfaceStability.Evolving public abstract class SecretManager { + + public static final Logger LOG = LoggerFactory.getLogger(SecretManager.class); /** * The token was invalid and the message explains why. */ @@ -107,16 +114,23 @@ public byte[] retriableRetrievePassword(T identifier) public void checkAvailableForRead() throws StandbyException { // Default to being available for read. } - - /** - * The name of the hashing algorithm. - */ - private static final String DEFAULT_HMAC_ALGORITHM = "HmacSHA1"; - /** - * The length of the random keys to use. - */ - private static final int KEY_LENGTH = 64; + private static final String SELECTED_ALGORITHM; + private static final int SELECTED_LENGTH; + + static { + Configuration conf = new Configuration(); + String algorithm = conf.get( + CommonConfigurationKeysPublic.HMAC_ALGORITHM, + CommonConfigurationKeysPublic.DEFAULT_HMAC_ALGORITHM); + LOG.info("Selected hash algorithm: {}", algorithm); + SELECTED_ALGORITHM = algorithm; + int length = conf.getInt( + CommonConfigurationKeysPublic.HMAC_LENGTH, + CommonConfigurationKeysPublic.DEFAULT_HMAC_LENGTH); + LOG.info("Selected hash key length:{}", length); + SELECTED_LENGTH = length; + } /** * A thread local store for the Macs. @@ -126,10 +140,9 @@ public void checkAvailableForRead() throws StandbyException { @Override protected Mac initialValue() { try { - return Mac.getInstance(DEFAULT_HMAC_ALGORITHM); + return Mac.getInstance(SELECTED_ALGORITHM); } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + DEFAULT_HMAC_ALGORITHM + - " algorithm."); + throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM + " algorithm."); } } }; @@ -140,11 +153,10 @@ protected Mac initialValue() { private final KeyGenerator keyGen; { try { - keyGen = KeyGenerator.getInstance(DEFAULT_HMAC_ALGORITHM); - keyGen.init(KEY_LENGTH); + keyGen = KeyGenerator.getInstance(SELECTED_ALGORITHM); + keyGen.init(SELECTED_LENGTH); } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + DEFAULT_HMAC_ALGORITHM + - " algorithm."); + throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM + " algorithm."); } } @@ -185,6 +197,6 @@ public static byte[] createPassword(byte[] identifier, * @return the secret key */ protected static SecretKey createSecretKey(byte[] key) { - return new SecretKeySpec(key, DEFAULT_HMAC_ALGORITHM); + return new SecretKeySpec(key, SELECTED_ALGORITHM); } } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 4104e3043149e..0415d8d20f184 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1046,6 +1046,32 @@ + + hadoop.security.hmac-algorithm + HmacSHA1 + The configuration key specifying the hashing algorithm used for + HMAC (Hash-based Message Authentication Code) operations. + + The HMAC algorithm is used in token management to compute secure + message digests. This configuration allows users to specify the + algorithm to be used for HMAC operations. The algorithm must be a + valid cryptographic hash algorithm supported by the Java Cryptography + Architecture (JCA). Common examples include "HmacSHA1", "HmacSHA256", + and "HmacSHA512". + + + + hadoop.security.hmac-length + 64 + The configuration key specifying the key length for HMAC (Hash-based + Message Authentication Code) operations. + + This property determines the size of the secret keys generated + for HMAC computations. The key length must be appropriate for the + selected HMAC algorithm. For example, longer keys are generally + more secure but may not be supported by all algorithms. + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java index 5c1f3a1c31064..153b650c20485 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java @@ -60,6 +60,8 @@ public void testRecovery() throws IOException { secretMgr.setNodeId(nodeId); MasterKey currentKey = keygen.generateKey(); secretMgr.setMasterKey(currentKey); + // check key is 64 bit long (8 byte) + assertEquals(8, currentKey.getBytes().array().length); NMTokenIdentifier attemptToken1 = getNMTokenId(secretMgr.createNMToken(attempt1, nodeId, "user1")); NMTokenIdentifier attemptToken2 = From 38759d63b0e0b2f1967e79046425fdd0835c025c Mon Sep 17 00:00:00 2001 From: Bence Kosztolnik Date: Fri, 22 Nov 2024 08:41:51 +0100 Subject: [PATCH 2/4] Update by review Change-Id: I8feab0406d46892daa954a1e41eb193f031ee77a --- .../fs/CommonConfigurationKeysPublic.java | 21 ++++-------- .../hadoop/security/token/SecretManager.java | 12 +++---- .../src/main/resources/core-default.xml | 32 ++++++++----------- 3 files changed, 27 insertions(+), 38 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 9725bdbbeb14e..48f848e71ad9f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -1005,21 +1005,14 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY = "hadoop.security.credstore.java-keystore-provider.password-file"; - /** - * @see - * - * core-default.xml - */ - public static final String HMAC_ALGORITHM = "hadoop.security.hmac-algorithm"; - public static final String DEFAULT_HMAC_ALGORITHM = "HmacSHA1"; + public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY = + "secret-manager.key-generator.algorith"; + public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT = + "HmacSHA1"; - /** - * @see - * - * core-default.xml - */ - public static final String HMAC_LENGTH = "hadoop.security.hmac-length"; - public static final int DEFAULT_HMAC_LENGTH = 64; + public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY = + "hadoop.security.secret-manager.key-length"; + public static final int HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT = 64; /** * @see diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java index 757d9fc811ac3..002917e6d15b5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java @@ -121,13 +121,13 @@ public void checkAvailableForRead() throws StandbyException { static { Configuration conf = new Configuration(); String algorithm = conf.get( - CommonConfigurationKeysPublic.HMAC_ALGORITHM, - CommonConfigurationKeysPublic.DEFAULT_HMAC_ALGORITHM); + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY, + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT); LOG.info("Selected hash algorithm: {}", algorithm); SELECTED_ALGORITHM = algorithm; int length = conf.getInt( - CommonConfigurationKeysPublic.HMAC_LENGTH, - CommonConfigurationKeysPublic.DEFAULT_HMAC_LENGTH); + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY, + CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT); LOG.info("Selected hash key length:{}", length); SELECTED_LENGTH = length; } @@ -142,7 +142,7 @@ protected Mac initialValue() { try { return Mac.getInstance(SELECTED_ALGORITHM); } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM + " algorithm."); + throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM, nsa); } } }; @@ -156,7 +156,7 @@ protected Mac initialValue() { keyGen = KeyGenerator.getInstance(SELECTED_ALGORITHM); keyGen.init(SELECTED_LENGTH); } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM + " algorithm."); + throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM, nsa); } } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 0415d8d20f184..a6e7cdd8e3ffd 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1047,29 +1047,25 @@ - hadoop.security.hmac-algorithm + secret-manager.key-generator.algorith HmacSHA1 - The configuration key specifying the hashing algorithm used for - HMAC (Hash-based Message Authentication Code) operations. - - The HMAC algorithm is used in token management to compute secure - message digests. This configuration allows users to specify the - algorithm to be used for HMAC operations. The algorithm must be a - valid cryptographic hash algorithm supported by the Java Cryptography - Architecture (JCA). Common examples include "HmacSHA1", "HmacSHA256", - and "HmacSHA512". + + The configuration key specifying the KeyGenerator algorithm used in SecretManager + for generating secret keys. The algorithm must be a KeyGenerator algorithm supported by + the Java Cryptography Architecture (JCA). Common examples include "HmacSHA1", + "HmacSHA256", and "HmacSHA512". + - hadoop.security.hmac-length + hadoop.security.secret-manager.key-length 64 - The configuration key specifying the key length for HMAC (Hash-based - Message Authentication Code) operations. - - This property determines the size of the secret keys generated - for HMAC computations. The key length must be appropriate for the - selected HMAC algorithm. For example, longer keys are generally - more secure but may not be supported by all algorithms. + + The configuration key specifying the key length of the generated secret keys + in SecretManager. The key length must be appropriate for the algorithm. + For example, longer keys are generally more secure but may not be supported + by all algorithms. + From 6d41273269933e813a6ce9d8880d149970c79aca Mon Sep 17 00:00:00 2001 From: Bence Kosztolnik Date: Fri, 22 Nov 2024 18:59:44 +0100 Subject: [PATCH 3/4] Update by review Change-Id: I8feab0406d46892daa954a1e41eb193f031ee77a --- .../org/apache/hadoop/fs/CommonConfigurationKeysPublic.java | 2 +- .../hadoop-common/src/main/resources/core-default.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 48f848e71ad9f..e35747f458dda 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -1006,7 +1006,7 @@ public class CommonConfigurationKeysPublic { "hadoop.security.credstore.java-keystore-provider.password-file"; public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY = - "secret-manager.key-generator.algorith"; + "secret-manager.key-generator.algorithm"; public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT = "HmacSHA1"; diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index a6e7cdd8e3ffd..305cc32889183 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1047,7 +1047,7 @@ - secret-manager.key-generator.algorith + secret-manager.key-generator.algorithm HmacSHA1 The configuration key specifying the KeyGenerator algorithm used in SecretManager From 75a062c435cfe384f365fb96770fbcd3ae8c0217 Mon Sep 17 00:00:00 2001 From: Bence Kosztolnik Date: Fri, 22 Nov 2024 22:58:47 +0100 Subject: [PATCH 4/4] Update by review Change-Id: I8feab0406d46892daa954a1e41eb193f031ee77a --- .../org/apache/hadoop/fs/CommonConfigurationKeysPublic.java | 2 +- .../hadoop-common/src/main/resources/core-default.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index e35747f458dda..c3364c934eb4e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -1006,7 +1006,7 @@ public class CommonConfigurationKeysPublic { "hadoop.security.credstore.java-keystore-provider.password-file"; public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY = - "secret-manager.key-generator.algorithm"; + "hadoop.security.secret-manager.key-generator.algorithm"; public static final String HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT = "HmacSHA1"; diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 305cc32889183..9d1bc79c1598a 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1047,7 +1047,7 @@ - secret-manager.key-generator.algorithm + hadoop.security.secret-manager.key-generator.algorithm HmacSHA1 The configuration key specifying the KeyGenerator algorithm used in SecretManager