Update hive-lineage.sh (#1306)

* [hive-lineage] update hive confwhitelist to allow hive openlineage properties at runtime Signed-off-by: Pahulpreet Singh <[email protected]> * [hive-lineage] add KERBEROS clusters to hive lineage test Signed-off-by: Pahulpreet Singh <[email protected]> * [hive-lineage] specify openlineage namespace while submitting job in test_hive_lineage Signed-off-by: Pahulpreet Singh <[email protected]> --------- Signed-off-by: Pahulpreet Singh <[email protected]>
GoogleCloudDataproc · Feb 26, 2025 · cf2e0f8 · cf2e0f8
1 parent f458d05
commit cf2e0f8
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/hive-lineage/hive-lineage.sh b/hive-lineage/hive-lineage.sh
@@ -34,7 +34,8 @@ function set_hive_lineage_conf() {
     ["hive.exec.post.hooks"]="$HIVE_OL_HOOK"
     ["hive.exec.failure.hooks"]="$HIVE_OL_HOOK"
     ["hive.openlineage.transport.type"]="gcplineage"
-    ["hive.conf.validation"]="false" # to allow custom properties, like hive.openlineage.namespace
+    ["hive.security.authorization.sqlstd.confwhitelist.append"]="tez.application.tags|hive.openlineage.*"
+    ["hive.conf.validation"]="false"
   )
   echo "Setting hive conf to enable lineage"
   for key in "${!properties[@]}"; do

diff --git a/hive-lineage/test_hive_lineage.py b/hive-lineage/test_hive_lineage.py
@@ -9,15 +9,20 @@ class HiveLineageTestCase(DataprocTestCase):
   TEST_SCRIPT_FILE = "hive-lineage/hivetest.hive"
 
   def __submit_hive_job(self, cluster_name):
-    self.assert_dataproc_job(
-        cluster_name, 'hive', '--file={}/{}'.format(self.INIT_ACTIONS_REPO,
-                                                    self.TEST_SCRIPT_FILE))
+    properties = "hive.openlineage.namespace=init-actions-test"
+    self.assert_dataproc_job(cluster_name, 'hive',
+                             '--file={}/{} --properties={}'.format(
+                                 self.INIT_ACTIONS_REPO,
+                                 self.TEST_SCRIPT_FILE,
+                                 properties))
+
   def verify_cluster(self, name):
     self.__submit_hive_job(name)
 
   @parameterized.parameters(
       'STANDARD',
       'HA',
+      'KERBEROS',
   )
   def test_hive_job_success(self, configuration):
     self.createCluster(configuration,