Adds optional step param to expect_column_values_to_be_{in,de}creas…

…ing (#316) If set, the test will check that the values are not only strictly increasing/decreasing, but the difference between subsequent values is exactly `step`.
calogica · Sep 10, 2024 · c9abed7 · c9abed7
1 parent 4249dcc
commit c9abed7
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -566,6 +566,7 @@ tests:
       row_condition: "id is not null" # (Optional)
       strictly: true # (Optional for comparison operator. Default is 'true', and it uses '>'. If set to 'false' it uses '>='.)
       group_by: [group_id, other_group_id, ...] # (Optional)
+      step: 1 # (Optional. If set, it requires the difference between values to be exactly this step. Requires numeric columns.)
 ```
 
 ### [expect_column_values_to_be_decreasing](macros/schema_tests/column_values_basic/expect_column_values_to_be_decreasing.sql)
@@ -583,6 +584,7 @@ tests:
       row_condition: "id is not null" # (Optional)
       strictly: true # (Optional for comparison operator. Default is 'true' and it uses '<'. If set to 'false', it uses '<='.)
       group_by: [group_id, other_group_id, ...] # (Optional)
+      step: 1 # (Optional. If set, it requires the difference between values to be exactly this step. Requires numeric columns.)
 ```
 
 ### [expect_column_value_lengths_to_be_between](macros/schema_tests/string_matching/expect_column_value_lengths_to_be_between.sql)

diff --git a/integration_tests/models/schema_tests/data_test.sql b/integration_tests/models/schema_tests/data_test.sql
@@ -7,7 +7,9 @@ select
     'b' as col_string_b,
     cast(null as {{ dbt.type_string() }}) as col_null,
     cast(null as {{ dbt.type_string() }}) as col_null_2,
-    1.0 as col_numeric_a_plus_b
+    1.0 as col_numeric_a_plus_b,
+    2 as idx_multiplied_by_2,
+    -2 as idx_multiplied_by_minus_2
 
 union all
 
@@ -20,7 +22,9 @@ select
     'ab' as col_string_b,
     null as col_null,
     null as col_null_2,
-    1.0 as col_numeric_a_plus_b
+    1.0 as col_numeric_a_plus_b,
+    4 as idx_multiplied_by_2,
+    -4 as idx_multiplied_by_minus_2
 
 union all
 
@@ -33,7 +37,9 @@ select
     'abc' as col_string_b,
     null as col_null,
     null as col_null_2,
-    1.0 as col_numeric_a_plus_b
+    1.0 as col_numeric_a_plus_b,
+    6 as idx_multiplied_by_2,
+    -6 as idx_multiplied_by_minus_2
 
 union all
 
@@ -46,4 +52,6 @@ select
     'abcd' as col_string_b,
     null as col_null,
     null as col_null_2,
-    1.0 as col_numeric_a_plus_b
+    1.0 as col_numeric_a_plus_b,
+    8 as idx_multiplied_by_2,
+    -8 as idx_multiplied_by_minus_2
diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml
@@ -417,20 +417,20 @@ models:
           row_condition: 1=1
           compare_row_condition: 1=1
       - dbt_expectations.expect_table_column_count_to_equal:
-          value: 9
+          value: 11
       - dbt_expectations.expect_table_column_count_to_be_between:
           min_value: 1
-          max_value: 10
+          max_value: 11
       - dbt_expectations.expect_table_column_count_to_be_between:
           min_value: 1
       - dbt_expectations.expect_table_column_count_to_be_between:
-          max_value: 10
+          max_value: 11
       - dbt_expectations.expect_table_columns_to_contain_set:
           column_list: ["col_numeric_b", "col_string_a"]
       - dbt_expectations.expect_table_columns_to_match_set:
-          column_list: ["idx", "date_col", "col_numeric_a", "col_numeric_b", "col_string_a", "col_string_b", "col_null", "col_null_2", "col_numeric_a_plus_b"]
+          column_list: ["idx", "date_col", "col_numeric_a", "col_numeric_b", "col_string_a", "col_string_b", "col_null", "col_null_2", "col_numeric_a_plus_b", "idx_multiplied_by_2", "idx_multiplied_by_minus_2"]
       - dbt_expectations.expect_table_columns_to_match_ordered_list:
-          column_list: ["idx", "date_col", "col_numeric_a", "col_numeric_b", "col_string_a", "col_string_b", "col_null", "col_null_2", "col_numeric_a_plus_b"]
+          column_list: ["idx", "date_col", "col_numeric_a", "col_numeric_b", "col_string_a", "col_string_b", "col_null", "col_null_2", "col_numeric_a_plus_b", "idx_multiplied_by_2", "idx_multiplied_by_minus_2"]
       - dbt_expectations.expect_table_column_count_to_equal_other_table:
           compare_model: ref("data_test")
       - dbt_expectations.expect_table_columns_to_not_contain_set:
@@ -579,6 +579,18 @@ models:
         tests:
           - dbt_expectations.expect_column_values_to_be_null
 
+      - name: idx_multiplied_by_2
+        tests:
+          - dbt_expectations.expect_column_values_to_be_increasing:
+              sort_column: idx
+              step: 2
+
+      - name: idx_multiplied_by_minus_2
+        tests:
+          - dbt_expectations.expect_column_values_to_be_decreasing:
+              sort_column: idx
+              step: 2
+
   - name : data_test_factored
     tests :
       - dbt_expectations.expect_table_row_count_to_equal_other_table_times_factor:

diff --git a/macros/schema_tests/column_values_basic/expect_column_values_to_be_decreasing.sql b/macros/schema_tests/column_values_basic/expect_column_values_to_be_decreasing.sql
@@ -2,7 +2,8 @@
                                                    sort_column=None,
                                                    strictly=True,
                                                    row_condition=None,
-                                                   group_by=None) %}
+                                                   group_by=None,
+                                                   step=None) %}
 
 {%- set sort_column = column_name if not sort_column else sort_column -%}
 {%- set operator = "<" if strictly else "<=" %}
@@ -44,7 +45,12 @@ validation_errors as (
     where
         value_field_lag is not null
         and
-        not (value_field {{ operator }} value_field_lag)
+        not (
+            (value_field {{ operator }} value_field_lag)
+            {%- if step %}
+            and ((value_field_lag - value_field) = {{ step }})
+            {%- endif %}
+        )
 
 )
 select *

diff --git a/macros/schema_tests/column_values_basic/expect_column_values_to_be_increasing.sql b/macros/schema_tests/column_values_basic/expect_column_values_to_be_increasing.sql
@@ -2,7 +2,8 @@
                                                    sort_column=None,
                                                    strictly=True,
                                                    row_condition=None,
-                                                   group_by=None) %}
+                                                   group_by=None,
+                                                   step=None) %}
 
 {%- set sort_column = column_name if not sort_column else sort_column -%}
 {%- set operator = ">" if strictly else ">=" -%}
@@ -46,7 +47,12 @@ validation_errors as (
     where
         value_field_lag is not null
         and
-        not (value_field {{ operator }} value_field_lag)
+        not (
+            (value_field {{ operator }} value_field_lag)
+            {%- if step %}
+            and ((value_field - value_field_lag) = {{ step }})
+            {%- endif %}
+        )
 
 )
 select *