Add unique constraints for data validation

src/etl/data-validation.schema.json

@@ -71,7 +71,9 @@
                 "properties": {
                   "column_list": {
                     "type": "array",
-                    "items": { "type": "string" }
+                    "items": { "type": "string" },
+                    "minItems": 2,
+                    "uniqueItems": true
                   }
                 },
                 "required": ["column_list"]
@@ -96,7 +98,8 @@
                   },
                   "column_set": {
                     "type": "array",
-                    "items": { "type": "string" }
+                    "items": { "type": "string" },
+                    "uniqueItems": true
                   }
                 },
                 "required": ["column_set"],
@@ -136,7 +139,8 @@
                         { "required": ["dtype"] },
                         { "required": ["dtype_list"] }
                       ]
-                    }
+                    },
+                    "uniqueItems": true
                   },
                   "exact_match": {
                     "type": "boolean"

tests/fixtures/etl/data-validation/fail.yaml

+jobs:
+  job1:
+    inputs:
+      input1:
+        - type: great-expectations
+          name: expect_column_values_to_be_unique
+          opts:
+            column: user_id
+  job2:
+    inputs:
+      input1:
+        - type: great-expectations
+          name: expect_compound_columns_to_be_unique
+          opts:
+            column_list:
+              - user_id
+              - user_id

tests/fixtures/etl/data-validation/job.yaml

@@ -74,3 +74,12 @@ jobs:
                   - double
               - name: account_id
                 dtype: int64
+  job6:
+    inputs:
+      input1:
+        - type: great-expectations
+          name: expect_compound_columns_to_be_unique
+          opts:
+            column_list:
+              - user_id
+              - account_id

tests/test_schemas.py

@@ -7,10 +7,19 @@ def test_etl_spec_schemas():
     for file in ["tests/fixtures/etl/specs/dummy.yaml", "tests/fixtures/etl/specs/job.yaml"]:
         validate_yaml(schema_file=etl_spec, input_file=file)
 
-def test_etl_data_validation_schemas():
+def test_etl_data_validation_schemas_success():
     """
     Test ETL Data Validation Schema
     """
     etl_spec = "src/etl/data-validation.schema.json"
     for file in ["tests/fixtures/etl/data-validation/dummy.yaml", "tests/fixtures/etl/data-validation/job.yaml"]:
         validate_yaml(schema_file=etl_spec, input_file=file)
+
+
+def test_etl_data_validation_schemas_fail():
+    """
+    Test ETL Data Validation Schema
+    """
+    etl_spec = "src/etl/data-validation.schema.json"
+    for file in ["tests/fixtures/etl/data-validation/fail.yaml",]:
+        validate_yaml(schema_file=etl_spec, input_file=file)