refactor the process of the validation of the metadata

src/syngen/VERSION

@@ -1 +1 @@
-0.9.32
+0.9.33

src/syngen/ml/config/validation.py

@@ -58,13 +58,22 @@ def _define_mapping(self):
                     "parent_columns": key_data["references"]["columns"],
                 }
 
+    def _check_conditions(self, metadata: Dict) -> bool:
+        """
+        Check conditions whether to launch validation or not
+        """
+        print_report = metadata.get("train_settings", {}).get("print_report", False)
+        return (
+            self.type_of_process == "infer"
+            or (self.type_of_process == "train" and print_report is True)
+        )
+
     def _validate_metadata(self, table_name: str):
         """
         Validate the metadata
         """
         metadata_of_the_table = self.metadata[table_name]
         table_keys = metadata_of_the_table.get("keys", {})
-        print_report = metadata_of_the_table.get("train_settings", {}).get("print_report", False)
         for key, config in table_keys.items():
             if config["type"] not in self.type_of_fk_keys:
                 continue
@@ -75,9 +84,7 @@ def _validate_metadata(self, table_name: str):
             )
             parent_table = self.mapping[key]["parent_table"]
             if parent_table not in self.metadata:
-                if self.type_of_process == "infer" or (
-                    self.type_of_process == "train" and print_report is True
-                ):
+                if self._check_conditions(metadata_of_the_table):
                     self._check_existence_of_success_file(parent_table)
                     self._check_existence_of_generated_data(parent_table)
                 elif self.type_of_process == "train":

src/syngen/ml/vae/models/dataset.py

@@ -711,6 +711,7 @@ def __define_date_format(date_text: pd.DataFrame):
             r"[A-Z][a-z]+ \d{1,2} \d{4}|"
             r"[A-Z][a-z]+ \d{1,2}, \d{4}|"
             r"\d{2} [A-Z][a-z]+ \d{4}|"
+            r"\d{2}-[A-Z]{3}-\d{2}|"
             r"\d{2}[-][A-Z][a-z]+[-]\d{2}|"
             r"\d{4}[-/\\]\d{1,2}"
         )

src/tests/unit/dataset/test_dataset.py

@@ -299,23 +299,30 @@ def test_check_non_existent_columns(rp_logger):
 
 
 @pytest.mark.parametrize(
-    "initial_date_format, expected_date_format",
+    "initial_date_format, expected_date_format, upper_case",
     [
-        ("%m-%d-%Y", "%m-%d-%Y"),
-        ("%d-%m-%Y", "%d-%m-%Y"),
-        ("%m/%d/%Y", "%m/%d/%Y"),
-        ("%d/%m/%Y", "%d/%m/%Y"),
-        ("%Y/%m/%d", "%Y/%m/%d"),
-        ("%Y-%m-%d", "%Y-%m-%d"),
-        ("%B %d, %Y", "%B %d, %Y"),
-        ("%b %d, %Y", "%b %d, %Y"),
-        ("%d %B %Y", "%d %B %Y"),
-        ("%b %d %Y", "%b %d %Y"),
-        ("%d.%m.%Y", "%d.%m.%Y"),
-        ("%m-%b-%y", "%d-%m-%Y"),
+        ("%m-%d-%Y", "%m-%d-%Y", False),
+        ("%d-%b-%y", "%d-%m-%Y", True),
+        ("%d-%m-%y", "%d-%m-%Y", False),
+        ("%d-%m-%Y", "%d-%m-%Y", False),
+        ("%m/%d/%Y", "%m/%d/%Y", False),
+        ("%d/%m/%Y", "%d/%m/%Y", False),
+        ("%Y/%m/%d", "%Y/%m/%d", False),
+        ("%Y-%m-%d", "%Y-%m-%d", False),
+        ("%B %d, %Y", "%B %d, %Y", False),
+        ("%b %d, %Y", "%b %d, %Y", False),
+        ("%d %B %Y", "%d %B %Y", False),
+        ("%b %d %Y", "%b %d %Y", False),
+        ("%d.%m.%Y", "%d.%m.%Y", False),
+        ("%m-%b-%y", "%d-%m-%Y", False),
     ]
 )
-def test_define_date_format_with_diff_format(initial_date_format, expected_date_format, rp_logger):
+def test_define_date_format_with_diff_format(
+    initial_date_format,
+    expected_date_format,
+    upper_case,
+    rp_logger
+):
     rp_logger.info(
         "Test the process of identifying the date format in the date column "
         "where the initial date format - %s and expected date format: %s",
@@ -327,10 +334,18 @@ def test_define_date_format_with_diff_format(initial_date_format, expected_date_
             "keys": {}
         }
     }
-    data = {
-        "Date": [(datetime.datetime(2020, 1, 1) + datetime.timedelta(days=x)).
-                 strftime(initial_date_format) for x in range(10000)]
-    }
+
+    if upper_case:
+        data = {
+            "Date": [(datetime.datetime(2020, 1, 1) + datetime.timedelta(days=x)).
+                     strftime(initial_date_format).upper() for x in range(10000)]
+        }
+    else:
+        data = {
+            "Date": [(datetime.datetime(2020, 1, 1) + datetime.timedelta(days=x)).
+                     strftime(initial_date_format) for x in range(10000)]
+        }
+
     df = pd.DataFrame(data, columns=["Date"])
     with patch("syngen.ml.vae.models.dataset.fetch_config", lambda x: MagicMock()):
         mock_dataset = Dataset(

src/tests/unit/validation_metadata/test_validation_metadata.py

@@ -777,6 +777,7 @@ def test_validate_metadata_of_related_tables_with_several_fk_key_in_infer_proces
     rp_logger.info(SUCCESSFUL_MESSAGE)
 
 
+@patch.object(Validator, "_check_existence_of_generated_data")
 @patch.object(Validator, "_check_existence_of_success_file")
 @patch.object(Validator, "_validate_referential_integrity")
 @patch.object(Validator, "_check_key_columns")
@@ -786,6 +787,7 @@ def test_validate_incomplete_metadata_contained_fk_key_in_train_process_without_
     mock_check_key_columns,
     mock_validate_referential_integrity,
     mock_check_existence_of_success_file,
+    mock_check_existence_of_generated_data,
     test_metadata_storage,
     rp_logger
 ):
@@ -870,6 +872,7 @@ def test_validate_incomplete_metadata_contained_fk_key_in_train_process_without_
     assert mock_check_key_columns.call_count == 2
     mock_validate_referential_integrity.assert_called_once()
     mock_check_existence_of_success_file.assert_called_once()
+    mock_check_existence_of_generated_data.assert_not_called()
     rp_logger.info(SUCCESSFUL_MESSAGE)