diff --git a/CHANGES.md b/CHANGES.md index b8ff482..2397094 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,8 @@ ## Unreleased - DynamoDB: Change CrateDB data model to use (`pk`, `data`, `aux`) columns Attention: This is a breaking change. +- MongoDB: Handle too large `$date.$numberLong` values gracefully +- Zyp/Moksha: Improve error reporting when rule evaluation fails ## 2024/09/26 v0.0.19 - DynamoDB CDC: Fix `MODIFY` operation by propagating `NewImage` fully diff --git a/src/commons_codec/transform/mongodb.py b/src/commons_codec/transform/mongodb.py index 2b67f96..744ed54 100644 --- a/src/commons_codec/transform/mongodb.py +++ b/src/commons_codec/transform/mongodb.py @@ -123,7 +123,11 @@ def decode_extended_json(self, value: t.Dict[str, t.Any]) -> t.Any: type_ = next(iter(value)) # Get key of first item in dictionary. is_date_numberlong = type_ == "$date" and "$numberLong" in value["$date"] if is_date_numberlong: - out = dt.datetime.fromtimestamp(int(value["$date"]["$numberLong"]) / 1000, tz=dt.timezone.utc) + try: + out = dt.datetime.fromtimestamp(int(value["$date"]["$numberLong"]) / 1000, tz=dt.timezone.utc) + except ValueError as ex: + logger.error(f"Decoding legacy timestamp failed: {ex}. value={value}") + out = 0 else: out = object_hook(value) diff --git a/src/zyp/model/moksha.py b/src/zyp/model/moksha.py index 6b6dd07..f901274 100644 --- a/src/zyp/model/moksha.py +++ b/src/zyp/model/moksha.py @@ -22,11 +22,14 @@ class MokshaRule: disabled: t.Optional[bool] = False def compile(self): - return MokshaRuntimeRule(self.type, compile_expression(self.type, self.expression), disabled=self.disabled) + return MokshaRuntimeRule( + self, self.type, compile_expression(self.type, self.expression), disabled=self.disabled + ) @define class MokshaRuntimeRule: + source: MokshaRule type: str transformer: MokshaTransformer disabled: t.Optional[bool] = False @@ -76,10 +79,11 @@ def apply(self, data: t.Any) -> t.Any: for rule in self._runtime_rules: try: data = rule.evaluate(data) - except Exception: - logger.exception(f"Error evaluating rule: {rule}") - if isinstance(data, map): - data = list(data) - logger.debug(f"Error payload:\n{data}") + except Exception as ex: + logger.error(f"Error evaluating rule: {ex}. Expression: {rule.source.expression}") + if logger.level is logging.DEBUG: + if isinstance(data, map): + data = list(data) + logger.debug(f"Error payload:\n{data}") raise return data diff --git a/tests/transform/mongodb/data.py b/tests/transform/mongodb/data.py index 037ae2a..82580df 100644 --- a/tests/transform/mongodb/data.py +++ b/tests/transform/mongodb/data.py @@ -58,7 +58,10 @@ "code_bytes": {"$code": "ab\u0000ab\u0000"}, "code_scope": {"$code": "abab", "$scope": {"x": {"$numberInt": "42"}}}, "date_iso8601": {"$date": "2015-09-23T10:32:42.33Z"}, - "date_numberlong": {"$date": {"$numberLong": "1356351330000"}}, + "date_numberlong_valid": {"$date": {"$numberLong": "1356351330000"}}, + "date_numberlong_invalid": { + "$date": {"$numberLong": "-9223372036854775808"} + }, # year -292275055 is out of range "dbref": { "$id": {"$oid": "56027fcae4b09385a85f9344"}, "$ref": "foo", @@ -169,7 +172,8 @@ }, }, "date_iso8601": 1443004362000, - "date_numberlong": 1356351330000, + "date_numberlong_valid": 1356351330000, + "date_numberlong_invalid": 0, "dbref": { "$id": "56027fcae4b09385a85f9344", "$ref": "foo", diff --git a/tests/zyp/moksha/test_model.py b/tests/zyp/moksha/test_model.py index 44370d6..94c3267 100644 --- a/tests/zyp/moksha/test_model.py +++ b/tests/zyp/moksha/test_model.py @@ -1,3 +1,4 @@ +import logging import re import pytest @@ -50,22 +51,27 @@ def test_moksha_transformation_success_jq(): def test_moksha_transformation_error_jq_scalar(caplog): + logging.getLogger("zyp.model.moksha").setLevel(logging.DEBUG) moksha = MokshaTransformation().jq(". /= 100") with pytest.raises(ValueError) as ex: moksha.apply("foo") assert ex.match(re.escape('string ("foo") and number (100) cannot be divided')) - assert "Error evaluating rule: MokshaRuntimeRule(type='jq'" in caplog.text + assert ( + 'Error evaluating rule: string ("foo") and number (100) cannot be divided. Expression: . /= 100' + in caplog.messages + ) assert "Error payload:\nfoo" in caplog.messages def test_moksha_transformation_error_jq_map(caplog): + logging.getLogger("zyp.model.moksha").setLevel(logging.DEBUG) moksha = MokshaTransformation().jq(".foo") with pytest.raises(ValueError) as ex: moksha.apply(map(lambda x: x, ["foo"])) # noqa: C417 assert ex.match(re.escape('Cannot index array with string "foo"')) - assert "Error evaluating rule: MokshaRuntimeRule(type='jq'" in caplog.text + assert 'Error evaluating rule: Cannot index array with string "foo". Expression: .foo' in caplog.messages assert "Error payload:\n[]" in caplog.messages