diff --git a/iotdb-client/client-py/iotdb/utils/rpc_utils.py b/iotdb-client/client-py/iotdb/utils/rpc_utils.py index 0912fd14d95de..601cd99cf7406 100644 --- a/iotdb-client/client-py/iotdb/utils/rpc_utils.py +++ b/iotdb-client/client-py/iotdb/utils/rpc_utils.py @@ -17,6 +17,7 @@ # import logging +import numpy as np import pandas as pd from pandas._libs import OutOfBoundsDatetime from tzlocal import get_localzone_name @@ -81,10 +82,12 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): try: ts = pd.Timestamp(time, unit=precision, tz=timezone) except OutOfBoundsDatetime: + # Preserve the input unit: pandas 3.1 may overflow when constructing + # extreme millisecond timestamps directly from an integer. try: - ts = pd.Timestamp(time, unit=precision).tz_localize(timezone) + ts = pd.Timestamp(np.datetime64(int(time), precision)).tz_localize(timezone) except NotImplementedError: - return pd.Timestamp(time, unit=precision) + return pd.Timestamp(np.datetime64(int(time), precision)) except ValueError: logger.warning( f"Timezone string '{timezone}' cannot be recognized by pandas. " @@ -95,7 +98,7 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): # Timestamp falls outside Python's stdlib datetime range (year < 1 or # > 9999). pandas >= 3.0 cannot attach a timezone to such Timestamps # because tz conversion relies on toordinal(). Return naive instead. - return pd.Timestamp(time, unit=precision) + return pd.Timestamp(np.datetime64(int(time), precision)) # Construction succeeded but utcoffset() may still fail downstream (e.g. # in pd.DataFrame's tz alignment) when the Timestamp's year is outside # 1..9999 and tz is not the canonical UTC. Probe early and fall back to @@ -105,7 +108,7 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): try: ts.utcoffset() except NotImplementedError: - return pd.Timestamp(time, unit=precision) + return pd.Timestamp(np.datetime64(int(time), precision)) return ts diff --git a/iotdb-client/client-py/tests/unit/test_rpc_utils.py b/iotdb-client/client-py/tests/unit/test_rpc_utils.py new file mode 100644 index 0000000000000..c34e6e4a961de --- /dev/null +++ b/iotdb-client/client-py/tests/unit/test_rpc_utils.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from datetime import timedelta + +import numpy as np +import pandas as pd +import pytest + +from iotdb.utils.rpc_utils import convert_to_timestamp, isoformat + + +@pytest.mark.parametrize("precision", ["ms", "us", "ns"]) +@pytest.mark.parametrize("value", [-(2**63) + 1, 2**63 - 1]) +def test_timestamp_boundaries_keep_value_and_precision(precision, value): + for scalar in (value, np.int64(value)): + for timezone in ("UTC", "Etc/UTC"): + timestamp = convert_to_timestamp(scalar, precision, timezone) + assert timestamp.asm8.dtype == np.dtype(f"datetime64[{precision}]") + assert timestamp.asm8.astype(np.int64) == value + if timestamp.tzinfo is not None: + assert timestamp.utcoffset() == timedelta(0) + expected = np.datetime_as_string(np.datetime64(value, precision)) + formatted = isoformat(timestamp, precision) + if formatted.endswith("+00:00"): + formatted = formatted[:-6] + assert formatted == expected + + +@pytest.mark.parametrize("precision", ["ms", "us", "ns"]) +@pytest.mark.parametrize("timezone", ["UTC", "Asia/Shanghai"]) +def test_normal_timestamp_keeps_timezone(precision, timezone): + value = 123456789 + timestamp = convert_to_timestamp(value, precision, timezone) + assert timestamp == pd.Timestamp(value, unit=precision, tz=timezone) + assert str(timestamp.tzinfo) == timezone + + +def test_timestamp_nat_is_preserved(): + for precision in ("ms", "us", "ns"): + assert convert_to_timestamp(-(2**63), precision, "Etc/UTC") is pd.NaT