Skip to content

Preserve sub-second precision when serializing unixTimestamp parameters #10644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jonathan343 wants to merge 1 commit into
base: v2
Choose a base branch
from
+196 −18
Open

Preserve sub-second precision when serializing unixTimestamp parameters #10644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changes/next-release/bugfix-Serialization-52170.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"type": "bugfix",
"category": "Serialization",
"description": "Preserve sub-second precision when serializing ``unixTimestamp`` request parameters. Timestamps with a fractional component are now sent as a fractional value (e.g. ``1704110400.123456``) instead of being truncated to whole seconds. Whole-second timestamps are unchanged."
}
29 changes: 13 additions & 16 deletions awscli/botocore/serialize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,14 @@ def _timestamp_iso8601(self, value):
return value.strftime(timestamp_format)

def _timestamp_unixtimestamp(self, value):
return int(calendar.timegm(value.timetuple()))
timestamp = calendar.timegm(value.timetuple())
if value.microsecond > 0:
# Add the microseconds as integers before dividing so the float is only
# rounded once. Dividing first and then adding rounds twice, which can
# produce a slightly different value, e.g. 1 + 3691 / 10**6 gives
# 1.0036909999999999 instead of 1.003691.
return (timestamp * 10**6 + value.microsecond) / 10**6
return timestamp

def _timestamp_rfc822(self, value):
if isinstance(value, datetime.datetime):
Expand Down Expand Up @@ -617,22 +624,12 @@ def _serialize_type_timestamp(self, serialized, value, shape, key):
tag = 1 # Use tag 1 for unix timestamp
initial_byte = self._get_initial_byte(self.TAG_MAJOR_TYPE, tag)
serialized.extend(initial_byte) # Tagging the timestamp
additional_info, num_bytes = self._get_additional_info_and_num_bytes(
timestamp
)

if num_bytes == 0:
initial_byte = self._get_initial_byte(
self.UNSIGNED_INT_MAJOR_TYPE, timestamp
)
serialized.extend(initial_byte)
# Tag 1 permits either an integer or a floating-point epoch seconds

@GarrettBeatty GarrettBeatty Sep 16, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

code seems fine. just for my understanding why did we have all the _get_additional_info_and_num_bytes logic before and why can it be removed now?

@jonathan343 jonathan343 Sep 17, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The _serialize_type_integer we now use calls the same _get_additional_info_and_num_bytes helper under the hood. I'm honestly not sure why this wasn't called in the initial implementation.

But the current behavior is incorrect since we can't serialize negative timestamps. A timestamp value like datetime(1968, 12, 31, 23, 59, 59) currently results in a OverflowError: can't convert negative int to unsigned error which this new approach addresses.

# value; a float is used when sub-second precision is present.
if isinstance(timestamp, float):
self._serialize_type_double(serialized, timestamp, shape, key)
else:
initial_byte = self._get_initial_byte(
self.UNSIGNED_INT_MAJOR_TYPE, additional_info
)
serialized.extend(
initial_byte + timestamp.to_bytes(num_bytes, "big")
)
self._serialize_type_integer(serialized, timestamp, shape, key)

def _serialize_type_float(self, serialized, value, shape, key):
if self._is_special_number(value):
Expand Down
180 changes: 178 additions & 2 deletions tests/unit/botocore/test_serialize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import decimal
import io
import json
import struct

import dateutil.tz
from botocore import serialize
Expand Down Expand Up @@ -615,6 +616,177 @@ def test_restxml_serializes_unicode(self):
except UnicodeEncodeError:
self.fail("RestXML serializer failed to serialize unicode text.")


class TestTimestampPrecision(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'UnixTimestamp': {'shape': 'UnixTimestampType'},
'IsoTimestamp': {'shape': 'IsoTimestampType'},
'Rfc822Timestamp': {'shape': 'Rfc822TimestampType'},
},
},
'IsoTimestampType': {
'type': 'timestamp',
"timestampFormat": "iso8601",
},
'UnixTimestampType': {
'type': 'timestamp',
"timestampFormat": "unixTimestamp",
},
'Rfc822TimestampType': {
'type': 'timestamp',
"timestampFormat": "rfc822",
},
},
}
self.service_model = ServiceModel(self.model)

def serialize_to_request(self, input_params):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol']
)
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation')
)

def test_whole_seconds_serialize_without_fraction(self):
test_datetime = datetime.datetime(2024, 1, 1, 12, 0, 0)
request = self.serialize_to_request(
{'UnixTimestamp': test_datetime, 'IsoTimestamp': test_datetime}
)
self.assertEqual(request['body']['UnixTimestamp'], 1704110400)
self.assertIsInstance(request['body']['UnixTimestamp'], int)
self.assertEqual(
request['body']['IsoTimestamp'], '2024-01-01T12:00:00Z'
)

def test_millisecond_precision_is_preserved(self):
test_datetime = datetime.datetime(2024, 1, 1, 12, 0, 0, 123000)
request = self.serialize_to_request(
{'UnixTimestamp': test_datetime, 'IsoTimestamp': test_datetime}
)
self.assertEqual(request['body']['UnixTimestamp'], 1704110400.123)
self.assertEqual(
request['body']['IsoTimestamp'], '2024-01-01T12:00:00.123000Z'
)

def test_microsecond_precision_is_preserved(self):
test_datetime = datetime.datetime(2024, 1, 1, 12, 0, 0, 123456)
request = self.serialize_to_request(
{'UnixTimestamp': test_datetime, 'IsoTimestamp': test_datetime}
)
self.assertEqual(request['body']['UnixTimestamp'], 1704110400.123456)
self.assertEqual(
request['body']['IsoTimestamp'], '2024-01-01T12:00:00.123456Z'
)

def test_unix_timestamp_fraction_round_trips_through_repr(self):
# The float must render with exactly the digits the caller provided
# (no binary floating point noise) when written to a request body.
test_datetime = datetime.datetime(2024, 1, 1, 12, 0, 0, 123456)
request = self.serialize_to_request({'UnixTimestamp': test_datetime})
self.assertEqual(
str(request['body']['UnixTimestamp']), '1704110400.123456'
)

def test_unix_timestamp_before_epoch_with_fraction(self):
test_datetime = datetime.datetime(1969, 12, 31, 23, 59, 59, 250000)
request = self.serialize_to_request({'UnixTimestamp': test_datetime})
self.assertEqual(request['body']['UnixTimestamp'], -0.75)

def test_rfc822_timestamp_always_uses_second_precision(self):
# RFC822 format doesn't support sub-second precision.
test_datetime = datetime.datetime(2024, 1, 1, 12, 0, 0, 123456)
request = self.serialize_to_request({'Rfc822Timestamp': test_datetime})
self.assertEqual(
request['body']['Rfc822Timestamp'],
'Mon, 01 Jan 2024 12:00:00 GMT',
)


class TestRpcV2CBORTimestampSerialization(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {
'protocol': 'smithy-rpc-v2-cbor',
'apiVersion': '2014-01-01',
'serviceId': 'MyService',
'targetPrefix': 'sampleservice',
'documentation': '',
},
'operations': {
'TestOperation': {
'name': 'TestOperation',
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'Timestamp': {'shape': 'TimestampType'},
},
},
'TimestampType': {'type': 'timestamp'},
},
}
self.service_model = ServiceModel(self.model)

def serialize_timestamp(self, value):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol']
)
request = request_serializer.serialize_to_request(
{'Timestamp': value},
self.service_model.operation_model('TestOperation'),
)
# Skip the leading map header and "Timestamp" key; return the
# encoded value only.
body = bytes(request['body'])
prefix = b'\xa1\x69Timestamp'
self.assertTrue(body.startswith(prefix))
return body[len(prefix) :]

def test_whole_seconds_serialize_as_tagged_integer(self):
encoded = self.serialize_timestamp(
datetime.datetime(2024, 1, 1, 12, 0, 0)
)
# Tag 1, then uint32 1704110400
self.assertEqual(encoded, b'\xc1\x1a\x65\x92\xa9\x40')

def test_fractional_seconds_serialize_as_tagged_double(self):
encoded = self.serialize_timestamp(
datetime.datetime(2024, 1, 1, 12, 0, 0, 500000)
)
# Tag 1, then float64 1704110400.5
self.assertEqual(
encoded, b'\xc1\xfb' + struct.pack('>d', 1704110400.5)
)

def test_negative_timestamp_serializes_as_tagged_negative_integer(self):
encoded = self.serialize_timestamp(
datetime.datetime(1969, 12, 31, 23, 59, 59)
)
# Tag 1, then negative integer -1 (major type 1, value 0)
self.assertEqual(encoded, b'\xc1\x20')


class TestRpcV2CBORHostPrefix(unittest.TestCase):
def setUp(self):
self.model = {
Expand Down Expand Up @@ -649,7 +821,9 @@ def setUp(self):
self.service_model = ServiceModel(self.model)

def test_host_prefix_added_to_serialized_request(self):
operation_model = self.service_model.operation_model('TestHostPrefixOperation')
operation_model = self.service_model.operation_model(
'TestHostPrefixOperation'
)
serializer = serialize.create_serializer('smithy-rpc-v2-cbor')

params = {'Foo': 'bound'}
Expand All @@ -658,7 +832,9 @@ def test_host_prefix_added_to_serialized_request(self):
self.assertEqual(serialized['host_prefix'], 'bound')

def test_no_host_prefix_when_not_configured(self):
operation_model = self.service_model.operation_model('TestNoHostPrefixOperation')
operation_model = self.service_model.operation_model(
'TestNoHostPrefixOperation'
)
serializer = serialize.create_serializer('smithy-rpc-v2-cbor')

params = {'Foo': 'bound'}
Expand Down
Loading