diff --git a/eventsourcingdb/container.py b/eventsourcingdb/container.py
index 484b9b9..c6646d3 100644
--- a/eventsourcingdb/container.py
+++ b/eventsourcingdb/container.py
@@ -64,6 +64,8 @@ def _create_container(self) -> None:
             "--data-directory-temporary",
             "--http-enabled",
             "--https-enabled=false",
+            "--http-port",
+            str(self._internal_port),
         ]
 
         if self._signing_key is not None:
diff --git a/tests/test_container.py b/tests/test_container.py
new file mode 100644
index 0000000..7af6d8c
--- /dev/null
+++ b/tests/test_container.py
@@ -0,0 +1,22 @@
+import pytest
+
+from eventsourcingdb import Container
+
+from .shared.database import Database
+
+
+class TestContainer:
+    @staticmethod
+    @pytest.mark.asyncio
+    async def test_starts_with_a_custom_port() -> None:
+        image_tag = Database._get_image_tag_from_dockerfile()
+        container = Container().with_image_tag(image_tag).with_port(4000)
+        container.start()
+
+        try:
+            client = container.get_client()
+
+            # Should not throw.
+            await client.ping()
+        finally:
+            container.stop()