From 94c5cbdd7698a5d4e817c5f8823fc3ebe2ec5f7e Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:09:25 +0000 Subject: [PATCH 1/2] test(django-google-spanner): enable system tests and share a single per-run database --- packages/django-google-spanner/noxfile.py | 5 -- .../django-google-spanner/tests/settings.py | 7 +- .../django_spanner/test_check_constraint.py | 2 + .../system/django_spanner/test_decimal.py | 2 + .../system/django_spanner/test_json_field.py | 2 + .../system/django_spanner/test_queries.py | 2 + .../tests/system/django_spanner/utils.py | 83 +++++++++++-------- 7 files changed, 64 insertions(+), 39 deletions(-) diff --git a/packages/django-google-spanner/noxfile.py b/packages/django-google-spanner/noxfile.py index 1889aa16ef87..b951e3f2c475 100644 --- a/packages/django-google-spanner/noxfile.py +++ b/packages/django-google-spanner/noxfile.py @@ -232,11 +232,6 @@ def system_test(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): - # TODO(https://github.com/googleapis/google-cloud-python/issues/16099): - # System tests were disabled in the split repo to speed up the build process. - # See: https://github.com/googleapis/python-spanner-django/pull/929. - session.skip("System tests are temporarily disabled.") - print("System tests with django 5.2") system_test(session) diff --git a/packages/django-google-spanner/tests/settings.py b/packages/django-google-spanner/tests/settings.py index 77631946de8f..5eb4555bad94 100644 --- a/packages/django-google-spanner/tests/settings.py +++ b/packages/django-google-spanner/tests/settings.py @@ -5,6 +5,8 @@ # https://developers.google.com/open-source/licenses/bsd import os +import time +import uuid DEBUG = True USE_TZ = True @@ -29,7 +31,10 @@ PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project") # _get_test_db_name method in creation.py addes prefix of 'test_' to db name. -DATABASE_NAME = os.getenv("DJANGO_SPANNER_DB", "django_test_db") +DATABASE_NAME = os.getenv( + "DJANGO_SPANNER_DB", + f"dj_sys_{int(time.time())}_{uuid.uuid4().hex[:6]}", +) DATABASES = { "default": { diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py b/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py index ae546f2cd736..46efaac0232f 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py @@ -25,6 +25,8 @@ @unittest.skipIf(USE_EMULATOR, "Check Constraint is not implemented in emulator.") class TestCheckConstraint(TransactionTestCase): + available_apps = [] + @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py b/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py index 691a68882e29..1731c8b99b8a 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py @@ -20,6 +20,8 @@ class TestDecimal(TransactionTestCase): + available_apps = [] + @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py index 396bd8c4cfb1..c47a8fadb9e6 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py @@ -22,6 +22,8 @@ @unittest.skipIf(USE_EMULATOR, "Jsonfield is not implemented in emulator.") class TestJsonField(TransactionTestCase): + available_apps = [] + @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_queries.py b/packages/django-google-spanner/tests/system/django_spanner/test_queries.py index 2db465917716..a5cb4ac409ee 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_queries.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_queries.py @@ -19,6 +19,8 @@ class TestQueries(TransactionTestCase): + available_apps = [] + @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/utils.py b/packages/django-google-spanner/tests/system/django_spanner/utils.py index b4dd759570f8..5424403d56ec 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/utils.py +++ b/packages/django-google-spanner/tests/system/django_spanner/utils.py @@ -4,30 +4,37 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +import atexit import os import time -from django.core.management import call_command -from django.db import connection +from django.conf import settings +from django.db import connections from google.api_core import exceptions from google.cloud.spanner_v1 import Client from google.cloud.spanner_v1.instance import Backup, Instance from test_utils.retry import RetryErrors from django_spanner import USE_EMULATOR -from django_spanner.creation import DatabaseCreation CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int( - os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 60) + os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 240) ) EXISTING_INSTANCES = [] INSTANCE_ID = os.environ.get( - "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE", "spanner-django-python-systest" + "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE", + settings.DATABASES["default"]["INSTANCE"], +) +PROJECT_ID = os.getenv( + "GOOGLE_CLOUD_PROJECT", + settings.DATABASES["default"]["PROJECT"], +) +DATABASE_NAME = os.getenv( + "DJANGO_SPANNER_DB", + settings.DATABASES["default"]["NAME"], ) -PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project") -DATABASE_NAME = os.getenv("DJANGO_SPANNER_DB", "django_test_db") class Config(object): @@ -48,12 +55,15 @@ def _list_instances(): def setup_instance(): + if Config.INSTANCE is not None: + return + if USE_EMULATOR: from google.auth.credentials import AnonymousCredentials Config.CLIENT = Client(project=PROJECT_ID, credentials=AnonymousCredentials()) else: - Config.CLIENT = Client() + Config.CLIENT = Client(project=PROJECT_ID) retry = RetryErrors(exceptions.ServiceUnavailable) @@ -79,7 +89,8 @@ def setup_instance(): backup.delete() instance.delete() - if CREATE_INSTANCE: + Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) + if not Config.INSTANCE.exists(): if not USE_EMULATOR: # Defend against back-end returning configs for regions we aren't # actually allowed to use. @@ -96,48 +107,54 @@ def setup_instance(): Config.INSTANCE = Config.CLIENT.instance( INSTANCE_ID, config_name, labels=labels ) - if not Config.INSTANCE.exists(): + try: created_op = Config.INSTANCE.create() created_op.result( SPANNER_OPERATION_TIMEOUT_IN_SECONDS ) # block until completion - else: + except exceptions.AlreadyExists: Config.INSTANCE.reload() - else: - Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload() def teardown_instance(): - if CREATE_INSTANCE: - Config.INSTANCE.delete() + pass def setup_database(): + if Config.DATABASE is not None: + return + Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME) if not Config.DATABASE.exists(): - creation = DatabaseCreation(connection) - creation._create_test_db(verbosity=0, autoclobber=False, keepdb=True) - - # Running migrations on the db. - call_command("migrate", interactive=False) + try: + created_op = Config.DATABASE.create() + created_op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS) + except exceptions.AlreadyExists: + pass def teardown_database(): - from django.db import connections - connections.close_all() - Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME) - if Config.DATABASE.exists(): - Config.DATABASE.drop() - # Clear the global client cache to avoid Stale Session errors - # when the next test class creates a new database. - try: - import django_spanner.base - - django_spanner.base._SPANNER_CLIENT_CACHE = None - except ImportError: - pass +def _cleanup_test_resources(): + connections.close_all() + if Config.DATABASE is not None: + try: + if Config.DATABASE.exists(): + Config.DATABASE.drop() + except Exception: + pass + Config.DATABASE = None + if CREATE_INSTANCE and Config.INSTANCE is not None: + try: + if Config.INSTANCE.exists(): + Config.INSTANCE.delete() + except Exception: + pass + Config.INSTANCE = None + + +atexit.register(_cleanup_test_resources) From f3421be3db45719c502170cbdb71fba6194b0896 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:37:17 +0000 Subject: [PATCH 2/2] Addressed comments --- .../django-google-spanner/tests/settings.py | 6 -- .../django_spanner/test_check_constraint.py | 2 - .../system/django_spanner/test_decimal.py | 2 - .../system/django_spanner/test_json_field.py | 2 - .../system/django_spanner/test_queries.py | 2 - .../tests/system/django_spanner/utils.py | 98 ++++++------------- 6 files changed, 30 insertions(+), 82 deletions(-) diff --git a/packages/django-google-spanner/tests/settings.py b/packages/django-google-spanner/tests/settings.py index 5eb4555bad94..27942420e847 100644 --- a/packages/django-google-spanner/tests/settings.py +++ b/packages/django-google-spanner/tests/settings.py @@ -13,12 +13,6 @@ INSTALLED_APPS = [ "django_spanner", # Must be the first entry - "django.contrib.contenttypes", - "django.contrib.auth", - "django.contrib.sites", - "django.contrib.sessions", - "django.contrib.messages", - "django.contrib.staticfiles", "tests", ] diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py b/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py index 46efaac0232f..ae546f2cd736 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py @@ -25,8 +25,6 @@ @unittest.skipIf(USE_EMULATOR, "Check Constraint is not implemented in emulator.") class TestCheckConstraint(TransactionTestCase): - available_apps = [] - @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py b/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py index 1731c8b99b8a..691a68882e29 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py @@ -20,8 +20,6 @@ class TestDecimal(TransactionTestCase): - available_apps = [] - @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py index c47a8fadb9e6..396bd8c4cfb1 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py @@ -22,8 +22,6 @@ @unittest.skipIf(USE_EMULATOR, "Jsonfield is not implemented in emulator.") class TestJsonField(TransactionTestCase): - available_apps = [] - @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_queries.py b/packages/django-google-spanner/tests/system/django_spanner/test_queries.py index a5cb4ac409ee..2db465917716 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_queries.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_queries.py @@ -19,8 +19,6 @@ class TestQueries(TransactionTestCase): - available_apps = [] - @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/utils.py b/packages/django-google-spanner/tests/system/django_spanner/utils.py index 5424403d56ec..665cbcda3855 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/utils.py +++ b/packages/django-google-spanner/tests/system/django_spanner/utils.py @@ -12,17 +12,13 @@ from django.db import connections from google.api_core import exceptions from google.cloud.spanner_v1 import Client -from google.cloud.spanner_v1.instance import Backup, Instance -from test_utils.retry import RetryErrors +from google.cloud.spanner_v1.database import Database from django_spanner import USE_EMULATOR -CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None - SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int( os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 240) ) -EXISTING_INSTANCES = [] INSTANCE_ID = os.environ.get( "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE", settings.DATABASES["default"]["INSTANCE"], @@ -31,10 +27,6 @@ "GOOGLE_CLOUD_PROJECT", settings.DATABASES["default"]["PROJECT"], ) -DATABASE_NAME = os.getenv( - "DJANGO_SPANNER_DB", - settings.DATABASES["default"]["NAME"], -) class Config(object): @@ -45,13 +37,29 @@ class Config(object): """ CLIENT = None - INSTANCE_CONFIG = None INSTANCE = None DATABASE = None -def _list_instances(): - return list(Config.CLIENT.list_instances()) +def _cleanup_stale_databases(instance): + """Drop stale dj_sys__* databases older than 1 hour.""" + cutoff = int(time.time()) - 1 * 60 * 60 + try: + for db_pb in instance.list_databases(): + db = Database.from_pb(db_pb, instance) + parts = db.database_id.split("_") + if len(parts) >= 3 and parts[0] == "dj" and parts[1] == "sys": + try: + created_ts = int(parts[2]) + except ValueError: + continue + if created_ts < cutoff: + try: + db.drop() + except Exception: + pass + except Exception: + pass def setup_instance(): @@ -62,60 +70,20 @@ def setup_instance(): from google.auth.credentials import AnonymousCredentials Config.CLIENT = Client(project=PROJECT_ID, credentials=AnonymousCredentials()) - else: - Config.CLIENT = Client(project=PROJECT_ID) - - retry = RetryErrors(exceptions.ServiceUnavailable) - - configs = list(retry(Config.CLIENT.list_instance_configs)()) - - instances = retry(_list_instances)() - EXISTING_INSTANCES[:] = instances - - # Delete test instances that are older than an hour. - cutoff = int(time.time()) - 1 * 60 * 60 - instance_pbs = Config.CLIENT.list_instances("labels.python-spanner-systests:true") - for instance_pb in instance_pbs: - instance = Instance.from_pb(instance_pb, Config.CLIENT) - if "created" not in instance.labels: - continue - create_time = int(instance.labels["created"]) - if create_time > cutoff: - continue - if not USE_EMULATOR: - # Instance cannot be deleted while backups exist. - for backup_pb in instance.list_backups(): - backup = Backup.from_pb(backup_pb, instance) - backup.delete() - instance.delete() - - Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) - if not Config.INSTANCE.exists(): - if not USE_EMULATOR: - # Defend against back-end returning configs for regions we aren't - # actually allowed to use. - configs = [config for config in configs if "-us-" in config.name] - + configs = list(Config.CLIENT.list_instance_configs()) if not configs: raise ValueError("List instance configs failed in module set up.") - - Config.INSTANCE_CONFIG = configs[0] - config_name = configs[0].name - create_time = str(int(time.time())) - labels = {"django-spanner-systests": "true", "created": create_time} - - Config.INSTANCE = Config.CLIENT.instance( - INSTANCE_ID, config_name, labels=labels - ) - try: + Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, configs[0].name) + if not Config.INSTANCE.exists(): created_op = Config.INSTANCE.create() - created_op.result( - SPANNER_OPERATION_TIMEOUT_IN_SECONDS - ) # block until completion - except exceptions.AlreadyExists: + created_op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS) + else: Config.INSTANCE.reload() else: + Config.CLIENT = Client(project=PROJECT_ID) + Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload() + _cleanup_stale_databases(Config.INSTANCE) def teardown_instance(): @@ -126,7 +94,8 @@ def setup_database(): if Config.DATABASE is not None: return - Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME) + active_db_name = connections["default"].settings_dict["NAME"] + Config.DATABASE = Config.INSTANCE.database(active_db_name) if not Config.DATABASE.exists(): try: created_op = Config.DATABASE.create() @@ -148,13 +117,6 @@ def _cleanup_test_resources(): except Exception: pass Config.DATABASE = None - if CREATE_INSTANCE and Config.INSTANCE is not None: - try: - if Config.INSTANCE.exists(): - Config.INSTANCE.delete() - except Exception: - pass - Config.INSTANCE = None atexit.register(_cleanup_test_resources)