test(django-google-spanner): enable system tests and share a single per-run database

packages/django-google-spanner/noxfile.py

@@ -232,11 +232,6 @@ def system_test(session):
 
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
 def system(session):
-    # TODO(https://github.com/googleapis/google-cloud-python/issues/16099):
-    # System tests were disabled in the split repo to speed up the build process.
-    # See: https://github.com/googleapis/python-spanner-django/pull/929.
-    session.skip("System tests are temporarily disabled.")
-
     print("System tests with django 5.2")
     system_test(session)
 

packages/django-google-spanner/tests/settings.py

@@ -5,18 +5,14 @@
 # https://developers.google.com/open-source/licenses/bsd
 
 import os
+import time
+import uuid
 
 DEBUG = True
 USE_TZ = True
 
 INSTALLED_APPS = [
     "django_spanner",  # Must be the first entry
-    "django.contrib.contenttypes",
-    "django.contrib.auth",
-    "django.contrib.sites",
-    "django.contrib.sessions",
-    "django.contrib.messages",
-    "django.contrib.staticfiles",
     "tests",
 ]
 
@@ -29,7 +25,10 @@
 PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project")
 
 # _get_test_db_name method in creation.py addes prefix of 'test_' to db name.
-DATABASE_NAME = os.getenv("DJANGO_SPANNER_DB", "django_test_db")
+DATABASE_NAME = os.getenv(
+    "DJANGO_SPANNER_DB",
+    f"dj_sys_{int(time.time())}_{uuid.uuid4().hex[:6]}",
+)
 
 DATABASES = {
     "default": {

packages/django-google-spanner/tests/system/django_spanner/utils.py

@@ -4,30 +4,29 @@
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
 
+import atexit
 import os
 import time
 
-from django.core.management import call_command
-from django.db import connection
+from django.conf import settings
+from django.db import connections
 from google.api_core import exceptions
 from google.cloud.spanner_v1 import Client
-from google.cloud.spanner_v1.instance import Backup, Instance
-from test_utils.retry import RetryErrors
+from google.cloud.spanner_v1.database import Database
 
 from django_spanner import USE_EMULATOR
-from django_spanner.creation import DatabaseCreation
-
-CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None
 
 SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int(
-    os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 60)
+    os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 240)
 )
-EXISTING_INSTANCES = []
 INSTANCE_ID = os.environ.get(
-    "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE", "spanner-django-python-systest"
+    "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE",
+    settings.DATABASES["default"]["INSTANCE"],
+)
+PROJECT_ID = os.getenv(
+    "GOOGLE_CLOUD_PROJECT",
+    settings.DATABASES["default"]["PROJECT"],
 )
-PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project")
-DATABASE_NAME = os.getenv("DJANGO_SPANNER_DB", "django_test_db")
 
 
 class Config(object):
@@ -38,106 +37,86 @@ class Config(object):
     """
 
     CLIENT = None
-    INSTANCE_CONFIG = None
     INSTANCE = None
     DATABASE = None
 
 
-def _list_instances():
-    return list(Config.CLIENT.list_instances())
+def _cleanup_stale_databases(instance):
+    """Drop stale dj_sys_<timestamp>_* databases older than 1 hour."""
+    cutoff = int(time.time()) - 1 * 60 * 60
+    try:
+        for db_pb in instance.list_databases():
+            db = Database.from_pb(db_pb, instance)
+            parts = db.database_id.split("_")
+            if len(parts) >= 3 and parts[0] == "dj" and parts[1] == "sys":
+                try:
+                    created_ts = int(parts[2])
+                except ValueError:
+                    continue
+                if created_ts < cutoff:
+                    try:
+                        db.drop()
+                    except Exception:
+                        pass
+    except Exception:
+        pass
 
 
 def setup_instance():
+    if Config.INSTANCE is not None:
+        return
+
     if USE_EMULATOR:
         from google.auth.credentials import AnonymousCredentials
 
         Config.CLIENT = Client(project=PROJECT_ID, credentials=AnonymousCredentials())
-    else:
-        Config.CLIENT = Client()
-
-    retry = RetryErrors(exceptions.ServiceUnavailable)
-
-    configs = list(retry(Config.CLIENT.list_instance_configs)())
-
-    instances = retry(_list_instances)()
-    EXISTING_INSTANCES[:] = instances
-
-    # Delete test instances that are older than an hour.
-    cutoff = int(time.time()) - 1 * 60 * 60
-    instance_pbs = Config.CLIENT.list_instances("labels.python-spanner-systests:true")
-    for instance_pb in instance_pbs:
-        instance = Instance.from_pb(instance_pb, Config.CLIENT)
-        if "created" not in instance.labels:
-            continue
-        create_time = int(instance.labels["created"])
-        if create_time > cutoff:
-            continue
-        if not USE_EMULATOR:
-            # Instance cannot be deleted while backups exist.
-            for backup_pb in instance.list_backups():
-                backup = Backup.from_pb(backup_pb, instance)
-                backup.delete()
-        instance.delete()
-
-    if CREATE_INSTANCE:
-        if not USE_EMULATOR:
-            # Defend against back-end returning configs for regions we aren't
-            # actually allowed to use.
-            configs = [config for config in configs if "-us-" in config.name]
-
+        configs = list(Config.CLIENT.list_instance_configs())
         if not configs:
             raise ValueError("List instance configs failed in module set up.")
-
-        Config.INSTANCE_CONFIG = configs[0]
-        config_name = configs[0].name
-        create_time = str(int(time.time()))
-        labels = {"django-spanner-systests": "true", "created": create_time}
-
-        Config.INSTANCE = Config.CLIENT.instance(
-            INSTANCE_ID, config_name, labels=labels
-        )
+        Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, configs[0].name)
         if not Config.INSTANCE.exists():
             created_op = Config.INSTANCE.create()
-            created_op.result(
-                SPANNER_OPERATION_TIMEOUT_IN_SECONDS
-            )  # block until completion
+            created_op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS)
         else:
             Config.INSTANCE.reload()
-
     else:
+        Config.CLIENT = Client(project=PROJECT_ID)
         Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID)
         Config.INSTANCE.reload()
+        _cleanup_stale_databases(Config.INSTANCE)
 
 
 def teardown_instance():
-    if CREATE_INSTANCE:
-        Config.INSTANCE.delete()
+    pass
 
 
 def setup_database():
-    Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME)
-    if not Config.DATABASE.exists():
-        creation = DatabaseCreation(connection)
-        creation._create_test_db(verbosity=0, autoclobber=False, keepdb=True)
+    if Config.DATABASE is not None:
+        return
 
-    # Running migrations on the db.
-    call_command("migrate", interactive=False)
+    active_db_name = connections["default"].settings_dict["NAME"]
+    Config.DATABASE = Config.INSTANCE.database(active_db_name)
+    if not Config.DATABASE.exists():
+        try:
+            created_op = Config.DATABASE.create()
+            created_op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS)
+        except exceptions.AlreadyExists:
+            pass
 
 
 def teardown_database():
-    from django.db import connections
-
     connections.close_all()
 
-    Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME)
-    if Config.DATABASE.exists():
-        Config.DATABASE.drop()
 
-    # Clear the global client cache to avoid Stale Session errors
-    # when the next test class creates a new database.
-    try:
-        import django_spanner.base
+def _cleanup_test_resources():
+    connections.close_all()
+    if Config.DATABASE is not None:
+        try:
+            if Config.DATABASE.exists():
+                Config.DATABASE.drop()
+        except Exception:
+            pass
+        Config.DATABASE = None
 
-        django_spanner.base._SPANNER_CLIENT_CACHE = None
-    except ImportError:
-        pass
+
+atexit.register(_cleanup_test_resources)