Add anonymous analytics and TOTP-protected staff administration

README.md

@@ -71,11 +71,18 @@ Deactivate with `deactivate` when you're done.
 ## Optional
 
 - __Create an admin user__
-  - With uv:
+  - Set `LITEFILE_STAFF_BOOTSTRAP_USERNAME` and a unique
+    `LITEFILE_STAFF_BOOTSTRAP_PASSWORD` through your shell or secret manager, then:
     ```bash
-    uv run python manage.py createsuperuser
+    cd efile_app
+    uv run python manage.py bootstrap_staff
     ```
-  Admin will be available at `/admin/` after you start the server.
+  Store the generated TOTP setup URI in your password manager. Alternatively,
+  supply its Base32 secret using `LITEFILE_STAFF_BOOTSTRAP_TOTP_SECRET`.
+  With the default private path, open http://127.0.0.1:8000/staff-7c83f0a2/.
+  Every staff role requires a local password and TOTP, including superusers.
+  See the [staff setup and local demo runbook](docs/developer-notes/staff-privacy-and-analytics.md)
+  for local sample data, deployment switches, and recovery.
 
 - __Static files__
   During development, static files are served automatically. No `collectstatic` is needed.

efile_app/.env.example

@@ -1,7 +1,22 @@
-# Django Settings
+# Django settings
 DJANGO_SECRET_KEY=your-secret-key-here
 DJANGO_LOG_LEVEL=DEBUG
 
+# Private staff tools: no shared/default administrator credentials.
+# Set these in your shell or secret manager, then run:
+# uv run python manage.py bootstrap_staff
+# Existing accounts are never promoted or overwritten by bootstrap.
+LITEFILE_STAFF_BOOTSTRAP_USERNAME=
+LITEFILE_STAFF_BOOTSTRAP_PASSWORD=
+LITEFILE_STAFF_BOOTSTRAP_EMAIL=
+# Optional Base32 TOTP setup secret (20–40 random bytes).
+# Leave blank to print a newly generated setup URI once on creation.
+LITEFILE_STAFF_BOOTSTRAP_TOTP_SECRET=
+LITEFILE_STAFF_PATH=staff-7c83f0a2
+LITEFILE_ANALYTICS_ENABLED=false
+LITEFILE_ANALYTICS_EXPORT_ENABLED=false
+LITEFILE_PRIVACY_DELETION_ENABLED=false
+
 # Database (optional - leave empty to use SQLite)
 DATABASE_URL=postgresql://user:password@localhost:5432/dbname
 

efile_app/efile/apps.py

@@ -4,6 +4,7 @@
 class EfileConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "efile"
+    verbose_name = "LITEFile"
 
     def ready(self):
         # Registers the checks in efile/checks.py by importing them.

efile_app/efile/authentication.py

@@ -13,12 +13,14 @@
 
 class SuffolkEFileBackend(BaseBackend):
     def authenticate(self, request, username=None, password=None, **kwargs):
-        logger.info("Trying auth?")
-
         jurisdiction = kwargs.get("jurisdiction", get_jurisdiction_from_request(request))
         if not username or not password or not jurisdiction:
             return None
 
+        # A staff password must also stay local if entered on a filer form.
+        if User.objects.filter(username__iexact=username, is_staff=True).exists():
+            return None
+
         try:
             try:
                 auth_data = auth_with_tyler_api(username, password, jurisdiction)
@@ -29,23 +31,27 @@ def authenticate(self, request, username=None, password=None, **kwargs):
                     request.efsp_unavailable = True
                 return None
             if not auth_data or "tokens" not in auth_data:
-                logger.info("Tyler auth failed for user %s", username)
+                logger.info("Court authentication failed")
                 return None
 
             request.session["auth_tokens"] = auth_data["tokens"]
 
-            logger.info("Auth data: %s", auth_data)
-
             user = self._get_or_create_user(username, auth_data, jurisdiction)
+            if not user.is_active or user.is_staff:
+                request.session.pop("auth_tokens", None)
+                return None
             # TODO(brycew): actually write these?
             # if request:
             #    self._store_tokens_in_session(request, auth_data, jurisdiction)
 
-            logger.info("Successfully auth'd user: %s", username)
+            logger.info("Court authentication succeeded")
             request.session["user_email"] = user.email
             return user
         except Exception:
-            logger.exception("Error during auth for user: %s", username)
+            if request is not None:
+                request.session.pop("auth_tokens", None)
+                request.session.pop("user_email", None)
+            logger.error("Court authentication failed unexpectedly")
             return None
 
     def get_user(self, user_id):

efile_app/efile/management/commands/bootstrap_staff.py

@@ -0,0 +1,80 @@
+"""Explicit, one-time environment bootstrap with no default credentials."""
+
+import base64
+import binascii
+import os
+import secrets
+
+from django.conf import settings
+from django.contrib.auth.password_validation import validate_password
+from django.core.exceptions import ValidationError
+from django.core.management.base import BaseCommand, CommandError
+from django.core.validators import validate_email
+from django.db import IntegrityError, transaction
+from django_otp.plugins.otp_totp.models import TOTPDevice
+
+from efile.models import UserProfile
+
+
+class Command(BaseCommand):
+    help = "Bootstrap a local-password/TOTP superuser from LITEFILE_STAFF_BOOTSTRAP_* environment variables."
+
+    def handle(self, *args, **options):
+        username = os.getenv("LITEFILE_STAFF_BOOTSTRAP_USERNAME", "").strip()
+        if not username:
+            raise CommandError("Set LITEFILE_STAFF_BOOTSTRAP_USERNAME; there is no default account.")
+        existing = UserProfile.objects.filter(username=username).first()
+        if existing:
+            if not (existing.is_active and existing.is_staff and existing.is_superuser):
+                raise CommandError("An incompatible account already exists. Bootstrap cannot promote or overwrite it.")
+            if not TOTPDevice.objects.filter(user=existing, confirmed=True).exists():
+                raise CommandError(
+                    "The account exists without TOTP. Use provision_staff_totp from the trusted console."
+                )
+            self.stdout.write("Staff account already provisioned; password, roles, and TOTP unchanged.")
+            return
+
+        password = os.getenv("LITEFILE_STAFF_BOOTSTRAP_PASSWORD", "")
+        if not password:
+            raise CommandError("Set LITEFILE_STAFF_BOOTSTRAP_PASSWORD to a unique password; there is no default.")
+        email = os.getenv("LITEFILE_STAFF_BOOTSTRAP_EMAIL", "").strip()
+        user = UserProfile(username=username, email=email, is_active=True, is_staff=True, is_superuser=True)
+        try:
+            UserProfile._meta.get_field("username").clean(username, user)
+            if email:
+                validate_email(email)
+            validate_password(password, user=user)
+        except ValidationError as error:
+            # Never echo supplied values, including a password or TOTP secret.
+            raise CommandError(
+                "Invalid username, email, or password. Use a valid username and a strong unique password."
+            ) from error
+
+        secret = os.getenv("LITEFILE_STAFF_BOOTSTRAP_TOTP_SECRET", "").strip().upper().replace(" ", "").rstrip("=")
+        supplied_secret = bool(secret)
+        if supplied_secret:
+            try:
+                key = base64.b32decode(secret + "=" * (-len(secret) % 8))
+            except (binascii.Error, ValueError) as error:
+                raise CommandError("LITEFILE_STAFF_BOOTSTRAP_TOTP_SECRET must be a valid Base32 secret.") from error
+            if not 20 <= len(key) <= 40:
+                raise CommandError("Use a TOTP secret containing 20–40 random bytes (32–64 Base32 characters).")
+        else:
+            key = secrets.token_bytes(20)
+
+        try:
+            with transaction.atomic():
+                user.set_password(password)
+                user.save()
+                device = TOTPDevice.objects.create(user=user, name="Staff authenticator", key=key.hex(), confirmed=True)
+        except IntegrityError as error:
+            raise CommandError(
+                "Bootstrap conflicted with another account creation. No credentials were overwritten."
+            ) from error
+
+        self.stdout.write(f"Staff superuser created. Sign in at /{settings.LITEFILE_STAFF_PATH}/.")
+        if supplied_secret:
+            self.stdout.write("TOTP configured from the supplied secret; credentials are not printed.")
+        else:
+            self.stdout.write("Store this newly generated TOTP setup URI securely; it is printed only on creation:")
+            self.stdout.write(device.config_url)

efile_app/efile/management/commands/exclude_analytics_account.py

@@ -0,0 +1,15 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from efile.models import UserProfile
+
+
+class Command(BaseCommand):
+    help = "Exclude a designated test account from future reporting; existing anonymous counters are not changed."
+
+    def add_arguments(self, parser):
+        parser.add_argument("account_id", type=int)
+
+    def handle(self, *args, **options):
+        if not UserProfile.objects.filter(pk=options["account_id"]).update(analytics_excluded=True):
+            raise CommandError("Local account not found.")
+        self.stdout.write("Account excluded from future reporting.")

efile_app/efile/management/commands/process_document_extractions.py

@@ -1,8 +1,10 @@
+import io
 import logging
 import multiprocessing
 import time
 
 from django.conf import settings
+from django.core.management import call_command
 from django.core.management.base import BaseCommand
 from django.db import close_old_connections
 
@@ -51,8 +53,22 @@ def handle(self, *args, **options):
             renew_extraction_lease,
         )
 
+        next_rollup = 0.0
+
+        def rollup_if_due():
+            nonlocal next_rollup
+            now = time.monotonic()
+            if now < next_rollup:
+                return
+            next_rollup = now + 60
+            try:
+                call_command("process_usage_events", stdout=io.StringIO())
+            except Exception:
+                logger.error("Usage rollup failed; queued events retained for retry")
+
         while True:
             close_old_connections()
+            rollup_if_due()
             job = claim_next_extraction()
             if job is None:
                 if options["once"]:
@@ -73,6 +89,7 @@ def handle(self, *args, **options):
                     if not child.is_alive():
                         break
                     close_old_connections()
+                    rollup_if_due()
                     if time.monotonic() >= deadline or not renew_extraction_lease(job.pk, job.claim_token):
                         child.terminate()
                         break

efile_app/efile/management/commands/process_privacy_requests.py

@@ -0,0 +1,28 @@
+from django.core.management.base import BaseCommand
+
+from efile.models import PrivacyRequest
+from efile.services.privacy import process_request
+
+
+class Command(BaseCommand):
+    help = "Resume previously confirmed deletion work. Preview by default; --apply retries storage failures/interrupted processing."
+
+    def add_arguments(self, parser):
+        parser.add_argument("--apply", action="store_true")
+
+    def handle(self, *args, **options):
+        requests = PrivacyRequest.objects.filter(
+            status__in=["processing", "attention"], outcome__in=["", "storage_failed"]
+        )
+        for item in requests.order_by("created_at"):
+            if not item.verified_at or not item.operator_id:
+                continue
+            if not options["apply"]:
+                self.stdout.write(f"Pending confirmed request {item.reference}.")
+                continue
+            try:
+                result = process_request(item.pk, item.operator)
+            except Exception:
+                self.stderr.write(f"Request {item.reference} requires operator attention.")
+            else:
+                self.stdout.write(f"Request {item.reference}: {result.status} / {result.outcome}.")

efile_app/efile/management/commands/process_usage_events.py

@@ -0,0 +1,19 @@
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from efile.models import StaffLoginThrottle
+from efile.services.analytics import drain_events
+
+
+class Command(BaseCommand):
+    help = "Roll up queued usage events without double-counting. Run at least every minute."
+
+    def handle(self, *args, **options):
+        try:
+            drain_events()
+        except Exception:
+            raise CommandError("Usage rollup failed; queued events remain available for retry.") from None
+        StaffLoginThrottle.objects.filter(window_started__lt=timezone.now() - timedelta(days=1)).delete()
+        self.stdout.write("Usage event batch processed.")

efile_app/efile/management/commands/provision_staff_totp.py

@@ -0,0 +1,37 @@
+"""Console bootstrap/recovery: never create a password-only staff bypass."""
+
+from django.contrib.sessions.models import Session
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+from django_otp.plugins.otp_totp.models import TOTPDevice
+
+from efile.models import UserProfile
+from efile.services.privacy import account_sessions
+
+
+class Command(BaseCommand):
+    help = "Provision a staff TOTP authenticator. Deliver the printed secret securely; --reset revokes all sessions."
+
+    def add_arguments(self, parser):
+        parser.add_argument("username")
+        parser.add_argument("--reset", action="store_true")
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        user = (
+            UserProfile.objects.select_for_update()
+            .filter(username=options["username"], is_staff=True, is_active=True)
+            .first()
+        )
+        if user is None:
+            raise CommandError("An active local staff account is required.")
+        devices = TOTPDevice.objects.filter(user=user)
+        if devices.exists() and not options["reset"]:
+            raise CommandError(
+                "An authenticator already exists. Use --reset only after verifying the recovery request."
+            )
+        if options["reset"]:
+            devices.delete()
+            Session.objects.filter(pk__in=[s.pk for s in account_sessions(user)]).delete()
+        device = TOTPDevice.objects.create(user=user, name="Staff authenticator", confirmed=True)
+        self.stdout.write(device.config_url)

efile_app/efile/management/commands/prune_staff_requests.py

@@ -0,0 +1,21 @@
+from datetime import timedelta
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from efile.models import PrivacyRequest, StaffAudit
+
+
+class Command(BaseCommand):
+    help = "Prune resolved request/audit records after the configured retention period; never discard open manifests."
+
+    def handle(self, *args, **options):
+        cutoff = timezone.now() - timedelta(days=max(1, settings.LITEFILE_STAFF_REQUEST_RETENTION_DAYS))
+        PrivacyRequest.objects.filter(
+            status="completed", external_cleanup_pending=False, completed_at__lt=cutoff
+        ).delete()
+        # Retain audits for unresolved requests even if their processing is old.
+        references = PrivacyRequest.objects.values("reference")
+        StaffAudit.objects.filter(created_at__lt=cutoff).exclude(reference__in=references).delete()
+        self.stdout.write("Resolved staff records pruned; open work retained.")