245 cache timestamp (#318)

Author: arthanson

netbox_custom_objects/__init__.py

@@ -1,48 +1,13 @@
 import sys
 import warnings
 
+from django.db import transaction
 from django.db.utils import DatabaseError, OperationalError, ProgrammingError
 from netbox.plugins import PluginConfig
 
 from .constants import APP_LABEL as APP_LABEL
 
 
-def is_running_migration():
-    """
-    Check if the code is currently running during a Django migration.
-    """
-    # Check if 'makemigrations' or 'migrate' command is in sys.argv
-    if any(cmd in sys.argv for cmd in ["makemigrations", "migrate"]):
-        return True
-
-    return False
-
-
-def check_custom_object_type_table_exists():
-    """
-    Check if the CustomObjectType table exists in the database.
-    Returns True if the table exists, False otherwise.
-    """
-    from django.db import connection
-    from .models import CustomObjectType
-
-    try:
-        # Use raw SQL to check table existence without generating ORM errors
-        with connection.cursor() as cursor:
-            table_name = CustomObjectType._meta.db_table
-            cursor.execute("""
-                SELECT EXISTS (
-                    SELECT FROM information_schema.tables
-                    WHERE table_name = %s
-                )
-            """, [table_name])
-            table_exists = cursor.fetchone()[0]
-            return table_exists
-    except (OperationalError, ProgrammingError, DatabaseError):
-        # Catch database-specific errors (permission issues, etc.)
-        return False
-
-
 # Plugin Configuration
 class CustomObjectsPluginConfig(PluginConfig):
     name = "netbox_custom_objects"
@@ -60,6 +25,47 @@ class CustomObjectsPluginConfig(PluginConfig):
     required_settings = []
     template_extensions = "template_content.template_extensions"
 
+    @staticmethod
+    def _is_running_migration():
+        """
+        Check if the code is currently running during a Django migration.
+        """
+        # Check if 'makemigrations' or 'migrate' command is in sys.argv
+        return any(cmd in sys.argv for cmd in ["makemigrations", "migrate"])
+
+    @staticmethod
+    def _is_running_test():
+        """
+        Check if the code is currently running during Django tests.
+        """
+        # Check if 'test' command is in sys.argv
+        return "test" in sys.argv
+
+    @staticmethod
+    def _check_custom_object_type_table_exists():
+        """
+        Check if the CustomObjectType table exists in the database.
+        Returns True if the table exists, False otherwise.
+        """
+        from django.db import connection
+        from .models import CustomObjectType
+
+        try:
+            # Use raw SQL to check table existence without generating ORM errors
+            with connection.cursor() as cursor:
+                table_name = CustomObjectType._meta.db_table
+                cursor.execute("""
+                    SELECT EXISTS (
+                        SELECT FROM information_schema.tables
+                        WHERE table_name = %s
+                    )
+                """, [table_name])
+                table_exists = cursor.fetchone()[0]
+                return table_exists
+        except (OperationalError, ProgrammingError, DatabaseError):
+            # Catch database-specific errors (permission issues, etc.)
+            return False
+
     def ready(self):
         from .models import CustomObjectType
         from netbox_custom_objects.api.serializers import get_serializer_class
@@ -74,14 +80,24 @@ def ready(self):
             )
 
             # Skip database calls if running during migration or if table doesn't exist
-            if is_running_migration() or not check_custom_object_type_table_exists():
+            if self._is_running_migration() or not self._check_custom_object_type_table_exists():
                 super().ready()
                 return
 
-            qs = CustomObjectType.objects.all()
-            for obj in qs:
-                model = obj.get_model()
-                get_serializer_class(model)
+            try:
+                with transaction.atomic():
+                    qs = CustomObjectType.objects.all()
+                    for obj in qs:
+                        model = obj.get_model()
+                        get_serializer_class(model)
+            except (DatabaseError, OperationalError, ProgrammingError):
+                # Only suppress exceptions during tests when schema may not match model
+                # During normal operation, re-raise to alert of actual problems
+                if self._is_running_test():
+                    # The transaction.atomic() block will automatically rollback
+                    pass
+                else:
+                    raise
 
         super().ready()
 
@@ -132,22 +148,33 @@ def get_models(self, include_auto_created=False, include_swapped=False):
             )
 
             # Skip custom object type model loading if running during migration
-            if is_running_migration() or not check_custom_object_type_table_exists():
+            if self._is_running_migration() or not self._check_custom_object_type_table_exists():
                 return
 
             # Add custom object type models
             from .models import CustomObjectType
 
-            custom_object_types = CustomObjectType.objects.all()
-            for custom_type in custom_object_types:
-                model = custom_type.get_model()
-                if model:
-                    yield model
-
-                    # If include_auto_created is True, also yield through models
-                    if include_auto_created and hasattr(model, '_through_models'):
-                        for through_model in model._through_models:
-                            yield through_model
+            try:
+                with transaction.atomic():
+                    custom_object_types = CustomObjectType.objects.all()
+                    for custom_type in custom_object_types:
+                        model = custom_type.get_model()
+                        if model:
+                            yield model
+
+                            # If include_auto_created is True, also yield through models
+                            if include_auto_created and hasattr(model, '_through_models'):
+                                for through_model in model._through_models:
+                                    yield through_model
+            except (DatabaseError, OperationalError, ProgrammingError):
+                # Only suppress exceptions during tests when schema may not match model
+                # (e.g., cache_timestamp column doesn't exist yet during test setup)
+                # During normal operation, re-raise to alert of actual problems
+                if self._is_running_test():
+                    # The transaction.atomic() block will automatically rollback
+                    pass
+                else:
+                    raise
 
 
 config = CustomObjectsPluginConfig

netbox_custom_objects/migrations/0002_customobjecttype_cache_timestamp.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.8 on 2025-12-05 00:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("netbox_custom_objects", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="customobjecttype",
+            name="cache_timestamp",
+            field=models.DateTimeField(auto_now=True),
+        ),
+    ]

…migrations/0002_ensure_fk_constraints.py …migrations/0003_ensure_fk_constraints.pynetbox_custom_objects/migrations/0002_ensure_fk_constraints.py renamed to netbox_custom_objects/migrations/0003_ensure_fk_constraints.py netbox_custom_objects/migrations/0002_ensure_fk_constraints.py renamed to netbox_custom_objects/migrations/0003_ensure_fk_constraints.py

@@ -20,7 +20,7 @@ def ensure_existing_fk_constraints(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('netbox_custom_objects', '0001_initial'),
+        ('netbox_custom_objects', '0002_customobjecttype_cache_timestamp'),
     ]
 
     operations = [

netbox_custom_objects/models.py

@@ -200,6 +200,10 @@ class CustomObjectType(NetBoxModel):
     verbose_name = models.CharField(max_length=100, blank=True)
     verbose_name_plural = models.CharField(max_length=100, blank=True)
     slug = models.SlugField(max_length=100, unique=True, db_index=True)
+    cache_timestamp = models.DateTimeField(
+        auto_now=True,
+        help_text=_("Timestamp used for cache invalidation")
+    )
     object_type = models.OneToOneField(
         ObjectType,
         on_delete=models.CASCADE,
@@ -265,7 +269,25 @@ def get_cached_model(cls, custom_object_type_id):
         :param custom_object_type_id: ID of the CustomObjectType
         :return: The cached model or None if not found
         """
-        return cls._model_cache.get(custom_object_type_id)
+        cache_entry = cls._model_cache.get(custom_object_type_id)
+        if cache_entry:
+            # Cache stores (model, timestamp) tuples
+            return cache_entry[0]
+        return None
+
+    @classmethod
+    def get_cached_timestamp(cls, custom_object_type_id):
+        """
+        Get the timestamp of a cached model for a specific CustomObjectType.
+
+        :param custom_object_type_id: ID of the CustomObjectType
+        :return: The cached timestamp or None if not found
+        """
+        cache_entry = cls._model_cache.get(custom_object_type_id)
+        if cache_entry:
+            # Cache stores (model, timestamp) tuples
+            return cache_entry[1]
+        return None
 
     @classmethod
     def is_model_cached(cls, custom_object_type_id):
@@ -468,11 +490,15 @@ def get_model(
         :rtype: Model
         """
 
-        # Double-check pattern: check cache again after acquiring lock
         with self._global_lock:
             if self.is_model_cached(self.id) and not no_cache:
-                model = self.get_cached_model(self.id)
-                return model
+                cached_timestamp = self.get_cached_timestamp(self.id)
+                # Only use cache if the timestamps are available and match
+                if cached_timestamp and self.cache_timestamp and cached_timestamp == self.cache_timestamp:
+                    model = self.get_cached_model(self.id)
+                    return model
+                else:
+                    self.clear_model_cache(self.id)
 
         # Generate the model outside the lock to avoid holding it during expensive operations
         model_name = self.get_table_model_name(self.pk)
@@ -547,9 +573,9 @@ def wrapped_post_through_setup(self, cls):
 
         self._after_model_generation(attrs, model)
 
-        # Cache the generated model (protected by lock for thread safety)
+        # Cache the generated model with its timestamp (protected by lock for thread safety)
         with self._global_lock:
-            self._model_cache[self.id] = model
+            self._model_cache[self.id] = (model, self.cache_timestamp)
 
         # Do the clear cache now that we have it in the cache so there
         # is no recursion.
@@ -1594,6 +1620,9 @@ def save(self, *args, **kwargs):
         # Clear and refresh the model cache for this CustomObjectType when a field is modified
         self.custom_object_type.clear_model_cache(self.custom_object_type.id)
 
+        # Update parent's cache_timestamp to invalidate cache across all workers
+        self.custom_object_type.save(update_fields=['cache_timestamp'])
+
         super().save(*args, **kwargs)
 
         # Ensure FK constraints AFTER the transaction commits to avoid "pending trigger events" errors
@@ -1622,6 +1651,9 @@ def delete(self, *args, **kwargs):
         # Clear the model cache for this CustomObjectType when a field is deleted
         self.custom_object_type.clear_model_cache(self.custom_object_type.id)
 
+        # Update parent's cache_timestamp to invalidate cache across all workers
+        self.custom_object_type.save(update_fields=['cache_timestamp'])
+
         super().delete(*args, **kwargs)
 
         # Reregister SearchIndex with new set of searchable fields