【发布时间】：2022-01-23 13:18:34
【问题描述】：
我最近一直想知道确保数据库记录的创建是幂等的通常最好的方法是什么。我能想到的两种方法是：
- 在执行 INSERT 之前检查记录是否已存在
- 对相关列使用唯一约束以确保不存在具有相同值的两条记录
这似乎是一个先看再跳/更容易要求宽恕比许可二分法的例子。在 Python 社区中，我知道后一种方法是可以接受的，甚至是首选方法。我想知道这是否同样适用于使用关系数据库。
越快越好吗？
根据下面的一些测试，似乎具有唯一约束的 EAFP 方法对于插入新记录和优雅地处理重复记录都更快。但是，我可以想象在每次 INSERT 之前使用 SELECT 的 LBYL 方法可能更可取的情况。
- 如果表架构发生更改，更新约束以包含新列可能会很棘手。在生产环境中更改代码肯定比迁移数据库更容易。
- 如果表包含数百万条记录，则在生产环境中添加和删除索引可能会很棘手。
- 在我的 Django 示例中，我用来避免在错误异常
create_permission_EAFP上静默失败的字符串搜索方法看起来很老套。 （虽然这可能比一般方法更能说明我的实现）。
性能测试
以下测试是在我笔记本电脑上的 Docker 容器中使用 Postgres 14 和 Django 3.2 运行的。我决定为此使用 Django 测试框架，因为每次运行测试都从一个空 DB 开始。
创建 10,000 条记录的结果
在tests.py 中针对一万条记录运行测试的输出：
======================================================================
FAIL: test_look_before_you_leap_faster_existing_records (idempodentinserts.tests.TestPermissionCreation)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/code/mainapp/idempodentinserts/tests.py", line 82, in test_look_before_you_leap_faster_existing_records
self.assertLess(duration_LBYL, duration_EAFP, f"LBYL took longer with existing records. {report}")
AssertionError: 4.998060464859009 not less than 2.5420615673065186 : LBYL took longer with existing records.
For 10000 create calls...
The Look-before-you-leap strategy took 4.998 seconds (average: 0.500 milliseconds).
The Ask-forgiveness-not-permission strategy took 2.542 seconds (average: 0.254 milliseconds).
======================================================================
FAIL: test_look_before_you_leap_faster_new_records (idempodentinserts.tests.TestPermissionCreation)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/code/mainapp/idempodentinserts/tests.py", line 103, in test_look_before_you_leap_faster_new_records
self.assertLess(duration_LBYL, duration_EAFP, f"LBYL took longer with new records. {report}")
AssertionError: 31.07089853286743 not less than 20.387959241867065 : LBYL took longer with new records.
For 10000 create calls...
The Look-before-you-leap strategy took 31.071 seconds (average: 3.107 milliseconds).
The Ask-forgiveness-not-permission strategy took 20.388 seconds (average: 2.039 milliseconds).
----------------------------------------------------------------------
Ran 4 tests in 122.848s
FAILED (failures=2)
创建 1,000,000 条记录的结果
即使创建一百万条记录，测试也支持 EAFP 作为更快的方法。虽然所有插入都变慢了，但首先检查记录是否存在并没有帮助。
======================================================================
FAIL: test_look_before_you_leap_faster_existing_records (idempodentinserts.tests.TestPermissionCreation)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/code/mainapp/idempodentinserts/tests.py", line 82, in test_look_before_you_leap_faster_existing_records
self.assertLess(duration_LBYL, duration_EAFP, f"LBYL took longer with existing records. {report}")
AssertionError: 445.97691440582275 not less than 247.20186638832092 : LBYL took longer with existing records.
For 1000000 create calls...
The Look-before-you-leap strategy took 445.977 seconds (average: 0.446 milliseconds).
The Ask-forgiveness-not-permission strategy took 247.202 seconds (average: 0.247 milliseconds).
======================================================================
FAIL: test_look_before_you_leap_faster_new_records (idempodentinserts.tests.TestPermissionCreation)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/code/mainapp/idempodentinserts/tests.py", line 103, in test_look_before_you_leap_faster_new_records
self.assertLess(duration_LBYL, duration_EAFP, f"LBYL took longer with new records. {report}")
AssertionError: 6323.6987335681915 not less than 4435.961817026138 : LBYL took longer with new records.
For 1000000 create calls...
The Look-before-you-leap strategy took 6323.699 seconds (average: 6.324 milliseconds).
The Ask-forgiveness-not-permission strategy took 4435.962 seconds (average: 4.436 milliseconds).
----------------------------------------------------------------------
Ran 4 tests in 23372.856s
FAILED (failures=2)
代码
models.py
from django.db import models
class Permission(models.Model):
subject_uuid = models.UUIDField(db_index=True)
object_uuid = models.UUIDField(db_index=True)
verb = models.CharField(max_length=255)
class UniquePermission(models.Model):
subject_uuid = models.UUIDField(db_index=True)
object_uuid = models.UUIDField(db_index=True)
verb = models.CharField(max_length=255)
class Meta:
unique_together = ("subject_uuid", "object_uuid","verb")
permissions.py
from django import db
import psycopg2
from . import models
def create_permission_LBYL(subject_uuid, object_uuid, verb):
if not models.Permission.objects.filter(
subject_uuid=subject_uuid, object_uuid=object_uuid, verb=verb
).exists():
models.Permission.objects.create(
subject_uuid=subject_uuid, object_uuid=object_uuid, verb=verb
)
def create_permission_EAFP(subject_uuid, object_uuid, verb):
try:
models.UniquePermission.objects.create(
subject_uuid=subject_uuid, object_uuid=object_uuid, verb=verb
)
except db.IntegrityError as e:
# This hack wouldn't work if Postgres wasn't the database backend
if not isinstance(e.__cause__, psycopg2.errors.UniqueViolation):
raise e
tests.py
from django import test
from . import permissions
from . import models
VERB_LENGTH = 10
THOUSAND = 1000
VERB_COUNT = 10 * THOUSAND
class TestPermissionCreation(test.TransactionTestCase):
"""
Compares performance between the LBYL (look before you leap)
and EAFP (it's easier to ask forgiveness than permission) approaches to idempotent
database inserts.
"""
@classmethod
def setUpClass(cls):
super().setUpClass()
letter_combinations = itertools.combinations(string.ascii_lowercase, VERB_LENGTH)
unique_words = ("".join(combination) for combination in letter_combinations)
cls.verbs = list(itertools.islice(unique_words, VERB_COUNT))
cls.existing_subject_uuids = [uuid.uuid4() for _ in cls.verbs]
cls.existing_object_uuids = [uuid.uuid4() for _ in cls.verbs]
cls.new_subject_uuids = [uuid.uuid4() for _ in cls.verbs]
cls.new_object_uuids = [uuid.uuid4() for _ in cls.verbs]
def setUp(self):
models.Permission.objects.bulk_create(
models.Permission(subject_uuid=sub, object_uuid=obj, verb=verb)
for sub, obj, verb in zip(self.existing_subject_uuids,
self.existing_object_uuids,
self.verbs)
)
models.UniquePermission.objects.bulk_create(
models.UniquePermission(subject_uuid=sub, object_uuid=obj, verb=verb)
for sub, obj, verb in zip(self.existing_subject_uuids,
self.existing_object_uuids,
self.verbs)
)
def report_durations(self, duration_LBYL, duration_EAFP):
verb_count = len(self.verbs)
LBYL_ave_ms = (duration_LBYL / verb_count) * 1000
EAFP_ave_ms = (duration_EAFP / verb_count) * 1000
return (
f"For {verb_count} create calls... "
f"The Look-before-you-leap strategy took {duration_LBYL:.3f} seconds "
f"(average: {LBYL_ave_ms:.3f} milliseconds). "
f"The Ask-forgiveness-not-permission strategy took {duration_EAFP:.3f} seconds "
f"(average: {EAFP_ave_ms:.3f} milliseconds)."
)
def test_look_before_you_leap_faster_existing_records(self):
start_LBYL = time.time()
for sub, obj, verb in zip(self.existing_subject_uuids,
self.existing_object_uuids,
self.verbs):
permissions.create_permission_LBYL(subject_uuid=sub, object_uuid=obj, verb=verb)
duration_LBYL = time.time() - start_LBYL
start_EAFP = time.time()
for sub, obj, verb in zip(self.existing_subject_uuids,
self.existing_object_uuids,
self.verbs):
permissions.create_permission_EAFP(subject_uuid=sub, object_uuid=obj, verb=verb)
duration_EAFP = time.time() - start_EAFP
report = self.report_durations(duration_EAFP=duration_EAFP,duration_LBYL=duration_LBYL)
self.assertLess(duration_LBYL, duration_EAFP, f"LBYL took longer with existing records. {report}")
def test_look_before_you_leap_faster_new_records(self):
start_LBYL = time.time()
for sub, obj, verb in zip(self.new_subject_uuids,
self.new_object_uuids,
self.verbs):
permissions.create_permission_LBYL(subject_uuid=sub, object_uuid=obj, verb=verb)
duration_LBYL = time.time() - start_LBYL
start_EAFP = time.time()
for sub, obj, verb in zip(self.new_subject_uuids,
self.new_object_uuids,
self.verbs):
permissions.create_permission_EAFP(subject_uuid=sub, object_uuid=obj, verb=verb)
duration_EAFP = time.time() - start_EAFP
report = self.report_durations(duration_EAFP=duration_EAFP,duration_LBYL=duration_LBYL)
self.assertLess(duration_LBYL, duration_EAFP, f"LBYL took longer with new records. {report}")
def test_ask_forgiveness_not_permission_faster_existing_records(self):
start_LBYL = time.time()
for sub, obj, verb in zip(self.existing_subject_uuids,
self.existing_object_uuids,
self.verbs):
permissions.create_permission_LBYL(subject_uuid=sub, object_uuid=obj, verb=verb)
duration_LBYL = time.time() - start_LBYL
start_EAFP = time.time()
for sub, obj, verb in zip(self.existing_subject_uuids,
self.existing_object_uuids,
self.verbs,
):
permissions.create_permission_EAFP(subject_uuid=sub, object_uuid=obj, verb=verb)
duration_EAFP = time.time() - start_EAFP
report = self.report_durations(duration_EAFP=duration_EAFP,duration_LBYL=duration_LBYL)
self.assertLess(duration_EAFP, duration_LBYL, f"LBYL took longer with existing records. {report}")
def test_ask_forgiveness_not_permission_faster_new_records(self):
start_LBYL = time.time()
for sub, obj, verb in zip(self.new_subject_uuids,
self.new_object_uuids,
self.verbs):
permissions.create_permission_LBYL(subject_uuid=sub, object_uuid=obj, verb=verb)
duration_LBYL = time.time() - start_LBYL
start_EAFP = time.time()
for sub, obj, verb in zip(self.new_subject_uuids,
self.new_object_uuids,
self.verbs):
permissions.create_permission_EAFP(subject_uuid=sub, object_uuid=obj, verb=verb)
duration_EAFP = time.time() - start_EAFP
report = self.report_durations(duration_EAFP=duration_EAFP,duration_LBYL=duration_LBYL)
self.assertLess(duration_EAFP, duration_LBYL, f"LBYL took longer with new records. {report}")
【问题讨论】：
-
LBYL 不会阻止另一个进程在您的选择和插入之间插入具有您独特特征的记录。
-
@snakecharmerb 非常感谢。
标签： python sql python-3.x postgresql django-models