Actions
Issue #9129
closed
Task #8488: [EPIC] Upgrade to django 3.2
Syncing is running out of DB connections
Start date:
Due date:
Estimated time:
(Total: 0:00 h)
Severity:
2. Medium
Version:
Platform Release:
OS:
Triaged:
Yes
Groomed:
No
Sprint Candidate:
No
Tags:
Sprint:
Sprint 103
Quarter:
Description
pulp [bd3e6e1fe5bb4586b75163ddd7ad1521]: pulpcore.tasking.pulpcore_worker:INFO: File "/usr/local/lib/python3.8/site-packages/pulpcore/tasking/pulpcore_worker.py", line 274, in _perform_task
result = func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 130, in sync
repo_version = d_version.create()
File "/usr/local/lib/python3.8/site-packages/pulpcore/plugin/stages/declarative_version.py", line 151, in create
loop.run_until_complete(pipeline)
File "/usr/lib64/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/site-packages/pulpcore/plugin/stages/api.py", line 225, in create_pipeline
await asyncio.gather(*futures)
File "/usr/local/lib/python3.8/site-packages/pulpcore/plugin/stages/api.py", line 43, in __call__
await self.run()
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 778, in run
await asyncio.gather(*tasks)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 592, in _fetch_collection_metadata
await self._fetch_paginated_collection_metadata(
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 548, in _fetch_paginated_collection_metadata
await asyncio.gather(*tasks)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 441, in _fetch_collection_version_metadata
await self._add_collection_version(api_version, collection_version_url, metadata)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 469, in _add_collection_version
await asyncio.gather(*tasks)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 592, in _fetch_collection_metadata
await self._fetch_paginated_collection_metadata(
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 548, in _fetch_paginated_collection_metadata
await asyncio.gather(*tasks)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 441, in _fetch_collection_version_metadata
await self._add_collection_version(api_version, collection_version_url, metadata)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 496, in _add_collection_version
self.parsing_metadata_progress_bar.increment()
File "/usr/local/lib/python3.8/site-packages/pulpcore/app/models/progress.py", line 185, in increment
self.increase_by(1)
File "/usr/local/lib/python3.8/site-packages/pulpcore/app/models/progress.py", line 198, in increase_by
self.save()
File "/usr/local/lib/python3.8/site-packages/pulpcore/app/models/progress.py", line 142, in save
super().save(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django_lifecycle/mixins.py", line 134, in save
save(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 726, in save
self.save_base(using=using, force_insert=force_insert,
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 763, in save_base
updated = self._save_table(
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 845, in _save_table
updated = self._do_update(base_qs, using, pk_val, values, update_fields,
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 899, in _do_update
return filtered._update(values) > 0
File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 802, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1559, in execute_sql
cursor = super().execute_sql(result_type)
File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1173, in execute_sql
cursor = self.connection.cursor()
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 259, in cursor
return self._cursor()
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 235, in _cursor
self.ensure_connection()
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 219, in ensure_connection
self.connect()
File "/usr/local/lib/python3.8/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 219, in ensure_connection
self.connect()
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 200, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/postgresql/base.py", line 187, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib64/python3.8/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 219, in ensure_connection
self.connect()
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 200, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/postgresql/base.py", line 187, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib64/python3.8/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: FATAL: remaining connection slots are reserved for non-replication superuser connections
Related issues
Actions
.
Ensure that the Stages API uses a single connection to the DB.
Django 3.2 opens up a new connection to the DB for each coroutine that accesses the DB. All ORM interactions need to be wrapped in 'asgiref.sync.sync_to_async' in order to ensure that all DB querries are executed using a single connection in a serial manner. This patch also provides utility function
pulpcore.plugin.sync.sync_to_async_iterablebuilt on top ofsync_to_async. This functions is needed to operate on QuerySet objects and other synchronous iterables.ProgressReport is enhanced with async interfaces: asave(), aincrease_by(), aincrement(), aenter(), aexit(). Plugins should switch to the async interfaces in their Stages.
This patch also ensures that all transactions in the Stages API are run synchronously. This requires deprecating the ContentStage._pre_save() and _post_save() coroutines in favor of synchronous interfaces with the same name.
Each task that uses async code to access the DB now creates 2 connections to the DB. One to communicate on the main thread and another connection for everything that accesses the DB from coroutines such as the Stages API.
fixes: #9129