Project

Profile

Help

Issue #7073

Updated by dkliban@redhat.com 5 months ago

The test_import test_pulpimport test is failing intermittently on Travis:

<pre>
=================================== FAILURES ===================================

________________________ PulpImportTestCase.test_import ________________________

self = <pulpcore.tests.functional.api.using_plugin.test_pulpimport.PulpImportTestCase testMethod=test_import>

def test_import(self):

"""Test an import."""

importer = self._create_importer()

> task_group = self._perform_import(importer)

pulpcore/tests/functional/api/using_plugin/test_pulpimport.py:194:

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

pulpcore/tests/functional/api/using_plugin/test_pulpimport.py:167: in _perform_import

task_group = monitor_task_group(task_group_href)

pulpcore/tests/functional/utils.py:54: in monitor_task_group

exit()

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = Use exit() or Ctrl-D (i.e. EOF) to exit, code = None

def __call__(self, code=None):

# Shells like IDLE catch the SystemExit, but listen when their

# stdin wrapper is closed.

try:

sys.stdin.close()

except:

pass

> raise SystemExit(code)

E SystemExit: None

../../../virtualenv/python3.7.6/lib/python3.7/site.py:403: SystemExit

----------------------------- Captured stdout call -----------------------------

The task group failed: {'all_tasks_dispatched': True,

'canceled': 0,

'completed': 1,

'description': 'Import of '

'/tmp/0cf747ab-d5b3-454a-88f0-7aedf5fad6f3/export-5a84c6c2-4905-4400-b2ea-f48e8c603e8b-20200701_1100.tar.gz',

'failed': 1,

'group_progress_reports': [],

'pulp_href': '/pulp/api/v3/task-groups/a8fc525b-eb7d-442d-a732-694648485198/',

'running': 0,

'skipped': 0,

'waiting': 0}
</pre>

The following logs appear.

<pre>
pulp: rq.worker:ERROR: django.db.utils.OperationalError: deadlock detected

DETAIL: Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT: See server log for query details.

CONTEXT: while updating tuple (0,29) in relation "core_content"

Traceback (most recent call last):

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

return self.cursor.execute(sql, params)

psycopg2.extensions.TransactionRollbackError: deadlock detected

DETAIL: Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT: See server log for query details.

CONTEXT: while updating tuple (0,29) in relation "core_content"

The above exception was the direct cause of the following exception:

Traceback (most recent call last):

File "/usr/local/lib/python3.7/site-packages/rq/worker.py", line 883, in perform_job

rv = job.perform()

File "/usr/local/lib/python3.7/site-packages/rq/job.py", line 657, in perform

self._result = self._execute()

File "/usr/local/lib/python3.7/site-packages/rq/job.py", line 663, in _execute

return self.func(*self.args, **self.kwargs)

File "/usr/local/lib/python3.7/site-packages/pulpcore/app/tasks/importer.py", line 126, in import_repository_version

c_result = _import_file(content_path, ContentResource)

File "/usr/local/lib/python3.7/site-packages/pulpcore/app/tasks/importer.py", line 47, in _import_file

return resource.import_data(data, raise_errors=True)

File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 627, in import_data

return self.import_data_inner(dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs)

File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 673, in import_data_inner

raise row_result.errors[-1].error

File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 569, in import_row

self.save_instance(instance, using_transactions, dry_run)

File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 352, in save_instance

instance.save()

File "/usr/local/lib/python3.7/site-packages/pulpcore/app/models/base.py", line 110, in save

return super().save(*args, **kwargs)

File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 741, in save

force_update=force_update, update_fields=update_fields)

File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 779, in save_base

force_update, using, update_fields,

File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 851, in _save_table

forced_update)

File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 900, in _do_update

return filtered._update(values) > 0

File "/usr/local/lib/python3.7/site-packages/django/db/models/query.py", line 760, in _update

return query.get_compiler(self.db).execute_sql(CURSOR)

File "/usr/local/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1469, in execute_sql

cursor = super().execute_sql(result_type)

File "/usr/local/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1140, in execute_sql

cursor.execute(sql, params)

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 67, in execute

return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers

return executor(sql, params, many, context)

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

return self.cursor.execute(sql, params)

File "/usr/local/lib/python3.7/site-packages/django/db/utils.py", line 89, in __exit__

raise dj_exc_value.with_traceback(traceback) from exc_value

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

return self.cursor.execute(sql, params)

django.db.utils.OperationalError: deadlock detected

DETAIL: Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT: See server log for query details.

CONTEXT: while updating tuple (0,29) in relation "core_content"

Traceback (most recent call last):

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

return self.cursor.execute(sql, params)

psycopg2.extensions.TransactionRollbackError: deadlock detected

DETAIL: Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT: See server log for query details.

CONTEXT: while updating tuple (0,29) in relation "core_content"

The above exception was the direct cause of the following exception:

Traceback (most recent call last):

File "/usr/local/lib/python3.7/site-packages/rq/worker.py", line 883, in perform_job

rv = job.perform()

File "/usr/local/lib/python3.7/site-packages/rq/job.py", line 657, in perform

self._result = self._execute()

File "/usr/local/lib/python3.7/site-packages/rq/job.py", line 663, in _execute

return self.func(*self.args, **self.kwargs)

File "/usr/local/lib/python3.7/site-packages/pulpcore/app/tasks/importer.py", line 126, in import_repository_version

c_result = _import_file(content_path, ContentResource)

File "/usr/local/lib/python3.7/site-packages/pulpcore/app/tasks/importer.py", line 47, in _import_file

return resource.import_data(data, raise_errors=True)

File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 627, in import_data

return self.import_data_inner(dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs)

File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 673, in import_data_inner

raise row_result.errors[-1].error

File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 569, in import_row

self.save_instance(instance, using_transactions, dry_run)

File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 352, in save_instance

instance.save()

File "/usr/local/lib/python3.7/site-packages/pulpcore/app/models/base.py", line 110, in save

return super().save(*args, **kwargs)

File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 741, in save

force_update=force_update, update_fields=update_fields)

File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 779, in save_base

force_update, using, update_fields,

File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 851, in _save_table

forced_update)

File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 900, in _do_update

return filtered._update(values) > 0

File "/usr/local/lib/python3.7/site-packages/django/db/models/query.py", line 760, in _update

return query.get_compiler(self.db).execute_sql(CURSOR)

File "/usr/local/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1469, in execute_sql

cursor = super().execute_sql(result_type)

File "/usr/local/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1140, in execute_sql

cursor.execute(sql, params)

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 67, in execute

return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers

return executor(sql, params, many, context)

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

return self.cursor.execute(sql, params)

File "/usr/local/lib/python3.7/site-packages/django/db/utils.py", line 89, in __exit__

raise dj_exc_value.with_traceback(traceback) from exc_value

File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

return self.cursor.execute(sql, params)

django.db.utils.OperationalError: deadlock detected

DETAIL: Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT: See server log for query details.

CONTEXT: while updating tuple (0,29) in relation "core_content"
</pre>

There is a theory behind such a deadlock in this stack overflow post: https://stackoverflow.com/a/42731706

Back