Pulp

The test_import test is failing intermittently on Travis:

=================================== FAILURES ===================================

________________________ PulpImportTestCase.test_import ________________________

self = 

    def test_import(self):

        """Test an import."""

        importer = self._create_importer()

>       task_group = self._perform_import(importer)

pulpcore/tests/functional/api/using_plugin/test_pulpimport.py:194: 

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

pulpcore/tests/functional/api/using_plugin/test_pulpimport.py:167: in _perform_import

    task_group = monitor_task_group(task_group_href)

pulpcore/tests/functional/utils.py:54: in monitor_task_group

    exit()

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Use exit() or Ctrl-D (i.e. EOF) to exit, code = None

    def __call__(self, code=None):

        # Shells like IDLE catch the SystemExit, but listen when their

        # stdin wrapper is closed.

        try:

            sys.stdin.close()

        except:

            pass

>       raise SystemExit(code)

E       SystemExit: None

../../../virtualenv/python3.7.6/lib/python3.7/site.py:403: SystemExit

----------------------------- Captured stdout call -----------------------------

The task group failed: {'all_tasks_dispatched': True,

 'canceled': 0,

 'completed': 1,

 'description': 'Import of '

                '/tmp/0cf747ab-d5b3-454a-88f0-7aedf5fad6f3/export-5a84c6c2-4905-4400-b2ea-f48e8c603e8b-20200701_1100.tar.gz',

 'failed': 1,

 'group_progress_reports': [],

 'pulp_href': '/pulp/api/v3/task-groups/a8fc525b-eb7d-442d-a732-694648485198/',

 'running': 0,

 'skipped': 0,

 'waiting': 0}

The following logs appear.

pulp: rq.worker:ERROR: django.db.utils.OperationalError: deadlock detected

DETAIL:  Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT:  See server log for query details.

CONTEXT:  while updating tuple (0,29) in relation "core_content"

Traceback (most recent call last):

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

    return self.cursor.execute(sql, params)

psycopg2.extensions.TransactionRollbackError: deadlock detected

DETAIL:  Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT:  See server log for query details.

CONTEXT:  while updating tuple (0,29) in relation "core_content"

The above exception was the direct cause of the following exception:

Traceback (most recent call last):

  File "/usr/local/lib/python3.7/site-packages/rq/worker.py", line 883, in perform_job

    rv = job.perform()

  File "/usr/local/lib/python3.7/site-packages/rq/job.py", line 657, in perform

    self._result = self._execute()

  File "/usr/local/lib/python3.7/site-packages/rq/job.py", line 663, in _execute

    return self.func(*self.args, **self.kwargs)

  File "/usr/local/lib/python3.7/site-packages/pulpcore/app/tasks/importer.py", line 126, in import_repository_version

    c_result = _import_file(content_path, ContentResource)

  File "/usr/local/lib/python3.7/site-packages/pulpcore/app/tasks/importer.py", line 47, in _import_file

    return resource.import_data(data, raise_errors=True)

  File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 627, in import_data

    return self.import_data_inner(dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs)

  File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 673, in import_data_inner

    raise row_result.errors[-1].error

  File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 569, in import_row

    self.save_instance(instance, using_transactions, dry_run)

  File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 352, in save_instance

    instance.save()

  File "/usr/local/lib/python3.7/site-packages/pulpcore/app/models/base.py", line 110, in save

    return super().save(*args, **kwargs)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 741, in save

    force_update=force_update, update_fields=update_fields)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 779, in save_base

    force_update, using, update_fields,

  File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 851, in _save_table

    forced_update)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 900, in _do_update

    return filtered._update(values) > 0

  File "/usr/local/lib/python3.7/site-packages/django/db/models/query.py", line 760, in _update

    return query.get_compiler(self.db).execute_sql(CURSOR)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1469, in execute_sql

    cursor = super().execute_sql(result_type)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1140, in execute_sql

    cursor.execute(sql, params)

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 67, in execute

    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers

    return executor(sql, params, many, context)

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

    return self.cursor.execute(sql, params)

  File "/usr/local/lib/python3.7/site-packages/django/db/utils.py", line 89, in __exit__

    raise dj_exc_value.with_traceback(traceback) from exc_value

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

    return self.cursor.execute(sql, params)

django.db.utils.OperationalError: deadlock detected

DETAIL:  Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT:  See server log for query details.

CONTEXT:  while updating tuple (0,29) in relation "core_content"

Traceback (most recent call last):

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

    return self.cursor.execute(sql, params)

psycopg2.extensions.TransactionRollbackError: deadlock detected

DETAIL:  Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT:  See server log for query details.

CONTEXT:  while updating tuple (0,29) in relation "core_content"

The above exception was the direct cause of the following exception:

Traceback (most recent call last):

  File "/usr/local/lib/python3.7/site-packages/rq/worker.py", line 883, in perform_job

    rv = job.perform()

  File "/usr/local/lib/python3.7/site-packages/rq/job.py", line 657, in perform

    self._result = self._execute()

  File "/usr/local/lib/python3.7/site-packages/rq/job.py", line 663, in _execute

    return self.func(*self.args, **self.kwargs)

  File "/usr/local/lib/python3.7/site-packages/pulpcore/app/tasks/importer.py", line 126, in import_repository_version

    c_result = _import_file(content_path, ContentResource)

  File "/usr/local/lib/python3.7/site-packages/pulpcore/app/tasks/importer.py", line 47, in _import_file

    return resource.import_data(data, raise_errors=True)

  File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 627, in import_data

    return self.import_data_inner(dataset, dry_run, raise_errors, using_transactions, collect_failed_rows, **kwargs)

  File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 673, in import_data_inner

    raise row_result.errors[-1].error

  File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 569, in import_row

    self.save_instance(instance, using_transactions, dry_run)

  File "/usr/local/lib/python3.7/site-packages/import_export/resources.py", line 352, in save_instance

    instance.save()

  File "/usr/local/lib/python3.7/site-packages/pulpcore/app/models/base.py", line 110, in save

    return super().save(*args, **kwargs)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 741, in save

    force_update=force_update, update_fields=update_fields)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 779, in save_base

    force_update, using, update_fields,

  File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 851, in _save_table

    forced_update)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 900, in _do_update

    return filtered._update(values) > 0

  File "/usr/local/lib/python3.7/site-packages/django/db/models/query.py", line 760, in _update

    return query.get_compiler(self.db).execute_sql(CURSOR)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1469, in execute_sql

    cursor = super().execute_sql(result_type)

  File "/usr/local/lib/python3.7/site-packages/django/db/models/sql/compiler.py", line 1140, in execute_sql

    cursor.execute(sql, params)

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 67, in execute

    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers

    return executor(sql, params, many, context)

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

    return self.cursor.execute(sql, params)

  File "/usr/local/lib/python3.7/site-packages/django/db/utils.py", line 89, in __exit__

    raise dj_exc_value.with_traceback(traceback) from exc_value

  File "/usr/local/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute

    return self.cursor.execute(sql, params)

django.db.utils.OperationalError: deadlock detected

DETAIL:  Process 4137 waits for ShareLock on transaction 6954; blocked by process 4128.

Process 4128 waits for ShareLock on transaction 6951; blocked by process 4137.

HINT:  See server log for query details.

CONTEXT:  while updating tuple (0,29) in relation "core_content"

There is a theory behind such a deadlock in this stack overflow post: https://stackoverflow.com/a/42731706