Debian Support

Ticket moved to GitHub: "pulp/pulp_deb/407":https://github.com/pulp/pulp_deb/issues/407

pulpcore version: 3.12.1

pulp_deb version: 2.11.1

Attempts to sync a repository with a remote that has Debian components defined consistently fail with the error "Response payload is not completed".

Failing remote config:

{
    "architectures": "amd64",
    "ca_cert": null,
    "client_cert": null,
    "components": "main contrib non-free",
    "connect_timeout": null,
    "distributions": "bullseye",
    "download_concurrency": 10,
    "gpgkey": null,
    "headers": null,
    "ignore_missing_package_indices": false,
    "name": "remote_debian_bullseye",
    "policy": "immediate",
    "proxy_url": null,
    "pulp_created": "2021-04-29T11:17:38.824675Z",
    "pulp_href": "/pulp/api/v3/remotes/deb/apt/18600e5b-4914-4bbd-ae2a-bc0488c30e07/",
    "pulp_labels": {},
    "pulp_last_updated": "2021-04-30T00:47:55.224174Z",
    "rate_limit": null,
    "sock_connect_timeout": null,
    "sock_read_timeout": null,
    "sync_installer": false,
    "sync_sources": false,
    "sync_udebs": false,
    "tls_validation": true,
    "total_timeout": null,
    "url": "http://ftp.au.debian.org/debian/"
}

Error reported to API:

{
            "child_tasks": [],
            "created_resources": [],
            "error": {
                "description": "Response payload is not completed",
                "traceback": "  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/rq/worker.py\", line 1008, in perform_job\n    rv = job.perform()\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/rq/job.py\", line 706, in perform\n    self._result = self._execute()\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/rq/job.py\", line 729, in _execute\n    result = self.func(*self.args, **self.kwargs)\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulp_deb/app/tasks/synchronizing.py\", line 122, in synchronize\n    DebDeclarativeVersion(first_stage, repository, mirror=mirror).create()\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/declarative_version.py\", line 149, in create\n    loop.run_until_complete(pipeline)\n  File \"/usr/lib/python3.6/asyncio/base_events.py\", line 484, in run_until_complete\n    return future.result()\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py\", line 225, in create_pipeline\n    await asyncio.gather(*futures)\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py\", line 43, in __call__\n    await self.run()\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/artifact_stages.py\", line 174, in run\n    pb.done += task.result()  # download_count\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/artifact_stages.py\", line 200, in _handle_content_unit\n    await asyncio.gather(*downloaders_for_content)\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/models.py\", line 89, in download\n    download_result = await downloader.run(extra_data=self.extra_data)\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/base.py\", line 241, in run\n    return await self._run(extra_data=extra_data)\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/backoff/_async.py\", line 133, in retry\n    ret = await target(*args, **kwargs)\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/http.py\", line 218, in _run\n    to_return = await self._handle_response(response)\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/http.py\", line 185, in _handle_response\n    chunk = await response.content.read(1048576)  # 1 megabyte\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/aiohttp/streams.py\", line 380, in read\n    await self._wait(\"read\")\n  File \"/usr/local/lib/pulp/lib/python3.6/site-packages/aiohttp/streams.py\", line 306, in _wait\n    await waiter\n"
            },
            "finished_at": "2021-04-29T22:03:38.950771Z",
            "logging_cid": "2367609e386d4f3b8c94830deea2053e",
            "name": "pulp_deb.app.tasks.synchronizing.synchronize",
            "parent_task": null,
            "progress_reports": [
                {
                    "code": "update.release_file",
                    "done": 1,
                    "message": "Update ReleaseFile units",
                    "state": "canceled",
                    "suffix": null,
                    "total": null
                },
                {
                    "code": "update.packageindex",
                    "done": 3,
                    "message": "Update PackageIndex units",
                    "state": "canceled",
                    "suffix": null,
                    "total": null
                },
                {
                    "code": "associating.content",
                    "done": 10017,
                    "message": "Associating Content",
                    "state": "canceled",
                    "suffix": null,
                    "total": null
                },
                {
                    "code": "unassociating.content",
                    "done": 0,
                    "message": "Un-Associating Content",
                    "state": "canceled",
                    "suffix": null,
                    "total": null
                },
                {
                    "code": "sync.downloading.artifacts",
                    "done": 200,
                    "message": "Downloading Artifacts",
                    "state": "failed",
                    "suffix": null,
                    "total": null
                }
            ],
            "pulp_created": "2021-04-29T22:02:51.216445Z",
            "pulp_href": "/pulp/api/v3/tasks/c6ca7102-7b9a-4774-9455-5eb486b527b3/",
            "reserved_resources_record": [
                "/pulp/api/v3/remotes/deb/apt/2889a55e-8ece-40ae-a811-d39d89deda4e/",
                "/pulp/api/v3/repositories/deb/apt/35664f88-071a-4bec-a1be-b7e0befe369e/"
            ],
            "started_at": "2021-04-29T22:02:51.391384Z",
            "state": "failed",
            "task_group": null,
            "worker": "/pulp/api/v3/workers/ebece415-1026-44a0-8668-0de82c2ac9e9/"
        }

When failing in this manner multiple attempts will progress the sync to a point beyond which it will not progress, in the example above that point was when the "associating.content" value was 10017.

As a further test the remote was recreated with a null value for "components":

{
    "architectures": "amd64",
    "ca_cert": null,
    "client_cert": null,
    "components": null,
    "connect_timeout": null,
    "distributions": "bullseye",
    "download_concurrency": 10,
    "gpgkey": null,
    "headers": null,
    "ignore_missing_package_indices": true,
    "name": "remote_debian_bullseye",
    "policy": "immediate",
    "proxy_url": null,
    "pulp_created": "2021-05-04T02:08:55.188864Z",
    "pulp_href": "/pulp/api/v3/remotes/deb/apt/3f95927d-b39c-4bda-a24e-f850d73f2095/",
    "pulp_labels": {},
    "pulp_last_updated": "2021-05-04T02:08:55.188881Z",
    "rate_limit": null,
    "sock_connect_timeout": null,
    "sock_read_timeout": null,
    "sync_installer": false,
    "sync_sources": false,
    "sync_udebs": false,
    "tls_validation": true,
    "total_timeout": null,
    "url": "http://ftp.au.debian.org/debian/"
}

While the sync attempts for this configuration still failed multiple times it is always with a "Server disconnected" error and the sync progressed further st each attempt until a complete sync was achieved.

{
   "child_tasks": [],
   "created_resources": [
       "/pulp/api/v3/repositories/deb/apt/dbd1bbf7-a5ef-46b6-922a-0ddf62b92ebc/versions/4/"
   ],
   "error": null,
   "finished_at": "2021-05-04T06:24:54.428619Z",
   "logging_cid": "1121eb0f04d742aa80b0a1cb70b0cedb",
   "name": "pulp_deb.app.tasks.synchronizing.synchronize",
   "parent_task": null,
   "progress_reports": [
       {
           "code": "sync.downloading.artifacts",
           "done": 3157,
           "message": "Downloading Artifacts",
           "state": "completed",
           "suffix": null,
           "total": null
       },
       {
           "code": "update.release_file",
           "done": 1,
           "message": "Update ReleaseFile units",
           "state": "completed",
           "suffix": null,
           "total": null
       },
       {
           "code": "update.packageindex",
           "done": 3,
           "message": "Update PackageIndex units",
           "state": "completed",
           "suffix": null,
           "total": null
       },
       {
           "code": "associating.content",
           "done": 119400,
           "message": "Associating Content",
           "state": "completed",
           "suffix": null,
           "total": null
       },
       {
           "code": "unassociating.content",
           "done": 1,
           "message": "Un-Associating Content",
           "state": "completed",
           "suffix": null,
           "total": null
       }
   ],
   "pulp_created": "2021-05-04T06:09:38.553551Z",
   "pulp_href": "/pulp/api/v3/tasks/c9674fbd-5d30-4932-bbc1-1c50e8d6e595/",
   "reserved_resources_record": [
       "/pulp/api/v3/remotes/deb/apt/3f95927d-b39c-4bda-a24e-f850d73f2095/",
       "/pulp/api/v3/repositories/deb/apt/dbd1bbf7-a5ef-46b6-922a-0ddf62b92ebc/"
   ],
   "started_at": "2021-05-04T06:09:38.704737Z",
   "state": "completed",
   "task_group": null,
   "worker": "/pulp/api/v3/workers/a25d6c8a-b4ea-4500-ab08-92104d08bca7/"
}

Full log of a sync failing with the "Response payload is not completed" error:

Apr 30 00:29:19 pulphost.domain.local gunicorn[1381]: pulp [28ac6dceafad4e07933ddfbeead891e7]:  - admin [30/Apr/2021:00:29:19 +0000] "POST /pulp/api/v3/repositories/deb/apt/dbd1bbf7-a5ef-46b6-922a-
0ddf62b92ebc/sync/ HTTP/1.0" 202 67 "-" "HTTPie/0.9.8"
Apr 30 00:29:19 pulphost.domain.local rq[1374]: pulp [None]: rq.worker:INFO: 1374@pulphost.domain.local: 69878f8d-7e56-4322-8f4c-5687c9ae7aeb
Apr 30 00:29:19 pulphost.domain.local rq[1383]: pulp [28ac6dceafad4e07933ddfbeead891e7]: rq.worker:INFO: resource-manager: Job OK (46efd531-9651-48db-8aa3-73521c79de42)
Apr 30 00:29:19 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: pulp_deb.app.tasks.synchronizing:INFO: Downloading Release file for distribution: "bullseye"
Apr 30 00:29:20 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: pulp_deb.app.tasks.synchronizing:INFO: Parsing Release file at distribution="bullseye"
Apr 30 00:29:20 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: pulp_deb.app.tasks.synchronizing:INFO: Downloading: contrib/binary-amd64/Packages
Apr 30 00:29:20 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: pulp_deb.app.tasks.synchronizing:INFO: Downloading: non-free/binary-amd64/Packages
Apr 30 00:29:20 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: pulp_deb.app.tasks.synchronizing:INFO: Downloading: main/binary-amd64/Packages
Apr 30 00:29:25 pulphost.domain.local rq[1374]: /usr/local/lib/pulp/lib/python3.6/site-packages/debian/deb822.py:740: UserWarning: Parsing of Deb822 data with python3-apt's apt_pkg was requested bu
t this package is not importable. Is python3-apt installed?
Apr 30 00:29:25 pulphost.domain.local rq[1374]:   warnings.warn(msg)
Apr 30 00:29:25 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: pulp_deb.app.tasks.synchronizing:INFO: Artifact digest not matched. Ignored
<snip>
Apr 30 00:29:28 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: pulp_deb.app.tasks.synchronizing:INFO: Artifact digest not matched. Ignored
Apr 30 00:29:30 pulphost.domain.local gunicorn[1381]: pulp [953559dc9eb149849798a84899c1fa04]:  - admin [30/Apr/2021:00:29:30 +0000] "GET /pulp/api/v3/tasks/cb2c2cb4-c3ca-4758-be66-faa930487dad/ HT
TP/1.0" 200 1391 "-" "HTTPie/0.9.8"
Apr 30 00:30:03 pulphost.domain.local gunicorn[1381]: pulp [c2c1187c01de41d3bd0ea2ca6ff96079]:  - admin [30/Apr/2021:00:30:03 +0000] "GET /pulp/api/v3/tasks/8f2574c3-56f4-40f3-b0c3-31227bef42fd/ HT
TP/1.0" 200 602 "-" "HTTPie/0.9.8"
Apr 30 00:30:37 pulphost.domain.local rq[1388]: pulp [None]: rq.worker:INFO: Cleaning registries for queue: 1388@pulphost.domain.local
Apr 30 00:30:47 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: asyncio:ERROR: Task was destroyed but it is pending!
Apr 30 00:30:47 pulphost.domain.local rq[1374]: task: <Task pending coro=<Event.wait() running at /usr/lib/python3.6/asyncio/locks.py:283> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object
 at 0x7d07428b8558>()]> cb=[_wait.<locals>._on_completion() at /usr/lib/python3.6/asyncio/tasks.py:380]>
Apr 30 00:30:47 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: asyncio:ERROR: Task was destroyed but it is pending!
Apr 30 00:30:47 pulphost.domain.local rq[1374]: task: <Task pending coro=<Event.wait() running at /usr/lib/python3.6/asyncio/locks.py:283> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object
 at 0x7d07428b8498>()]> cb=[_wait.<locals>._on_completion() at /usr/lib/python3.6/asyncio/tasks.py:380]>
Apr 30 00:30:47 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: asyncio:ERROR: Task was destroyed but it is pending!
Apr 30 00:30:47 pulphost.domain.local rq[1374]: task: <Task pending coro=<Event.wait() running at /usr/lib/python3.6/asyncio/locks.py:283> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object
 at 0x7d07428b83d8>()]> cb=[_wait.<locals>._on_completion() at /usr/lib/python3.6/asyncio/tasks.py:380]>
Apr 30 00:30:47 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: asyncio:ERROR: Task was destroyed but it is pending!
Apr 30 00:30:47 pulphost.domain.local rq[1374]: task: <Task pending coro=<Event.wait() running at /usr/lib/python3.6/asyncio/locks.py:283> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object
 at 0x7d07428b8318>()]> cb=[_wait.<locals>._on_completion() at /usr/lib/python3.6/asyncio/tasks.py:380]>
Apr 30 00:30:47 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: asyncio:ERROR: Task was destroyed but it is pending!
Apr 30 00:30:47 pulphost.domain.local rq[1374]: task: <Task pending coro=<Event.wait() running at /usr/lib/python3.6/asyncio/locks.py:283> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object
 at 0x7d07428b8138>()]>>
Apr 30 00:30:47 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: asyncio:ERROR: Task was destroyed but it is pending!
Apr 30 00:30:47 pulphost.domain.local rq[1374]: task: <Task pending coro=<Event.wait() running at /usr/lib/python3.6/asyncio/locks.py:283> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object
 at 0x7d07428b8258>()]> cb=[_wait.<locals>._on_completion() at /usr/lib/python3.6/asyncio/tasks.py:380]>
Apr 30 00:30:48 pulphost.domain.local rq[1374]: pulp [28ac6dceafad4e07933ddfbeead891e7]: rq.worker:ERROR: Traceback (most recent call last):
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/rq/worker.py", line 1008, in perform_job
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     rv = job.perform()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/rq/job.py", line 706, in perform
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     self._result = self._execute()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/rq/job.py", line 729, in _execute
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     result = self.func(*self.args, **self.kwargs)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulp_deb/app/tasks/synchronizing.py", line 122, in synchronize
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     DebDeclarativeVersion(first_stage, repository, mirror=mirror).create()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/declarative_version.py", line 149, in create
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     loop.run_until_complete(pipeline)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/lib/python3.6/asyncio/base_events.py", line 484, in run_until_complete
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     return future.result()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py", line 225, in create_pipeline
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await asyncio.gather(*futures)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py", line 43, in __call__
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await self.run()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/artifact_stages.py", line 174, in run
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     pb.done += task.result()  # download_count
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/artifact_stages.py", line 200, in _handle_content_unit
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await asyncio.gather(*downloaders_for_content)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/models.py", line 89, in download
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     download_result = await downloader.run(extra_data=self.extra_data)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/base.py", line 241, in run
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     return await self._run(extra_data=extra_data)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/backoff/_async.py", line 133, in retry
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     ret = await target(*args, **kwargs)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/http.py", line 218, in _run
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     to_return = await self._handle_response(response)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/http.py", line 185, in _handle_response
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     chunk = await response.content.read(1048576)  # 1 megabyte
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/aiohttp/streams.py", line 380, in read
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await self._wait("read")
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/aiohttp/streams.py", line 306, in _wait
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await waiter
Apr 30 00:30:48 pulphost.domain.local rq[1374]: aiohttp.client_exceptions.ClientPayloadError: Response payload is not completed
Apr 30 00:30:48 pulphost.domain.local rq[1374]: Traceback (most recent call last):
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/rq/worker.py", line 1008, in perform_job
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     rv = job.perform()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/rq/job.py", line 706, in perform
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     self._result = self._execute()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/rq/job.py", line 729, in _execute
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     result = self.func(*self.args, **self.kwargs)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulp_deb/app/tasks/synchronizing.py", line 122, in synchronize
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     DebDeclarativeVersion(first_stage, repository, mirror=mirror).create()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/declarative_version.py", line 149, in create
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     loop.run_until_complete(pipeline)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/lib/python3.6/asyncio/base_events.py", line 484, in run_until_complete
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     return future.result()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py", line 225, in create_pipeline
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await asyncio.gather(*futures)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py", line 43, in __call__
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await self.run()
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/artifact_stages.py", line 174, in run
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     pb.done += task.result()  # download_count
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/artifact_stages.py", line 200, in _handle_content_unit
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await asyncio.gather(*downloaders_for_content)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/plugin/stages/models.py", line 89, in download
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     download_result = await downloader.run(extra_data=self.extra_data)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/base.py", line 241, in run
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     return await self._run(extra_data=extra_data)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/backoff/_async.py", line 133, in retry
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     ret = await target(*args, **kwargs)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/http.py", line 218, in _run
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     to_return = await self._handle_response(response)
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/pulpcore/download/http.py", line 185, in _handle_response
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     chunk = await response.content.read(1048576)  # 1 megabyte
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/aiohttp/streams.py", line 380, in read
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await self._wait("read")
Apr 30 00:30:48 pulphost.domain.local rq[1374]:   File "/usr/local/lib/pulp/lib/python3.6/site-packages/aiohttp/streams.py", line 306, in _wait
Apr 30 00:30:48 pulphost.domain.local rq[1374]:     await waiter
Apr 30 00:30:48 pulphost.domain.local rq[1374]: aiohttp.client_exceptions.ClientPayloadError: Response payload is not completed