Issue #6121
Updated by dalley about 4 years ago
A Pulp 2 immediate-mode sync of 150,000 files using this repository [0] takes 37.5 minutes With Pulp 3, the same sync takes about 3x as long (about 2 hours). hours) and then fails at the end with the following error: ~~~ Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: pulp: rq.worker:ERROR: aiohttp.client_exceptions.ServerDisconnectedError Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: Traceback (most recent call last): Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/worker.py", line 822, in perform_job Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: rv = job.perform() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/job.py", line 605, in perform Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: self._result = self._execute() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/job.py", line 611, in _execute Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: return self.func(*self.args, **self.kwargs) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulp_file/pulp_file/app/tasks/synchronizing.py", line 45, in synchronize Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: dv.create() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/declarative_version.py", line 149, in create Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: loop.run_until_complete(pipeline) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/lib64/python3.6/asyncio/base_events.py", line 484, in run_until_complete Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: return future.result() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/api.py", line 209, in create_pipeline Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await asyncio.gather(*futures) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/api.py", line 43, in __call__ Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await self.run() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/artifact_stages.py", line 132, in run Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: pb.done += task.result() # download_count Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/artifact_stages.py", line 157, in _handle_content_unit Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await asyncio.gather(*downloaders_for_content) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/models.py", line 81, in download Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: download_result = await downloader.run(extra_data=self.extra_data) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/download/base.py", line 221, in run Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: return await self._run(extra_data=extra_data) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/backoff/_async.py", line 133, in retry Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: ret = await target(*args, **kwargs) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/download/http.py", line 184, in _run Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: async with self.session.get(self.url, proxy=self.proxy) as response: Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/aiohttp/client.py", line 1012, in __aenter__ Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: self._resp = await self._coro Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/aiohttp/client.py", line 504, in _request Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await resp.start(conn) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/aiohttp/client_reqrep.py", line 847, in start Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: message, payload = await self._protocol.read() # type: ignore # noqa Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/aiohttp/streams.py", line 591, in read Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await self._waiter Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: aiohttp.client_exceptions.ServerDisconnectedError Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: Traceback (most recent call last): Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/worker.py", line 822, in perform_job Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: rv = job.perform() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/job.py", line 605, in perform Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: self._result = self._execute() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/job.py", line 611, in _execute Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: return self.func(*self.args, **self.kwargs) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulp_file/pulp_file/app/tasks/synchronizing.py", line 45, in synchronize Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: dv.create() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/declarative_version.py", line 149, in create Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: loop.run_until_complete(pipeline) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/lib64/python3.6/asyncio/base_events.py", line 484, in run_until_complete Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: return future.result() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/api.py", line 209, in create_pipeline Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await asyncio.gather(*futures) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/api.py", line 43, in __call__ Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await self.run() Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/artifact_stages.py", line 132, in run Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: pb.done += task.result() # download_count Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/artifact_stages.py", line 157, in _handle_content_unit Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await asyncio.gather(*downloaders_for_content) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/plugin/stages/models.py", line 81, in download Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: download_result = await downloader.run(extra_data=self.extra_data) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/download/base.py", line 221, in run Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: return await self._run(extra_data=extra_data) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/backoff/_async.py", line 133, in retry Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: ret = await target(*args, **kwargs) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/home/vagrant/devel/pulpcore/pulpcore/download/http.py", line 184, in _run Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: async with self.session.get(self.url, proxy=self.proxy) as response: Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/aiohttp/client.py", line 1012, in __aenter__ Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: self._resp = await self._coro Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/aiohttp/client.py", line 504, in _request Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await resp.start(conn) Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/aiohttp/client_reqrep.py", line 847, in start Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: message, payload = await self._protocol.read() # type: ignore # noqa Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: File "/usr/local/lib/pulp/lib64/python3.6/site-packages/aiohttp/streams.py", line 591, in read Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: await self._waiter Feb 09 19:52:24 pulp2-nightly-pulp3-source-centos7.localhost.example.com rq[9501]: aiohttp.client_exceptions.ServerDisconnectedError ~~~ An on_demand sync of the same repo in Pulp 3 is able to successfully finish, but it still takes about 90 minutes, still much longer than Pulp 2. The first 50k files seem sync roughly as fast as they do for Pulp 2, but once the number grows higher to progress starts being made very very noticeably slower than at the beginning. [0] http://quartet.usersys.redhat.com/pub/fake-repos/very_large_file_150k/
.