Issue #2580
closedTruncated iso file when os runs of out space
Description
Pulp produced truncated iso file during publish of export distributor because of not enough space in working directory.
We were publishing 38 repo groups - 38 publish operations were started and then one big join() for all of them.
Errors that were produced during publish:
Traceback:
Traceback (most recent call last):
File "/usr/lib/python2.6/site-packages/celery/app/trace.py", line 240, in trace_task
R = retval = fun(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/pulp/server/async/tasks.py", line 393, in __call__
return super(Task, self).__call__(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/celery/app/trace.py", line 437, in __protected_call__
return self.run(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/pulp/server/managers/repo/group/publish.py", line 59, in publish
conduit, call_config)
File "/usr/lib/python2.6/site-packages/pulp/server/managers/repo/group/publish.py", line 71, in _do_publish
report = distributor_instance.publish_group(group, conduit, call_config)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/groupdistributor.py", line 113, in publish_group
return self._publisher.publish()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 557, in publish
return self.process_lifecycle()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 502, in process_lifecycle
super(PluginStep, self).process_lifecycle()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 127, in process_lifecycle
step.process()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 203, in process
self._process_block()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 245, in _process_block
self.process_main()
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/yum/publish.py", line 920, in process_main
generate_iso.create_iso(self.content_dir, self.output_dir, image_prefix, image_size)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/generate_iso.py", line 54, in create_iso
_make_iso(image_list[i], target_dir, output_dir, name)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/generate_iso.py", line 81, in _make_iso
pathspec_file = _get_pathspec_file(file_list, target_dir)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/generate_iso.py", line 227, in _get_pathspec_file
os.write(file_descriptor, graft + '\n')
OSError: [Errno 28] No space left on device
Traceback (most recent call last):
File "/usr/lib/python2.6/site-packages/celery/app/trace.py", line 240, in trace_task
R = retval = fun(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/pulp/server/async/tasks.py", line 393, in __call__
return super(Task, self).__call__(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/celery/app/trace.py", line 437, in __protected_call__
return self.run(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/pulp/server/managers/repo/group/publish.py", line 59, in publish
conduit, call_config)
File "/usr/lib/python2.6/site-packages/pulp/server/managers/repo/group/publish.py", line 71, in _do_publish
report = distributor_instance.publish_group(group, conduit, call_config)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/groupdistributor.py", line 113, in publish_group
return self._publisher.publish()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 557, in publish
return self.process_lifecycle()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 502, in process_lifecycle
super(PluginStep, self).process_lifecycle()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 127, in process_lifecycle
step.process()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 203, in process
self._process_block()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 245, in _process_block
self.process_main()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 921, in process_main
copytree(self.source_dir, self.target_dir, symlinks=self.preserve_symlinks)
File "/usr/lib/python2.6/site-packages/pulp/server/util.py", line 245, in copytree
raise Error(errors)
Beacuse of retries in our publishing systems, all publishes eventually succeeded. Next step was to synchronized published repositories back to pulp. It failed with
{
"_href": "/pulp/api/v2/tasks/828dccc4-db02-42b4-9776-103aaf94f203/",
"_id": {
"$oid": "586ff988ba5149b0e287d808"
},
"_ns": "task_status",
"error": {
"code": "PLP0000",
"data": {},
"description": "Importer indicated a failed response",
"sub_errors": []
},
"exception": null,
"finish_time": "2017-01-06T20:15:48Z",
"id": "586ff988ba5149b0e287d808",
"progress_report": {
"iso_importer": {
"error_message": null,
"finished_bytes": 13255024640,
"iso_error_messages": [
{
"error": {},
"name": "sat-6-isos--rhev-3_DOT_5-2017-01-06T08.14-05.iso"
}
],
"num_isos": 3,
"num_isos_finished": 2,
"state": "isos_failed",
"state_times": {
"isos_failed": "2017-01-06T20:15:48",
"isos_in_progress": "2017-01-06T20:09:46",
"manifest_in_progress": "2017-01-06T20:09:44",
"not_started": "2017-01-06T20:09:44"
},
"total_bytes": 13299017728,
"traceback": null
}
},
"result": null,
"spawned_tasks": [],
"start_time": "2017-01-06T20:09:44Z",
"state": "error",
"tags": [
"pulp:repository:all-iso-content",
"pulp:action:sync"
],
"task_id": "828dccc4-db02-42b4-9776-103aaf94f203",
"task_type": "pulp.server.managers.repo.sync.sync",
"traceback": "Traceback (most recent call last):\n File '/usr/lib/python2.6/site-packages/celery/app/trace.py', line 240, in trace_task\n R = retval = fun(*args, **kwargs)\n File '/usr/lib/python2.6/site-packages/pulp/server/async/tasks.py', line 393, in __call__\n return super(Task, self).__call__(*args, **kwargs)\n File '/usr/lib/python2.6/site-packages/celery/app/trace.py', line 437, in __protected_call__\n return self.run(*args, **kwargs)\n File '/usr/lib/python2.6/site-packages/pulp/server/managers/repo/sync.py', line 100, in sync\n raise PulpExecutionException(_('Importer indicated a failed response'))\nPulpExecutionException: Importer indicated a failed response\n",
"worker_name": "reserved_resource_worker-3@pulp07.web.prod.ext.phx2.redhat.com"
}
<pre>
But then is succeeded in last retry attempt. Unfortunately I don't have task details for that, because task has been already reaped.
I think this could be related to https://pulp.plan.io/issues/2142
Updated by mhrivnak about 7 years ago
I confirmed that during sync, Pulp will refuse to import a file that doesn't match the size and checksum in the PULP_MANIFEST file.
So the bug here seems to be about cleanup. If export fails, Pulp should perhaps try to clean up at least any partially-written files, if not the entire partial publication.
Updated by bmbouter about 5 years ago
- Status changed from NEW to CLOSED - WONTFIX
Pulp 2 is approaching maintenance mode, and this Pulp 2 ticket is not being actively worked on. As such, it is being closed as WONTFIX. Pulp 2 is still accepting contributions though, so if you want to contribute a fix for this ticket, please reopen or comment on it. If you don't have permissions to reopen this ticket, or you want to discuss an issue, please reach out via the developer mailing list.