Actions
Issue #2580
closedTruncated iso file when os runs of out space
Status:
CLOSED - WONTFIX
Priority:
Normal
Assignee:
-
Sprint/Milestone:
-
Start date:
Due date:
Estimated time:
Severity:
2. Medium
Version:
Platform Release:
OS:
Triaged:
Yes
Groomed:
No
Sprint Candidate:
No
Tags:
Pulp 2
Sprint:
Quarter:
Description
Pulp produced truncated iso file during publish of export distributor because of not enough space in working directory.
We were publishing 38 repo groups - 38 publish operations were started and then one big join() for all of them.
Errors that were produced during publish:
Traceback:
Traceback (most recent call last):
File "/usr/lib/python2.6/site-packages/celery/app/trace.py", line 240, in trace_task
R = retval = fun(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/pulp/server/async/tasks.py", line 393, in __call__
return super(Task, self).__call__(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/celery/app/trace.py", line 437, in __protected_call__
return self.run(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/pulp/server/managers/repo/group/publish.py", line 59, in publish
conduit, call_config)
File "/usr/lib/python2.6/site-packages/pulp/server/managers/repo/group/publish.py", line 71, in _do_publish
report = distributor_instance.publish_group(group, conduit, call_config)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/groupdistributor.py", line 113, in publish_group
return self._publisher.publish()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 557, in publish
return self.process_lifecycle()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 502, in process_lifecycle
super(PluginStep, self).process_lifecycle()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 127, in process_lifecycle
step.process()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 203, in process
self._process_block()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 245, in _process_block
self.process_main()
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/yum/publish.py", line 920, in process_main
generate_iso.create_iso(self.content_dir, self.output_dir, image_prefix, image_size)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/generate_iso.py", line 54, in create_iso
_make_iso(image_list[i], target_dir, output_dir, name)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/generate_iso.py", line 81, in _make_iso
pathspec_file = _get_pathspec_file(file_list, target_dir)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/generate_iso.py", line 227, in _get_pathspec_file
os.write(file_descriptor, graft + '\n')
OSError: [Errno 28] No space left on device
Traceback (most recent call last):
File "/usr/lib/python2.6/site-packages/celery/app/trace.py", line 240, in trace_task
R = retval = fun(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/pulp/server/async/tasks.py", line 393, in __call__
return super(Task, self).__call__(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/celery/app/trace.py", line 437, in __protected_call__
return self.run(*args, **kwargs)
File "/usr/lib/python2.6/site-packages/pulp/server/managers/repo/group/publish.py", line 59, in publish
conduit, call_config)
File "/usr/lib/python2.6/site-packages/pulp/server/managers/repo/group/publish.py", line 71, in _do_publish
report = distributor_instance.publish_group(group, conduit, call_config)
File "/usr/lib/python2.6/site-packages/pulp_rpm/plugins/distributors/export_distributor/groupdistributor.py", line 113, in publish_group
return self._publisher.publish()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 557, in publish
return self.process_lifecycle()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 502, in process_lifecycle
super(PluginStep, self).process_lifecycle()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 127, in process_lifecycle
step.process()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 203, in process
self._process_block()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 245, in _process_block
self.process_main()
File "/usr/lib/python2.6/site-packages/pulp/plugins/util/publish_step.py", line 921, in process_main
copytree(self.source_dir, self.target_dir, symlinks=self.preserve_symlinks)
File "/usr/lib/python2.6/site-packages/pulp/server/util.py", line 245, in copytree
raise Error(errors)
Beacuse of retries in our publishing systems, all publishes eventually succeeded. Next step was to synchronized published repositories back to pulp. It failed with
{
"_href": "/pulp/api/v2/tasks/828dccc4-db02-42b4-9776-103aaf94f203/",
"_id": {
"$oid": "586ff988ba5149b0e287d808"
},
"_ns": "task_status",
"error": {
"code": "PLP0000",
"data": {},
"description": "Importer indicated a failed response",
"sub_errors": []
},
"exception": null,
"finish_time": "2017-01-06T20:15:48Z",
"id": "586ff988ba5149b0e287d808",
"progress_report": {
"iso_importer": {
"error_message": null,
"finished_bytes": 13255024640,
"iso_error_messages": [
{
"error": {},
"name": "sat-6-isos--rhev-3_DOT_5-2017-01-06T08.14-05.iso"
}
],
"num_isos": 3,
"num_isos_finished": 2,
"state": "isos_failed",
"state_times": {
"isos_failed": "2017-01-06T20:15:48",
"isos_in_progress": "2017-01-06T20:09:46",
"manifest_in_progress": "2017-01-06T20:09:44",
"not_started": "2017-01-06T20:09:44"
},
"total_bytes": 13299017728,
"traceback": null
}
},
"result": null,
"spawned_tasks": [],
"start_time": "2017-01-06T20:09:44Z",
"state": "error",
"tags": [
"pulp:repository:all-iso-content",
"pulp:action:sync"
],
"task_id": "828dccc4-db02-42b4-9776-103aaf94f203",
"task_type": "pulp.server.managers.repo.sync.sync",
"traceback": "Traceback (most recent call last):\n File '/usr/lib/python2.6/site-packages/celery/app/trace.py', line 240, in trace_task\n R = retval = fun(*args, **kwargs)\n File '/usr/lib/python2.6/site-packages/pulp/server/async/tasks.py', line 393, in __call__\n return super(Task, self).__call__(*args, **kwargs)\n File '/usr/lib/python2.6/site-packages/celery/app/trace.py', line 437, in __protected_call__\n return self.run(*args, **kwargs)\n File '/usr/lib/python2.6/site-packages/pulp/server/managers/repo/sync.py', line 100, in sync\n raise PulpExecutionException(_('Importer indicated a failed response'))\nPulpExecutionException: Importer indicated a failed response\n",
"worker_name": "reserved_resource_worker-3@pulp07.web.prod.ext.phx2.redhat.com"
}
<pre>
But then is succeeded in last retry attempt. Unfortunately I don't have task details for that, because task has been already reaped.
I think this could be related to https://pulp.plan.io/issues/2142
Actions