Pulp

commit 3313998ed12557ed63ae4c55bc6b5d9c07dbbf58

Author: Daniel Alley <dalley@redhat.com>

Date: Mon Jun 21 19:16:00 2021 -0400

Provide an option to fall back to on_demand if artifact download fails

closes: #5286

https://pulp.plan.io/issues/5286

diff --git a/CHANGES/5286.feature b/CHANGES/5286.feature

new file mode 100644

index 000000000..bf6641dd9

--- /dev/null

+++ b/CHANGES/5286.feature

@@ -0,0 +1 @@

+Files which are unable to be downloaded will fall back to "on_demand" and log a warning rather than causing a sync to fail.

diff --git a/pulpcore/app/migrations/0066_download_concurrency_and_retry_changes.py b/pulpcore/app/migrations/0066_remote_field_changes.py

similarity index 80%

rename from pulpcore/app/migrations/0066_download_concurrency_and_retry_changes.py

rename to pulpcore/app/migrations/0066_remote_field_changes.py

index 836cd5e51..08c468ee1 100644

--- a/pulpcore/app/migrations/0066_download_concurrency_and_retry_changes.py

+++ b/pulpcore/app/migrations/0066_remote_field_changes.py

@@ -21,4 +21,9 @@ class Migration(migrations.Migration):

name='max_retries',

field=models.PositiveIntegerField(null=True),

),

+ migrations.AddField(

+ model_name='remote',

+ name='continue_on_error',

+ field=models.BooleanField(default=False),

+ ),

]

diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py

index e1dbd28a5..cea59a573 100644

--- a/pulpcore/app/models/repository.py

+++ b/pulpcore/app/models/repository.py

@@ -295,6 +295,7 @@ class Remote(MasterModel):

null=True, validators=[MinValueValidator(1, "Download concurrency must be at least 1")]

)

max_retries = models.PositiveIntegerField(null=True)

+ continue_on_error = models.BooleanField(default=False)

policy = models.TextField(choices=POLICY_CHOICES, default=IMMEDIATE)

total_timeout = models.FloatField(

diff --git a/pulpcore/app/serializers/repository.py b/pulpcore/app/serializers/repository.py

index 4504b353b..2c2d5e422 100644

--- a/pulpcore/app/serializers/repository.py

+++ b/pulpcore/app/serializers/repository.py

@@ -152,6 +152,15 @@ class RemoteSerializer(ModelSerializer):

required=False,

allow_null=True,

)

+ continue_on_error = serializers.BooleanField(

+ help_text=(

+ "In the event that a sync encounters a recoverable error such as a 404 while "

+ "downloading an artifact, attempt to recover and continue. Caution: use of this "

+ "option may result in 'successful' syncs which do not fully conform to the "

+ "expected sync settings, and defer failure to the time of content consumption."

+ ),

+ required=False,

+ )

policy = serializers.ChoiceField(

help_text="The policy to use when downloading content.",

choices=((models.Remote.IMMEDIATE, "When syncing, download all metadata and content now.")),

diff --git a/pulpcore/plugin/stages/artifact_stages.py b/pulpcore/plugin/stages/artifact_stages.py

index 60d50d14e..34a6eded7 100644

--- a/pulpcore/plugin/stages/artifact_stages.py

+++ b/pulpcore/plugin/stages/artifact_stages.py

@@ -124,13 +124,16 @@ class ArtifactDownloader(Stage):

max_concurrent_content (int): The maximum number of

:class:`~pulpcore.plugin.stages.DeclarativeContent` instances to handle simultaneously.

Default is 200.

+ continue_on_error (bool): Whether to continue if an artifact fails to be downloaded.

+ In this case, the artifact will fall back to 'deferred_download'.

args: unused positional arguments passed along to :class:`~pulpcore.plugin.stages.Stage`.

kwargs: unused keyword arguments passed along to :class:`~pulpcore.plugin.stages.Stage`.

"""

- def __init__(self, max_concurrent_content=200, *args, **kwargs):

+ def __init__(self, max_concurrent_content=200, continue_on_error=False, *args, **kwargs):

super().__init__(*args, **kwargs)

self.max_concurrent_content = max_concurrent_content

+ self.continue_on_error = continue_on_error

async def run(self):

"""

@@ -196,10 +199,22 @@ class ArtifactDownloader(Stage):

and not d_artifact.deferred_download

and not d_artifact.artifact.file

]

+ downloaded_count = 0

if downloaders_for_content:

- await asyncio.gather(*downloaders_for_content)

+ results = await asyncio.gather(*downloaders_for_content, return_exceptions=True)

+ for d_artifact, result in zip(d_content.d_artifacts, results):

+ if isinstance(result, Exception):

+ if self.continue_on_error:

+ log.warn(

+ "Download of {} failed due to error: {}.".format(d_artifact.url, result)

+ )

+ d_artifact.deferred_download = True

+ else:

+ raise result

+ else:

+ downloaded_count += 1

await self.put(d_content)

- return len(downloaders_for_content)

+ return downloaded_count

class ArtifactSaver(Stage):