Migration Plugin

I hit this when migrating the following repos:

CentOS 7 RPMs CentOS 8 AppStream CentOS 8 BaseOS RHEL 8 AppStream RHEL 8 BaseOS

There was also 1 container repo and 1 file repo but those are not related I suspect.

From past bugs I suspect the RHEL 8 repos are the culprit. Previous syncing bug with same error:

Task info:

{"pulp_tasks"=>
  [{"pulp_href"=>"/pulp/api/v3/tasks/597a0881-ad9c-4635-b4e2-98f85746c376/",
    "pulp_created"=>"2020-09-09T19:45:56.126+00:00",
    "state"=>"failed",
    "name"=>"pulp_2to3_migration.app.tasks.migrate.migrate_from_pulp2",
    "started_at"=>"2020-09-09T19:45:56.269+00:00",
    "finished_at"=>"2020-09-09T19:56:49.218+00:00",
    "error"=>
     {"traceback"=>
       "  File \"/usr/lib/python3.6/site-packages/rq/worker.py\", line 883, in perform_job\n" +
       "    rv = job.perform()\n" +
       "  File \"/usr/lib/python3.6/site-packages/rq/job.py\", line 657, in perform\n" +
       "    self._result = self._execute()\n" +
       "  File \"/usr/lib/python3.6/site-packages/rq/job.py\", line 663, in _execute\n" +
       "    return self.func(*self.args, **self.kwargs)\n" +
       "  File \"/usr/lib/python3.6/site-packages/pulp_2to3_migration/app/tasks/migrate.py\", line 141, in migrate_from_pulp2\n" +
       "    migrate_content(plan)\n" +
       "  File \"/usr/lib/python3.6/site-packages/pulp_2to3_migration/app/migration.py\", line 36, in migrate_content\n" +
       "    plugin.migrator.migrate_content_to_pulp3()\n" +
       "  File \"/usr/lib/python3.6/site-packages/pulp_2to3_migration/app/plugin/rpm/migrator.py\", line 141, in migrate_content_to_pulp3\n" +
       "    loop.run_until_complete(dm.create())\n" +
       "  File \"/usr/lib64/python3.6/asyncio/base_events.py\", line 484, in run_until_complete\n" +
       "    return future.result()\n" +
       "  File \"/usr/lib/python3.6/site-packages/pulp_2to3_migration/app/plugin/content.py\", line 86, in create\n" +
       "    await pipeline\n" +
       "  File \"/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py\", line 225, in create_pipeline\n" +
       "    await asyncio.gather(*futures)\n" +
       "  File \"/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py\", line 43, in __call__\n" +
       "    await self.run()\n" +
       "  File \"/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/artifact_stages.py\", line 244, in run\n" +
       "    RemoteArtifact.objects.bulk_get_or_create(self._needed_remote_artifacts(batch))\n" +
       "  File \"/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/artifact_stages.py\", line 288, in _needed_remote_artifacts\n" +
       "    msg.format(rp=content_artifact.relative_path, c=d_content.content)\n",
      "description"=>
       "No declared artifact with relative path \"repodata/a22b89e3-765e-45bf-97f0-60e53435f3e5\" for content \"\""},
    "worker"=>"/pulp/api/v3/workers/c2ab7f57-5c17-4f3c-b0d9-071ca78cce21/",
    "child_tasks"=>[],
    "task_group"=>
     "/pulp/api/v3/task-groups/70d63d76-6dad-4f99-b1c4-51977ee6d7f4/",
    "progress_reports"=>
     [{"message"=>
        "Pre-migrating Pulp 2 PACKAGE_ENVIRONMENT content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>24,
       "done"=>24},
      {"message"=>"Pre-migrating Pulp 2 SRPM content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>0,
       "done"=>0},
      {"message"=>"Processing Pulp 2 repositories, importers, distributors",
       "code"=>"processing.repositories",
       "state"=>"completed",
       "total"=>7,
       "done"=>7},
      {"message"=>"Pre-migrating Pulp 2 ISO content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>250,
       "done"=>250},
      {"message"=>"Pre-migrating Pulp 2 ISO content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>250,
       "done"=>250},
      {"message"=>"Pre-migrating Pulp 2 DOCKER_BLOB content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>571,
       "done"=>571},
      {"message"=>"Pre-migrating Pulp 2 DOCKER_BLOB content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>571,
       "done"=>571},
      {"message"=>"Pre-migrating Pulp 2 SRPM content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>0,
       "done"=>0},
      {"message"=>
        "Pre-migrating Pulp 2 YUM_REPO_METADATA_FILE content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>2,
       "done"=>2},
      {"message"=>
        "Pre-migrating Pulp 2 DOCKER_MANIFEST content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>427,
       "done"=>427},
      {"message"=>"Pre-migrating Pulp 2 DOCKER_MANIFEST content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>427,
       "done"=>427},
      {"message"=>
        "Pre-migrating Pulp 2 YUM_REPO_METADATA_FILE content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>2,
       "done"=>2},
      {"message"=>"Pre-migrating Pulp 2 MODULEMD content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>269,
       "done"=>269},
      {"message"=>"Pre-migrating Pulp 2 MODULEMD content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>269,
       "done"=>269},
      {"message"=>"Pre-migrating Pulp 2 DISTRIBUTION content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>3,
       "done"=>3},
      {"message"=>
        "Pre-migrating Pulp 2 DOCKER_MANIFEST_LIST content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>57,
       "done"=>57},
      {"message"=>
        "Pre-migrating Pulp 2 DOCKER_MANIFEST_LIST content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>57,
       "done"=>57},
      {"message"=>"Pre-migrating Pulp 2 DISTRIBUTION content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>3,
       "done"=>3},
      {"message"=>"Pre-migrating Pulp 2 DOCKER_TAG content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>147,
       "done"=>147},
      {"message"=>"Pre-migrating Pulp 2 DOCKER_TAG content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>147,
       "done"=>147},
      {"message"=>"Creating repositories in Pulp 3",
       "code"=>"creating.repositories",
       "state"=>"completed",
       "total"=>7,
       "done"=>7},
      {"message"=>"Pre-migrating Pulp 2 ERRATUM content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>1265,
       "done"=>1265},
      {"message"=>"Pre-migrating Pulp 2 ERRATUM content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>1265,
       "done"=>1265},
      {"message"=>"Pre-migrating Pulp 2 RPM content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>34325,
       "done"=>34325},
      {"message"=>"Pre-migrating Pulp 2 RPM content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>34325,
       "done"=>34325},
      {"message"=>
        "Pre-migrating Pulp 2 MODULEMD_DEFAULTS content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>84,
       "done"=>84},
      {"message"=>
        "Pre-migrating Pulp 2 MODULEMD_DEFAULTS content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>84,
       "done"=>84},
      {"message"=>
        "Pre-migrating Pulp 2 PACKAGE_CATEGORY content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>29,
       "done"=>29},
      {"message"=>"Pre-migrating Pulp 2 PACKAGE_GROUP content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>274,
       "done"=>274},
      {"message"=>"Pre-migrating Pulp 2 PACKAGE_GROUP content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>274,
       "done"=>274},
      {"message"=>
        "Pre-migrating Pulp 2 PACKAGE_LANGPACKS content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>1,
       "done"=>1},
      {"message"=>
        "Pre-migrating Pulp 2 PACKAGE_LANGPACKS content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>1,
       "done"=>1},
      {"message"=>
        "Pre-migrating Pulp 2 PACKAGE_CATEGORY content (detail info)",
       "code"=>"premigrating.content.detail",
       "state"=>"completed",
       "total"=>29,
       "done"=>29},
      {"message"=>
        "Pre-migrating Pulp 2 PACKAGE_ENVIRONMENT content (general info)",
       "code"=>"premigrating.content.general",
       "state"=>"completed",
       "total"=>24,
       "done"=>24},
      {"message"=>"Migrating importers to Pulp 3",
       "code"=>"migrating.importers",
       "state"=>"completed",
       "total"=>7,
       "done"=>7},
      {"message"=>"Migrating docker content to Pulp 3 docker_blob",
       "code"=>"migrating.docker.content",
       "state"=>"completed",
       "total"=>571,
       "done"=>571},
      {"message"=>"Migrating docker content to Pulp 3 docker_manifest",
       "code"=>"migrating.docker.content",
       "state"=>"completed",
       "total"=>427,
       "done"=>427},
      {"message"=>"Migrating docker content to Pulp 3 docker_manifest_list",
       "code"=>"migrating.docker.content",
       "state"=>"completed",
       "total"=>57,
       "done"=>57},
      {"message"=>"Migrating docker content to Pulp 3 docker_tag",
       "code"=>"migrating.docker.content",
       "state"=>"completed",
       "total"=>147,
       "done"=>147},
      {"message"=>"Migrating iso content to Pulp 3 iso",
       "code"=>"migrating.iso.content",
       "state"=>"completed",
       "total"=>250,
       "done"=>250},
      {"message"=>"Migrating content to Pulp 3",
       "code"=>"migrating.content",
       "state"=>"failed",
       "total"=>0,
       "done"=>0},
      {"message"=>"Migrating rpm content to Pulp 3 erratum",
       "code"=>"migrating.rpm.content",
       "state"=>"completed",
       "total"=>1265,
       "done"=>1265},
      {"message"=>"Migrating rpm content to Pulp 3 modulemd",
       "code"=>"migrating.rpm.content",
       "state"=>"completed",
       "total"=>269,
       "done"=>269},
      {"message"=>"Migrating rpm content to Pulp 3 modulemd_defaults",
       "code"=>"migrating.rpm.content",
       "state"=>"completed",
       "total"=>84,
       "done"=>84},
      {"message"=>"Migrating rpm content to Pulp 3 yum_repo_metadata_file",
       "code"=>"migrating.rpm.content",
       "state"=>"completed",
       "total"=>2,
       "done"=>2},
      {"message"=>"Migrating rpm content to Pulp 3 package_langpacks",
       "code"=>"migrating.rpm.content",
       "state"=>"completed",
       "total"=>1,
       "done"=>1},
      {"message"=>"Migrating rpm content to Pulp 3 package_group",
       "code"=>"migrating.rpm.content",
       "state"=>"canceled",
       "total"=>274,
       "done"=>274},
      {"message"=>"Migrating rpm content to Pulp 3 rpm",
       "code"=>"migrating.rpm.content",
       "state"=>"completed",
       "total"=>34325,
       "done"=>34325},
      {"message"=>"Migrating rpm content to Pulp 3 srpm",
       "code"=>"migrating.rpm.content",
       "state"=>"completed",
       "total"=>0,
       "done"=>0},
      {"message"=>"Migrating rpm content to Pulp 3 distribution",
       "code"=>"migrating.rpm.content",
       "state"=>"completed",
       "total"=>3,
       "done"=>2}],
    "created_resources"=>
     ["/pulp/api/v3/task-groups/70d63d76-6dad-4f99-b1c4-51977ee6d7f4/"],
    "reserved_resources_record"=>["pulp_2to3_migration"]}],
 "task_groups"=>
  [{"pulp_href"=>
     "/pulp/api/v3/task-groups/70d63d76-6dad-4f99-b1c4-51977ee6d7f4/",
    "description"=>"Migration Sub-tasks",
    "all_tasks_dispatched"=>false,
    "waiting"=>0,
    "skipped"=>0,
    "running"=>0,
    "completed"=>0,
    "canceled"=>0,
    "failed"=>1,
    "group_progress_reports"=>
     [{"message"=>"Repo version creation",
       "code"=>"create.repo_version",
       "total"=>0,
       "done"=>0},
      {"message"=>"Distribution creation",
       "code"=>"create.distribution",
       "total"=>0,
       "done"=>0}]}],
 "poll_attempts"=>{"total"=>59, "failed"=>1}}

Exception:

Katello::Errors::Pulp3Error: No declared artifact with relative path "repodata/a22b89e3-765e-45bf-97f0-60e53435f3e5" for content ""