Migration Plugin

1. Create and run MP

$  http POST :24817/pulp/api/v3/migration-plans/ plan='{"plugins": [{"type": "iso","repositories": [{"name": "file","repository_versions": [{"pulp2_repository_id": "file","pulp2_distributor_repository_ids": ["file", "file2"]}],"pulp2_importer_repository_id": "file2"}]}]}
'
HTTP/1.1 201 Created
Allow: GET, POST, HEAD, OPTIONS
Connection: close
Content-Length: 342
Content-Type: application/json
Date: Thu, 12 Nov 2020 10:25:41 GMT
Location: /pulp/api/v3/migration-plans/c4dee9d9-836d-4735-8caa-cdc1f74781f9/
Server: gunicorn/20.0.4
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "plan": {
        "plugins": [
            {
                "repositories": [
                    {
                        "name": "file",
                        "pulp2_importer_repository_id": "file2",
                        "repository_versions": [
                            {
                                "pulp2_distributor_repository_ids": [
                                    "file",
                                    "file2"
                                ],
                                "pulp2_repository_id": "file"
                            }
                        ]
                    }
                ],
                "type": "iso"
            }
        ]
    },
    "pulp_created": "2020-11-12T10:25:41.255970Z",
    "pulp_href": "/pulp/api/v3/migration-plans/c4dee9d9-836d-4735-8caa-cdc1f74781f9/"
}

2. Create and run second MP ( remove 'file2' distributor from plan)

$  http POST :24817/pulp/api/v3/migration-plans/ plan='{"plugins": [{"type": "iso","repositories": [{"name": "file","repository_versions": [{"pulp2_repository_id": "file","pulp2_distributor_repository_ids": ["file"]}],"pulp2_importer_repository_id": "file2"}]}]}
'
HTTP/1.1 201 Created
Allow: GET, POST, HEAD, OPTIONS
Connection: close
Content-Length: 334
Content-Type: application/json
Date: Thu, 12 Nov 2020 10:26:04 GMT
Location: /pulp/api/v3/migration-plans/cda7f196-0066-4612-bd5d-31818745ef4a/
Server: gunicorn/20.0.4
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "plan": {
        "plugins": [
            {
                "repositories": [
                    {
                        "name": "file",
                        "pulp2_importer_repository_id": "file2",
                        "repository_versions": [
                            {
                                "pulp2_distributor_repository_ids": [
                                    "file"
                                ],
                                "pulp2_repository_id": "file"
                            }
                        ]
                    }
                ],
                "type": "iso"
            }
        ]
    },
    "pulp_created": "2020-11-12T10:26:04.134864Z",
    "pulp_href": "/pulp/api/v3/migration-plans/cda7f196-0066-4612-bd5d-31818745ef4a/"
}

3. Observe in the db for both pulp2distributors not_in_plan=False however file2 distributor should be marked ad True because it was not specified in the 2nd plan

Potential place where to start with https://github.com/pulp/pulp-2to3-migration/blob/master/pulp_2to3_migration/app/pre_migration.py#L662

we are comparing premigrated data with mongo data but not with data specified in plan

        removed_dist_object_ids = premigrated_dist_object_ids - mongo_dist_object_ids

        Pulp2Distributor.objects.filter(
            pulp2_object_id__in=removed_dist_object_ids).update(not_in_plan=True)

Seems like there is same issue with repositories

After file migration i have created an rpm plan(simple plan), when i list pulp2 repos i see both file and rpm repos marked as in plan.

If looking into the code, we do look into the plan's data for the repos but we compare it with the mongo only, we should also compare the data specified in plan with the premigrated data( all data not just data type from the plugin) and mongo data

# filter by repo type
        repos_to_consider = type_to_repo_ids[plugin_plan.type]

        # in case only certain repositories are specified in the migration plan
        if repos:
            repos_to_consider = set(repos).intersection(repos_to_consider)

        mongo_repo_q = mongo_Q(repo_id__in=repos_to_consider)

        mongo_repo_object_ids = set(
            str(i.id) for i in Repository.objects(mongo_repo_q).only('id'))

        premigrated_repos = Pulp2Repository.objects.filter(pulp2_repo_type=plugin_plan.type)
        premigrated_repo_object_ids = set(
            premigrated_repos.values_list('pulp2_object_id', flat=True))
        removed_repo_object_ids = premigrated_repo_object_ids - mongo_repo_object_ids

        Pulp2Repository.objects.filter(
            pulp2_object_id__in=removed_repo_object_ids).update(not_in_plan=True)

$ http GET :24817/pulp/api/v3/pulp2repositories/
HTTP/1.1 200 OK
Allow: GET, HEAD, OPTIONS
Connection: close
Content-Length: 1672
Content-Type: application/json
Date: Thu, 12 Nov 2020 10:39:56 GMT
Server: gunicorn/20.0.4
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "count": 2,
    "next": null,
    "previous": null,
    "results": [
        {
            "is_migrated": true,
            "not_in_plan": false,
            "pulp2_object_id": "5f906f7fc998ac1fff39a8b3",
            "pulp2_repo_id": "rpm_unsigned",
            "pulp2_repo_type": "rpm",
            "pulp3_distribution_hrefs": [
                "/pulp/api/v3/distributions/rpm/rpm/72c60e3a-ec44-4d9b-afbc-8bef576efb34/"
            ],
            "pulp3_publication_href": "/pulp/api/v3/publications/rpm/rpm/cf213db8-0b3b-4219-93fb-3e69b4e3bdc0/",
            "pulp3_remote_href": "/pulp/api/v3/remotes/rpm/rpm/09cf4b5a-4201-4eeb-ac8a-3e8be6826f76/",
            "pulp3_repository_href": "/pulp/api/v3/repositories/rpm/rpm/369c9d79-3a60-4be9-9b15-4296272180b5/",
            "pulp3_repository_version": "/pulp/api/v3/repositories/rpm/rpm/369c9d79-3a60-4be9-9b15-4296272180b5/versions/1/",
            "pulp_created": "2020-11-12T10:39:50.204433Z",
            "pulp_href": "/pulp/api/v3/pulp2repositories/870f3fcf-47e9-40e7-aa07-d2df0130fc5f/"
        },
        {
            "is_migrated": true,
            "not_in_plan": false,
            "pulp2_object_id": "5f8f16d9c998ac29ad889a50",
            "pulp2_repo_id": "file",
            "pulp2_repo_type": "iso",
            "pulp3_distribution_hrefs": [
                "/pulp/api/v3/distributions/file/file/3d45eac9-e231-4271-a506-75d56bd2b231/",
                "/pulp/api/v3/distributions/file/file/3e957d65-771d-4b1d-aadd-09710f7e796b/"
            ],
            "pulp3_publication_href": "/pulp/api/v3/publications/file/file/ec81dd40-b3a2-493d-acea-41953b986e10/",
            "pulp3_remote_href": "/pulp/api/v3/remotes/file/file/6efb13e2-10ec-4f60-9a76-a43e0c19378c/",
            "pulp3_repository_href": "/pulp/api/v3/repositories/file/file/78270194-4696-4c65-8e4f-b63cfdf5f121/",
            "pulp3_repository_version": "/pulp/api/v3/repositories/file/file/78270194-4696-4c65-8e4f-b63cfdf5f121/versions/1/",
            "pulp_created": "2020-11-12T10:25:49.296619Z",
            "pulp_href": "/pulp/api/v3/pulp2repositories/fd9e9119-5fb5-492c-8abc-9c19d9d5b25c/"
        }
    ]
}