Container Support

Tested pulp2 to pulp3 sync

$ pulp-admin docker repo create --repo-id pulp-docker --feed https://registry-1.docker.io/ --upstream-name pulp/test-fixture-1
Repository [pulp-docker] successfully created

$ pulp-admin docker repo sync run --repo-id pulp-docker
+----------------------------------------------------------------------+
                 Synchronizing Repository [pulp-docker]
+----------------------------------------------------------------------+

This command may be exited via ctrl+c without affecting the request.

Downloading manifests
[|]
... completed

Copying units already in pulp
[-]
... completed

Copying units already in pulp
[-]
... completed

Downloading remote files
[==================================================] 100%
10 of 10 items
... completed

Saving Manifests and Blobs
[-]
... completed

Saving Tags
[==================================================] 100%
18 of 18 items
... completed

Task Succeeded

Task Succeeded

$ http POST http://localhost:24817/pulp/api/v3/repositories/ name=foo_pulp
HTTP/1.1 201 Created
Allow: GET, POST, HEAD, OPTIONS
Connection: close
Content-Length: 273
Content-Type: application/json
Date: Tue, 13 Aug 2019 16:05:39 GMT
Location: /pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "_created": "2019-08-13T16:05:39.371511Z", 
    "_href": "/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/", 
    "_latest_version_href": null, 
    "_versions_href": "/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/", 
    "description": null, 
    "name": "foo_pulp"
}

$ http POST http://localhost:24817/pulp/api/v3/remotes/docker/docker/ name='pulp-docker' upstream_name='pulp-docker' url='http://localhost:5000/' policy='on_demand'
HTTP/1.1 201 Created
Allow: GET, POST, HEAD, OPTIONS
Connection: close
Content-Length: 466
Content-Type: application/json
Date: Tue, 13 Aug 2019 16:07:04 GMT
Location: /pulp/api/v3/remotes/docker/docker/2003a9ab-d48a-4da5-b048-6b2905f86b87/
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "_created": "2019-08-13T16:07:04.604251Z", 
    "_href": "/pulp/api/v3/remotes/docker/docker/2003a9ab-d48a-4da5-b048-6b2905f86b87/", 
    "_last_updated": "2019-08-13T16:07:04.604267Z", 
    "_type": "docker.docker", 
    "download_concurrency": 20, 
    "name": "pulp-docker", 
    "policy": "on_demand", 
    "proxy_url": null, 
    "ssl_ca_certificate": null, 
    "ssl_client_certificate": null, 
    "ssl_client_key": null, 
    "ssl_validation": true, 
    "upstream_name": "pulp-docker", 
    "url": "http://localhost:5000/v2/", 
    "whitelist_tags": null
}

(pulp) [vagrant@pulp2-nightly-pulp3-source-centos7 pulpcore]$ http POST ':24817/pulp/api/v3/remotes/docker/docker/2003a9ab-d48a-4da5-b048-6b2905f86b87/sync/' repository=/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/
HTTP/1.1 202 Accepted
Allow: POST, OPTIONS
Connection: close
Content-Length: 67
Content-Type: application/json
Date: Tue, 13 Aug 2019 16:07:36 GMT
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "task": "/pulp/api/v3/tasks/15d93c58-1633-4279-bcf4-c50a4947fc39/"
}

(pulp) [vagrant@pulp2-nightly-pulp3-source-centos7 pulpcore]$ http GET http://localhost:24817/pulp/api/v3/tasks/15d93c58-1633-4279-bcf4-c50a4947fc39/
HTTP/1.1 200 OK
Allow: GET, PATCH, DELETE, HEAD, OPTIONS
Connection: close
Content-Length: 878
Content-Type: application/json
Date: Tue, 13 Aug 2019 16:07:52 GMT
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "_created": "2019-08-13T16:07:36.507399Z", 
    "_href": "/pulp/api/v3/tasks/15d93c58-1633-4279-bcf4-c50a4947fc39/", 
    "created_resources": [
        "/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/"
    ], 
    "error": null, 
    "finished_at": "2019-08-13T16:07:43.086543Z", 
    "name": "pulp_docker.app.tasks.synchronize.synchronize", 
    "non_fatal_errors": [], 
    "parent": null, 
    "progress_reports": [
        {
            "done": 1, 
            "message": "Downloading tag list", 
            "state": "completed", 
            "suffix": null, 
            "total": 1
        }, 
        {
            "done": 9, 
            "message": "Processing Tags", 
            "state": "completed", 
            "suffix": null, 
            "total": 9
        }, 
        {
            "done": 8, 
            "message": "Downloading Artifacts", 
            "state": "completed", 
            "suffix": null, 
            "total": 8
        }, 
        {
            "done": 35, 
            "message": "Associating Content", 
            "state": "completed", 
            "suffix": null, 
            "total": 35
        }
    ], 
    "spawned_tasks": [], 
    "started_at": "2019-08-13T16:07:36.606703Z", 
    "state": "completed", 
    "worker": "/pulp/api/v3/workers/2ccf02d6-419b-4d91-adf2-ac02d8b01650/"
}

(pulp) [vagrant@pulp2-nightly-pulp3-source-centos7 pulpcore]$ http GET http://localhost:24817/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/
HTTP/1.1 200 OK
Allow: GET, DELETE, HEAD, OPTIONS
Connection: close
Content-Length: 1229
Content-Type: application/json
Date: Tue, 13 Aug 2019 16:08:09 GMT
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "_created": "2019-08-13T16:07:36.630378Z", 
    "_href": "/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/", 
    "base_version": null, 
    "content_summary": {
        "added": {
            "docker.blob": {
                "count": 10, 
                "href": "/pulp/api/v3/content/docker/blobs/?repository_version_added=/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/"
            }, 
            "docker.manifest": {
                "count": 9, 
                "href": "/pulp/api/v3/content/docker/manifests/?repository_version_added=/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/"
            }, 
            "docker.tag": {
                "count": 9, 
                "href": "/pulp/api/v3/content/docker/tags/?repository_version_added=/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/"
            }
        }, 
        "present": {
            "docker.blob": {
                "count": 10, 
                "href": "/pulp/api/v3/content/docker/blobs/?repository_version=/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/"
            }, 
            "docker.manifest": {
                "count": 9, 
                "href": "/pulp/api/v3/content/docker/manifests/?repository_version=/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/"
            }, 
            "docker.tag": {
                "count": 9, 
                "href": "/pulp/api/v3/content/docker/tags/?repository_version=/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/versions/1/"
            }
        }, 
        "removed": {}
    }, 
    "number": 1
}

$ http POST http://localhost:24817/pulp/api/v3/distributions/docker/docker/ name='bazx' base_path='foox' repository=/pulp/api/v3/repositories/9490efab-5953-4673-9c67-0aaa15504f35/
HTTP/1.1 202 Accepted
Allow: GET, POST, HEAD, OPTIONS
Connection: close
Content-Length: 67
Content-Type: application/json
Date: Tue, 13 Aug 2019 16:19:21 GMT
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "task": "/pulp/api/v3/tasks/35b22593-e7f3-4c82-896e-4a3fd18c39d6/"
}

$ sudo docker  pull localhost:24816/foox:manifest_a
Trying to pull repository localhost:24816/foox ... 
manifest_a: Pulling from localhost:24816/foox
686209d53cbd: Pull complete 
Digest: sha256:21e3caae28758329318c8a868a80daa37ad8851705155fc28767852c73d36af5
Status: Downloaded newer image for localhost:24816/foox:manifest_a

(pulp) [vagrant@pulp2-nightly-pulp3-source-centos7 pulpcore]$ pulp-admin repo list
+----------------------------------------------------------------------+
                              Repositories
+----------------------------------------------------------------------+

Id:                  pulp-docker
Display Name:        None
Description:         None
Content Unit Counts: 
  Docker Blob:          10
  Docker Manifest:      14
  Docker Manifest List: 4
  Docker Tag:           18

(pulp) [vagrant@pulp2-nightly-pulp3-source-centos7 pulpcore]$ mongo
MongoDB shell version v3.4.22
connecting to: mongodb://127.0.0.1:27017
MongoDB server version: 3.4.22
Server has startup warnings: 
2019-08-13T10:01:18.612+0000 I CONTROL  [initandlisten] 
2019-08-13T10:01:18.612+0000 I CONTROL  [initandlisten] ** WARNING: Access control is not enabled for the database.
2019-08-13T10:01:18.612+0000 I CONTROL  [initandlisten] **          Read and write access to data and configuration is unrestricted.
2019-08-13T10:01:18.612+0000 I CONTROL  [initandlisten] 
2019-08-13T10:01:18.613+0000 I CONTROL  [initandlisten] 
2019-08-13T10:01:18.613+0000 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/enabled is 'always'.
2019-08-13T10:01:18.613+0000 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2019-08-13T10:01:18.613+0000 I CONTROL  [initandlisten] 
2019-08-13T10:01:18.613+0000 I CONTROL  [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/defrag is 'always'.
2019-08-13T10:01:18.613+0000 I CONTROL  [initandlisten] **        We suggest setting it to 'never'
2019-08-13T10:01:18.613+0000 I CONTROL  [initandlisten] 
> use pulp_database
switched to db pulp_database
> db.units_docker_tag.count({'schema_version': 2})
9

Sync from pulp3 with pulp3

$ http POST http://localhost:24817/pulp/api/v3/remotes/docker/docker/ name='foox1' upstream_name='foox' url='http://192.168.121.142:24816/' policy='on_demand'
HTTP/1.1 201 Created
Allow: GET, POST, HEAD, OPTIONS
Connection: close
Content-Length: 457
Content-Type: application/json
Date: Thu, 15 Aug 2019 10:17:10 GMT
Location: /pulp/api/v3/remotes/docker/docker/9342f5eb-4f5a-4b5c-8e48-7fe00c327ee0/
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "_created": "2019-08-15T10:17:10.518577Z",
    "_href": "/pulp/api/v3/remotes/docker/docker/9342f5eb-4f5a-4b5c-8e48-7fe00c327ee0/",
    "_last_updated": "2019-08-15T10:17:10.518592Z",
    "_type": "docker.docker",
    "download_concurrency": 20,
    "name": "foox1",
    "policy": "on_demand",
    "proxy_url": null,
    "ssl_ca_certificate": null,
    "ssl_client_certificate": null,
    "ssl_client_key": null,
    "ssl_validation": true,
    "upstream_name": "foox",
    "url": "http://192.168.121.142:24816/",
    "whitelist_tags": null

$ http POST http://localhost:24817/pulp/api/v3/remotes/docker/docker/9342f5eb-4f5a-4b5c-8e48-7fe00c327ee0/sync/ repository=/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/
HTTP/1.1 202 Accepted
Allow: POST, OPTIONS
Connection: close
Content-Length: 67
Content-Type: application/json
Date: Thu, 15 Aug 2019 12:43:25 GMT
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "task": "/pulp/api/v3/tasks/a9fc8a4b-16d0-4c88-af2c-768c5ac9d6ad/"
}

(pulp) [vagrant@pulp3-source-fedora29 pulp_docker]$ http GET http://localhost:24817/pulp/api/v3/tasks/a9fc8a4b-16d0-4c88-af2c-768c5ac9d6ad/
HTTP/1.1 200 OK
Allow: GET, PATCH, DELETE, HEAD, OPTIONS
Connection: close
Content-Length: 1049
Content-Type: application/json
Date: Thu, 15 Aug 2019 12:43:36 GMT
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "_created": "2019-08-15T12:43:25.609563Z",
    "_href": "/pulp/api/v3/tasks/a9fc8a4b-16d0-4c88-af2c-768c5ac9d6ad/",
    "created_resources": [
        "/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/"
    ],
    "error": null,
    "finished_at": "2019-08-15T12:43:26.791018Z",
    "name": "pulp_docker.app.tasks.synchronize.synchronize",
    "non_fatal_errors": [],
    "parent": null,
    "progress_reports": [
        {
            "done": 1,
            "message": "Downloading tag list",
            "state": "completed",
            "suffix": null,
            "total": 1
        },
        {
            "done": 9,
            "message": "Processing Tags",
            "state": "completed",
            "suffix": null,
            "total": 9
        },
        {
            "done": 8,
            "message": "Downloading Artifacts",
            "state": "completed",
            "suffix": null,
            "total": 8
        },
        {
            "done": 35,
            "message": "Associating Content",
            "state": "completed",
            "suffix": null,
            "total": 35
        }
    ],
    "reserved_resources_record": [
        "/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/",
        "/pulp/api/v3/remotes/docker/docker/9342f5eb-4f5a-4b5c-8e48-7fe00c327ee0/"
    ],
    "spawned_tasks": [],
    "started_at": "2019-08-15T12:43:25.694853Z",
    "state": "completed",
    "worker": "/pulp/api/v3/workers/94b0db8f-310d-4a71-b2dd-9bdecb748dbc/"
}

(pulp) [vagrant@pulp3-source-fedora29 pulp_docker]$ http GET http://localhost:24817/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/
HTTP/1.1 200 OK
Allow: GET, DELETE, HEAD, OPTIONS
Connection: close
Content-Length: 1229
Content-Type: application/json
Date: Thu, 15 Aug 2019 12:43:45 GMT
Server: gunicorn/19.9.0
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN

{
    "_created": "2019-08-15T12:43:25.715464Z",
    "_href": "/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/",
    "base_version": null,
    "content_summary": {
        "added": {
            "docker.blob": {
                "count": 10,
                "href": "/pulp/api/v3/content/docker/blobs/?repository_version_added=/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/"
            },
            "docker.manifest": {
                "count": 9,
                "href": "/pulp/api/v3/content/docker/manifests/?repository_version_added=/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/"
            },
            "docker.tag": {
                "count": 9,
                "href": "/pulp/api/v3/content/docker/tags/?repository_version_added=/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/"
            }
        },
        "present": {
            "docker.blob": {
                "count": 10,
                "href": "/pulp/api/v3/content/docker/blobs/?repository_version=/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/"
            },
            "docker.manifest": {
                "count": 9,
                "href": "/pulp/api/v3/content/docker/manifests/?repository_version=/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/"
            },
            "docker.tag": {
                "count": 9,
                "href": "/pulp/api/v3/content/docker/tags/?repository_version=/pulp/api/v3/repositories/c3685cbd-9bf1-4080-97cf-56376c9347df/versions/1/"
            }
        },
        "removed": {}
    },
    "number": 1
}

docker pull from pulp3 ( different machine)

add  192.168.121.142:24816 to insecure registry

$ sudo docker  pull 192.168.121.142:24816/foox:manifest_a
Trying to pull repository 192.168.121.142:24816/foox ... 
sha256:21e3caae28758329318c8a868a80daa37ad8851705155fc28767852c73d36af5: Pulling from 192.168.121.142:24816/foox
Digest: sha256:21e3caae28758329318c8a868a80daa37ad8851705155fc28767852c73d36af5
Status: Image is up to date for 192.168.121.142:24816/foox:manifest_a
(pulp) [vagrant@pulp3-source-fedora29 pulp_docker]$ http http://192.168.121.142:24816/v2/foox/tags/list
HTTP/1.1 200 OK
Content-Length: 132
Content-Type: application/json; charset=utf-8
Date: Thu, 15 Aug 2019 12:58:40 GMT
Server: Python/3.6 aiohttp/3.5.4

{
    "name": "foox",
    "tags": [
        "manifest_e",
        "ml_iv",
        "manifest_d",
        "ml_iii",
        "manifest_c",
        "manifest_a",
        "ml_i",
        "ml_ii",
        "manifest_b"
    ]
}

(pulp) [vagrant@pulp3-source-fedora29 pulp_docker]$ http http://192.168.121.142:24816/v2/foox/manifests/manifest_a Accept:application/vnd.docker.distribution.manifest.v2+json
HTTP/1.1 200 OK
Accept-Ranges: bytes
Content-Disposition: attachment; filename=e3caae28758329318c8a868a80daa37ad8851705155fc28767852c73d36af5
Content-Length: 524
Content-Type: application/vnd.docker.distribution.manifest.v2+json
Date: Thu, 15 Aug 2019 12:58:55 GMT
Docker-Content-Digest: sha256:21e3caae28758329318c8a868a80daa37ad8851705155fc28767852c73d36af5
Docker-Distribution-API-Version: registry/2.0
Last-Modified: Thu, 15 Aug 2019 10:58:20 GMT
Server: Python/3.6 aiohttp/3.5.4

{
    "config": {
        "digest": "sha256:d21d863f69b5de1a973a41344488f2ec89a625f2624195f51b4e2d54a97fc53b",
        "mediaType": "application/vnd.docker.container.image.v1+json",
        "size": 1178
    },
    "layers": [
        {
            "digest": "sha256:686209d53cbd832d0c9a5f77ae8acf87c58f7880581cf132f4022857d23e9182",
            "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
            "size": 130
        }
    ],
    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
    "schemaVersion": 2
}