Pulp: Issueshttps://pulp.plan.io/https://pulp.plan.io/favicon.ico2021-08-25T20:39:56ZPulp
Planio Migration Plugin - Issue #9295 (CLOSED - CURRENTRELEASE): After migration interruption, incomplet...https://pulp.plan.io/issues/92952021-08-25T20:39:56Zttereshcttereshc@redhat.com
<p>To reproduce:</p>
<ul>
<li>interrupt migration at the moment when a publication is created but not completed (good luck)</li>
<li>re-run migration and observe that pulp2repositories endpoint refers to incomplete publications. If you try to access them via content app, you'll get 404.</li>
</ul> Migration Plugin - Issue #9283 (CLOSED - CURRENTRELEASE): Syncing is running out of DB connectionshttps://pulp.plan.io/issues/92832021-08-23T21:08:54Zfao89
<pre><code>pulp [bd3e6e1fe5bb4586b75163ddd7ad1521]: pulpcore.tasking.pulpcore_worker:INFO: File "/usr/local/lib/python3.8/site-packages/pulpcore/tasking/pulpcore_worker.py", line 274, in _perform_task
result = func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 130, in sync
repo_version = d_version.create()
File "/usr/local/lib/python3.8/site-packages/pulpcore/plugin/stages/declarative_version.py", line 151, in create
loop.run_until_complete(pipeline)
File "/usr/lib64/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/usr/local/lib/python3.8/site-packages/pulpcore/plugin/stages/api.py", line 225, in create_pipeline
await asyncio.gather(*futures)
File "/usr/local/lib/python3.8/site-packages/pulpcore/plugin/stages/api.py", line 43, in __call__
await self.run()
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 778, in run
await asyncio.gather(*tasks)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 592, in _fetch_collection_metadata
await self._fetch_paginated_collection_metadata(
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 548, in _fetch_paginated_collection_metadata
await asyncio.gather(*tasks)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 441, in _fetch_collection_version_metadata
await self._add_collection_version(api_version, collection_version_url, metadata)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 469, in _add_collection_version
await asyncio.gather(*tasks)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 592, in _fetch_collection_metadata
await self._fetch_paginated_collection_metadata(
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 548, in _fetch_paginated_collection_metadata
await asyncio.gather(*tasks)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 441, in _fetch_collection_version_metadata
await self._add_collection_version(api_version, collection_version_url, metadata)
File "/usr/local/lib/python3.8/site-packages/pulp_ansible/app/tasks/collections.py", line 496, in _add_collection_version
self.parsing_metadata_progress_bar.increment()
File "/usr/local/lib/python3.8/site-packages/pulpcore/app/models/progress.py", line 185, in increment
self.increase_by(1)
File "/usr/local/lib/python3.8/site-packages/pulpcore/app/models/progress.py", line 198, in increase_by
self.save()
File "/usr/local/lib/python3.8/site-packages/pulpcore/app/models/progress.py", line 142, in save
super().save(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django_lifecycle/mixins.py", line 134, in save
save(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 726, in save
self.save_base(using=using, force_insert=force_insert,
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 763, in save_base
updated = self._save_table(
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 845, in _save_table
updated = self._do_update(base_qs, using, pk_val, values, update_fields,
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 899, in _do_update
return filtered._update(values) > 0
File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 802, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1559, in execute_sql
cursor = super().execute_sql(result_type)
File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1173, in execute_sql
cursor = self.connection.cursor()
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 259, in cursor
return self._cursor()
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 235, in _cursor
self.ensure_connection()
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 219, in ensure_connection
self.connect()
File "/usr/local/lib/python3.8/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 219, in ensure_connection
self.connect()
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 200, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/postgresql/base.py", line 187, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib64/python3.8/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
</code></pre>
<pre><code>Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 219, in ensure_connection
self.connect()
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 200, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/postgresql/base.py", line 187, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib64/python3.8/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: FATAL: remaining connection slots are reserved for non-replication superuser connections
</code></pre> Migration Plugin - Issue #9254 (CLOSED - CURRENTRELEASE): Pulp 2to3 Migration of SLES repositorie...https://pulp.plan.io/issues/92542021-08-19T06:30:52Zlaugmanuel@gmail.com
<p>Hello,</p>
<p>I've got a Foreman/Katello installation and recently migrated all the repos to Pulp3.
During that, the SLES repos got successfully migrated, but the sles_auth_token was not set.</p>
<p>In Pulp2, the auth token could be set by specifying it as a query parameter as part of the url.
This seems to cause the Pulp 2to3 Migration plugin to treat the auth_token as part of the URL and just keep the token as is.</p>
<p><strong>Example:</strong></p>
<p>Given we have a Pulp2 remote with the following URL:</p>
<pre><code class="text syntaxhl" data-language="text">https://updates.suse.com/SUSE/Products/SLE-Module-Legacy/12/x86_64/product/?abc-i-removed-the-auth-token-manually
</code></pre>
<p>Then the resulting Pulp 3 remote then looks like this:</p>
<pre><code class="json syntaxhl" data-language="json"><span class="p">{</span><span class="w">
</span><span class="nl">"pulp_href"</span><span class="p">:</span><span class="w"> </span><span class="s2">"/pulp/api/v3/remotes/rpm/rpm/e11dc866-b38c-43a1-85ce-3c378bf12198/"</span><span class="p">,</span><span class="w">
</span><span class="nl">"pulp_created"</span><span class="p">:</span><span class="w"> </span><span class="s2">"2021-08-14T13:33:37.551888Z"</span><span class="p">,</span><span class="w">
</span><span class="nl">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"5ea944d18a0c8441a6fcaac4-db9d7b99-f62d-499e-b2ff-297db52a32ad"</span><span class="p">,</span><span class="w">
</span><span class="nl">"url"</span><span class="p">:</span><span class="w"> </span><span class="s2">"https://updates.suse.com/SUSE/Products/SLE-Module-Legacy/12/x86_64/product/?abc-i-removed-the-auth-token-manually"</span><span class="p">,</span><span class="w">
</span><span class="nl">"ca_cert"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="p">,</span><span class="w">
</span><span class="nl">"client_cert"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="p">,</span><span class="w">
</span><span class="nl">"client_key"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="p">,</span><span class="w">
</span><span class="nl">"tls_validation"</span><span class="p">:</span><span class="w"> </span><span class="kc">true</span><span class="p">,</span><span class="w">
</span><span class="nl">"proxy_url"</span><span class="p">:</span><span class="w"> </span><span class="s2">"http://proxy.example.com:8000"</span><span class="p">,</span><span class="w">
</span><span class="nl">"username"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="p">,</span><span class="w">
</span><span class="nl">"password"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="p">,</span><span class="w">
</span><span class="nl">"pulp_last_updated"</span><span class="p">:</span><span class="w"> </span><span class="s2">"2021-08-14T13:33:37.551923Z"</span><span class="p">,</span><span class="w">
</span><span class="nl">"download_concurrency"</span><span class="p">:</span><span class="w"> </span><span class="mi">20</span><span class="p">,</span><span class="w">
</span><span class="nl">"policy"</span><span class="p">:</span><span class="w"> </span><span class="s2">"immediate"</span><span class="p">,</span><span class="w">
</span><span class="nl">"sles_auth_token"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span><span class="w">
</span><span class="p">}</span><span class="w">
</span></code></pre>
<p>You might find more information here (Foreman plugin used to manage SLES repositories): <a href="https://github.com/ATIX-AG/foreman_scc_manager/issues/43" class="external">https://github.com/ATIX-AG/foreman_scc_manager/issues/43</a></p>
<p>This seems to be related to: <a href="https://pulp.plan.io/issues/6927" class="external">https://pulp.plan.io/issues/6927</a></p> Migration Plugin - Issue #9242 (CLOSED - CURRENTRELEASE): Filter out incomplete publications from...https://pulp.plan.io/issues/92422021-08-17T12:08:42Zipanova@redhat.comipanova@redhat.com
<p>If the only publication a repo version has is incomplete, it would be shown in a mapping as migrated.</p>
<p>The problematic code:
<a href="https://github.com/pulp/pulp-2to3-migration/blob/master/pulp_2to3_migration/app/serializers.py#L236" class="external">https://github.com/pulp/pulp-2to3-migration/blob/master/pulp_2to3_migration/app/serializers.py#L236</a></p> Migration Plugin - Issue #8895 (CLOSED - CURRENTRELEASE): django.core.exceptions.FieldError: Unsu...https://pulp.plan.io/issues/88952021-06-14T18:28:21Zttereshcttereshc@redhat.com
<p><del>Trying to run a migration with the future pulpcore 3.14 fails in the following way</del>
pulpcore 3.14 will deprecate it and 3.15 will remove it.</p>
<p>This needs to be fixed for 3.15.</p>
<pre><code>pulp [d42e1bccd6444d1e9992fd63aadf7625]: django.request:ERROR: Internal Server Error: /pulp/api/v3/migration-plans/5172b292-c16d-45d0-80cb-4577328dbd7f/run/
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/usr/local/lib/python3.6/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/usr/local/lib/python3.6/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/usr/local/lib/python3.6/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/rest_framework/viewsets.py", line 125, in view
return self.dispatch(request, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 509, in dispatch
response = self.handle_exception(exc)
File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 469, in handle_exception
self.raise_uncaught_exception(exc)
File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 480, in raise_uncaught_exception
raise exc
File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 506, in dispatch
response = handler(request, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/pulp_2to3_migration/app/viewsets.py", line 94, in run
if is_migration_plan_running():
File "/usr/local/lib/python3.6/site-packages/pulp_2to3_migration/app/viewsets.py", line 47, in is_migration_plan_running
reserved_resources_record__resource='pulp_2to3_migration')
File "/usr/local/lib/python3.6/site-packages/django/db/models/manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 892, in filter
return self._filter_or_exclude(False, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 910, in _filter_or_exclude
clone.query.add_q(Q(*args, **kwargs))
File "/usr/local/lib/python3.6/site-packages/django/db/models/sql/query.py", line 1290, in add_q
clause, _ = self._add_q(q_object, self.used_aliases)
File "/usr/local/lib/python3.6/site-packages/django/db/models/sql/query.py", line 1318, in _add_q
split_subq=split_subq, simple_col=simple_col,
File "/usr/local/lib/python3.6/site-packages/django/db/models/sql/query.py", line 1251, in build_filter
condition = self.build_lookup(lookups, col, value)
File "/usr/local/lib/python3.6/site-packages/django/db/models/sql/query.py", line 1110, in build_lookup
lhs = self.try_transform(lhs, lookup_name)
File "/usr/local/lib/python3.6/site-packages/django/db/models/sql/query.py", line 1151, in try_transform
"permitted%s" % (name, output_field.__name__, suggestion)
django.core.exceptions.FieldError: Unsupported lookup 'resource' for ArrayField or join on the field not permitted.
</code></pre>