RPM Support

Hello,

I'm having following issues while trying to synchronize repository:

# /usr/local/bin/pulp rpm repository create --name=centos_8_3_base
{
  "pulp_href": "/pulp/api/v3/repositories/rpm/rpm/de4b4a2d-4c3e-4f40-b225-ef6e9cdbea93/",
  "pulp_created": "2021-03-22T10:37:29.513788Z",
  "versions_href": "/pulp/api/v3/repositories/rpm/rpm/de4b4a2d-4c3e-4f40-b225-ef6e9cdbea93/versions/",
  "latest_version_href": "/pulp/api/v3/repositories/rpm/rpm/de4b4a2d-4c3e-4f40-b225-ef6e9cdbea93/versions/0/",
  "name": "centos_8_3_base",
  "description": null,
  "remote": null,
  "metadata_signing_service": null,
  "retain_package_versions": 0
}

# /usr/local/bin/pulp rpm remote create --name=centos_8_3_base --url=http://mirror.centos.org/centos/8.3.2011/BaseOS/x86_64/os/
{
  "pulp_href": "/pulp/api/v3/remotes/rpm/rpm/136aef71-a90b-4e04-a593-1d1aaeb5db89/",
  "pulp_created": "2021-03-22T10:38:01.007355Z",
  "name": "centos_8_3_base",
  "url": "http://mirror.centos.org/centos/8.3.2011/BaseOS/x86_64/os/",
  "ca_cert": null,
  "client_cert": null,
  "client_key": null,
  "tls_validation": true,
  "proxy_url": null,
  "username": null,
  "password": null,
  "pulp_last_updated": "2021-03-22T10:38:01.007391Z",
  "download_concurrency": 10,
  "policy": "immediate",
  "total_timeout": null,
  "connect_timeout": null,
  "sock_connect_timeout": null,
  "sock_read_timeout": null,
  "sles_auth_token": null
}

# /usr/local/bin/pulp rpm repository sync --name=centos_8_3_base --remote=centos_8_3_base
Started background task /pulp/api/v3/tasks/60f50a37-44b1-47b4-852f-2ed8cba569a6/
.......................................................................Error: Task /pulp/api/v3/tasks/60f50a37-44b1-47b4-852f-2ed8cba569a6/ failed: 'malformed record literal: ""
LINE 1: ....3.8', '6.module_el8.3.0+604+ab7bf9cc', 'x86_64', '', 'e25f6...
                                                             ^
DETAIL:  Missing left parenthesis.
'

Following errors in logs:

Mar 22 10:40:06 hostname gunicorn[23229]: pulp [e32eba41066f4576a18540dd19cc772a]: 127.0.0.1 - admin [22/Mar/2021:10:40:06 +0000] "GET /pulp/api/v3/tasks/60f50a37-44b1-47b4-852f-2ed8cba569a6/ HTTP/1.0" 200 1478 "-" "python-requests/2.25.1"
Mar 22 10:40:07 hostname rq[23370]: pulp [82b62bafeb7f4cce9f1a1eb06c9b4905]: rq.worker:ERROR: Traceback (most recent call last):
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 84, in _execute
Mar 22 10:40:07 hostname rq[23370]:    return self.cursor.execute(sql, params)
Mar 22 10:40:07 hostname rq[23370]: psycopg2.errors.InvalidTextRepresentation: malformed record literal: ""
Mar 22 10:40:07 hostname rq[23370]: LINE 1: ....3.8', '6.module_el8.3.0+604+ab7bf9cc', 'x86_64', '', 'e25f6...
Mar 22 10:40:07 hostname rq[23370]:                                                             ^
Mar 22 10:40:07 hostname rq[23370]: DETAIL:  Missing left parenthesis.
Mar 22 10:40:07 hostname rq[23370]: The above exception was the direct cause of the following exception:
Mar 22 10:40:07 hostname rq[23370]: Traceback (most recent call last):
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/rq/worker.py", line 975, in perform_job
Mar 22 10:40:07 hostname rq[23370]:    rv = job.perform()
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/rq/job.py", line 696, in perform
Mar 22 10:40:07 hostname rq[23370]:    self._result = self._execute()
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/rq/job.py", line 719, in _execute
Mar 22 10:40:07 hostname rq[23370]:    return self.func(*self.args, **self.kwargs)
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulp_rpm/app/tasks/synchronizing.py", line 257, in synchronize
Mar 22 10:40:07 hostname rq[23370]:    dv.create()
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/declarative_version.py", line 148, in create
Mar 22 10:40:07 hostname rq[23370]:    loop.run_until_complete(pipeline)
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib64/python3.6/asyncio/base_events.py", line 484, in run_until_complete
Mar 22 10:40:07 hostname rq[23370]:    return future.result()
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py", line 225, in create_pipeline
Mar 22 10:40:07 hostname rq[23370]:    await asyncio.gather(*futures)
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py", line 43, in __call__
Mar 22 10:40:07 hostname rq[23370]:    await self.run()
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/content_stages.py", line 95, in run
Mar 22 10:40:07 hostname rq[23370]:    d_content.content.save()
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/app/models/base.py", line 115, in save
Mar 22 10:40:07 hostname rq[23370]:    return super().save(*args, **kwargs)
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django_lifecycle/mixins.py", line 129, in save
Mar 22 10:40:07 hostname rq[23370]:    save(*args, **kwargs)
Mar 22 10:40:07 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 744, in save
Mar 22 10:40:08 hostname rq[23370]:    force_update=force_update, update_fields=update_fields)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 782, in save_base
Mar 22 10:40:08 hostname rq[23370]:    force_update, using, update_fields,
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 873, in _save_table
Mar 22 10:40:08 hostname rq[23370]:    result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 911, in _do_insert
Mar 22 10:40:08 hostname rq[23370]:    using=using, raw=raw)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/manager.py", line 82, in manager_method
Mar 22 10:40:08 hostname rq[23370]:    return getattr(self.get_queryset(), name)(*args, **kwargs)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/query.py", line 1186, in _insert
Mar 22 10:40:08 hostname rq[23370]:    return query.get_compiler(using=using).execute_sql(return_id)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1377, in execute_sql
Mar 22 10:40:08 hostname rq[23370]:    cursor.execute(sql, params)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 67, in execute
Mar 22 10:40:08 hostname rq[23370]:    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
Mar 22 10:40:08 hostname rq[23370]:    return executor(sql, params, many, context)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 84, in _execute
Mar 22 10:40:08 hostname rq[23370]:    return self.cursor.execute(sql, params)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/utils.py", line 89, in __exit__
Mar 22 10:40:08 hostname rq[23370]:    raise dj_exc_value.with_traceback(traceback) from exc_value
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 84, in _execute
Mar 22 10:40:08 hostname rq[23370]:    return self.cursor.execute(sql, params)
Mar 22 10:40:08 hostname rq[23370]: django.db.utils.DataError: malformed record literal: ""
Mar 22 10:40:08 hostname rq[23370]: LINE 1: ....3.8', '6.module_el8.3.0+604+ab7bf9cc', 'x86_64', '', 'e25f6...
Mar 22 10:40:08 hostname rq[23370]:                                                             ^
Mar 22 10:40:08 hostname rq[23370]: DETAIL:  Missing left parenthesis.
Mar 22 10:40:08 hostname rq[23370]: Traceback (most recent call last):
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 84, in _execute
Mar 22 10:40:08 hostname rq[23370]:    return self.cursor.execute(sql, params)
Mar 22 10:40:08 hostname rq[23370]: psycopg2.errors.InvalidTextRepresentation: malformed record literal: ""
Mar 22 10:40:08 hostname rq[23370]: LINE 1: ....3.8', '6.module_el8.3.0+604+ab7bf9cc', 'x86_64', '', 'e25f6...
Mar 22 10:40:08 hostname rq[23370]:                                                             ^
Mar 22 10:40:08 hostname rq[23370]: DETAIL:  Missing left parenthesis.
Mar 22 10:40:08 hostname rq[23370]: The above exception was the direct cause of the following exception:
Mar 22 10:40:08 hostname rq[23370]: Traceback (most recent call last):
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/rq/worker.py", line 975, in perform_job
Mar 22 10:40:08 hostname rq[23370]:    rv = job.perform()
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/rq/job.py", line 696, in perform
Mar 22 10:40:08 hostname rq[23370]:    self._result = self._execute()
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/rq/job.py", line 719, in _execute
Mar 22 10:40:08 hostname rq[23370]:    return self.func(*self.args, **self.kwargs)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulp_rpm/app/tasks/synchronizing.py", line 257, in synchronize
Mar 22 10:40:08 hostname rq[23370]:    dv.create()
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/declarative_version.py", line 148, in create
Mar 22 10:40:08 hostname rq[23370]:    loop.run_until_complete(pipeline)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib64/python3.6/asyncio/base_events.py", line 484, in run_until_complete
Mar 22 10:40:08 hostname rq[23370]:    return future.result()
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py", line 225, in create_pipeline
Mar 22 10:40:08 hostname rq[23370]:    await asyncio.gather(*futures)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/api.py", line 43, in __call__
Mar 22 10:40:08 hostname rq[23370]:    await self.run()
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/plugin/stages/content_stages.py", line 95, in run
Mar 22 10:40:08 hostname rq[23370]:    d_content.content.save()
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/pulpcore/app/models/base.py", line 115, in save
Mar 22 10:40:08 hostname rq[23370]:    return super().save(*args, **kwargs)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django_lifecycle/mixins.py", line 129, in save
Mar 22 10:40:08 hostname rq[23370]:    save(*args, **kwargs)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 744, in save
Mar 22 10:40:08 hostname rq[23370]:    force_update=force_update, update_fields=update_fields)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 782, in save_base
Mar 22 10:40:08 hostname rq[23370]:    force_update, using, update_fields,
Mar 22 10:40:08 hostname gunicorn[23229]: pulp [6e4108df743c4f6dafcd1411234e9250]: 127.0.0.1 - admin [22/Mar/2021:10:40:08 +0000] "GET /pulp/api/v3/tasks/60f50a37-44b1-47b4-852f-2ed8cba569a6/ HTTP/1.0" 200 4947 "-" "python-requests/2.25.1"
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 873, in _save_table
Mar 22 10:40:08 hostname rq[23370]:    result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 911, in _do_insert
Mar 22 10:40:08 hostname rq[23370]:    using=using, raw=raw)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/manager.py", line 82, in manager_method
Mar 22 10:40:08 hostname rq[23370]:    return getattr(self.get_queryset(), name)(*args, **kwargs)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/query.py", line 1186, in _insert
Mar 22 10:40:08 hostname rq[23370]:    return query.get_compiler(using=using).execute_sql(return_id)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1377, in execute_sql
Mar 22 10:40:08 hostname rq[23370]:    cursor.execute(sql, params)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 67, in execute
Mar 22 10:40:08 hostname rq[23370]:    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
Mar 22 10:40:08 hostname rq[23370]:    return executor(sql, params, many, context)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 84, in _execute
Mar 22 10:40:08 hostname rq[23370]:    return self.cursor.execute(sql, params)
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/utils.py", line 89, in __exit__
Mar 22 10:40:08 hostname rq[23370]:    raise dj_exc_value.with_traceback(traceback) from exc_value
Mar 22 10:40:08 hostname rq[23370]:  File "/usr/lib/python3.6/site-packages/django/db/backends/utils.py", line 84, in _execute
Mar 22 10:40:08 hostname rq[23370]:    return self.cursor.execute(sql, params)
Mar 22 10:40:08 hostname rq[23370]: django.db.utils.DataError: malformed record literal: ""
Mar 22 10:40:08 hostname rq[23370]: LINE 1: ....3.8', '6.module_el8.3.0+604+ab7bf9cc', 'x86_64', '', 'e25f6...
Mar 22 10:40:08 hostname rq[23370]:                                                             ^
Mar 22 10:40:08 hostname rq[23370]: DETAIL:  Missing left parenthesis.
Mar 22 10:40:08 hostname rq[23370]: pulp [None]: rq.worker:INFO: 23370@hostname.sns.sec: 5a36c8da-85e4-41b3-9636-59d281645352
Mar 22 10:40:10 hostname rq[23370]: pulp [None]: rq.worker:INFO: 23370@hostname.sns.sec: Job OK (5a36c8da-85e4-41b3-9636-59d281645352

It is VM running Oracle Linux 8.3, also tried it with version 3.11.1 with the same issue:

# /usr/local/bin/pulp status
{
  "versions": [
    {
      "component": "pulpcore",
      "version": "3.9.1"
    },
    {
      "component": "pulp_rpm",
      "version": "3.9.0"
    }
  ],
  "online_workers": [
    {
      "pulp_created": "2021-03-22T07:12:16.765141Z",
      "pulp_href": "/pulp/api/v3/workers/b3a0a25a-0f50-42e6-8a39-1b68226992e3/",
      "name": "23370@hostname.sns.sec",
      "last_heartbeat": "2021-03-22T10:48:30.806559Z"
    },
    {
      "pulp_created": "2021-03-22T07:12:10.076036Z",
      "pulp_href": "/pulp/api/v3/workers/8c65bc91-19e1-4e30-ba6a-05beaa4ca573/",
      "name": "resource-manager",
      "last_heartbeat": "2021-03-22T10:48:34.517400Z"
    },
    {
      "pulp_created": "2021-03-22T07:12:17.186151Z",
      "pulp_href": "/pulp/api/v3/workers/cefd5028-dab6-4a2b-a593-b7a50bf3bba1/",
      "name": "23509@hostname.sns.sec",
      "last_heartbeat": "2021-03-22T10:48:57.177400Z"
    }
  ],
  "online_content_apps": [
    {
      "name": "23136@hostname",
      "last_heartbeat": "2021-03-22T10:48:54.277865Z"
    },
    {
      "name": "23192@hostname",
      "last_heartbeat": "2021-03-22T10:48:55.040598Z"
    },
    {
      "name": "23154@hostname",
      "last_heartbeat": "2021-03-22T10:48:55.045353Z"
    },
    {
      "name": "23155@hostname",
      "last_heartbeat": "2021-03-22T10:48:55.080843Z"
    },
    {
      "name": "23138@hostname",
      "last_heartbeat": "2021-03-22T10:48:55.111686Z"
    },
    {
      "name": "23200@hostname",
      "last_heartbeat": "2021-03-22T10:48:55.528796Z"
    },
    {
      "name": "23172@hostname",
      "last_heartbeat": "2021-03-22T10:48:55.567173Z"
    },
    {
      "name": "23199@hostname",
      "last_heartbeat": "2021-03-22T10:48:56.001181Z"
    }
  ],
  "database_connection": {
    "connected": true
  },
  "redis_connection": {
    "connected": true
  },
  "storage": {
    "total": 85851357184,
    "used": 3087319040,
    "free": 78358970368
  }
}

Any help in solving this issue would be appreciated. Same issue happens also with other RPM repositories and I've verfied that URLs are correct and accessible from host where Pulp is running.