Project

Profile

Help

Issue #3535

Error syncing Oracle EPEL repository

Added by steveh1138 about 2 years ago. Updated 12 months ago.

Status:
CLOSED - CURRENTRELEASE
Priority:
Normal
Assignee:
Category:
-
Sprint/Milestone:
-
Start date:
Due date:
Severity:
2. Medium
Version:
Platform Release:
2.16.2
Blocks Release:
OS:
CentOS 7
Backwards Incompatible:
No
Triaged:
Yes
Groomed:
No
Sprint Candidate:
No
Tags:
Pulp 2
QA Contact:
Complexity:
Smash Test:
Verified:
No
Verification Required:
No
Sprint:
Sprint 38

Description

When attempting to sync the official public Oracle EPEL repository I receive the following error in Foreman/Katello. As this is a pulp error I was directed over to your project. Foreman / Katello 1.16.0 / 3.5.1

PLP0000: Invalid block tag: '}gmail{*}com>'

Here is the error output I receive

{"contents_changed"=>true,
 "pulp_tasks"=>
  [{"exception"=>nil,
    "task_type"=>"pulp.server.managers.repo.sync.sync",
    "_href"=>"/pulp/api/v2/tasks/57edd0fc-dcf8-453f-a5ac-2ede318d2311/",
    "task_id"=>"57edd0fc-dcf8-453f-a5ac-2ede318d2311",
    "tags"=>
     ["pulp:repository:4d73ea9e-7776-486e-811b-d914dc823105",
      "pulp:action:sync"],
    "finish_time"=>"2018-03-28T04:15:52Z",
    "_ns"=>"task_status",
    "start_time"=>"2018-03-28T04:02:38Z",
    "traceback"=>nil,
    "spawned_tasks"=>
     [{"_href"=>"/pulp/api/v2/tasks/18a119c8-9f71-4d05-972b-d9932e635ec8/",
       "task_id"=>"18a119c8-9f71-4d05-972b-d9932e635ec8"}],
    "progress_report"=>
     {"yum_importer"=>
       {"content"=>
         {"items_total"=>10,
          "state"=>"FINISHED",
          "error_details"=>[],
          "details"=>
           {"rpm_total"=>10, "rpm_done"=>10, "drpm_total"=>0, "drpm_done"=>0},
          "size_total"=>15409527,
          "size_left"=>0,
          "items_left"=>0},
        "comps"=>{"state"=>"FINISHED"},
        "purge_duplicates"=>{"state"=>"FINISHED"},
        "distribution"=>
         {"items_total"=>0,
          "state"=>"FINISHED",
          "error_details"=>[],
          "items_left"=>0},
        "errata"=>{"state"=>"FINISHED"},
        "metadata"=>{"state"=>"FINISHED"}}},
    "queue"=>"reserved_resource_worker-2@ussl-foreman.usanainc.com.dq",
    "state"=>"finished",
    "worker_name"=>"reserved_resource_worker-2@ussl-foreman.usanainc.com",
    "result"=>
     {"result"=>"success",
      "importer_id"=>"yum_importer",
      "exception"=>nil,
      "repo_id"=>"4d73ea9e-7776-486e-811b-d914dc823105",
      "traceback"=>nil,
      "started"=>"2018-03-28T04:02:40Z",
      "_ns"=>"repo_sync_results",
      "completed"=>"2018-03-28T04:15:50Z",
      "importer_type_id"=>"yum_importer",
      "error_message"=>nil,
      "summary"=>
       {"content"=>{"state"=>"FINISHED"},
        "comps"=>{"state"=>"FINISHED"},
        "purge_duplicates"=>{"state"=>"FINISHED"},
        "distribution"=>{"state"=>"FINISHED"},
        "errata"=>{"state"=>"FINISHED"},
        "metadata"=>{"state"=>"FINISHED"}},
      "added_count"=>10,
      "removed_count"=>0,
      "updated_count"=>2,
      "id"=>"5abb16f7a55b78667ce16f7a",
      "details"=>
       {"content"=>
         {"size_total"=>15409527,
          "items_left"=>0,
          "items_total"=>10,
          "state"=>"FINISHED",
          "size_left"=>0,
          "details"=>
           {"rpm_total"=>10, "rpm_done"=>10, "drpm_total"=>0, "drpm_done"=>0},
          "error_details"=>[]},
        "comps"=>{"state"=>"FINISHED"},
        "purge_duplicates"=>{"state"=>"FINISHED"},
        "distribution"=>
         {"items_total"=>0,
          "state"=>"FINISHED",
          "error_details"=>[],
          "items_left"=>0},
        "errata"=>{"state"=>"FINISHED"},
        "metadata"=>{"state"=>"FINISHED"}}},
    "error"=>nil,
    "_id"=>{"$oid"=>"5abb134bde37317496f9af38"},
    "id"=>"5abb134bde37317496f9af38"},
   {"exception"=>nil,
    "task_type"=>"pulp.server.managers.repo.publish.publish",
    "_href"=>"/pulp/api/v2/tasks/18a119c8-9f71-4d05-972b-d9932e635ec8/",
    "task_id"=>"18a119c8-9f71-4d05-972b-d9932e635ec8",
    "tags"=>
     ["pulp:repository:4d73ea9e-7776-486e-811b-d914dc823105",
      "pulp:action:publish"],
    "finish_time"=>"2018-03-28T04:20:36Z",
    "_ns"=>"task_status",
    "start_time"=>"2018-03-28T04:20:02Z",
    "traceback"=>
     "Traceback (most recent call last):\n  File \"/usr/lib/python2.7/site-packages/celery/app/trace.py\", line 240, in trace_task\n    R = retval = fun(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/async/tasks.py\", line 527, in __call__\n    return super(Task, self).__call__(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/async/tasks.py\", line 107, in __call__\n    return super(PulpTask, self).__call__(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/celery/app/trace.py\", line 438, in __protected_call__\n    return self.run(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/controllers/repository.py\", line 1106, in publish\n    result = check_publish(repo_obj, dist_id, dist_inst, transfer_repo, conduit, call_config)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/controllers/repository.py\", line 1200, in check_publish\n    result = _do_publish(repo_obj, dist_id, dist_inst, transfer_repo, conduit, call_config)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/controllers/repository.py\", line 1252, in _do_publish\n    publish_report = publish_repo(transfer_repo, conduit, call_config)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/async/tasks.py\", line 730, in wrap_f\n    return f(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/distributors/yum/distributor.py\", line 174, in publish_repo\n    return self._publisher.process_lifecycle()\n  File \"/usr/lib/python2.7/site-packages/pulp/plugins/util/publish_step.py\", line 572, in process_lifecycle\n    super(PluginStep, self).process_lifecycle()\n  File \"/usr/lib/python2.7/site-packages/pulp/plugins/util/publish_step.py\", line 163, in process_lifecycle\n    step.process()\n  File \"/usr/lib/python2.7/site-packages/pulp/plugins/util/publish_step.py\", line 239, in process\n    self._process_block(item=item)\n  File \"/usr/lib/python2.7/site-packages/pulp/plugins/util/publish_step.py\", line 301, in _process_block\n    self.process_main(item=item)\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/distributors/yum/publish.py\", line 485, in process_main\n    context.add_unit_metadata(unit)\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/distributors/yum/metadata/other.py\", line 42, in add_unit_metadata\n    self.metadata_file_handle.write(unit.render_other(self.checksum_type))\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/db/models.py\", line 850, in render_other\n    return self._render(metadata, context)\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/db/models.py\", line 879, in _render\n    t = Template(template)\n  File \"/usr/lib/python2.7/site-packages/django/template/base.py\", line 125, in __init__\n    self.nodelist = compile_string(template_string, origin)\n  File \"/usr/lib/python2.7/site-packages/django/template/base.py\", line 153, in compile_string\n    return parser.parse()\n  File \"/usr/lib/python2.7/site-packages/django/template/base.py\", line 276, in parse\n    self.invalid_block_tag(token, command, parse_until)\n  File \"/usr/lib/python2.7/site-packages/django/template/base.py\", line 332, in invalid_block_tag\n    raise self.error(token, \"Invalid block tag: '%s'\" % command)\nTemplateSyntaxError: Invalid block tag: '}gmail{*}com>'\n",
    "spawned_tasks"=>[],
    "progress_report"=>
     {"4d73ea9e-7776-486e-811b-d914dc823105"=>
       [{"num_success"=>1,
         "description"=>"Copying files",
         "step_type"=>"save_tar",
         "items_total"=>1,
         "state"=>"FINISHED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"9c354dd3-4837-4ea8-a3e1-7a03861942ff",
         "num_processed"=>1},
        {"num_success"=>1,
         "description"=>"Initializing repo metadata",
         "step_type"=>"initialize_repo_metadata",
         "items_total"=>1,
         "state"=>"FINISHED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"23f96ff2-525e-451b-bfda-b558d113f17f",
         "num_processed"=>1},
        {"num_success"=>0,
         "description"=>"Publishing Distribution files",
         "step_type"=>"distribution",
         "items_total"=>0,
         "state"=>"FINISHED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"848c3bd5-0fed-4a5e-bc76-05b886952c19",
         "num_processed"=>0},
        {"num_success"=>299,
         "description"=>"Publishing RPMs",
         "step_type"=>"rpms",
         "items_total"=>17849,
         "state"=>"FAILED",
         "error_details"=>
          [{"traceback"=>
             "  File \"/usr/lib/python2.7/site-packages/pulp/plugins/util/publish_step.py\", line 239, in process\n    self._process_block(item=item)\n\n  File \"/usr/lib/python2.7/site-packages/pulp/plugins/util/publish_step.py\", line 301, in _process_block\n    self.process_main(item=item)\n\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/distributors/yum/publish.py\", line 485, in process_main\n    context.add_unit_metadata(unit)\n\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/distributors/yum/metadata/other.py\", line 42, in add_unit_metadata\n    self.metadata_file_handle.write(unit.render_other(self.checksum_type))\n\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/db/models.py\", line 850, in render_other\n    return self._render(metadata, context)\n\n  File \"/usr/lib/python2.7/site-packages/pulp_rpm/plugins/db/models.py\", line 879, in _render\n    t = Template(template)\n\n  File \"/usr/lib/python2.7/site-packages/django/template/base.py\", line 125, in __init__\n    self.nodelist = compile_string(template_string, origin)\n\n  File \"/usr/lib/python2.7/site-packages/django/template/base.py\", line 153, in compile_string\n    return parser.parse()\n\n  File \"/usr/lib/python2.7/site-packages/django/template/base.py\", line 276, in parse\n    self.invalid_block_tag(token, command, parse_until)\n\n  File \"/usr/lib/python2.7/site-packages/django/template/base.py\", line 332, in invalid_block_tag\n    raise self.error(token, \"Invalid block tag: '%s'\" % command)\n",
            "error"=>"Invalid block tag: '}gmail{*}com>'"}],
         "details"=>"",
         "num_failures"=>1,
         "step_id"=>"c772a9bf-e9f4-43b7-bde8-302909f9dd9b",
         "num_processed"=>300},
        {"num_success"=>0,
         "description"=>"Publishing Delta RPMs",
         "step_type"=>"drpms",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"d1431e03-4eb4-474f-aec2-bd850013fd91",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Publishing Errata",
         "step_type"=>"errata",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"d148e811-003c-4402-bbd7-db155bd0d486",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Publishing Comps file",
         "step_type"=>"comps",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"6231f6da-1020-4fd4-92fd-131e57dce3e7",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Publishing Metadata.",
         "step_type"=>"metadata",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"1eb2afec-4d4e-4273-bb61-d7280cb86e2a",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Closing repo metadata",
         "step_type"=>"close_repo_metadata",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"ec3585fb-9eb5-4731-b14a-ab7ab3944713",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Generating sqlite files",
         "step_type"=>"generate sqlite",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"f2f4bedc-2662-4107-b08e-91593d7ad2cb",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Generating HTML files",
         "step_type"=>"repoview",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"2b9388b9-622a-4e91-a85b-6f9cc743e389",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Publishing files to web",
         "step_type"=>"publish_directory",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"777f72ca-23b9-4a38-9257-1025123a0e2d",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Writing Listings File",
         "step_type"=>"initialize_repo_metadata",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"1b2738a1-31d0-47b3-b658-f866245b97a8",
         "num_processed"=>0},
        {"num_success"=>0,
         "description"=>"Writing Listings File",
         "step_type"=>"initialize_repo_metadata",
         "items_total"=>1,
         "state"=>"NOT_STARTED",
         "error_details"=>[],
         "details"=>"",
         "num_failures"=>0,
         "step_id"=>"3488c45f-94cb-43ab-976c-bbcf65275c5f",
         "num_processed"=>0}]},
    "queue"=>"reserved_resource_worker-0@ussl-foreman.usanainc.com.dq",
    "state"=>"error",
    "worker_name"=>"reserved_resource_worker-0@ussl-foreman.usanainc.com",
    "result"=>nil,
    "error"=>
     {"code"=>"PLP0000",
      "data"=>{},
      "description"=>"Invalid block tag: '}gmail{*}com>'",
      "sub_errors"=>[]},
    "_id"=>{"$oid"=>"5abb16f8de37317496f9af98"},
    "id"=>"5abb16f8de37317496f9af98"}],
 "poll_attempts"=>{"total"=>37, "failed"=>1}}

Here are the versions of every pulp package currently installed

python-kombu-3.0.33-8.pulp.el7.noarch
rubygem-smart_proxy_pulp-1.3.0-1.el7.noarch
python-pulp-streamer-2.13.4-1.el7.noarch
pulp-puppet-tools-2.13.4-1.el7.noarch
python-pulp-ostree-common-1.2.1-3.el7.noarch
python-pulp-puppet-common-2.13.4-1.el7.noarch
python-pulp-docker-common-2.4.1-2.el7.noarch
python-isodate-0.5.0-4.pulp.el7.noarch
python-pulp-client-lib-2.13.4-1.el7.noarch
python-pulp-common-2.13.4-1.el7.noarch
python-pulp-rpm-common-2.13.4-1.el7.noarch
pulp-selinux-2.13.4-1.el7.noarch
pulp-puppet-plugins-2.13.4-1.el7.noarch
pulp-rpm-admin-extensions-2.13.4-1.el7.noarch
python-pulp-bindings-2.13.4-1.el7.noarch
pulp-katello-1.0.2-1.el7.noarch
pulp-ostree-plugins-1.2.1-3.el7.noarch
pulp-ostree-admin-extensions-1.2.1-3.el7.noarch
python-pulp-repoauth-2.13.4-1.el7.noarch
pulp-docker-plugins-2.4.1-2.el7.noarch
pulp-client-1.0-1.noarch
pulp-rpm-plugins-2.13.4-1.el7.noarch
pulp-server-2.13.4-1.el7.noarch
python-pulp-oid_validation-2.13.4-1.el7.noarch
pulp-admin-client-2.13.4-1.el7.noarch

Associated revisions

Revision 4a9965fc View on GitHub
Added by bmbouter almost 2 years ago

Sanitize Invalid Changelogs Too

The diff below causes the selection of input to be sanitized to be greedy instead of non-greedy. This causes it work sanitize more data in the cases of invalid entries such as Changelogs with too many sections in one rpm.

This fix is very safe for several reasons:

  1. Escaping all data for the template layer should be safe. This is essentially user-provided data given that we got it from the internet, so it probably all should be escaped. Since this fixes this issue I'm not going to rewrite the sanitizer now to keep it low-risk.

  2. The extra-greedyness added is still limited by the end tag matched

https://pulp.plan.io/issues/3535 closes #3535

Revision 62edf22a View on GitHub
Added by bmbouter almost 2 years ago

Sanitize Invalid Changelogs Too

The diff below causes the selection of input to be sanitized to be greedy instead of non-greedy. This causes it work sanitize more data in the cases of invalid entries such as Changelogs with too many sections in one rpm.

This fix is very safe for several reasons:

  1. Escaping all data for the template layer should be safe. This is essentially user-provided data given that we got it from the internet, so it probably all should be escaped. Since this fixes this issue I'm not going to rewrite the sanitizer now to keep it low-risk.

  2. The extra-greedyness added is still limited by the end tag matched

https://pulp.plan.io/issues/3535 closes #3535

(cherry picked from commit 4a9965fc100f1414d3be0845138fd66981b0ba31)

History

#1 Updated by daviddavis about 2 years ago

  • Project changed from File Support to RPM Support

#2 Updated by ttereshc about 2 years ago

  • Description updated (diff)

Could you provide a feed URL to help with reproducing your issue?
If the remote repository is not publicly available, it will be helpful if you can find the metadata of a package which causes the error. I would look for a substring }gmail{*}com in the primary.xml file of the remote repository.
Thanks

#3 Updated by ttereshc about 2 years ago

FWIW, my guess would be that a bug is somewhere around a logic for a substitution of special characters.
https://github.com/pulp/pulp_rpm/blob/f0ffdac0390ecf78ac2c02a2f5a2a840dda7fdf1/plugins/pulp_rpm/plugins/db/models.py#L929

#4 Updated by steveh1138 about 2 years ago

This is the URL for the repository I'm attempting to sync

http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64

#5 Updated by dalley almost 2 years ago

  • Triaged changed from No to Yes
  • Sprint set to Sprint 35

#6 Updated by mansari almost 2 years ago

  • Status changed from NEW to ASSIGNED
  • Assignee set to mansari

#7 Updated by rchan almost 2 years ago

  • Sprint changed from Sprint 35 to Sprint 36

#8 Updated by rchan almost 2 years ago

  • Sprint changed from Sprint 36 to Sprint 37

#9 Updated by rchan almost 2 years ago

  • Status changed from ASSIGNED to NEW
  • Assignee deleted (mansari)

#10 Updated by terjetrane almost 2 years ago

I'm seeing the same thing using Red Hat Satellite 6.3 which has pulp-server-2.13.4.9-1.el7sat.noarch

The error occurs twice when trying to sync EPEL6 (from a mirror at http://fedora.uib.no/epel/6/x86_64/)

PLP0000: Invalid block tag: '}gmail{*}com>'

Grepping through the xml files in repodata I find these to lines in 9bae1663c2fef450ad66c4113bc1d70c1acc0852bfcb6bbe07d7c711b883c7da-other.xml:

<changelog author="Neal Gompa <ngompa13{%}gmail{*}com> - 0.11.2-1" date="1437048000">- Update to 0.11.2</changelog>

<changelog author="Neal Gompa <ngompa13{%}gmail{*}com> - 0.3.3-1" date="1274702400">- Update to OggConvert 0.3.3</changelog>

Seems he is replacing @ with {%} to avoid spammers harvesting his email address, and the {% sequence is then interpreted as "openblock" for a Django template or something, that expects a tag (like 'load', 'comment', 'for', 'if', etc.) and gets confused when it sees '}gmail{*}com>' instead.

I believe this worked for our previous version of Satellite, 6.2 that used pulp 2.8.7

#11 Updated by dkliban@redhat.com almost 2 years ago

If the above is true, then the problem is most likely in this method[0].

The problem is described here[1]. Need to escape the django syntax character somehow.

[0] https://github.com/pulp/pulp_rpm/blob/master/plugins/pulp_rpm/plugins/db/models.py#L887
[1] https://docs.djangoproject.com/en/1.10/ref/templates/api/#limitations-with-string-literals

#12 Updated by bmbouter almost 2 years ago

  • Status changed from NEW to ASSIGNED
  • Assignee set to bmbouter

#13 Updated by rchan almost 2 years ago

  • Sprint changed from Sprint 37 to Sprint 38

#14 Updated by bmbouter almost 2 years ago

I was not able to reproduce the emitted exception. I verified the exception was not present in the logs and also in the raw json returned from the server. I tested the following:

Pulp 2.16.1 source checkout on F28 w/ Django 1.11.13
Satellite 6.3, on EL7 w/ Django 1.6.11

I tried syncing from http://archive.linux.duke.edu/pub/epel/6Server/x86_64/ which should be effectively the same as http://fedora.uib.no/epel/6/x86_64/.

I verified that the <changelog author="Neal Gompa <ngompa13{%}gmail{*}com> - 0.11.2-1" date="1437048000">- Update to 0.11.2</changelog> was present in the other.xml published data so we have confirmation that in the test cases Pulp's template processing handled this entry.

I'm going to try http://fedora.uib.no/epel/6/x86_64/ just to be sure it also doesn't produce the issue.

#15 Updated by bmbouter almost 2 years ago

I was also not able to reproduce when syncing from http://fedora.uib.no/epel/6/x86_64/ either.

Specifically to test w/ pulp-admin I create the repo and sync it with:

pulp-admin rpm repo create --repo-id epel6 --download-policy=on_demand --feed="http://fedora.uib.no/epel/6/x86_64/"
pulp-admin rpm repo sync run --repo-id epel6

I'm going to try http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64/ next with:

pulp-admin rpm repo create --repo-id OL7 --download-policy=on_demand --feed="http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64/"
pulp-admin rpm repo sync run --repo-id OL7

#16 Updated by bmbouter almost 2 years ago

Actually this does reproduce w/ the oracle linux repo.

Task Failed

Invalid block tag on line 7: '}gmail{*}com&gt;'. Did you forget to register or
load this tag?

#17 Updated by bmbouter almost 2 years ago

Actually the escaping code is not even performing that transform. The exception occurs when Template(foo) tries to instantiate sanitized data and it contains {%}gmail{*}com&gt not {% templatetag openblock %}}gmail{*}com&gt. I'm looking into why this data isn't sanitized like it should be.

#18 Updated by bmbouter almost 2 years ago

This issue occurs because the regex that is used to select the <changelog>...</changelog> entry for sanitization matches non greedily. The package python2-pika-doc (details below) has multiple changelogs. Here is the snippet stored in Pulp's db that is being published:

- merge changelog- Rebuild for Python 3.6- Honor %_smp_ncpus_max setting on testing- enable parallel testing with nose - enable python-twisted-core and python-tornado on epel - drop obsolete Group tag- https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages- drop some duplications - validate b0rken tests on epel7- add %check with execution of both unit and acceptance tests - enable adapters for both tornado and twisted - generate additional documentation, split into subpackage- Fix builds by defining python3_pkgversion if it doesn't exist - Add missing BRs for py3-other variant (for EPEL 7)- Actually make the python 3 bcond work- Upgrade to version 0.10.0 - Refactor to meet current Fedora guidelines - Add Python 3 subpackage (with EPEL 7 compatibility)- Rebuilt for https://fedoraproject.org/wiki/Fedora_24_Mass_Rebuild- Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild- Rebuilt for https://fedoraproject.org/wiki/Fedora_21_Mass_Rebuild- Rebuilt for https://fedoraproject.org/wiki/Fedora_20_Mass_Rebuild- Rebuilt for https://fedoraproject.org/wiki/Fedora_19_Mass_Rebuild- Rebuilt for https://fedoraproject.org/wiki/Fedora_18_Mass_Rebuild- Bump pika release version to fix upgrade path for f17 -\> f18- Patch pika/adapters/blocking_connection.py- Rebuilt for https://fedoraproject.org/wiki/Fedora_17_Mass_Rebuild- Upgrade to version 0.9.5- Upgrade to version 0.9.4- Upgrade to version 0.9.3- Initial Package
package details:

<package arch="noarch" name="python2-pika-doc" pkgid="{{ pkgid }}"><version epoch="0" rel="9.el7" ver="0.10.0" /

#19 Updated by bmbouter almost 2 years ago

The diff below causes the section selected for sanitization to be greedy instead of non-greedy and resolves the issue so sync+publish on that repo work. This fix is very safe for several reasons:

1. Escaping all data for the template layer should be safe. This is essentially user-provided data given that we got it from the internet so it probably all should be escaped. Since this fixes this issue I'm not going to rewrite the sanitizer now to keep it low-risk.

2. The extra-greedyness added is still limited by the end tag matched

I'm making a PR from this now.

diff --git a/plugins/pulp_rpm/plugins/db/models.py b/plugins/pulp_rpm/plugins/db/models.py
index 4d49ff2c..77f455a2 100644
--- a/plugins/pulp_rpm/plugins/db/models.py
+++ b/plugins/pulp_rpm/plugins/db/models.py
@@ -903,7 +903,7 @@ class RpmBase(NonMetadataPackage):
         """
         start_tag_pattern = r'<%s.*?(?<!/)>' % tag_name
         end_tag_pattern = r'</%s>' % tag_name
-        complete_tag_pattern = r'(%s)(.*?)(%s)' % (start_tag_pattern, end_tag_pattern)
+        complete_tag_pattern = r'(%s)(.*)(%s)' % (start_tag_pattern, end_tag_pattern)
         tag_re = re.compile(complete_tag_pattern, flags=re.DOTALL)
         template = tag_re.sub(RpmBase._generate_tag_replacement_str, template)
         return template

#20 Updated by bmbouter almost 2 years ago

  • Smash Test set to 1065

#21 Updated by bmbouter almost 2 years ago

  • Status changed from ASSIGNED to MODIFIED

#22 Updated by dkliban@redhat.com almost 2 years ago

  • Platform Release set to 2.16.2

#25 Updated by ipanova@redhat.com over 1 year ago

  • Status changed from MODIFIED to CLOSED - CURRENTRELEASE

#26 Updated by bmbouter 12 months ago

  • Tags Pulp 2 added

Please register to edit this issue

Also available in: Atom PDF