Pulp

Consider either scenario:

1. A user installs pulp for the 1st time with pulp_installer, including a plugin with a prereq role (pulp_rpm).
2. A user upgrades pulp with pulp_instaler, including a new version of the prereq role.

The following happens:

1. The preflight check happens. It actually fails to evaluate compatibility due to to (new) C dependencies to build the plugin with a prereq role being installed.
2. The output from that failure to evaluate is not caught, it is returned as success regardless.
3. The plugin prereq role gets run immediately after this.
4. pulpcore & plugins get installed, regardless of compatibility or not.

Possible solutions:

1. Run the pre-flight check after the prereq role(s). The system already gets modified a little before then anyway, which could break plugins just like the prereq role does.
2. Record failure (ansible "rescue" task/block attribute) and re-run the pre-flight check later if it failed to evaluate.

Output (I added this task after the check):

        - debug: var=compatibility

TASK [pulp : debug] ************************************************************
ok: [pulp3-sandbox-fedora31] => {
    "compatibility": {
        "changed": false,
        "cmd": [
            "/usr/local/lib/pulp/bin/pip-compile"
        ],
        "delta": "0:00:08.244381",
        "end": "2020-05-04 21:41:15.578898",
        "failed": false,
        "failed_when_result": false,
        "msg": "non-zero return code",
        "rc": 1,
        "start": "2020-05-04 21:41:07.334517",
        "stderr": "EDITED: A GIANT BLOB VERSION OF stderr_lines"
        "stderr_lines": [
            "    ERROR: Command errored out with exit status 1:",
            "     command: /usr/local/lib/pulp/bin/python3 /usr/local/lib/pulp/lib64/python3.7/site-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-q6uy_p34/overlay --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools wheel pycairo",
            "         cwd: None",
            "    Complete output (31 lines):",
            "    Collecting setuptools",
            "      Using cached setuptools-46.1.3-py3-none-any.whl (582 kB)",
            "    Collecting wheel",
            "      Downloading wheel-0.34.2-py2.py3-none-any.whl (26 kB)",
            "    Collecting pycairo",
            "      Downloading pycairo-1.19.1.tar.gz (205 kB)",
            "    Could not build wheels for pycairo, since package 'wheel' is not installed.",
            "    Installing collected packages: setuptools, wheel, pycairo",
            "        Running setup.py install for pycairo: started",
            "        Running setup.py install for pycairo: finished with status 'error'",
            "        ERROR: Command errored out with exit status 1:",
            "         command: /usr/local/lib/pulp/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-install-3kgu2pft/pycairo/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-install-3kgu2pft/pycairo/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record /tmp/pip-record-5xpsg3b4/install-record.txt --single-version-externally-managed --prefix /tmp/pip-build-env-q6uy_p34/overlay --compile --install-headers /usr/local/lib/pulp/include/site/python3.7/pycairo",
            "             cwd: /tmp/pip-install-3kgu2pft/pycairo/",
            "        Complete output (15 lines):",
            "        running install",
            "        running build",
            "        running build_py",
            "        creating build",
            "        creating build/lib.linux-x86_64-3.7",
            "        creating build/lib.linux-x86_64-3.7/cairo",
            "        copying cairo/__init__.py -> build/lib.linux-x86_64-3.7/cairo",
            "        copying cairo/__init__.pyi -> build/lib.linux-x86_64-3.7/cairo",
            "        copying cairo/py.typed -> build/lib.linux-x86_64-3.7/cairo",
            "        running build_ext",
            "        Package cairo was not found in the pkg-config search path.",
            "        Perhaps you should add the directory containing `cairo.pc'",
            "        to the PKG_CONFIG_PATH environment variable",
            "        Package 'cairo', required by 'virtual:world', not found",
            "        Command '['pkg-config', '--print-errors', '--exists', 'cairo >= 1.13.1']' returned non-zero exit status 1.",
            "        ----------------------------------------",
            "    ERROR: Command errored out with exit status 1: /usr/local/lib/pulp/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-install-3kgu2pft/pycairo/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-install-3kgu2pft/pycairo/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record /tmp/pip-record-5xpsg3b4/install-record.txt --single-version-externally-managed --prefix /tmp/pip-build-env-q6uy_p34/overlay --compile --install-headers /usr/local/lib/pulp/include/site/python3.7/pycairo Check the logs for full command output.",
            "    ----------------------------------------",
            "Traceback (most recent call last):",
            "  File \"/usr/local/lib/pulp/bin/pip-compile\", line 8, in <module>",
            "    sys.exit(cli())",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/click/core.py\", line 829, in __call__",
            "    return self.main(*args, **kwargs)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/click/core.py\", line 782, in main",
            "    rv = self.invoke(ctx)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/click/core.py\", line 1066, in invoke",
            "    return ctx.invoke(self.callback, **ctx.params)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/click/core.py\", line 610, in invoke",
            "    return callback(*args, **kwargs)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/click/decorators.py\", line 21, in new_func",
            "    return f(get_current_context(), *args, **kwargs)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/piptools/scripts/compile.py\", line 392, in cli",
            "    results = resolver.resolve(max_rounds=max_rounds)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/piptools/resolver.py\", line 166, in resolve",
            "    has_changed, best_matches = self._resolve_one_round()",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/piptools/resolver.py\", line 257, in _resolve_one_round",
            "    their_constraints.extend(self._iter_dependencies(best_match))",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/piptools/resolver.py\", line 363, in _iter_dependencies",
            "    dependencies = self.repository.get_dependencies(ireq)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/piptools/repositories/local.py\", line 74, in get_dependencies",
            "    return self.repository.get_dependencies(ireq)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/piptools/repositories/pypi.py\", line 224, in get_dependencies",
            "    download_dir, ireq, wheel_cache",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/piptools/repositories/pypi.py\", line 180, in resolve_reqs",
            "    results = resolver._resolve_one(reqset, ireq)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/pip/_internal/resolution/legacy/resolver.py\", line 362, in _resolve_one",
            "    abstract_dist = self._get_abstract_dist_for(req_to_install)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/pip/_internal/resolution/legacy/resolver.py\", line 314, in _get_abstract_dist_for",
            "    abstract_dist = self.preparer.prepare_linked_requirement(req)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/pip/_internal/operations/prepare.py\", line 431, in prepare_linked_requirement",
            "    req, self.req_tracker, self.finder, self.build_isolation,",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/pip/_internal/operations/prepare.py\", line 85, in _get_prepared_distribution",
            "    abstract_dist.prepare_distribution_metadata(finder, build_isolation)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/pip/_internal/distributions/sdist.py\", line 38, in prepare_distribution_metadata",
            "    self._setup_isolation(finder)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/pip/_internal/distributions/sdist.py\", line 68, in _setup_isolation",
            "    \"Installing build dependencies\"",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/pip/_internal/build_env.py\", line 199, in install_requirements",
            "    call_subprocess(args, spinner=spinner)",
            "  File \"/usr/local/lib/pulp/lib64/python3.7/site-packages/pip/_internal/utils/subprocess.py\", line 241, in call_subprocess",
            "    raise InstallationError(exc_msg)",
            "pip._internal.exceptions.InstallationError: Command errored out with exit status 1: /usr/local/lib/pulp/bin/python3 /usr/local/lib/pulp/lib64/python3.7/site-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-q6uy_p34/overlay --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools wheel pycairo Check the logs for full command output."
        ],
        "stdout": "",
        "stdout_lines": []
    }
}