Actions
Issue #5904
closed
This backend doesn't support absolute paths error when using s3 as a backend
Start date:
Due date:
Estimated time:
Severity:
2. Medium
Version:
Platform Release:
OS:
Triaged:
Yes
Groomed:
No
Sprint Candidate:
No
Tags:
Sprint:
Quarter:
Description
Following the examples from the documentation https://pulp-rpm.readthedocs.io/en/latest/workflows/upload.html
Ansible vars
---
pulp_use_system_wide_pkgs: true
pulp_install_plugins:
pulp-certguard:
source_dir: "/var/lib/pulp/devel/cert-guard/"
pulp-deb: {}
pulp-file: {}
pulp-rpm:
prereq_role: "pulp.pulp_rpm_prerequisites"
prereq_pip_packages:
- boto3
- django-storages
pulp_api_bind: '0.0.0.0:24817'
pulp_content_bind: '0.0.0.0:24816'
src_cert_guard_dir: "/var/lib/pulp/devel/cert-guard/"
pulp_settings:
secret_key: SECRET
content_origin: "http://pulp.domain:24816"
default_file_storage: "storages.backends.s3boto3.S3Boto3Storage"
aws_storage_bucket_name: "bucketname"
pulp_default_admin_password: SECRET
Code used to reproduce
#!/usr/bin/env bash
wait_until_task_finished() {
echo "Polling the task until it has reached a final state."
local task_url=$1
while true
do
local response=$(http $task_url)
local state=$(jq -r .state <<< ${response})
jq . <<< "${response}"
case ${state} in
failed|canceled)
echo "Task in final state: ${state}"
exit 1
;;
completed)
echo "$task_url complete."
break
;;
*)
echo "Still waiting..."
sleep 1
;;
esac
done
}
message() {
echo -e "\n######################################################################"
echo "# $1"
echo "######################################################################"
}
uploadArtifacts() {
# Get an RPM package
PKG="$@"
echo $PKG
# Upload it as an Artifact
echo "Upload an RPM package."
ARTIFACT=$(http --form POST $BASE_ADDR/pulp/api/v3/artifacts/ \
file@./$PKG)
ARTIFACT_HREF=$(echo $ARTIFACT | jq -r '.pulp_href')
if [ $ARTIFACT_HREF == "null" ]; then
message "Uploading artifact failed for some reason. Check the output below."
echo $ARTIFACT
exit 1
fi
echo "Inspecting artifact."
http $BASE_ADDR$ARTIFACT_HREF
message "Finished uploading artifact $PKG"
createRPM $ARTIFACT_HREF
}
createRPM() {
ARTIFACT=$@
# Create RPM package from an artifact
echo "Create RPM content from artifact."
TASK_URL=$(http POST $BASE_ADDR/pulp/api/v3/content/rpm/packages/ \
artifact=$ARTIFACT relative_path=$PKG | jq -r '.task')
# Poll the task (here we use a function defined in docs/_scripts/base.sh)
wait_until_task_finished $BASE_ADDR$TASK_URL
# After the task is complete, it gives us a new package (RPM content)
echo "Set PACKAGE_HREF from finished task."
PACKAGE_HREF=$(http $BASE_ADDR$TASK_URL| jq -r '.created_resources | first')
echo "Inspecting Package."
http $BASE_ADDR$PACKAGE_HREF
message "Finished uploading artifact $ARTIFACT"
repoAdd $PACKAGE_HREF
}
repoAdd() {
# Add created RPM content to repository
echo "Add created RPM Package to repository."
TASK_URL=$(http POST $BASE_ADDR$REPO_HREF'modify/' \
add_content_units:="[\"$@\"]" | jq -r '.task')
# Poll the task (here we use a function defined in docs/_scripts/base.sh)
wait_until_task_finished $BASE_ADDR$TASK_URL
# After the task is complete, it gives us a new repository version
echo "Set REPOVERSION_HREF from finished task."
REPOVERSION_HREF_WITH_PKG=$(http $BASE_ADDR$TASK_URL| jq -r '.created_resources | first')
echo "Inspecting RepositoryVersion."
http $BASE_ADDR$REPOVERSION_HREF_WITH_PKG
}
export BASE_ADDR="pulp.domain"
export REPO_HREF="/pulp/api/v3/repositories/rpm/rpm/9fe975ec-8e1c-480e-8ef7-8734b382659d/"
PKG="package.rpm"
uploadArtifacts $PKG
Versions
~$ pip3 list
DEPRECATION: The default format will switch to columns in the future. You can use --format=(legacy|columns) (or define a format=(legacy|columns) in your pip.conf under the [list] section) to disable this warning.
aiofiles (0.4.0)
aiohttp (3.6.2)
asgiref (3.2.3)
async-timeout (3.0.1)
attrs (19.3.0)
backoff (1.10.0)
boto3 (1.10.41)
botocore (1.13.41)
certifi (2019.11.28)
cffi (1.13.2)
chardet (3.0.4)
Click (7.0)
coreapi (2.3.3)
coreschema (0.0.4)
createrepo-c (0.15.4)
cryptography (2.8)
Django (2.2.9)
django-filter (2.2.0)
django-storages (1.8)
djangorestframework (3.10.3)
djangorestframework-queryfields (1.0.0)
docutils (0.15.2)
drf-nested-routers (0.91)
drf-yasg (1.17.0)
dynaconf (2.2.1)
gunicorn (20.0.4)
idna (2.8)
idna-ssl (1.1.0)
inflection (0.3.1)
itypes (1.1.0)
Jinja2 (2.10.3)
jmespath (0.9.4)
libcomps (0.1.14.post1)
MarkupSafe (1.1.1)
multidict (4.7.1)
nose (1.3.7)
packaging (19.2)
pip (9.0.3)
productmd (1.23)
psycopg2 (2.8.4)
pulp-certguard (0.1.0rc3.dev0, /var/lib/pulp/devel/cert-guard)
pulp-deb (2.0.0b3)
pulp-file (0.1.0)
pulp-rpm (3.0.0)
pulpcore (3.0.0)
pycparser (2.19)
pygobject (3.22.0)
pygtrie (2.3.2)
pyOpenSSL (19.1.0)
pyparsing (2.4.5)
python-box (3.4.6)
python-dateutil (2.8.0)
python-debian (0.1.36)
python-dotenv (0.10.3)
pytz (2019.3)
PyYAML (5.2)
redis (3.1.0)
requests (2.22.0)
rq (1.1.0)
ruamel.yaml (0.16.5)
ruamel.yaml.clib (0.2.0)
s3transfer (0.2.1)
scikit-build (0.10.0)
setuptools (39.2.0)
six (1.13.0)
sqlparse (0.3.0)
toml (0.10.0)
typing-extensions (3.7.4.1)
uritemplate (3.0.0)
urllib3 (1.25.7)
wheel (0.33.6)
whitenoise (4.1.4)
yarl (1.4.2)
Script output
Upload an RPM package.
Inspecting artifact.
HTTP/1.1 200 OK
Allow: GET, DELETE, HEAD, OPTIONS
Connection: keep-alive
Content-Length: 704
Content-Type: application/json
Date: Thu, 19 Dec 2019 14:37:37 GMT
Server: nginx/1.16.1
Vary: Accept, Cookie
X-Frame-Options: SAMEORIGIN
{
"file": "artifact/de/46eb3bbad80328e5b235b1d417edd17ee86a6edb96d25032bf16ccf6b8eb86",
"md5": "82c35e7d9ce9933a19b57ce122e0c185",
"pulp_created": "2019-12-19T14:37:36.097806Z",
"pulp_href": "/pulp/api/v3/artifacts/448a78ef-4ce8-4de5-b9d8-5fa8704c0273/",
"sha1": "c1e8e6a06e434a7ed7f9c8010bbf7416e6497f76",
"sha224": "2affd187cca5c2165893d22ace6f123ffbfc359077621d5f925c02e8",
"sha256": "de46eb3bbad80328e5b235b1d417edd17ee86a6edb96d25032bf16ccf6b8eb86",
"sha384": "ce0ae324b83ccccc6e8383f4917925c31dd5410c289c5278039e3a651f7009ea4b38bc014a69b852703442e5155f3d4b",
"sha512": "1f1a02b67c536f6a69839f9b3f4c543569e4620b0feb6c2fa8ae192db12a95463307322f365832db7135b75ea28729c6aa50ed6850617ea8e4368a7a36593c17",
"size": 26562555
}
######################################################################
# Finished uploading xxxxxx.rpm
######################################################################
Create RPM content from artifact.
Polling the task until it has reached a final state.
{
"pulp_href": "/pulp/api/v3/tasks/b1fc4cfc-c4df-4b7a-842a-0d9c24bc5517/",
"pulp_created": "2019-12-19T14:37:37.839018Z",
"state": "failed",
"name": "pulpcore.app.tasks.base.general_create",
"started_at": "2019-12-19T14:37:37.917992Z",
"finished_at": "2019-12-19T14:37:38.117287Z",
"error": {
"traceback": " File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/worker.py\", line 822, in perform_job\n rv = job.perform()\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/job.py\", line 605, in perform\n self._result = self._execute()\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/rq/job.py\", line 611, in _execute\n return self.func(*self.args, **self.kwargs)\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/pulpcore/app/tasks/base.py\", line 17, in general_create\n serializer.is_valid(raise_exception=True)\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/rest_framework/serializers.py\", line 235, in is_valid\n self._validated_data = self.run_validation(self.initial_data)\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/rest_framework/serializers.py\", line 433, in run_validation\n value = self.validate(value)\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/pulpcore/plugin/serializers/content.py\", line 64, in validate\n data = self.deferred_validate(data)\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/pulp_rpm/app/serializers.py\", line 246, in deferred_validate\n new_pkg = _prepare_package(data[\"artifact\"], data[\"relative_path\"])\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/pulp_rpm/app/shared_utils.py\", line 25, in _prepare_package\n shutil.copy2(artifact.file.path, temp_path)\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/django/db/models/fields/files.py\", line 57, in path\n return self.storage.path(self.name)\n File \"/usr/local/lib/pulp/lib64/python3.6/site-packages/django/core/files/storage.py\", line 109, in path\n raise NotImplementedError(\"This backend doesn't support absolute paths.\")\n",
"description": "This backend doesn't support absolute paths."
},
"worker": "/pulp/api/v3/workers/5ff16059-bcc9-4b8d-ac6f-5e06455a1971/",
"progress_reports": [],
"created_resources": [],
"reserved_resources_record": [
"/pulp/api/v3/artifacts/448a78ef-4ce8-4de5-b9d8-5fa8704c0273/"
]
}
Task in final state: failed
Actions
.
Dealing with Packages from S3
https://pulp.plan.io/issues/5904 closes #5904