Issue #5964: pulpcore.plugin.repo_version_utils.remove_duplicates does not handle base_version != None correctly.

Issue #5964

# Problem 

 

 `pulpcore.plugin.repo_version_utils.remove_duplicates` does not handle the case were `base_version != None` correctly as it operates on the wrong content sets. 

 

 # Example 

 

 Assume that there are two artifacts: 

 

 \- a1 sha256: 4355...   
   
 \- a2 sha256: 53c2... 

 

 and two file content units: 

 

 \- c1 "relative_path": "test_upload.txt", artifact is a1   
   
 \- c2 "relative_path": "test_upload.txt", artifact is a2 

 

 These two collide w.r.t. the repo_key and `pulpcore.plugin.repo_version_utils.remove_duplicates` must remove duplicates. 

 

 Now create the following repo versions: 

 

 0: empty inital repo version 

 

 1: Post to /modify adding "c1". Expected content "c1" 

 

 2: Post to /modify adding "c2". Expected content "c2" (c1 has to removed because we are adding newer conflicting content) 

 

 3: Post to /modify adding "c2" to base_version 1. Note that, semantically, this is exactly the same operation as the one in version 2, i.e. add "c2" to a repo version containing "c1". Expected content "c2" (c1 has to removed because we are adding newer conflicting content) 

 

 However, version 3 contains c1! 

 

 # Reproducer 

 

 Run the following script on a Pulp3 install with an empty DB. 

 

 ~~~text 
 
 #!/usr/bin/env bash 
 
 set -e 

 

 echo "Setting environment variables for default hostname/port for the API and the Content app" 
 
 export BASE_ADDR=${BASE_ADDR:-http://localhost:24817} 
 
 export CONTENT_ADDR=${CONTENT_ADDR:-http://localhost:24816} 

 

 # Necessary for `django-admin` 
 
 export DJANGO_SETTINGS_MODULE=pulpcore.app.settings 

 

 # Poll a Pulp task until it is finished. 
 
 wait_until_task_finished() { 
     
     echo "Polling the task until it has reached a final state." 
     
     local task_url=$1 
     
     while true 
     
     do 
         
         local response=$(http $task_url) 
         
         local state=$(jq -r .state <<< ${response}) 
         
         jq . <<< "${response}" 
         
         case ${state} in 
             
             failed|canceled) 
                 
                 echo "Task in final state: ${state}" 
                 
                 exit 1 
                 
                 ;; 
             
             completed) 
                 
                 echo "$task_url complete." 
                 
                 break 
                 
                 ;; 
             
             *) 
                 
                 echo "Still waiting..." 
                 
                 sleep 1 
                 
                 ;; 
         
         esac 
     
     done 
 
 } 

 

 echo "Creating a file "1" at path FILE_CONTENT to upload." 
 
 export FILE_CONTENT="1" 
 
 echo $FILE_CONTENT > test_upload.txt 
 
 DIGEST1=4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865 

 

 echo "Uploading the file to Pulp, creating an artifact, storing ARTIFACT1_HREF." 
 
 export ARTIFACT1_HREF=$(http --form POST $BASE_ADDR/pulp/api/v3/artifacts/ \ 
     
     file@./test_upload.txt \ 
     
     | jq -r '.pulp_href') 

 

 echo "Inspecting new artifact." 
 
 http $BASE_ADDR$ARTIFACT1_HREF 

 

 echo 'Create File Content from the artifact and save as environment variable' 
 
 export TASK_URL=$(http POST $BASE_ADDR/pulp/api/v3/content/file/files/ \ 
     
     relative_path="test_upload.txt" \ 
     
     artifact=$ARTIFACT1_HREF \ 
     
     | jq -r '.task') 

 

 wait_until_task_finished $BASE_ADDR$TASK_URL 

 

 export CONTENT1_HREF=$(http $BASE_ADDR$TASK_URL| jq -r '.created_resources | first') 

 

 echo "Inspecting new file content 1" 
 
 http $BASE_ADDR$CONTENT1_HREF 

 

 echo "Creating a file "2" at path FILE_CONTENT to upload." 
 
 export FILE_CONTENT="2" 
 
 echo $FILE_CONTENT > test_upload.txt 

 

 echo "Uploading the file to Pulp, creating an artifact, storing ARTIFACT1_HREF." 
 
 export ARTIFACT2_HREF=$(http --form POST $BASE_ADDR/pulp/api/v3/artifacts/ \ 
     
     file@./test_upload.txt \ 
     
     | jq -r '.pulp_href') 

 

 echo "Inspecting new artifact." 
 
 http $BASE_ADDR$ARTIFACT2_HREF 

 

 echo 'Create File Content from the artifact and save as environment variable' 
 
 export TASK_URL=$(http POST $BASE_ADDR/pulp/api/v3/content/file/files/ \ 
     
     relative_path="test_upload.txt" \ 
     
     artifact=$ARTIFACT2_HREF \ 
     
     | jq -r '.task') 

 

 wait_until_task_finished $BASE_ADDR$TASK_URL 

 

 export CONTENT2_HREF=$(http $BASE_ADDR$TASK_URL| jq -r '.created_resources | first') 

 

 echo "Inspecting new file content 2" 
 
 http $BASE_ADDR$CONTENT2_HREF 

 

 export REPO_NAME=dup 

 

 echo "Creating a new repository named $REPO_NAME." 
 
 export REPO_HREF=$(http POST $BASE_ADDR/pulp/api/v3/repositories/file/file/ name=$REPO_NAME \ 
   
   | jq -r '.pulp_href') 

 

 echo "Inspecting repository." 
 
 http $BASE_ADDR$REPO_HREF 

 

 echo "Kick off a task to add content 1 to a repository, storing TASK_URL env variable" 
 
 export TASK_URL=$(http POST $BASE_ADDR$REPO_HREF'modify/' \ 
     
     add_content_units:="[\"$CONTENT1_HREF\"]" \ 
     
     | jq -r '.task') 

 

 # Poll the task (here we use a function defined in docs/_scripts/base.sh) 
 
 wait_until_task_finished $BASE_ADDR$TASK_URL 

 

 echo "Retrieving REPOVERSION_HREF from task" 
 
 export REPOVERSION_HREF=$(http $BASE_ADDR$TASK_URL| jq -r '.created_resources | first') 
 
 export REPOVERSION_WITH_1_HREF=$REPOVERSION_HREF 

 

 echo "Inspecting repository version." 
 
 http $BASE_ADDR$REPOVERSION_HREF 

 

 echo "Inspecting the content of the repository version" 
 
 export FILE_CONTENT_REPO_VERSION_HREF=$(http $BASE_ADDR$REPOVERSION_HREF | jq -r '.content_summary.present["file.file"].href') 
 
 http $BASE_ADDR$FILE_CONTENT_REPO_VERSION_HREF 

 

 echo "Kick off a task to add content 2 to a repository, storing TASK_URL env variable" 
 
 export TASK_URL=$(http POST $BASE_ADDR$REPO_HREF'modify/' \ 
     
     add_content_units:="[\"$CONTENT2_HREF\"]" \ 
     
     | jq -r '.task') 

 

 # Poll the task (here we use a function defined in docs/_scripts/base.sh) 
 
 wait_until_task_finished $BASE_ADDR$TASK_URL 

 

 echo "Retrieving REPOVERSION_HREF from task" 
 
 export REPOVERSION_HREF=$(http $BASE_ADDR$TASK_URL| jq -r '.created_resources | first') 

 

 echo "Inspecting repository version." 
 
 http $BASE_ADDR$REPOVERSION_HREF 

 

 echo "Inspecting the content of the repository version" 
 
 export FILE_CONTENT_REPO_VERSION_HREF=$(http $BASE_ADDR$REPOVERSION_HREF | jq -r '.content_summary.present["file.file"].href') 
 
 http $BASE_ADDR$FILE_CONTENT_REPO_VERSION_HREF 

 

 echo "Kick off a task to add content 2 to the version containing 1, storing TASK_URL env variable" 
 
 export TASK_URL=$(http POST $BASE_ADDR$REPO_HREF'modify/' \ 
     
     add_content_units:="[\"$CONTENT2_HREF\"]" \ 
     
     base_version=$REPOVERSION_WITH_1_HREF \ 
     
     | jq -r '.task') 

 

 # Poll the task (here we use a function defined in docs/_scripts/base.sh) 
 
 wait_until_task_finished $BASE_ADDR$TASK_URL 

 

 echo "Retrieving REPOVERSION_HREF from task" 
 
 export REPOVERSION_HREF=$(http $BASE_ADDR$TASK_URL| jq -r '.created_resources | first') 

 

 echo "Inspecting repository version." 
 
 http $BASE_ADDR$REPOVERSION_HREF 

 

 echo "Inspecting the content of the repository version" 
 
 export FILE_CONTENT_REPO_VERSION_HREF=$(http $BASE_ADDR$REPOVERSION_HREF | jq -r '.content_summary.present["file.file"].href') 
 
 http $BASE_ADDR$FILE_CONTENT_REPO_VERSION_HREF 
 
 ~~~
Back
Project

Profile

Help

Pulp

Issue #5964
