|
# coding=utf-8
|
|
"""Basic test for rpm rsync distributor.
|
|
|
|
This test do basic publish operation in rsync distributor with rpm repository
|
|
which was previously published in yum_distributor
|
|
|
|
"""
|
|
from __future__ import unicode_literals
|
|
|
|
import json
|
|
from packaging.version import Version
|
|
import os
|
|
import sys
|
|
import unittest2
|
|
import pprint
|
|
|
|
from pulp_smash import api, config, utils, cli, selectors, exceptions
|
|
from pulp_smash.compat import urljoin
|
|
from pulp_smash.constants import REPOSITORY_PATH, RPM_FEED_URL
|
|
from pulp_smash.tests.rpm.api_v2.utils import gen_distributor, gen_repo
|
|
from pulp_smash.tests.rpm.utils import set_up_module
|
|
|
|
|
|
_REPO = None
|
|
|
|
|
|
def get_remote_repo_packages(yum_distributor, rsync_distributor):
|
|
"""Return 'ls' output of packages directory of repo in remote server."""
|
|
cli_client = cli.Client(config.get_config())
|
|
repo_relative = yum_distributor['config']['relative_url']
|
|
remote_root = rsync_distributor['config']['remote']['root']
|
|
ret = cli_client.run(['ls', '%s/%s' % (remote_root, repo_relative)])
|
|
if ret.returncode:
|
|
return (False, ret.stderr)
|
|
else:
|
|
return (True, ret.stdout)
|
|
|
|
|
|
def gen_rsync_distributor():
|
|
"""Return a dict for use in creating a rsync distributor."""
|
|
return {
|
|
'auto_publish': False,
|
|
'distributor_id': utils.uuid4(),
|
|
'distributor_type_id': 'rpm_rsync_distributor',
|
|
'distributor_config': {
|
|
'remote': {
|
|
'auth_type': 'password',
|
|
'ssh_identity_file': '/etc/rsync_key',
|
|
'ssh_user': 'cdn_user',
|
|
'ssh_password': 'cdn_user',
|
|
'root': '/home/cdn_user/cdn/',
|
|
'host': 'dev'},
|
|
'remote_units_path': 'content/origin/units/',
|
|
'http': True,
|
|
'https': False,
|
|
'handler_type': 'rsync'
|
|
},
|
|
}
|
|
|
|
|
|
def setUpModule(): # pylint:disable=invalid-name
|
|
"""Prepare remote environemnt for rsync distributor.
|
|
|
|
1. Add cdn_user
|
|
2. Setup password for cdn_user
|
|
3. Setup ssh rsa-key and key authentication for ssh
|
|
|
|
Skip all tests if rsync_distributor features is not testable in pulp.
|
|
"""
|
|
set_up_module()
|
|
cfg = config.get_config()
|
|
if cfg.version < Version('2.9'):
|
|
raise unittest2.SkipTest('This module requires Pulp 2.9 or greater.')
|
|
#if selectors.bug_is_untestable(1759, cfg.version):
|
|
# raise unittest2.SkipTest(
|
|
# 'https://pulp.plan.io/issues/1759 is not testable'
|
|
# )
|
|
|
|
cli_client = cli.Client(config.get_config(), cli.echo_handler)
|
|
cli_client.run(['adduser', 'cdn_user'])
|
|
cmd = cli_client.machine['echo']['cdn_user']
|
|
cmd |= cli_client.machine['passwd']['cdn_user']['--stdin']
|
|
cmd.run()
|
|
cli_client.run(['ssh-keygen', '-f', '/etc/rsync_key',
|
|
'-t', 'rsa', '-N', '', '-P', ''])
|
|
cli_client.run('chown apache /etc/rsync_key'.split())
|
|
cli_client.run('sudo -u cdn_user mkdir -p /home/cdn_user/.ssh/'.split())
|
|
cli_client.run(('sudo -u cdn_user cp /etc/rsync_key.pub ' +
|
|
'/home/cdn_user/.ssh/authorized_keys').split())
|
|
cli_client.run('chown cdn_user /home/cdn_user/.ssh/authorized_keys'.split())
|
|
|
|
|
|
def tearDownModule(): # pylint:disable=invalid-name
|
|
"""Delete the repository created by :meth:`setUpModule`."""
|
|
cli_client = cli.Client(config.get_config())
|
|
cli_client.run('userdel cdn_user'.split())
|
|
|
|
|
|
def get_repomd_xml_path(distributor_rel_url):
|
|
"""Construct the path to a repository's ``repomd.xml`` file.
|
|
|
|
:param distributor_rel_url: A distributor's ``relative_url`` option.
|
|
:returns: An string path to a ``repomd.xml`` file.
|
|
"""
|
|
return urljoin(
|
|
urljoin('/pulp/repos/', distributor_rel_url),
|
|
'repodata/repomd.xml',
|
|
)
|
|
|
|
|
|
|
|
class RSyncTestBase(utils.BaseAPITestCase):
|
|
def get_rsynced_units_content(self, units, root, content_unit_path):
|
|
cli_client = cli.Client(config.get_config())
|
|
remote_root = root
|
|
stats = {}
|
|
for unit in units:
|
|
storage_path = unit['metadata']['_storage_path']
|
|
rel_unit_path = storage_path.replace('/var/lib/pulp/content/units/', '')
|
|
remote_path = os.path.join(remote_root, content_unit_path,
|
|
rel_unit_path)
|
|
ret = cli_client.run(['stat', remote_path])
|
|
stats[unit['metadata']['_storage_path']] = ret.returncode
|
|
return stats
|
|
|
|
def get_rsynced_units_dest(self, units):
|
|
(success, ret) = get_remote_repo_packages(self.yum_distributor,
|
|
self.rsync_distributor)
|
|
repo_packages = set([x for x in ret.split('\n') if x])
|
|
db_packages = set([unit['metadata']['filename']
|
|
for unit in units])
|
|
return (db_packages - repo_packages, repo_packages - db_packages)
|
|
|
|
class TestRsyncDistributorDelete(RSyncTestBase):
|
|
"""Test if files are removed from remote if they are removed from pulp repo
|
|
|
|
Test sequence:
|
|
1. Create rpm repo with feed
|
|
2. Sync repo
|
|
4. Associate yum distributor with repository
|
|
4. Associate rsync distributor with repository
|
|
3. Get list of packages in the repository
|
|
4. Publish in yum distributor
|
|
5. Publish in rsync distributor
|
|
6. Update remote_content_location of rsync distributor
|
|
7. Publish in rsync distributor
|
|
|
|
T1. Test if all synced packages are available in remote_content_location
|
|
Test if all synced packages are available in repo destination
|
|
|
|
T2. Test if all synced packages are available in remote_content_location
|
|
repository after it was updated
|
|
Test if all synced packages are available in repo destination
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Create a repository with a distributor, and populate it.
|
|
|
|
Associate yum and rsync distributor to repository and publish
|
|
repository in those.
|
|
In addition, create several variables for use by the test methods.
|
|
"""
|
|
super(TestRsyncDistributorDelete, cls).setUpClass()
|
|
cls.responses = {}
|
|
|
|
client = api.Client(cls.cfg, api.json_handler)
|
|
body = gen_repo()
|
|
body['importer_config']['feed'] = RPM_FEED_URL
|
|
cls.repo_href = client.post(REPOSITORY_PATH, body)['_href']
|
|
cls.resources.add(cls.repo_href) # mark for deletion
|
|
cls.responses['sync'] = utils.sync_repo(cls.cfg, cls.repo_href)
|
|
rsync_dist = gen_rsync_distributor()
|
|
cls.yum_distributor = client.post(urljoin(cls.repo_href,
|
|
'distributors/'),
|
|
gen_distributor())
|
|
|
|
# Get contents of repository
|
|
cls.synced_units = client.post(
|
|
urljoin(cls.repo_href, 'search/units/'),
|
|
{'criteria': {}},
|
|
)
|
|
|
|
rsync_dist_conf = rsync_dist['distributor_config']
|
|
rsync_dist_conf['predistributor_id'] = cls.yum_distributor['id']
|
|
cls.rsync_distributor = client.post(urljoin(cls.repo_href,
|
|
'distributors/'),
|
|
rsync_dist)
|
|
|
|
cls.responses['rpm publish'] = client.post(
|
|
urljoin(cls.repo_href, 'actions/publish/'),
|
|
{'id': cls.yum_distributor['id']},
|
|
)
|
|
|
|
cls.responses['rsync publish'] = client.post(
|
|
urljoin(cls.repo_href, 'actions/publish/'),
|
|
{'id': cls.rsync_distributor['id']},
|
|
)
|
|
|
|
|
|
def test_01_removed_data(self):
|
|
""""Test if rpms are removed from remote server.
|
|
|
|
Test if packages are removed from the remote if they are
|
|
unassociated from the repo, published in yum and then published in
|
|
rsync distributor with delete == True
|
|
"""
|
|
|
|
db_rpm_units = []
|
|
for unit in self.synced_units:
|
|
if unit['metadata']['_content_type_id'] != 'rpm':
|
|
continue
|
|
db_rpm_units.append(unit)
|
|
|
|
filenames = []
|
|
for unit in db_rpm_units[:2]:
|
|
filenames.append(unit["metadata"]["filename"])
|
|
|
|
client = api.Client(self.cfg, api.json_handler)
|
|
|
|
criteria = {"type_ids": ["rpm"],
|
|
"filters": {"unit": {"filename": {"$in": filenames}}}}
|
|
report = client.post(urljoin(self.repo_href, 'actions/unassociate/'),
|
|
{"criteria": criteria})
|
|
unassociate_result = next(api.poll_spawned_tasks(self.cfg, report))
|
|
|
|
publish_ret1 = client.post(urljoin(self.repo_href, 'actions/publish/'),
|
|
{'id': self.yum_distributor['id']})
|
|
publish_ret2 = client.post(urljoin(self.repo_href, 'actions/publish/'),
|
|
{'id': self.rsync_distributor['id'],
|
|
'override_config': {"delete": True}})
|
|
|
|
(success, ret) = get_remote_repo_packages(self.yum_distributor,
|
|
self.rsync_distributor)
|
|
|
|
missing_in_remote, missing_in_db = self.get_rsynced_units_dest(db_rpm_units)
|
|
|
|
self.assertEqual(set(missing_in_remote), set(filenames))
|
|
self.assertEqual(set(missing_in_db), set(['repodata']), missing_in_db)
|
|
|
|
def test_02_repo_clean(self):
|
|
""""Test if it's possible to clean up whole repo and rsync empty.
|
|
|
|
Test if it's possible to rsync empty repo to remote server.
|
|
"""
|
|
|
|
db_rpm_units = []
|
|
filenames = []
|
|
for unit in self.synced_units:
|
|
if unit['metadata']['_content_type_id'] != 'rpm':
|
|
continue
|
|
db_rpm_units.append(unit)
|
|
filenames.append(unit["metadata"]["filename"])
|
|
client = api.Client(self.cfg, api.json_handler)
|
|
|
|
criteria = {"type_ids": ["rpm"]}
|
|
report = client.post(urljoin(self.repo_href, 'actions/unassociate/'),
|
|
{"criteria": criteria})
|
|
unassociate_result = next(api.poll_spawned_tasks(self.cfg, report))
|
|
|
|
publish_ret1 = client.post(urljoin(self.repo_href, 'actions/publish/'),
|
|
{'id': self.yum_distributor['id']})
|
|
print >> sys.stderr, publish_ret1
|
|
publish_ret2 = client.post(urljoin(self.repo_href, 'actions/publish/'),
|
|
{'id': self.rsync_distributor['id'],
|
|
'override_config': {"delete": True}})
|
|
print >> sys.stderr, publish_ret2
|
|
|
|
(success, ret) = get_remote_repo_packages(self.yum_distributor,
|
|
self.rsync_distributor)
|
|
|
|
missing_in_remote, missing_in_db = self.get_rsynced_units_dest(db_rpm_units)
|
|
|
|
self.assertEqual(set(missing_in_remote), set(filenames))
|
|
self.assertEqual(set(missing_in_db), set(['repodata']), missing_in_db)
|
.