Project

Profile

Help

Task #2449 » consumer_applicability_stats.py

To get applicability related stats from db - ttereshc, 07/04/2017 12:14 AM

 
1
import logging
2
import logging.handlers
3

    
4
from pulp.server.db import connection
5

    
6
_logger = logging.getLogger(__name__)
7
_logger.setLevel(logging.INFO)
8
_logger.addHandler(logging.handlers.SysLogHandler(address='/dev/log'))
9

    
10
connection.initialize()
11
db = connection.get_database()
12
consumers_c = db['consumers']
13
bindings_c = db['consumer_bindings']
14
profiles_c = db['consumer_unit_profiles']
15
repos_c = db['repos']
16
rpa_c = db['repo_profile_applicability']
17
rcu_c = db['repo_content_units']
18

    
19

    
20
def get_consumer_stats():
21
    unique_consumers_in_profiles = profiles_c.distinct('consumer_id')
22
    consumers_in_consumers = consumers_c.distinct('id')
23
    no_profile_consumers = list(set(consumers_in_consumers) - set(unique_consumers_in_profiles))
24
    unique_profiles = profiles_c.distinct('profile_hash')
25
    profiles_count = len(unique_profiles)
26
    profiles_packages_stats = []
27
    packages_set = set()
28
    for profile in unique_profiles:
29
        profile = profiles_c.find({'profile_hash': profile}, projection=['profile'])[0]
30
        profiles_packages_stats.append(len(profile['profile']))
31
        packages_set.update(tuple(pkg.values())for pkg in profile['profile'])
32

    
33
    consumer_bindings = []
34
    consumer_bindings_stats = []
35
    for con in consumers_c.find(projection=['id']):
36
        con_bind = bindings_c.find({'consumer_id': con['id']}, projection=['repo_id'])
37
        con_bind = [c['repo_id'] for c in con_bind]
38
        if con_bind:
39
            consumer_bindings.append(con_bind)
40
            consumer_bindings_stats.append(len(con_bind))
41

    
42
    _logger.info('=== Consumers ===')
43
    _logger.info('Consumers: %s' % consumers_c.count())
44
    _logger.info('Consumers with profile: %s' % len(unique_consumers_in_profiles))
45
    _logger.info('Consumers without profile: %s' % len(no_profile_consumers))
46
    _logger.info('Consumer profiles, total: %s' % profiles_count)
47
    _logger.info('Consumer profiles, min/avg/max num of packages: %s/%s/%s' % (
48
        min(profiles_packages_stats),
49
        sum(profiles_packages_stats)/profiles_count,
50
        max(profiles_packages_stats)))
51
    _logger.info('Consumer profiles, total unique papckages: %s' % len(packages_set))
52
    _logger.info('Consumers with bindings, total: %s' % len(consumer_bindings_stats))
53
    _logger.info('Consumer bindings, total: %s' % bindings_c.count())
54
    _logger.info('Consumer bindings, min/avg/max per consumer: %s/%s/%s' % (
55
        min(consumer_bindings_stats),
56
        sum(consumer_bindings_stats)/len(consumer_bindings_stats),
57
        max(consumer_bindings_stats)))
58

    
59

    
60
def get_repository_stats():
61
    repos_repo_ids = rpa_c.distinct('repo_id')
62
    repo_rpm_counts = []
63
    repo_erratum_counts = []
64
    for repo_id in repos_repo_ids:
65
        rpm_count = rcu_c.find({'repo_id': repo_id, 'unit_type_id': 'rpm'}).count()
66
        erratum_count = rcu_c.find({'repo_id': repo_id, 'unit_type_id': 'erratum'}).count()
67
        repo_rpm_counts.append(rpm_count)
68
        repo_erratum_counts.append(erratum_count)
69

    
70
    _logger.info('=== Repositories ===')
71
    _logger.info('Repositories, total: %s' % len(repos_repo_ids))
72
    _logger.info('Repositories, N of RPMs, min/avg/max: %s/%s/%s' % (
73
        min(repo_rpm_counts),
74
        sum(repo_rpm_counts)/len(repo_rpm_counts),
75
        max(repo_rpm_counts)))
76
    _logger.info('Repositories, N of errata, min/avg/max: %s/%s/%s' % (
77
        min(repo_erratum_counts),
78
        sum(repo_erratum_counts)/len(repo_erratum_counts),
79
        max(repo_erratum_counts)))
80

    
81

    
82
def get_applicability_stats():
83
    rpa_repo_ids = rpa_c.distinct('repo_id')
84
    repos_repo_ids = repos_c.distinct('repo_id')
85
    missing_repo_ids = list(set(rpa_repo_ids) - set(repos_repo_ids))
86
    rpa_orphaned_by_repo_id = rpa_c.find({'repo_id': {'$in': missing_repo_ids}})
87
    rpa_profiles = rpa_c.distinct('profile_hash')
88
    profiles = profiles_c.distinct('profile_hash')
89
    missing_profiles = list(set(rpa_profiles) - set(profiles))
90
    rpa_orphaned_by_profile = rpa_c.find({'profile_hash': {'$in': missing_profiles}})
91

    
92
    actual_applicability_profiles = rpa_c.find({'profile_hash': {'$in': profiles}})
93

    
94
    _logger.info('=== Applicability ===')
95
    _logger.info('Repositories, total: %s' % len(repos_repo_ids))
96
    _logger.info('Repositories in applicability profiles: %s' % len(rpa_repo_ids))
97
    _logger.info('Repositories not in applicability profiles: %s' % len(missing_repo_ids))
98
    _logger.info('Consumer profiles, total: %s' % len(profiles))
99
    _logger.info('Consumer profiles with applicability: %s' % len(rpa_profiles))
100
    _logger.info('Consumer profiles without applicability: %s' % len(missing_profiles))
101
    _logger.info('Applicability profiles, total: %s' % rpa_c.count())
102
    _logger.info('Actual applicability profiles: %s' % actual_applicability_profiles.count())
103
    _logger.info('Orphaned applicability by repo: %s' % rpa_orphaned_by_repo_id.count())
104
    _logger.info('Orphaned applicability by consumer profile: %s' % rpa_orphaned_by_profile.count())
105

    
106

    
107
if __name__ == '__main__':
108
    get_consumer_stats()
109
    get_repository_stats()
110
    get_applicability_stats()
(1-1/2)