|
"""
|
|
This module proves that the json metadata for each package has identical keys
|
|
"""
|
|
import pprint
|
|
import requests
|
|
from twine import package as tp
|
|
|
|
import os
|
|
|
|
|
|
packages = [
|
|
# 'scipy',
|
|
# 'numpy', 'shelf-reader', 'django',
|
|
# "0-._.-._.-._.-._.-._.-._.-0",
|
|
# "00SMALINUX",
|
|
# "01changer",
|
|
# "02exercicio",
|
|
# "0805nexter",
|
|
# "0x10c-asm",
|
|
# "1",
|
|
# "1020-nester",
|
|
# "115wangpan",
|
|
# "131228_pytest_1",
|
|
# "1337",
|
|
# "17MonIP",
|
|
|
|
# "18-e",
|
|
# "199Fix",
|
|
# "1ee",
|
|
# "1pass",
|
|
# "1to001",
|
|
# "2",
|
|
# "2013007_pyh",
|
|
# "2112",
|
|
# "2969nester",
|
|
# "2C.py",
|
|
# "2factorcli",
|
|
# "2gis",
|
|
# "2lazy2rest",
|
|
# "2mp3",
|
|
# "2or3",
|
|
# "3-1",
|
|
# "36ban_commons",
|
|
# "3color-Press",
|
|
# "3d-wallet-generator",
|
|
# "3lwg",
|
|
# "3t",
|
|
# "3to2",
|
|
# "3to2_py3k",
|
|
# "3xsd",
|
|
# "42",
|
|
# "42cc-pystyle",
|
|
# "42qucc",
|
|
# "4ch",
|
|
# "4chan",
|
|
# "4chandownloader",
|
|
# "4scanner",
|
|
# "4Suite",
|
|
|
|
|
|
"IPGenerator",
|
|
"ipgeo",
|
|
"ipgeobase-importer",
|
|
"ipget",
|
|
"ipgetter",
|
|
"ipgettercarlos",
|
|
"ipgroup",
|
|
"iphoto",
|
|
"ipify",
|
|
"ipinfo",
|
|
"ip_info",
|
|
"ipipe",
|
|
"ipkg",
|
|
"ipkiss24ce",
|
|
"IPKISS_Manager",
|
|
"IPlantUML",
|
|
"iplib",
|
|
"iplocate",
|
|
"iplocation",
|
|
"iplotter",
|
|
"ipm",
|
|
"ipmisim",
|
|
"ipmitools",
|
|
"ipmongo",
|
|
"ipmsg",
|
|
"ipnb",
|
|
"ipnext.site.editablefooter",
|
|
"ipnext.viewlet.blog",
|
|
"ipngnc",
|
|
"ipngxr2d",
|
|
"ipodio",
|
|
"ipodshuffle",
|
|
]
|
|
dl_urls = []
|
|
skipped = []
|
|
|
|
|
|
# Get a list of keys from json
|
|
def json_keys(packages_list):
|
|
all_keys = set()
|
|
for pkg_name in packages_list:
|
|
resp = requests.get(
|
|
'https://pypi.python.org/pypi/{pkg_name}/json/'.format(pkg_name=pkg_name))
|
|
try:
|
|
keys = resp.json()['info'].keys()
|
|
all_keys = all_keys.union(keys)
|
|
except Exception, e:
|
|
skipped.append(pkg_name)
|
|
continue
|
|
|
|
for version, files_list in resp.json()['releases'].iteritems():
|
|
for file_dict in files_list:
|
|
inner_keys = file_dict.keys()
|
|
dl_urls.append(file_dict['url'])
|
|
all_keys = all_keys.union(inner_keys)
|
|
return all_keys
|
|
|
|
|
|
keys_list = json_keys(packages)
|
|
keys_from_sources = {'json': keys_list}
|
|
|
|
# For each file, create a metadata dict and store its keys by metadata version
|
|
for i, url in enumerate(dl_urls):
|
|
filename = os.path.split(url)[-1]
|
|
if filename.endswith('.exe'):
|
|
continue
|
|
if not os.path.exists(filename):
|
|
print "downloading {} out of {}".format(i, str(len(dl_urls)))
|
|
resp = requests.get(url)
|
|
with open(filename, 'wb') as handle:
|
|
for block in resp.iter_content(1024):
|
|
handle.write(block)
|
|
else:
|
|
print "aleady downloaded {}".format(i)
|
|
|
|
print "twine is building package from {}".format(filename)
|
|
try:
|
|
twine_p = tp.PackageFile.from_filename(filename, comment='')
|
|
twine_md = twine_p.metadata_dictionary()
|
|
md_ver = twine_md['metadata_version']
|
|
keys_from_sources[md_ver] = keys_from_sources.get(md_ver) or set()
|
|
keys_from_sources[md_ver] = keys_from_sources[md_ver].union(set(twine_md.keys()))
|
|
except:
|
|
pass
|
|
|
|
print "\n\nSkipped:"
|
|
for s in skipped:
|
|
print s
|
|
|
|
print keys_from_sources
|
|
|
|
pp = pprint.PrettyPrinter(indent=4)
|
|
pp.pprint(keys_from_sources)
|
.