diff --git a/pkgs/development/compilers/icedtea/update.py b/pkgs/development/compilers/icedtea/update.py new file mode 100755 index 00000000000..ba3c5fc51ad --- /dev/null +++ b/pkgs/development/compilers/icedtea/update.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 + +import subprocess, urllib.request, re, os, tarfile +from html.parser import HTMLParser + +HG_URL = 'http://icedtea.classpath.org/hg/release/icedtea{}-forest-{}' +DOWNLOAD_URL = 'http://icedtea.wildebeest.org/download/source/' +DOWNLOAD_HTML = DOWNLOAD_URL + '?C=M;O=D' + +ICEDTEA_JDKS = [7] + +BUNDLES = ['openjdk', 'corba', 'jaxp', 'jaxws', 'jdk', 'langtools', 'hotspot'] + +SRC_PATH = './sources.nix' + +def get_output(cmd, env = None): + try: + proc = subprocess.Popen(cmd, env = env, stdout = subprocess.PIPE) + out = proc.communicate()[0] + except subprocess.CalledProcessError as e: + return None + + return out.decode('utf-8').strip() + +def nix_prefetch_url(url): + env = os.environ.copy() + env['PRINT_PATH'] = '1' + out = get_output(['nix-prefetch-url', url], env = env) + + return out.split('\n') + +def get_nix_attr(path, attr): + out = get_output(['nix-instantiate', '--eval-only', '-A', attr, path]) + + if len(out) < 2 or out[0] != '"' or out[-1] != '"': + raise Exception('Cannot find Nix attribute "{}" (parsing failure?)'.format(attr)) + + # Strip quotes + return out[1:-1] + +def get_jdk_attr(jdk, attr): + return get_nix_attr(SRC_PATH, 'icedtea{}.{}'.format(jdk, attr)) + +class Parser(HTMLParser): + def __init__(self, link_regex): + HTMLParser.__init__(self) + + self.regex = link_regex + self.href = None + self.version = None + + def handle_starttag(self, tag, attrs): + if self.href != None or tag != 'a': + return + + href = None + for attr in attrs: + if attr[0] == 'href': + href = attr[1] + if href == None: + return + + m = re.match(self.regex, href) + if m != None: + self.href = href + self.version = m.group(1) + +def get_latest_version_url(major): + f = urllib.request.urlopen(DOWNLOAD_HTML) + html = f.read().decode('utf-8') + f.close() + + parser = Parser(r'^icedtea\d?-({}\.\d[\d.]*)\.tar\.xz$'.format(major)) + parser.feed(html) + parser.close() + + if parser.href == None: + raise Exception('Error: could not find download url for major version "{}"'.format(major)) + + return parser.version, DOWNLOAD_URL + parser.href + +def get_old_bundle_attrs(jdk, bundle): + attrs = {} + for attr in ('changeset', 'url', 'sha256'): + attrs[attr] = get_jdk_attr(jdk, 'bundles.{}.{}'.format(bundle, attr)) + + return attrs + +def get_old_attrs(jdk): + attrs = {} + + for attr in ('branch', 'version', 'url', 'sha256'): + attrs[attr] = get_jdk_attr(jdk, attr) + + attrs['bundles'] = {} + + for bundle in BUNDLES: + attrs['bundles'][bundle] = get_old_bundle_attrs(jdk, bundle) + + return attrs + +def get_member_filename(tarball, name): + for fname in tarball.getnames(): + m = re.match(r'^icedtea\d?-\d[\d.]*/{}$'.format(name), fname) + if m != None: + return m.group(0) + + return None + +def get_member_file(tarball, name): + path = get_member_filename(tarball, name) + if path == None: + raise Exception('Could not find "{}" inside tarball'.format(name)) + + f = tarball.extractfile(path) + data = f.read().decode('utf-8') + f.close() + + return data + +def get_new_bundle_attr(makefile, bundle, attr): + var = '{}_{}'.format(bundle.upper(), attr.upper()) + regex = r'^{} = (.*?)$'.format(var) + + m = re.search(regex, makefile, re.MULTILINE) + if m == None: + raise Exception('Could not find variable "{}" in Makefile.am'.format(var)) + + return m.group(1) + +def get_new_bundle_attrs(jdk, branch, path): + hg_url = HG_URL.format(jdk, branch) + + attrs = {} + + print('Opening file: "{}"'.format(path)) + tar = tarfile.open(name = path, mode = 'r:xz') + + makefile = get_member_file(tar, 'Makefile.am') + hotspot_map = get_member_file(tar, 'hotspot.map') + + for bundle in BUNDLES: + battrs = {} + + if bundle == 'hotspot': + m = re.search(r'^default (.*?) (.*?) (.*?)$', hotspot_map, re.MULTILINE) + if m == None: + raise Exception('Could not find info for hotspot bundle in hotspot.map') + + battrs['url'] = '{}/archive/{}.tar.gz'.format(m.group(1), m.group(2)) + battrs['changeset'] = m.group(2) + battrs['sha256'] = m.group(3) + + attrs[bundle] = battrs + continue + + changeset = get_new_bundle_attr(makefile, bundle, 'changeset') + battrs['changeset'] = changeset + battrs['sha256'] = get_new_bundle_attr(makefile, bundle, 'sha256sum') + + if bundle == 'openjdk': + battrs['url'] = '{}/archive/{}.tar.gz'.format(hg_url, changeset) + else: + battrs['url'] = '{}/{}/archive/{}.tar.gz'.format(hg_url, bundle, changeset) + + attrs[bundle] = battrs + + tar.close() + + return attrs + +def get_new_attrs(jdk): + print('Getting old attributes for JDK {}...'.format(jdk)) + old_attrs = get_old_attrs(jdk) + attrs = {} + + # The major version corresponds to a specific JDK (1 = OpenJDK6, 2 = OpenJDK7, 3 = OpenJDK8) + major = jdk - 5 + + print('Getting latest version for JDK {}...'.format(jdk)) + version, url = get_latest_version_url(major) + + print() + print('Old version: {}'.format(old_attrs['version'])) + print('New version: {}'.format(version)) + print() + + if version == old_attrs['version']: + print('No update available, skipping...') + print() + return old_attrs + + print('Update available, generating new attributes for JDK {}...'.format(jdk)) + + attrs['version'] = version + attrs['branch'] = '.'.join(version.split('.')[:2]) + attrs['url'] = url + + print('Downloading tarball from url "{}"...'.format(url)) + print() + attrs['sha256'], path = nix_prefetch_url(url) + print() + + print('Inspecting tarball for bundle information...') + + attrs['bundles'] = get_new_bundle_attrs(jdk, attrs['branch'], path) + + print('Done!') + + return attrs + +def generate_jdk(jdk): + attrs = get_new_attrs(jdk) + + branch = attrs['branch'] + src_version = attrs['version'].replace(branch, '${branch}') + src_url = attrs['url'].replace(attrs['version'], '${version}') + + hg_url = HG_URL.format(jdk, branch) + src_hg_url = HG_URL.format(jdk, '${branch}') + + src = ' icedtea{} = rec {{\n'.format(jdk) + src += ' branch = "{}";\n'.format(branch) + src += ' version = "{}";\n'.format(src_version) + src += '\n' + src += ' url = "{}";\n'.format(src_url) + src += ' sha256 = "{}";\n'.format(attrs['sha256']) + src += '\n' + src += ' hg_url = "{}";\n'.format(src_hg_url) + src += '\n' + src += ' bundles = {\n' + + for bundle in BUNDLES: + battrs = attrs['bundles'][bundle] + + b_url = battrs['url'] + b_url = b_url.replace(hg_url, '${hg_url}') + b_url = b_url.replace(battrs['changeset'], '${changeset}') + + src += ' {} = rec {{\n'.format(bundle) + src += ' changeset = "{}";\n'.format(battrs['changeset']) + src += ' url = "{}";\n'.format(b_url) + src += ' sha256 = "{}";\n'.format(battrs['sha256']) + src += ' };\n' + + if bundle != BUNDLES[-1]: + src += '\n' + + src += ' };\n' + src += ' };\n' + + return src + +def generate_sources(jdks): + src = '# This file is autogenerated from update.py in the same directory.\n' + src += '{\n' + + for jdk in jdks: + print() + print('Generating sources for JDK {}...'.format(jdk)) + src += generate_jdk(jdk) + + src += '}\n' + return src + +if __name__ == '__main__': + print('Generating {}...'.format(SRC_PATH)) + src = generate_sources(ICEDTEA_JDKS) + + f = open(SRC_PATH, 'w', encoding = 'utf-8') + f.write(src) + f.close() + + print() + print('Update complete!')