#!/usr/bin/env python3 import subprocess, urllib.request, re, os, tarfile from html.parser import HTMLParser HG_URL = 'http://icedtea.classpath.org/hg/release/icedtea{}-forest-{}' DOWNLOAD_URL = 'http://icedtea.wildebeest.org/download/source/' DOWNLOAD_HTML = DOWNLOAD_URL + '?C=M;O=D' ICEDTEA_JDKS = [7] BUNDLES = ['openjdk', 'corba', 'jaxp', 'jaxws', 'jdk', 'langtools', 'hotspot'] SRC_PATH = './sources.nix' def get_output(cmd, env = None): try: proc = subprocess.Popen(cmd, env = env, stdout = subprocess.PIPE) out = proc.communicate()[0] except subprocess.CalledProcessError as e: return None return out.decode('utf-8').strip() def nix_prefetch_url(url): env = os.environ.copy() env['PRINT_PATH'] = '1' out = get_output(['nix-prefetch-url', url], env = env) return out.split('\n') def get_nix_attr(path, attr): out = get_output(['nix-instantiate', '--eval-only', '-A', attr, path]) if len(out) < 2 or out[0] != '"' or out[-1] != '"': raise Exception('Cannot find Nix attribute "{}" (parsing failure?)'.format(attr)) # Strip quotes return out[1:-1] def get_jdk_attr(jdk, attr): return get_nix_attr(SRC_PATH, 'icedtea{}.{}'.format(jdk, attr)) class Parser(HTMLParser): def __init__(self, link_regex): HTMLParser.__init__(self) self.regex = link_regex self.href = None self.version = None def handle_starttag(self, tag, attrs): if self.href != None or tag != 'a': return href = None for attr in attrs: if attr[0] == 'href': href = attr[1] if href == None: return m = re.match(self.regex, href) if m != None: self.href = href self.version = m.group(1) def get_latest_version_url(major): f = urllib.request.urlopen(DOWNLOAD_HTML) html = f.read().decode('utf-8') f.close() parser = Parser(r'^icedtea\d?-({}\.\d[\d.]*)\.tar\.xz$'.format(major)) parser.feed(html) parser.close() if parser.href == None: raise Exception('Error: could not find download url for major version "{}"'.format(major)) return parser.version, DOWNLOAD_URL + parser.href def get_old_bundle_attrs(jdk, bundle): attrs = {} for attr in ('changeset', 'url', 'sha256'): attrs[attr] = get_jdk_attr(jdk, 'bundles.{}.{}'.format(bundle, attr)) return attrs def get_old_attrs(jdk): attrs = {} for attr in ('branch', 'version', 'url', 'sha256'): attrs[attr] = get_jdk_attr(jdk, attr) attrs['bundles'] = {} for bundle in BUNDLES: attrs['bundles'][bundle] = get_old_bundle_attrs(jdk, bundle) return attrs def get_member_filename(tarball, name): for fname in tarball.getnames(): m = re.match(r'^icedtea\d?-\d[\d.]*/{}$'.format(name), fname) if m != None: return m.group(0) return None def get_member_file(tarball, name): path = get_member_filename(tarball, name) if path == None: raise Exception('Could not find "{}" inside tarball'.format(name)) f = tarball.extractfile(path) data = f.read().decode('utf-8') f.close() return data def get_new_bundle_attr(makefile, bundle, attr): var = '{}_{}'.format(bundle.upper(), attr.upper()) regex = r'^{} = (.*?)$'.format(var) m = re.search(regex, makefile, re.MULTILINE) if m == None: raise Exception('Could not find variable "{}" in Makefile.am'.format(var)) return m.group(1) def get_new_bundle_attrs(jdk, branch, path): hg_url = HG_URL.format(jdk, branch) attrs = {} print('Opening file: "{}"'.format(path)) tar = tarfile.open(name = path, mode = 'r:xz') makefile = get_member_file(tar, 'Makefile.am') hotspot_map = get_member_file(tar, 'hotspot.map') for bundle in BUNDLES: battrs = {} if bundle == 'hotspot': m = re.search(r'^default (.*?) (.*?) (.*?)$', hotspot_map, re.MULTILINE) if m == None: raise Exception('Could not find info for hotspot bundle in hotspot.map') battrs['url'] = '{}/archive/{}.tar.gz'.format(m.group(1), m.group(2)) battrs['changeset'] = m.group(2) battrs['sha256'] = m.group(3) attrs[bundle] = battrs continue changeset = get_new_bundle_attr(makefile, bundle, 'changeset') battrs['changeset'] = changeset battrs['sha256'] = get_new_bundle_attr(makefile, bundle, 'sha256sum') if bundle == 'openjdk': battrs['url'] = '{}/archive/{}.tar.gz'.format(hg_url, changeset) else: battrs['url'] = '{}/{}/archive/{}.tar.gz'.format(hg_url, bundle, changeset) attrs[bundle] = battrs tar.close() return attrs def get_new_attrs(jdk): print('Getting old attributes for JDK {}...'.format(jdk)) old_attrs = get_old_attrs(jdk) attrs = {} # The major version corresponds to a specific JDK (1 = OpenJDK6, 2 = OpenJDK7, 3 = OpenJDK8) major = jdk - 5 print('Getting latest version for JDK {}...'.format(jdk)) version, url = get_latest_version_url(major) print() print('Old version: {}'.format(old_attrs['version'])) print('New version: {}'.format(version)) print() if version == old_attrs['version']: print('No update available, skipping...') print() return old_attrs print('Update available, generating new attributes for JDK {}...'.format(jdk)) attrs['version'] = version attrs['branch'] = '.'.join(version.split('.')[:2]) attrs['url'] = url print('Downloading tarball from url "{}"...'.format(url)) print() attrs['sha256'], path = nix_prefetch_url(url) print() print('Inspecting tarball for bundle information...') attrs['bundles'] = get_new_bundle_attrs(jdk, attrs['branch'], path) print('Done!') return attrs def generate_jdk(jdk): attrs = get_new_attrs(jdk) branch = attrs['branch'] src_version = attrs['version'].replace(branch, '${branch}') src_url = attrs['url'].replace(attrs['version'], '${version}') hg_url = HG_URL.format(jdk, branch) src_hg_url = HG_URL.format(jdk, '${branch}') src = ' icedtea{} = rec {{\n'.format(jdk) src += ' branch = "{}";\n'.format(branch) src += ' version = "{}";\n'.format(src_version) src += '\n' src += ' url = "{}";\n'.format(src_url) src += ' sha256 = "{}";\n'.format(attrs['sha256']) src += '\n' src += ' hg_url = "{}";\n'.format(src_hg_url) src += '\n' src += ' bundles = {\n' for bundle in BUNDLES: battrs = attrs['bundles'][bundle] b_url = battrs['url'] b_url = b_url.replace(hg_url, '${hg_url}') b_url = b_url.replace(battrs['changeset'], '${changeset}') src += ' {} = rec {{\n'.format(bundle) src += ' changeset = "{}";\n'.format(battrs['changeset']) src += ' url = "{}";\n'.format(b_url) src += ' sha256 = "{}";\n'.format(battrs['sha256']) src += ' };\n' if bundle != BUNDLES[-1]: src += '\n' src += ' };\n' src += ' };\n' return src def generate_sources(jdks): src = '# This file is autogenerated from update.py in the same directory.\n' src += '{\n' for jdk in jdks: print() print('Generating sources for JDK {}...'.format(jdk)) src += generate_jdk(jdk) src += '}\n' return src if __name__ == '__main__': print('Generating {}...'.format(SRC_PATH)) src = generate_sources(ICEDTEA_JDKS) f = open(SRC_PATH, 'w', encoding = 'utf-8') f.write(src) f.close() print() print('Update complete!')