path: root/rel-html.py

#!/usr/bin/env python

# Provided an index URL and a few project hints page this
# will spit out a shiny HTML 5 W3C compliant releases page.

# Copyright (C) 2012-2014 Luis R. Rodriguez <mcgrof@do-not-panic.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from HTMLParser import HTMLParser
import urllib
import urllib2
import ConfigParser
import re
import sys
import os
import getopt
from operator import itemgetter
from datetime import date

debug = 0


def rel_html_license_verbose():
    print '-----------------------------------------------------------------------'
    print 'Copyright (C) 2012-2014 Luis R. Rodriguez <mcgrof@do-not-panic.com>'
    print ''
    print 'This program is free software: you can redistribute it and/or modify'
    print 'it under the terms of the GNU Affero General Public License as'
    print 'published by the Free Software Foundation, either version 3 of the'
    print 'License, or (at your option) any later version.'
    print ''
    print 'This program is distributed in the hope that it will be useful,'
    print 'but WITHOUT ANY WARRANTY; without even the implied warranty of'
    print 'MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the'
    print 'GNU Affero General Public License for more details.'
    print ''
    print 'You should have received a copy of the GNU Affero General Public License'
    print 'along with this program.  If not, see <http://www.gnu.org/licenses/>.'
    print '-----------------------------------------------------------------------'


def rel_html_license():
    return "AGPL"


def rel_html_href():
    return '<a href="https://github.com/mcgrof/rel-html">rel-html</a>'


def __compute_rel_weight(rel_specs):
    weight = 0
    extra = 0
    sublevel = 0
    relmod = 0
    relmod_type = 0

    if (debug):
        sys.stdout.write("VERSION       = %s\n" % rel_specs['VERSION'])
        sys.stdout.write("PATCHLEVEL    = %s\n" % rel_specs['PATCHLEVEL'])
        sys.stdout.write("SUBLEVEL      = %s\n" % rel_specs['SUBLEVEL'])
        sys.stdout.write("EXTRAVERSION  = %s\n" % rel_specs['EXTRAVERSION'])
        sys.stdout.write("RELMOD_UPDATE = %s\n" % rel_specs['RELMOD_UPDATE'])
        sys.stdout.write("RELMOD_TYPE   = %s\n" % rel_specs['RELMOD_TYPE'])

    if (rel_specs['EXTRAVERSION'] != ''):
        if ("." in rel_specs['EXTRAVERSION'] or
                "rc" in rel_specs['EXTRAVERSION']):
            rc = rel_specs['EXTRAVERSION'].lstrip("-rc")
            if (rc == ""):
                rc = 0
            else:
                rc = int(rc) - 20
            extra = int(rc)
        else:
            extra = int(rel_specs['EXTRAVERSION']) + 10

    if (rel_specs['SUBLEVEL'] != ''):
        sublevel = int(rel_specs['SUBLEVEL'].lstrip(".")) * 20
    else:
        sublevel = 5

    if (rel_specs['RELMOD_UPDATE'] != ''):
        mod = rel_specs['RELMOD_UPDATE']
        if (mod == ""):
            mod = 0
        else:
            mod = int(mod)
        relmod = int(mod)

    if (rel_specs['RELMOD_TYPE'] != ''):
        rtype = rel_specs['RELMOD_TYPE']
        if ("c" in rtype):
            relmod_type = relmod_type + 6
        if ("u" in rtype):
            relmod_type = relmod_type + 7
        if ("p" in rtype):
            relmod_type = relmod_type + 8
        if ("n" in rtype):
            relmod_type = relmod_type + 9
        if ("s" in rtype):
            relmod_type = relmod_type + 10

    weight = (int(rel_specs['VERSION'])    << 32) + \
             (int(rel_specs['PATCHLEVEL']) << 16) + \
             (sublevel   		       << 8 ) + \
             (extra * 60) + (relmod * 2) + relmod_type

    return weight


def get_rel_spec(rel):
    if ("rc" in rel):
        m = re.match(r"v*(?P<VERSION>\d+)\.+"
                     "(?P<PATCHLEVEL>\d+)[.]*"
                     "(?P<SUBLEVEL>\d*)"
                     "(?P<EXTRAVERSION>[-rc]+\w*)\-*"
                     "(?P<RELMOD_UPDATE>\d*)[-]*"
                     "(?P<RELMOD_TYPE>[usnpc]*)",
                     rel)
    else:
        m = re.match(r"v*(?P<VERSION>\d+)\.+"
                     "(?P<PATCHLEVEL>\d+)[.]*"
                     "(?P<SUBLEVEL>\d*)[.]*"
                     "(?P<EXTRAVERSION>\w*)\-*"
                     "(?P<RELMOD_UPDATE>\d*)[-]*"
                     "(?P<RELMOD_TYPE>[usnpc]*)",
                     rel)
    if (not m):
        return m
    rel_specs = m.groupdict()
    return rel_specs


def compute_rel_weight(rel):
    rel_specs = get_rel_spec(rel)
    if (not rel_specs):
        return 0
    return __compute_rel_weight(rel_specs)


def __compute_rel_weight_next(rel_specs):
    weight = 0
    date = 0
    relmod_update = 0
    relmod_type = 0

    if (debug):
        sys.stdout.write("DATE_VERSION  = %s\n" % rel_specs['DATE_VERSION'])
        sys.stdout.write("RELMOD_UPDATE = %s\n" % rel_specs['RELMOD_UPDATE'])
        sys.stdout.write("RELMOD_TYPE   = %s\n" % rel_specs['RELMOD_TYPE'])

    if (rel_specs['DATE_VERSION'] != ''):
        date = rel_specs['DATE_VERSION'].lstrip("rc")
        date = int(date.translate(None, "-"))

    if (rel_specs['RELMOD_UPDATE'] != ''):
        mod = rel_specs['RELMOD_UPDATE']
        if (mod == ""):
            mod = 0
        else:
            mod = int(mod)
        relmod = int(mod)

    if (rel_specs['RELMOD_TYPE'] != ''):
        rtype = rel_specs['RELMOD_TYPE']
        if ("c" in rtype):
            relmod_type = relmod_type + 6
        if ("u" in rtype):
            relmod_type = relmod_type + 7
        if ("p" in rtype):
            relmod_type = relmod_type + 8
        if ("n" in rtype):
            relmod_type = relmod_type + 9
        if ("s" in rtype):
            relmod_type = relmod_type + 10

    weight = (999                << 32) + \
             (int(date)          << 16) + \
             (int(relmod_update) << 8 ) + \
             (int(relmod_type))

    return weight


def compute_rel_weight_next(rel):
    if (len(rel) < 4):
        return 0
    if (rel[4] == "-"):
        m = re.match(r"v*(?P<DATE_VERSION>\w+-*\w*-*\w*)[-]*"
                     "(?P<RELMOD_UPDATE>\d*)[-]*"
                     "(?P<RELMOD_TYPE>[usnpc]*)",
                     rel)
    else:
        m = re.match(r"v*(?P<DATE_VERSION>\w+)[-]*"
                     "(?P<RELMOD_UPDATE>\d*)[-]*"
                     "(?P<RELMOD_TYPE>[usnpc]*)",
                     rel)
    if (not m):
        return 0

    rel_specs = m.groupdict()

    return __compute_rel_weight_next(rel_specs)


def sort_rels_weight(col):
    return sorted(col, key=lambda k: k['weight'], reverse=True)


class index_tarball_hunter(HTMLParser):

    "Goes through an index page with releases and adds tarball targets to the index parser"

    def parse(self, html):
        "Parse the given string 's'."
        self.feed(html)
        self.close()

    def handle_decl(self, decl):
        pass

    def tarball_add_stable(self, t_new):
        s_new = t_new.get('specifics')
        for t_old in self.tarballs:
            s_old = t_old.get('specifics')
            idx = self.tarballs.index(t_old)

            if (s_old['VERSION'] != s_new['VERSION']):
                break

            if (s_old['PATCHLEVEL'] != s_new['PATCHLEVEL']):
                break

            if (s_old['EXTRAVERSION'] != '' and
                    s_new['EXTRAVERSION'] != ''):
                if (s_new['EXTRAVERSION'] > s_old['EXTRAVERSION']):
                    self.tarballs.remove(t_old)
                    self.tarballs.insert(idx, t_new)
                    return
                if (s_new['EXTRAVERSION'] < s_old['EXTRAVERSION']):
                    return
            if (s_old['RELMOD_UPDATE'] != '' and
                    s_new['RELMOD_UPDATE'] != ''):
                if (s_old['RELMOD_UPDATE'] == s_new['RELMOD_UPDATE']):
                    if (s_new['RELMOD_TYPE'] == ''):
                        self.tarballs.insert(idx - 1, t_new)
                        return
                    self.tarballs.append(t_new)
                    return
                if (s_new['RELMOD_UPDATE'] > s_old['RELMOD_UPDATE']):
                    self.tarballs.remove(t_old)
                    self.tarballs.insert(idx, t_new)
                    return
                else:
                    return
        self.tarballs.append(t_new)

    def tarball_add_next(self, t_new):
        index_parser = self.index_parser
        s_new = t_new.get('specifics')
        for t_old in self.tarballs:
            s_old = t_old.get('specifics')
            idx = self.tarballs.index(t_old)
            for next_date in index_parser.next_rel_dates:
                if (next_date in t_old.get('rel')):
                    self.tarballs.insert(idx - 1, t_new)
                    return
        self.tarballs.append(t_new)

    def is_rel_eol(self, rel_specs):
        index_parser = self.index_parser
        for eol in index_parser.eol:
            m = re.match(r"v*(?P<VERSION>\d+)\.+"
                         "(?P<PATCHLEVEL>\d+)[.]*"
                         "(?P<SUBLEVEL>\w*)[.-]*"
                         "(?P<EXTRAVERSION>\w*)",
                         eol)
            if (not m):
                continue
            eol_specs = m.groupdict()
            if (eol_specs['VERSION'] == rel_specs['VERSION'] and
                    eol_specs['PATCHLEVEL'] == rel_specs['PATCHLEVEL']):
                return True
        return False

    def get_rel_match(self, value):
        index_parser = self.index_parser
        rel_match = dict(m=None, rel_name="")
        for rel_name in index_parser.rel_names:
            m = re.match(r'' + rel_name + '-+'
                         "v*(?P<VERSION>\d+)\.+"
                         "(?P<PATCHLEVEL>\d+)\.*"
                         "(?P<SUBLEVEL>\w*)[.-]*"
                         "(?P<EXTRAVERSION>\w*)[-]*"
                         "(?P<RELMOD_UPDATE>\d*)[-]*"
                         "(?P<RELMOD_TYPE>[usnpc]*)",
                         value)
            if (m):
                rel_match['m'] = m
                rel_match['rel_name'] = rel_name
                return rel_match
        return rel_match

    def get_rel_match_next(self, value):
        index_parser = self.index_parser
        rel_match = dict(m=None, rel_name="")
        for rel_name in index_parser.rel_names:
            for next_date in index_parser.next_rel_dates:
                m = re.match(r'' + rel_name + '+'
                             + '\-(?P<DATE_VERSION>' + next_date + '+)'
                             + '\-*(?P<EXTRAVERSION>\d*)'
                             + '\-*(?P<RELMOD>\w*)',
                             value)
                if (m):
                    rel_match['m'] = m
                    rel_match['rel_name'] = rel_name
                    return rel_match
        return rel_match

    def get_rel_name(self, value):
        for rel in self.releases:
            if (rel in value):
                return rel
        return ""

    def update_latest_tarball_stable(self, value):
        index_parser = self.index_parser
        release = self.get_rel_name(value)
        if (not release):
            return
        if ('tar.sign' in value):
            return
        if (index_parser.release_extension not in value):
            return

        rel_match = self.get_rel_match(value)
        if (not rel_match['m']):
            return

        rel_specifics = rel_match['m'].groupdict()

        supported = True
        if (self.is_rel_eol(rel_specifics)):
            supported = False

        p = re.compile(index_parser.release_extension + '$')
        rel_name = p.sub("", value)

        base_release = ''
        for r in index_parser.supported:
            if (r in value):
                base_release = r
                break

        ver = rel_name.lstrip(rel_match['rel_name'] + '-')

        p = re.compile('-[usnpc]*$')
        short_ver = p.sub("", ver)
        tmp_changelog = 'ChangeLog-' + short_ver
        tmp_changelog_signed = tmp_changelog + ".sign"

        if (index_parser.ignore_changelogs):
            chanlog_req = False
        else:
            if ("rc" in ver):
                chanlog_req = False
            else:
                chanlog_req = True

        w = compute_rel_weight(ver)
        if (not w):
            return

        tar = dict(version=short_ver,
                   base_release=r,
                   weight=w,
                   rel=rel_name,
                   specifics=rel_specifics,
                   base_url=self.base_url,
                   base_url_validated=False,
                   url=self.base_url + '/' + value,
                   maintained=supported,
                   longterm=False,
                   next_rel=False,
                   tarball=value,
                   tarball_exists=True,
                   ignore_signature=index_parser.ignore_signatures,
                   signed_tarball=rel_name + '.tar.sign',
                   signed_tarball_exists=False,
                   changelog=tmp_changelog,
                   changelog_url=self.base_url + '/' + tmp_changelog,
                   changelog_exists=False,
                   changelog_required=chanlog_req,
                   signed_changelog=tmp_changelog_signed,
                   signed_changelog_exists=False,
                   verified=False)

        self.tarball_add_stable(tar)

    def update_latest_tarball_next(self, value):
        index_parser = self.index_parser
        rel_match = self.get_rel_match_next(value)
        if (not rel_match['m']):
            return

        rel_specifics = rel_match['m'].groupdict()

        rel_name_next = rel_match[
            'rel_name'] + '-' + rel_specifics['DATE_VERSION']
        next_version = rel_specifics['DATE_VERSION']

        if (rel_specifics['EXTRAVERSION'] != ''):
            rel_name_next = rel_name_next + '-' + rel_specifics['EXTRAVERSION']
            next_version = next_version + '-' + rel_specifics['EXTRAVERSION']

        if (rel_specifics['RELMOD'] != ''):
            rel_name_next = rel_name_next + '-' + rel_specifics['RELMOD']
            next_version = next_version + '-' + rel_specifics['RELMOD']

        tar_next = rel_name_next + index_parser.release_extension
        s_tarball_next = rel_name_next + ".tar.sign"

        tmp_changelog = 'ChangeLog-' + next_version
        tmp_changelog_signed = 'ChangeLog-' + next_version + ".sign"

        w = compute_rel_weight_next(next_version)

        tar_next = dict(version=next_version,
                        weight=w,
                        rel=rel_name_next,
                        url=self.base_url + '/' + tar_next,
                        specifics=rel_specifics,
                        base_url=self.base_url,
                        base_url_validated=False,
                        maintained=True,
                        longterm=False,
                        next_rel=True,
                        tarball=tar_next,
                        tarball_exists=True,
                        ignore_signature=index_parser.ignore_signatures,
                        signed_tarball=s_tarball_next,
                        signed_tarball_exists=False,
                        changelog=tmp_changelog,
                        changelog_url=self.base_url + '/' + tmp_changelog,
                        changelog_exists=False,
                        changelog_required=False,
                        signed_changelog=tmp_changelog_signed,
                        signed_changelog_exists=False,
                        verified=False)
        self.tarball_add_next(tar_next)

    def print_tarballs(self):
        for tar in self.tarballs:
            specifics = tar.get('specifics')
            sys.stdout.write("Tarball: %s<br>----extra: %s mod_update: %s mod_type: %s<br>" %
                             (tar.get('url'),
                              specifics['EXTRAVERSION'],
                              specifics['RELMOD_UPDATE'],
                              specifics['RELMOD_TYPE']))

    def is_biggest_tarball(self, pivot_tar, tars):
        other_greater = 0
        for tar in tars:
            if pivot_tar.get('rel') == tar.get('rel'):
                continue
            if pivot_tar.get('base_release') == tar.get('base_release'):
                if pivot_tar.get('weight') < tar.get('weight'):
                    other_greater = other_greater + 1

        if (other_greater == 0):
            return True

        return False

    def update_rel_candidates(self):
        index_parser = self.index_parser
        for tar in self.tarballs:
            index_parser.rel_html_rels.append(tar)
        for tar in index_parser.rel_html_rels:
            if not self.is_biggest_tarball(tar, index_parser.rel_html_rels):
                index_parser.rel_html_rels.remove(tar)

    def is_next_rel(self, value):
        index_parser = self.index_parser
        for next_date in index_parser.next_rel_dates:
            if (next_date != '' and
                next_date in value and
                    index_parser.release_extension in value):
                return True
        return False

    def handle_starttag(self, tag, attributes):
        "Process a tags and its 'attributes'."
        index_parser = self.index_parser
        if tag != 'a':
            pass
        for name, value in attributes:
            if name != 'href':
                pass
            release = self.get_rel_name(value)
            if (release not in value):
                pass
            if (self.is_next_rel(value)):
                self.update_latest_tarball_next(value)
                pass

            self.update_latest_tarball_stable(value)

    def handle_endtag(self, tag):
        pass

    def handle_data(self, data):
        pass

    def handle_comment(self, data):
        pass

    def __init__(self, index_parser, releases, url):
        HTMLParser.__init__(self)
        self.index_parser = index_parser
        self.base_url = url.rstrip("/")
        self.releases = releases
        self.tarballs = []


class index_rel_inferrer(HTMLParser):

    "Goes through an index page with releases and update the inferred release"

    def parse(self, html):
        "Parse the given string 's'."
        self.feed(html)
        self.close()

    def handle_decl(self, decl):
        pass

    def revise_inference(self, rel, value, rel_name):
        index_parser = self.index_parser

        value = value.lstrip(rel_name + "-")

        p = re.compile(index_parser.release_extension + '$')
        value = p.sub("", value)

        base_specs = get_rel_spec(rel.get('base'))
        if (not base_specs):
            return

        inferred_specs = get_rel_spec(value)
        if (not inferred_specs):
            return

        if (inferred_specs['VERSION'] != base_specs['VERSION']):
            return
        if (inferred_specs['PATCHLEVEL'] != base_specs['PATCHLEVEL']):
            return
        if (base_specs['SUBLEVEL'] != ''):
            if (inferred_specs['SUBLEVEL'] != base_specs['SUBLEVEL']):
                return

        w = compute_rel_weight(value)
        if (not w):
            return

        inferred_rel = dict(base=rel,
                            url=self.base_url,
                            highest_release=value,
                            weight=w)

        # XXX: better way to do this?
        if (rel.get('highest_release') == ''):
            rel['url'] = self.base_url
            rel['highest_release'] = value
            rel['weight'] = w
        if (rel.get('weight') < inferred_rel.get('weight')):
            rel['url'] = self.base_url
            rel['highest_release'] = value
            rel['weight'] = w

    def handle_starttag(self, tag, attributes):
        "Process a tags and its 'attributes'."
        index_parser = self.index_parser
        if tag != 'a':
            return
        for name, value in attributes:
            if name != 'href':
                return
            rel_name = index_parser.search_rel_name(value)
            if (not rel_name):
                return
            if (index_parser.release_extension not in value):
                return
            if (".sign" in value):
                return
            for rel in index_parser.inferred_releases:
                if (rel.get('base') not in value):
                    continue
                self.revise_inference(rel, value, rel_name)

    def handle_endtag(self, tag):
        pass

    def handle_data(self, data):
        pass

    def handle_comment(self, data):
        pass

    def __init__(self, index_parser, url):
        HTMLParser.__init__(self)
        self.index_parser = index_parser
        self.base_url = url


class stable_url_parser(HTMLParser):

    "Goes through an index page and returns a URL for a release"

    def parse(self, html):
        "Parse the given string 's'."
        self.feed(html)
        self.close()

    def update_url(self, rel_target, rel_target_string, r, r_string):

        rel_string = r_string.lstrip("/v")
        rel_string = rel_string.rstrip("/")
        w = compute_rel_weight(rel_string)

        rel = dict(release_base=rel_target_string,
                   release=rel_string,
                   weight=w,
                   version=r['VERSION'],
                   patchlevel=r['PATCHLEVEL'],
                   extraversion=r['EXTRAVERSION'],
                   sublevel=r['SUBLEVEL'],
                   relmod=r['RELMOD_UPDATE'],
                   rel_url=self.base_url.rstrip("/") + '/' + r_string.rstrip("/"))

        if (len(self.stable_urls) == 0):
            self.stable_urls.append(rel)
            return

        for r_tmp in self.stable_urls:
            if (r_tmp.get('release_base') != rel_target_string):
                continue
            if (r_tmp.get('release') == r_string):
                return
            if (r_tmp.get('weight') < w):
                self.stable_urls.remove(r_tmp)
                self.stable_urls.append(rel)
                return

        self.stable_urls.append(rel)

    def update_stable_urls(self, rel, value):
        rel_target = get_rel_spec(rel)
        if (not rel_target):
            return

        r = get_rel_spec(value)
        if (not r):
            return

        if (rel_target['VERSION'] == ''):
            return
        if (rel_target['VERSION'] != r['VERSION']):
            return

        if (rel_target['PATCHLEVEL'] == ''):
            return
        if (rel_target['PATCHLEVEL'] != r['PATCHLEVEL']):
            return

        self.update_url(rel_target, rel, r, value)

    def handle_decl(self, decl):
        pass

    def handle_starttag(self, tag, attributes):
        "Process a tags and its 'attributes'."
        if tag != 'a':
            pass
        for name, value in attributes:
            if name != 'href':
                pass
            for rel in self.index_parser.supported:
                if (rel in value):
                    self.update_stable_urls(rel, value)

    def handle_endtag(self, tag):
        pass

    def handle_data(self, data):
        pass

    def handle_comment(self, data):
        pass

    def __init__(self, index_parser, url):
        HTMLParser.__init__(self)
        self.index_parser = index_parser
        self.base_url = url
        self.stable_urls = []


class index_parser(HTMLParser):

    "HTML index parser for software releases class."

    def parse(self, html, url):
        "Parse the given string 's'."
        self.feed(html)
        self.close()

    def __init__(self, config_file):

        HTMLParser.__init__(self)

        self.config = ConfigParser.SafeConfigParser()
        self.config.read(config_file)

        self.rel_html_proj = self.config.get("project", "rel_html_proj")
        if (self.config.has_option("project", "rel_html_proj_aliases")):
            self.rel_html_proj_aliases = self.config.get(
                "project", "rel_html_proj_aliases").split()
        else:
            self.rel_html_proj_aliases = list()

        self.rel_names = self.rel_html_proj_aliases
        self.rel_names.insert(0, self.rel_html_proj)

        self.inferred_releases = []

        if (self.config.has_option("project", "supported")):
            self.supported = self.config.get("project", "supported").split()
            for rel in self.supported:
                inferred_rel = dict(base=rel,
                                    url='',
                                    highest_release='',
                                    weight=0)
                self.inferred_releases.append(inferred_rel)
        else:
            self.supported = list()
        if (self.config.has_option("project", "eol")):
            self.eol = self.config.get("project", "eol").split()
            for rel in self.eol:
                inferred_rel = dict(base=rel, url='',
                                    highest_release='',
                                    weight=0)
                self.inferred_releases.append(inferred_rel)
        else:
            self.eol = list()

        self.stable_urls = []
        self.rel_html_release_urls = []

        urls = self.config.get("project", "rel_html_url_releases").split()

        for url in urls:
            self.rel_html_release_urls.append(url.strip())

        self.rel_html_rels = []

        if (self.config.has_option("project", "ignore_signatures")):
            self.ignore_signatures = self.config.get(
                "project", "ignore_signatures")
        else:
            self.ignore_signatures = False
        if (self.config.has_option("project", "ignore_changelogs")):
            self.ignore_changelogs = self.config.get(
                "project", "ignore_changelogs")
        else:
            self.ignore_changelogs = False
        if (self.config.has_option("project", "release_extension")):
            self.release_extension = "." + \
                self.config.get("project", "release_extension")
        else:
            self.release_extension = ".tar.xz"

        self.next_rel_day = 0
        self.next_rel_month = 0
        self.next_rel_url = ''
        self.next_rel_date = ''
        self.next_rel_date_rfc3339 = ''
        self.next_rel_dates = list()

        self.rel_license = self.config.get("project", "rel_license")
        self.html_title = self.config.get("html", "title")

        self.html_nav_dict = ({'url': self.config.get("html", "nav_01_url"),
                               'txt': self.config.get("html", "nav_01_txt")},
                              {'url': self.config.get("html", "nav_02_url"),
                               'txt': self.config.get("html", "nav_02_txt")},
                              {'url': self.config.get("html", "nav_03_url"),
                               'txt': self.config.get("html", "nav_03_txt")})

        self.html_release_title = self.config.get("html", "release_title")
        if (self.config.has_option("html", "release_title_next")):
            self.html_release_title_next = self.config.get(
                "html", "release_title_next")
        else:
            self.html_release_title_next = ''
        self.html_about_title = self.config.get("html", "about_title")
        self.html_about = self.config.get("html", "about")

        self.rels = []
        self.signed = False
        self.changelog = ''
        self.signed_changelog = False

    def get_stable_ext_urls(self, url):
        url_parser = stable_url_parser(self, url)
        try:
            rel = urllib2.urlopen(url)
            html = rel.read()
            url_parser.parse(html)
            self.stable_urls = url_parser.stable_urls
        except urllib2.HTTPError as error:
            return

    def search_rel_name(self, value):
        for rel_name in self.rel_names:
            if (rel_name in value):
                return rel_name
        return ""

    def search_stable_tarballs(self, ver, url):
        try:
            tarball_hunter = index_tarball_hunter(self, [ver], url)

            f = urllib2.urlopen(url)
            html = f.read()
            tarball_hunter.parse(html)
            tarball_hunter.update_rel_candidates()
        except urllib2.HTTPError as error:
            return
        except urllib2.URLError as e:
            return

    def evaluate_stable_ext_urls(self):
        for r in self.stable_urls:
            self.search_stable_tarballs(r.get('release'), r.get('rel_url'))

    def update_inferred_releases(self, url):
        try:
            rel_inferrer = index_rel_inferrer(self, url)
            f = urllib2.urlopen(url)
            html = f.read()
            rel_inferrer.parse(html)
        except urllib2.HTTPError as error:
            return
        except urllib2.URLError as e:
            return

    def evaluate_inferred_releases(self):
        for r in self.inferred_releases:
            if (r.get('url') == ''):
                continue
            self.search_stable_tarballs(r.get('highest_release'), r.get('url'))

    def __get_next_rel_page(self, url):
        r = urllib2.urlopen(url)
        html = r.read()
        num_parser = largest_num_href_parser()
        return num_parser.parse(html)

    def get_next_url(self, url):
        self.next_rel_month = self.__get_next_rel_page(url)
        self.next_rel_day = self.__get_next_rel_page(
            url + self.next_rel_month)
        self.next_rel_url   = url + \
            self.next_rel_month + '/' + self.next_rel_day
        self.next_rel_date_rfc3339  = str(date.today().year) + '-' + \
            self.next_rel_month + '-' + self.next_rel_day
        self.next_rel_date = self.next_rel_date_rfc3339.replace("-", "")
        self.next_rel_dates.append(self.next_rel_date_rfc3339)
        self.next_rel_dates.append(self.next_rel_date)

    def evaluate_next_url(self):
        try:
            tarball_hunter = index_tarball_hunter(self,
                                                  self.next_rel_dates,
                                                  self.next_rel_url)

            f = urllib2.urlopen(self.next_rel_url)
            html = f.read()
            tarball_hunter.parse(html)
            tarball_hunter.update_rel_candidates()
        except urllib2.HTTPError as error:
            return

    def scrape_for_releases(self):
        for url in self.rel_html_release_urls:
            if url.endswith('stable/'):
                self.get_stable_ext_urls(url)
                self.evaluate_stable_ext_urls()
            elif url.endswith(str(date.today().year) + '/'):
                self.get_next_url(url)
                self.evaluate_next_url()
            else:
                self.update_inferred_releases(url)
        self.evaluate_inferred_releases()

    def review_base_url(self, ver, url):
        try:
            f_rel = urllib2.urlopen(url)
            html = f_rel.read()
            self.parse(html, url)
        except urllib2.HTTPError as error:
            return

    def validate_releases(self):
        for r in self.rel_html_rels:
            if (r.get('base_url_reviewed')):
                continue
            if (r.get('base_url') == ''):
                continue
            self.review_base_url(self, r.get('base_url'))
        self.rel_html_rels = sort_rels_weight(self.rel_html_rels)

    def handle_starttag(self, tag, attributes):
        "Process a tags and its 'attributes'."
        if tag != 'a':
            pass
        for name, value in attributes:
            if name != 'href':
                pass
            for r in self.rel_html_rels:
                # sys.stdout.write('%s<br>\n' % value)
                if r.get('version') not in value:
                    continue
                if r.get('signed_tarball') in value:
                    r['signed_tarball_exists'] = True
                elif r.get('tarball') in value:
                    if "longerm" in value:
                        r['longterm'] = True
                elif (r.get('changelog') == value):
                    r['changelog_exists'] = True
                elif (r.get('signed_changelog') == value):
                    r['signed_changelog_exists'] = True

    def handle_endtag(self, tag):
        pass

    def handle_data(self, data):
        pass

    def releases_verified(self):
        "Verify releases"

        all_verified = True

        for r in self.rel_html_rels:
            if (not r['tarball_exists']):
                all_verified = False
                sys.stdout.write('No tarball: %s<br>\n' % r['tarball'])
                break
            if (not r['ignore_signature']):
                if (not r['signed_tarball_exists']):
                    all_verified = False
                    sys.stdout.write(
                        'No signed tarball: %s<br>\n' % r['signed_tarball'])
                    break
            if (r['changelog_required']):
                if (not (r['changelog_exists'])):
                    all_verified = False
                    sys.stdout.write(
                        'No changelog (%s): %s<br>\n' % (r['changelog'], r['version']))
                    break
                if (not (r['signed_changelog_exists'])):
                    sys.stdout.write('No signed changelog (%s): %s<br>\n' %
                                     (r['signed_changelog'], r['version']))
                    all_verified = False
                    break
            else:
                if (r['changelog_exists'] and
                        (not (r['signed_changelog_exists']))):
                    sys.stdout.write("Although a ChangeLog is not "
                                     "required for this release (%s), one does "
                                     "but it is not digitally signed. The "
                                     "file %s does not exist<br>" %
                                    (r['version'], r['signed_changelog']))
                    all_verified = False
            r['verified'] = True

        return all_verified


class largest_num_href_parser(HTMLParser):

    "Will take an index page and return the highest numbered link"

    def parse(self, html):
        "Parse the given string 's'."
        self.feed(html)
        self.close()
        return self.number

    def handle_decl(self, decl):
        pass

    def handle_starttag(self, tag, attributes):
        "Process a tags and its 'attributes'."
        if tag != 'a':
            pass
        for name, value in attributes:
            if name != 'href':
                pass
            if (re.match(r'\d+', value)):
                number = re.sub(r'\D', "", value)
                if (number > self.number):
                    self.number = number

    def handle_endtag(self, tag):
        pass

    def handle_data(self, data):
        pass

    def handle_comment(self, data):
        pass

    def __init__(self):
        HTMLParser.__init__(self)
        self.number = 0

# Example full html output parser, this can be used to
# simply read and output a full HTML file, you can modify
# this class to help you modify the contents. We do that
# later.


class html_base(HTMLParser):

    "HTML 5 generator from parsed index parser content."

    def parse(self, html):
        "Parse the given string 's'."
        self.feed(html)
        self.close()

    def handle_decl(self, decl):
        sys.stdout.write('<!%s>' % decl)

    def handle_starttag(self, tag, attributes):
        sys.stdout.write('<%s' % tag)
        for name, value in attributes:
            sys.stdout.write(' %s="%s"' % (name, value))
        sys.stdout.write('>')

    def handle_endtag(self, tag):
        sys.stdout.write('</%s>' % tag)

    def handle_data(self, data):
        sys.stdout.write('%s' % data)

    def handle_comment(self, data):
        sys.stdout.write('<!--%s-->' % data)

    def __init__(self):
        HTMLParser.__init__(self)


def license_url(license):
    if (license == 'GPLv2'):
        return "http://www.gnu.org/licenses/gpl-2.0.html"
    elif (license == 'ISC'):
        return "http://opensource.org/licenses/ISC"
    elif (license == 'AGPL'):
        return "http://www.gnu.org/licenses/agpl-3.0.html"
    elif (license == 'copyleft-next'):
        return "http://copyleft-next.org/"
    else:
        return "http://opensource.org/licenses/alphabetical"


class rel_html_gen(HTMLParser):

    "HTML 5 generator from parsed index parser content."

    def __init__(self, parser):
        HTMLParser.__init__(self)
        self.parser = parser
        self.skip_endtag = False
        self.latest_stable = {}
        self.next_rels = []
        self.next_rel_count = 0

    def handle_title(self, tag, attributes):
        sys.stdout.write('<%s>%s' % (tag, self.parser.html_title))

    def handle_def_start(self, tag, attributes):
        sys.stdout.write('<%s' % tag)
        for name, value in attributes:
            sys.stdout.write(' %s="%s"' % (name, value))
        sys.stdout.write('>')

    def handle_h1_top(self, tag, attributes):
        self.skip_endtag = True
        sys.stdout.write('%s</h1>\n' % (self.parser.html_title))
        sys.stdout.write('\t\t<nav>\n')
        sys.stdout.write('\t\t\t<ul>\n')
        for nav in self.parser.html_nav_dict:
            sys.stdout.write(
                '\t\t\t\t<li><a href="%s">%s</a></li>\n' % (nav['url'], nav['txt']))
        sys.stdout.write('\t\t\t</ul>\n')
        sys.stdout.write('\t\t</nav>\n')

    def handle_h1_release(self, tag, attributes):
        self.skip_endtag = True
        sys.stdout.write('%s</h1>\n' % (self.parser.html_release_title))
        sys.stdout.write('\t\t\t<table border="0">\n')

        count = 0

        for r in self.parser.rel_html_rels:
            count += 1
            if (not r.get('verified')):
                continue

            if (count == 2):
                latest_stable = r

            if (r.get('next_rel')):
                self.next_rels.append(r)
                self.next_rel_count = self.next_rel_count + 1
                continue

            sys.stdout.write('\t\t\t\t<tr>')
            sys.stdout.write('\t\t\t\t<td><a href="%s">%s</a></td>\n' %
                             (r.get('url'), r.get('rel')))
            if (not r.get('ignore_signature')):
                sys.stdout.write('\t\t\t\t<td><a href="%s">signed</a></td>\n' %
                                 (r.get('base_url') + '/' + r.get('signed_tarball')))
            else:
                sys.stdout.write('\t\t\t\t<td></td>\n')
            if (r.get('maintained')):
                sys.stdout.write('\t\t\t\t<td></td>\n')
            else:
                sys.stdout.write(
                    '\t\t\t\t<td><font color="FF0000">EOL</font></td>\n')
            if (not r.get('longterm')):
                sys.stdout.write('\t\t\t\t<td></td>\n')
            else:
                sys.stdout.write(
                    '\t\t\t\t<td><font color="00FF00">Longterm</font></td>\n')

            if (r.get('changelog_required')):
                sys.stdout.write('\t\t\t\t<td><a href="%s">%s</a></td>\n' %
                                 (r.get('changelog_url'), "ChangeLog"))
            else:
                sys.stdout.write('\t\t\t\t<td></td>\n')
            sys.stdout.write('\t\t\t\t</tr>')

        sys.stdout.write('\t\t\t</table>\n')

    def handle_h1_release_next(self, tag, attributes):
        if (self.next_rel_count <= 0):
            return
        if (not len(self.next_rels)):
            return
        sys.stdout.write('%s</h1>\n' % (self.parser.html_release_title_next))
        sys.stdout.write('\t\t\t<table border="0">\n')

        for r in self.next_rels:
            if (not r.get('verified')):
                continue

            sys.stdout.write('\t\t\t\t<tr>')
            sys.stdout.write('\t\t\t\t<td><a href="%s">%s</a></td>\n' %
                             (r.get('url'), r.get('rel')))
            if (not r.get('ignore_signature')):
                sys.stdout.write('\t\t\t\t<td><a href="%s">signed</a></td>\n' %
                                 (r.get('base_url') + '/' + r.get('signed_tarball')))
            else:
                sys.stdout.write('\t\t\t\t<td></td>\n')
            if (r.get('maintained')):
                sys.stdout.write('\t\t\t\t<td></td>\n')
            else:
                sys.stdout.write(
                    '\t\t\t\t<td><font color="FF0000">EOL</font></td>\n')

            if (not r.get('longterm')):
                sys.stdout.write('\t\t\t\t<td></td>\n')
            else:
                sys.stdout.write(
                    '\t\t\t\t<td><font color="00FF00">Longterm</font></td>\n')

            if (r.get('changelog_required')):
                sys.stdout.write('\t\t\t\t<td><a href="%s">%s</a></td>\n' %
                                 (r.get('changelog_url'), "ChangeLog"))
            else:
                sys.stdout.write('\t\t\t\t<td></td>\n')
            sys.stdout.write('\t\t\t\t</tr>')

        sys.stdout.write('\t\t\t</table>\n')

    def handle_h1_about(self, tag, attributes):
        self.skip_endtag = True
        sys.stdout.write('%s</h1>\n' % (self.parser.html_about_title))
        sys.stdout.write('<p>%s</p>\n' % (self.parser.html_about))

    def handle_h_license(self, tag, attributes):
        self.skip_endtag = True
        sys.stdout.write('License</h1>\n')
        sys.stdout.write('\t\t<p>%s is licensed under the <a href="%s">%s</a>. \n' %
                         (self.parser.rel_html_proj,
                          license_url(self.parser.rel_license),
                          self.parser.rel_license))
        sys.stdout.write('This page was generated by %s licensed under the <a href="%s">%s</a></p>\n' %
                         (rel_html_href(),
                          license_url(rel_html_license()),
                          rel_html_license()))

    def handle_h1_pass(self, tag, attributes):
        pass

    def handle_h(self, tag, attributes):
        def_run = self.handle_h1_pass

        for name, value in attributes:
            if (name == 'id'):
                if (value == 'top_content'):
                    def_run = self.handle_h1_top
                elif (value == 'release_title'):
                    def_run = self.handle_h1_release
                elif (value == 'release_title_next'):
                    def_run = self.handle_h1_release_next
                elif (value == 'about'):
                    def_run = self.handle_h1_about
                elif (value == 'license'):
                    def_run = self.handle_h_license

        sys.stdout.write('<%s' % tag)
        for name, value in attributes:
            sys.stdout.write(' %s="%s"' % (name, value))
        sys.stdout.write('>')

        def_run(tag, attributes)

    def parse(self, html):
        "Parse the given string 's'."
        self.feed(html)
        self.close()

    def handle_decl(self, decl):
        sys.stdout.write('<!%s>' % decl)

    def handle_starttag(self, tag, attributes):
        self.skip_endtag = False
        if (tag == 'title'):
            self.handle_title(tag, attributes)
        elif (tag in {'h1', 'h2', 'h3'}):
            self.handle_h(tag, attributes)
        else:
            self.handle_def_start(tag, attributes)

    def handle_endtag(self, tag):
        if (self.skip_endtag):
            pass
        sys.stdout.write('</%s>' % tag)

    def handle_data(self, data):
        sys.stdout.write('%s' % data)

    def handle_comment(self, data):
        sys.stdout.write('<!--%s-->' % data)


def check_file(file_input):
    if not os.path.isfile(file_input):
        print 'File not found: %(file)s' % {"file": file_input}
        usage()


def try_rel_next(rel):
    sys.stdout.write("----------------------------------------\n")
    sys.stdout.write("Rel: %s\n" % rel)
    w = compute_rel_weight_next(rel)
    sys.stdout.write("Weight: %s\n" % w)


def try_rel(rel_dict):
    sys.stdout.write("----------------------------------------\n")
    sys.stdout.write("Rel: %s\n" % rel_dict.get('version'))
    rel_dict["weight"] = compute_rel_weight(rel_dict.get('version'))
    sys.stdout.write("Weight: %s\n" % rel_dict.get("weight"))


def print_rels_weight(rels):
    for r in rels:
        sys.stdout.write("Rel: %20s\t%20s\n" %
                         (r.get('version'), r.get('weight')))


def try_rels(rels):
    col = []
    rsorted = []
    max_weight = 0
    for rel in rels:
        rel_d = dict(version=rel, weight=0)
        col.append(rel_d)

    for r in col:
        try_rel(r)

    col = sort_rels_weight(col)

    print_rels_weight(col)


def debug_rel_tests():
    try_rel_next("20130510-2-u")
    try_rel_next("2013-01-10-2-u")
    try_rel_next("20130110-2-u")
    try_rel_next("2013-03-07-u")
    try_rel_next("2013-03-07")

    rels = ["2.6.32.3",
            "3.8",
            "2.6.32.1",
            "2.6.32.40",
            "2.6.32",
            "3.8.2",
            "3.5.1",
            "3.2.1",
            "3.7.1",
            "3.8.2-1-usnpc",
            "3.8-rc1",
            "3.8-rc1-1-usnpc",
            "3.8-rc2-2-usnpc",
            "3.8-rc2-2-c",
            "3.8-rc2-2-s",
            "3.8-rc2-2",
            "3.8-rc3-1-u"]

    try_rels(rels)


def main():

    if (debug):
        debug_rel_tests()
        sys.exit(1)

    config_file = ''
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hf:")
    except getopt.GetoptError as err:
        print str(err)
        usage()

    for o, a in opts:
        if o in ("-f"):
            check_file(a)
            config_file = a
        elif o in ("-h", "--help"):
            usage()

    if len(config_file) == 0:
        config_file = 'rel-html.cfg'

    parser = index_parser(config_file)

    html = ""

    # We go through two passes on the provided URLs:
    #
    # 1. Scraping for releases
    # 2. Validation
    #
    # The first pass is based on finding the
    # highest release extraversion tarball. We
    # require a second pass as validation entails
    # searching for a ChangeLog and signature file
    # for all known existing releases.

    parser.scrape_for_releases()
    parser.validate_releases()

    if (not parser.releases_verified()):
        sys.stdout.write("Releases not verified\n")
        sys.exit(1)

    gen = rel_html_gen(parser)
    f = open('html/template.html', 'r')
    html = f.read()

    gen.parse(html)


def usage():
    print ''
    print '%(cmd)s' % {"cmd": sys.argv[0]}
    print ''
    print 'Provided an index URL and a few project hints page this'
    print 'will spit out a shiny HTML 5 W3C compliant releases page.'
    print ''
    rel_html_license_verbose()
    print ''
    print 'This program can be run without arguments or with a project file passed'
    print 'as an argument. If no arguments are given it will assume you have the'
    print 'file rel-html.cfg present on your current directory.'
    print ''
    print 'Usage: %(cmd)s [ -f rel-html.cfg ]' % {"cmd": sys.argv[0]}
    sys.exit(2)

if __name__ == "__main__":
    main()