SWDB/install

#!/usr/bin/python3





import urllib3
import sys
import os
import requests
import urllib.request
import sqlite3
import wget





"""
Toggles.
"""
OPT_CREATE_DB = True
OPT_DOWNLOAD = True
OPT_SKILLS = True
OPT_ESSENCES = True
OPT_BASE = True





"""
Initializes the database file and creates all the tables.

:param name: The path to the sqlite database.
:returns: Connection to the database.
:raises IntegrityError: The queryes couldn't bre executed.
:raises IOError: The sqlite file couldn't be created.
"""
def createDatabase(name):
    global OPT_CREATE_DB
    print('Configuring database...')
    try:
        db = sqlite3.connect(name)
        if OPT_CREATE_DB == False:
            return db
        cursor = db.cursor()
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monster(
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            base INTEGER,
            stars INT,
            element TEXT,
            name TEXT,
            name_aw TEXT,
            type TEXT,
            awakeable INT,
            img TEXT,
            img_aw TEXT
            );
        ''')
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monster_base(
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            name TEXT
            );
        ''')
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monster_skill(
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            monster INT,
            idx INT,
            name TEXT,
            desc TEXT,
            passive INT,
            awaken INT,
            upgrade INT,
            leader INT,
            img TEXT,
            cooldown INT
            );
        ''')
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monster_stat(
            monster INTEGER,
            stars INT,
            awake INT,
            stat TEXT,
            min INT,
            max INT
            );
        ''')
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monster_essence(
            monster INT,
            element TEXT,
            grade TEXT,
            num INT
            );
        ''')
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monster_location(
            monster INT,
            location TEXT
            );
        ''')
        cursor.execute('''
            CREATE TABLE IF NOT EXISTS monster_collection (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            monster INT,
            stars   INT,
            level   INT,
            awaken  INT
            );
        ''')
        db.commit()
        cursor.close()
        return db
    except sqlite3.IntegrityError as e:
        print("Error executing statement: " + str(e))
        raise
    except IOError as e:
        print("I/O Error creating database " + name + ": " + str(e))
        raise





"""
Inserts a row into the database.

:param db: Connection to the database.
:param table: Name of the table to insert into.
:param values: List of values to insert.
:raises IntegrityError: The insert query was unsuccesfull.
"""
def insert(db, table, values):
    cursor = db.cursor()
    placeholders = ''
    for x in range(0, len(values)):
        placeholders = placeholders + '?, '
    placeholders = placeholders[:len(placeholders) - 2]
    query = 'INSERT INTO ' + table + ' VALUES (' + placeholders + ')'
    try:
        cursor.execute(query, values)
    except sqlite3.IntegrityError as e:
        print('Error Inserting into ' + table + ' with values (' + str(values) + '): ' + str(e))
        raise
    cursor.close;





def download(url, target):
    if OPT_DOWNLOAD == False:
        return True
    try:
        urllib.request.urlretrieve(url, target)
        return True
    except:
        return False


def getDetails(page):
    details = []
    name = ''
    name_aw = ''
    img = ''
    img_aw = ''
    awakeable = 0
    mtype = ''
    stars = 1
    if ('alt="Purple Star"' in page):
        if ('alt="Gold Star"' in page):
            awakeable = 1 # Awakeable (Gold)
        else:
            awakeable = 2 # Only awaken (Purple)
    else:
        awakeable = 0 # No awakeable (Silver)

    try:
        stars = int(page[page.index('x', page.index('_Star.png"')) + 1:page.index(')', page.index('x', page.index('_Star.png"')))])
    except:
        stars = 1
        pass

    header = page[page.index('>') + 1:page.index('<', page.index('>'))]
    if ') - ' in header:
        name = header[:header.index(' (')]
        name_aw = header[header.index(') - ') + 4:]
    else:
        if ' (' in header:
            name = header[:header.index(' (')]
        else:
            name = header

    imgs = page[page.index('<div id="images">'):page.index('</div>', page.index('<div id="images">'))]
    if 'data-src="' in imgs:
        im = imgs[imgs.index('data-src="') + 10:imgs.index('"', imgs.index('data-src="') + 11)]
        imgs = imgs[imgs.index('"', imgs.index('data-src="') + 11):]
    else:
        im = imgs[imgs.index('<img src="') + 10:imgs.index('"', imgs.index('<img src="') + 11)]
        imgs = imgs[imgs.index('"', imgs.index('<img src="') + 11):]

    if 'data-src="' in imgs:
        img = im
        img_aw = imgs[imgs.index('data-src="') + 10:imgs.index('"', imgs.index('data-src="') + 11)]
    elif '<img src="' in imgs:
        img = im
        img_aw = imgs[imgs.index('<img src="') + 10:imgs.index('"', imgs.index('<img src="') + 11)]
    else:
        img = im
        img_aw = ''

    if awakeable == 2 and name_aw == '':
        name_aw = name
        name = ''
    if awakeable == 2 and img_aw == '':
        img_aw = img
        img = ''

    mtype = page[page.index('<b>Type:</b>') + 13:page.index('\n', page.index('<b>Type:</b>'))]
    details = [name, name_aw, img, img_aw, awakeable, mtype, stars]
    return details





def getObtainable(page):
    obtainable = []
    while 'title=' in page:
        obtainable.append(page[page.index('title="') + 7:page.index('"', page.index('title="') + 7)])
        page = page[page.index('"', page.index('title="') + 7):]
    return obtainable





def getEssences(page):
    essences = []
    while '\n' in page:
        line = page[:page.index('\n')]
        if '<span class="essence-number">' in line:
            num = line[line.index('<span class="essence-number">') + 29:line.index('</span>', line.index('<span class="essence-number">'))]
            desc = line[line.index('alt="') + 5:line.index('"', line.index('alt="') + 6)].split(" ")
            elm = desc[2]
            if elm == 'Darkness':
                elm = 'Dark'
            grade = desc[3]
            grade = grade[1:len(grade) - 1]
            essences.append([num, elm, grade])
        page = page[page.index('\n') + 1:]
    return essences





def getSkills(page):
    skills = []
    while '<td style="text-align' in page:
        passive = 0
        awaken = 0
        upgrade = 0
        leader = 0
        image = ''
        name = ''
        description = ''
        cooldown = 0

        name = page[page.index('alt="') + 5:page.index('"', page.index('alt="') + 6)]
        if ' (Passive)' in name:
            passive = 1
            name = name[:name.index(' (Passive)')]
        if 'Strengthened ' in name:
            upgrade = 1
            awaken = 1
            name = name[13:]
        if len(name) > 12 and name[:13] == 'Leader Skill ':
            leader = 1
            name = 'LEADER SKILL'
        if leader == 1:
            subawaken = page[:page.index(':</b> ')]
        else:
            subawaken = page[:page.index('</b>: ')]
        if '(Awakened)' in subawaken:
            awaken = 1
        image = page[page.index('data-src="') + 10:page.index('"', page.index('data-src="') + 11)]
        if leader == 1:
            description = page[page.index(':</b> ') + 6:page.index('\n', page.index(':</b> '))]
        else:
            description = page[page.index('</b>: ') + 6:page.index('\n', page.index('</b>: '))]
        if ' (Reusable in ' in description:
            try:
                cooldown = int(description[description.index(' (Reusable in ') + 14:description.index(' turns)', description.index(' (Reusable in '))])
                description = description[:description.index(' (Reusable in ')]
            except:
                pass
        if ' [Automatic Effect]' in description:
            description = description[:description.index(' [Automatic Effect]')]

        skills.append([name, description, passive, awaken, upgrade, leader, image, cooldown])
        page = page[page.index('</tr>', page.index('<td style="text-align')) + 10:]
    return skills





def getStats(page, awakeable):
    stats = []
    ahp = []
    aatk = []
    adef = []
    wahp = []
    waatk = []
    wadef = []
    spd = 0
    crir = 0
    crid = 0
    res = 0
    acc = 0
    wspd = 0
    wcrir = 0
    wcrid = 0
    wres = 0
    wacc = 0

    thp = page[page.index('<b>HP</b>'):page.index('</tr>', page.index('<b>HP</b>'))]
    while '<td class="monstercell">' in thp:
        n = thp[thp.index('<td class="monstercell">') + 24:thp.index('\n', thp.index('<td class="monstercell">'))]
        try:
            ahp.append(int(thp[thp.index('<td class="monstercell">') + 24:thp.index('\n', thp.index('<td class="monstercell">'))]))
        except Exception as e:
            pass
        thp = thp[thp.index('<td class="monstercell">') + 1:]
    tatk = page[page.index('<b>ATK</b>'):page.index('</tr>', page.index('<b>ATK</b>'))]
    while '<td class="monstercell">' in tatk:
        n = tatk[tatk.index('<td class="monstercell">') + 24:tatk.index('\n', tatk.index('<td class="monstercell">'))]
        try:
            aatk.append(int(tatk[tatk.index('<td class="monstercell">') + 24:tatk.index('\n', tatk.index('<td class="monstercell">'))]))
        except Exception as e:
            pass
        tatk = tatk[tatk.index('<td class="monstercell">') + 1:]
    tdef = page[page.index('<b>ATK</b>'):page.index('</tr>', page.index('<b>ATK</b>'))]
    while '<td class="monstercell">' in tdef:
        n = tdef[tdef.index('<td class="monstercell">') + 24:tdef.index('\n', tdef.index('<td class="monstercell">'))]
        try:
            adef.append(int(tdef[tdef.index('<td class="monstercell">') + 24:tdef.index('\n', tdef.index('<td class="monstercell">'))]))
        except Exception as e:
            pass
        tdef = tdef[tdef.index('<td class="monstercell">') + 1:]
    # STATS (awakened)
    if '#246BB2; width:10%;">Awakened' in page:
        mpag = page[page.index('#246BB2; width:10%;">Awakened'):]
        thp = page[page.index('<b>HP</b>'):page.index('</tr>', page.index('<b>HP</b>'))]
        while '<td class="monstercell">' in thp:
            n = thp[thp.index('<td class="monstercell">') + 24:thp.index('\n', thp.index('<td class="monstercell">'))]
            try:
                wahp.append(int(thp[thp.index('<td class="monstercell">') + 24:thp.index('\n', thp.index('<td class="monstercell">'))]))
            except Exception as e:
                pass
            thp = thp[thp.index('<td class="monstercell">') + 1:]
        tatk = page[page.index('<b>ATK</b>'):page.index('</tr>', page.index('<b>ATK</b>'))]
        while '<td class="monstercell">' in tatk:
            n = tatk[tatk.index('<td class="monstercell">') + 24:tatk.index('\n', tatk.index('<td class="monstercell">'))]
            try:
                waatk.append(int(tatk[tatk.index('<td class="monstercell">') + 24:tatk.index('\n', tatk.index('<td class="monstercell">'))]))
            except Exception as e:
                pass
            tatk = tatk[tatk.index('<td class="monstercell">') + 1:]
        tdef = page[page.index('<b>ATK</b>'):page.index('</tr>', page.index('<b>ATK</b>'))]
        while '<td class="monstercell">' in tdef:
            n = tdef[tdef.index('<td class="monstercell">') + 24:tdef.index('\n', tdef.index('<td class="monstercell">'))]
            try:
                wadef.append(int(tdef[tdef.index('<td class="monstercell">') + 24:tdef.index('\n', tdef.index('<td class="monstercell">'))]))
            except Exception as e:
                pass
            tdef = tdef[tdef.index('<td class="monstercell">') + 1:]
        mpag = page[page.index('Static Attributes'):]
    # STATIC STATS
    if awakeable == 1: # For awakeable monsters, skip one line
        page = page[page.index('\n', page.index('style="width: 9%;">')):]
    spd = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
    page = page[page.index('\n', page.index('style="width: 9%;">')):]
    crir = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
    page = page[page.index('\n', page.index('style="width: 9%;">')):]
    crid = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
    page = page[page.index('\n', page.index('style="width: 9%;">')):]
    res = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
    page = page[page.index('\n', page.index('style="width: 9%;">')):]
    acc = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
    page = page[page.index('\n', page.index('style="width: 9%;">')):]
    # Remove percent symbols and cast
    spd = int(spd.replace('<b>', '').replace('</b>', ''))
    crir = int(crir[:len(crir) - 1].replace('<b>', '').replace('</b>', ''))
    crid = int(crid[:len(crid) - 1].replace('<b>', '').replace('</b>', ''))
    res = int(res[:len(res) - 1].replace('<b>', '').replace('</b>', ''))
    acc = int(acc[:len(acc) - 1].replace('<b>', '').replace('</b>', ''))
    # Static stats, awakened
    if 'style="width: 9%;"><b>Awakened</b>' in page:
        page = page[page.index('style="width: 9%;"><b>Awakened</b>') + 10:]
        wspd = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
        page = page[page.index('\n', page.index('style="width: 9%;">')):]
        wcrir = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
        page = page[page.index('\n', page.index('style="width: 9%;">')):]
        wcrid = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
        page = page[page.index('\n', page.index('style="width: 9%;">')):]
        wres = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
        page = page[page.index('\n', page.index('style="width: 9%;">')):]
        wacc = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
        page = page[page.index('\n', page.index('style="width: 9%;">')):]
        # Remove percent symbols and cast
        wspd = int(wspd.replace('<b>', '').replace('</b>', ''))
        wcrir = int(wcrir[:len(wcrir) - 1].replace('<b>', '').replace('</b>', ''))
        wcrid = int(wcrid[:len(wcrid) - 1].replace('<b>', '').replace('</b>', ''))
        wres = int(wres[:len(wres) - 1].replace('<b>', '').replace('</b>', ''))
        wacc = int(wacc[:len(wacc) - 1].replace('<b>', '').replace('</b>', ''))
    else:
        wspd = None
        wcrir = None
        wcrid = None
        wres = None
        wacc = None
    stats = [ahp, aatk, adef, wahp, waatk, wadef, spd, crir, crid, res, acc, wspd, wcrir, wcrid, wres, wacc]
    return stats





"""
Parses the url with info abount one monster, extracts the info and saves it to
the database.

:param db: Connection to the database.
:param url: URL with the info to parse.
:param url: Element of the monster.
:raises Exception: The monster data couldn't be saved.
"""
def parseMonster(db, url, element):
    http = urllib3.PoolManager()
    try:
        cursor = db.cursor()

        name = '';
        name_aw = '';
        img = '';
        img_aw = '';
        awakeable = 0;
        mtype = ''
        stars = 0
        obtainable = []
        essences = []
        skills = []
        ahp = []
        aatk = []
        adef = []
        wahp = []
        waatk = []
        wadef = []
        spd = 0
        crir = 0
        crid = 0
        res = 0
        acc = 0
        wspd = 0
        wcrir = 0
        wcrid = 0
        wres = 0
        wacc = 0

        # DEBUG - GOLD
        #url = 'http://summonerswar.wikia.com/wiki/Fairy_(Water)_-_Elucia'
        # DEBUG - SILVER
        #url = 'http://summonerswar.wikia.com/wiki/Forest_Keeper_(Water)'
        # DEBUG - PURPLE
        #url = 'http://summonerswar.wikia.com/wiki/Rainbowmon'
        # DEBUG - Upgradeable skill
        #url = 'http://summonerswar.wikia.com/wiki/Paladin_(Wind)_-_Louise'

        mpage = http.request('GET', url).data.decode('utf-8');

        # Basic details
        detailsPage = mpage[mpage.index('<h1 class="page-header__title">'):mpage.index('</tr>', mpage.index('<b>Type:</b>', mpage.index('<h1 class="page-header__title">')))]
        details = getDetails(detailsPage)
        name = details[0]
        name_aw = details[1]
        img = details[2]
        img_aw = details[3]
        awakeable = details[4]
        mtype = details[5]
        stars = details[6]

        # Check if monster already parsed
        cursor.execute('SELECT count(id) AS c FROM monster WHERE element = ? AND name = ? AND name_aw = ?', (element, name, name_aw))
        if cursor.fetchone()[0] == 0:

            # Regions where the monster can be obtained
            obtainablePage = mpage[mpage.index('<b>Obtainable from:</b>'):mpage.index('</td>', mpage.index('<b>Obtainable from:</b>'))]
            obtainable = getObtainable(obtainablePage)

            # Essences required to awaken the monster
            if awakeable == 1:
                essencesPage = mpage[mpage.index('<div id="essences">'):mpage.index('</table>', mpage.index('<div id="essences">'))]
                essences = getEssences(essencesPage)

            # Skills
            skillsPage = mpage[mpage.index('<table', mpage.index('<table', mpage.index('<div id="skills">') + 1) + 1):mpage.index('</table>', mpage.index('<table', mpage.index('<table', mpage.index('<div id="skills">') + 1) + 1))]
            skills = getSkills(skillsPage)

            # Stats
            statsPage = mpage[mpage.index('Evolution Stats'):]
            stats = getStats(statsPage, awakeable)
            ahp = stats[0]
            aatk = stats[1]
            adef = stats[2]
            wahp = stats[3]
            waatk = stats[4]
            wadef = stats[5]
            spd = stats[6]
            crir = stats[7]
            crid = stats[8]
            res = stats[9]
            acc = stats[10]
            wspd = stats[11]
            wcrir = stats[12]
            wcrid = stats[13]
            wres = stats[14]
            wacc = stats[15]

            # Save monster
            strname = element  + ' ' + name + " - " + name_aw
            if awakeable == 0:
                strname = element  + ' ' + name
            elif awakeable == 2:
                strname = name_aw
            print ("    Saving monster: " + strname + "...")
            if img != '':
                if download(img, 'public/img/content/monster/' + str(id["monster"]).zfill(3) + '.png'):
                    img = str(id["monster"]).zfill(3) + '.png'
                else:
                    print("Error downloading image for monster " + str(id["monster"]))
            if img != '':
                if download(img_aw, 'public/img/content/monster/' + str(id["monster"]).zfill(3) + '_aw.png'):
                    img_aw = str(id["monster"]).zfill(3) + '_aw.png'
                else:
                    print("Error downloading image for awakened monster " + str(id["monster"]))
            insert(db, 'monster', (id["monster"], None, stars, element, name, name_aw, mtype, awakeable, img, img_aw))

            # Save locations
            for o in obtainable:
                insert(db, 'monster_location', (id["monster"], o))
            for e in essences:
                insert(db, 'monster_essence', (id["monster"], e[1], e[2], e[0]))

            # Save skills
            idx = 0
            for s in skills:
                #                0    1    2       3      4       5      6   7
                # id monster idx name desc passive awaken upgrade leader img cooldown
                if download(s[6], 'public/img/content/skill/' + str(id["monster_skill"]).zfill(4) + '.png'):
                    s[6] = str(id["monster_skill"]).zfill(4) + '.png'
                else:
                    s[6] = ''
                if s[4] == 1:
                    s[4] = id["monster_skill"] - 1
                insert(db, 'monster_skill', (id["monster_skill"], id["monster"], idx, s[0], s[1], s[2], s[3], None, s[5], s[6], s[7]))
                idx = idx + 1
                id["monster_skill"] = id["monster_skill"] + 1

            # Save stats
            i = 0
            s = stars
            while i < len(ahp):
                if i + 1 < len(ahp):
                    insert(db, 'monster_stat', (id["monster"], s, 0, 'HP', ahp[i], ahp[i + 1]))
                    insert(db, 'monster_stat', (id["monster"], s, 0, 'ATK', aatk[i], aatk[i + 1]))
                    insert(db, 'monster_stat', (id["monster"], s, 0, 'DEF', aatk[i], aatk[i + 1]))
                    insert(db, 'monster_stat', (id["monster"], s, 0, 'SPD', spd, spd))
                    insert(db, 'monster_stat', (id["monster"], s, 0, 'CRIR', crir, crir))
                    insert(db, 'monster_stat', (id["monster"], s, 0, 'CRID', crid, crid))
                    insert(db, 'monster_stat', (id["monster"], s, 0, 'RES', res, res))
                    insert(db, 'monster_stat', (id["monster"], s, 0, 'ACC', acc, acc))
                if i + 1 < len(wahp):
                    insert(db, 'monster_stat', (id["monster"], s, 1, 'HP', wahp[i], wahp[i + 1]))
                    insert(db, 'monster_stat', (id["monster"], s, 1, 'ATK', waatk[i], waatk[i + 1]))
                    insert(db, 'monster_stat', (id["monster"], s, 1, 'DEF', waatk[i], waatk[i + 1]))
                    insert(db, 'monster_stat', (id["monster"], s, 1, 'SPD', wspd, wspd))
                    insert(db, 'monster_stat', (id["monster"], s, 1, 'CRIR', wcrir, wcrir))
                    insert(db, 'monster_stat', (id["monster"], s, 1, 'CRID', wcrid, wcrid))
                    insert(db, 'monster_stat', (id["monster"], s, 1, 'RES', wres, wres))
                    insert(db, 'monster_stat', (id["monster"], s, 1, 'ACC', wacc, wacc))
                s = s + 1
                i = i + 2

            # Save and proceed
            db.commit()
            id["monster"] = id["monster"] + 1

    except IOError as e:
        db.rollback()
        print('Error saving monster with id ' + str(id['monster']) + ': ' + str(e))
        print('  Current data:')
        print('    Name:          ' + name)
        print('    Name (awaken): ' + name_aw)
        print('    Element:       ' + element)
        print('    Type:          ' + mtype)
        print('    Awakeable:     ' + str(awakeable))
        print('    Obtainable:')
        for o in obtainable:
            print('      ' + o)
        print('    Essences:')
        for e in essences:
            print('      ' + e[0] + ' x ' + e[2] + ' ' + e[1] + ' essences')
        print('    Skills:')
        for s in skills:
            print('      ' + s[1])
        pass
        #sys.exit("DEBUG EXIT")



def makeDirs():
    dirs = ['public/img/content/monster/', 'public/img/content/skill/', 'public/img/content/essence/']
    for dir in dirs:
        try:
            if os.path.isdir(dir) == False:
                os.makedirs(dir)
        except IOError as e:
            print('Unable to create directory: ' + dir);
            raise



"""
Create the entries for the table monster_base, reading data from the table
monster.

:param db: Connection to the database.
:raise IntegrityError: One or more bases could not be created.
"""
def createBases(db):
    cursor = db.cursor()
    cursor_base = db.cursor()
    try:
        cursor.execute('SELECT DISTINCT name FROM monster;')

        for row in cursor:
            try:
                cursor_base.execute('INSERT INTO monster_base VALUES (?, ?);', (id["monster_base"], row[0]));
                cursor.execute('UPDATE monster SET base = ? WHERE name = ?;', (id["monster_base"], row[0]));
            except sqlite3.IntegrityError as e:
                print("Error creating monster bases: " + str(e))
                raise
            id["monster_base"] = id["monster_base"] + 1;
        db.commit()
        cursor_base.close();
    except sqlite3.IntegrityError as e:
        print("Error creating monster bases: " + str(e))
        raise





"""
Downloads the essence icons.

"""
def getEssenceIcons():
    url = 'http://summonerswar.wikia.com/wiki/Essences'
    http = urllib3.PoolManager()
    page = http.request('GET', url).data.decode('utf-8')
    page = page[page.index('id="List_of_Essences"'):page.index('id="Essence_fusion"')]
    while 'data-src="' in page:
        name = page[page.index('data-image-name="') + 7:page.index('"', page.index('data-image-name="') + 17)].split(" ")
        element = name[2]
        if element == 'Darkness':
            element = 'Dark'
        grade = name[3][1:len(name)]
        file = page[page.index('data-src="') + 10:page.index('"', page.index('data-src="') + 11)]
        if download(file, 'public/img/content/essence/' + element + '_' + grade + '.png') == False:
            print("Unable to save image for essence: " + element + " (" + grade + ")");
        page = page[page.index('data-src="') + 10:]





def setBases(db):
    cursor = db.cursor()
    cursor_base = db.cursor()
    cursor.execute('SELECT DISTINCT name FROM monster;')
    for row in cursor:
        cursor_base.execute('INSERT INTO monster_base VALUES (?, ?);', (id["monster_base"], row[0]));
        cursor_base.execute('UPDATE monster SET base = ? WHERE name = ?;', (id["monster_base"], row[0]));
        id["monster_base"] = id["monster_base"] + 1;
    db.commit()
    cursor_base.close();
    cursor.close()





"""
Starts scrapping th page.

:param db: Connection to the database.
:param baseUrl: URL of the list of monsters of any element, where the element
                name has been replaced by  '#'.
:param elements: Array with all five elements.
:raise IndexError: The page doesn't match the expected format.
"""
def parse(db, baseUrl, elements):
    print('Fetching monster data...')
    http = urllib3.PoolManager()
    makeDirs()
    for elm in elements:
        url = baseUrl.replace('#', elm)
        try:
            mlist = http.request('GET', url).data.decode('utf-8')
            mlist = mlist[mlist.index('</th></tr>'):]
            element = url[url.rfind('/') + 1:url.rfind('_')]
            while '</td></tr>' in mlist:
                murl = 'http://summonerswar.wikia.com' + mlist[mlist.index('a href="') + 8:mlist.index('"', mlist.index('a href="') + 8)]
                parseMonster(db, murl, elm)
                mlist = mlist[mlist.index('</td></tr>') + 1:]
        except IndexError as e:
            print("Error processing " + elm + " monster list page: " + str(e))
            print("Maybe the page has changed and this parser is no longer valid.")
            raise
        #print("ENDED FIRE")
        #sys.exit("DEBUG EXIT")

    # Detect upgrade skills
    cursor = db.cursor()
    cursor.execute('UPDATE monster_skill SET upgrade = id + 1 WHERE name = (SELECT name FROM monster_skill s WHERE s.id = monster_skill.id + 1)')
    cursor.execute('UPDATE monster_skill SET monster = null WHERE id IN (SELECT upgrade FROM monster_skill);')
    cursor.close;

    # Get essence icons
    getEssenceIcons();

    # Create base monsters
    setBases(db)



"""
Indexes to keep track of the number of insertions in each affected table.
"""
id = {}
id["monster"] = 1;
id["monster_base"] = 1;
id["monster_skill"] = 1;
id["monster_essence"] = 1;
id["monster_location"] = 1;


"""
Begin script
"""
db = createDatabase('application/sw.sqlite')
parse(db, 'http://summonerswar.wikia.com/wiki/#_Monsters', ['Fire', 'Water', 'Wind', 'Light', 'Dark'])
db.close()