|
|
@@ -1,771 +0,0 @@
|
|
|
-#!/usr/bin/python3
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-import urllib3
|
|
|
-import sys
|
|
|
-import os
|
|
|
-import requests
|
|
|
-import urllib.request
|
|
|
-import sqlite3
|
|
|
-import wget
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Toggles.
|
|
|
-"""
|
|
|
-OPT_CREATE_DB = True
|
|
|
-OPT_DOWNLOAD = True
|
|
|
-OPT_SKILLS = True
|
|
|
-OPT_ESSENCES = True
|
|
|
-OPT_BASE = True
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Initializes the database file and creates all the tables.
|
|
|
-
|
|
|
-:param name: The path to the sqlite database.
|
|
|
-:returns: Connection to the database.
|
|
|
-:raises IntegrityError: The queryes couldn't bre executed.
|
|
|
-:raises IOError: The sqlite file couldn't be created.
|
|
|
-"""
|
|
|
-def createDatabase(name):
|
|
|
- global OPT_CREATE_DB
|
|
|
- print('Configuring database...')
|
|
|
- try:
|
|
|
- db = sqlite3.connect(name)
|
|
|
- if OPT_CREATE_DB == False:
|
|
|
- return db
|
|
|
- cursor = db.cursor()
|
|
|
- cursor.execute('''
|
|
|
- CREATE TABLE IF NOT EXISTS monster(
|
|
|
- id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
- base INTEGER,
|
|
|
- stars INT,
|
|
|
- element TEXT,
|
|
|
- name TEXT,
|
|
|
- name_aw TEXT,
|
|
|
- type TEXT,
|
|
|
- awakeable INT,
|
|
|
- img TEXT,
|
|
|
- img_aw TEXT
|
|
|
- );
|
|
|
- ''')
|
|
|
- cursor.execute('''
|
|
|
- CREATE TABLE IF NOT EXISTS monster_base(
|
|
|
- id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
- name TEXT
|
|
|
- );
|
|
|
- ''')
|
|
|
- cursor.execute('''
|
|
|
- CREATE TABLE IF NOT EXISTS monster_skill(
|
|
|
- id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
- monster INT,
|
|
|
- idx INT,
|
|
|
- name TEXT,
|
|
|
- desc TEXT,
|
|
|
- passive INT,
|
|
|
- awaken INT,
|
|
|
- upgrade INT,
|
|
|
- leader INT,
|
|
|
- img TEXT,
|
|
|
- cooldown INT
|
|
|
- );
|
|
|
- ''')
|
|
|
- cursor.execute('''
|
|
|
- CREATE TABLE IF NOT EXISTS monster_stat(
|
|
|
- monster INTEGER,
|
|
|
- stars INT,
|
|
|
- awake INT,
|
|
|
- stat TEXT,
|
|
|
- min INT,
|
|
|
- max INT
|
|
|
- );
|
|
|
- ''')
|
|
|
- cursor.execute('''
|
|
|
- CREATE TABLE IF NOT EXISTS monster_essence(
|
|
|
- monster INT,
|
|
|
- element TEXT,
|
|
|
- grade TEXT,
|
|
|
- num INT
|
|
|
- );
|
|
|
- ''')
|
|
|
- cursor.execute('''
|
|
|
- CREATE TABLE IF NOT EXISTS monster_location(
|
|
|
- monster INT,
|
|
|
- location TEXT
|
|
|
- );
|
|
|
- ''')
|
|
|
- cursor.execute('''
|
|
|
- CREATE TABLE IF NOT EXISTS monster_collection (
|
|
|
- id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
- monster INT,
|
|
|
- stars INT,
|
|
|
- level INT,
|
|
|
- awaken INT
|
|
|
- );
|
|
|
- ''')
|
|
|
- db.commit()
|
|
|
- cursor.close()
|
|
|
- return db
|
|
|
- except sqlite3.IntegrityError as e:
|
|
|
- print("Error executing statement: " + str(e))
|
|
|
- raise
|
|
|
- except IOError as e:
|
|
|
- print("I/O Error creating database " + name + ": " + str(e))
|
|
|
- raise
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Inserts a row into the database.
|
|
|
-
|
|
|
-:param db: Connection to the database.
|
|
|
-:param table: Name of the table to insert into.
|
|
|
-:param values: List of values to insert.
|
|
|
-:raises IntegrityError: The insert query was unsuccesfull.
|
|
|
-"""
|
|
|
-def insert(db, table, values):
|
|
|
- cursor = db.cursor()
|
|
|
- placeholders = ''
|
|
|
- for x in range(0, len(values)):
|
|
|
- placeholders = placeholders + '?, '
|
|
|
- placeholders = placeholders[:len(placeholders) - 2]
|
|
|
- query = 'INSERT INTO ' + table + ' VALUES (' + placeholders + ')'
|
|
|
- try:
|
|
|
- cursor.execute(query, values)
|
|
|
- except sqlite3.IntegrityError as e:
|
|
|
- print('Error Inserting into ' + table + ' with values (' + str(values) + '): ' + str(e))
|
|
|
- raise
|
|
|
- cursor.close;
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-def download(url, target):
|
|
|
- if OPT_DOWNLOAD == False:
|
|
|
- return True
|
|
|
- try:
|
|
|
- urllib.request.urlretrieve(url, target)
|
|
|
- return True
|
|
|
- except:
|
|
|
- return False
|
|
|
-
|
|
|
-
|
|
|
-def getDetails(page):
|
|
|
- details = []
|
|
|
- name = ''
|
|
|
- name_aw = ''
|
|
|
- img = ''
|
|
|
- img_aw = ''
|
|
|
- awakeable = 0
|
|
|
- mtype = ''
|
|
|
- stars = 1
|
|
|
- if ('alt="Purple Star"' in page):
|
|
|
- if ('alt="Gold Star"' in page):
|
|
|
- awakeable = 1 # Awakeable (Gold)
|
|
|
- else:
|
|
|
- awakeable = 2 # Only awaken (Purple)
|
|
|
- else:
|
|
|
- awakeable = 0 # No awakeable (Silver)
|
|
|
-
|
|
|
- try:
|
|
|
- stars = int(page[page.index('x', page.index('_Star.png"')) + 1:page.index(')', page.index('x', page.index('_Star.png"')))])
|
|
|
- except:
|
|
|
- stars = 1
|
|
|
- pass
|
|
|
-
|
|
|
- header = page[page.index('>') + 1:page.index('<', page.index('>'))]
|
|
|
- if ') - ' in header:
|
|
|
- name = header[:header.index(' (')]
|
|
|
- name_aw = header[header.index(') - ') + 4:]
|
|
|
- else:
|
|
|
- if ' (' in header:
|
|
|
- name = header[:header.index(' (')]
|
|
|
- else:
|
|
|
- name = header
|
|
|
-
|
|
|
- imgs = page[page.index('<div id="images">'):page.index('</div>', page.index('<div id="images">'))]
|
|
|
- if 'data-src="' in imgs:
|
|
|
- im = imgs[imgs.index('data-src="') + 10:imgs.index('"', imgs.index('data-src="') + 11)]
|
|
|
- imgs = imgs[imgs.index('"', imgs.index('data-src="') + 11):]
|
|
|
- else:
|
|
|
- im = imgs[imgs.index('<img src="') + 10:imgs.index('"', imgs.index('<img src="') + 11)]
|
|
|
- imgs = imgs[imgs.index('"', imgs.index('<img src="') + 11):]
|
|
|
-
|
|
|
- if 'data-src="' in imgs:
|
|
|
- img = im
|
|
|
- img_aw = imgs[imgs.index('data-src="') + 10:imgs.index('"', imgs.index('data-src="') + 11)]
|
|
|
- elif '<img src="' in imgs:
|
|
|
- img = im
|
|
|
- img_aw = imgs[imgs.index('<img src="') + 10:imgs.index('"', imgs.index('<img src="') + 11)]
|
|
|
- else:
|
|
|
- img = im
|
|
|
- img_aw = ''
|
|
|
-
|
|
|
- if awakeable == 2 and name_aw == '':
|
|
|
- name_aw = name
|
|
|
- name = ''
|
|
|
- if awakeable == 2 and img_aw == '':
|
|
|
- img_aw = img
|
|
|
- img = ''
|
|
|
-
|
|
|
- mtype = page[page.index('<b>Type:</b>') + 13:page.index('\n', page.index('<b>Type:</b>'))]
|
|
|
- details = [name, name_aw, img, img_aw, awakeable, mtype, stars]
|
|
|
- return details
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-def getObtainable(page):
|
|
|
- obtainable = []
|
|
|
- while 'title=' in page:
|
|
|
- obtainable.append(page[page.index('title="') + 7:page.index('"', page.index('title="') + 7)])
|
|
|
- page = page[page.index('"', page.index('title="') + 7):]
|
|
|
- return obtainable
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-def getEssences(page):
|
|
|
- essences = []
|
|
|
- while '\n' in page:
|
|
|
- line = page[:page.index('\n')]
|
|
|
- if '<span class="essence-number">' in line:
|
|
|
- num = line[line.index('<span class="essence-number">') + 29:line.index('</span>', line.index('<span class="essence-number">'))]
|
|
|
- desc = line[line.index('alt="') + 5:line.index('"', line.index('alt="') + 6)].split(" ")
|
|
|
- elm = desc[2]
|
|
|
- if elm == 'Darkness':
|
|
|
- elm = 'Dark'
|
|
|
- grade = desc[3]
|
|
|
- grade = grade[1:len(grade) - 1]
|
|
|
- essences.append([num, elm, grade])
|
|
|
- page = page[page.index('\n') + 1:]
|
|
|
- return essences
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-def getSkills(page):
|
|
|
- skills = []
|
|
|
- while '<td style="text-align' in page:
|
|
|
- passive = 0
|
|
|
- awaken = 0
|
|
|
- upgrade = 0
|
|
|
- leader = 0
|
|
|
- image = ''
|
|
|
- name = ''
|
|
|
- description = ''
|
|
|
- cooldown = 0
|
|
|
-
|
|
|
- name = page[page.index('alt="') + 5:page.index('"', page.index('alt="') + 6)]
|
|
|
- if ' (Passive)' in name:
|
|
|
- passive = 1
|
|
|
- name = name[:name.index(' (Passive)')]
|
|
|
- if 'Strengthened ' in name:
|
|
|
- upgrade = 1
|
|
|
- awaken = 1
|
|
|
- name = name[13:]
|
|
|
- if len(name) > 12 and name[:13] == 'Leader Skill ':
|
|
|
- leader = 1
|
|
|
- name = 'LEADER SKILL'
|
|
|
- if leader == 1:
|
|
|
- subawaken = page[:page.index(':</b> ')]
|
|
|
- else:
|
|
|
- subawaken = page[:page.index('</b>: ')]
|
|
|
- if '(Awakened)' in subawaken:
|
|
|
- awaken = 1
|
|
|
- image = page[page.index('data-src="') + 10:page.index('"', page.index('data-src="') + 11)]
|
|
|
- if leader == 1:
|
|
|
- description = page[page.index(':</b> ') + 6:page.index('\n', page.index(':</b> '))]
|
|
|
- else:
|
|
|
- description = page[page.index('</b>: ') + 6:page.index('\n', page.index('</b>: '))]
|
|
|
- if ' (Reusable in ' in description:
|
|
|
- try:
|
|
|
- cooldown = int(description[description.index(' (Reusable in ') + 14:description.index(' turns)', description.index(' (Reusable in '))])
|
|
|
- description = description[:description.index(' (Reusable in ')]
|
|
|
- except:
|
|
|
- pass
|
|
|
- if ' [Automatic Effect]' in description:
|
|
|
- description = description[:description.index(' [Automatic Effect]')]
|
|
|
-
|
|
|
- skills.append([name, description, passive, awaken, upgrade, leader, image, cooldown])
|
|
|
- page = page[page.index('</tr>', page.index('<td style="text-align')) + 10:]
|
|
|
- return skills
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-def getStats(page, awakeable):
|
|
|
- stats = []
|
|
|
- ahp = []
|
|
|
- aatk = []
|
|
|
- adef = []
|
|
|
- wahp = []
|
|
|
- waatk = []
|
|
|
- wadef = []
|
|
|
- spd = 0
|
|
|
- crir = 0
|
|
|
- crid = 0
|
|
|
- res = 0
|
|
|
- acc = 0
|
|
|
- wspd = 0
|
|
|
- wcrir = 0
|
|
|
- wcrid = 0
|
|
|
- wres = 0
|
|
|
- wacc = 0
|
|
|
-
|
|
|
- thp = page[page.index('<b>HP</b>'):page.index('</tr>', page.index('<b>HP</b>'))]
|
|
|
- while '<td class="monstercell">' in thp:
|
|
|
- n = thp[thp.index('<td class="monstercell">') + 24:thp.index('\n', thp.index('<td class="monstercell">'))]
|
|
|
- try:
|
|
|
- ahp.append(int(thp[thp.index('<td class="monstercell">') + 24:thp.index('\n', thp.index('<td class="monstercell">'))]))
|
|
|
- except Exception as e:
|
|
|
- pass
|
|
|
- thp = thp[thp.index('<td class="monstercell">') + 1:]
|
|
|
- tatk = page[page.index('<b>ATK</b>'):page.index('</tr>', page.index('<b>ATK</b>'))]
|
|
|
- while '<td class="monstercell">' in tatk:
|
|
|
- n = tatk[tatk.index('<td class="monstercell">') + 24:tatk.index('\n', tatk.index('<td class="monstercell">'))]
|
|
|
- try:
|
|
|
- aatk.append(int(tatk[tatk.index('<td class="monstercell">') + 24:tatk.index('\n', tatk.index('<td class="monstercell">'))]))
|
|
|
- except Exception as e:
|
|
|
- pass
|
|
|
- tatk = tatk[tatk.index('<td class="monstercell">') + 1:]
|
|
|
- tdef = page[page.index('<b>ATK</b>'):page.index('</tr>', page.index('<b>ATK</b>'))]
|
|
|
- while '<td class="monstercell">' in tdef:
|
|
|
- n = tdef[tdef.index('<td class="monstercell">') + 24:tdef.index('\n', tdef.index('<td class="monstercell">'))]
|
|
|
- try:
|
|
|
- adef.append(int(tdef[tdef.index('<td class="monstercell">') + 24:tdef.index('\n', tdef.index('<td class="monstercell">'))]))
|
|
|
- except Exception as e:
|
|
|
- pass
|
|
|
- tdef = tdef[tdef.index('<td class="monstercell">') + 1:]
|
|
|
- # STATS (awakened)
|
|
|
- if '#246BB2; width:10%;">Awakened' in page:
|
|
|
- mpag = page[page.index('#246BB2; width:10%;">Awakened'):]
|
|
|
- thp = page[page.index('<b>HP</b>'):page.index('</tr>', page.index('<b>HP</b>'))]
|
|
|
- while '<td class="monstercell">' in thp:
|
|
|
- n = thp[thp.index('<td class="monstercell">') + 24:thp.index('\n', thp.index('<td class="monstercell">'))]
|
|
|
- try:
|
|
|
- wahp.append(int(thp[thp.index('<td class="monstercell">') + 24:thp.index('\n', thp.index('<td class="monstercell">'))]))
|
|
|
- except Exception as e:
|
|
|
- pass
|
|
|
- thp = thp[thp.index('<td class="monstercell">') + 1:]
|
|
|
- tatk = page[page.index('<b>ATK</b>'):page.index('</tr>', page.index('<b>ATK</b>'))]
|
|
|
- while '<td class="monstercell">' in tatk:
|
|
|
- n = tatk[tatk.index('<td class="monstercell">') + 24:tatk.index('\n', tatk.index('<td class="monstercell">'))]
|
|
|
- try:
|
|
|
- waatk.append(int(tatk[tatk.index('<td class="monstercell">') + 24:tatk.index('\n', tatk.index('<td class="monstercell">'))]))
|
|
|
- except Exception as e:
|
|
|
- pass
|
|
|
- tatk = tatk[tatk.index('<td class="monstercell">') + 1:]
|
|
|
- tdef = page[page.index('<b>ATK</b>'):page.index('</tr>', page.index('<b>ATK</b>'))]
|
|
|
- while '<td class="monstercell">' in tdef:
|
|
|
- n = tdef[tdef.index('<td class="monstercell">') + 24:tdef.index('\n', tdef.index('<td class="monstercell">'))]
|
|
|
- try:
|
|
|
- wadef.append(int(tdef[tdef.index('<td class="monstercell">') + 24:tdef.index('\n', tdef.index('<td class="monstercell">'))]))
|
|
|
- except Exception as e:
|
|
|
- pass
|
|
|
- tdef = tdef[tdef.index('<td class="monstercell">') + 1:]
|
|
|
- mpag = page[page.index('Static Attributes'):]
|
|
|
- # STATIC STATS
|
|
|
- if awakeable == 1: # For awakeable monsters, skip one line
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- spd = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- crir = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- crid = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- res = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- acc = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- # Remove percent symbols and cast
|
|
|
- spd = int(spd.replace('<b>', '').replace('</b>', ''))
|
|
|
- crir = int(crir[:len(crir) - 1].replace('<b>', '').replace('</b>', ''))
|
|
|
- crid = int(crid[:len(crid) - 1].replace('<b>', '').replace('</b>', ''))
|
|
|
- res = int(res[:len(res) - 1].replace('<b>', '').replace('</b>', ''))
|
|
|
- acc = int(acc[:len(acc) - 1].replace('<b>', '').replace('</b>', ''))
|
|
|
- # Static stats, awakened
|
|
|
- if 'style="width: 9%;"><b>Awakened</b>' in page:
|
|
|
- page = page[page.index('style="width: 9%;"><b>Awakened</b>') + 10:]
|
|
|
- wspd = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- wcrir = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- wcrid = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- wres = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- wacc = page[page.index('style="width: 9%;">') + 19:page.index('\n', page.index('style="width: 9%;">'))]
|
|
|
- page = page[page.index('\n', page.index('style="width: 9%;">')):]
|
|
|
- # Remove percent symbols and cast
|
|
|
- wspd = int(wspd.replace('<b>', '').replace('</b>', ''))
|
|
|
- wcrir = int(wcrir[:len(wcrir) - 1].replace('<b>', '').replace('</b>', ''))
|
|
|
- wcrid = int(wcrid[:len(wcrid) - 1].replace('<b>', '').replace('</b>', ''))
|
|
|
- wres = int(wres[:len(wres) - 1].replace('<b>', '').replace('</b>', ''))
|
|
|
- wacc = int(wacc[:len(wacc) - 1].replace('<b>', '').replace('</b>', ''))
|
|
|
- else:
|
|
|
- wspd = None
|
|
|
- wcrir = None
|
|
|
- wcrid = None
|
|
|
- wres = None
|
|
|
- wacc = None
|
|
|
- stats = [ahp, aatk, adef, wahp, waatk, wadef, spd, crir, crid, res, acc, wspd, wcrir, wcrid, wres, wacc]
|
|
|
- return stats
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Parses the url with info abount one monster, extracts the info and saves it to
|
|
|
-the database.
|
|
|
-
|
|
|
-:param db: Connection to the database.
|
|
|
-:param url: URL with the info to parse.
|
|
|
-:param url: Element of the monster.
|
|
|
-:raises Exception: The monster data couldn't be saved.
|
|
|
-"""
|
|
|
-def parseMonster(db, url, element):
|
|
|
- http = urllib3.PoolManager()
|
|
|
- try:
|
|
|
- cursor = db.cursor()
|
|
|
-
|
|
|
- name = '';
|
|
|
- name_aw = '';
|
|
|
- img = '';
|
|
|
- img_aw = '';
|
|
|
- awakeable = 0;
|
|
|
- mtype = ''
|
|
|
- stars = 0
|
|
|
- obtainable = []
|
|
|
- essences = []
|
|
|
- skills = []
|
|
|
- ahp = []
|
|
|
- aatk = []
|
|
|
- adef = []
|
|
|
- wahp = []
|
|
|
- waatk = []
|
|
|
- wadef = []
|
|
|
- spd = 0
|
|
|
- crir = 0
|
|
|
- crid = 0
|
|
|
- res = 0
|
|
|
- acc = 0
|
|
|
- wspd = 0
|
|
|
- wcrir = 0
|
|
|
- wcrid = 0
|
|
|
- wres = 0
|
|
|
- wacc = 0
|
|
|
-
|
|
|
- # DEBUG - GOLD
|
|
|
- #url = 'http://summonerswar.wikia.com/wiki/Fairy_(Water)_-_Elucia'
|
|
|
- # DEBUG - SILVER
|
|
|
- #url = 'http://summonerswar.wikia.com/wiki/Forest_Keeper_(Water)'
|
|
|
- # DEBUG - PURPLE
|
|
|
- #url = 'http://summonerswar.wikia.com/wiki/Rainbowmon'
|
|
|
- # DEBUG - Upgradeable skill
|
|
|
- #url = 'http://summonerswar.wikia.com/wiki/Paladin_(Wind)_-_Louise'
|
|
|
-
|
|
|
- mpage = http.request('GET', url).data.decode('utf-8');
|
|
|
-
|
|
|
- # Basic details
|
|
|
- detailsPage = mpage[mpage.index('<h1 class="page-header__title">'):mpage.index('</tr>', mpage.index('<b>Type:</b>', mpage.index('<h1 class="page-header__title">')))]
|
|
|
- details = getDetails(detailsPage)
|
|
|
- name = details[0]
|
|
|
- name_aw = details[1]
|
|
|
- img = details[2]
|
|
|
- img_aw = details[3]
|
|
|
- awakeable = details[4]
|
|
|
- mtype = details[5]
|
|
|
- stars = details[6]
|
|
|
-
|
|
|
- # Check if monster already parsed
|
|
|
- cursor.execute('SELECT count(id) AS c FROM monster WHERE element = ? AND name = ? AND name_aw = ?', (element, name, name_aw))
|
|
|
- if cursor.fetchone()[0] == 0:
|
|
|
-
|
|
|
- # Regions where the monster can be obtained
|
|
|
- obtainablePage = mpage[mpage.index('<b>Obtainable from:</b>'):mpage.index('</td>', mpage.index('<b>Obtainable from:</b>'))]
|
|
|
- obtainable = getObtainable(obtainablePage)
|
|
|
-
|
|
|
- # Essences required to awaken the monster
|
|
|
- if awakeable == 1:
|
|
|
- essencesPage = mpage[mpage.index('<div id="essences">'):mpage.index('</table>', mpage.index('<div id="essences">'))]
|
|
|
- essences = getEssences(essencesPage)
|
|
|
-
|
|
|
- # Skills
|
|
|
- skillsPage = mpage[mpage.index('<table', mpage.index('<table', mpage.index('<div id="skills">') + 1) + 1):mpage.index('</table>', mpage.index('<table', mpage.index('<table', mpage.index('<div id="skills">') + 1) + 1))]
|
|
|
- skills = getSkills(skillsPage)
|
|
|
-
|
|
|
- # Stats
|
|
|
- statsPage = mpage[mpage.index('Evolution Stats'):]
|
|
|
- stats = getStats(statsPage, awakeable)
|
|
|
- ahp = stats[0]
|
|
|
- aatk = stats[1]
|
|
|
- adef = stats[2]
|
|
|
- wahp = stats[3]
|
|
|
- waatk = stats[4]
|
|
|
- wadef = stats[5]
|
|
|
- spd = stats[6]
|
|
|
- crir = stats[7]
|
|
|
- crid = stats[8]
|
|
|
- res = stats[9]
|
|
|
- acc = stats[10]
|
|
|
- wspd = stats[11]
|
|
|
- wcrir = stats[12]
|
|
|
- wcrid = stats[13]
|
|
|
- wres = stats[14]
|
|
|
- wacc = stats[15]
|
|
|
-
|
|
|
- # Save monster
|
|
|
- strname = element + ' ' + name + " - " + name_aw
|
|
|
- if awakeable == 0:
|
|
|
- strname = element + ' ' + name
|
|
|
- elif awakeable == 2:
|
|
|
- strname = name_aw
|
|
|
- print (" Saving monster: " + strname + "...")
|
|
|
- if img != '':
|
|
|
- if download(img, 'public/img/content/monster/' + str(id["monster"]).zfill(3) + '.png'):
|
|
|
- img = str(id["monster"]).zfill(3) + '.png'
|
|
|
- else:
|
|
|
- print("Error downloading image for monster " + str(id["monster"]))
|
|
|
- if img != '':
|
|
|
- if download(img_aw, 'public/img/content/monster/' + str(id["monster"]).zfill(3) + '_aw.png'):
|
|
|
- img_aw = str(id["monster"]).zfill(3) + '_aw.png'
|
|
|
- else:
|
|
|
- print("Error downloading image for awakened monster " + str(id["monster"]))
|
|
|
- insert(db, 'monster', (id["monster"], None, stars, element, name, name_aw, mtype, awakeable, img, img_aw))
|
|
|
-
|
|
|
- # Save locations
|
|
|
- for o in obtainable:
|
|
|
- insert(db, 'monster_location', (id["monster"], o))
|
|
|
- for e in essences:
|
|
|
- insert(db, 'monster_essence', (id["monster"], e[1], e[2], e[0]))
|
|
|
-
|
|
|
- # Save skills
|
|
|
- idx = 0
|
|
|
- for s in skills:
|
|
|
- # 0 1 2 3 4 5 6 7
|
|
|
- # id monster idx name desc passive awaken upgrade leader img cooldown
|
|
|
- if download(s[6], 'public/img/content/skill/' + str(id["monster_skill"]).zfill(4) + '.png'):
|
|
|
- s[6] = str(id["monster_skill"]).zfill(4) + '.png'
|
|
|
- else:
|
|
|
- s[6] = ''
|
|
|
- if s[4] == 1:
|
|
|
- s[4] = id["monster_skill"] - 1
|
|
|
- insert(db, 'monster_skill', (id["monster_skill"], id["monster"], idx, s[0], s[1], s[2], s[3], None, s[5], s[6], s[7]))
|
|
|
- idx = idx + 1
|
|
|
- id["monster_skill"] = id["monster_skill"] + 1
|
|
|
-
|
|
|
- # Save stats
|
|
|
- i = 0
|
|
|
- s = stars
|
|
|
- while i < len(ahp):
|
|
|
- if i + 1 < len(ahp):
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 0, 'HP', ahp[i], ahp[i + 1]))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 0, 'ATK', aatk[i], aatk[i + 1]))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 0, 'DEF', aatk[i], aatk[i + 1]))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 0, 'SPD', spd, spd))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 0, 'CRIR', crir, crir))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 0, 'CRID', crid, crid))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 0, 'RES', res, res))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 0, 'ACC', acc, acc))
|
|
|
- if i + 1 < len(wahp):
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 1, 'HP', wahp[i], wahp[i + 1]))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 1, 'ATK', waatk[i], waatk[i + 1]))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 1, 'DEF', waatk[i], waatk[i + 1]))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 1, 'SPD', wspd, wspd))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 1, 'CRIR', wcrir, wcrir))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 1, 'CRID', wcrid, wcrid))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 1, 'RES', wres, wres))
|
|
|
- insert(db, 'monster_stat', (id["monster"], s, 1, 'ACC', wacc, wacc))
|
|
|
- s = s + 1
|
|
|
- i = i + 2
|
|
|
-
|
|
|
- # Save and proceed
|
|
|
- db.commit()
|
|
|
- id["monster"] = id["monster"] + 1
|
|
|
-
|
|
|
- except IOError as e:
|
|
|
- db.rollback()
|
|
|
- print('Error saving monster with id ' + str(id['monster']) + ': ' + str(e))
|
|
|
- print(' Current data:')
|
|
|
- print(' Name: ' + name)
|
|
|
- print(' Name (awaken): ' + name_aw)
|
|
|
- print(' Element: ' + element)
|
|
|
- print(' Type: ' + mtype)
|
|
|
- print(' Awakeable: ' + str(awakeable))
|
|
|
- print(' Obtainable:')
|
|
|
- for o in obtainable:
|
|
|
- print(' ' + o)
|
|
|
- print(' Essences:')
|
|
|
- for e in essences:
|
|
|
- print(' ' + e[0] + ' x ' + e[2] + ' ' + e[1] + ' essences')
|
|
|
- print(' Skills:')
|
|
|
- for s in skills:
|
|
|
- print(' ' + s[1])
|
|
|
- pass
|
|
|
- #sys.exit("DEBUG EXIT")
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-def makeDirs():
|
|
|
- dirs = ['public/img/content/monster/', 'public/img/content/skill/', 'public/img/content/essence/']
|
|
|
- for dir in dirs:
|
|
|
- try:
|
|
|
- if os.path.isdir(dir) == False:
|
|
|
- os.makedirs(dir)
|
|
|
- except IOError as e:
|
|
|
- print('Unable to create directory: ' + dir);
|
|
|
- raise
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Create the entries for the table monster_base, reading data from the table
|
|
|
-monster.
|
|
|
-
|
|
|
-:param db: Connection to the database.
|
|
|
-:raise IntegrityError: One or more bases could not be created.
|
|
|
-"""
|
|
|
-def createBases(db):
|
|
|
- cursor = db.cursor()
|
|
|
- cursor_base = db.cursor()
|
|
|
- try:
|
|
|
- cursor.execute('SELECT DISTINCT name FROM monster;')
|
|
|
-
|
|
|
- for row in cursor:
|
|
|
- try:
|
|
|
- cursor_base.execute('INSERT INTO monster_base VALUES (?, ?);', (id["monster_base"], row[0]));
|
|
|
- cursor.execute('UPDATE monster SET base = ? WHERE name = ?;', (id["monster_base"], row[0]));
|
|
|
- except sqlite3.IntegrityError as e:
|
|
|
- print("Error creating monster bases: " + str(e))
|
|
|
- raise
|
|
|
- id["monster_base"] = id["monster_base"] + 1;
|
|
|
- db.commit()
|
|
|
- cursor_base.close();
|
|
|
- except sqlite3.IntegrityError as e:
|
|
|
- print("Error creating monster bases: " + str(e))
|
|
|
- raise
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Downloads the essence icons.
|
|
|
-
|
|
|
-"""
|
|
|
-def getEssenceIcons():
|
|
|
- url = 'http://summonerswar.wikia.com/wiki/Essences'
|
|
|
- http = urllib3.PoolManager()
|
|
|
- page = http.request('GET', url).data.decode('utf-8')
|
|
|
- page = page[page.index('id="List_of_Essences"'):page.index('id="Essence_fusion"')]
|
|
|
- while 'data-src="' in page:
|
|
|
- name = page[page.index('data-image-name="') + 7:page.index('"', page.index('data-image-name="') + 17)].split(" ")
|
|
|
- element = name[2]
|
|
|
- if element == 'Darkness':
|
|
|
- element = 'Dark'
|
|
|
- grade = name[3][1:len(name)]
|
|
|
- file = page[page.index('data-src="') + 10:page.index('"', page.index('data-src="') + 11)]
|
|
|
- if download(file, 'public/img/content/essence/' + element + '_' + grade + '.png') == False:
|
|
|
- print("Unable to save image for essence: " + element + " (" + grade + ")");
|
|
|
- page = page[page.index('data-src="') + 10:]
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-def setBases(db):
|
|
|
- cursor = db.cursor()
|
|
|
- cursor_base = db.cursor()
|
|
|
- cursor.execute('SELECT DISTINCT name FROM monster;')
|
|
|
- for row in cursor:
|
|
|
- cursor_base.execute('INSERT INTO monster_base VALUES (?, ?);', (id["monster_base"], row[0]));
|
|
|
- cursor_base.execute('UPDATE monster SET base = ? WHERE name = ?;', (id["monster_base"], row[0]));
|
|
|
- id["monster_base"] = id["monster_base"] + 1;
|
|
|
- db.commit()
|
|
|
- cursor_base.close();
|
|
|
- cursor.close()
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Starts scrapping th page.
|
|
|
-
|
|
|
-:param db: Connection to the database.
|
|
|
-:param baseUrl: URL of the list of monsters of any element, where the element
|
|
|
- name has been replaced by '#'.
|
|
|
-:param elements: Array with all five elements.
|
|
|
-:raise IndexError: The page doesn't match the expected format.
|
|
|
-"""
|
|
|
-def parse(db, baseUrl, elements):
|
|
|
- print('Fetching monster data...')
|
|
|
- http = urllib3.PoolManager()
|
|
|
- makeDirs()
|
|
|
- for elm in elements:
|
|
|
- url = baseUrl.replace('#', elm)
|
|
|
- try:
|
|
|
- mlist = http.request('GET', url).data.decode('utf-8')
|
|
|
- mlist = mlist[mlist.index('</th></tr>'):]
|
|
|
- element = url[url.rfind('/') + 1:url.rfind('_')]
|
|
|
- while '</td></tr>' in mlist:
|
|
|
- murl = 'http://summonerswar.wikia.com' + mlist[mlist.index('a href="') + 8:mlist.index('"', mlist.index('a href="') + 8)]
|
|
|
- parseMonster(db, murl, elm)
|
|
|
- mlist = mlist[mlist.index('</td></tr>') + 1:]
|
|
|
- except IndexError as e:
|
|
|
- print("Error processing " + elm + " monster list page: " + str(e))
|
|
|
- print("Maybe the page has changed and this parser is no longer valid.")
|
|
|
- raise
|
|
|
- #print("ENDED FIRE")
|
|
|
- #sys.exit("DEBUG EXIT")
|
|
|
-
|
|
|
- # Detect upgrade skills
|
|
|
- cursor = db.cursor()
|
|
|
- cursor.execute('UPDATE monster_skill SET upgrade = id + 1 WHERE name = (SELECT name FROM monster_skill s WHERE s.id = monster_skill.id + 1)')
|
|
|
- cursor.execute('UPDATE monster_skill SET monster = null WHERE id IN (SELECT upgrade FROM monster_skill);')
|
|
|
- cursor.close;
|
|
|
-
|
|
|
- # Get essence icons
|
|
|
- getEssenceIcons();
|
|
|
-
|
|
|
- # Create base monsters
|
|
|
- setBases(db)
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Indexes to keep track of the number of insertions in each affected table.
|
|
|
-"""
|
|
|
-id = {}
|
|
|
-id["monster"] = 1;
|
|
|
-id["monster_base"] = 1;
|
|
|
-id["monster_skill"] = 1;
|
|
|
-id["monster_essence"] = 1;
|
|
|
-id["monster_location"] = 1;
|
|
|
-
|
|
|
-
|
|
|
-"""
|
|
|
-Begin script
|
|
|
-"""
|
|
|
-db = createDatabase('application/sw.sqlite')
|
|
|
-parse(db, 'http://summonerswar.wikia.com/wiki/#_Monsters', ['Fire', 'Water', 'Wind', 'Light', 'Dark'])
|
|
|
-db.close()
|