Basic parser added

.gitignore

@@ -1,7 +1,7 @@
 # Private files
 application/config/auth.php_*
 util/
-parser/
+parser/OLD/
 data/*.sqlite
 
 # Autogenerated files

parser/parse.py

@@ -1,83 +0,0 @@
-#!/usr/bin/python3
-
-import urllib3
-import sys
-import os
-import requests
-
-
-urls = [
-  'http://summonerswar.wikia.com/wiki/Fire_Monsters',
-  'http://summonerswar.wikia.com/wiki/Water_Monsters',
-  'http://summonerswar.wikia.com/wiki/Wind_Monsters',
-  'http://summonerswar.wikia.com/wiki/Light_Monsters',
-  'http://summonerswar.wikia.com/wiki/Dark_Monsters'
-]
-http = urllib3.PoolManager()
-fsql = open('data.sql', 'w')
-fget = open('img.sh', 'w')
-id = 1;
-
-for url in urls:
-    page = http.request('GET', url).data.decode('utf-8')
-    page = page[page.index('</th></tr>'):]
-    element = url[url.rfind('/') + 1:url.rfind('_')]
-
-    while '</td></tr>' in page:
-        
-        murl = 'http://summonerswar.wikia.com/' + page[page.index('a href="') + 8:page.index('"', page.index('a href="') + 8)]
-        print(murl)
-        page = page[page.index('</td></tr>') + 1:]
-
-        '''page = page[page.index('<td style="text-align:center;">') + 1:]
-        monster = page[:page.index('</tr>') + 5]
-        name = monster[monster.index('title="') + 7:monster.index(' (', monster.index('title="') + 8)]
-        monster = monster[monster.index('"', monster.index('title="') + 8):]
-        img = monster[monster.index('data-src="') + 10:monster.index('"', monster.index('data-src="') + 11)]
-        monster = monster[monster.index('<td style="text-align:center;">') + 1:]
-        monster = monster[monster.index('<td style="text-align:center;">') + 1:]
-        tpe = monster[monster.index('>') + 1:monster.index('\n')]
-        monster = monster[monster.index('<td style="text-align:center;">') + 1:]
-        monster = monster[monster.index('<td style="text-align:center;">') + 1:]
-        hp = monster[monster.index('>') + 1:monster.index('\n')]
-        monster = monster[monster.index('<td style="text-align:center;">') + 1:]
-        atk = monster[monster.index('>') + 1:monster.index('\n')]
-        monster = monster[monster.index('<td style="text-align:center;">') + 1:]
-        df = monster[monster.index('>') + 1:monster.index('\n')]
-        monster = monster[monster.index('<td style="text-align:center;">') + 1:]
-        spd = monster[monster.index('>') + 1:monster.index('\n')]
-        print("NAME:    " + name)
-        print("ELEMENT: " + element)
-        print("IMG:     " + img)
-        print("TYPE:    " + tpe)
-        print("HP:      " + hp)
-        print("ATK:     " + atk)
-        print("DEF:     " + df)
-        print("SPD:     " + spd)
-        sys.exit("DEBUG EXIT")
-
-        
-
-        element = monster[monster.index('"element">') + 10:monster.index('<', monster.index('"element">'))]
-        print("ELEMENT: " + element)
-        monster = monster[monster.index('<', monster.index('"element">')):]
-
-        name = monster[monster.index('<h3>') + 4:monster.index('</h3>')]
-        print("NAME: " + name)
-        monster = monster[monster.index('</h3>'):]
-
-        img_aw = monster[monster.index('data-src="', monster.index('icon')) + 10:monster.index('"', monster.index('data-src="', monster.index('icon')) + 11)]
-        print("IMAGE (AW): " + img_aw)
-        monster = monster[monster.index('"', monster.index('data-src="', monster.index('icon')) + 11):]
-
-        name_aw = monster[monster.index('>', monster.index('"name"')) + 1:monster.index('<', monster.index('"name"'))]
-        print("NAME (AW): " + name_aw)
-        monster = monster[monster.index('<', monster.index('"name"')):]
-
-        print("wget " + img + " -O monster/" + "{:04d}".format(id) + ".png", file=fget)
-        print("wget " + img_aw + " -O monster/awaken/" + "{:04d}".format(id) + ".png", file=fget)
-        print("INSERT INTO monster (id, name, name_aw, element, stars, base) VALUES (" + str(id) + ", '" + name + "', '" + name_aw + "', '" + element + "', " + str(star) + ", null);", file=fsql)
-
-        id = id + 1
-        #sys.exit("DEBUG EXIT")
-        '''

parser/swarfarm_parser

@@ -0,0 +1,731 @@
+#!/bin/php
+<?php
+
+    /**
+     * Swarfarm URL of the monster list (first page).
+     */
+    CONST URL_UNIT = 'https://swarfarm.com/api/v2/monsters/';
+
+    /**
+     * Swarfarm URL of the skill list (first page).
+     */
+    CONST URL_SKILL = 'https://swarfarm.com/api/v2/skills/';
+
+    /**
+     * Path to the application
+     */
+    CONST PATH = __DIR__ . "/../";
+
+    /**
+     * Script verbosity
+     *
+     * 0: Critical
+     * 1: Error
+     * 2: Warning
+     * 3: Info
+     * 4: Debug
+     * 5: Trace
+     */
+    $verbosity = 4;
+
+    /**
+     * Echoes an output message to stdout, depending on the verbosity level.
+     *
+     * @param int $level Message priority.
+     * @param string $message content.
+     */
+    function output($level, $message){
+        global $verbosity;
+        if ($level <= $verbosity){
+            if (substr("testers", -1) != PHP_EOL){
+                $message .= PHP_EOL;
+            }
+            echo $message;
+        }
+    }
+
+    /**
+     * Gets an entity com2us_id from it's Swarfarm id.
+     *
+     * @param int $swarfarm_id Entity's Swarfarm ID.
+     * @return int Entity's com2us_id, -1 on error.
+     */
+    function get_com2us_id($swarfarm_id){
+        if ($swarfarm_id == null){
+            return null;
+        }
+        $contents = file_get_contents(URL_UNIT . $swarfarm_id . "/");
+        if($contents === false){
+            //Print out the contents.
+            output(1, "Unable to retrieve monster.");
+            return -1;
+        }
+        $contents = json_decode($contents);
+        return $contents->com2us_id;
+    }
+
+    /**
+     * Gets an element ID from it's name
+     *
+     * @param string $name Element name (case insensitive).
+     * @return int Element ID, -1 on unknown element.
+     */
+    function get_element_id($name){
+        $id = -1;
+        switch (strtoupper($name)){
+            case "MAGIC":
+                $id = 0;
+                break;
+            case "WATER":
+                $id = 1;
+                break;
+            case "FIRE":
+                $id = 2;
+                break;
+            case "WIND":
+                $id = 3;
+                break;
+            case "LIGHT":
+                $id = 4;
+                break;
+            case "DARK":
+                $id = 5;
+                break;
+            case "PURE":
+                $id = 6;
+                break;
+        }
+        return $id;
+    }
+
+    /**
+     * Gets an archetype ID from it's name
+     *
+     * @param string $name Archetype name (case insensitive).
+     * @return int Archetype ID, -1 on unknown archetype.
+     */
+    function get_archetype_id($name){
+        $id = 0; // NONE
+        switch (strtoupper($name)){
+            case "ATTACK":
+                $id = 1;
+                break;
+            case "DEFENSE":
+                $id = 2;
+                break;
+            case "HP":
+                $id = 3;
+                break;
+            case "SUPPORT":
+                $id = 4;
+                break;
+            case "MATERIAL":
+                $id = 5;
+                break;
+        }
+        return $id;
+    }
+
+    /**
+     * Gets an archetype ID from it's name
+     *
+     * @param string $name Archetype name (case insensitive).
+     * @return int Archetype ID, -1 on unknown archetype.
+     */
+    function connect_to_database(){
+        // Connect to the database
+        $db = new SQLite3(PATH . "application/sw.sqlite");
+        if (!$db) {
+            output(0, "Error connecting to th database: " . $db->lastErrorMsg());
+            exit();
+        }
+        // Disable integrity
+        $db->query("PRAGMA foreign_keys=OFF;");
+        return $db;
+    }
+
+    // TODO: Parse arguments
+
+    // Connect to the database
+    $db = connect_to_database();
+
+    // Loop skills pages
+    $next = URL_SKILL;
+    while ($next != null){
+        output(3, "Reading skills page: $next");
+        $contents = file_get_contents($next);
+        if ($contents === false){
+            output(0, "Unable to read skill page: $next");
+            return -1;
+        }
+        $contents = json_decode($contents);
+        $next = $contents->next;
+        $list = $contents->results;
+        foreach ($list as $skill){
+            $statement = $db->prepare("SELECT * FROM k_skill WHERE id = :id");
+            $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+            $q = $statement->execute();
+            $r = $q->fetchArray(SQLITE3_ASSOC);
+            if (!$r){
+                $statement = $db->prepare("
+                  INSERT INTO k_skill(
+                    id,
+                    name,
+                    description,
+                    slot,
+                    cooltime,
+                    hits,
+                    passive,
+                    aoe,
+                    max_level,
+                    multiplier_formula
+                  ) VALUES (
+                    :id,
+                    :name,
+                    :description,
+                    :slot,
+                    :cooltime,
+                    :hits,
+                    :passive,
+                    :aoe,
+                    :max_level,
+                    :multiplier_formula
+                  );
+                ");
+                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                $statement->bindValue(":name", $skill->name, SQLITE3_TEXT);
+                $statement->bindValue(":description", $skill->name, SQLITE3_TEXT);
+                $statement->bindValue(":slot", $skill->slot, SQLITE3_TEXT);
+                $statement->bindValue(":cooltime", $skill->cooltime, SQLITE3_INTEGER);
+                $statement->bindValue(":hits", $skill->hits, SQLITE3_INTEGER);
+                $statement->bindValue(":passive", $skill->passive, SQLITE3_INTEGER);
+                $statement->bindValue(":aoe", $skill->aoe, SQLITE3_INTEGER);
+                $statement->bindValue(":max_level", $skill->max_level, SQLITE3_INTEGER);
+                $statement->bindValue(":multiplier_formula", $skill->multiplier_formula, SQLITE3_INTEGER);
+                output(4, "Execute query: ");
+                output(4, $statement->getSQL(true));
+                $statement->execute();
+                $url = 'https://swarfarm.com/static/herders/images/skills/' . $skill->icon_filename;
+                $out = PATH . "public/img/skill/" . sprintf('%08d', $skill->com2us_id) . ".png";
+                output(4, "Downloading file '$url' to: '$out'");
+                if (!file_put_contents($out, file_get_contents($url))) { 
+                    output(1, "Error downloading file '$url' to: '$out'");
+                }
+                $level = 1;
+                $statement = $db->prepare("INSERT INTO k_skill_level (skill, level, description) VALUES (:skill, :level, :description);");
+                $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
+                $statement->bindValue(":level", $level, SQLITE3_INTEGER);
+                foreach ($skill->upgrades as $upgrade){
+                    $level ++;
+                    $statement = $db->prepare("INSERT INTO k_skill_level (skill, level, description) VALUES (:skill, :level, :description);");
+                    $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
+                    $statement->bindValue(":level", $level, SQLITE3_INTEGER);
+                    $statement->bindValue(":description", str_replace("{0}", $upgrade->amount, $upgrade->effect), SQLITE3_TEXT);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                // TODO: Get data for other tables: 
+                // k_skill_effect
+            }
+            else{
+                // Compare all values
+                if ($r["name"] != $skill->name){
+                    $statement = $db->prepare("UPDATE k_skill SET name = :name WHERE id = :id;");
+                    $statement->bindValue(":name", $skill->name, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["description"] != $skill->description){
+                    $statement = $db->prepare("UPDATE k_skill SET description = :description WHERE id = :id;");
+                    $statement->bindValue(":description", $skill->description, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["slot"] != $skill->slot){
+                    $statement = $db->prepare("UPDATE k_skill SET slot = :slot WHERE id = :id;");
+                    $statement->bindValue(":slot", $skill->slot, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["cooltime"] != $skill->cooltime){
+                    $statement = $db->prepare("UPDATE k_skill SET cooltime = :cooltime WHERE id = :id;");
+                    $statement->bindValue(":cooltime", $skill->cooltime, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["hits"] != $skill->hits){
+                    $statement = $db->prepare("UPDATE k_skill SET hits = :hits WHERE id = :id;");
+                    $statement->bindValue(":hits", $skill->hits, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                $passive = ($skill->passive == 1) ? (1) : (0);
+                if ($r["passive"] != $passive){
+                    $statement = $db->prepare("UPDATE k_skill SET passive = :passive WHERE id = :id;");
+                    $statement->bindValue(":passive", $passive, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                $aoe = ($skill->aoe == 1) ? (1) : (0);
+                if ($r["aoe"] != $aoe){
+                    $statement = $db->prepare("UPDATE k_skill SET aoe = :aoe WHERE id = :id;");
+                    $statement->bindValue(":aoe", $aoe, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["max_level"] != $skill->max_level){
+                    $statement = $db->prepare("UPDATE k_skill SET max_level = :max_level WHERE id = :id;");
+                    $statement->bindValue(":max_level", $skill->max_level, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["multiplier_formula"] != $skill->multiplier_formula){
+                    $statement = $db->prepare("UPDATE k_skill SET multiplier_formula = :multiplier_formula WHERE id = :id;");
+                    $statement->bindValue(":multiplier_formula", $skill->multiplier_formula, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+
+                // Image download
+                $out = PATH . "public/img/skill/" . sprintf('%08d', $skill->com2us_id) . ".png";
+                if (!file_exists($out)){
+                    $url = 'https://swarfarm.com/static/herders/images/skills/' . $skill->icon_filename;   
+                    output(4, "Downloading file '$url' to: '$out'");
+                    if (!file_put_contents($out, file_get_contents($url))) { 
+                        output(1, "Error downloading file '$url' to: '$out'");
+                    } 
+                }
+            }
+        }
+        //echo "NEXTPAGE: " . $next;
+    }
+
+    // Loop units pages
+    $next = URL_UNIT;
+    while ($next != null){
+        output(3, "Reading units page: $next");
+        $contents = file_get_contents($next);
+        if($contents === false){
+            output(0, "Unable to retrieve monster list.");
+            return -1;
+        }
+        $contents = json_decode($contents);
+        $next = $contents->next;
+        $list = $contents->results;
+        foreach ($list as $unit){
+            $statement = $db->prepare("SELECT * FROM k_unit WHERE id = :id");
+            $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+            $q = $statement->execute();
+            $r = $q->fetchArray(SQLITE3_ASSOC);
+            if (!$r){
+                $statement = $db->prepare("
+                  INSERT INTO k_unit(
+                    id,
+                    family,
+                    name,
+                    element,
+                    archetype,
+                    base_stars,
+                    natural_stars,
+                    obtainable,
+                    can_awaken,
+                    awaken_bonus,
+                    skill_ups_to_max,
+                    leader_skill,
+                    hp,
+                    attack,
+                    defense,
+                    speed,
+                    crit_rate,
+                    crit_damage,
+                    resistance,
+                    accuracy,
+                    raw_hp,
+                    raw_attack,
+                    raw_defense,
+                    max_lvl_hp,
+                    max_lvl_attack,
+                    max_lvl_defense,
+                    awakens_from,
+                    awakens_to,
+                    fusion_food,
+                    homunculus,
+                    craft_cost
+                  ) VALUES (
+                    :id,
+                    :family,
+                    :name,
+                    :element,
+                    :archetype,
+                    :base_stars,
+                    :natural_stars,
+                    :obtainable,
+                    :can_awaken,
+                    :awaken_bonus,
+                    :skill_ups_to_max,
+                    :leader_skill,
+                    :hp,
+                    :attack,
+                    :defense,
+                    :speed,
+                    :crit_rate,
+                    :crit_damage,
+                    :resistance,
+                    :accuracy,
+                    :raw_hp,
+                    :raw_attack,
+                    :raw_defense,
+                    :max_lvl_hp,
+                    :max_lvl_attack,
+                    :max_lvl_defense,
+                    :awakens_from,
+                    :awakens_to,
+                    :fusion_food,
+                    :homunculus,
+                    :craft_cost
+                  );
+                ");
+                $statement->bindValue(":id", $unit->com2us_id, SQLITE3_TEXT);
+                $statement->bindValue(":family", $unit->family_id, SQLITE3_INTEGER);
+                $statement->bindValue(":name", $unit->name, SQLITE3_TEXT);
+                $statement->bindValue(":element", get_element_id($unit->element), SQLITE3_INTEGER);
+                $statement->bindValue(":archetype", get_archetype_id($unit->archetype), SQLITE3_INTEGER);
+                $statement->bindValue(":base_stars", $unit->base_stars, SQLITE3_INTEGER);
+                $statement->bindValue(":natural_stars", $unit->natural_stars, SQLITE3_INTEGER);
+                $statement->bindValue(":obtainable", $unit->obtainable, SQLITE3_INTEGER);
+                $statement->bindValue(":can_awaken", $unit->can_awaken, SQLITE3_INTEGER);
+                $statement->bindValue(":awaken_bonus", $unit->awaken_bonus, SQLITE3_TEXT);
+                $statement->bindValue(":skill_ups_to_max", $unit->skill_ups_to_max, SQLITE3_INTEGER);
+                $statement->bindValue(":leader_skill", $unit->leader_skill, SQLITE3_INTEGER);
+                $statement->bindValue(":hp", $unit->base_hp, SQLITE3_INTEGER);
+                $statement->bindValue(":attack", $unit->base_attack, SQLITE3_INTEGER);
+                $statement->bindValue(":defense", $unit->base_defense, SQLITE3_INTEGER);
+                $statement->bindValue(":speed", $unit->speed, SQLITE3_INTEGER);
+                $statement->bindValue(":crit_rate", $unit->crit_rate, SQLITE3_INTEGER);
+                $statement->bindValue(":crit_damage", $unit->crit_damage, SQLITE3_INTEGER);
+                $statement->bindValue(":resistance", $unit->resistance, SQLITE3_INTEGER);
+                $statement->bindValue(":accuracy", $unit->accuracy, SQLITE3_INTEGER);
+                $statement->bindValue(":raw_hp", $unit->raw_hp, SQLITE3_INTEGER);
+                $statement->bindValue(":raw_attack", $unit->raw_attack, SQLITE3_INTEGER);
+                $statement->bindValue(":raw_defense", $unit->raw_defense, SQLITE3_INTEGER);
+                $statement->bindValue(":max_lvl_hp", $unit->max_lvl_hp, SQLITE3_INTEGER);
+                $statement->bindValue(":max_lvl_attack", $unit->max_lvl_attack, SQLITE3_INTEGER);
+                $statement->bindValue(":max_lvl_defense", $unit->max_lvl_defense, SQLITE3_INTEGER);
+                $statement->bindValue(":awakens_from", get_com2us_id($unit->awakens_from), SQLITE3_INTEGER);
+                $statement->bindValue(":awakens_to", get_com2us_id($unit->awakens_to), SQLITE3_TEXT);
+                $statement->bindValue(":fusion_food", $unit->fusion_food, SQLITE3_INTEGER);
+                $statement->bindValue(":homunculus", $unit->homunculus, SQLITE3_INTEGER);
+                $statement->bindValue(":craft_cost", 0, SQLITE3_INTEGER);
+                output(4, "Execute query: ");
+                output(4, $statement->getSQL(true));
+                $statement->execute();
+
+                $url = 'https://swarfarm.com/static/herders/images/monsters/' . $unit->image_filename;   
+                $out = PATH . "public/img/unit/" . sprintf('%08d', $unit->com2us_id) . ".png";
+                output(4, "Downloading file '$url' to: '$out'");
+                if (!file_put_contents($out, file_get_contents($url))) { 
+                    output(1, "Error downloading file '$url' to: '$out'");
+                }
+
+                foreach ($unit->skills as $skill){
+                    $statement = $db->prepare("INSERT INTO k_unit_skill (unit, skill) VALUES (:unit, :skill);");
+                    $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
+                    $statement->bindValue(":skill", $skill, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                // TODO: Get data for other tables: 
+                // k_unit_essence
+                // k_unit_source
+            }
+            else{
+                // Compare all values
+                if ($r["family"] != $unit->family_id){
+                    $statement = $db->prepare("UPDATE k_unit SET family = :family WHERE id = :id;");
+                    $statement->bindValue(":family", $unit->family_id, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["name"] != $unit->name){
+                    $statement = $db->prepare("UPDATE k_unit SET name = :name WHERE id = :id;");
+                    $statement->bindValue(":name", $unit->name, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                $element = get_element_id($unit->element);
+                if ($r["element"] != $element){
+                    $statement = $db->prepare("UPDATE k_unit SET element = :element WHERE id = :id;");
+                    $statement->bindValue(":element", $element, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                $archetype = get_archetype_id($unit->archetype);
+                if ($r["archetype"] != $archetype){
+                    $statement = $db->prepare("UPDATE k_unit SET archetype = :archetype WHERE id = :id;");
+                    $statement->bindValue(":archetype", $archetype, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["base_stars"] != $unit->base_stars){
+                    $statement = $db->prepare("UPDATE k_unit SET base_stars = :base_stars WHERE id = :id;");
+                    $statement->bindValue(":base_stars", $unit->base_stars, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["natural_stars"] != $unit->natural_stars){
+                    $statement = $db->prepare("UPDATE k_unit SET natural_stars = :natural_stars WHERE id = :id;");
+                    $statement->bindValue(":natural_stars", $unit->natural_stars, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["obtainable"] != $unit->obtainable){
+                    $statement = $db->prepare("UPDATE k_unit SET obtainable = :obtainable WHERE id = :id;");
+                    $statement->bindValue(":obtainable", $unit->obtainable, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["can_awaken"] != $unit->can_awaken){
+                    $statement = $db->prepare("UPDATE k_unit SET can_awaken = :can_awaken WHERE id = :id;");
+                    $statement->bindValue(":can_awaken", $unit->can_awaken, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["awaken_bonus"] != $unit->awaken_bonus && $unit->awaken_bonus != 0){
+                    $statement = $db->prepare("UPDATE k_unit SET awaken_bonus = :awaken_bonus WHERE id = :id;");
+                    $statement->bindValue(":awaken_bonus", $unit->awaken_bonus, SQLITE3_TEXT);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_TEXT);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["skill_ups_to_max"] != $unit->skill_ups_to_max){
+                    $statement = $db->prepare("UPDATE k_unit SET skill_ups_to_max = :skill_ups_to_max WHERE id = :id;");
+                    $statement->bindValue(":skill_ups_to_max", $unit->skill_ups_to_max, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["hp"] != $unit->base_hp){
+                    $statement = $db->prepare("UPDATE k_unit SET hp = :hp WHERE id = :id;");
+                    $statement->bindValue(":hp", $unit->base_hp, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["attack"] != $unit->base_attack){
+                    $statement = $db->prepare("UPDATE k_unit SET attack = :attack WHERE id = :id;");
+                    $statement->bindValue(":attack", $unit->base_attack, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["defense"] != $unit->base_defense){
+                    $statement = $db->prepare("UPDATE k_unit SET defense = :defense WHERE id = :id;");
+                    $statement->bindValue(":defense", $unit->base_defense, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["speed"] != $unit->speed){
+                    $statement = $db->prepare("UPDATE k_unit SET speed = :speed WHERE id = :id;");
+                    $statement->bindValue(":speed", $unit->speed, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["crit_rate"] != $unit->crit_rate){
+                    $statement = $db->prepare("UPDATE k_unit SET crit_rate = :crit_rate WHERE id = :id;");
+                    $statement->bindValue(":crit_rate", $unit->crit_rate, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["crit_damage"] != $unit->crit_damage){
+                    $statement = $db->prepare("UPDATE k_unit SET crit_damage = :crit_damage WHERE id = :id;");
+                    $statement->bindValue(":crit_damage", $unit->crit_damage, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["resistance"] != $unit->resistance){
+                    $statement = $db->prepare("UPDATE k_unit SET resistance = :resistance WHERE id = :id;");
+                    $statement->bindValue(":resistance", $unit->resistance, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["accuracy"] != $unit->accuracy){
+                    $statement = $db->prepare("UPDATE k_unit SET accuracy = :accuracy WHERE id = :id;");
+                    $statement->bindValue(":accuracy", $unit->accuracy, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["raw_hp"] != $unit->raw_hp){
+                    $statement = $db->prepare("UPDATE k_unit SET raw_hp = :raw_hp WHERE id = :id;");
+                    $statement->bindValue(":raw_hp", $unit->raw_hp, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["raw_attack"] != $unit->raw_attack){
+                    $statement = $db->prepare("UPDATE k_unit SET raw_attack = :raw_attack WHERE id = :id;");
+                    $statement->bindValue(":raw_attack", $unit->raw_attack, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["raw_defense"] != $unit->raw_defense){
+                    $statement = $db->prepare("UPDATE k_unit SET raw_defense = :raw_defense WHERE id = :id;");
+                    $statement->bindValue(":raw_defense", $unit->raw_defense, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["max_lvl_hp"] != $unit->max_lvl_hp){
+                    $statement = $db->prepare("UPDATE k_unit SET max_lvl_hp = :max_lvl_hp WHERE id = :id;");
+                    $statement->bindValue(":max_lvl_hp", $unit->max_lvl_hp, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["max_lvl_attack"] != $unit->max_lvl_attack){
+                    $statement = $db->prepare("UPDATE k_unit SET max_lvl_attack = :max_lvl_attack WHERE id = :id;");
+                    $statement->bindValue(":max_lvl_attack", $unit->max_lvl_attack, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["max_lvl_defense"] != $unit->max_lvl_defense){
+                    $statement = $db->prepare("UPDATE k_unit SET max_lvl_defense = :max_lvl_defense WHERE id = :id;");
+                    $statement->bindValue(":max_lvl_defense", $unit->max_lvl_defense, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                $com2us_id = get_com2us_id($unit->awakens_from);
+                if ($r["awakens_from"] != $com2us_id){
+                    $statement = $db->prepare("UPDATE k_unit SET awakens_from = :awakens_from WHERE id = :id;");
+                    $statement->bindValue(":awakens_from", $com2us_id, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                $com2us_id = get_com2us_id($unit->awakens_to);
+                if ($r["awakens_to"] != $com2us_id){
+                    $statement = $db->prepare("UPDATE k_unit SET awakens_to = :awakens_to WHERE id = :id;");
+                    $statement->bindValue(":awakens_to", $com2us_id, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["fusion_food"] != $unit->fusion_food){
+                    $statement = $db->prepare("UPDATE k_unit SET fusion_food = :fusion_food WHERE id = :id;");
+                    $statement->bindValue(":fusion_food", $unit->fusion_food, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+                if ($r["homunculus"] != $unit->homunculus){
+                    $statement = $db->prepare("UPDATE k_unit SET homunculus = :homunculus WHERE id = :id;");
+                    $statement->bindValue(":homunculus", $unit->homunculus, SQLITE3_INTEGER);
+                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                }
+
+                // Loop skills. Only for inserting in k_skill and k_unit_skill
+                $statement = $db->prepare("SELECT count(skill) AS c FROM k_unit_skill WHERE unit = :unit;");
+                $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
+                $q = $statement->execute();
+                $r = $q->fetchArray(SQLITE3_ASSOC);
+                if ($r["c"] != sizeof($unit->skills)){
+                    $statement = $db->prepare("DELETE FROM k_unit_skill WHERE unit = :unit;");
+                    $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
+                    output(4, "Execute query: ");
+                    output(4, $statement->getSQL(true));
+                    $statement->execute();
+                    foreach ($unit->skills as $skill){
+                        $statement = $db->prepare("INSERT INTO k_unit_skill (unit, skill) VALUES (:unit, :skill);");
+                        $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
+                        $statement->bindValue(":skill", $skill, SQLITE3_INTEGER);
+                        output(4, "Execute query: ");
+                        output(4, $statement->getSQL(true));
+                        $statement->execute();
+                    }
+                }
+
+                // Image download
+                $out = PATH . "public/img/unit/" . sprintf('%08d', $unit->com2us_id) . ".png";
+                if (!file_exists($out)){
+                    //echo("  Downloading image " . $unit->image_filename . " to " .  sprintf('%08d', $unit->com2us_id) . ".png");
+                    $url = 'https://swarfarm.com/static/herders/images/monsters/' . $unit->image_filename;   
+                    output(4, "Downloading file '$url' to: '$out'");
+                    if (!file_put_contents($out, file_get_contents($url))) { 
+                        output(1, "Error downloading file '$url' to: '$out'");
+                    } 
+                }
+            }
+        }
+        //echo "NEXTPAGE: " . $next;
+    }
+
+    
+
+
+?>