SWDB/parser/swarfarm_parser

#!/bin/php
<?php
/**
 * Swarfarm data parser
 *
 * Script that parses game data from swarfarm.com.
 *
 * @author Iñigo Valentin <i@inigovalentin.com>
 * @license https://www.gnu.org/licenses/gpl-3.0.en.html GNU General Public License V3
 * @package SWDB
 */

/**
 * Swarfarm URL of the monster list (first page).
 */
CONST URL_UNIT = 'https://swarfarm.com/api/v2/monsters/';

/**
 * Swarfarm URL of the skill list (first page).
 */
CONST URL_SKILL = 'https://swarfarm.com/api/v2/skills/';

/**
 * Path to the application
 */
CONST PATH = __DIR__ . "/../";

/*
 * Script verbosity
 *
 * 0: Critical
 * 1: Error
 * 2: Warning
 * 3: Info
 * 4: Debug
 * 5: Trace
 */
$args = array(
    "verbosity" => 3,
    "override_images" => false,
    "skip_downloads" => false,
    "only_downloads" => false,
    "only_new" => false
);

/**
 * Echoes an output message to stdout, depending on the verbosity level.
 *
 * @param int $level Message priority.
 * @param string $message content.
 */
function output($level, $message, $keep_format = false){
    global $args;
    if ($level <= $args["verbosity"]){
        if ($keep_format == false){
            $message = str_replace("\n", " ", $message);
            while (strpos($message, "  ") !== false){
                $message = str_replace("  ", " ", $message);
            }
        }
        if (substr($message, -1) != PHP_EOL && $keep_format == false){
            $message .= PHP_EOL;
        }
        echo $message;
    }
}

/**
 * Downloads a file.
 *
 * @param string $url The file to download.
 * @param string $target File destination.
 * @return boolean true on success, false on error.
 */
function download($url, $target){
    global $args;
    if ($args["skip_downloads"] == true){
        return true;
    }
    if (!file_exists($target) || $args["override_images"] == true){
        output(4, "Downloading file '$url' to: '$target'");
        if (!file_put_contents($target, file_get_contents($url))) { 
            output(1, "Error downloading file '$url' to: '$target'");
            return false;
        }
        else{
            return true;
        }
    }
    else{
        return true;
    }
}

/**
 * Gets an unit com2us_id from it's Swarfarm id.
 *
 * @param int $swarfarm_id Unit's Swarfarm ID.
 * @return int Skill's com2us_id, -1 on error.
 */
function get_com2us_id($swarfarm_id){
    if ($swarfarm_id == null){
        return null;
    }
    $contents = file_get_contents(URL_UNIT . $swarfarm_id . "/");
    if($contents === false){
        output(1, "Unable to retrieve unit ID by swarfarm ID: " . $swarfarm_id);
        return -1;
    }
    $contents = json_decode($contents);
    return $contents->com2us_id;
}

/**
 * Gets an skill com2us_id from it's Swarfarm id.
 *
 * @param int $swarfarm_id Skill's Swarfarm ID.
 * @return int Skill's com2us_id, -1 on error.
 */
function get_com2us_skill_id($swarfarm_id){
    if ($swarfarm_id == null){
        return null;
    }
    $contents = file_get_contents(URL_SKILL . $swarfarm_id . "/");
    if($contents === false){
        output(1, "Unable to retrieve skill ID by swarfarm ID: " . $swarfarm_id);
        return -1;
    }
    $contents = json_decode($contents);
    return $contents->com2us_id;
}

/**
 * Gets an element ID from it's name
 *
 * @param string $name Element name (case insensitive).
 * @return int Element ID, -1 on unknown element.
 */
function get_element_id($name){
    $id = -1;
    switch (strtoupper($name)){
        case "MAGIC":
            $id = 0;
            break;
        case "WATER":
            $id = 1;
            break;
        case "FIRE":
            $id = 2;
            break;
        case "WIND":
            $id = 3;
            break;
        case "LIGHT":
            $id = 4;
            break;
        case "DARK":
            $id = 5;
            break;
        case "PURE":
            $id = 6;
            break;
    }
    return $id;
}

/**
 * Gets an archetype ID from it's name
 *
 * @param string $name Archetype name (case insensitive).
 * @return int Archetype ID, -1 on unknown archetype.
 */
function get_archetype_id($name){
    $id = 0; // NONE
    switch (strtoupper($name)){
        case "ATTACK":
            $id = 1;
            break;
        case "DEFENSE":
            $id = 2;
            break;
        case "HP":
            $id = 3;
            break;
        case "SUPPORT":
            $id = 4;
            break;
        case "MATERIAL":
            $id = 5;
            break;
    }
    return $id;
}

/**
 * Prints the help message.
 */
function show_help(){
    echo "\nSwarfarm parser\n\n";
    echo "Refreshes data using the Swarfarm API.\n\n";
    echo "Usage:\n\n";
    echo "    swarfarm_parser <options...>\n\n";
    echo "Options:\n\n";
    echo " +--------+------------------+---------------------------------------------------+\n";
    echo " | -s     | --skip-downloads |  Don't download images.                           |\n";
    echo " +--------+------------------+---------------------------------------------------+\n";
    echo " | -o     | --only-downloads |  Download images, but don't make changes to       |\n";
    echo " |        |                  |  the database.                                    |\n";
    echo " +--------+------------------+---------------------------------------------------+\n";
    echo " | -n     | --only-new       |  Look for new monsters or skills, but don't       |\n";
    echo " |        |                  |  update existing ones.                            |\n";
    echo " +--------+------------------+---------------------------------------------------+\n";
    echo " | -v [n] | --verbosity [n]  |  Controls the script output to the console.       |\n";
    echo " |        |                  |  Values for [n]:                                  |\n";
    echo " |        |                  |    0: Only show critical errors.                  |\n";
    echo " |        |                  |    1: Only error messages.                        |\n";
    echo " |        |                  |    2: Only error and warning messages.            |\n";
    echo " |        |                  |    3: Standard output.                            |\n";
    echo " |        |                  |    4: Include debug messages.                     |\n";
    echo " |        |                  |    5: Print traces (lots of lines!).              |\n";
    echo " +--------+------------------+---------------------------------------------------+\n";
}

/**
 * Establishes a connection with the database.
 *
 * @param string $name Archetype name (case insensitive).
 * @return resource Database connection
 */
function connect_to_database(){
    // Connect to the database
    $db = new SQLite3(PATH . "application/key.sqlite");
    if (!$db) {
        output(0, "Error connecting to th database: " . $db->lastErrorMsg());
        exit();
    }
    // Disable integrity
    $db->query("PRAGMA foreign_keys=OFF;");
    output(4, "Connected to dataabse: " . PATH . "application/key.sqlite");
    return $db;
}

// Parse arguments
$accepted = array(
  "--verbosity", "-v",
  "--skip-downloads", "-s",
  "--only-downloads", "-d",
  "--only-new", "-n",
  "--override-images", "-o",
  "--help", "-h"
);
for ($i = 1; $i < count($argv); $i++){
    if ($argv[$i] == "--help" || $argv[$i] == "-h"){
        show_help();
        return 0;
    }
    elseif (!in_array($argv[$i], $accepted)){
        echo "Unrecognized option " . $argv[$i] . "\n";
        return -1;
    }
    elseif ($argv[$i] == "--verbosity" || $argv[$i] == "-v"){
        if (count($argv) >= $i + 2 && ctype_digit($argv[$i + 1]) && intval($argv[$i + 1]) >= 0 && intval($argv[$i + 1]) <= 5){
            $args["verbosity"] = intval($argv[$i + 1]);
            $i ++;
        }
        else{
            echo "Unrecognized value for parameter " . $argv[$i] . ": " . $argv[$i + 1] . "\n";
            echo "Accepted values are:\n";
            echo "  0 (critical)\n";
            echo "  1 (error)\n";
            echo "  2 (warning)\n";
            echo "  3 (info)\n";
            echo "  4 (debug)\n";
            echo "  5 (trace)\n";
        }
    }
    elseif ($argv[$i] == "--skip-downloads" || $argv[$i] == "-s"){
        if (in_array("-d", $argv)){
            echo "Conflicting arguments: '-d' and '" . $argv[$i] . "'\n";
            return -1;
        }
        elseif (in_array("--only-downloads", $argv)){
            echo "Conflicting arguments: '--only-downloads' and '" . $argv[$i] . "'\n";
            return -1;
        }
        else{
            $args["skip_downloads"] = true;
        }
    }
    elseif ($argv[$i] == "--only-downloads" || $argv[$i] == "-s"){
        if (in_array("-s", $argv)){
            echo "Conflicting arguments: '-s' and '" . $argv[$i] . "'\n";
            return -1;
        }
        elseif (in_array("--skip-downloads", $argv)){
            echo "Conflicting arguments: '--skip-downloads' and '" . $argv[$i] . "'\n";
            return -1;
        }
        else{
            $args["only_downloads"] = true;
        }
    }
    elseif ($argv[$i] == "--only-new" || $argv[$i] == "-n"){
        $args["only_new"] = true;
    }
    elseif ($argv[$i] == "--override-images" || $argv[$i] == "-o"){
        if (in_array("-s", $argv)){
            echo "Conflicting arguments: '-s' and '" . $argv[$i] . "'\n";
            return -1;
        }
        elseif (in_array("--skip-downloads", $argv)){
            echo "Conflicting arguments: '--skip-downloads' and '" . $argv[$i] . "'\n";
            return -1;
        }
        else{
            $args["override_images"] = true;
        }
    }
}
output(4, "Resolved arguments:");
foreach($args as $key => $value) {
    output(4, "  $key => " . var_export($value, true));
}

// Connect to the database
$db = connect_to_database();

// Loop skills pages
$next = URL_SKILL;
while ($next != null){
    output(3, "Reading skills page: $next");
    $contents = file_get_contents($next);
    if ($contents === false){
        output(0, "Unable to read skill page: $next");
        return -1;
    }
    $contents = json_decode($contents);
    $next = $contents->next;
    $list = $contents->results;
    foreach ($list as $skill){
        $statement = $db->prepare("SELECT * FROM skill WHERE id = :id");
        $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
        $q = $statement->execute();
        $r = $q->fetchArray(SQLITE3_ASSOC);
        if (!$r){
            // New skill, insert directly
            $statement = $db->prepare("
              INSERT INTO skill(
                id,
                name,
                description,
                slot,
                cooltime,
                hits,
                passive,
                aoe,
                max_level,
                multiplier_formula
              ) VALUES (
                :id,
                :name,
                :description,
                :slot,
                :cooltime,
                :hits,
                :passive,
                :aoe,
                :max_level,
                :multiplier_formula
              );
            ");
            $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
            $statement->bindValue(":name", $skill->name, SQLITE3_TEXT);
            $statement->bindValue(":description", $skill->name, SQLITE3_TEXT);
            $statement->bindValue(":slot", $skill->slot, SQLITE3_TEXT);
            $statement->bindValue(":cooltime", $skill->cooltime, SQLITE3_INTEGER);
            $statement->bindValue(":hits", $skill->hits, SQLITE3_INTEGER);
            $statement->bindValue(":passive", $skill->passive, SQLITE3_INTEGER);
            $statement->bindValue(":aoe", $skill->aoe, SQLITE3_INTEGER);
            $statement->bindValue(":max_level", $skill->max_level, SQLITE3_INTEGER);
            $statement->bindValue(":multiplier_formula", $skill->multiplier_formula, SQLITE3_TEXT);
            output(4, "QUERY: " . $statement->getSQL(true));
            $statement->execute();
            $url = 'https://swarfarm.com/static/herders/images/skills/' . $skill->icon_filename;
            $out = PATH . "public/img/skill/" . sprintf('%08d', $skill->com2us_id) . ".png";
            download($url, $out);
            $level = 1;
            $statement = $db->prepare("INSERT INTO skill_level (skill, level, description) VALUES (:skill, :level, :description);");
            $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
            $statement->bindValue(":level", $level, SQLITE3_INTEGER);
            foreach ($skill->upgrades as $upgrade){
                $level ++;
                $statement = $db->prepare("INSERT INTO skill_level (skill, level, description) VALUES (:skill, :level, :description);");
                $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":level", $level, SQLITE3_INTEGER);
                $statement->bindValue(":description", str_replace("{0}", $upgrade->amount, $upgrade->effect), SQLITE3_TEXT);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            foreach ($skill->effects as $effect){
                $statement = $db->prepare("INSERT INTO skill_effect (skill, effect) VALUES (:skill, :effect);");
                $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":effect", $effect->effect->id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
        }
        else{
            // Existing skill, compare all values
            if ($r["name"] != $skill->name){
                $statement = $db->prepare("UPDATE skill SET name = :name WHERE id = :id;");
                $statement->bindValue(":name", $skill->name, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["description"] != $skill->description){
                $statement = $db->prepare("UPDATE skill SET description = :description WHERE id = :id;");
                $statement->bindValue(":description", $skill->description, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["slot"] != $skill->slot){
                $statement = $db->prepare("UPDATE skill SET slot = :slot WHERE id = :id;");
                $statement->bindValue(":slot", $skill->slot, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["cooltime"] != $skill->cooltime){
                $statement = $db->prepare("UPDATE skill SET cooltime = :cooltime WHERE id = :id;");
                $statement->bindValue(":cooltime", $skill->cooltime, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["hits"] != $skill->hits){
                $statement = $db->prepare("UPDATE skill SET hits = :hits WHERE id = :id;");
                $statement->bindValue(":hits", $skill->hits, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            $passive = ($skill->passive == 1) ? (1) : (0);
            if ($r["passive"] != $passive){
                $statement = $db->prepare("UPDATE skill SET passive = :passive WHERE id = :id;");
                $statement->bindValue(":passive", $passive, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            $aoe = ($skill->aoe == 1) ? (1) : (0);
            if ($r["aoe"] != $aoe){
                $statement = $db->prepare("UPDATE skill SET aoe = :aoe WHERE id = :id;");
                $statement->bindValue(":aoe", $aoe, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["max_level"] != $skill->max_level){
                $statement = $db->prepare("UPDATE skill SET max_level = :max_level WHERE id = :id;");
                $statement->bindValue(":max_level", $skill->max_level, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["multiplier_formula"] != $skill->multiplier_formula){
                $statement = $db->prepare("UPDATE skill SET multiplier_formula = :multiplier_formula WHERE id = :id;");
                $statement->bindValue(":multiplier_formula", $skill->multiplier_formula, SQLITE3_TEXT);
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }

            // Compare existing effects
            $statement = $db->prepare("SELECT count(effect) AS c FROM skill_effect WHERE skill = :skill;");
            $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
            $q = $statement->execute();
            $r = $q->fetchArray(SQLITE3_ASSOC);
            if ($r["c"] != sizeof($skill->effects)){
                $statement = $db->prepare("DELETE FROM skill_effect WHERE skill = :skill;");
                $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
                foreach ($skill->effects as $effect){
                    $statement = $db->prepare("INSERT INTO skill_effect (skill, effect) VALUES (:skill, :effect);");
                    $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":effect", $effect->effect->id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
            }

            // Image download
            $out = PATH . "public/img/skill/" . sprintf('%08d', $skill->com2us_id) . ".png";
            $url = 'https://swarfarm.com/static/herders/images/skills/' . $skill->icon_filename;   
            download($url, $out);
        }
    }
    //echo "NEXTPAGE: " . $next;
}

// Loop monsters pages
$next = URL_UNIT;
while ($next != null){
    output(3, "Reading monsters page: $next");
    $contents = file_get_contents($next);
    if($contents === false){
        output(0, "Unable to retrieve monster list.");
        return -1;
    }
    $contents = json_decode($contents);
    $next = $contents->next;
    $list = $contents->results;
    foreach ($list as $monster){
        $statement = $db->prepare("SELECT * FROM monster WHERE id = :id");
        $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
        $q = $statement->execute();
        $r = $q->fetchArray(SQLITE3_ASSOC);
        if (!$r){
            $statement = $db->prepare("
              INSERT INTO monster(
                id,
                family,
                name,
                element,
                archetype,
                base_stars,
                natural_stars,
                obtainable,
                can_awaken,
                awaken_bonus,
                skill_ups_to_max,
                leader_skill,
                hp,
                attack,
                defense,
                speed,
                crit_rate,
                crit_damage,
                resistance,
                accuracy,
                raw_hp,
                raw_attack,
                raw_defense,
                max_lvl_hp,
                max_lvl_attack,
                max_lvl_defense,
                awakens_from,
                awakens_to,
                fusion_food,
                homunculus,
                craft_cost
              ) VALUES (
                :id,
                :family,
                :name,
                :element,
                :archetype,
                :base_stars,
                :natural_stars,
                :obtainable,
                :can_awaken,
                :awaken_bonus,
                :skill_ups_to_max,
                :leader_skill,
                :hp,
                :attack,
                :defense,
                :speed,
                :crit_rate,
                :crit_damage,
                :resistance,
                :accuracy,
                :raw_hp,
                :raw_attack,
                :raw_defense,
                :max_lvl_hp,
                :max_lvl_attack,
                :max_lvl_defense,
                :awakens_from,
                :awakens_to,
                :fusion_food,
                :homunculus,
                :craft_cost
              );
            ");
            $statement->bindValue(":id", $monster->com2us_id, SQLITE3_TEXT);
            $statement->bindValue(":family", $monster->family_id, SQLITE3_INTEGER);
            $statement->bindValue(":name", $monster->name, SQLITE3_TEXT);
            $statement->bindValue(":element", get_element_id($monster->element), SQLITE3_INTEGER);
            $statement->bindValue(":archetype", get_archetype_id($monster->archetype), SQLITE3_INTEGER);
            $statement->bindValue(":base_stars", $monster->base_stars, SQLITE3_INTEGER);
            $statement->bindValue(":natural_stars", $monster->natural_stars, SQLITE3_INTEGER);
            $statement->bindValue(":obtainable", $monster->obtainable, SQLITE3_INTEGER);
            $statement->bindValue(":can_awaken", $monster->can_awaken, SQLITE3_INTEGER);
            $statement->bindValue(":awaken_bonus", $monster->awaken_bonus, SQLITE3_TEXT);
            $statement->bindValue(":skill_ups_to_max", $monster->skill_ups_to_max, SQLITE3_INTEGER);
            if (isset($monster->leader_skill->id)){
                $statement->bindValue(":leader_skill", $monster->leader_skill, SQLITE3_INTEGER);
            }
            else{
                $statement->bindValue(":leader_skill", null, SQLITE3_INTEGER);
            }
            $statement->bindValue(":hp", $monster->base_hp, SQLITE3_INTEGER);
            $statement->bindValue(":attack", $monster->base_attack, SQLITE3_INTEGER);
            $statement->bindValue(":defense", $monster->base_defense, SQLITE3_INTEGER);
            $statement->bindValue(":speed", $monster->speed, SQLITE3_INTEGER);
            $statement->bindValue(":crit_rate", $monster->crit_rate, SQLITE3_INTEGER);
            $statement->bindValue(":crit_damage", $monster->crit_damage, SQLITE3_INTEGER);
            $statement->bindValue(":resistance", $monster->resistance, SQLITE3_INTEGER);
            $statement->bindValue(":accuracy", $monster->accuracy, SQLITE3_INTEGER);
            $statement->bindValue(":raw_hp", $monster->raw_hp, SQLITE3_INTEGER);
            $statement->bindValue(":raw_attack", $monster->raw_attack, SQLITE3_INTEGER);
            $statement->bindValue(":raw_defense", $monster->raw_defense, SQLITE3_INTEGER);
            $statement->bindValue(":max_lvl_hp", $monster->max_lvl_hp, SQLITE3_INTEGER);
            $statement->bindValue(":max_lvl_attack", $monster->max_lvl_attack, SQLITE3_INTEGER);
            $statement->bindValue(":max_lvl_defense", $monster->max_lvl_defense, SQLITE3_INTEGER);
            $statement->bindValue(":awakens_from", get_com2us_id($monster->awakens_from), SQLITE3_TEXT);
            $statement->bindValue(":awakens_to", get_com2us_id($monster->awakens_to), SQLITE3_TEXT);
            $statement->bindValue(":fusion_food", $monster->fusion_food, SQLITE3_INTEGER);
            $statement->bindValue(":homunculus", $monster->homunculus, SQLITE3_INTEGER);
            $statement->bindValue(":craft_cost", 0, SQLITE3_INTEGER);
            output(4, "QUERY: " . $statement->getSQL(true));
            $statement->execute();

            $url = 'https://swarfarm.com/static/herders/images/monsters/' . $monster->image_filename;   
            $out = PATH . "public/img/unit/" . sprintf('%08d', $monster->com2us_id) . ".png";
            download($url, $out);

            // Skills (not in database yet, but they will be once skills are parsed)
            foreach ($monster->skills as $skill){
                $statement = $db->prepare("INSERT INTO monster_skill (monster, skill) VALUES (:monster, :skill);");
                $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":skill", get_com2us_skill_id($skill), SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }

            // Essences to awaken
            foreach ($monster->awaken_cost as $essence){
                $statement = $db->prepare("INSERT INTO monster_essence (monster, item, amount) VALUES (:monster, :item, :amount);");
                $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":item", $essence->item->com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":amount", $essence->quantity, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }

            // Unit sources
            foreach ($monster->source as $source){
                $statement = $db->prepare("INSERT INTO monster_source (monster, source) VALUES (:monster, :source);");
                $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":source", $source->id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
        }
        else{
            // Compare all values
            if ($r["family"] != $monster->family_id){
                $statement = $db->prepare("UPDATE monster SET family = :family WHERE id = :id;");
                $statement->bindValue(":family", $monster->family_id, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["name"] != $monster->name){
                $statement = $db->prepare("UPDATE monster SET name = :name WHERE id = :id;");
                $statement->bindValue(":name", $monster->name, SQLITE3_TEXT);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            $element = get_element_id($monster->element);
            if ($r["element"] != $element){
                $statement = $db->prepare("UPDATE monster SET element = :element WHERE id = :id;");
                $statement->bindValue(":element", $element, SQLITE3_TEXT);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            $archetype = get_archetype_id($monster->archetype);
            if ($r["archetype"] != $archetype){
                $statement = $db->prepare("UPDATE monster SET archetype = :archetype WHERE id = :id;");
                $statement->bindValue(":archetype", $archetype, SQLITE3_TEXT);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["base_stars"] != $monster->base_stars){
                $statement = $db->prepare("UPDATE monster SET base_stars = :base_stars WHERE id = :id;");
                $statement->bindValue(":base_stars", $monster->base_stars, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["natural_stars"] != $monster->natural_stars){
                $statement = $db->prepare("UPDATE monster SET natural_stars = :natural_stars WHERE id = :id;");
                $statement->bindValue(":natural_stars", $monster->natural_stars, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["obtainable"] != $monster->obtainable){
                $statement = $db->prepare("UPDATE monster SET obtainable = :obtainable WHERE id = :id;");
                $statement->bindValue(":obtainable", $monster->obtainable, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["can_awaken"] != $monster->can_awaken){
                $statement = $db->prepare("UPDATE monster SET can_awaken = :can_awaken WHERE id = :id;");
                $statement->bindValue(":can_awaken", $monster->can_awaken, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["awaken_bonus"] != $monster->awaken_bonus && $monster->awaken_bonus != 0){
                $statement = $db->prepare("UPDATE monster SET awaken_bonus = :awaken_bonus WHERE id = :id;");
                $statement->bindValue(":awaken_bonus", $monster->awaken_bonus, SQLITE3_TEXT);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_TEXT);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["skill_ups_to_max"] != $monster->skill_ups_to_max){
                $statement = $db->prepare("UPDATE monster SET skill_ups_to_max = :skill_ups_to_max WHERE id = :id;");
                $statement->bindValue(":skill_ups_to_max", $monster->skill_ups_to_max, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if (isset($monster->leader_skill->id) && $r["leader_skill"] != $monster->leader_skill->id){
                $statement = $db->prepare("UPDATE monster SET leader_skill = :leader_skill WHERE id = :id;");
                $statement->bindValue(":leader_skill", $monster->leader_skill->id, SQLITE3_TEXT);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["hp"] != $monster->base_hp){
                $statement = $db->prepare("UPDATE monster SET hp = :hp WHERE id = :id;");
                $statement->bindValue(":hp", $monster->base_hp, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["attack"] != $monster->base_attack){
                $statement = $db->prepare("UPDATE monster SET attack = :attack WHERE id = :id;");
                $statement->bindValue(":attack", $monster->base_attack, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["defense"] != $monster->base_defense){
                $statement = $db->prepare("UPDATE monster SET defense = :defense WHERE id = :id;");
                $statement->bindValue(":defense", $monster->base_defense, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["speed"] != $monster->speed){
                $statement = $db->prepare("UPDATE monster SET speed = :speed WHERE id = :id;");
                $statement->bindValue(":speed", $monster->speed, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["crit_rate"] != $monster->crit_rate){
                $statement = $db->prepare("UPDATE monster SET crit_rate = :crit_rate WHERE id = :id;");
                $statement->bindValue(":crit_rate", $monster->crit_rate, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["crit_damage"] != $monster->crit_damage){
                $statement = $db->prepare("UPDATE monster SET crit_damage = :crit_damage WHERE id = :id;");
                $statement->bindValue(":crit_damage", $monster->crit_damage, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["resistance"] != $monster->resistance){
                $statement = $db->prepare("UPDATE monster SET resistance = :resistance WHERE id = :id;");
                $statement->bindValue(":resistance", $monster->resistance, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["accuracy"] != $monster->accuracy){
                $statement = $db->prepare("UPDATE monster SET accuracy = :accuracy WHERE id = :id;");
                $statement->bindValue(":accuracy", $monster->accuracy, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["raw_hp"] != $monster->raw_hp){
                $statement = $db->prepare("UPDATE monster SET raw_hp = :raw_hp WHERE id = :id;");
                $statement->bindValue(":raw_hp", $monster->raw_hp, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["raw_attack"] != $monster->raw_attack){
                $statement = $db->prepare("UPDATE monster SET raw_attack = :raw_attack WHERE id = :id;");
                $statement->bindValue(":raw_attack", $monster->raw_attack, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["raw_defense"] != $monster->raw_defense){
                $statement = $db->prepare("UPDATE monster SET raw_defense = :raw_defense WHERE id = :id;");
                $statement->bindValue(":raw_defense", $monster->raw_defense, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["max_lvl_hp"] != $monster->max_lvl_hp){
                $statement = $db->prepare("UPDATE monster SET max_lvl_hp = :max_lvl_hp WHERE id = :id;");
                $statement->bindValue(":max_lvl_hp", $monster->max_lvl_hp, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["max_lvl_attack"] != $monster->max_lvl_attack){
                $statement = $db->prepare("UPDATE monster SET max_lvl_attack = :max_lvl_attack WHERE id = :id;");
                $statement->bindValue(":max_lvl_attack", $monster->max_lvl_attack, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["max_lvl_defense"] != $monster->max_lvl_defense){
                $statement = $db->prepare("UPDATE monster SET max_lvl_defense = :max_lvl_defense WHERE id = :id;");
                $statement->bindValue(":max_lvl_defense", $monster->max_lvl_defense, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            $com2us_id = get_com2us_id($monster->awakens_from);
            if ($r["awakens_from"] != $com2us_id){
                $statement = $db->prepare("UPDATE monster SET awakens_from = :awakens_from WHERE id = :id;");
                $statement->bindValue(":awakens_from", $com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            $com2us_id = get_com2us_id($monster->awakens_to);
            if ($r["awakens_to"] != $com2us_id){
                $statement = $db->prepare("UPDATE monster SET awakens_to = :awakens_to WHERE id = :id;");
                $statement->bindValue(":awakens_to", $com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["fusion_food"] != $monster->fusion_food){
                $statement = $db->prepare("UPDATE monster SET fusion_food = :fusion_food WHERE id = :id;");
                $statement->bindValue(":fusion_food", $monster->fusion_food, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }
            if ($r["homunculus"] != $monster->homunculus){
                $statement = $db->prepare("UPDATE monster SET homunculus = :homunculus WHERE id = :id;");
                $statement->bindValue(":homunculus", $monster->homunculus, SQLITE3_INTEGER);
                $statement->bindValue(":id", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
            }

            // Loop skills. Only for inserting in skill and monster_skill
            $statement = $db->prepare("SELECT count(skill) AS c FROM monster_skill WHERE monster = :monster;");
            $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
            $q = $statement->execute();
            $r = $q->fetchArray(SQLITE3_ASSOC);
            // If some errors happens and the skill list has to be rebuilt,
            // force this block to execute (it is slow).
            // if ($r["c"] != sizeof($monster->skills))
            if ($r["c"] != sizeof($monster->skills) || 1 == 1){
                $statement = $db->prepare("DELETE FROM monster_skill WHERE monster = :monster;");
                $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
                foreach ($monster->skills as $skill){
                    $statement = $db->prepare("INSERT INTO monster_skill (monster, skill) VALUES (:monster, :skill);");
                    $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":skill", get_com2us_skill_id($skill), SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
            }

            // Essences to awaken
            $statement = $db->prepare("SELECT count(item) AS c FROM monster_essence WHERE monster = :monster;");
            $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
            $q = $statement->execute();
            $r = $q->fetchArray(SQLITE3_ASSOC);
            if ($r["c"] != sizeof($monster->awaken_cost)){
                $statement = $db->prepare("DELETE FROM monster_essence WHERE monster = :monster;");
                $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
                foreach ($monster->awaken_cost as $essence){
                    $statement = $db->prepare("INSERT INTO monster_essence (monster, item, amount) VALUES (:monster, :item, :amount);");
                    $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":item", $essence->item->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":amount", $essence->quantity, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
            }

            // Unit sources
            $statement = $db->prepare("SELECT count(source) AS c FROM monster_source WHERE monster = :monster;");
            $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
            $q = $statement->execute();
            $r = $q->fetchArray(SQLITE3_ASSOC);
            if ($r["c"] != sizeof($monster->source)){
                $statement = $db->prepare("DELETE FROM monster_source WHERE monster = :monster;");
                $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
                foreach ($monster->source as $source){
                    $statement = $db->prepare("INSERT INTO monster_source (monster, source) VALUES (:monster, :source);");
                    $statement->bindValue(":monster", $monster->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":source", $source->id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
            }

            // Image download
            $out = PATH . "public/img/unit/" . sprintf('%08d', $monster->com2us_id) . ".png";
            $url = 'https://swarfarm.com/static/herders/images/monsters/' . $monster->image_filename;   
            download($url, $out);
        }
    }
    //echo "NEXTPAGE: " . $next;
}