SWDB/parser/swarfarm_parser

#!/bin/php
<?php

    /**
     * Swarfarm URL of the monster list (first page).
     */
    CONST URL_UNIT = 'https://swarfarm.com/api/v2/monsters/';

    /**
     * Swarfarm URL of the skill list (first page).
     */
    CONST URL_SKILL = 'https://swarfarm.com/api/v2/skills/';

    /**
     * Path to the application
     */
    CONST PATH = __DIR__ . "/../";

    /*
     * Script verbosity
     *
     * 0: Critical
     * 1: Error
     * 2: Warning
     * 3: Info
     * 4: Debug
     * 5: Trace
     */
    $args = array(
        "verbosity" => 3,
        "override_images" => false,
        "skip_downloads" => false,
        "only_downloads" => false,
        "only_new" => false
    );

    /**
     * Echoes an output message to stdout, depending on the verbosity level.
     *
     * @param int $level Message priority.
     * @param string $message content.
     */
    function output($level, $message, $keep_format = false){
        global $args;
        if ($level <= $args["verbosity"]){
            if ($keep_format == false){
                $message = str_replace("\n", " ", $message);
                while (strpos($message, "  ") !== false){
                    $message = str_replace("  ", " ", $message);
                }
            }
            if (substr($message, -1) != PHP_EOL && $keep_format == false){
                $message .= PHP_EOL;
            }
            echo $message;
        }
    }

    /**
     * Downloads a file.
     *
     * @param string $url The file to download.
     * @param string $target File destination.
     * @return boolean true on success, false on error.
     */
    function download($url, $target){
        global $args;
        if ($args["skip_downloads"] == true){
            return true;
        }
        if (!file_exists($target) || $args["override_images"] == true){
            output(4, "Downloading file '$url' to: '$target'");
            if (!file_put_contents($target, file_get_contents($url))) { 
                output(1, "Error downloading file '$url' to: '$target'");
                return false;
            }
            else{
                return true;
            }
        }
        else{
            return true;
        }
    }

    /**
     * Gets an entity com2us_id from it's Swarfarm id.
     *
     * @param int $swarfarm_id Entity's Swarfarm ID.
     * @return int Entity's com2us_id, -1 on error.
     */
    function get_com2us_id($swarfarm_id){
        if ($swarfarm_id == null){
            return null;
        }
        $contents = file_get_contents(URL_UNIT . $swarfarm_id . "/");
        if($contents === false){
            //Print out the contents.
            output(1, "Unable to retrieve monster.");
            return -1;
        }
        $contents = json_decode($contents);
        return $contents->com2us_id;
    }

    /**
     * Gets an element ID from it's name
     *
     * @param string $name Element name (case insensitive).
     * @return int Element ID, -1 on unknown element.
     */
    function get_element_id($name){
        $id = -1;
        switch (strtoupper($name)){
            case "MAGIC":
                $id = 0;
                break;
            case "WATER":
                $id = 1;
                break;
            case "FIRE":
                $id = 2;
                break;
            case "WIND":
                $id = 3;
                break;
            case "LIGHT":
                $id = 4;
                break;
            case "DARK":
                $id = 5;
                break;
            case "PURE":
                $id = 6;
                break;
        }
        return $id;
    }

    /**
     * Gets an archetype ID from it's name
     *
     * @param string $name Archetype name (case insensitive).
     * @return int Archetype ID, -1 on unknown archetype.
     */
    function get_archetype_id($name){
        $id = 0; // NONE
        switch (strtoupper($name)){
            case "ATTACK":
                $id = 1;
                break;
            case "DEFENSE":
                $id = 2;
                break;
            case "HP":
                $id = 3;
                break;
            case "SUPPORT":
                $id = 4;
                break;
            case "MATERIAL":
                $id = 5;
                break;
        }
        return $id;
    }

    /**
     * Prints the help message.
     */
    function show_help(){
        echo "\nSwarfarm parser\n\n";
        echo "Refreshes data using the Swarfarm API.\n\n";
        echo "Usage:\n\n";
        echo "    swarfarm_parser <options...>\n\n";
        echo "Options:\n\n";
        echo " +--------+------------------+---------------------------------------------------+\n";
        echo " | -s     | --skip-downloads |  Don't download images.                           |\n";
        echo " +--------+------------------+---------------------------------------------------+\n";
        echo " | -o     | --only-downloads |  Download images, but don't make changes to       |\n";
        echo " |        |                  |  the database.                                    |\n";
        echo " +--------+------------------+---------------------------------------------------+\n";
        echo " | -n     | --only-new       |  Look for new units or skills, but don't update   |\n";
        echo " |        |                  |  existing ones.                                   |\n";
        echo " +--------+------------------+---------------------------------------------------+\n";
        echo " | -v [n] | --verbosity [n]  |  Controls the script output to the console.       |\n";
        echo " |        |                  |  Values for [n]:                                  |\n";
        echo " |        |                  |    0: Only show critical errors.                  |\n";
        echo " |        |                  |    1: Only error messages.                        |\n";
        echo " |        |                  |    2: Only error and warning messages.            |\n";
        echo " |        |                  |    3: Standard output.                            |\n";
        echo " |        |                  |    4: Include debug messages.                     |\n";
        echo " |        |                  |    5: Print traces (lots of lines!).              |\n";
        echo " +--------+------------------+---------------------------------------------------+\n";
    }

    /**
     * Establishes a connection with the database.
     *
     * @param string $name Archetype name (case insensitive).
     * @return resource Database connection
     */
    function connect_to_database(){
        // Connect to the database
        $db = new SQLite3(PATH . "application/sw.sqlite");
        if (!$db) {
            output(0, "Error connecting to th database: " . $db->lastErrorMsg());
            exit();
        }
        // Disable integrity
        $db->query("PRAGMA foreign_keys=OFF;");
        output(4, "Connected to databse: " . PATH . "application/sw.sqlite");
        return $db;
    }

    // Parse arguments
    $accepted = array(
      "--verbosity", "-v",
      "--skip-downloads", "-s",
      "--only-downloads", "-d",
      "--only-new", "-n",
      "--override-images", "-o",
      "--help", "-h"
    );
    for ($i = 1; $i < count($argv); $i++){
        if ($argv[$i] == "--help" || $argv[$i] == "-h"){
            show_help();
            return 0;
        }
        elseif (!in_array($argv[$i], $accepted)){
            echo "Unrecognized option " . $argv[$i] . "\n";
            return -1;
        }
        elseif ($argv[$i] == "--verbosity" || $argv[$i] == "-v"){
            if (count($argv) >= $i + 2 && is_int($argv[$i + 1]) && intval($argv[$i + 1]) >= 0 && intval($argv[$i + 1]) <= 5){
                $args["verbosity"] = intval($argv[$i + 1]);
                $i ++;
            }
            else{
                echo "Unrecognized value for parameter " . $argv[$i] . ": " . $argv[$i + 1] . "\n";
                echo "Accepted values are:\n";
                echo "  0 (critical)\n";
                echo "  1 (error)\n";
                echo "  2 (warning)\n";
                echo "  3 (info)\n";
                echo "  4 (debug)\n";
                echo "  5 (trace)\n";
            }
        }
        elseif ($argv[$i] == "--skip-downloads" || $argv[$i] == "-s"){
            if (in_array("-d", $argv)){
                echo "Conflicting arguments: '-d' and '" . $argv[$i] . "'\n";
                return -1;
            }
            elseif (in_array("--only-downloads", $argv)){
                echo "Conflicting arguments: '--only-downloads' and '" . $argv[$i] . "'\n";
                return -1;
            }
            else{
                $args["skip_downloads"] = true;
            }
        }
        elseif ($argv[$i] == "--only-downloads" || $argv[$i] == "-s"){
            if (in_array("-s", $argv)){
                echo "Conflicting arguments: '-s' and '" . $argv[$i] . "'\n";
                return -1;
            }
            elseif (in_array("--skip-downloads", $argv)){
                echo "Conflicting arguments: '--skip-downloads' and '" . $argv[$i] . "'\n";
                return -1;
            }
            else{
                $args["only_downloads"] = true;
            }
        }
        elseif ($argv[$i] == "--only-new" || $argv[$i] == "-n"){
            $args["only_new"] = true;
        }
        elseif ($argv[$i] == "--override-images" || $argv[$i] == "-o"){
            if (in_array("-s", $argv)){
                echo "Conflicting arguments: '-s' and '" . $argv[$i] . "'\n";
                return -1;
            }
            elseif (in_array("--skip-downloads", $argv)){
                echo "Conflicting arguments: '--skip-downloads' and '" . $argv[$i] . "'\n";
                return -1;
            }
            else{
                $args["override_images"] = true;
            }
        }
    }
    output(4, "Resolved arguments:");
    foreach($args as $key => $value) {
        output(4, "  $key => " . var_export($value, true));
    }

    // Connect to the database
    $db = connect_to_database();

    // Loop skills pages
    $next = URL_SKILL;
    while ($next != null){
        output(3, "Reading skills page: $next");
        $contents = file_get_contents($next);
        if ($contents === false){
            output(0, "Unable to read skill page: $next");
            return -1;
        }
        $contents = json_decode($contents);
        $next = $contents->next;
        $list = $contents->results;
        foreach ($list as $skill){
            $statement = $db->prepare("SELECT * FROM k_skill WHERE id = :id");
            $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
            $q = $statement->execute();
            $r = $q->fetchArray(SQLITE3_ASSOC);
            if (!$r){
                // New skill, insert directly
                $statement = $db->prepare("
                  INSERT INTO k_skill(
                    id,
                    name,
                    description,
                    slot,
                    cooltime,
                    hits,
                    passive,
                    aoe,
                    max_level,
                    multiplier_formula
                  ) VALUES (
                    :id,
                    :name,
                    :description,
                    :slot,
                    :cooltime,
                    :hits,
                    :passive,
                    :aoe,
                    :max_level,
                    :multiplier_formula
                  );
                ");
                $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":name", $skill->name, SQLITE3_TEXT);
                $statement->bindValue(":description", $skill->name, SQLITE3_TEXT);
                $statement->bindValue(":slot", $skill->slot, SQLITE3_TEXT);
                $statement->bindValue(":cooltime", $skill->cooltime, SQLITE3_INTEGER);
                $statement->bindValue(":hits", $skill->hits, SQLITE3_INTEGER);
                $statement->bindValue(":passive", $skill->passive, SQLITE3_INTEGER);
                $statement->bindValue(":aoe", $skill->aoe, SQLITE3_INTEGER);
                $statement->bindValue(":max_level", $skill->max_level, SQLITE3_INTEGER);
                $statement->bindValue(":multiplier_formula", $skill->multiplier_formula, SQLITE3_TEXT);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();
                $url = 'https://swarfarm.com/static/herders/images/skills/' . $skill->icon_filename;
                $out = PATH . "public/img/skill/" . sprintf('%08d', $skill->com2us_id) . ".png";
                download($url, $out);
                $level = 1;
                $statement = $db->prepare("INSERT INTO k_skill_level (skill, level, description) VALUES (:skill, :level, :description);");
                $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                $statement->bindValue(":level", $level, SQLITE3_INTEGER);
                foreach ($skill->upgrades as $upgrade){
                    $level ++;
                    $statement = $db->prepare("INSERT INTO k_skill_level (skill, level, description) VALUES (:skill, :level, :description);");
                    $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":level", $level, SQLITE3_INTEGER);
                    $statement->bindValue(":description", str_replace("{0}", $upgrade->amount, $upgrade->effect), SQLITE3_TEXT);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                foreach ($skill->effects as $effect){
                    $statement = $db->prepare("INSERT INTO k_skill_effect (skill, effect) VALUES (:skill, :effect);");
                    $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":effect", $effect->effect->id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
            }
            else{
                // Existing skill, compare all values
                if ($r["name"] != $skill->name){
                    $statement = $db->prepare("UPDATE k_skill SET name = :name WHERE id = :id;");
                    $statement->bindValue(":name", $skill->name, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["description"] != $skill->description){
                    $statement = $db->prepare("UPDATE k_skill SET description = :description WHERE id = :id;");
                    $statement->bindValue(":description", $skill->description, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["slot"] != $skill->slot){
                    $statement = $db->prepare("UPDATE k_skill SET slot = :slot WHERE id = :id;");
                    $statement->bindValue(":slot", $skill->slot, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["cooltime"] != $skill->cooltime){
                    $statement = $db->prepare("UPDATE k_skill SET cooltime = :cooltime WHERE id = :id;");
                    $statement->bindValue(":cooltime", $skill->cooltime, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["hits"] != $skill->hits){
                    $statement = $db->prepare("UPDATE k_skill SET hits = :hits WHERE id = :id;");
                    $statement->bindValue(":hits", $skill->hits, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                $passive = ($skill->passive == 1) ? (1) : (0);
                if ($r["passive"] != $passive){
                    $statement = $db->prepare("UPDATE k_skill SET passive = :passive WHERE id = :id;");
                    $statement->bindValue(":passive", $passive, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                $aoe = ($skill->aoe == 1) ? (1) : (0);
                if ($r["aoe"] != $aoe){
                    $statement = $db->prepare("UPDATE k_skill SET aoe = :aoe WHERE id = :id;");
                    $statement->bindValue(":aoe", $aoe, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["max_level"] != $skill->max_level){
                    $statement = $db->prepare("UPDATE k_skill SET max_level = :max_level WHERE id = :id;");
                    $statement->bindValue(":max_level", $skill->max_level, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["multiplier_formula"] != $skill->multiplier_formula){
                    $statement = $db->prepare("UPDATE k_skill SET multiplier_formula = :multiplier_formula WHERE id = :id;");
                    $statement->bindValue(":multiplier_formula", $skill->multiplier_formula, SQLITE3_TEXT);
                    $statement->bindValue(":id", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }

                // Compare existing effects
                $statement = $db->prepare("SELECT count(effect) AS c FROM k_skill_effect WHERE skill = :skill;");
                $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                $q = $statement->execute();
                $r = $q->fetchArray(SQLITE3_ASSOC);
                if ($r["c"] != sizeof($skill->effects)){
                    $statement = $db->prepare("DELETE FROM k_skill_effect WHERE skill = :skill;");
                    $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                    foreach ($skill->effects as $effect){
                        $statement = $db->prepare("INSERT INTO k_skill_effect (skill, effect) VALUES (:skill, :effect);");
                        $statement->bindValue(":skill", $skill->com2us_id, SQLITE3_INTEGER);
                        $statement->bindValue(":effect", $effect->effect->id, SQLITE3_INTEGER);
                        output(4, "QUERY: " . $statement->getSQL(true));
                        $statement->execute();
                    }
                }

                // Image download
                $out = PATH . "public/img/skill/" . sprintf('%08d', $skill->com2us_id) . ".png";
                $url = 'https://swarfarm.com/static/herders/images/skills/' . $skill->icon_filename;   
                download($url, $out);
            }
        }
        //echo "NEXTPAGE: " . $next;
    }

    // Loop units pages
    $next = URL_UNIT;
    while ($next != null){
        output(3, "Reading units page: $next");
        $contents = file_get_contents($next);
        if($contents === false){
            output(0, "Unable to retrieve monster list.");
            return -1;
        }
        $contents = json_decode($contents);
        $next = $contents->next;
        $list = $contents->results;
        foreach ($list as $unit){
            $statement = $db->prepare("SELECT * FROM k_unit WHERE id = :id");
            $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
            $q = $statement->execute();
            $r = $q->fetchArray(SQLITE3_ASSOC);
            if (!$r){
                $statement = $db->prepare("
                  INSERT INTO k_unit(
                    id,
                    family,
                    name,
                    element,
                    archetype,
                    base_stars,
                    natural_stars,
                    obtainable,
                    can_awaken,
                    awaken_bonus,
                    skill_ups_to_max,
                    leader_skill,
                    hp,
                    attack,
                    defense,
                    speed,
                    crit_rate,
                    crit_damage,
                    resistance,
                    accuracy,
                    raw_hp,
                    raw_attack,
                    raw_defense,
                    max_lvl_hp,
                    max_lvl_attack,
                    max_lvl_defense,
                    awakens_from,
                    awakens_to,
                    fusion_food,
                    homunculus,
                    craft_cost
                  ) VALUES (
                    :id,
                    :family,
                    :name,
                    :element,
                    :archetype,
                    :base_stars,
                    :natural_stars,
                    :obtainable,
                    :can_awaken,
                    :awaken_bonus,
                    :skill_ups_to_max,
                    :leader_skill,
                    :hp,
                    :attack,
                    :defense,
                    :speed,
                    :crit_rate,
                    :crit_damage,
                    :resistance,
                    :accuracy,
                    :raw_hp,
                    :raw_attack,
                    :raw_defense,
                    :max_lvl_hp,
                    :max_lvl_attack,
                    :max_lvl_defense,
                    :awakens_from,
                    :awakens_to,
                    :fusion_food,
                    :homunculus,
                    :craft_cost
                  );
                ");
                $statement->bindValue(":id", $unit->com2us_id, SQLITE3_TEXT);
                $statement->bindValue(":family", $unit->family_id, SQLITE3_INTEGER);
                $statement->bindValue(":name", $unit->name, SQLITE3_TEXT);
                $statement->bindValue(":element", get_element_id($unit->element), SQLITE3_INTEGER);
                $statement->bindValue(":archetype", get_archetype_id($unit->archetype), SQLITE3_INTEGER);
                $statement->bindValue(":base_stars", $unit->base_stars, SQLITE3_INTEGER);
                $statement->bindValue(":natural_stars", $unit->natural_stars, SQLITE3_INTEGER);
                $statement->bindValue(":obtainable", $unit->obtainable, SQLITE3_INTEGER);
                $statement->bindValue(":can_awaken", $unit->can_awaken, SQLITE3_INTEGER);
                $statement->bindValue(":awaken_bonus", $unit->awaken_bonus, SQLITE3_TEXT);
                $statement->bindValue(":skill_ups_to_max", $unit->skill_ups_to_max, SQLITE3_INTEGER);
                if (isset($unit->leader_skill->id)){
                    $statement->bindValue(":leader_skill", $unit->leader_skill, SQLITE3_INTEGER);
                }
                else{
                    $statement->bindValue(":leader_skill", null, SQLITE3_INTEGER);
                }
                $statement->bindValue(":hp", $unit->base_hp, SQLITE3_INTEGER);
                $statement->bindValue(":attack", $unit->base_attack, SQLITE3_INTEGER);
                $statement->bindValue(":defense", $unit->base_defense, SQLITE3_INTEGER);
                $statement->bindValue(":speed", $unit->speed, SQLITE3_INTEGER);
                $statement->bindValue(":crit_rate", $unit->crit_rate, SQLITE3_INTEGER);
                $statement->bindValue(":crit_damage", $unit->crit_damage, SQLITE3_INTEGER);
                $statement->bindValue(":resistance", $unit->resistance, SQLITE3_INTEGER);
                $statement->bindValue(":accuracy", $unit->accuracy, SQLITE3_INTEGER);
                $statement->bindValue(":raw_hp", $unit->raw_hp, SQLITE3_INTEGER);
                $statement->bindValue(":raw_attack", $unit->raw_attack, SQLITE3_INTEGER);
                $statement->bindValue(":raw_defense", $unit->raw_defense, SQLITE3_INTEGER);
                $statement->bindValue(":max_lvl_hp", $unit->max_lvl_hp, SQLITE3_INTEGER);
                $statement->bindValue(":max_lvl_attack", $unit->max_lvl_attack, SQLITE3_INTEGER);
                $statement->bindValue(":max_lvl_defense", $unit->max_lvl_defense, SQLITE3_INTEGER);
                $statement->bindValue(":awakens_from", get_com2us_id($unit->awakens_from), SQLITE3_TEXT);
                $statement->bindValue(":awakens_to", get_com2us_id($unit->awakens_to), SQLITE3_TEXT);
                $statement->bindValue(":fusion_food", $unit->fusion_food, SQLITE3_INTEGER);
                $statement->bindValue(":homunculus", $unit->homunculus, SQLITE3_INTEGER);
                $statement->bindValue(":craft_cost", 0, SQLITE3_INTEGER);
                output(4, "QUERY: " . $statement->getSQL(true));
                $statement->execute();

                $url = 'https://swarfarm.com/static/herders/images/monsters/' . $unit->image_filename;   
                $out = PATH . "public/img/unit/" . sprintf('%08d', $unit->com2us_id) . ".png";
                download($url, $out);

                // Skills (not in database yet, but they will be once skills are parsed)
                foreach ($unit->skills as $skill){
                    $statement = $db->prepare("INSERT INTO k_unit_skill (unit, skill) VALUES (:unit, :skill);");
                    $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":skill", $skill, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }

                // Essences to awaken
                foreach ($unit->awaken_cost as $essence){
                    $statement = $db->prepare("INSERT INTO k_unit_essence (unit, item, amount) VALUES (:unit, :item, :amount);");
                    $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":item", $essence->item->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":amount", $essence->quantity, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }

                // Unit sources
                foreach ($unit->source as $source){
                    $statement = $db->prepare("INSERT INTO k_unit_source (unit, source) VALUES (:unit, :source);");
                    $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":source", $source->id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
            }
            else{
                // Compare all values
                if ($r["family"] != $unit->family_id){
                    $statement = $db->prepare("UPDATE k_unit SET family = :family WHERE id = :id;");
                    $statement->bindValue(":family", $unit->family_id, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["name"] != $unit->name){
                    $statement = $db->prepare("UPDATE k_unit SET name = :name WHERE id = :id;");
                    $statement->bindValue(":name", $unit->name, SQLITE3_TEXT);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                $element = get_element_id($unit->element);
                if ($r["element"] != $element){
                    $statement = $db->prepare("UPDATE k_unit SET element = :element WHERE id = :id;");
                    $statement->bindValue(":element", $element, SQLITE3_TEXT);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                $archetype = get_archetype_id($unit->archetype);
                if ($r["archetype"] != $archetype){
                    $statement = $db->prepare("UPDATE k_unit SET archetype = :archetype WHERE id = :id;");
                    $statement->bindValue(":archetype", $archetype, SQLITE3_TEXT);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["base_stars"] != $unit->base_stars){
                    $statement = $db->prepare("UPDATE k_unit SET base_stars = :base_stars WHERE id = :id;");
                    $statement->bindValue(":base_stars", $unit->base_stars, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["natural_stars"] != $unit->natural_stars){
                    $statement = $db->prepare("UPDATE k_unit SET natural_stars = :natural_stars WHERE id = :id;");
                    $statement->bindValue(":natural_stars", $unit->natural_stars, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["obtainable"] != $unit->obtainable){
                    $statement = $db->prepare("UPDATE k_unit SET obtainable = :obtainable WHERE id = :id;");
                    $statement->bindValue(":obtainable", $unit->obtainable, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["can_awaken"] != $unit->can_awaken){
                    $statement = $db->prepare("UPDATE k_unit SET can_awaken = :can_awaken WHERE id = :id;");
                    $statement->bindValue(":can_awaken", $unit->can_awaken, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["awaken_bonus"] != $unit->awaken_bonus && $unit->awaken_bonus != 0){
                    $statement = $db->prepare("UPDATE k_unit SET awaken_bonus = :awaken_bonus WHERE id = :id;");
                    $statement->bindValue(":awaken_bonus", $unit->awaken_bonus, SQLITE3_TEXT);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_TEXT);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["skill_ups_to_max"] != $unit->skill_ups_to_max){
                    $statement = $db->prepare("UPDATE k_unit SET skill_ups_to_max = :skill_ups_to_max WHERE id = :id;");
                    $statement->bindValue(":skill_ups_to_max", $unit->skill_ups_to_max, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if (isset($unit->leader_skill->id) && $r["leader_skill"] != $unit->leader_skill->id){
                    $statement = $db->prepare("UPDATE k_unit SET leader_skill = :leader_skill WHERE id = :id;");
                    $statement->bindValue(":leader_skill", $unit->leader_skill->id, SQLITE3_TEXT);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["hp"] != $unit->base_hp){
                    $statement = $db->prepare("UPDATE k_unit SET hp = :hp WHERE id = :id;");
                    $statement->bindValue(":hp", $unit->base_hp, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["attack"] != $unit->base_attack){
                    $statement = $db->prepare("UPDATE k_unit SET attack = :attack WHERE id = :id;");
                    $statement->bindValue(":attack", $unit->base_attack, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["defense"] != $unit->base_defense){
                    $statement = $db->prepare("UPDATE k_unit SET defense = :defense WHERE id = :id;");
                    $statement->bindValue(":defense", $unit->base_defense, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["speed"] != $unit->speed){
                    $statement = $db->prepare("UPDATE k_unit SET speed = :speed WHERE id = :id;");
                    $statement->bindValue(":speed", $unit->speed, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["crit_rate"] != $unit->crit_rate){
                    $statement = $db->prepare("UPDATE k_unit SET crit_rate = :crit_rate WHERE id = :id;");
                    $statement->bindValue(":crit_rate", $unit->crit_rate, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["crit_damage"] != $unit->crit_damage){
                    $statement = $db->prepare("UPDATE k_unit SET crit_damage = :crit_damage WHERE id = :id;");
                    $statement->bindValue(":crit_damage", $unit->crit_damage, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["resistance"] != $unit->resistance){
                    $statement = $db->prepare("UPDATE k_unit SET resistance = :resistance WHERE id = :id;");
                    $statement->bindValue(":resistance", $unit->resistance, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["accuracy"] != $unit->accuracy){
                    $statement = $db->prepare("UPDATE k_unit SET accuracy = :accuracy WHERE id = :id;");
                    $statement->bindValue(":accuracy", $unit->accuracy, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["raw_hp"] != $unit->raw_hp){
                    $statement = $db->prepare("UPDATE k_unit SET raw_hp = :raw_hp WHERE id = :id;");
                    $statement->bindValue(":raw_hp", $unit->raw_hp, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["raw_attack"] != $unit->raw_attack){
                    $statement = $db->prepare("UPDATE k_unit SET raw_attack = :raw_attack WHERE id = :id;");
                    $statement->bindValue(":raw_attack", $unit->raw_attack, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["raw_defense"] != $unit->raw_defense){
                    $statement = $db->prepare("UPDATE k_unit SET raw_defense = :raw_defense WHERE id = :id;");
                    $statement->bindValue(":raw_defense", $unit->raw_defense, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["max_lvl_hp"] != $unit->max_lvl_hp){
                    $statement = $db->prepare("UPDATE k_unit SET max_lvl_hp = :max_lvl_hp WHERE id = :id;");
                    $statement->bindValue(":max_lvl_hp", $unit->max_lvl_hp, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["max_lvl_attack"] != $unit->max_lvl_attack){
                    $statement = $db->prepare("UPDATE k_unit SET max_lvl_attack = :max_lvl_attack WHERE id = :id;");
                    $statement->bindValue(":max_lvl_attack", $unit->max_lvl_attack, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["max_lvl_defense"] != $unit->max_lvl_defense){
                    $statement = $db->prepare("UPDATE k_unit SET max_lvl_defense = :max_lvl_defense WHERE id = :id;");
                    $statement->bindValue(":max_lvl_defense", $unit->max_lvl_defense, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                $com2us_id = get_com2us_id($unit->awakens_from);
                if ($r["awakens_from"] != $com2us_id){
                    $statement = $db->prepare("UPDATE k_unit SET awakens_from = :awakens_from WHERE id = :id;");
                    $statement->bindValue(":awakens_from", $com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                $com2us_id = get_com2us_id($unit->awakens_to);
                if ($r["awakens_to"] != $com2us_id){
                    $statement = $db->prepare("UPDATE k_unit SET awakens_to = :awakens_to WHERE id = :id;");
                    $statement->bindValue(":awakens_to", $com2us_id, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["fusion_food"] != $unit->fusion_food){
                    $statement = $db->prepare("UPDATE k_unit SET fusion_food = :fusion_food WHERE id = :id;");
                    $statement->bindValue(":fusion_food", $unit->fusion_food, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }
                if ($r["homunculus"] != $unit->homunculus){
                    $statement = $db->prepare("UPDATE k_unit SET homunculus = :homunculus WHERE id = :id;");
                    $statement->bindValue(":homunculus", $unit->homunculus, SQLITE3_INTEGER);
                    $statement->bindValue(":id", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                }

                // Loop skills. Only for inserting in k_skill and k_unit_skill
                $statement = $db->prepare("SELECT count(skill) AS c FROM k_unit_skill WHERE unit = :unit;");
                $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                $q = $statement->execute();
                $r = $q->fetchArray(SQLITE3_ASSOC);
                if ($r["c"] != sizeof($unit->skills)){
                    $statement = $db->prepare("DELETE FROM k_unit_skill WHERE unit = :unit;");
                    $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                    foreach ($unit->skills as $skill){
                        $statement = $db->prepare("INSERT INTO k_unit_skill (unit, skill) VALUES (:unit, :skill);");
                        $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                        $statement->bindValue(":skill", $skill, SQLITE3_INTEGER);
                        output(4, "QUERY: " . $statement->getSQL(true));
                        $statement->execute();
                    }
                }

                // Essences to awaken
                $statement = $db->prepare("SELECT count(item) AS c FROM k_unit_essence WHERE unit = :unit;");
                $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                $q = $statement->execute();
                $r = $q->fetchArray(SQLITE3_ASSOC);
                if ($r["c"] != sizeof($unit->awaken_cost)){
                    $statement = $db->prepare("DELETE FROM k_unit_essence WHERE unit = :unit;");
                    $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                    foreach ($unit->awaken_cost as $essence){
                        $statement = $db->prepare("INSERT INTO k_unit_essence (unit, item, amount) VALUES (:unit, :item, :amount);");
                        $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                        $statement->bindValue(":item", $essence->item->com2us_id, SQLITE3_INTEGER);
                        $statement->bindValue(":amount", $essence->quantity, SQLITE3_INTEGER);
                        output(4, "QUERY: " . $statement->getSQL(true));
                        $statement->execute();
                    }
                }

                // Unit sources
                $statement = $db->prepare("SELECT count(source) AS c FROM k_unit_source WHERE unit = :unit;");
                $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                $q = $statement->execute();
                $r = $q->fetchArray(SQLITE3_ASSOC);
                if ($r["c"] != sizeof($unit->source)){
                    $statement = $db->prepare("DELETE FROM k_unit_source WHERE unit = :unit;");
                    $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                    output(4, "QUERY: " . $statement->getSQL(true));
                    $statement->execute();
                    foreach ($unit->source as $source){
                        $statement = $db->prepare("INSERT INTO k_unit_source (unit, source) VALUES (:unit, :source);");
                        $statement->bindValue(":unit", $unit->com2us_id, SQLITE3_INTEGER);
                        $statement->bindValue(":source", $source->id, SQLITE3_INTEGER);
                        output(4, "QUERY: " . $statement->getSQL(true));
                        $statement->execute();
                    }
                }

                // Image download
                $out = PATH . "public/img/unit/" . sprintf('%08d', $unit->com2us_id) . ".png";
                $url = 'https://swarfarm.com/static/herders/images/monsters/' . $unit->image_filename;   
                download($url, $out);
            }
        }
        //echo "NEXTPAGE: " . $next;
    }

    


?>