<?php
/**
 * Classes for building the package database file
 *
 * @copyright (c) 2017, Snapcreek LLC
 * @license	https://opensource.org/licenses/GPL-3.0 GNU Public License
 */
defined('ABSPATH') || defined('DUPXABSPATH') || exit;

require_once DUPLICATOR_PRO_PLUGIN_PATH.'/classes/entities/class.global.entity.php';

require_once dirname(__FILE__).'/class.pack.database.build.progress.php';
require_once dirname(__FILE__).'/class.pack.database.build.iterator.php';
require_once dirname(__FILE__).'/class.pack.database.info.php';

/**
 * Class used to do the actual working of building the database file
 * There are currently three modes: PHP, MYSQLDUMP, PHPCHUNKING
 * PHPCHUNKING and PHP will eventually be combined as one routine
 */
class DUP_PRO_Database
{

    //IDE HELPERS
    /* @var $global DUP_PRO_Global_Entity  */

    //CONSTANTS
    /**
     * Marks the end of the CREATEs in the SQL file which have to be
     * run together in one chunk during install
     */
    const TABLE_CREATION_END_MARKER = "/* DUPLICATOR PRO TABLE CREATION END */\n";

    /**
     * Updating the percentage of progression in the serialized structure in the database is a heavy action so every TOT entries are made. 
     * 
     */
    const ROWS_NUM_TO_UPDATE_PROGRESS = 10000;

    /**
     * prefix of the file used to save the offsets of the inserted tables
     */
    const STORE_DB_PROGRESS_FILE_PREFIX = 'dup_pro_db_export_progress_';

    /**
     * 
     */
    const CLOSE_INSERT_QUERY = ";\n\n";

    /**
     * 
     */
    const PHP_DUMP_CHUNK_WORKER_TIME = 5; // in sec

    //PUBLIC
    /**
     *
     * @var DUP_PRO_DatabaseInfo
     */
    public $info;
    //PUBLIC: Legacy Style
    public $Type     = 'MySQL';
    public $Size;
    public $File;
    public $FilterTables;
    public $FilterOn;
    public $DBMode;
    public $Compatible;
    public $Comments = '';
    public $dbStorePathPublic;
    //PRIVATE
    private $endFileMarker;
    private $traceLogEnabled;

    /**
     *
     * @var DUP_PRO_Package 
     */
    private $Package;
    private $throttleDelayInUs = 0;

    /**
     *
     * @global wpdb $wpdb
     * @param DUP_PRO_Package $package
     */
    public function __construct(DUP_PRO_Package $package)
    {
        global $wpdb;

        $this->Package         = $package;
        $this->endFileMarker   = '';
        $this->traceLogEnabled = DUP_PRO_Log::isTraceLogEnabled();
        $this->info            = new DUP_PRO_DatabaseInfo();
        $global                = DUP_PRO_Global_Entity::get_instance();
        if (!($global instanceof DUP_PRO_Global_Entity)) {
            if (is_admin()) {
                add_action('admin_notices', array('DUP_PRO_UI_Alert', 'showTablesCorrupted'));
                add_action('network_admin_notices', array('DUP_PRO_UI_Alert', 'showTablesCorrupted'));
            }
            throw new Exception("Global Entity is null!");
        }
        $this->throttleDelayInUs = $global->getMicrosecLoadReduction();
        $wpdb->query("SET SESSION wait_timeout = ".DUPLICATOR_PRO_DB_MAX_TIME);
    }

    public function __destruct()
    {
        $this->Package = null;
    }

    public function __clone()
    {
        DUP_PRO_LOG::trace("CLONE ".__CLASS__);

        $this->info = clone $this->info;
    }

    /**
     * Runs the build process for the database
     *
     * @return null
     */
    public function build()
    {
        DUP_PRO_LOG::trace("BUILDING DATABASE");
        try {
            do_action('duplicator_pro_build_database_before_start', $this->Package);

            $global = DUP_PRO_Global_Entity::get_instance();

            $this->Package->db_build_progress->startTime = DUP_PRO_U::getMicrotime();
            $this->Package->set_status(DUP_PRO_PackageStatus::DBSTART);

            $this->dbStorePathPublic = "{$this->Package->StorePath}/{$this->File}";
            $mysqlDumpPath           = DUP_PRO_DB::getMySqlDumpPath();
            $mode                    = DUP_PRO_DB::getBuildMode(); //($mysqlDumpPath && $global->package_mysqldump) ? 'MYSQLDUMP' : 'PHP';

            $mysqlDumpSupport = ($mysqlDumpPath) ? 'Is Supported' : 'Not Supported';

            $log = "\n********************************************************************************\n";
            $log .= "DATABASE:\n";
            $log .= "********************************************************************************\n";
            $log .= "BUILD MODE:   {$mode} ";

            if (($mode == 'MYSQLDUMP') && strlen($this->Compatible)) {
                $log .= " (Legacy SQL)";
            }

            if ($mode == 'PHP') {
                $log .= "(query limit - {$global->package_phpdump_qrylimit})\n";
            } else {
                $log .= "(query limit - {$global->package_mysqldump_qrylimit})\n";
            }

            $log .= "MYSQLDUMP:    {$mysqlDumpSupport}\n";
            $log .= "MYSQLTIMEOUT: ".DUPLICATOR_PRO_DB_MAX_TIME;
            DUP_PRO_Log::info($log);
            $log = null;

            do_action('duplicator_pro_build_database_start', $this->Package);

            switch ($mode) {
                case 'MYSQLDUMP':
                    $this->runMysqlDump($mysqlDumpPath);
                    break;
                case 'PHP' :
                    $this->runPHPDump();
                    $this->validateStage1();
                    break;
            }

            $this->doFinish();
        }
        catch (Exception $e) {
            $this->Package->set_status(DUP_PRO_PackageStatus::ERROR);
            do_action('duplicator_pro_build_database_fail', $this->Package);
            DUP_PRO_Log::error("Runtime error in DUP_PRO_Database::Build", "Exception: {$e}");
        }

        DUP_PRO_LOG::trace("Done building database");
    }

    /**
     * Gets the database.sql file path and name
     *
     * @return string	Returns the full file path and file name of the database.sql file
     */
    public function getSafeFilePath()
    {
        return DupProSnapLibIOU::safePath(DUPLICATOR_PRO_SSDIR_PATH."/{$this->File}");
    }

    /**
     * @return string Returns the URL to the sql file
     */
    public function getURL()
    {
        return DUPLICATOR_PRO_SSDIR_URL."/{$this->File}";
    }

    protected function getStoreProgressFile()
    {
        return trailingslashit(DUPLICATOR_PRO_SSDIR_PATH_TMP).self::STORE_DB_PROGRESS_FILE_PREFIX.$this->Package->Hash.'.json';
    }

    /**
     *  Gets all the scanner information about the database
     *
     * 	@return array Returns an array of information about the database
     */
    public function getScanData()
    {
        global $wpdb;
        $filterTables         = isset($this->FilterTables) ? explode(',', $this->FilterTables) : null;
        $tblBaseCount         = 0;
        $tblFinalCount        = 0;
        $muFilteredTableCount = 0;

        $tables                    = $wpdb->get_results("SHOW TABLE STATUS", ARRAY_A);
        $views                     = $wpdb->get_results("SHOW FULL TABLES WHERE Table_Type = 'VIEW'", ARRAY_A);
        $procs                     = $wpdb->get_results("SHOW PROCEDURE STATUS", ARRAY_A);
        $info                      = array();
        $info['Status']['Success'] = is_null($tables) ? false : true;
        $info['Status']['Size']    = 'Good';
        $info['Status']['Rows']    = 'Good';

        $info['Size']       = 0;
        $info['Rows']       = 0;
        $info['TableCount'] = 0;
        $info['TableList']  = array();
        $tblCaseFound       = 0;

        $ms_tables_to_filter    = $this->Package->Multisite->getTablesToFilter();
        $this->info->tablesList = array();

        //Only return what we really need
        foreach ($tables as $table) {
            $tblBaseCount++;
            $name = $table["Name"];
            if (in_array($name, $ms_tables_to_filter)) {
                $muFilteredTableCount++;
                continue;
            }

            if ($this->FilterOn && is_array($filterTables)) {
                if (in_array($name, $filterTables)) {
                    continue;
                }
            }

            $size = ($table["Data_length"] + $table["Index_length"]);

            $info['Size']                      += $size;
            $info['Rows']                      += ($table["Rows"]);
            $info['TableList'][$name]['Case']  = preg_match('/[A-Z]/', $name) ? 1 : 0;
            $info['TableList'][$name]['Rows']  = empty($table["Rows"]) ? '0' : number_format($table["Rows"]);
            $info['TableList'][$name]['Size']  = DUP_PRO_U::byteSize($size);
            $info['TableList'][$name]['USize'] = $size;
            $tblFinalCount++;

            $this->info->addTableInList($name, $table["Rows"], $size);

            //Table Uppercase
            if ($info['TableList'][$name]['Case']) {
                if (!$tblCaseFound) {
                    $tblCaseFound = 1;
                }
            }
        }

        $info['Status']['Size'] = ($info['Size'] > DUPLICATOR_PRO_SCAN_DB_ALL_SIZE) ? 'Warn' : 'Good';
        $info['Status']['Rows'] = ($info['Rows'] > DUPLICATOR_PRO_SCAN_DB_ALL_ROWS) ? 'Warn' : 'Good';
        $info['TableCount']     = $tblFinalCount;

        $this->info->name                 = $wpdb->dbname;
        $this->info->isNameUpperCase      = preg_match('/[A-Z]/', $wpdb->dbname) ? 1 : 0;
        $this->info->isTablesUpperCase    = $tblCaseFound;
        $this->info->tablesBaseCount      = $tblBaseCount;
        $this->info->tablesFinalCount     = $tblFinalCount;
        $this->info->muFilteredTableCount = $muFilteredTableCount;
        $this->info->tablesRowCount       = $info['Rows'];
        $this->info->tablesSizeOnDisk     = $info['Size'];
        $this->info->dbEngine             = DupProSnapLibDB::getDBEngine($wpdb->dbh);
        $this->info->version              = DUP_PRO_DB::getVersion();
        $this->info->versionComment       = DUP_PRO_DB::getVariable('version_comment');
        $this->info->varLowerCaseTables   = DUP_PRO_DB::getVariable('lower_case_table_names');
        $tables                           = $this->getFilteredTables();
        $this->info->charSetList          = DUP_PRO_DB::getTableCharSetList($tables);
        $this->info->collationList        = DUP_PRO_DB::getTableCollationList($filterTables);
        $this->info->buildMode            = DUP_PRO_DB::getBuildMode();
        $this->info->viewCount            = count($views);
        $this->info->procCount            = count($procs);

        return $info;
    }

    /**
     * Runs the mysqldump process to build the database.sql script
     *
     * @param string $exePath The path to the mysqldump executable
     *
     * @return bool	Returns true if the mysqldump process ran without issues
     */
    private function runMysqlDump($exePath)
    {
        DUP_PRO_LOG::trace("RUN MYSQL DUMP");
        $sql_header = "/* DUPLICATOR-PRO (MYSQL-DUMP BUILD MODE) MYSQL SCRIPT CREATED ON : ".@date("Y-m-d H:i:s")." */\n\n";

        if (file_put_contents($this->dbStorePathPublic, $sql_header, FILE_APPEND) === false) {
            DUP_PRO_Log::error("file_put_content failed", "file_put_content failed while writing to {$this->dbStorePathPublic}", false);
            return false;
        }

        if ($this->mysqlDumpWriteCreates($exePath) != true) {
            DUP_PRO_Log::trace("Mysqldump error while writing CREATE queries");
            return false;
        }

        if ($this->mysqlDumpWriteInserts($exePath) != true) {
            DUP_PRO_Log::trace("Mysqldump error while writing INSERT queries");
            return false;
        }

        return true;
    }

    /**
     * @param string $exePath The path to the mysqldump executable
     * @return bool returns true if successful
     */
    private function mysqlDumpWriteCreates($exePath)
    {
        DUP_PRO_LOG::trace("START WRITING CREATES TO SQL FILE");
        $cmd         = $this->getMysqlDumpCmd($exePath, array('--no-data', '--routines'));
        $mysqlResult = $this->mysqlDumpWriteCmd($cmd, $exePath);
        if (file_put_contents($this->dbStorePathPublic, self::TABLE_CREATION_END_MARKER."\n", FILE_APPEND) === false) {
            DUP_PRO_Log::error("file_put_content failed", "file_put_content failed while writing to {$this->dbStorePathPublic}", false);
            return false;
        }
        return $this->mysqlDumpEvaluateResult($mysqlResult);
    }

    /**
     * @param string $exePath The path to the mysqldump executable
     * @return bool returns true if successful
     */
    private function mysqlDumpWriteInserts($exePath)
    {
        DUP_PRO_LOG::trace("START WRITING INSERTS TO SQL FILE");
        $cmd         = $this->getMysqlDumpCmd($exePath, array('--no-create-info'));
        $mysqlResult = $this->mysqlDumpWriteCmd($cmd, $exePath);
        $sql_footer  = "\n\n/* Duplicator WordPress Timestamp: ".date("Y-m-d H:i:s")."*/\n";
        $sql_footer  .= "/* ".DUPLICATOR_PRO_DB_EOF_MARKER." */\n";
        if (file_put_contents($this->dbStorePathPublic, $sql_footer, FILE_APPEND) === false) {
            DUP_PRO_Log::error("file_put_content failed", "file_put_content failed while writing to {$this->dbStorePathPublic}", false);
            return false;
        }
        return $this->mysqlDumpEvaluateResult($mysqlResult);
    }

    /**
     * @param string $cmd The mysqldump command to be run
     * @param string $exePath The path to the mysqldump executable
     * @return int The result of the mysql dump
     */
    private function mysqlDumpWriteCmd($cmd, $exePath)
    {
        DUP_PRO_LOG::trace("WRITING CREATES TO SQL FILE");
        $mysqlResult = 0;

        if (DUP_PRO_Shell_U::isPopenEnabled()) {
            $tables              = $this->getFilteredTables(true);
            $caseSensitiveTables = array_map(array('DUP_PRO_DB', 'updateCaseSensitivePrefix'), $tables);

            $findReplaceTableNames = array();
            foreach ($tables as $index => $tableName) {
                $csTable = $caseSensitiveTables[$index];
                if ($tableName !== $csTable) {
                    $findReplaceTableNames[$tableName] = $csTable;
                }
            }
            $needToRewrite = count($findReplaceTableNames) > 0;

            $firstLine = '';
            DUP_PRO_LOG::trace("POPEN mysqldump: $cmd");
            $handle    = popen($cmd, "r");
            if ($handle) {
                while (!feof($handle)) {
                    $line = fgets($handle); //get only one line
                    if ($line) {
                        if (empty($firstLine)) {
                            $firstLine = $line;
                            if (false !== stripos($line, 'Using a password on the command line interface can be insecure')) {
                                continue;
                            }
                        }

                        if ($needToRewrite) {
                            $replaceCount = 1;

                            if (preg_match('/CREATE TABLE `(.*?)`/', $line, $matches)) {
                                $tableName = $matches[1];
                                if (isset($findReplaceTableNames[$tableName])) {
                                    $rewriteTableAs = $findReplaceTableNames[$tableName];
                                    $line           = str_replace('CREATE TABLE `'.$tableName.'`', 'CREATE TABLE `'.$rewriteTableAs.'`', $line, $replaceCount);
                                }
                            } elseif (preg_match('/INSERT INTO `(.*?)`/', $line, $matches)) {
                                $tableName = $matches[1];
                                if (isset($findReplaceTableNames[$tableName])) {
                                    $rewriteTableAs = $findReplaceTableNames[$tableName];
                                    $line           = str_replace('INSERT INTO `'.$tableName.'`', 'INSERT INTO `'.$rewriteTableAs.'`', $line, $replaceCount);
                                }
                            } elseif (preg_match('/LOCK TABLES `(.*?)`/', $line, $matches)) {
                                $tableName = $matches[1];
                                if (isset($findReplaceTableNames[$tableName])) {
                                    $rewriteTableAs = $findReplaceTableNames[$tableName];
                                    $line           = str_replace('LOCK TABLES `'.$tableName.'`', 'LOCK TABLES `'.$rewriteTableAs.'`', $line, $replaceCount);
                                }
                            }
                        }

                        if (file_put_contents($this->dbStorePathPublic, $line, FILE_APPEND) === false) {
                            DUP_PRO_Log::error("file_put_content failed", "file_put_content failed while writing to {$this->dbStorePathPublic}", false);
                            //return mysql result warning value
                            $mysqlResult = 1;
                            return $mysqlResult;
                        }
                        $output = "Ran from {$exePath}";
                    }
                }
                $mysqlResult = pclose($handle);
            } else {
                $output = '';
            }

            // Password bug > 5.6 (@see http://bugs.mysql.com/bug.php?id=66546)
            if (empty($output) && trim($firstLine) === 'Warning: Using a password on the command line interface can be insecure.') {
                $output = '';
            }
        } else {
            DUP_PRO_LOG::trace("SHELL_EXEC mysqldump: $cmd");
            //$output = shell_exec($cmd);
            exec($cmd, $output, $mysqlResult);
            $output = implode("\n", $output);

            // Password bug > 5.6 (@see http://bugs.mysql.com/bug.php?id=66546)
            if (trim($output) === 'Warning: Using a password on the command line interface can be insecure.') {
                $output = '';
            }
            $output = (strlen($output)) ? $output : "Ran from {$exePath}";
            DUP_PRO_Log::info("RESPONSE: {$output}");
        }

        return $mysqlResult;
    }

    /**
     * @param int $mysqlResult The result of the mysql dump
     * @return bool returns true if the result was valid
     */
    private function mysqlDumpEvaluateResult($mysqlResult)
    {
        if ($mysqlResult !== 0) {
            /**
             * -1 error command shell
             * mysqldump return
             * 0 - Success
             * 1 - Warning
             * 2 - Exception
             */
            DUP_PRO_LOG::infoTrace('MYSQL DUMP ERROR '.print_r($mysqlResult, true));
            DUP_PRO_Log::error(DUP_PRO_U::__('Shell mysql dump failed. Last 10 lines of dump file below.'),
                implode("\n", DupProSnapLibIOU::getLastLinesOfFile($this->dbStorePathPublic, DUPLICATOR_PRO_DB_MYSQLDUMP_ERROR_CONTAINING_LINE_COUNT, DUPLICATOR_PRO_DB_MYSQLDUMP_ERROR_CHARS_IN_LINE_COUNT)), false);
            $this->setError(DUP_PRO_U::__('Shell mysql dump error. Take a look at the package log for details.'), DUP_PRO_U::__('Change SQL engine to PHP'), 'global:{package_mysqldump:0}');
            return false;
        }
        DUP_PRO_Log::trace("Operation was successful");
        return true;
    }

    /**
     * @param string $exePath The path to the mysqldump executable
     * @param array $extraFlags Extra flags to be added to the command
     * @return string The command to be executed
     */
    private function getMysqlDumpCmd($exePath, $extraFlags = array())
    {
        $global = DUP_PRO_Global_Entity::get_instance();

        $host           = explode(':', DB_HOST);
        $host           = reset($host);
        $port           = strpos(DB_HOST, ':') ? end(explode(':', DB_HOST)) : '';
        $name           = DB_NAME;
        $mysqlcompat_on = isset($this->Compatible) && strlen($this->Compatible);

        //Build command
        $cmd = escapeshellarg($exePath);
        $cmd .= ' --no-create-db';
        $cmd .= ' --single-transaction';
        $cmd .= ' --hex-blob';
        $cmd .= ' --skip-add-drop-table';
        $cmd .= ' --quote-names';
        $cmd .= ' --skip-comments';
        $cmd .= ' --skip-set-charset';
        $cmd .= ' --allow-keywords';
        $cmd .= ' --net_buffer_length='.DupProSnapLibUtil::getIntBetween($global->package_mysqldump_qrylimit, DUP_PRO_Constants::MYSQL_DUMP_CHUNK_SIZE_MIN_LIMIT, DUP_PRO_Constants::MYSQL_DUMP_CHUNK_SIZE_MAX_LIMIT);
        $cmd .= ' --no-tablespaces';

        if (!empty($extraFlags)) {
            foreach ($extraFlags as $flag) {
                $cmd .= ' '.$flag;
            }
        }

        //Compatibility mode
        if ($mysqlcompat_on) {
            DUP_PRO_Log::info("COMPATIBLE: [{$this->Compatible}]");
            $cmd .= " --compatible={$this->Compatible}";
        }

        // get excluded table list
        $tables = $this->getFilteredTables(true);

        foreach ($tables as $val) {
            $cmd .= " --ignore-table={$name}.{$val} ";
        }

        $cmd .= ' -u '.escapeshellarg(DB_USER);
        $cmd .= (DB_PASSWORD) ?
            ' -p'.DUP_PRO_Shell_U::escapeshellargWindowsSupport(DB_PASSWORD) : '';
        $cmd .= ' -h '.escapeshellarg($host);
        $cmd .= (!empty($port) && is_numeric($port)) ?
            ' -P '.$port : '';

        $cmd .= ' '.escapeshellarg(DB_NAME);
        $cmd .= ' 2>&1';

        if (!DUP_PRO_Shell_U::isPopenEnabled()) {
            // >> makes sure that the result is appended to the file and doesn't overwrite it
            $cmd .= ' >> '.escapeshellarg($this->dbStorePathPublic);
        }

        return $cmd;
    }

    /**
     * return a tables list. 
     * If $getExcludedTables is false return the included tables list else return the filtered table list
     * 
     * @global wpdb $wpdb
     * @param bool $getExcludedTables
     * @return string[]
     */
    private function getFilteredTables($getExcludedTables = false)
    {
        global $wpdb;

        $result = array();

        // ALL TABLES
        $allTables       = $wpdb->get_col("SHOW FULL TABLES WHERE Table_Type != 'VIEW'");
        // MANUAL FILTER TABLE 
        $filterTables    = ($this->FilterOn && isset($this->FilterTables)) ? explode(',', $this->FilterTables) : array();
        // SUB SITE FILTER TABLE
        $muFilterTables  = $this->Package->Multisite->getTablesToFilter();
        // TOTAL FILTER TABLES
        $allFilterTables = array_unique(array_merge($filterTables, $muFilterTables));

        $allTablesCount = count($allTables);
        $allFilterCount = count($allFilterTables);
        $createCount    = $allTablesCount - $allFilterCount;

        DUP_PRO_Log::infoTrace("TABLES: total: ".$allTablesCount." | filtered:".$allFilterCount." | create:".$createCount);
        if (!empty($filterTables)) {
            DUP_PRO_Log::infoTrace("MANUAL FILTER TABLES: \n\t".implode("\n\t", $filterTables));
        }
        if (!empty($muFilterTables)) {
            DUP_PRO_Log::infoTrace("MU SITE FILTER TABLES: \n\t".implode("\n\t", $muFilterTables));
        }

        if ($getExcludedTables) {
            $result = $allFilterTables;
        } else {
            if (empty($allFilterTables)) {
                $result = $allTables;
            } else {
                foreach ($allTables as $val) {
                    if (!in_array($val, $allFilterTables)) {
                        $result[] = $val;
                    }
                }
            }
        }

        return $result;
    }

    /**
     * Callback called in the insert iterator at the beginning of the current table dump.
     * 
     * @param DUP_PRO_DB_Build_Iterator $iterator
     */
    public function startTableIteratorCallback(DUP_PRO_DB_Build_Iterator $iterator)
    {
        $this->Package->db_build_progress->tableCountStart($iterator->current());
    }

    /**
     * Callback called in the insert iterator at the end of the current table dump.
     * 
     * @param DUP_PRO_DB_Build_Iterator $iterator
     */
    public function endTableIteratorCallback(DUP_PRO_DB_Build_Iterator $iterator)
    {
        $this->Package->db_build_progress->tableCountEnd($iterator->current(), $iterator->getCurrentOffset());
    }

    /**
     * Creates the database.sql script using PHP code
     *
     * @return null
     */
    private function runPHPDump()
    {
        DUP_PRO_LOG::trace("RUN PHP DUMP");
        global $wpdb;
        $global = DUP_PRO_Global_Entity::get_instance();

        $wpdb->query("SET session wait_timeout = ".DUPLICATOR_PRO_DB_MAX_TIME);
        $this->doFiltering();
        $this->writeCreates();

        $handle           = @fopen($this->dbStorePathPublic, 'a');
        $dbInsertIterator = $this->getDbBuildIterator();

        //BUILD INSERTS:
        for (; $dbInsertIterator->valid(); $dbInsertIterator->next()) {
            if ($dbInsertIterator->getCurrentRows() <= 0) {
                continue;
            }

            $table = $dbInsertIterator->current();

            $page_count = ceil($dbInsertIterator->getCurrentRows() / $global->package_phpdump_qrylimit);
            $dbInsertIterator->addFileSize(DupProSnapLibIOU::fwrite($handle, "\n/* INSERT TABLE DATA: {$table} */\n"));

            $row_offset = 0;

            for ($i = 0; $i < $page_count; $i++) {
                $result = DupProSnapLibDB::selectUsingPrimaryKeyAsOffset(
                        $wpdb->dbh,
                        'SELECT * FROM `'.$table.'` WHERE 1',
                        $table,
                        $row_offset,
                        $global->package_phpdump_qrylimit,
                        $row_offset);

                if ($result === false) {
                    $this->setErrorSelectFrom($table);
                    return;
                }

                if (DupProSnapLibDB::numRows($result) > 0) {
                    $first = true;
                    while (($row   = DupProSnapLibDB::fetchAssoc($result))) {
                        if ($first) {
                            $line  = 'INSERT INTO `'.$table.'` VALUES '."\n";
                            $first = false;
                        } else {
                            $line = ",\n";
                        }
                        $line       .= '('.implode(',', array_map(array('DUP_PRO_DB', 'escValueToQueryString'), $row)).')';
                        $totalCount = $dbInsertIterator->nextRow(0, DupProSnapLibIOU::fwriteChunked($handle, $line));

                        if (0 == ($totalCount % self::ROWS_NUM_TO_UPDATE_PROGRESS)) {
                            $this->setProgressPer($totalCount);
                        }
                    }
                    $dbInsertIterator->addFileSize(DupProSnapLibIOU::fwrite($handle, self::CLOSE_INSERT_QUERY));

                    if ($this->throttleDelayInUs > 0) {
                        usleep($this->throttleDelayInUs * $global->package_phpdump_qrylimit);
                    }
                }
                DupProSnapLibDB::freeResult($result);
            }
        }

        $this->writeSQLFooter($handle);
        $wpdb->flush();
        DupProSnapLibIOU::fclose($handle);
    }

    /**
     * Initialize the build iterator, based on the phpdumpmode, the storeprogress file is used or not.
     * 
     * @staticvar type $iterator
     * @return \DUP_PRO_DB_Build_Iterator
     */
    private function getDbBuildIterator()
    {
        static $iterator = null;

        if (is_null($iterator)) {
            $iterator = new DUP_PRO_DB_Build_Iterator(
                $this->Package->db_build_progress->tablesToProcess,
                (DUP_PRO_DB::getBuildMode() === DUP_PRO_DB::BUILD_MODE_PHP_MULTI_THREAD ? $this->getStoreProgressFile() : null),
                array($this, 'startTableIteratorCallback'),
                array($this, 'endTableIteratorCallback'));
        }
        return $iterator;
    }

    private function setError($message, $fix, $quickFix = false)
    {
        DUP_PRO_Log::trace($message);

        $this->Package->build_progress->failed = true;
        DUP_PRO_LOG::trace('Database: buildInChunks Failed');
        $this->Package->update();

        DUP_PRO_Log::error("**RECOMMENDATION:  $fix.", $message, false);

        $system_global = DUP_PRO_System_Global_Entity::get_instance();
        if ($quickFix === false) {
            $system_global->add_recommended_text_fix($message, $fix);
        } else {
            $system_global->add_recommended_quick_fix($message, $fix, $quickFix);
        }
        $system_global->save();
    }

    /**
     * Manage the error if it comes from a SELECT on the table
     * 
     * @global wpdb $wpdb
     * @param string $table
     */
    private function setErrorSelectFrom($table)
    {
        global $wpdb;
        DUP_PRO_Log::info($wpdb->last_error);
        if (false !== stripos($wpdb->last_error, 'is marked as crashed and should be repaired')) {
            $repair_query = "REPAIR TABLE `".$table."`;";
            $fix          = sprintf(DUP_PRO_U::__('Detected that database table %1$s is corrupt. Run repair tool or execute SQL command %2$s'), $table, $repair_query);
        } else {
            $fix = DUP_PRO_U::__('Please contact your DataBase administrator to fix the error.');
        }
        $this->setError($wpdb->last_error, $fix);
    }

    /**
     * Uses PHP to build the SQL file in chunks over multiple http requests
     *
     * @return void
     */
    public function buildInChunks()
    {
        DUP_PRO_LOG::trace("Database: buildInChunks Start");
        if ($this->Package->db_build_progress->wasInterrupted) {
            $this->Package->db_build_progress->failureCount++;
            $log_msg = 'Database: buildInChunks failure count increased to  '.$this->Package->db_build_progress->failureCount;
            DUP_PRO_LOG::trace($log_msg);
            error_log($log_msg);
        }

        if ($this->Package->db_build_progress->errorOut || $this->Package->db_build_progress->failureCount > DUPLICATOR_PRO_SQL_SCRIPT_PHP_CODE_MULTI_THREADED_MAX_RETRIES) {
            $this->Package->build_progress->failed = true;
            DUP_PRO_LOG::trace('Database: buildInChunks Failed');
            $this->Package->update();
            return;
        }

        $this->Package->db_build_progress->wasInterrupted = true;
        $this->Package->update();
        //TODO: See where else it needs to directly error out
        if (!$this->Package->db_build_progress->doneInit) {
            DUP_PRO_LOG::trace("Database: buildInChunks Init");
            $this->doInit();
            $this->Package->db_build_progress->doneInit = true;
        } elseif (!$this->Package->db_build_progress->doneFiltering) {
            DUP_PRO_LOG::trace("Database: buildInChunks Filtering");
            $this->doFiltering();
            $this->Package->db_build_progress->doneFiltering = true;
        } elseif (!$this->Package->db_build_progress->doneCreates) {
            DUP_PRO_LOG::trace("Database: buildInChunks WriteCreates");
            $this->writeCreates();
            $this->Package->db_build_progress->doneCreates = true;
        } elseif (!$this->Package->db_build_progress->completed) {
            DUP_PRO_LOG::trace("Database: buildInChunks WriteInsertChunk");
            $this->writeInsertChunk();
        }

        $this->Package->build_progress->database_script_built = false;
        if ($this->Package->db_build_progress->completed) {
            if (!$this->Package->db_build_progress->validationStage1) {
                $this->validateStage1();
            } else {
                DUP_PRO_LOG::trace("Database: buildInChunks completed");
                $this->Package->build_progress->database_script_built = true;
                $this->doFinish();
            }
        }

        DUP_PRO_LOG::trace("Database: buildInChunks End");
        // Resetting failure count since we if it recovers after a single failure we won't count it against it.
        $this->Package->db_build_progress->failureCount   = 0;
        $this->Package->db_build_progress->wasInterrupted = false;
        $this->Package->update();
    }

    /**
     * Performs validation of the values entered based on build progress counts
     * 
     * @throws Exception
     */
    protected function validateStage1()
    {
        DUP_PRO_LOG::trace("DB VALIDATION 1");
        $isValid = true;

        // SEARCH END MARKER
        $lastLines = DUP_PRO_U::tailFile($this->dbStorePathPublic, 3);
        if (strpos($lastLines, DUPLICATOR_PRO_DB_EOF_MARKER) === false) {
            DUP_PRO_LOG::infoTrace('DB VALIDATION 1: can\'t find SQL EOR MARKER in sql file');
            $isValid = false;
        }

        foreach ($this->Package->db_build_progress->countCheckData['tables'] as $table => $tableInfo) {
            $minVal = min($tableInfo['start'], $tableInfo['end']);
            $maxVal = max($tableInfo['start'], $tableInfo['end']);

            // The rows entered must be between the start value of the dump on the table and the end value.
            if ($tableInfo['count'] < $minVal || $tableInfo['count'] > $maxVal) {
                DUP_PRO_LOG::infoTrace('DB VALIDATION 1 FAIL: count check table "'.$table.'" START: '.$tableInfo['start'].' END: '.$tableInfo['end'].' COUNT: '.$tableInfo['count']);
                $isValid = false;
            } else {
                $this->info->addInsertedRowsInTableList($table, $tableInfo['count']);
                DUP_PRO_LOG::trace('DB VALIDATION 1 SUCCESS: count check table "'.$table.'" START: '.$tableInfo['start'].' END: '.$tableInfo['end'].' COUNT: '.$tableInfo['count']);
            }
        }

        $dbInsertIterator = $this->getDbBuildIterator();
        clearstatcache();
        if (filesize($this->dbStorePathPublic) !== $dbInsertIterator->getFileSize()) {
            DUP_PRO_LOG::infoTrace('SQL FILE SIZE CHECK FAILED, EXPECTED: '.$dbInsertIterator->getFileSize().' FILE SIZE: '.filesize($this->dbStorePathPublic).' OF FILE '.$this->dbStorePathPublic);
            $isValid = false;
        } else {
            DUP_PRO_LOG::infoTrace('SQL FILE SIZE CHECK OK, SIZE: '.$dbInsertIterator->getFileSize());
        }

        $dbInsertIterator->removeCounterFile();

        if ($isValid) {
            DUP_PRO_LOG::trace("DB VALIDATION 1: succeffully");
            $this->Package->db_build_progress->validationStage1 = true;
            $this->Package->update();
        } else {
            DUP_PRO_LOG::infoTrace("DB VALIDATION 1: failed to validate");
            throw new Exception("DB VALIDATION 1: failed to validate");
        }
    }

    /**
     * Used to initialize the PHP chunking logic
     *     *
     * @return void
     */
    private function doInit()
    {
        $global = DUP_PRO_Global_Entity::get_instance();

        do_action('duplicator_pro_build_database_before_start', $this->Package);

        $this->Package->db_build_progress->startTime = DUP_PRO_U::getMicrotime();
        $this->Package->set_status(DUP_PRO_PackageStatus::DBSTART);
        $this->dbStorePathPublic                     = "{$this->Package->StorePath}/{$this->File}";

        $log = "\n********************************************************************************\n";
        $log .= "DATABASE:\n";
        $log .= "********************************************************************************\n";
        $log .= "BUILD MODE:   PHP + CHUNKING ";
        $log .= "(chunk size - {$global->package_phpdump_qrylimit} rows)\n";

        DUP_PRO_Log::info($log);

        do_action('duplicator_pro_build_database_start', $this->Package);
        $this->Package->update();
    }

    /**
     * Initialize the table to be processed for the dump.
     * 
     * @global wpdb $wpdb
     */
    private function doFiltering()
    {
        global $wpdb;
        $wpdb->query("SET session wait_timeout = ".DUPLICATOR_PRO_DB_MAX_TIME);
        $tables                                            = $this->getFilteredTables();
        $this->Package->db_build_progress->tablesToProcess = array_map(array('DUP_PRO_DB', 'updateCaseSensitivePrefix'), $tables);
        $this->Package->db_build_progress->countCheckSetStart();
        $this->Package->db_build_progress->doneFiltering   = true;
        $this->Package->update();

        // MAKE SURE THE ITERATOR IS RESET
        $dbInsertIterator = $this->getDbBuildIterator();
        $dbInsertIterator->rewind();
    }

    /**
     * Dumps the structure of the view table and procedures.
     * 
     * @global wpdb $wpdb
     */
    private function writeCreates()
    {
        global $wpdb;

        $handle = @fopen($this->dbStorePathPublic, 'a');

        //Added 'NO_AUTO_VALUE_ON_ZERO' at plugin version 3.4.8 to fix :
        //**ERROR** database error write 'Invalid default value for for older mysql versions
        $sql_header = "/* DUPLICATOR-PRO (";
        $sql_header .= (DUP_PRO_DB::getBuildMode() === DUP_PRO_DB::BUILD_MODE_PHP_MULTI_THREAD ? 'PHP MULTI-THREADED BUILD MODE' : 'PHP SINGLE-THREAD BUILD MODE');
        $sql_header .= ") MYSQL SCRIPT CREATED ON : ".date("Y-m-d H:i:s")." */\n\n";
        $sql_header .= "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;\n";
        $sql_header .= "/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;\n";
        $sql_header .= "/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;\n\n";
        DupProSnapLibIOU::fwrite($handle, $sql_header);

        //BUILD CREATES:
        //All creates must be created before inserts do to foreign key constraints
        foreach ($this->Package->db_build_progress->tablesToProcess as $table) {
            $create             = $wpdb->get_row("SHOW CREATE TABLE `{$table}`", ARRAY_N);
            $count              = 1;
            // UPDATE CASE SENSITIVE TABLE PREFIX NAME
            $create_table_query = str_ireplace($table, $table, $create[1], $count);

            DupProSnapLibIOU::fwrite($handle, "{$create_table_query};\n\n");
            DUP_PRO_LOG::trace("DATABASE CREATE TABLE: ".$table." OK");
        }

        $procedures = $wpdb->get_col("SHOW PROCEDURE STATUS WHERE `Db` = '{$wpdb->dbname}'", 1);
        if (count($procedures)) {
            foreach ($procedures as $procedure) {
                DupProSnapLibIOU::fwrite($handle, "DELIMITER ;;\n");
                $create = $wpdb->get_row("SHOW CREATE PROCEDURE `{$procedure}`", ARRAY_N);
                DupProSnapLibIOU::fwrite($handle, "{$create[2]} ;;\n");
                DupProSnapLibIOU::fwrite($handle, "DELIMITER ;\n\n");
            }
        }

        $views = $wpdb->get_col("SHOW FULL TABLES WHERE Table_Type = 'VIEW'");
        if (count($views)) {
            foreach ($views as $view) {
                $create = $wpdb->get_row("SHOW CREATE VIEW `{$view}`", ARRAY_N);
                DupProSnapLibIOU::fwrite($handle, "{$create[1]};\n\n");
            }
        }
        DupProSnapLibIOU::fwrite($handle, self::TABLE_CREATION_END_MARKER);

        $dbInsertIterator = $this->getDbBuildIterator();
        $fileStat         = fstat($handle);
        $dbInsertIterator->addFileSize($fileStat['size']);
        DupProSnapLibIOU::fclose($handle);

        $this->Package->db_build_progress->errorOut    = true;
        $this->Package->db_build_progress->doneCreates = true;
        $this->Package->update();
        $this->Package->db_build_progress->errorOut    = false;
    }

    /**
     * 
     * @global wpdb $wpdb
     * @return void
     * 
     * @throws Exception
     */
    private function writeInsertChunk()
    {
        global $wpdb;
        $startTime   = microtime(true);
        $elapsedTime = 0;
        $global      = DUP_PRO_Global_Entity::get_instance();

        $dbInsertIterator = $this->getDbBuildIterator();
        $dbBuildProgress  = $this->Package->db_build_progress;

        $this->truncateSqlFileOnExpectedSize($dbInsertIterator->getFileSize());

        if (($handle = fopen($this->dbStorePathPublic, 'a')) === false) {
            $msg = print_r(error_get_last(), true);
            throw new Exception("FILE READ ERROR: Could not open file {$this->dbStorePathPublic} {$msg}");
        }

        if (!$dbInsertIterator->lastIsCompleteInsert()) {
            $dbInsertIterator->setLastIsCompleteInsert(
                DupProSnapLibIOU::fwrite($handle, self::CLOSE_INSERT_QUERY)
            );
        }

        for (; $dbInsertIterator->valid(); $dbInsertIterator->next()) {
            $table = $dbInsertIterator->current();

            if ($this->traceLogEnabled) {
                $table_number  = $dbInsertIterator->key() + 1;
                DUP_PRO_Log::trace("------------ DB SCAN CHUNK LOOP ------------");
                DUP_PRO_Log::trace("table: ".$table." (".$table_number." of ".$dbInsertIterator->count().")");
                DUP_PRO_Log::trace("worker_time: ".$elapsedTime." Max worker time: ".self::PHP_DUMP_CHUNK_WORKER_TIME);
                DUP_PRO_Log::trace("row_offset: ".$dbInsertIterator->getCurrentOffset()." of ".$dbInsertIterator->getCurrentRows());
                if (($primaryColumn = DupProSnapLibDB::getUniqueIndexColumn($wpdb->dbh, $table)) === false) {
                    DUP_PRO_Log::trace("no key column found, use normal offset ");
                } else {
                    DUP_PRO_Log::trace("primary column for offset system: ".DupProSnapLibLogger::varToString($primaryColumn));
                }
                DUP_PRO_Log::trace("last_index_offset: ".DupProSnapLibLogger::varToString($dbInsertIterator->getLastIndexOffset()));

                DUP_PRO_Log::trace("chunk_size: ".$global->package_phpdump_qrylimit);
            }

            $indexColumns = DupProSnapLibDB::getUniqueIndexColumn($wpdb->dbh, $table);

            while ($dbInsertIterator->isCurrentTableOffsetValid()) {
                $result = DupProSnapLibDB::selectUsingPrimaryKeyAsOffset(
                        $wpdb->dbh,
                        'SELECT * FROM `'.$table.'` WHERE 1',
                        $table,
                        $dbInsertIterator->getLastIndexOffset(),
                        $global->package_phpdump_qrylimit);

                if ($result === false) {
                    $this->setErrorSelectFrom($table);
                    return;
                }

                if (DupProSnapLibDB::numRows($result) > 0) {
                    $first = true;
                    while (($row   = DupProSnapLibDB::fetchAssoc($result))) {
                        if ($first) {
                            $line  = 'INSERT INTO `'.$table.'` VALUES '."\n";
                            $first = false;
                        } else {
                            $line = ",\n";
                        }
                        $line .= '('.implode(',', array_map(array('DUP_PRO_DB', 'escValueToQueryString'), $row)).')';

                        /* TEST INTERRUPTION START *** */
                        /* mt_srand((double) microtime() * 1000000);
                          if (mt_rand(1, 1000) > 997) {
                          die();
                          } */
                        /* TEST INTERRUPTION END *** */

                        $totalCount = $dbInsertIterator->nextRow(
                            DupProSnapLibDB::getOffsetFromRowAssoc($row, $indexColumns, $dbInsertIterator->getLastIndexOffset()),
                            DupProSnapLibIOU::fwriteChunked($handle, $line)
                        );

                        if (0 == ($totalCount % self::ROWS_NUM_TO_UPDATE_PROGRESS)) {
                            $this->setProgressPer($totalCount);
                        }

                        if (($elapsedTime = microtime(true) - $startTime) >= self::PHP_DUMP_CHUNK_WORKER_TIME) {
                            break;
                        }
                    }

                    $dbInsertIterator->setLastIsCompleteInsert(
                        DupProSnapLibIOU::fwrite($handle, self::CLOSE_INSERT_QUERY)
                    );

                    if ($this->throttleDelayInUs > 0) {
                        usleep($this->throttleDelayInUs * $global->package_phpdump_qrylimit);
                    }
                    DupProSnapLibDB::freeResult($result);
                } else {
                    DupProSnapLibDB::freeResult($result);
                    break;
                }

                if ($elapsedTime >= self::PHP_DUMP_CHUNK_WORKER_TIME) {
                    break 2;
                }
            }
        }

        if (($dbBuildProgress->completed = !$dbInsertIterator->valid())) {
            $this->writeSQLFooter($handle);
            $this->Package->update();
        } else {
            $this->setProgressPer($totalCount);
        }

        DupProSnapLibIOU::fclose($handle);
    }

    /**
     * 
     * @param int $size
     * @return boolean
     * @throws Exception
     */
    private function truncateSqlFileOnExpectedSize($size)
    {
        clearstatcache();
        if (filesize($this->dbStorePathPublic) === $size) {
            return true;
        }

        $handle = @fopen($this->dbStorePathPublic, 'r+');
        if ($handle === false) {
            $msg = print_r(error_get_last(), true);
            throw new Exception("FILE READ ERROR: Could not open file {$this->dbStorePathPublic} {$msg}");
        }

        if (ftruncate($handle, $size)) {
            DUP_PRO_LOG::trace("SQL FILE DON'T MATCH SIZE, TRUNCATE AT ".$size);
        } else {
            throw new Exception("FILE TRUNCATE ERROR: Could not truncate to file size ".$size);
        }

        DupProSnapLibIOU::fclose($handle);
    }

    private function writeSQLFooter($fileHandle)
    {
        $sql_footer       = "\n/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;\n";
        $sql_footer       .= "/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;\n";
        $sql_footer       .= "/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;\n\n";
        $sql_footer       .= "/* Duplicator WordPress Timestamp: ".date("Y-m-d H:i:s")."*/\n";
        $sql_footer       .= "/* ".DUPLICATOR_PRO_DB_EOF_MARKER." */\n";
        $dbInsertIterator = $this->getDbBuildIterator();
        $dbInsertIterator->addFileSize(DupProSnapLibIOU::fwrite($fileHandle, $sql_footer));
    }

    private function setProgressPer($offset)
    {
        $per = DupProSnapLibUtil::getWorkPercent(
                DUP_PRO_PackageStatus::DBSTART,
                DUP_PRO_PackageStatus::DBDONE,
                $this->Package->db_build_progress->countCheckData['impreciseTotalRows'],
                $offset);
        $this->Package->set_status($per);
    }

    private function doFinish()
    {
        DUP_PRO_Log::info("SQL CREATED: {$this->File}");
        $time_end     = DUP_PRO_U::getMicrotime();
        $elapsed_time = DUP_PRO_U::elapsedTime($time_end, $this->Package->db_build_progress->startTime);

        $sql_file_size = filesize($this->dbStorePathPublic);
        if ($sql_file_size <= 0) {
            DUP_PRO_Log::error("SQL file generated zero bytes.", "No data was written to the sql file.  Check permission on file and parent directory at [{$this->dbStorePathPublic}]");
        }
        DUP_PRO_Log::info("SQL FILE SIZE: ".DUP_PRO_U::byteSize($sql_file_size));
        DUP_PRO_Log::info("SQL FILE TIME: ".date("Y-m-d H:i:s"));
        DUP_PRO_Log::info("SQL RUNTIME: {$elapsed_time}");
        DUP_PRO_Log::info("MEMORY STACK: ".DUP_PRO_Server::getPHPMemory());

        $this->Size = @filesize($this->dbStorePathPublic);
        $this->Package->set_status(DUP_PRO_PackageStatus::DBDONE);
        do_action('duplicator_pro_build_database_completed', $this->Package);
    }

    private function getIntFieldsStruct($table)
    {
        if (!isset($this->intFieldsStruct[$table])) {
            $table_structure             = $GLOBALS['wpdb']->get_results("DESCRIBE `$table`");
            $int_fields_struct_for_table = array();
            foreach ($table_structure as $struct) {
                if ((0 === strpos($struct->Type, 'tinyint')) || (0 === strpos(strtolower($struct->Type), 'smallint')) || (0 === strpos(strtolower($struct->Type), 'mediumint')) || (0 === strpos(strtolower($struct->Type), 'int')) || (0 === strpos(strtolower($struct->Type), 'bigint'))
                ) {
                    $int_fields_struct_for_table[$struct->Field] = (null === $struct->Default ) ? 'NULL' : $struct->Default;
                }
            }
            $this->intFieldsStruct[$table] = $int_fields_struct_for_table;
        }
        return $this->intFieldsStruct[$table];
    }
}