<?php
verifyCsrfGetToken();

require dirname(__FILE__) . "/../structure.php";
require dirname(__FILE__). '/../importlib.php';

@ob_end_flush();
$status = 'FAIL';
output('<p class="information">' . $GLOBALS['I18N']->get('Reading emails from file ... '));
flush();
if (filesize($_SESSION['import_file']) > 50000) {
  @ ini_set("memory_limit", memory_get_usage() + 50 * filesize($_SESSION['import_file']));
}
$email_list = file_get_contents($_SESSION["import_file"]);
flush();
// Clean up email file
$email_list = trim($email_list);
$email_list = str_replace("\r", "\n", $email_list);
$email_list = str_replace("\n\r", "\n", $email_list);
$email_list = str_replace("\n\n", "\n", $email_list);

if ($_SESSION["import_record_delimiter"] != "\n") {
  $email_list = str_replace($_SESSION["import_record_delimiter"], "\n", $email_list);
};


// Split file/emails into array
$email_list = explode("\n", $email_list); //WARNING the file contents get replace by an array
output(sprintf('..' . $GLOBALS['I18N']->get('ok, %d lines') . '</p>', sizeof($email_list)));
$header = array_shift($email_list);
$header = str_replace('"', '', $header);
$total = sizeof($email_list);
$headers = explode($_SESSION["import_field_delimiter"], $header);
$headers = array_unique($headers);
$_SESSION['columnnames'] = $headers;
  
### show progress and adjust working space
if (sizeof($email_list)) {
  $import_field_delimiter = $_SESSION["import_field_delimiter"];
  if (sizeof($email_list) > 300 && !$_SESSION["test_import"]) {
    # this is a possibly a time consuming process, so show a progress bar
    print '<script language="Javascript" type="text/javascript"> document.write(progressmeter); start();</script>';
    flush();
    # increase the memory to make sure we are not running out
    #    $mem = sizeof($email_list);
    ini_set("memory_limit", "32M");
  }

  ### store the chosen mappings in the $system_attribute_mapping list
  # print "A: ".sizeof($import_attribute);
  foreach ($system_attributes as $key => $val) {
    if (isset ($_SESSION["systemindex"][$key])) {
      $system_attribute_mapping[$key] = $_SESSION["systemindex"][$key]; 
    }
  }

  ### Parse the lines into records
  #  print "<br/>Loading emails .. ";
  flush();
  $count = array ();
  $count["email_add"] = 0;
  $count["exist"] = 0;
  $count["list_add"] = 0;
  $count["group_add"] = 0;
  $count['foundblacklisted'] = 0;
  $c = 1;
  $count["invalid_email"] = 0;
  $num_lists = sizeof($_SESSION["lists"]);
  $total = sizeof($email_list);
  $cnt = 0;
  $count["emailmatch"] = 0;
  $count["fkeymatch"] = 0;
  $count["dataupdate"] = 0;
  $count['duplicate'] = 0;
  $additional_emails = 0;
  foreach ($email_list as $line) {
    set_time_limit(60);
    # will contain attributes to store / change
    $user = array ();
    # get rid of text delimiters generally added by spreadsheet apps
    $line = str_replace('"', '', $line);

    $values = explode($_SESSION["import_field_delimiter"], $line);
    $system_values = array ();
    foreach ($system_attribute_mapping as $column => $index) {
   #   print '<br/>'.$column . ' = '. $values[$index];
      if (!empty($values[$index])) {
        $system_values[$column] = $values[$index];
      } else {
        $system_values[$column] = '';
      }
    }
    ## Check completeness
    $index = clean($system_values["email"]);
    $invalid = 0;
    if (!$index) {
      if ($_SESSION["show_warnings"]) {
        Warn($GLOBALS['I18N']->get('Record has no email') .
        ": $c -> $line");
      }
      $index = $GLOBALS['I18N']->get('Invalid Email') . " $c";
      $system_values["email"] = $_SESSION["assign_invalid"];
      $invalid = 1;
      $count["invalid_email"]++;
    }

    //print ("<pre>" . var_dump($_SESSION["import_attribute"]) . "</pre>"); // debug
    //    dbg('_SESSION["import_attribute"',$_SESSION["import_attribute"]); //debug
    if (sizeof($values) != (sizeof($_SESSION["import_attribute"]) + sizeof($system_attributes) - sizeof($unused_systemattr)) && !empty($_SESSION['test_import']) && !empty($_SESSION["show_warnings"]))
      Warn("Record has more values than header indicated (" .
      sizeof($values) . "!=" .
       (sizeof($_SESSION["import_attribute"]) + sizeof($system_attributes) - sizeof($unused_systemattr)) .
      "), this may cause trouble: $index");
    if (!$invalid || ($invalid && $_SESSION["omit_invalid"] != "yes")) {
      $user["systemvalues"] = $system_values;
      reset($_SESSION["import_attribute"]);
      $replace = array ();
      while (list ($key, $val) = each($_SESSION["import_attribute"])) {
        if (!empty($values[$val["index"]])) {
          $user[$val["index"]] = addslashes($values[$val["index"]]);
          $replace[$key] = addslashes($values[$val["index"]]);
        }
      }
    } else {
      # Warn("Omitting invalid one: $email");
    }
    $user["systemvalues"]["email"] = parsePlaceHolders($system_values["email"], array_merge($replace, $system_values, array (
      "number" => $c
    )));
    $user["systemvalues"]["email"] = cleanEmail($user["systemvalues"]["email"]);
    $c++;
    if (!isset($user["systemvalues"]["htmlemail"])) {
      $user["systemvalues"]["htmlemail"] = 1;
    }
    if ($_SESSION["test_import"]) {
      
#      var_dump($user["systemvalues"]);exit;
      $html = '';
      foreach ($user["systemvalues"] as $column => $value) {
        if (strpos($column,'grouptype_') === 0) {
          if (isset($system_attributes[$column])) {
            $column = $system_attributes[$column];
          }
        }
        if (!empty($column)) {
          if ($value) {
            $html .= "$column -> $value<br/>\n";
          } else {
            $html .= "$column -> " . $GLOBALS['I18N']->get('clear value') . "<br/>\n";
          }
        }
      }
    #  var_dump($_SESSION["systemindex"]);

      reset($_SESSION["import_attribute"]);
      foreach ($_SESSION["import_attribute"] as $column => $item) {
        if (!empty($user[$item["index"]])) {
          if ($item["record"] == "new") {
            $html .= ' ' . $GLOBALS['I18N']->get('New Attribute') . ': ' . $item["column"];
          }
          elseif ($item["record"] == "skip") {
            # forget about it
            $html .= ' ' . $GLOBALS['I18N']->get('Skip value') . ' '.$column.': ';
          } elseif ($item["record"] != 'system') {
            $html .= $attributes[$item["record"]];
#            var_dump($attributes[$item['record']]);
          } else {
            $html .= $item["column"];
          }
          $html .= " -> " . $user[$item["index"]] . "<br/>";
        }
      }
      if ($html) {
        print '<blockquote>' . $html . '</blockquote><hr />';
      }
    } else {
      if ($cnt % 5 == 0) {
        print '<script type="text/javascript">
        var parentJQuery = window.parent.jQuery;
        parentJQuery("#progressbar").updateProgress("'.$cnt.','.$total.'");
        </script>';      
        flush();
      }
      $cnt++;

	if (!$invalid || ($invalid && $_SESSION["omit_invalid"] != "yes")) {
      # do import
      ## create new attributes
      foreach ($_SESSION["import_attribute"] as $column => $item) {
        if ($item["record"] == "new") {
          Sql_Query(sprintf('insert into %s (name,type) values("%s","textline")', $tables["attribute"], addslashes($column)));
          $attid = Sql_Insert_id();
          Sql_Query(sprintf('update %s set tablename = "attr%d" where id = %d', $tables["attribute"], $attid, $attid));
          Sql_Query("create table " . $GLOBALS["table_prefix"] . "listattr_attr" . $attid . "
                      (id integer not null primary key auto_increment, name varchar(255) unique,
                      listorder integer default 0)");
          $_SESSION["import_attribute"][$column]["record"] = $attid;
        }
      }
      $new = 0;
/*      $cnt++;
      if ($cnt % 5 == 0) {
        print '<script type="text/javascript">
        var parentJQuery = window.parent.jQuery;
        parentJQuery("#progressbar").updateProgress("'.$cnt.','.$total.'");
        </script>';      
        flush();
      }*/
      if (!empty($user["systemvalues"]["foreignkey"])) {
        dbg('Importing on FK '.$user["systemvalues"]["foreignkey"].' email :'.$user["systemvalues"]["email"]);
        $result = Sql_query(sprintf('select id,uniqid from %s where foreignkey = "%s"', $tables["user"], $user["systemvalues"]["foreignkey"]));
        # print "<br/>Using foreign key for matching: ".$user["systemvalues"]["foreign key"];
        $count["fkeymatch"]++;
        $exists = Sql_Affected_Rows();
        $existing_user = Sql_fetch_array($result);
        # check whether the email will clash
        $clashcheck = Sql_Fetch_Row_Query(sprintf('select id from %s
                  where email = "%s"', $tables["user"], $user["systemvalues"]["email"]));
        if ($clashcheck[0] != $existing_user["id"]) {
          $count['duplicate']++;
          $notduplicate = 0;
          $c = 0;
          while (!$notduplicate) {
            $c++;
            $req = Sql_Query(sprintf('select id from %s where email = "%s"', $tables["user"], $GLOBALS['I18N']->get('duplicate') .
            "$c " . $user["systemvalues"]["email"]));
            $notduplicate = !Sql_Affected_Rows();
          }
          if (!$_SESSION["retainold"]) {
            Sql_Query(sprintf('update %s set email = "%s" where email = "%s"', $tables["user"], "duplicate$c " .
            $user["systemvalues"]["email"], $user["systemvalues"]["email"]));
            addUserHistory("duplicate$c " . $user["systemvalues"]["email"], "Duplication clash ", ' User marked duplicate email after clash with imported record');
          } else {
            if ($_SESSION["show_warnings"])
              print Warn($GLOBALS['I18N']->get('Duplicate Email') . ' ' . $user["systemvalues"]["email"] . $GLOBALS['I18N']->get(' user imported as ') . '&quot;' . $GLOBALS['I18N']->get('duplicate') . "$c " . $user["systemvalues"]["email"] . "&quot;");
            $user["systemvalues"]["email"] = $GLOBALS['I18N']->get('duplicate') . "$c " . $user["systemvalues"]["email"];
          }
        }
      } else {
        dbg('Importing on email '.$user["systemvalues"]["email"]);
        $result = Sql_query(sprintf('select id,uniqid from %s where email = "%s"', $tables["user"], $user["systemvalues"]["email"]));
        # print "<br/>Using email for matching: ".$user["systemvalues"]["email"];
        $count["emailmatch"]++;
        $exists = Sql_Affected_Rows();
        $existing_user = Sql_fetch_array($result);
      }
      if ($exists) {
        // User exist, remember some values to add them to the lists
        $count["exist"]++;
        $userid = $existing_user["id"];
        $uniqid = $existing_user["uniqid"];
      } else {
        // user does not exist
        $new = 1;
        // Create unique number
        mt_srand((double) microtime() * 1000000);
        $randval = mt_rand();
        # this is very time consuming when importing loads of users as it does a lookup
        # needs speeding up if possible
        $uniqid = getUniqid();
        $confirmed = $_SESSION["notify"] != "yes" && !preg_match("/Invalid Email/i", $index);

        $query = sprintf('INSERT INTO %s (email,entered,confirmed,uniqid,htmlemail)
                  values("%s",now(),%d,"%s",1)', $tables["user"], $user["systemvalues"]["email"], $confirmed, $uniqid);
        $result = Sql_query($query, 1);
        $userid = Sql_insert_id();
        if (!$userid) {
          # no id returned, so it must have been a duplicate entry
          if ($_SESSION["show_warnings"])
            print Warn($GLOBALS['I18N']->get('Duplicate Email') . ' ' . $user["systemvalues"]["email"]);
          $c = 0;
          while (!$userid) {
            $c++;
            $query = sprintf('INSERT INTO %s (email,entered,confirmed,uniqid,htmlemail)
                          values("%s",now(),%d,"%s",1)', $tables["user"], $user["systemvalues"]["email"] .
            " ($c)", 0, $uniqid);
            $result = Sql_query($query, 1);
            $userid = Sql_insert_id();
          }
          $user["systemvalues"]["email"] = $user["systemvalues"]["email"] . " ($c)";
        }

        $count["email_add"]++;
        $some = 1;
      }

      reset($_SESSION["import_attribute"]);
   #   var_dump($_SESSION);exit;
      if ($new || (!$new && $_SESSION["overwrite"] == "yes")) {
        $query = "";
        $count["dataupdate"]++;
        $old_data = Sql_Fetch_Array_Query(sprintf('select * from %s where id = %d', $tables["user"], $userid));
        $old_data = array_merge($old_data, getUserAttributeValues('', $userid));
        $history_entry = $GLOBALS['scheme'] . '://' . getConfig("website") . $GLOBALS["adminpages"] . '/?page=user&id=' . $userid . "\n\n";
        foreach ($user["systemvalues"] as $column => $value) {
          if (!empty($column)) { # && !empty($value)) {
            if ($column == 'groupmapping' || strpos($column,'grouptype_') === 0) {
              ## specifically request this group, so that it doesn't interfere with the "groups" which are the ones 
              ## submitted in the form
              
              if (strpos($column,'grouptype_') === 0) {
                list($tmp,$type) = explode('_',$column);
              } else {
                $type = $_SESSION['grouptype'];
              }
              $type = sprintf('%d',$type);
              ## verify the type is set
              if (!in_array($type,array_keys($GLOBALS['config']['usergroup_types']))) {
                Warn('Invalid group membership type'.$type);
                dbg($type,'Type not found');
              }
              
              $columnGroups = explode(',',$value);
              foreach ($columnGroups as $sGroup) {
                $sGroup = trim($sGroup);
                $groupIdReq = Sql_Fetch_Row_Query(sprintf('select id from groups where name = "%s"',$sGroup));
                if (empty($groupIdReq[0])) {
                  Sql_Query(sprintf('insert into groups (name) values("%s")',$sGroup));
                  Warn("Group $sGroup added");
                  $groupIdReq[0] = Sql_Insert_id();
                }
                dbg('Adding to group '.$sGroup.' with type '.$GLOBALS['config']['usergroup_types'][$type]);
                ## @@ this may cause problems on not-upgraded DBs
                Sql_Query(sprintf('replace into user_group (userid,groupid,type) values(%d,%d,%d)',$userid,$groupIdReq[0],$type));
              } 
            } else {
              $query .= sprintf('%s = "%s",', $column, $value);
            }
          }
        }
        if ($query) {
          $query = substr($query, 0, -1);
          # this may cause a duplicate error on email, so add ignore
          Sql_Query("update ignore {$tables["user"]} set $query where id = $userid");
        }
        foreach ($_SESSION["import_attribute"] as $item) {
          if (isset ($user[$item["index"]]) && is_numeric($item['record']) && strpos($item['record'],'grouptype_') !== 0) {
            $attribute_index = $item["record"];
            $uservalue = $user[$item["index"]];
            # check whether this is a textline or a selectable item
            $att = Sql_Fetch_Row_Query("select type,tablename,name from " . $tables["attribute"] . " where id = $attribute_index"); ////
            switch ($att[0]) {
              case "select" :
              case "radio" :
                $val = Sql_Query("select id from $table_prefix" . "listattr_$att[1] where name = \"$uservalue\"");
                # if we do not have this value add it
                if (!Sql_Affected_Rows()) {
                  Sql_Query("insert into $table_prefix" . "listattr_$att[1] (name) values(\"$uservalue\")");
                  Warn("Value $uservalue added to attribute $att[2]");
                  $user_att_value = Sql_Insert_Id();
                } else {
                  $d = Sql_Fetch_Row($val);
                  $user_att_value = $d[0];
                }
                break;
              case "checkboxgroup":
                $values = explode(',',$uservalue);
                $valueIds = array();
                foreach ($values as $importValue) {
                  $val = Sql_Query("select id from $table_prefix" . "listattr_$att[1] where name = \"$importValue\"");
                  # if we do not have this value add it
                  if (!Sql_Affected_Rows()) {
                    Sql_Query("insert into $table_prefix" . "listattr_$att[1] (name) values(\"$importValue\")");
                    Warn("Value $importValue added to attribute $att[2]");
                    $valueIds[] = Sql_Insert_Id();
                  } else {
                    $d = Sql_Fetch_Row($val);
                    $valueIds[] = $d[0];
                  }
                }
                $user_att_value = join(',',$valueIds);
                break;
              case "checkbox" :
                $uservalue = trim($uservalue);
                #print $uservalue;exit;
                if (!empty($uservalue) && $uservalue != "off")
                  $user_att_value = "on";
                else
                  $user_att_value = "";
                break;
              case "date" :
#                $user_att_value = parseDate($uservalue);
                $user_att_value = $uservalue;
                break;
              default :
                $user_att_value = $uservalue;
                break;
            }

            Sql_query(sprintf('replace into %s (attributeid,userid,value) values(%d,%d,"%s")', $tables["user_attribute"], $attribute_index, $userid, $user_att_value));
          } else {
            if ($item["record"] != "skip") {
              # add an empty entry if none existed
              Sql_Query(sprintf('insert ignore into %s (attributeid,userid,value) values(%d,%d,"")', $tables["user_attribute"], $item["record"], $userid));
            }
          }
        }
        $current_data = Sql_Fetch_Array_Query(sprintf('select * from %s where id = %d', $tables["user"], $userid));
        $current_data = array_merge($current_data, getUserAttributeValues('', $userid));
        $information_changed = 0;
        foreach ($current_data as $key => $val) {
          if (!is_numeric($key))
            if (isset($old_data[$key]) && $old_data[$key] != $val && $old_data[$key] && $key != "password" && $key != "modified") {
              $information_changed = 1;
              $history_entry .= "$key = $val\n*changed* from $old_data[$key]\n";
            }
        }
        if (!$information_changed) {
          $history_entry .= "\nNo user details changed";
        }
        addUserHistory($user["systemvalues"]["email"], "Import by " . adminName(), $history_entry);
      }

      #add this user to the lists identified, except when they are blacklisted
      $isBlackListed = isBlackListed($user["systemvalues"]["email"]);
      if (!$isBlackListed && is_array($_SESSION["lists"])) {
        reset($_SESSION["lists"]);
        $addition = 0;
        $listoflists = "";
        while (list ($key, $listid) = each($_SESSION["lists"])) {
          $query = "replace INTO " . $tables["listuser"] . " (userid,listid,entered) values($userid,$listid,now())";
          $result = Sql_query($query, 1);
          # if the affected rows is 2, the user was already subscribed
          $addition = $addition || Sql_Affected_Rows() == 1;
          $listoflists .= "  * " . listName($key)."\n";# $_SESSION["listname"][$key] . "\n";
        }
        if ($addition) {
          $count["list_add"]++;
        }
        if (!TEST && $_SESSION["notify"] == "yes" && $addition) {
          $subscribemessage = str_replace('[LISTS]', $listoflists, getUserConfig("subscribemessage", $userid));
          if (function_exists('sendmail')) {
            sendMail($user["systemvalues"]["email"], getConfig("subscribesubject"), $subscribemessage, system_messageheaders(), $envelope);
            if (isset ($_SESSION["throttle_import"])) {
              sleep($_SESSION["throttle_import"]);
            }
          }
        }
      } elseif ($isBlackListed) {
        $count['foundblacklisted']++;
      }
      if (!is_array($_SESSION["groups"])) {
        $groups = array ();
      } else {
        $groups = $_SESSION["groups"];
      }
      if (isset ($everyone_groupid) && !in_array($everyone_groupid, $groups)) {
        array_push($groups, $everyone_groupid);
      }
      if (defined('IN_WEBBLER') && is_array($groups)) {
        #add this user to the groups identified
        reset($groups);
        $groupaddition = 0;
        while (list ($key, $groupid) = each($groups)) {
          if ($groupid) {
            $query = sprintf('replace INTO user_group (userid,groupid,type) values(%d,%d,%d)',$userid,$groupid,$_SESSION['grouptype']);
            $result = Sql_query($query);
            # if the affected rows is 2, the user was already subscribed
            $groupaddition = $groupaddition || Sql_Affected_Rows() == 1;
          }
        }
        if ($groupaddition) {
          $count["group_add"]++;
        }
      }
	}
    } // end else
    if ($_SESSION["test_import"] && $c > 50)
      break;
  }


  $report = "";
  if (empty($some) && !$count["list_add"]) {
    $report .= '<br/>' . $GLOBALS['I18N']->get('All the emails already exist in the database and are member of the lists');
  } else {
    $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('%s emails succesfully imported to the database and added to %d lists.'), $count["email_add"], $num_lists);
    $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('%d emails subscribed to the lists'), $count["list_add"]);
    if ($count["exist"]) {
      $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('%s emails already existed in the database'), $count["exist"]);
    }
  }
  if ($count["invalid_email"]) {
    $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('%d Invalid Emails found.'), $count["invalid_email"]);
    if (!$_SESSION["omit_invalid"]) {
      $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('These records were added, but the email has been made up from ') . $_SESSION["assign_invalid"]);
    } else {
      $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('These records were deleted. Check your source and reimport the data. Duplicates will be identified.'));
    }
  }
  if ($count["duplicate"]) {
    $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('%d duplicate emails found.'), $count["duplicate"]);
  }
  if ($_SESSION["overwrite"] == "yes") {
    $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('Subscriber data was updated for %d subscribers'), $count["dataupdate"]);
  }
  if ($count['foundblacklisted']) {
    $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('%s emails were on the blacklist and have not been added to the lists'), $count["foundblacklisted"]);
  }
  $report .= sprintf('<br/>' . $GLOBALS['I18N']->get('%d subscribers were matched by foreign key, %d by email'), $count["fkeymatch"], $count["emailmatch"]);
  if (!$GLOBALS['commandline']) {
    print $report;
    if (function_exists('sendmail')) {
      sendMail(getConfig("admin_address"), $GLOBALS['I18N']->get('phplist Import Results'), $report);
    }
    if (function_exists('logevent')) {
      logEvent($report);
    }
    foreach ($GLOBALS['plugins'] as $pluginname => $plugin) {
      $plugin->importReport($report);
    }
  } else {
    output($report);
  }
  $htmlupdate = $report.'<br/>'.PageLinkButton("import2",s('Import some more emails'));
  $htmlupdate = str_replace("'","\'",$htmlupdate);
  
  clearImport();
  $status = '<script type="text/javascript">
      var parentJQuery = window.parent.jQuery;
      parentJQuery("#progressbar").progressbar("destroy");
      parentJQuery("#busyimage").hide();
      parentJQuery("#progresscount").html(\''.$htmlupdate.'\');
      </script>';
      
}

