<?php
if (!defined('VB_ENTRY')) die('Access denied.');
/*======================================================================*\
|| #################################################################### ||
|| # vBulletin 5.0.0 Beta 10
|| # ---------------------------------------------------------------- # ||
|| # Copyright 2000-2012 vBulletin Solutions Inc. All Rights Reserved. ||
|| # This file may not be redistributed in whole or significant part. # ||
|| # ---------------- VBULLETIN IS NOT FREE SOFTWARE ---------------- # ||
|| # http://www.vbulletin.com | http://www.vbulletin.com/license.html # ||
|| #################################################################### ||
\*======================================================================*/

/**
 * vB_Api_Search
 *
 * @package vBApi
 * @access public
 */
class vB_Api_Search extends vB_Api
{
	private static $badwords;
	private static $allbadwords;
	private static $goodwords;
	const asymptote = 5;
	const slope = 20;
	protected $search_json;
	protected $criteria;
	protected $window = 3;
	protected $maxpagesize = 20;
	const FILTER_STARTER_ONLY = 'starter_only';
	const FILTER_LASTVISIT = 'lastVisit';
	const FILTER_LASTDAY = 'lastDay';
	const FILTER_LASTWEEK = 'lastWeek';
	const FILTER_LASTMONTH = 'lastMonth';
	const FILTER_LASTYEAR = 'lastYear';
//	const FILTER_VIEW_CHANNEL = 'channel';
	const FILTER_VIEW_ACTIVITY = 'activity';
	const FILTER_VIEW_TOPIC = 'topic';
	const FILTER_VIEW_CONVERSATION_THREAD = 'conversation_thread';
	const FILTER_VIEW_CONVERSATION_THREAD_SEARCH = 'conversation_thread_search';
	const FILTER_VIEW_CONVERSATION_STREAM = 'conversation_stream';
	const FILTER_FOLLOWING_USERS = 'followMembers';
	const FILTER_FOLLOWING_CHANNEL = 'followChannel';
	const FILTER_FOLLOWING_ALL = 'followAll';
	const FILTER_FOLLOWING_BOTH = 'followBoth';
	const FILTER_FOLLOWING_CONTENT = 'followContent';
	const FILTER_FOLLOW = 'follow';

	const SEARCH_TYPE_SYSTEM = 0; // for system searches
	const SEARCH_TYPE_USER = 1; // for user request searches

	protected function __construct()
	{
		parent::__construct();
	}

	// based on vb/search/core.php::flood_check
	protected function floodCheck()
	{
		$searchFloodTime = vB::getDatastore()->getOption('searchfloodtime');

		//if we don't have a search limit then skip check
		if ($searchFloodTime == 0)
		{
			return true;
		}

		$userContext = vB::getUserContext();

		//if the user is an admin or a moderater, skip the check
		if ($userContext->hasPermission('adminpermissions', 'cancontrolpanel') /* OR $user->isModerator()*/)
		{
			return true;
		}

		$request = vB::getRequest();
		$userId = vB::getCurrentSession()->get('userid');

		// get last search for this user and check floodcheck
		$filter = array(
			'type' => self::SEARCH_TYPE_USER
		);
		if ($userId == 0)
		{
			// it's a guest
			$filter['ipaddress'] = $request->getIpAddress();
		}
		else
		{
			$filter['userid'] = $userId;
		}

//		$prevsearch = $db->query_first("
//			SELECT searchlogid, dateline
//			FROM " . TABLE_PREFIX . "searchlog AS searchlog
//			WHERE $filter AND
//			ORDER BY dateline DESC LIMIT 1
//		");
		$prevsearch = vB::getDbAssertor()->getRow('searchlog', $filter, array('field' => 'dateline', 'direction' => vB_dB_Query::SORT_DESC));
		if ($prevsearch)
		{
			$timepassed = $request->getTimeNow() - $prevsearch['dateline'];
			if ($timepassed < $searchFloodTime)
			{
				throw new vB_Exception_Api('searchfloodcheck', array($searchFloodTime, ($searchFloodTime - $timepassed)));
			}
		}

		return true;
	}

	/**
	 * Search for nodes
	 * @param string|array|object $search_json list of parameters that can be encoded in a json string
	 * @param int $searchType
	 * @return int result_id
	 */
	// TODO: $searchType shouldn't have a default value, but be explicitely declared in each call
	public function getSearchResult($search_json, $searchType = 0)
	{
		if (!is_string($search_json))
		{
			$search_json = json_encode($search_json);
		}
		$criteria = $this->json2criteria($search_json);
		if (!empty($search_json['error']))
		{
			$return_structure = $this->cacheResults(array(), $criteria, 0, $searchType);
			$return_structure['ignored_keywords'] = $criteria->get_ignored_keywords();
			$return_structure['totalRecords'] = 0;
			return $return_structure;
		}
		elseif (!$criteria->getIgnoreCache())
		{
			$hashkey = md5(serialize($criteria->getJSON())) ;

			//now see if we have a cached value.
			if ($searchlog = vB::getDbAssertor()->getRow('vBDBSearch:searchlog',
						array(vB_dB_Query::CONDITIONS_KEY => array(
							array(
								'field' => 'searchhash',
								'value' => $hashkey,
								'operator' => vB_dB_Query::OPERATOR_EQ
							),
							array(
								'field' => 'userid',
								'value' => vB::getCurrentSession()->get('userid'),
								'operator' => vB_dB_Query::OPERATOR_EQ
							),
							array(
								'field' => 'dateline',
								'value' => vB::getRequest()->getTimeNow() - 300 /* 5 minutes */,
								'operator' => vB_dB_Query::OPERATOR_GTE
							)
						)
					),
					array(
						'field' => array('dateline'),
						'direction' => vB_dB_Query::SORT_DESC,
					)
				)
			)
			{
				$searchlog['resultId'] = $searchlog['searchlogid'];
				$searchlog['totalRecords'] = $searchlog['results_count'];
				$searchlog['from_cache'] = true;
				$ignored_words = $criteria->get_ignored_keywords();
				if (!empty($ignored_words))
				{
					$searchlog['ignored_keywords'] = $ignored_words;
				}
				return $searchlog;
			}
		}

		return $this->getSearchResultsCriteria($criteria, $searchType);
	}

	/**
	 * Search for nodes
	 * @param vB_Search_Criteria $criteria a criteria object
	 * @return int result_id
	 */
	// shouldn't this be a protected method? It's only used in this class
	public function getSearchResultsCriteria(vB_Search_Criteria $criteria, $searchType)
	{
		//first let's see if this is valid.

		$userdata = vB::getUserContext()->getReadChannels();

		if (empty($userdata['canRead']))
		{
			throw new Exception('no_permission');
		}

		if (!empty($userdata['cantRead']))
		{
			$criteria->add_exclude_filter($userdata['cantRead']);
		}

		$json = $criteria->getJSON();
		$results = array();
		$records_nr = 0;

		$time_before = $time_after = microtime(true);

		if ($searchType == self::SEARCH_TYPE_USER)
		{
			// user requested this search, so we need to check flooding
			$this->floodCheck();
		}

		if (empty($json['error']))
		{
			$query = vB::getDbAssertor()->assertQuery(
				'vBDBSearch:getSearchResults',
					array(
						vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_METHOD,
						'criteria' => $criteria
					)
			);
			$time_after = microtime(true);
			if ($query AND $query->valid())
			{
				foreach ($query as $row)
				{
					$results[] = $row['nodeid'];
					$records_nr ++;
				}
			}
		}
		$return_structure = $this->cacheResults($results, $criteria, $time_after - $time_before, $searchType);

		$return_structure['ignored_keywords'] = $criteria->get_ignored_keywords();
		$return_structure['totalRecords'] = $records_nr;
		return $return_structure;
	}

	/**
	 *
	 * Caches the results
	 * @param array $nodeIds
	 * @return int resultId
	 */
	private function cacheResults($nodeIds, $criteria, $searchtime = 0, $searchType = 0)
	{
		$fields['type'] = intval($searchType);
		$fields['userid'] = vB::getCurrentSession()->get('userid');
		$fields['ipaddress'] =  vB::getRequest()->getIpAddress();
		$fields['searchhash'] = md5(serialize($criteria->getJSON())) ;
		$fields['sortby'] = $criteria->get_sort_field();
		$fields['sortorder'] = $criteria->get_sort_direction();
		$fields['searchtime'] = $searchtime;
		$fields['dateline'] = vB::getRequest()->getTimeNow();
		$fields['completed'] = 0;
		$fields['json'] = json_encode($criteria->getJSON());
		$fields['results'] = implode(',', $nodeIds);
		$fields['results_count'] = count($nodeIds);
		$fields['resultId'] = vB::getDbAssertor()->assertQuery('vBDBSearch:cacheResults',array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_METHOD,'fields'=>$fields));
		return $fields;
	}

	/**
	 * Search for nodeids and returns the resultid as well as the page value (avoids having to make a call for the resultid and immediately make another to fetch the page value)
	 * @param string|array|object $search_json list of parameters that can be encoded in a json string
	 * @param int $pagenumber pagination - the page number
	 * @param int $perpage pagination - the number of results per page
	 * @return array node_ids
	 */
	public function getInitialNodes($search_json, $perpage = false, $pagenumber = false, $getStarterInfo = false)
	{
		if (!is_string($search_json))
		{
			$search_json = json_encode($search_json);
		}

		$result = $this->getSearchResult($search_json);

		return $this->getMoreNodes($result, $perpage, $pagenumber, $getStarterInfo);
	}

	/**
	 * Search for nodes and returns the resultid as well as the page value (avoids having to make a call for the resultid and immediately make another to fetch the page value)
	 * @param string|array|object $search_json list of parameters that can be encoded in a json string
	 * @param int $perpage pagination - the number of results per page
	 * @param int $pagenumber pagination - the page number
	 * @param bool $getStarterInfo
	 * @param bool $doFloodCheck - most times we want to skip this check, except when coming from search module
	 * @return array search_result_structure
	 */
	public function getInitialResults($search_json, $perpage = 1000, $pagenumber = false, $getStarterInfo = false, $searchType = 0)
	{
		if (!is_string($search_json))
		{
			$search_json = json_encode($search_json);
		}
		$result = $this->getSearchResult($search_json, $searchType);
		try
		{
			return $this->getMoreResults($result, $perpage, $pagenumber, $getStarterInfo);
		}
		catch (vB_Exception_NodePermission $e)
		{

			if (!empty($result['from_cache']))
			{
				$this->purgeCacheForCurrentUser();
				// for the second attempt, we consider it a system search request (no need to perform flood check again)
				return $this->getInitialResults($search_json, $perpage, $pagenumber, $getStarterInfo, self::SEARCH_TYPE_SYSTEM);
			}
			else
			{
				$nodeid = $e->getMessage();
				$nodeids = explode(',', $result['results']);
				if (($index = array_search($nodeid, $nodeids)) !== false)
				{
					unset($nodeids[$index]);
					$result['results'] = implode(',', $nodeids);
					vB::getDbAssertor()->update('searchlog', array('results' => $result['results']), array('searchlogid' => $result['searchlogid']));
					return $this->getMoreResults($result, $perpage, $pagenumber, $getStarterInfo);
				}
				else
				{
					throw $e;
				}
			}
		}

	}

	/**
	 * Get the node_ids from a search resultId
	 * @param int $resultId id of the search result
	 * @param int $perpage pagination - the number of results per page
	 * @param int $pagenumber pagination - the page number
	 * @return array result structure without node content (only nodeids)
	 */
	public function getMoreNodes($resultId, $perpage = false, $pagenumber = false)
	{
		if (is_array($resultId))
		{
			$cache = $resultId;
			$resultId = $cache['resultId'];
			if (!is_numeric($resultId))
			{
				throw new vB_Exception_Api('invalid_search_syntax');
			}
		}
		elseif (is_numeric($resultId))
		{
			$cache = vB::getDbAssertor()->getRow('vBDBSearch:searchlog',array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_SELECT, 'searchlogid' => $resultId));
			if (empty($cache))
			{
				throw new Exception('results_not_found');
			}
			if ($cache['dateline'] < vB::getRequest()->getTimeNow() - 300)
			{
				return $this->getInitialNodes($cache['json'], $perpage, $pagenumber);
			}
		}
		else
		{
			throw new vB_Exception_Api('invalid_search_syntax');
		}
		$return_structure['resultId'] = $resultId;
		$return_structure['perpage'] = $perpage;
		$return_structure['pagenumber'] = $pagenumber;

		$return_structure['totalRecords'] = $cache['results_count'];
		$return_structure['searchJSON'] = $cache['json'];
		$return_structure['searchJSONStructure'] = json_decode($cache['json'],true);
		$return_structure['searchtime'] = number_format($cache['searchtime'] / 1000 , 2);
		$return_structure['hasMore'] = false;
		$options = vB::getDatastore()->get_value('options');

		$perpage = fetch_perpage($perpage, $options['maxresults'], $options['searchperpage']);
		$return_structure['perpage'] = $perpage;
		$pagenumber = max(intval($pagenumber), 1);
		$return_structure['totalpages'] = ceil($return_structure['totalRecords']/$perpage);
		$return_structure['pagenumber'] = min($pagenumber, $return_structure['totalpages']);
		$start = (($return_structure['pagenumber'] - 1) * $perpage);
		$return_structure['from'] = $start+1;
		$return_structure['nodeIds'] = array();
		if (!empty($cache['results']))
		{
			$return_structure['nodeIds'] = explode(',', $cache['results']);
		}
		if (!empty($cache['ignored_keywords']))
		{
			$return_structure['ignored_keywords'] = $cache['ignored_keywords'];
		}
		$return_structure['hasMore'] = count($return_structure['nodeIds']) > ($start + $perpage);
		if ($start > count($return_structure['nodeIds']))
		{
			return $return_structure;
			//throw new Exception('no_more_results');
		}
		if (!empty($return_structure['nodeIds']))
		{
			$return_structure['nodeIds'] = array_slice($return_structure['nodeIds'], $start, $perpage);
			$return_structure['nodeIds'] = array_combine($return_structure['nodeIds'], $return_structure['nodeIds']);
		}
		$return_structure['to'] = $start + count($return_structure['nodeIds']);
		$nextUrl = $prevUrl = '';
		$baseurl = class_exists('vB5_Config') ? vB5_Config::instance()->baseurl : vB::getDatastore()->getOption('bburl');
		$search_params = array (
				'r' => $resultId
			);
		$url = $baseurl . vB5_Route::buildUrl('search', array(),$search_params);
		if ($return_structure['pagenumber'] < $return_structure['totalpages'])
		{
			$search_params['p'] = $return_structure['pagenumber'] + 1;
			$nextUrl = $baseurl . vB5_Route::buildUrl('search', array(),$search_params);
		}
		if ($return_structure['pagenumber'] > 1)
		{
			$search_params['p'] = $return_structure['pagenumber'] - 1;
			$prevUrl = $baseurl . vB5_Route::buildUrl('search', array(),$search_params);
		}
		$return_structure['pagination'] = array(
			'startcount' => $return_structure['from'],
			'endcount' => $return_structure['to'],
			'totalcount' => $return_structure['totalRecords'],
			'currentpage' => $return_structure['pagenumber'],
			'prevurl' => $prevUrl,
			'nexturl' => $nextUrl,
			"baseurl" => $url,
			'totalpages' =>$return_structure['totalpages']
		);
		return $return_structure;
	}

	/**
	 * Get the nodes from a search resultId
	 * @param int $resultId id of the search result
	 * @param int $perpage pagination - the number of results per page
	 * @param int $pagenumber pagination - the page number
	 * @return array nodes - list of nodes in the resultId
	 */
	public function getMoreResults($resultId, $perpage = false, $pagenumber = false, $getStarterInfo = false)
	{
		$return_structure = $this->getMoreNodes($resultId, $perpage, $pagenumber);
		$return_structure['results'] = vB_Library::instance('Node')->getFullContentforNodes($return_structure['nodeIds'], array('withParent' => $getStarterInfo, 'showVM' => 1));
		return $return_structure;
	}

	public function getInfo($resultId)
	{
		$assertor = vB::getDbAssertor();
		$query = $assertor->assertQuery('vBDBSearch:searchlog',array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_SELECT, 'searchlogid' => $resultId));
		$return_structure['resultId'] = $resultId;
		if (!$query OR !$query->valid() OR !($cache = $query->current()))
		{
			throw new Exception('results_not_found');
		}
		$return_structure['totalRecords'] = $cache['results_count'];
		$return_structure['searchJSON'] = $cache['json'];
		$return_structure['searchJSONStructure'] = json_decode($cache['json'], true);
		return $return_structure;
	}

	/**
	 * Get the page number where the specified node is at on the node list.
	 * @param int $nodeid			The node id
	 * @param int $starter			The starter node of the node id
	 * @param int $perpage 			The number of nodes per page the node list is using
	 * @param int $depth			The depth - 0 means no stopping, otherwise 1= direct child, 2= grandchild, etc
	 * @param string $sort			The sort order of the node list, 'asc' or 'desc'
	 * @param bool $include_starter	The flag to indicate if the starter node should be included in the node list or not
	 * @return int					The page number of the specified node. 0 means the node was not found.
	 */
	public function getPageNumber($nodeid, $starter, $perpage = 20, $depth = 0, $sort = 'asc', $include_starter = true)
	{
		$pagenum = 0;
		if (intval($nodeid) > 0 AND !empty($starter) AND intval($starter['nodeid']) > 0)
		{
			$node = $this->getNode($nodeid);
			if (!empty($node))
			{
				//fetch records that fall between the starter's publishdate and the target node's publishdate
				//sort in asc order so that the last record is the node we're looking for
				//then from that, we can calculate the page number of the target node against the total nodes in the starter
				$searchParams = array(
					'channel'			=> $starter['nodeid'],
					'depth'				=> $depth,
					'include_starter'	=> $include_starter,
					'date'				=> array(
						'from'			=> ($include_starter) ? $starter['publishdate'] : $starter['publishdate'] + 1,
						'to'			=> $node['publishdate'],
					),
					'sort'				=> array(
						'publishdate'	=> 'asc',
					),
				);

				$search_json = json_encode($searchParams);
				if ($perpage <= 0 )
				{
					$perpage = 20;
				}

				$results = $this->getInitialResults($search_json, $perpage, 1);
				if (!empty($results) AND $results['totalpages'] > 0)
				{
					$pagenum = 	$results['totalpages'];

					//when sort is desc, the page number should be reversed and should be relative to the actual overall count of nodes in the starter
					if (strtolower($sort) == 'desc')
					{
						$totalrecords = ($depth == 1) ? $starter['textcount_1'] : $starter['totalcount_1']; //_1 means it includes the starter node itself
						$totalpages = ceil($totalrecords/$perpage);
						$searchtotal = $results['totalRecords'];
						$remainder = $totalrecords % $perpage;
						if ($remainder > 0)
						{
							$searchtotal -= $remainder;
							$pagenum = ceil($searchtotal/$perpage) + 1;
						}
						$pagenum = $totalpages - $pagenum + 1;
					}
				}
			}
		}
		return $pagenum;
	}

	/**
	 *
	 * Index a node
	 * @param int $node_id to index
	 * @param boolean $propagate flag to propagate the indexing to the nodes parents
	 */
	public function index($node_id, $propagate = true)
	{
		$node = $this->getNodeToIndex($node_id);
		if (empty($node))
		{
			return false;
		}
		list($title, $text) = $this->getTitleAndText($node);
		//getting the content type for the node
		$contenttype = vB_Api::instanceInternal('ContentType')->fetchContentTypeClassFromId($node['contenttypeid']);

		//this should really be part of the Attach indexable content but doing it that way means having to look up the
		//parent for each Attachment we index, which gets expensive when doing the propogation where we already have
		//the parent information and may end up fetching it for each and every attachment.  So let's handle this
		//as a special case
		if ($contenttype == 'Attach')
		{
			$parentid = $this->getParentNodeId($node['nodeid']);
			$parentnode = $this->getNodeToIndex($parentid);
			list($parentTitle, $parentText) = $this->getTitleAndText($parentnode);
			$title .= ' ' . $parentTitle;
			$text .= ' ' . $parentText;
		}

		$result = $this->indexText($node, $title, $text);
		if (!$result)
		{
			return false;
		}

		// need to index the parent when the attachment changes
		if(!empty($propagate) AND $contenttype == 'Attach')
		{
			$parentid = $this->getParentNodeId($node['nodeid']);
			$this->index($parentid, false);
		}

		//need to index the attachments when the parents change
		if (!empty($propagate) OR $contenttype == 'Attach')
		{
			$attachments = vB_Api::instanceInternal('Node')->getNodeAttachments($node_id);
			foreach ($attachments as $attach)
			{
				$this->indexChildAttachment($attach['nodeid'], $title, $text);
			}
		}
	}

	/**
	 * Index a node
	 * @param int $node_id to index
	 * @param boolean $propagate flag to propagate the indexing to the nodes parents
	 */
	protected function indexChildAttachment($node_id, $parentTitle, $parentText)
	{
		$node = $this->getNodeToIndex($node_id);
		if (empty($node))
		{
			return false;
		}

		list($title, $text) = $this->getTitleAndText($node);
		$title .= ' ' . $parentTitle;
		$text .= ' ' . $parentText;
		$result = $this->indexText($node, $title, $text);
		if (!$result)
		{
			return false;
		}

		return true;
	}

	protected function getParentNodeId($nodeid)
	{
			//todo -- We need to look at making this assertor call a node method.
			// propagate the indexing to the parents
			$queryinfo = vB::getDbAssertor()->assertQuery('vBForum:closure', array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_SELECT,
				'child' => $nodeid, 'depth' => 1));
			if ($queryinfo AND $queryinfo->valid() AND $closure = $queryinfo->current())
			{
				return $closure['parent'];
			}

			return false;
	}

	protected function getTitleAndText($node)
	{
		$indexableContent = vB_Api_Content::getContentApi($node['contenttypeid'])->getIndexableContent($node['nodeid']);
		$title = "";
		if (!empty($indexableContent['title']))
		{
			$title = strtolower($indexableContent['title']);
			unset($indexableContent['title']);
		}

		//need to put a space between segments or we could concatenate words and distort our index
		$text = implode(' ', $indexableContent);
		$text = strtolower($text);

		return array($title, $text);
	}

	protected function getNodeToIndex($node)
	{
		if (is_numeric($node))
		{
			return $this->getNode($node);
		}
		elseif (is_array($node) && !empty($node['nodeid']) && !empty($node['contenttypeid']))
		{
			return $node;
		}

		return null;
	}

	protected function indexText($node, $title, $text)
	{
		$assertor = vB::getDbAssertor();
		$title_words = $this->break_words($title);
		$words = array_merge($this->break_words($text), $title_words);
		// no meaningful words, no index
		if (empty($words))
		{
			return false;
		}

		//count the number of occurance of each word
		$word_occurances = array_count_values($words);
		$words = array_keys($word_occurances);
		$total_words = count($words);
		$crc = sprintf('%u', crc32(implode(' ', $words)));
		// check if the same content has already been indexed (update)
		// We reduce the check to checking the current CRC with the old(stored) CRC of the content
		if ($node['CRC32'] == $crc)
		{
			// no need to index if the content hasn't changed
				return false;
		}
		// fetch previously indexed word ids

		$indexed_words_res = $assertor->assertQuery('vBDBSearch:fetch_indexed_words', array('nodeid' => $node['nodeid']));
		$prev_index = array();
		while($indexed_words_res AND $indexed_words_res->valid())
		{
			$searchtoword = $indexed_words_res->current();
			$prev_index[$searchtoword['wordid']] = $searchtoword;
			$indexed_words_res->next();
		}
		// update the existing node with the new CRC
		$assertor->assertQuery('vBForum:node', array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_UPDATE, 'nodeid' => $node['nodeid'], 'CRC32' => $crc));
		//fetch the ids of the words
		$existing_words = array();
		$remaining_words = array_flip($words);
		// see if all the words are in the words table
		$existing_words_res = $assertor->assertQuery('vBDBSearch:words', array('word' => $words));

		foreach($existing_words_res as $existing_word)
		{
			$existing_words[$existing_word['word']] = $existing_word['wordid'];
			unset($remaining_words[$existing_word['word']]);
		}

		if (!empty($remaining_words))
		{
			//adding one missing word to the table
			if (count($remaining_words) == 1)
			{
				$word = key($remaining_words);
				$existing_words[$word] = $assertor->assertQuery('vBDBSearch:words',
					array(vB_dB_Query::TYPE_KEY=>vB_dB_Query::QUERY_INSERT, 'word' => $word));
				if (is_array($existing_words[$word]))
				{
					$existing_words[$word] = array_pop($existing_words[$word]);
				}
			}
			else
			{
				//adding all missing words to the database
				//$insert_words = $remaining_words;
				$insert_words = array();
				foreach ($remaining_words as $key => $value)
				{
					$insert_words[] = array($key);
				}
				$assertor->insertMultiple('vBDBSearch:words', array('word'), $insert_words);
				//get the missing(just added) wordids
				$existing_words_res = $assertor->assertQuery('vBDBSearch:words', array('word' => array_keys($remaining_words)));
				foreach ($existing_words_res as $existing_word)
				{
					$existing_words[$existing_word['word']] = $existing_word['wordid'];
				}
			}
		}

		$searchword_tables_unique = array();

		foreach ($words as $index => $word)
		{
			if (!isset($existing_words[$word]))
			{
				continue;
			}
			else
			{
				$wordid=$existing_words[$word];
			}
			// create a relation between the word and the searchword_table
			$suffix = $this->get_table_name($word);
			$score_params = array
			(
				'occurance_nr' => $word_occurances[$word],
				'total_words' => $total_words,
				'word' => $word,
				'position' => $index + 1, //index is 0 based
				'is_title' => 0
			);
			if (!empty($title_words) AND in_array($word, $title_words))
			{
				$score_params['is_title'] = 1;
			}
			$score = $this->get_score($score_params);

			//let's check if this word has previously been indexed
			if (array_key_exists($wordid, $prev_index))
			{
				//do we need to update it?
				if ($prev_index[$wordid]['score'] != $score OR $prev_index[$wordid]['is_title'] != $score_params['is_title'] OR $prev_index[$wordid]['position'] != ($index + 1) )
				{

					$assertor->assertQuery(
						'vBDBSearch:updateSearchtowords',
						array
						(
							'wordid' => $wordid,
							'nodeid' => $node['nodeid'],
							'suffix' => $prev_index[$wordid]['suffix'],
							'score' => $score,
							'is_title' => ($score_params['is_title']?1:0),
							'position' => ($index + 1)
						)
					);

				}
				//now that we know the previous index is accurate, we can remove it from the list of obsolete index
				unset($prev_index[$wordid]);
			}
			else // it looks like a new word that needs to be indexed
			{
				// creating a list of indexes partitioned by the searchtowords tables
				$searchword_tables_unique[$suffix][$wordid] = array
				(
					'is_title' => $score_params['is_title']?1:0,
					'score' => $score,
					'position' => $index + 1 //index is 0 based
				);
			}
		}

		foreach ($searchword_tables_unique as $suffix => $wordids)
		{
			if (empty($wordids))
			{
				continue;
			}
			$values = array();
			foreach ($wordids as $wordid => $info)
			{
				$values[] = array($node['nodeid'], $wordid, empty($info['is_title']) ? 0 : 1, $info['score'], $info['position']);
			}
			$assertor->insertMultiple('vBDBSearch:searchtowords_' . $suffix, array('nodeid', 'wordid', 'is_title', 'score', 'position'), $values);
		}

		// let's clean up the entries from the previous indexing that are not used anymore
		foreach ($prev_index as $wordid => $details)
		{
			$assertor->delete('vBDBSearch:searchtowords_' . $details['suffix'], array('wordid' => $wordid, 'nodeid' => $node['nodeid']));
		}

		return true;
	}

	public function emptyIndex()
	{
		$assertor = vB::getDbAssertor();
		$assertor->update(
			'node',
			array('CRC32' => ''),
			array(
					array('field' => 'nodeid','value' => 0, 'operator' => vB_dB_Query::OPERATOR_NE)
			)
		);
		$assertor->assertQuery('vBDBSearch:truncate_table', array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_METHOD, 'table' => 'words'));
		$data = array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_DELETE);
		$names = $this->get_table_name_suffixes();
		foreach ($names as $name)
		{
			$assertor->assertQuery('vBDBSearch:truncate_table', array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_METHOD, 'table' => 'searchtowords_' . $name));
		}
	}

	public function indexAll()
	{
		//$parent_node = $this->getTopNodeId();
		$query = vB::getDbAssertor()->assertQuery('vBForum:node',array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_SELECT));

		if ($query AND $query->valid())
		{
			$row = $query->current();
			while($query->valid())
			{
				try
				{
					$this->index($row['nodeid']);
				}
				catch (Exception $e)
				{
					// we are going to ignore the exception and keep indexing the rest
					unset($e);
				}
				$row = $query->next();
			}
		}
	}

	/**
	 *
	 * indexes a range of nodes
	 * @param int $start - where to start indexing from - used for pagination
	 * @param int $perpage - nr of nodes to index
	 * @param array $filter - only index this type of nodes
	 * @return bool - returns true if there are more nodes to index
	 */
	public function indexRange($start = 0, $perpage = false, $filter = false)
	{
		$params = array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_METHOD);
		if (!empty($start))
		{
			$params[vB_dB_Query::PARAM_LIMITSTART] = intval($start);
		}
		if (!empty($perpage))
		{
			$params['limit'] = intval($perpage);
		}
		if (!empty($filter) AND !empty($filter['type']))
		{
			$params['contenttypeid'] = vB_Types::instance()->getContentTypeID($filter['type']);
		}

		$query = vB::getDbAssertor()->assertQuery('vBForum:getNodes',$params);
		$nr = 0;
		foreach ($query as $node)
		{
			try
			{
				$this->index($node);
			}
			catch (Exception $e)
			{
				// we are going to ignore the exception and keep indexing the rest
				unset($e);
			}
			$nr ++;
		}
		if (empty($perpage))
		{
			return false;
		}
		return $nr == $perpage;
	}

	public function delete($nodeid, $node = false)
	{
		$data = array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_DELETE, 'nodeid' => $nodeid);
		$assertor = vB::getDbAssertor();
		$names = $this->get_table_name_suffixes();
		foreach ($names as $name)
		{
			$assertor->assertQuery('vBDBSearch:searchtowords_' . $name, $data);
		}
		//Make sure the node does not exist anymore, delete it if still there.
		//Don't use the API/Library as it uses the cache
		$node = $assertor->getRow('vBForum:node', array('nodeid' => $nodeid));
		if (!empty($node))
		{
			vB_Api_Content::getContentApi($node['contenttypeid'])->delete($nodeid);
		}
	}

	private function getTopNodeId()
	{
		$node = $this->getNode(false, array('parentid' => '0'));
		if (empty($node))
		{
			throw new Exception('cant_find_top_node');
		}

		return $node['nodeid'];
	}

	private function getNode($nodeId, $add_params = array())
	{
		$params = array(vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_SELECT);
		if (!empty($nodeId))
		{
			$params['nodeid'] = $nodeId;
		}

		if (!empty($add_params))
		{
			$params = array_merge($params, $add_params);
		}

		$query = vB::getDbAssertor()->assertQuery('vBForum:node', $params);

		if (!$query OR !$query->valid())
		{
			return false;
		}

		return $query->current();

	}

	/**
	 * Returns the Channel structure
	 * @param bool
	 * @param array (optional) filters for query. Available values:
	 *				- exclude_categories
	 * @return array channel structure
	 */
	public function getChannels($flat = false, $queryOptions = array('exclude_categories' => false))
	{
		$options = vB::getDatastore()->get_value('options');
		$query = vB::getDbAssertor()->assertQuery('vBForum:getChannel',array(
			vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_METHOD,
			'include_parent' => 1,
			vB_dB_Query::PARAM_LIMIT => $options['maxresults'],
			'contenttypeid' => vB_Api::instanceInternal('ContentType')->fetchContentTypeIdFromClass('Channel'),
			'sort' => array(
							'parentid' => 'asc',
							'displayorder' => 'asc'
							)
		));
		if (!$query OR !$query->valid())
		{
			return array();
		}

		if (isset($queryOptions['exclude_categories']) AND !empty($queryOptions['exclude_categories']))
		{
			$categoryIds = $channelIds = array();
			foreach($query AS $channel)
			{
				$channelIds[] = $channel['nodeid'];
			}

			if (!empty($channelIds))
			{
				$categories = vB::getDbAssertor()->assertQuery('channel', array(
					vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_SELECT,
					vB_dB_Query::COLUMNS_KEY => array('nodeid'),
					vB_dB_Query::CONDITIONS_KEY => array(
						array('field' => 'nodeid', 'value' => $channelIds),
						array('field' => 'category', 'value' => 1)
					)
				));

				if ($categories)
				{
					foreach($categories as $category)
					{
						$categoryIds[] = $category['nodeid'];
					}
				}
			}
		}

		$results = array();
		$resultsTree = array();
		$excluded = array();
		foreach ($query AS $channel)
		{
			$channel['displayorder'] = intval($channel['displayorder']);
			if (isset($queryOptions['exclude_categories']) AND !empty($queryOptions['exclude_categories']) AND
				in_array($channel['nodeid'], $categoryIds))
			{
				$excluded[] = $channel['nodeid'];
				continue;
			}

			if (!empty($results[$channel['nodeid']]['channels']))
			{
				$channel['channels'] = $results[$channel['nodeid']]['channels'];
			}
			$results[$channel['nodeid']] = $channel;
			if ($channel['depth'] != 0 AND !in_array($channel['parentid'], $excluded))
			{
				$results[$channel['parentid']]['channels'][$channel['nodeid']] = &$results[$channel['nodeid']];
			}
			else
			{
				$resultsTree[$channel['nodeid']] = &$results[$channel['nodeid']];
			}
		}
		if ($flat)
		{
			return self::flattenTree($resultsTree, 'channels');
		}
		return $resultsTree;
	}

	private function flattenTree($tree, $branchName)
	{
		$leaves = array();
		foreach ($tree as $key => $branches) {
			$branch = array();
			if (!empty($branches[$branchName]))
			{
				$branch = $branches[$branchName];
				//unset($branches[$branchName]);
			}
			$leaves[$key] = $branches;
			if (!empty($branch))
			{
				$templeaves = self::flattenTree($branch,$branchName);
				foreach ($templeaves as $index => $leaf) {
					$leaves[$index] = $leaf;
				}
			}
		}
		return $leaves;
	}




	public function getSearchableContentTypes()
	{
		return vB_Types::instance()->getSearchableContentTypes();
	}

	private function json2criteria(&$json)
	{
		if (is_string($json))
		{
			$search_structure = json_decode($json, true);
		}
		else
		{
			$search_structure = $json;
		}
		if (empty($search_structure))
		{
			throw new vB_Exception_Api('invalid_search_syntax');
		}

		require_once (DIR . '/vb/search/core.php');
		$search_core = vB_Search_Core::get_instance();
		$criteria = $search_core->create_criteria(vB_Search_Core::SEARCH_COMMON);

		$def_sort_field = 'publishdate';
		$def_sort_dir = 'DESC';

		$currentUserId = vB::getCurrentSession()->get('userid');

		$criteria->setUser($currentUserId);

		$normalized_criteria = array();
		if (!empty($search_structure['keywords']))
		{
			$words = $criteria->add_keyword_filter($search_structure['keywords'], !empty($search_structure['title_only']));
			$def_sort_field = 'relevance';
			if (!empty($words))
			{
				$normalized_criteria['keywords'] = '';
				foreach ($words as $word)
				{
					$normalized_criteria['keywords'] .= $word['word'] . ' ';
					if (!empty($word['joiner']))
					{
						$normalized_criteria['keywords'] .= strtoupper($word['joiner']) . ' ';
					}
				}
				if (!empty($search_structure['title_only']))
				{
					$normalized_criteria['title_only'] = 1;
				}
			}
			else
			{
				$normalized_criteria['error']= 'ignored_search_keywords';
			}
		}

		if (!empty($search_structure['contenttypeid']))
		{
			$contentypeids = $criteria->add_contenttype_filter($search_structure['contenttypeid']);
			$normalized_criteria['contenttypeid'] = $search_structure['contenttypeid'];
			$pmcontentypeid = vB_Api::instanceInternal('contenttype')->fetchContentTypeIdFromClass('PrivateMessage');
			if (count($contentypeids) == 1 AND in_array($pmcontentypeid, $contentypeids))
			{
				$search_structure['private_messages_only'] = 1;
			}
			$vmcontentypeid = vB_Api::instanceInternal('contenttype')->fetchContentTypeIdFromClass('VisitorMessage');
			if (count($contentypeids) == 1 AND in_array($vmcontentypeid, $contentypeids))
			{
				$search_structure['visitor_messages_only'] = 1;
			}
		}
		else if(!empty($search_structure['type']))
		{
			$contentypeids = $criteria->add_contenttype_filter($search_structure['type']);
			$normalized_criteria['type'] = $search_structure['type'];
			$pmcontentypeid = vB_Api::instanceInternal('contenttype')->fetchContentTypeIdFromClass('PrivateMessage');
			if (in_array($pmcontentypeid, $contentypeids))
			{
				if (count($contentypeids) == 1)
				{
					$search_structure['private_messages_only'] = 1;
				}
				else
				{
					$search_structure['include_private_messages'] = 1;
				}
			}
			$vmcontentypeid = vB_Api::instanceInternal('contenttype')->fetchContentTypeIdFromClass('VisitorMessage');
			if (in_array($vmcontentypeid, $contentypeids))
			{
				if (count($contentypeids) == 1)
				{
					$search_structure['visitor_messages_only'] = 1;
				}
//				else
//				{
//					$search_structure['include_visitor_messages'] = 1;
//				}
			}
		}
		elseif (empty($search_structure['exclude_type']) OR (is_string($search_structure['exclude_type']) AND $search_structure['exclude_type'] != 'vBForum_PrivateMessage') OR (is_array($search_structure['exclude_type']) AND !in_array('vBForum_PrivateMessage',$search_structure['exclude_type'])))
		{
			$search_structure['include_private_messages'] = 1;
//			$search_structure['include_visitor_messages'] = 1;
		}

		if (!empty($search_structure['author']))
		{
			//reducing users from array to single
			if (is_array($search_structure['author']) AND count($search_structure['author']) == 1)
			{
				$search_structure['author'] = array_pop($search_structure['author']);
				$search_structure['exactname'] = 1;
			}

			// converting username to userid
			if (is_string($search_structure['author']) AND !empty($search_structure['exactname']))
			{
				$search_structure['author'] = trim($search_structure['author']);

				if (vb_String::vbStrlen($search_structure['author']) >= 3)
				{
					if (isset($username))
					{	// $username is never set but since this was here, stop the undefined variable.
						$username = vB_String::htmlSpecialCharsUni($username);
					}
					$user = vB_Api::instanceInternal("User")->fetchByUsername($search_structure['author'], array(vB_Api_User::USERINFO_AVATAR));
					if (!empty($user['userid']))
					{
						$search_structure['author'] = $user['userid'];
					}
				}
			}

			if (is_numeric($search_structure['author']))
			{
				$is_owner = ($currentUserId == $search_structure['author']);

				$criteria->add_owner_filter('author', vB_Search_Core::OP_EQ, $search_structure['author']);

				if (!empty($search_structure['visitor_messages_only']))
				{
					$criteria->add_filter('visitor_messages_only', vB_Search_Core::OP_EQ, $search_structure['author']);
					$normalized_criteria['visitor_messages_only'] = 1;
//					$need_protected = true;
				}
				elseif (/*$is_owner AND */!empty($search_structure['include_private_messages']))
				{
					$criteria->add_owner_filter('include_private_messages', vB_Search_Core::OP_EQ, $search_structure['author']);
					$normalized_criteria['include_private_messages'] = 1;
				}
				elseif (/*$is_owner AND */!empty($search_structure['private_messages_only']))
				{
					$criteria->add_filter('private_messages_only', vB_Search_Core::OP_EQ, $search_structure['author']);
					$normalized_criteria['private_messages_only'] = 1;
				}

				$userinfo = vB_Api::instanceInternal("User")->fetchUserinfo($search_structure['author']);
				$normalized_criteria['author'] = array($userinfo['username']);
			}
			elseif (is_array($search_structure['author']))
			{
				$user_ids = array();
				$user_names = array();
				foreach ($search_structure['author'] as $author) {
					if (is_numeric($author))
					{
						$user_ids[] = intval($search_structure['author']);
					}
					elseif (is_string($author))
					{
						$author = vB_String::htmlSpecialCharsUni($author);
						$user = vB_Api::instanceInternal("User")->fetchByUsername($author);
						$userid = false;
						if (!empty($user['userid']))
						{
							$user_ids[] = $user['userid'];
						}
					}
				}
				$criteria->add_filter('userid', vB_Search_Core::OP_EQ, $user_ids, true);
				sort($user_ids);
				$normalized_criteria['author'] = $user_ids;
			}
			elseif (is_string($search_structure['author']))
			{
				switch ($search_structure['author'])
				{
					case 'myFriends':
						$userid = array();
						if (!empty($currentUserId))
						{
							$query = vB::getDbAssertor()->assertQuery('userlist', array(
								vB_dB_Query::TYPE_KEY => vB_dB_Query::QUERY_SELECT,
								'userid' => $currentUserId,
								'friend' => 'yes'
							));

							if ($query AND $query->valid())
							{
								$userlist = $query->current();
								while ($query->valid())
								{
									$userid[] = $user['relationid'];
									$userlist = $query->next();
								}
							}
						}
						$criteria->add_filter('userid', vB_Search_Core::OP_EQ, $userid, true);
					break;
					case 'iFollow':
						if (empty($search_structure[self::FILTER_FOLLOW]))
						{
							$search_structure[self::FILTER_FOLLOW] = self::FILTER_FOLLOWING_USERS;
						}
						elseif ($search_structure[self::FILTER_FOLLOW] == self::FILTER_FOLLOWING_CHANNEL)
						{
							$search_structure[self::FILTER_FOLLOW] = self::FILTER_FOLLOWING_BOTH;
						}
					break;
					default:
						$criteria->add_user_filter($search_structure['author'], !empty($search_structure['exactname']));
					break;
				}
				$normalized_criteria['author'] = strtolower($search_structure['author']);
				if (!empty($search_structure['exactname']))
				{
					$normalized_criteria['exactname'] = 1;
				}
			}
		}
		if (!empty($search_structure['tag']))
		{
			if (is_array($search_structure['tag']))
			{
				foreach ($search_structure['tag'] as $index => $tag) {
					$search_structure['tag'][$index] = vB_String::htmlSpecialCharsUni(trim($tag));
				}
			}
			else
			{
				$search_structure['tag'] = vB_String::htmlSpecialCharsUni(trim($search_structure['tag']));
			}
			$existing_tags = $criteria->add_tag_filter($search_structure['tag']);
			$normalized_criteria['tag'] = array_keys($existing_tags);
			vB_Api::instanceInternal('Tags')->logSearchTags($existing_tags);
		}

		if (!empty($search_structure['date']))
		{
			if ($search_structure['date'] == self::FILTER_LASTVISIT)
			{
				$current_user = new vB_Legacy_CurrentUser();
				$dateline = $current_user->get_field('lastvisit');
				if(!empty($dateline))
				{
					$criteria->add_date_filter(vB_Search_Core::OP_GT, $dateline);
					$normalized_criteria['date'] = self::FILTER_LASTVISIT;
				}
			}

			if (!empty($search_structure['date']['from']))
			{
				$dateline = $this->computeDateLine($search_structure['date']['from']);
				if(!empty($dateline))
				{
					$criteria->add_date_filter(vB_Search_Core::OP_GT, $dateline);
					$normalized_criteria['date']['from'] = strtolower($search_structure['date']['from']);
				}
			}

			if (!empty($search_structure['date']['to']))
			{
				$dateline = $this->computeDateLine($search_structure['date']['to']);
				if(!empty($dateline))
				{
					$criteria->add_date_filter(vB_Search_Core::OP_LT, $dateline);
					$normalized_criteria['date']['to'] = strtolower($search_structure['date']['from']);
				}
			}
		}

		if (!empty($search_structure['topicAge']))
		{
			$criteria->add_topic_agefilter();
		}
		else if (!empty($search_structure['channelAge']))
		{
			$criteria->add_channel_agefilter();
		}

		if (!empty($search_structure['exclude']))
		{
			$criteria->add_exclude_filter($search_structure['exclude']);
			$normalized_criteria['exclude'] = $search_structure['exclude'];
		}

		if (empty($search_structure['sort']))
		{
			$search_structure['sort'] = array($def_sort_field => $def_sort_dir);
		}

		if (!is_array($search_structure['sort']))
		{
			$sort_dir = $def_sort_dir;
			if (strtolower($search_structure['sort']) == 'title')
			{
				$sort_dir = 'ASC';
			}
			$search_structure['sort'] = array($search_structure['sort'] => $sort_dir);
		}

		foreach ($search_structure['sort'] as $sort_field => $sort_dir)
		{
			$criteria->set_sort($sort_field, $sort_dir);
		}

		if ($sort_field != $def_sort_field AND $sort_dir != $def_sort_dir)
		{
			$normalized_criteria['sort'] = array($sort_field => $sort_dir);
		}

		if (!empty($search_structure['view']))
		{
			if ($search_structure['view'] == vB_Api_Search::FILTER_VIEW_CONVERSATION_STREAM AND !empty($search_structure['channel']))
			{
				$search_structure['depth'] = 2;
				$search_structure['include_starter'] = true;
			}
			elseif ($search_structure['view'] == vB_Api_Search::FILTER_VIEW_CONVERSATION_THREAD AND !empty($search_structure['channel']))
			{
				$search_structure['depth'] = 1;
				$search_structure['include_starter'] = true;
			}
			else
			{
				$criteria->add_view_filter($search_structure['view']);
				$normalized_criteria['view'] = $search_structure['view'];
			}
		}

		if (!empty($search_structure['starter_only']))
		{
			$criteria->add_filter('starter_only', vB_Search_Core::OP_EQ, true);
			$search_structure['include_starter'] = true;
			$normalized_criteria['starter_only'] = 1;
		}

		if (!empty($search_structure['channel']))
		{
			$criteria->add_channel_filter($search_structure['channel'], empty($search_structure['depth']) ? false : $search_structure['depth'], empty($search_structure['include_starter'])?false:true);

			$normalized_criteria['channel'] = $search_structure['channel'];

			if (!empty($search_structure['depth']))
			{
				$normalized_criteria['depth'] = $search_structure['depth'];
			}

			if (!empty($search_structure['include_starter']))
			{
				$normalized_criteria['include_starter'] = 1;
			}
		}

		if (!empty($search_structure['featured']))
		{
			$criteria->add_filter('featured', vB_Search_Core::OP_EQ, $search_structure['featured']);
			$normalized_criteria['featured'] = 1;
		}

		if (!empty($search_structure[self::FILTER_FOLLOW]))
		{
			if (is_string($search_structure[self::FILTER_FOLLOW]) AND !empty($currentUserId))
			{
				$criteria->add_follow_filter($search_structure[self::FILTER_FOLLOW], $currentUserId);
				$normalized_criteria[self::FILTER_FOLLOW] = $search_structure[self::FILTER_FOLLOW];
			}
			elseif (is_array($search_structure[self::FILTER_FOLLOW]))
			{
				list($type, $userid) = each($search_structure[self::FILTER_FOLLOW]);
				//if ($userid == $currentUserId)
				//{
					$criteria->add_follow_filter($type, $userid);
					$normalized_criteria[self::FILTER_FOLLOW] = $search_structure[self::FILTER_FOLLOW];
				//}
			}
		}

		if (!empty($search_structure['my_following']) AND !empty($currentUserId))
		{
			$criteria->add_follow_filter(self::FILTER_FOLLOWING_CHANNEL, $currentUserId);
			$normalized_criteria['my_following'] = $search_structure['my_following'];
		}

		if (!empty($search_structure['exclude_type']))
		{
			$criteria->add_contenttype_filter($search_structure['exclude_type'], vB_Search_Core::OP_NEQ);
			$normalized_criteria['exclude_type'] = $search_structure['exclude_type'];
		}

		if (!empty($search_structure['sticky_only']))
		{
			$criteria->add_filter('sticky', vB_Search_Core::OP_EQ, '1');
			$normalized_criteria['sticky_only'] = 1;
		}

		if (!empty($search_structure['exclude_sticky']))
		{
			$criteria->add_filter('sticky', vB_Search_Core::OP_NEQ, '1');
			$normalized_criteria['exclude_sticky'] = 1;
		}

		if (!empty($search_structure['include_sticky']))
		{
			$criteria->set_include_sticky();
			$normalized_criteria['include_sticky'] = 1;
		}

		if (empty($search_structure['include_blocked']) OR empty($currentUserId) OR !vB::getUserContext()->hasPermission('moderatorpermissions', 'canbanusers'))
		{
			//block people on the global ignore list.
			$globalignore = trim(vB::getDatastore()->getOption('globalignore'));
			if (!empty($globalignore))
			{
				$blocked = preg_split('#\s+#s', $globalignore, -1, PREG_SPLIT_NO_EMPTY);
				//the user can always see their own posts, so if they are in the blocked list we remove them
				if (!empty($currentUserId))
				{
					$bbuserkey = array_search($currentUserId , $blocked);

					if ($bbuserkey !== FALSE AND $bbuserkey !== NULL)
					{
						unset($blocked["$bbuserkey"]);
					}
				}
				//Make sure we didn't just empty the list
				if (!empty($blocked))
				{
					$criteria->add_filter('userid', vB_Search_Core::OP_NEQ, $blocked, false, true);
				}
			}
		}
		elseif (!empty($currentUserId) AND vB::getUserContext()->hasPermission('moderatorpermissions', 'canbanusers'))
		{
			$normalized_criteria['include_blocked'] = 1;
		}
		if (!empty($search_structure['ignore_protected']))
		{
			$criteria->add_filter('protected', vB_Search_Core::OP_NEQ, '1');
			$normalized_criteria['ignore_protected'] = 1;
		}

		if (!empty($search_structure['deleted_only']) AND !empty($currentUserId))
		{
			//$criteria->add_filter('showpublished', vB_Search_Core::OP_EQ, '0');
			$criteria->add_filter('showpublished', vB_Search_Core::OP_EQ, '0');
			//$criteria->add_filter('deleteuserid', vB_Search_Core::OP_NEQ, '0');
			$criteria->add_filter('deleteuserid', vB_Search_Core::OP_NEQ, '0');
			if (!vB::getUserContext()->hasPermission('moderatorpermissions', 'canremoveposts'))
			{
				$criteria->add_filter('userid', vB_Search_Core::OP_EQ, $currentUserId);
			}
			$normalized_criteria['deleted_only'] = 1;
		}

		if (empty($search_structure['include_attach']))
		{
			$criteria->add_filter('inlist', vB_Search_Core::OP_EQ, 1);
			$normalized_criteria['include_attach'] = 1;
		}


		if (empty($normalized_criteria['deleted_only'])/* AND empty($search_structure['include_unpublished'])*/)
		{
			//$criteria->add_filter('unpublishdate', vB_Search_Core::OP_GT, time());
			$criteria->add_owner_filter('unpublishdate', vB_Search_Core::OP_GT, time());
		}
		if (!empty($search_structure['unapproved_only']) AND !empty($currentUserId))
		{
			$criteria->add_filter('approved', vB_Search_Core::OP_NEQ, '1');
			if (!vB::getUserContext()->hasPermission('moderatorpermissions', 'canmanagethreads'))
			{
				$criteria->add_filter('userid', vB_Search_Core::OP_EQ, $currentUserId);
			}
			//$criteria->add_owner_filter('approved', vB_Search_Core::OP_NEQ, '1');
			$normalized_criteria['unapproved_only'] = 1;
		}

		if (empty($normalized_criteria['unapproved_only'])/* AND empty($search_structure['include_unapproved'])*/)
		{
			//$criteria->add_filter('showapproved', vB_Search_Core::OP_EQ, '1');
			$criteria->add_owner_filter('showapproved', vB_Search_Core::OP_EQ, '1');
		}

		if (!empty($search_structure['specific']))
		{
			$criteria->add_filter('nodeid', vB_Search_Core::OP_EQ, $search_structure['specific']);
			$normalized_criteria['specific'] = $search_structure['specific'];
		}

		if (!empty($search_structure['prefix']))
		{
			$normalized_criteria['prefix'] = $search_structure['prefix'];
			$criteria->add_filter('prefixid', vB_Search_Core::OP_EQ, $search_structure['prefix']);
		}

		if (!empty($search_structure['ignore_cache']))
		{
			$criteria->setIgnoreCache(true);
		}
		$criteria->setJSON($normalized_criteria);
		$json = $normalized_criteria;
		return $criteria;
	}

	private function computeDateLine($date)
	{

		$dateline = false;
		if (is_numeric($date))
		{
			//if(strlen($date)  >= 9)//unix timestamp

			//direct number compare should be faster
			if($date >= 100000000)
			{
				$dateline = $date;
			}
			else
			{
				$timeNow = vB::getRequest()->getTimeNow();
				$dateline = $timeNow - ($date * 86400); //24 x 60 x 60 = 1 day
			}
		}
		elseif (is_string($date))
		{
			$timeNow = vB::getRequest()->getTimeNow();
			switch ($date){
				case self::FILTER_LASTDAY :
					$dateline = $timeNow - 86400; //24 x 60 x 60 = 1 day
				break;
				case self::FILTER_LASTWEEK :
					$dateline = $timeNow -  604800; //7 x 24 x 60 x 60 = 1 day
				break;
				case self::FILTER_LASTMONTH :
					$dateline = strtotime('-1 month', $timeNow); //7 x 24 x 60 x 60 = 1 day
				break;
				case self::FILTER_LASTYEAR :
					$dateline = strtotime('-1 year', $timeNow); //7 x 24 x 60 x 60 = 1 day
				break;
				default:
					if ($parseddate = strtotime($date))
					{
						$dateline = $parseddate;
					}
				break;
			}
		}
		return $dateline;
	}
	/**
	 * finds which searchtowords table a word belongs to
	 * @var string $word
	 * @return array
	 */

	public static function get_table_name($word)
	{
		if (empty($word))return false;
		$firstchar = $word[0];
		$suffixes = self::get_table_name_suffixes();
		// do we have a valid character?
		if (in_array($firstchar, $suffixes))
		{
			return $firstchar;
		}
		// all non-valid characters are stored in the 'searchtowords_other' table
		return 'other';
	}

	/**
	 * gets the suffixes that need to be appended to the searchtowords table names
	 * @return array
	 */
	public static function get_table_name_suffixes()
	{
		$suffixes = range('a', 'z');
		$suffixes[] = 'other';
		return $suffixes;
	}

	/**
	 * breaks up the text into words
	 * @param string $text
	 * @return string
	*/
	public static function break_words($text)
	{
		$text = strip_tags($text);
		$text = strip_bbcode($text,true, false, false, true);
		$is_mb = false;
		// multibyte
		if (strlen($text) != vB_String::vbStrlen($text))
		{
			//$words = preg_split('/[\pP\s]/i', $text, -1,PREG_SPLIT_NO_EMPTY);
			$words = preg_split('/[\s,.!?@#$%^&*\(\)\\/<>"\';:\[\]\{\}\+|-]/', $text, -1,PREG_SPLIT_NO_EMPTY);
			$is_mb = true;
		}
		else
		{
			$words = preg_split('/[^a-z0-9_]+/i', $text, -1,PREG_SPLIT_NO_EMPTY);
		}

		foreach ($words as $index => $word) {
			if (!self::is_index_word($word)){
				unset($words[$index]);
				continue;
			}
			if ($is_mb)
			{
				$words[$index] = vB_String::toUtf8($word);
			}
		}
		return $words;
	}

	public static function is_index_word($word, $isLower = false)
	{
		$badwords = self::get_all_bad_words();
		$goodwords = self::get_good_words();

		if (!$isLower)
		{
			$word = vB_String::vBStrToLower($word);
		}
		// is the word in the goodwords array?
		if (in_array($word, $goodwords))
		{
			return 1;
		}
		else
		{
			// is the word outside the min/max char lengths for indexing?
			$wordlength = vB_String::vbStrlen($word);
			$options = vB::getDatastore()->get_value('options');
			if ($wordlength < $options['minsearchlength'] OR $wordlength > $options['maxsearchlength'])
			{
				return 0;
			}
			// is the word a common/bad word?
			else if (in_array($word, $badwords))
			{
				return false;
			}
			// word is good
			else
			{
				return 1;
			}
		}
	}

	/**
	 *
	 * The function for the word weight uses an asymptotic function
	 * @param array $word_info contains the information about the word
	 */
	protected function get_score($word_info)
	{
		$score = self::asymptote - ((self::asymptote-1) * exp(-1 * ($word_info['occurance_nr']-1)/self::slope));
		return round($score * 10000);

//		$score = ceil($word_info['occurance_nr'] * 100 / $word_info['total_words']);
//		//if the word is in the title, add more weight to it
//		$score = min(100, $score + (empty($word_info['is_title'])?0:50));
//		return $score;
	}

	public static function get_good_words()
	{
		if (!isset(self::$goodwords))
		{
			$options = vB::getDatastore()->get_value('options');
			$goodwords = trim($options['goodwords']);
			if (!empty($goodwords))
			{
				self::$goodwords = preg_split('#[ \r\n\t]+#s', strtolower($goodwords), -1, PREG_SPLIT_NO_EMPTY);
			}
			else
			{
				self::$goodwords = array();
			}
		}
		return self::$goodwords;
	}

	public static function get_all_bad_words()
	{
		if (!isset(self::$allbadwords))
		{
			$options = vB::getDatastore()->get_value('options');
			self::$allbadwords = array_merge(self::get_bad_words(), preg_split('/\s+/s', $options['badwords'], -1, PREG_SPLIT_NO_EMPTY));
		}
		return self::$allbadwords;
	}

	function saveSearchAsModule($data)
	{
		$db = vB::getDbAssertor();

		// copy widget record
		$widget = $db->getRow('widget', array('template' => 'widget_search', 'title' => 'Search'));
		if (!$widget)
		{
			throw new Exception('search_module_not_found');
		}
		$title_widget = $db->getRow('widget', array('template' => 'widget_search', 'title' => $data['searchTitle']));
		if ($title_widget)
		{
			throw new Exception('search_module_already_exists');
		}
		$widgetAPI = vB_Api::instanceInternal('widget');
		$new_widget_array = $widgetAPI->cloneWidget($widget['widgetid'], array('title' => $data['searchTitle'], 'default' => $data));
		$new_widget = $new_widget_array['widget'];
		$savedData = $widgetAPI->saveDefaultConfig($new_widget['widgetid'], $data);

		return $savedData;
	}

	/**
	 * Purge search log cache for current logged-in user
	 */
	public function purgeCacheForCurrentUser($from = 301)
	{
		$userinfo = vB::getCurrentSession()->fetch_userinfo();

		if ($userinfo['userid'])
		{
			vB::getDbAssertor()->update('searchlog', array('dateline' => vB::getRequest()->getTimeNow() - $from), array('userid' => $userinfo['userid']));
		}
	}

	public function clean()
	{
		vB::getDbAssertor()->delete('searchlog', array('dateline' => vB::getRequest()->getTimeNow() - 86400));
	}


	public static function get_bad_words()
	{
		if (!isset(self::$badwords))
		{
			$badwords = array(
				'&amp',
				'&quot',
				'a',
				'able',
				'about',
				'above',
				'according',
				'accordingly',
				'across',
				'actually',
				'after',
				'afterwards',
				'again',
				'against',
				'aint',
				'all',
				'allow',
				'allows',
				'almost',
				'alone',
				'along',
				'already',
				'also',
				'although',
				'always',
				'am',
				'among',
				'amongst',
				'an',
				'and',
				'another',
				'any',
				'anybody',
				'anyhow',
				'anyone',
				'anything',
				'anyway',
				'anyways',
				'anywhere',
				'apart',
				'appear',
				'appreciate',
				'appropriate',
				'are',
				'arent',
				'around',
				'as',
				'aside',
				'ask',
				'asking',
				'associated',
				'at',
				'available',
				'away',
				'awfully',
				'b',
				'be',
				'became',
				'because',
				'become',
				'becomes',
				'becoming',
				'been',
				'before',
				'beforehand',
				'behind',
				'being',
				'believe',
				'below',
				'beside',
				'besides',
				'best',
				'better',
				'between',
				'beyond',
				'both',
				'brief',
				'but',
				'by',
				'c',
				'came',
				'can',
				'cannot',
				'cant',
				'cause',
				'causes',
				'certain',
				'certainly',
				'changes',
				'clearly',
				'cmon',
				'co',
				'com',
				'come',
				'comes',
				'concerning',
				'consequently',
				'consider',
				'considering',
				'contain',
				'containing',
				'contains',
				'corresponding',
				'could',
				'couldnt',
				'course',
				'cs',
				'currently',
				'd',
				'definitely',
				'described',
				'despite',
				'did',
				'didnt',
				'different',
				'do',
				'does',
				'doesnt',
				'doing',
				'done',
				'dont',
				'down',
				'downwards',
				'during',
				'e',
				'each',
				'edu',
				'eg',
				'eight',
				'either',
				'else',
				'elsewhere',
				'enough',
				'entirely',
				'especially',
				'et',
				'etc',
				'even',
				'ever',
				'every',
				'everybody',
				'everyone',
				'everything',
				'everywhere',
				'ex',
				'exactly',
				'example',
				'except',
				'f',
				'far',
				'few',
				'fifth',
				'first',
				'five',
				'followed',
				'following',
				'follows',
				'for',
				'former',
				'formerly',
				'forth',
				'four',
				'from',
				'further',
				'furthermore',
				'g',
				'get',
				'gets',
				'getting',
				'given',
				'gives',
				'go',
				'goes',
				'going',
				'gone',
				'got',
				'gotten',
				'greetings',
				'h',
				'had',
				'hadnt',
				'happens',
				'hardly',
				'has',
				'hasnt',
				'have',
				'havent',
				'having',
				'he',
				'hello',
				'help',
				'hence',
				'her',
				'here',
				'heres',
				'hereafter',
				'hereby',
				'herein',
				'hereupon',
				'hers',
				'herself',
				'hes',
				'hi',
				'him',
				'himself',
				'his',
				'hither',
				'hopefully',
				'how',
				'howbeit',
				'however',
				'i',
				'id',
				'ie',
				'if',
				'ignored',
				'ill',
				'im',
				'immediate',
				'in',
				'inasmuch',
				'inc',
				'indeed',
				'indicate',
				'indicated',
				'indicates',
				'inner',
				'insofar',
				'instead',
				'into',
				'inward',
				'is',
				'isnt',
				'ist',
				'it',
				'itd',
				'itll',
				'its',
				'itself',
				'ive',
				'j',
				'just',
				'k',
				'keep',
				'keeps',
				'kept',
				'know',
				'knows',
				'known',
				'l',
				'last',
				'lately',
				'later',
				'latter',
				'latterly',
				'least',
				'less',
				'lest',
				'let',
				'lets',
				'like',
				'liked',
				'likely',
				'little',
				'look',
				'looking',
				'looks',
				'ltd',
				'm',
				'mainly',
				'many',
				'may',
				'maybe',
				'me',
				'mean',
				'meanwhile',
				'merely',
				'might',
				'more',
				'moreover',
				'most',
				'mostly',
				'much',
				'must',
				'my',
				'myself',
				'n',
				'name',
				'namely',
				'nd',
				'near',
				'nearly',
				'necessary',
				'need',
				'needs',
				'neither',
				'never',
				'nevertheless',
				'new',
				'next',
				'nine',
				'no',
				'nobody',
				'non',
				'none',
				'noone',
				'nor',
				'normally',
				'not',
				'nothing',
				'novel',
				'now',
				'nowhere',
				'o',
				'obviously',
				'of',
				'off',
				'often',
				'oh',
				'ok',
				'okay',
				'old',
				'on',
				'once',
				'one',
				'ones',
				'only',
				'onto',
				'or',
				'originally',
				'other',
				'others',
				'otherwise',
				'ought',
				'our',
				'ours',
				'ourselves',
				'out',
				'outside',
				'over',
				'overall',
				'own',
				'p',
				'particular',
				'particularly',
				'per',
				'perhaps',
				'placed',
				'please',
				'plus',
				'possible',
				'posted',
				'presumably',
				'probably',
				'provides',
				'q',
				'que',
				'quite',
				'quote',
				'qv',
				'r',
				'rather',
				'rd',
				're',
				'really',
				'reasonably',
				'regarding',
				'regardless',
				'regards',
				'relatively',
				'respectively',
				'right',
				's',
				'said',
				'same',
				'saw',
				'say',
				'saying',
				'says',
				'second',
				'secondly',
				'see',
				'seeing',
				'seem',
				'seemed',
				'seeming',
				'seems',
				'seen',
				'self',
				'selves',
				'sensible',
				'sent',
				'seriously',
				'seven',
				'several',
				'shall',
				'she',
				'should',
				'shouldnt',
				'since',
				'six',
				'so',
				'some',
				'somebody',
				'somehow',
				'someone',
				'something',
				'sometime',
				'sometimes',
				'somewhat',
				'somewhere',
				'soon',
				'sorry',
				'specified',
				'specify',
				'specifying',
				'still',
				'sub',
				'such',
				'sup',
				'sure',
				't',
				'take',
				'taken',
				'tell',
				'tends',
				'th',
				'than',
				'thank',
				'thanks',
				'thanx',
				'that',
				'thats',
				'the',
				'their',
				'theirs',
				'them',
				'themselves',
				'then',
				'thence',
				'there',
				'theres',
				'thereafter',
				'thereby',
				'therefore',
				'therein',
				'theres',
				'thereupon',
				'these',
				'they',
				'theyd',
				'theyll',
				'theyre',
				'theyve',
				'think',
				'third',
				'this',
				'thorough',
				'thoroughly',
				'those',
				'though',
				'three',
				'through',
				'throughout',
				'thru',
				'thus',
				'to',
				'together',
				'too',
				'took',
				'toward',
				'towards',
				'tried',
				'tries',
				'truly',
				'try',
				'trying',
				'ts',
				'twice',
				'two',
				'u',
				'un',
				'under',
				'unfortunately',
				'unless',
				'unlikely',
				'until',
				'unto',
				'up',
				'upon',
				'us',
				'use',
				'used',
				'useful',
				'uses',
				'using',
				'usually',
				'v',
				'value',
				'various',
				'very',
				'via',
				'viz',
				'vs',
				'w',
				'want',
				'wants',
				'was',
				'wasnt',
				'way',
				'we',
				'wed',
				'welcome',
				'well',
				'went',
				'were',
				'weve',
				'werent',
				'what',
				'whats',
				'whatever',
				'when',
				'whence',
				'whenever',
				'where',
				'whereafter',
				'whereas',
				'whereby',
				'wherein',
				'whereupon',
				'wherever',
				'wheres',
				'whether',
				'which',
				'while',
				'whither',
				'who',
				'whoever',
				'whole',
				'whom',
				'whos',
				'whose',
				'why',
				'will',
				'willing',
				'wish',
				'with',
				'within',
				'without',
				'wonder',
				'wont',
				'would',
				'would',
				'wouldnt',
				'x',
				'y',
				'yes',
				'yet',
				'you',
				'youd',
				'youll',
				'your',
				'youre',
				'yours',
				'yourself',
				'yourselves',
				'youve',
				'z',
				'zero'
			);
			// Legacy Hook 'search_stopwords' Removed //
			self::$badwords = $badwords;
		}
		return self::$badwords;
	}
}
