';
- }
- $texts = Array();
- $textq = $db->sql_unbuffered_query($this->fetch_page_search_resource());
- if(!$textq) $db->_die('');
- while($row = $db->fetchrow())
- {
- if ( $verbose )
+ $offset = $j * $pages_in_batch;
+
+ $j++;
+
+ if ( $verbose && $debug )
{
- ob_start();
- echo "Indexing page " . $this->nslist[$row['namespace']] . sanitize_page_id($row['page_id']) . "
";
- ob_flush();
- while (@ob_end_flush());
- flush();
+ echo "Running indexing round $j of $loops (offset $offset)\n" . ( isset($_SERVER['REQUEST_URI']) ? '
' : '' );
}
- if ( isset($this->nslist[$row['namespace']]) )
+
+ $texts = $db->sql_query('SELECT p.name, t.page_id, t.namespace, t.page_text FROM ' . table_prefix . "page_text AS t\n"
+ . " LEFT JOIN " . table_prefix . "pages AS p\n"
+ . " ON ( p.urlname = t.page_id AND p.namespace = t.namespace )\n"
+ . " WHERE ( p.password = '' OR p.password = '$sha1_blank' )\n"
+ . " AND ( p.visible = 1 )\n"
+ . " LIMIT $offset, $pages_in_batch;", false);
+ if ( !$texts )
+ $db->_die();
+
+ $k = $offset;
+
+ if ( $row = $db->fetchrow($texts) )
{
- $idstring = $this->nslist[$row['namespace']] . sanitize_page_id($row['page_id']);
- if ( isset($this->pages[$idstring]) )
- {
- $page = $this->pages[$idstring];
- }
- else
+ do
{
- $page = array('name' => dirtify_page_id($row['page_id']));
+ $k++;
+ if ( $verbose )
+ {
+ $mu = memory_get_usage();
+ echo " Indexing page $k of $num_pages: {$row['namespace']}:{$row['page_id']}";
+ if ( $debug )
+ echo ", mem = $mu...";
+ flush();
+ }
+
+ // Indexing identifier for the page in the DB
+ $page_uniqid = "ns={$row['namespace']};pid=" . sanitize_page_id($row['page_id']);
+ $page_uniqid = $db->escape($page_uniqid);
+
+ // List of words on the page
+ $wordlist = $this->calculate_word_list($row['page_text'], $row['page_id'], $row['name']);
+
+ // Index calculation complete -- run inserts
+ $inserts = array();
+ foreach ( $wordlist as $word )
+ {
+ if ( in_array($word, $stopwords) || strval(intval($word)) === $word || strlen($word) < 3 )
+ continue;
+ $word_db = $db->escape($word);
+ if ( !in_array($word, $master_word_list) )
+ {
+ $inserts[] = "( '$word_db', '$page_uniqid' )";
+ }
+ else
+ {
+ if ( $verbose && $debug )
+ echo '.';
+ $pid_col = ( ENANO_DBLAYER == 'MYSQL' ) ?
+ "CONCAT( page_names, ',$page_uniqid' )":
+ "page_names || ',$page_uniqid'";
+ $q = $db->sql_query('UPDATE ' . table_prefix . "search_index SET page_names = $pid_col WHERE word = '$word_db';", false);
+ if ( !$q )
+ $db->_die();
+ }
+ }
+ if ( count($inserts) > 0 )
+ {
+ if ( $verbose && $debug )
+ echo 'i';
+ $inserts = implode(",\n ", $inserts);
+ $q = $db->sql_query('INSERT INTO ' . table_prefix . "search_index(word, page_names) VALUES\n $inserts;", false);
+ if ( !$q )
+ $db->_die();
+ }
+
+ $master_word_list = array_unique(array_merge($master_word_list, $wordlist));
+ if ( $verbose )
+ {
+ if ( isset($_SERVER['REQUEST_URI']) )
+ echo '
';
+ echo "\n";
+ }
+ unset($inserts, $wordlist, $page_uniqid, $word_db, $q, $word, $row);
}
+ while ( $row = $db->fetchrow($texts) );
}
- else
- {
- $page = array('name' => dirtify_page_id($row['page_id']));
- }
- $texts[(string)$row['page_idstring']] = $row['page_text'] . ' ' . $page['name'];
+ $db->free_result($texts);
}
if ( $verbose )
{
- ob_start();
- echo "Calculating word list...";
- ob_flush();
- while (@ob_end_flush());
- flush();
- }
- $search->buildIndex($texts);
- if ( $verbose )
- {
- echo '
'.print_r($search->index, true).''; - // return; - $q = $db->sql_query('DELETE FROM '.table_prefix.'search_index'); - if(!$q) return false; - $secs = Array(); - $q = 'INSERT INTO '.table_prefix.'search_index(word,page_names) VALUES'; - foreach($search->index as $word => $pages) - { - $secs[] = '(\''.$db->escape($word).'\', \''.$db->escape($pages).'\')'; - } - $q .= implode(',', $secs); - unset($secs); - $q .= ';'; - $result = $db->sql_query($q); - $db->free_result(); - if($result) - return true; - else - $db->_die('The search index was trying to rebuild itself when the error occured.'); + return true; } /** diff -r 3a1c99845ca8 -r 717e71109645 includes/rijndael.php --- a/includes/rijndael.php Sun Feb 24 12:52:07 2008 -0500 +++ b/includes/rijndael.php Sun Feb 24 17:50:38 2008 -0500 @@ -10,6 +10,7 @@ * @license BSD-style license */ +define ('ENC_HEX', 201); define ('ENC_BASE64', 202); define ('ENC_BINARY', 203); diff -r 3a1c99845ca8 -r 717e71109645 includes/search.php --- a/includes/search.php Sun Feb 24 12:52:07 2008 -0500 +++ b/includes/search.php Sun Feb 24 17:50:38 2008 -0500 @@ -577,7 +577,7 @@ // Divisor for calculating relevance scores $divisor = ( count($query['any']) + count($query_phrase['any']) + count($query['req']) + count($query['not']) ) * 1.5; - + foreach ( $scores as $page_id => $score ) { if ( !isset($page_data[$page_id]) ) @@ -924,9 +924,8 @@ } /** - * Returns a list of words that shouldn't under most circumstances be indexed for searching. Kudos to MySQL. + * Returns a list of words that shouldn't under most circumstances be indexed for searching. * @return array - * @see http://dev.mysql.com/doc/refman/5.0/en/fulltext-stopwords.html */ function get_stopwords() @@ -935,52 +934,10 @@ if ( is_array($stopwords) ) return $stopwords; - $stopwords = array('a\'s', 'able', 'after', 'afterwards', 'again', - 'against', 'ain\'t', 'all', 'almost', 'alone', 'along', 'already', 'also', 'although', 'always', - 'am', 'among', 'amongst', 'an', 'and', 'another', 'any', 'anybody', 'anyhow', 'anyone', 'anything', 'anyway', - 'anyways', 'anywhere', 'apart', 'appear', 'appreciate', 'appropriate', 'are', 'aren\'t', 'around', 'as', 'aside', - 'ask', 'asking', 'associated', 'at', 'available', 'away', 'awfully', 'be', 'became', 'because', 'become', 'becomes', - 'becoming', 'been', 'before', 'beforehand', 'behind', 'being', 'believe', 'below', 'beside', 'besides', 'best', - 'better', 'between', 'beyond', 'both', 'brief', 'but', 'by', 'c\'mon', 'c\'s', 'came', 'can', 'can\'t', 'cannot', - 'cant', 'cause', 'causes', 'certain', 'certainly', 'changes', 'clearly', 'co', 'com', 'come', 'comes', 'concerning', - 'consequently', 'consider', 'considering', 'contain', 'containing', 'contains', 'corresponding', 'could', - 'couldn\'t', 'course', 'despite', 'did', 'didn\'t', 'different', 'do', - 'does', 'doesn\'t', 'doing', 'don\'t', 'done', 'down', 'downwards', 'during', 'each', 'edu', 'eg', 'eight', - 'either', 'else', 'elsewhere', 'enough', 'entirely', 'especially', 'et', 'etc', 'even', 'ever', 'every', - 'everybody', 'everyone', 'everything', 'everywhere', 'ex', 'exactly', 'example', 'except', 'far', 'few', 'fifth', - 'first', 'five', 'followed', 'following', 'follows', 'for', 'former', 'formerly', 'forth', 'four', 'from', - 'further', 'get', 'gets', 'getting', 'given', 'gives', 'go', 'goes', 'going', 'gone', 'got', - 'gotten', 'had', 'hadn\'t', 'happens', 'hardly', 'has', 'hasn\'t', 'have', 'haven\'t', 'having', - 'he', 'he\'s', 'hello', 'help', 'hence', 'her', 'here', 'here\'s', 'hereafter', 'hereby', 'herein', 'hereupon', - 'hers', 'herself', 'hi', 'him', 'himself', 'his', 'hither', 'hopefully', 'how', 'howbeit', 'however', 'i\'d', - 'i\'ll', 'i\'m', 'i\'ve', 'ie', 'if', 'ignored', 'immediate', 'in', 'inasmuch', 'inc', 'indeed', 'indicate', - 'indicated', 'indicates', 'inner', 'insofar', 'instead', 'into', 'inward', 'is', 'isn\'t', 'it', 'it\'d', 'it\'ll', - 'it\'s', 'its', 'itself', 'just', 'keep', 'keeps', 'kept', 'know', 'knows', 'known', 'last', 'lately', 'later', - 'latter', 'latterly', 'least', 'less', 'lest', 'let', 'let\'s', 'like', 'liked', 'likely', 'little', 'look', - 'looking', 'looks', 'ltd', 'mainly', 'many', 'may', 'maybe', 'me', 'mean', 'meanwhile', 'merely', 'might', 'more', - 'moreover', 'most', 'mostly', 'much', 'must', 'my', 'myself', 'name', 'namely', 'nd', 'near', 'nearly', 'necessary', - 'need', 'needs', 'neither', 'never', 'nevertheless', 'new', 'next', 'nine', 'no', 'nobody', 'non', 'none', 'noone', - 'nor', 'normally', 'not', 'nothing', 'novel', 'now', 'nowhere', 'obviously', 'of', 'off', 'often', 'oh', 'ok', - 'okay', 'old', 'on', 'once', 'one', 'ones', 'only', 'onto', 'or', 'other', 'others', 'otherwise', 'ought', 'our', - 'ours', 'ourselves', 'out', 'outside', 'over', 'overall', 'own', 'particular', 'particularly', 'per', 'perhaps', - 'placed', 'please', 'plus', 'possible', 'presumably', 'probably', 'provides', 'que', 'quite', 'qv', 'rather', 'rd', - 're', 'really', 'reasonably', 'regarding', 'regardless', 'regards', 'relatively', 'respectively', 'right', 'said', - 'same', 'saw', 'say', 'saying', 'says', 'second', 'secondly', 'see', 'seeing', 'seem', 'seemed', 'seeming', 'seems', - 'seen', 'self', 'selves', 'sensible', 'sent', 'serious', 'seriously', 'seven', 'several', 'shall', 'she', 'should', - 'shouldn\'t', 'since', 'six', 'so', 'some', 'somebody', 'somehow', 'someone', 'something', 'sometime', 'sometimes', - 'somewhat', 'somewhere', 'soon', 'sorry', 'specified', 'specify', 'specifying', 'still', 'sub', 'such', 'sup', - 'sure', 't\'s', 'take', 'taken', 'tell', 'tends', 'th', 'than', 'thank', 'thanks', 'thanx', 'that', 'that\'s', - 'thats', 'the', 'their', 'theirs', 'them', 'then', 'thence', 'there', 'there\'s', 'thereafter', - 'thereby', 'therefore', 'therein', 'theres', 'thereupon', 'these', 'they', 'they\'d', 'they\'ll', 'they\'re', - 'they\'ve', 'think', 'third', 'this', 'thorough', 'thoroughly', 'those', 'though', 'three', 'through', 'throughout', - 'thru', 'thus', 'to', 'together', 'too', 'took', 'toward', 'towards', 'tried', 'tries', 'truly', 'try', 'trying', - 'twice', 'two', 'un', 'under', 'unfortunately', 'unless', 'unlikely', 'until', 'unto', 'upon', 'use', - 'used', 'useful', 'uses', 'using', 'usually', 'value', 'various', 'very', - 'was', 'wasn\'t', 'way', 'we', 'we\'d', 'we\'ll', 'we\'re', 'we\'ve', 'welcome', 'well', 'went', 'were', 'weren\'t', - 'what', 'what\'s', 'whatever', 'when', 'whence', 'whenever', 'where', 'where\'s', 'whereafter', 'whereas', - 'which', 'while', 'who', 'who\'s', 'whole', 'whom', 'whose', 'why', 'will', 'willing', 'wish', 'with', 'within', - 'without', 'won\'t', 'wonder', 'would', 'would', 'wouldn\'t', 'yes', 'yet', 'you', 'you\'d', 'you\'ll', 'you\'re', - 'you\'ve', 'your', 'yours', 'zero'); + $stopwords = array('I', 'a', 'about', 'an', 'are', 'as', 'at', 'be', 'by', 'com', 'de', 'en', 'for', 'from', 'how', 'in', 'is', 'it', + 'la', 'of', 'on', 'or', 'that', 'the', 'this', 'to', 'was', 'what', 'when', 'where', 'who', 'will', 'with', 'and', + 'the'); + return $stopwords; } diff -r 3a1c99845ca8 -r 717e71109645 plugins/SpecialSearch.php --- a/plugins/SpecialSearch.php Sun Feb 24 12:52:07 2008 -0500 +++ b/plugins/SpecialSearch.php Sun Feb 24 17:50:38 2008 -0500 @@ -46,8 +46,9 @@ } $template->header(); @set_time_limit(0); + echo '
'; if($paths->rebuild_search_index(true)) - echo '
Index rebuilt!
'; + echo 'Index rebuilt!
'; else echo 'Index was not rebuilt due to an error.'; $template->footer();