Make your own free website on
  This php code is converted into HTML using 
     Colorizator v1.0 (c) 2001 by Alexander Yanuar Koentjara

<? /* Program name : LexZEUS Indexer (in use as support after using LexZEUS Crawler v1.0) Version : 1.0.10032001 (10 - Mar - 2001) Purpose : Index the html page into database after LexZEUS Crawler do its job File name : lexz_indexer.php Author : Alexander Yanuar Koentjara Syntax : php -q lexz_indexer.php [URL_LIST.TXT] [URL_LIST.TXT] is an optional Text file contains URL to be indexed into database. The URL must be seperated by new line without commas. If URL_LIST.TXT is not found, the predefined array of URL will be used. Term of use : This program can be used freely (without notification). If you want to add/modify some functionality, please to do so. You can also redestribute it freely. Using/redestributing this portion/completed/modified code should retain the original author and the one who modified it (please don't forget to update the version number) DATABASE required : MySQL from ******************* TABLE required: *************** TABLE TB_KEYWORD KWD_ID bigint(10) UNSIGNED Not Null auto_increment primary KEYWORD varchar(100) Not Null indexed TABLE TB_KEYWORD_REFERENCE KWD_ID bigint(10) UNSIGNED Not Null index_2(1) URL_ID bigint(10) UNSIGNED Not Null index_1 META_ID int(3) UNSIGNED Null index_2(2) COUNT int(3) UNSIGNED Not Null TABLE TB_URL URL_ID bigint(10) UNSIGNED Not Null auto_increment primary URL_STRING varchar(255) Not Null indexed first 5 chars TITLE varchar(255) Not Null DESCRIPTION varchar(255) Not Null Example of SEARCH Query when searching "COMPUTER" or "PROGRAMMING" : ******************************************************************** SELECT TBU.URL_STRING, TBU.TITLE, TBU.DESCRIPTION, SUM(TBR.COUNT) AS RELEVANCE FROM TB_KEYWORD TBK, TB_KEYWORD_REFERENCE TBR, TB_URL TBU WHERE (TBK.KEYWORD = 'COMPUTER' OR TBK.KEYWORD = 'PROGRAMMING') AND TBK.KWD_ID = TBR.KWD_ID AND TBR.URL_ID = TBU.URL_ID GROUP BY TBR.URL_ID ORDER BY RELEVANCE DESC */ // connect to MySQL $conn= @mysql_connect('localhost','user','password'); if (!$conn) die ("Cannot connect to MySQL !") // if you use proxy server to connect to the site, please specify // the proxy setting (the setting can be seen from browser's connection setting): $use_proxy = 1; # Put 1 means use proxy, 0 means not $proxy_server = ""; # Your Proxy IP address or host, for example "" $proxy_port = 7777; # Your Proxy Port, usually larger than 1024 // if you use proxy and you use basic authentication to connect to proxy server, // please supply login and password below : $use_basic_auth = 0; # Put 1 means use authentication, 0 means not $proxy_login = "username"; # The login name is here $proxy_password = "password"; # The password is here // array of URL to be indexed (predefined URLs) : $arr_page = array( "http://www.somewhere.page1.php", "http://www.somewhere.page2.htm", "http://www.somewhere.page3.cgi" ); if ($argv[1]) $arr_page = @file($argv[1]); // exclude common english in indexer ... just add into array if necessary ... // all words must be in CAPITAL ... $exclude_words = array( "I", "AM", "WE", "ARE", "YOU", "YOUR", "OURS", "THEY", "THEIR", "THEM", "A", "AN", "MY", "HIS", "HER", "SHE", "HE", "IS", "WERE", "WAS", "TO", "FROM", "UNTIL", "WHAT", "WHERE", "WHO", "WHOM", "WHICH", "HAS", "HAVE", "WHY", "HAD", "BEEN", "BE", "THE", "OR", "AND", "FOR", "BY", "WILL", "WOULD", "SHALL", "SHOULD", "OUGHT", "MUST", "IF", "UPON", "ON", "IN", "ONTO", "THIS", "THESE", "THERE", "THOSE", "THAT", "BECAUSE", "SINCE", "AT", "YES", "NO", "SOME", "ANY", "ALTHOUGH", "THOUGH", "DO", "DOES", "NOT", "PLEASE", "WANT", "NEED", "WANTED", "NEEDED", "ANYONE", "ANYBODY", "ANYTHING", "ANYWHERE", "SOMETHING", "SOMEBODY", "SOMEWHERE", "SOMEONE", "NOBODY", "NOONE", "NOTHING", "NOWHERE" ); // construct $arr_exclude $arr_exclude=array(); for($i=0;$i<count($exclude_words);$i++) { $arr_exclude[$exclude_words[$i]]=1; } function server_name($url) { if (strtoupper(substr($url,0,7))=="HTTP://") $url = substr($url,7); $url = ereg_replace("/.*","",$url); return $url; } function get_content($url) { global $proxy_server, $proxy_port, $use_proxy, $use_basic_auth, $proxy_login, $proxy_password; $proxy_user = ""; if ($use_proxy) { $fp = fsockopen($proxy_server,$proxy_port); if (!$fp) die("Can't connect to proxy server !!"); if ($use_basic_auth) $proxy_user ="Proxy-Authorization: Basic ". base64_encode("$proxy_login:$proxy_password"). "\r\n"; } else { $host = server_name($url); $fp = fsockopen($host,80); if (!$fp) die("Can't connect to $host !!"); } fputs($fp,"GET $url HTTP/1.0\r\n". $proxy_user . "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*\r\n". "Accept-Encoding: gzip, deflate\r\n\r\n"); $data=fread($fp,4096); if (!eregi("html",$data)) { fclose($fp); return 0; } while (!feof($fp)) $data.=fread($fp,4096); fclose($fp); for($i=0;$i<1500;$i++) if (substr($data,$i,4)=="\r\n\r\n") {$pos=$i+4;$i=2000;} return substr($data,$pos); } function to_log($str,$flag=0) { // IMPLEMENT YOUR LOGGING SYSTEM HERE ... print $str."\n"; if ($flag_exit) die ("\n--ERROR OCCURS, TERMINATED--\n"); } function index_to_db($conn,$html_code,$url) { $META = array ("KEYWORD" => 1, "META_KEYWORDS" =>5, "META_DESCRIPTION"=>10, "TITLE" =>15); eregi("<TITLE>([^<]+)",$html_code,$title); $title=$title[1]; if (!trim($title)) $title="UNTITLED"; $htm2 = substr($html_code,0,2000); preg_match_all("/<META ([^>]*)/i",$htm2,$arr,PREG_SET_ORDER); $meta_k=array(); $meta_d=array(""); for ($i=0;$i<count($arr);$i++) { $meta = $arr[$i][1]; if (eregi("KEYWORD",$meta)) { if (count($meta_k)) continue; $meta=eregi_replace("name[[:space:]]*=[[:space:]]*[\"']*". "[^\"' ]+[\"']*[[:space:]]*","",$meta); $meta=eregi_replace("content[[:space:]]*=[[:space:]]*","",$meta); $meta=eregi_replace("[\"']","",$meta); $meta=eregi_replace("[[:space:]]+"," ",$meta); $meta=str_replace(", ",",",$meta); $meta_k = explode(",",trim($meta)); } if (eregi("DESCRIPTION",$meta)) { $meta=eregi_replace("name[[:space:]]*=[[:space:]]*[\"']*[^\"' ]+[\"']*[[:space:]]*", "",$meta); $meta=eregi_replace("content[[:space:]]*=[[:space:]]*","",$meta); $meta=eregi_replace("[\"']","",$meta); $meta=eregi_replace("[[:space:]]+"," ",$meta); $meta=str_replace(", ",",",$meta); $meta_d[0] = trim($meta); } } $txt = eregi_replace("<STYLE[^>]*>[^>]*</STYLE>","",$html_code); $txt = eregi_replace("<SCRIPT[^>]*>[^>]*</SCRIPT>","",$txt); $txt = eregi_replace("<OBJECT[^>]*>[^>]*</OBJECT>","",$txt); $txt = eregi_replace("<APPLET[^>]*>[^>]*</APPLET>","",$txt); $txt = eregi_replace("<EMBED[^>]*>[^>]*</EMBED>","",$txt); $txt = eregi_replace("&[a-z]+;","",$txt); $txt = strip_tags($txt); $txt = ereg_replace("[[:space:]]+"," ",$txt); if (!trim($meta_d[0])) { $meta_d[0] = trim(substr($txt,0,100)); $arr_text = split("[, ]",$meta_d[0]); unset($arr_text[count($arr_text)-1]); $meta_d[0]=join(" ",$arr_text); } // better to strip out http:// so we don't have unecessary string comparation // when look for URL_STRING $url2 = eregi_replace("^http://","",$url); $sql = mysql_query("SELECT URL_ID FROM TB_URL WHERE URL_STRING='$url2'",$conn); $row=@mysql_fetch_array($sql); // If URL is exist in database, it means do reindex : if ($row["URL_ID"]) { to_log("RE-INDEXing $url ..."); @mysql_free_result($sql); $URL_ID = $row["URL_ID"]; mysql_query("DELETE FROM TB_KEYWORD_REFERENCE WHERE URL_ID = $URL_ID",$conn); mysql_query("UPDATE TB_URL SET DESCRIPTION='".str_replace("'","''",$meta_d[0])."', TITLE='".str_replace("'","''",$title)."' WHERE URL_ID = $URL_ID",$conn); } else // A new URL to be indexed ... { to_log("INDEXing $url ..."); @mysql_free_result($sql); mysql_query("INSERT INTO TB_URL (URL_STRING, TITLE, DESCRIPTION) VALUES ('$url2', '".str_replace("'","''",$title). "', '".str_replace("'","''",$meta_d[0])."')",$conn); $URL_ID = mysql_insert_id($conn); } // index the title $arr_text = split("[, ]",$title); for ($i=0;$i<count($arr_text);$i++) index_word($arr_text[$i], $conn, $URL_ID, $META["TITLE"]); // index the meta description $arr_text = split("[, ]",$meta_d[0]); for ($i=0;$i<count($arr_text);$i++) index_word($arr_text[$i], $conn, $URL_ID, $META["META_DESCRIPTION"]); // index the meta keyword, remove the repetitive $meta_k2=array(); for ($i=0;$i<count($meta_k);$i++) $meta_k2[$meta_k[$i]]=1; reset($meta_k2); while($key=key($meta_k2)) { index_word($key, $conn, $URL_ID, $META["META_KEYWORDS"]); next($meta_k2); } // index the body of page $arr_text = split("[, ]",$txt); for ($i=0;$i<count($arr_text);$i++) index_word($arr_text[$i], $conn, $URL_ID, $META["KEYWORD"]); unset($arr_text); unset($meta_k); unset($meta_k2); } function index_word($word, $conn, $URL_ID, $meta_id) { if (strlen($word)>255) return; GLOBAL $arr_exclude; // remove the . , ! ? at the last word $word = strtoupper( ereg_replace("[.,!?]+$","",trim($word)) ); // remove the ( ) { } $word = ereg_replace("[(){}'\"]","",$word); if ($arr_exclude[$word] || strlen($word)<2) return; // check if keyword is exist ... $sql = mysql_query("SELECT KWD_ID FROM TB_KEYWORD WHERE KEYWORD='$word'",$conn); $row=@mysql_fetch_array($sql); if ($row["KWD_ID"]) { mysql_free_result($sql); $sql = mysql_query("SELECT KWD_ID FROM TB_KEYWORD_REFERENCE WHERE KWD_ID=".$row["KWD_ID"]." AND URL_ID=$URL_ID AND META_ID=$meta_id",$conn); // if the keyword reference is in database, update the count $row2=mysql_fetch_array($sql); if ($row2["KWD_ID"]) { mysql_free_result($sql); $sql = mysql_query("UPDATE TB_KEYWORD_REFERENCE SET COUNT=COUNT+$meta_id WHERE KWD_ID=".$row["KWD_ID"]." AND URL_ID=$URL_ID AND META_ID=$meta_id",$conn); } // if the keyword reference is not in database, insert it else { mysql_free_result($sql); $sql = mysql_query("INSERT TB_KEYWORD_REFERENCE (KWD_ID, URL_ID, META_ID, COUNT) VALUES (".$row["KWD_ID"].", $URL_ID, $meta_id, $meta_id)",$conn); } } else { @mysql_free_result($sql); $sql = mysql_query("INSERT TB_KEYWORD (KEYWORD) VALUES ('$word')",$conn); $KWD_ID = mysql_insert_id($conn); $sql = mysql_query("INSERT TB_KEYWORD_REFERENCE (KWD_ID, URL_ID, META_ID, COUNT) VALUES ($KWD_ID, $URL_ID, $meta_id, $meta_id)",$conn); } } if ($conn) { mysql_select_db("DB_SPIDER_SEARCH",$conn); for ($i=0; $i<count($arr_page); $i++) { $html_code = get_content(chop($arr_page[$i])); index_to_db($conn,$html_code,chop($arr_page[$i])); } } else { die("Cannot connect to database !\n"); } @mysql_close($conn); ?>