<?php

//There are some links that we don't want to visit.  This function 
//catches most of the ones we will encounter on the stat dept. website.

function UseLink($url, $avoid_cgi){
	$url = strtolower($url);
	$url_ext = substr($url, strlen($url)-4);
	$url_ext2 = substr($url, strlen($url)-3);
	
	if($avoid_cgi && strpos($url, "?") !== false){
		return(false);
	}else if(strpos($url, "#") !== false || strpos($url, "file:") !== false ||
		strpos($url, "mailto:") !== false || $url_ext == ".jpg" || 
		$url_ext == ".ini" || $url_ext == ".sam" || 
		$url_ext == ".pdf" || $url_ext == ".css" || 
		$url_ext2 == ".ps" || $url_ext == ".abs")
	{
		return(false);
	}else{
		return(true);
	}
}
 

//You will want to modify the arguments of CrawlPage so it returns the data you
//were supposed to collect.

//'depth' is a ticker which makes sure you don't crawl too many links deep in the site
//'base_url' is where we started (www-stat ... ), so we don't venture out of the stat
//			department site
//'link_list' is an array containing all of the links we've found.  We pass this in so
//			we don't traverse the same page multiple times
//'debug_info' lets us see which links are being visited/added to the list/etc as the 
//			function executes.
//'avoid_cgi' for what I was searching for, I could avoid the cgi links on craigslist.  These
//			have a '?' followed by some parameter.  You may want to crawl these
//			depending on what you're trying to collect.

function CrawlPage($base_url, $url, &$link_list, $avoid_cgi, $depth, $debug_info, 
					&$costs, &$summary_lengths){
	if($debug_info){ print("crawling ".$url."\n"); }
	
	$text=@file_get_contents($url);
	$tlen = strlen($text);
	
	$link_list_s = count($link_list);
	if(substr($url, strlen($url)-4) == "html" || substr($url, strlen($url)-4) == ".htm"){
		$url_base = (strpos($url, "/") === false) 
						? $url 
						: substr($url, 0, strrpos($url, "/"));
	}else{
		$url_base = $url;
		if($url[strlen($url)-1] == "/"){ $url = substr($url,0, strlen($url)-1); }
	}

	if ($text){
		//This regular expression is modified from one in a comment 
		//by "bgamrat at wirehopper dot com" on 06-Feb-2006 08:54 
		//at http://us3.php.net/manual/en/function.preg-match-all.php

  		preg_match_all(
   			"/(?:(?:href)\s*[=\(]\\s*[\"'`])".
   			"([\+\w:?=@&\/#._;-]+)(?:[\s\"'`])/i",
			$text,
			$src_href_url);
		
		if(!is_array($src_href_url)){
			if($debug_info){ print("no links found: ".$url."\n"); }
			return;
		}else{
			$src_href_url = $src_href_url[1];
		}
	}else{
		if($debug_info){ print("no text found: ".$url."\n"); }
			return;
	}

	for($i = 0; $i < count($src_href_url); $i++){
		if($debug_info){ print("found link ".$src_href_url[$i]."\n"); }
		
		if(UseLink($src_href_url[$i], $avoid_cgi) == false){
			//skip this link
			if($debug_info){ print("skipping ".$src_href_url[$i]."\n"); }
		}else if(strpos($src_href_url[$i] , "http://") !== false){
			//it's an absolute link.  We want to make sure it doesn't take us
			//outside of the stat department webpage
			if(	strpos($src_href_url[$i], $base_url) !== false && 
				array_search($src_href_url[$i], $link_list) === false)
			{
				if($debug_info){ 
					print("adding absolute link ".$src_href_url[$i]."\n"); 
				}
				$link_list[] = $src_href_url[$i];
			}else if($debug_info){ 
				if(array_search($src_href_url[$i], $link_list) === false){
					print("wrong domain: ".$src_href_url[$i]." ---- $base_url\n"); 
				}else{
					print("already added: ".$src_href_url[$i]." ----\n"); 
				}
			}
		}else{
			$tmp_str = $url_base;
			while(substr($src_href_url[$i], 0, 3) == "../"){
				$tmp_str = substr($tmp_str, 0, strrpos($tmp_str, "/"));
				$src_href_url[$i] = substr($src_href_url[$i], 3);	
			}
			$tmp_str = 
				$tmp_str . "/" . str_replace("./", "", $src_href_url[$i]);
			$tmp_str = substr($tmp_str, 0, strlen("http://")).
						str_replace("//", "/", substr($tmp_str, strlen("http://")) );

			if($link_list_s == 0 || 
				array_search($tmp_str, $link_list) === false){
				
				if($debug_info){ 
					print("adding relative link ".$tmp_str."\n"); 
				}
				$link_list[] = $tmp_str;
			}else{
				if($debug_info){ 
					print("already in list: ".$tmp_str."\n"); 
				}
			}
		}
	}
	
	//This block of code actually looks at the text of the page.  You will want to modify
	//this to collect what you're interested in.
	
	$ndollarsigns = substr_count($text, '$');
	if($ndollarsigns > 0 && $ndollarsigns < 10){
		$max_cost = 0;
		$pos = -1;
		while(($pos = strpos($text, '$', $pos+1)) !== false){
			$new_cost = (int)substr($text, $pos+1);
			if($new_cost > $max_cost){
				$max_cost = $new_cost;
				if($debug_info){
					print("found a price of $max_cost \n");
				}
			}
		}
		if($max_cost > 0){
			$costs[] = $max_cost;
			$summary_lengths[] = strlen($text);
		}
	}
	
	//Now the function will recursively walk through the new links that were found.
	if($depth > 1){
		$link_list_s2 = count($link_list);
		for($i = $link_list_s; $i < $link_list_s2; $i++){
			CrawlPage(	$base_url, $link_list[$i], $link_list, 
						$avoid_cgi, $depth-1, $debug_info, $costs, $summary_lengths);
		}
	}
}

$url_list = array();
$costs = array();
$summary_lengths = array();

CrawlPage("craigslist",  "http://www.craigslist.com/apa/", $url_list, 
				true, 3, true, $costs, $summary_lengths);

print("exited function --- \n");
for($i = 0; $i < count($url_list); $i++){
	print($url_list[$i]." ----\n");
}

print("cost,SummaryLength\n");
for($i = 0; $i < count($costs); $i++){
	print($costs[$i].",".$summary_lengths[$i]."\n");
}

?>