#!/usr/bin/env perl
 
#######################################################################
## lyle@digitalfoo.net    http://digitalfoo.net
##
## ABOUT
##   This program basically crawls the Web, endlessly looking for more
##   hosts to web scrape for even more hosts. It can easily be 
##   extended with more code to actually make use the hosts or even 
##   the HTTP dump... just imaging something neat and make it!
##
## LICENSE
##   "THE BEER-WARE LICENSE" (Revision 42):
##   lyle@digitalfoo.net wrote this file. As long as you retain this 
##   notice you can do whatever you want with this stuff. If we meet
##   some day, and you think this stuff is worth it, you can buy me a
##   beer in return.  Lyle Scott
##
## FEEDBACK
##   If you have a bug, comment, or suggestion for a feature, contact
##   me via my websites or the contact posted above.
##
#######################################################################
 
use strict;
use warnings;
 
use IO::Socket::INET;
 
### harvest the web by doing http dumps on port 80 
### and scrape hostnames prefixed with http:// to 
### gain more hosts to crawl.
### [0] = initial host to scrape
sub harvest ($)
{
    my $start_host = shift;
    my $extracted_hosts;
    my @queue;
    my @crawled_hosts;
    my $host;
    my $h;
    
    my $iteration_count = 0;
    my $found_count = 0;
    
 
    # do initial dump of start host
    my $dump = http_dump($start_host); 
    
    # get a few links to start crawling
    if (length($dump)) {
        $extracted_hosts = extract_http_hosts($dump);
 
        while ($h = shift(@$extracted_hosts)) {
            next if grep(/$h/, @queue);
            push(@queue, $h);
        }
        
        print "nfound ", scalar(@queue), " hosts on the starting host ($start_host)!nn";
        push(@crawled_hosts, $start_host);
    }
    else {
        print "ERROR, http_dump($start_host)n";
        exit;
    }
 
 
    # while we have hosts to crawl
    while ($host = shift(@queue)) {
 
        # total count of loops... good for stats 
        $iteration_count++;
        
        push (@crawled_hosts, $host);
        
        # print some statistics every 30 hosts
        if (($iteration_count % 30) == 0) {
            print "nSTATS:> iterations: $iteration_count     found: $found_count    queue: " .scalar(@queue). "nn";
        }   
        
        # get HTTP dump of port 80 on the host
        $dump = http_dump($host);
 
        # if we did not get output, skip it
        next unless length($dump);
        
        # extract http://(hostnames) from HTTP dump on port 80
        $extracted_hosts = extract_http_hosts($dump);
 
        # put what we find on queue for later processing
        my $c = 0;
        my $bail = 0;
 
        while ($h = shift(@$extracted_hosts)) {
            next if grep(/$h/, @crawled_hosts);
            next if grep(/$h/, @queue); 
            push(@queue, $h);
            $c++;
        }
 
        printf ("added %4d new hosts to crawl off of  %sn", $c, $host);
        $found_count++;
    }
    
    # TODO: change to query google for more links
    die "nn... all out of hosts!nn";
}
 
 
 
 
### Utility function to trim white spaces off either end
### [0] string to trim
sub trim ($) 
{
    my $str = $_[0];
    $str =~ s/^s+//;
    $str =~ s/s+$/0 (X11; U; FreeBSD i386; en-US; rv:1.9.0.10) Gecko0.10$EOL" .
                     "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8$EOL" .
                     "Connection: close$EOF";
 
        # send the header             
        print $sock $header;
        
        # read in all the result lines into one variable
        my $http_req_results;
 
        {
            # slurp up all lines in the output;
            local $/ = undef; 
        
            # assign entire output to variable
            $http_req_results = <$sock>;
        }
        
        # return entire http dump of page source
        return $http_req_results;
    }
    else {
        warn "WARN:> http_dump: Cant connect to $_[0] : $@n";       
        warn "WARN:> Skipping it and moving on...n"; 
        return -1;  
    }
}
 
 
 
#
# make sure to start off with somewhere with a lot of links
#
if ($#ARGV == 0) {
    harvest($ARGV[0]);
}
else {
    harvest("slashdot.org");
}