#!/usr/bin/perl -w
use strict;
$|=1;
use HTTP::Request::Common qw(POST GET);
use LWP::Simple;
use LWP::UserAgent;
use IO::File;
use URI::Escape;

my $ua= LWP::UserAgent->new();
$ua->agent("Mozilla/4.75 [en] (X11; U; Linux 2.2.17 i686)");

my $googlehtml= QueryGoogle("site:forum.xda-developers.com Oct 2004");
my $topresult= ParseGoogle($googlehtml);

#PrintResult($result);

my %pages;

$pages{$_} ||= 0 for (@$topresult);

while (1)
{
    my @pages= grep { $pages{$_}==0 } keys %pages;
    last if (!@pages);

    for my $url (@pages) 
    {
        my $page= GetFromGoogleCache($url);

        my $subpages= FindSubPages($url, $page);

        $pages{$_} ||= 0 for (@$subpages);

        $pages{$url}++;
    }
}
sub savefile {
    my $fn= shift;
    my $data= shift;
    my $fh= IO::File->new($fn, "w+") or die "$fn: $!\n";
    binmode $fh;
    $fh->write($data);
    $fh->close();
}
sub filename_escape {
    my $fn= shift;

    $fn =~ s/-/--/g;
    $fn =~ s/[^a-z0-9.A-Z]/sprintf("-%02x", ord($&))/ge;

    return $fn;
}
sub encodeurlasfile {
    my ($url)= @_;

    $url =~ s/^\///;

    return "cache/".join("_", map { filename_escape($_) } split /[?&=]/, $url);
}

sub readfile {
    my $fn= shift;
    my $fh= IO::File->new($fn, "r") or die "$fn: $!\n";
    binmode $fh;
    my $data;
    $fh->read($data, -s $fh);
    $fh->close();

    return $data;
}

sub GetFromGoogleCache {
    my $url= shift;

    my $cachename= encodeurlasfile($url);
    if (-e $cachename) {
        return readfile($cachename);
    }

    $url =~ s{http://}{};

    my $pagerq= GET "http://www.google.com/search?q=cache:".uri_escape($url);
    my $rp= $ua->request($pagerq);

    printf("%9d %s\n", length($rp->content), $url);

    savefile($cachename, $rp->content);

    return $rp->content;
}
sub FindSubPages {
    my ($url, $content)= @_;

    my $baseurl= $url;
    $baseurl =~ s{/[^/]+$}{};

    my @subpages;
    if ($content =~ /<span class="nav">Goto page(.*?)<\/span>/) {
        my $links= $1;
        $links =~ s/&amp;/&/g;
        for my $link ($links =~ /<a href="(.*?)">/g) {
            push @subpages, "$baseurl/$link";
        }
    }
    return \@subpages;
}
exit(0);

sub uniq {
    my ($list)= @_;
    my %ref;
    $ref{$_}++ for (@$list);
    return [keys %ref];
}

sub QueryGoogle {
    my $query= shift;

    my $result= "";
    my $start= 0;
    while (1)
    {
        my $pagerq= GET "http://www.google.com/search?q=".uri_escape($query)."&num=100&safe=off&hl=en&filter=0".($start?"&start=$start":"");
        my $rp= $ua->request($pagerq);

        $result .= $rp->content;

        if ($rp->content !~ /<a href[^>]*start=(\d+)[^>]*><img src=.nav_next/) {
            last;
        }
        $start += 100;
    }
    return $result;
}
sub GetQuery {
    my ($parts, $qid)= @_;

    my $query="";

    for my $part (@$parts) {
        my $n_choices= scalar @$part;
        my $choice= $qid%$n_choices;
        $qid= int($qid/$n_choices);
        $query .= $part->[$choice];
    }
    return $query;
}

sub getTotalChoices {
    my ($parts)= @_;
    my $total= 1;
    for my $part (@$parts) {
        my $n_choices= scalar @$part;
        $total *= $n_choices;
    }
    return $total;
}

sub ParseGoogle {
    my ($html)= @_;

    my @result;
    my @parts= split(/<p>/, $html);
    for my $part (@parts) {
        next if ($part !~ /href/);
        next if ($part =~ /Did you mean(?: to search for)?:.*\/search/);
        next if ($part =~ /Category:.*href=.*directory.google.com/);
        next if ($part =~ /Result.*?Search took.*?second/i);
        return [] if ($part =~ /<br>Your search - <b>.*?<\/b> - did not match any documents/i);
        my $url;
        if (($url)= ($part =~ /<a href=["']?(.*?)["']?>/i)) {
#                  && $url && ($url !~ /google\.com/) ) {

            if ($url !~ /google\.com/ && $url =~ /^\w/) {
                push @result, $url;
            }
        }
    }
    return \@result;
}
sub PrintResult {
    my ($results) = @_;
    for my $url (@$results) {
        print "$url\n";
    }
}

