#!/usr/bin/perl -w

use strict;

package main;

use Dumpvalue;
my $d= Dumpvalue->new();

#my $egroup= Egroups->new("wolbonet", "msgs");

#my $egroup->login("xxxxxxxxxxx", "xxxxxxxx");

$|=1;
$/=undef;
$d->dumpValue(Egroups::ParseExpandedList(<>));

exit(0);

package Egroups;

use LWP::Simple;
use LWP::UserAgent;
use HTTP::Request::Common qw(POST GET);
use LWP::Simple;
use URI;

sub new {
    my $class= shift;
    my $groupname= shift;
    my $messagedir= shift;

    my $useragent= LWP::UserAgent->new();
    $useragent->cookie_jar(HTTP::Cookies->new());
    $useragent->env_proxy();
    $useragent->agent("Mozilla/4.75 [en] (X11; U; Linux 2.2.17 i686)");

    return bless {
        ua => $useragent,
        url=> "http://groups.yahoo.com/group/$groupname/messages",
        group=> $groupname,
        messagedir=> $messagedir,
    }, $class;
}

sub CopyMessages {
    my $self= shift;

    my $messageNr= 1;
    while (1) {
        my $messageList= $self->ReadMessages($messageNr);

        last if (!keys %$messageList);

        $self->WriteMessages($messageList);

        $messageNr= 1 + max(keys %$messageList);
    }
}

sub max {
    my $max= shift;
    while (@_) {
        my $val= shift;
        $max= $val if  ($val>$max);
    }
    return $max;
}

sub WriteMessages {
    my $self= shift;
    my $list= shift;

    for my $msgnr (sort {$a<=>$b} keys %$list) {
        $self->WriteMessage($msgnr, $list->{$msgnr});
    }
}

sub WriteMessage {
    my $self= shift;
    my $msgnr= shift;
    my $message= shift;

    my $filename= $self->{messagedir}/$msgnr;
    open(FH, ">$filename") or die "error writing $filename : $@\n";
    print FH $message->{body};
    close(FH);

    for my $attachment (@{$message->{attachments}}) {
        $self->WriteAttachment($msgnr, $attachment);
    }
}

sub WriteAttachment {
    my $self= shift;
    my $msgnr= shift;
    my $attachmentnr= shift;
    my $attachment= shift;

    my $filename= $self->{messagedir}/$msgnr.$attachmentnr;
    open(FH, ">$filename") or die "error writing $filename : $@\n";
    print FH $attachment->{body};
    close(FH);
}

# returns a hash of 
# {
#   <msgid>=>{
#     header=>"header", body=>"message",
#     attachments=>[
#       {
#         body=>attachment, name=>name
#       }
#     ]
#   }
# }
sub ReadMessages {
    my $self= shift;
    my $nr= shift;

    my $request= GET "$self->{url}/$nr?expand=1";
    my $response= $self->{ua}->request($request);

    return ParseExpandedList($response->as_string);
}

#############################################################################

use HTML::TreeBuilder;

# returns dict of msg id => {from,date,subject,body,attachment => ... }
sub ParseExpandedList {
    my ($htmlpage)= @_;

    my $h = HTML::TreeBuilder->new();
    $h->parse($htmlpage);
    $h->eof();

    my $messagelist= {};
    my $messagetable= $h->address("0.1.6");
    for (my $tr_i=0 ; $tr_i< $messagetable->content_list() ; $tr_i+=2) {
        my $header= ParseExpandListHeader($messagetable->address(".$tr_i"));
        my $body= ParseExpandListBody($messagetable->address(".".($tr_i+1)));

        $messagelist->{$header->{msgid}}= { %$header, %$body };
    }

    return $messagelist;
}

# extract from, subject, date, msgid from header.
sub ParseExpandListHeader {
    my ($headertree)= @_;

    my $header= {};

    $header->{msgid} = $headertree->address(".0.0.0.0.0");

    $header->{from} = ParseEmailAddress($headertree->address(".1.0.0"));
    $header->{date} = ParseDate($headertree->address(".1.0.2"));
    $header->{subject} = ParseSubject($headertree->address(".1.0.4"));

    return $header;
}

sub ParseEmailAddress {
    my ($addrtree)= @_;

    return $addrtree->address(".1").$addrtree->address(".2.0").$addrtree->address(".3");
}
sub ParseDate {
    my ($datetree)= @_;

    return $datetree->address(".1");
}
sub ParseSubject {
    my ($subjtree)= @_;

    return $subjtree->address(".2.0");
}

# extract body, attachments from body.
sub ParseExpandListBody {
    my ($bodytree)= @_;

    my $body= {};
    $body->{body}= $bodytree->address(".1.0")->as_text();

    my $bodycell= $bodytree->address(".1");
    for (my $p_i=1 ; $p_i<$bodycell->content_list() ; $p_i++) {
        my $attachment= ParseExpandListAttachment($bodycell->address(".$p_i"));
        push(@{$body->{attachments}}, $attachment) if ($attachment);
    }

    return $body;
}
sub ParseExpandListAttachment {
    my ($atttree) = @_;

    my $attachment= {};

    return if $atttree->tag() ne "p";
    return if $atttree->content_list() != 1;
    return if $atttree->address(".0")->tag() ne "table";
    return if $atttree->address(".0")->content_list() < 1;
    return if $atttree->address(".0.0")->tag() ne "tr";
    return if $atttree->address(".0.0")->content_list() != 1;
    return if $atttree->address(".0.0.0")->tag() ne "td";
    return if $atttree->address(".0.0.0")->content_list() != 1;
    return if $atttree->address(".0.0.0.0")->tag() ne "a";
    return if $atttree->address(".0.0.0.0")->content_list() != 1;

    $attachment->{link}= $atttree->address(".0.0.0.0")->{href};
    $attachment->{name}= $atttree->address(".0.0.0.0.0");

    return $attachment;
}

#############################################################################

