#!/usr/bin/perl -w
use strict;
$|++;

use GIFgraph::lines;
use Date::Calc qw(Today Days_in_Month Compress Add_Delta_YMD Month_to_Text);

## begin configuration

my @LANGS = sort qw(java perl|perl5 html cgi cobol fortran python tcl);

my (@LOWER) = (1995,3,1);       # beginning of useful data
my (@UPPER) = Add_Delta_YMD(Today,0,-1,0); # last month

use constant GIFOUT => "/home/merlyn/Html/x.gif";
use constant MEMORY => "/home/merlyn/.dejajobscache";

## end configuration

my @plotdata = ();

my (@from) = @LOWER;
{
  my $days = Days_in_Month(@from[0,1]);
  my (@to) = (@from[0,1],$days);
  last if Compress(@to) >= Compress(@UPPER);
  my $fromdate = to_dejadate(@from);
  my $todate = to_dejadate(@to);
  push @{$plotdata[0]}, sprintf("%04d %02d", @from);
  my $id = 0;
  for my $lang (@LANGS) {
    my $hits = count_hits($lang, $fromdate, $todate);
    if (defined $hits) {
      print "$fromdate $todate $lang $hits\n";
      $hits /= $days;
    }
    push @{$plotdata[++$id]}, $hits;
  }
  @from = Add_Delta_YMD(@from, 0, 1, 0);
  redo;
}

my $graph = GIFgraph::lines->new(640,480);
$graph->set(
            x_labels_vertical => 1,
            title => 'Keyword hits per day in misc.jobs.offered from Dejanews',
            dclrs => [qw(
                        lred lorange lyellow lgreen lblue lpurple
                        dred  orange dyellow dgreen dblue dpurple
                        )],
           );
$graph->set_legend(@LANGS);
$graph->plot_to_gif(GIFOUT, \@plotdata);

## subroutines

sub to_dejadate {
  my($y,$m,$d) = @_;
  join " ", Month_to_Text($m), $d, $y;
}

BEGIN {
  my %HIT_CACHE;
  my $SEPARATOR = "\001";

  dbmopen(%HIT_CACHE, MEMORY, 0666);

  sub count_hits {
    my $tag = join $SEPARATOR, @_;
    my $response;

    if ($response = $HIT_CACHE{$tag}) {
      (split $SEPARATOR, $response)[0];
    } elsif (defined ($response = count_hits_from_deja(@_))) {
      $HIT_CACHE{$tag} = join $SEPARATOR, $response, time;
      $response;
    } else {
      undef;
    }
  }
}

BEGIN {
  my $ua;
  my $uri;

  sub count_hits_from_deja {
    my ($query,$fromdate,$todate) = @_;
    
    unless ($ua) {
      require LWP::UserAgent;
      require URI;

      $ua = LWP::UserAgent->new;
      $uri = URI->new('http://www.dejanews.com/[ST_rn=ps]/dnquery.xp');
    }
    $uri->query_form(
                     ST => "PS",        # hidden
                     QRY => $query,
                     "groups" => "misc.jobs.offered",
                     "fromdate" => $fromdate,
                     "todate" => $todate,
                    );

    require HTTP::Request;
    my $req = HTTP::Request->new('GET',$uri);
    
    for ($ua->request($req)->as_string) {
      if (/Messages.*of exactly.*?(\d+)/) {
        return "$1";
      } elsif (/did not match any/) {
        return 0;
      } else {
        return undef;
      }
    }
  }
}