#!/usr/bin/perl
use strict;
use warnings;
use utf8;
use Encode;
use Unicode::Japanese;
use Unicode::Normalize;
use HTML::TreeBuilder;
use MeCab;
use HTML::TagCloud;
use URI::Escape;
binmode STDOUT, ':utf8';
my $count = 0;
print header();
opendir(my $dh, '.') or die;
foreach my $file (readdir($dh)) {
if ($file =~ /^\d{8}\.html$/) {
print STDERR "processing $file...\n";
parsefile($file, ++$count);
}
}
closedir($dh);
print footer($count);
sub parsefile
{
my $file = shift;
my $count = shift;
open(my $fh, $file) or die;
our $/ = undef;
my $data = <$fh>;
close($fh);
my $tree = new HTML::TreeBuilder;
$tree->parse(NFKC(Unicode::Japanese->new($data, 'auto')->getu));
my $text = $tree->look_down('id', 'content2')->as_text;
# 記号は空白にしちゃうお
$text =~ s/[\x00-\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]/ /g;
my %words;
my $mecab = MeCab::Tagger->new;
my $node = $mecab->parseToNode(encode('EUC-JP', $text));
while ($node = $node->{next}) {
my $midasi = decode('EUC-JP', $node->{surface});
my @hinsi = split ',', decode('EUC-JP', $node->{feature});
if ($hinsi[0] eq '名詞' && $hinsi[1] !~ /(非自立|代名詞|接尾)/) {
$words{$midasi}++;
}
}
my $cloud = HTML::TagCloud->new;
while (my ($word, $count) = each(%words)) {
$cloud->add($word, '#' . uri_escape(encode 'UTF-8', $word), $count);
}
my $html = $cloud->html(200);
$html =~ s/id="htmltagcloud"/class="htmltagcloud"/;
my $date = sprintf('%04d/%02d/%02d', ($file =~ /^(\d{4})(\d{2})(\d{2})/));
print <<"EOD";
$date
EOD
print $html;
print <<"EOD";
EOD
}
sub header
{
return <
横浜市長施政方針演説タグクラウド
横浜市長施政方針演説タグクラウド
Inspired by Japanese Prime Minister Speeches Tag Cloud.
EOD
}
sub footer
{
my $count = shift;
return <<"EOD";
EOD
}