Andrew Lewman
update the mirrors page, have the script update the csv list with last status.
Andrew Lewman commited 3dceeeffc at 2013-12-06 16:54:04
update-mirrors.pl
#!/usr/bin/perl -w
use warnings;
use strict;
use Data::Dumper;
use LWP::Simple;
use HTML::LinkExtor;
use LWP;
use Date::Parse;
use Date::Format;
use Digest::SHA qw(sha256_hex);
# This is Free Software (GPLv3)
# http://www.gnu.org/licenses/gpl-3.0.txt
print "Creating LWP agent ($LWP::VERSION)...\n";
my $lua = LWP::UserAgent->new(
keep_alive => 1,
timeout => 30,
agent => "Tor MirrorCheck Agent"
);
sub sanitize {
my $taintedData = shift;
my $cleanedData;
my $whitelist = '-a-zA-Z0-9: +';
# clean the data, return cleaned data
$taintedData =~ s/[^$whitelist]//go;
$cleanedData = $taintedData;
return $cleanedData;
}
sub ExtractLinks {
my $content = shift;
my $url = shift;
my @links;
my $parser = HTML::LinkExtor->new(undef, $url);
$parser->parse($content);
foreach my $linkarray($parser->links)
{
my ($elt_type, $attr_name, $attr_value) = @$linkarray;
if ($elt_type eq 'a' && $attr_name eq 'href' && $attr_value =~ /\/$/ && $attr_value =~ /^$url/)
{
push @links, Fetch($attr_value, \&ExtractLinks);
}
elsif ($attr_value =~ /\.(xpi|dmg|exe|tar\.gz)$/)
#elsif ($attr_value =~ /\.(asc)$/) # small pgp files easier to test with
{
push @links, $attr_value;
}
 
 
הההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההה
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX