tor-webwml.git

1) #!/usr/bin/perl -w
2) use warnings;
3) use strict;
4) use LWP::Simple;
5) use LWP;
6) use Date::Parse;

7) use Date::Format;

8) 

9) #
10) # A quick hack by Jacob Appelbaum <jacob@appelbaum.net>
11) # LWP suggestions by Leigh Honeywell 
12) # This is Free Software (GPLv3)
13) # http://www.gnu.org/licenses/gpl-3.0.txt
14) #
15) 

16) print "Creating LWP agent ($LWP::VERSION)...\n";
17) my $lua = LWP::UserAgent->new(
18)     keep_alive => 1,
19)     timeout => 15, 
20)     agent => "Tor MirrorCheck Agent"
21) );
22) 
23) sub sanitize {
24)     my $taintedData = shift;
25)     my $cleanedData;
26)     my $whitelist = '-a-zA-Z0-9: +';
27) 
28)     # clean the data, return cleaned data
29)     $taintedData =~ s/[^$whitelist]//go;
30)     $cleanedData = $taintedData;
31) 
32)     return $cleanedData;
33) }
34) 
35) sub FetchDate {
36)     my $url = shift; # Base url for mirror
37)     my $trace = "project/trace/www.torproject.org"; # Location of recent update info
38)     $url = "$url$trace";
39) 
40)     print "Fetching possible date from: $url\n";
41) 
42)     my $request = new HTTP::Request GET => "$url";
43)     my $result = $lua->request($request);

44)     my $code = $result->code();
45)     print "Result code $code\n";

46) 

47)     if ($result->is_success && $code eq "200"){

48)        my $taint = $result->content;
49)        my $content = sanitize($taint);
50)        if ($content) {
51)             
52)             my $date = str2time($content);
53) 
54)             if ($date) {
55)                 print "We've fetched a date $date.\n";
56)                 return $date;
57)             } else {
58)                 print "We've haven't fetched a date.\n";
59)                 return "Unknown";
60)             }
61) 
62)         } else {
63)             print "Unable to fetch date, empty content returned.\n";
64)             return "Unknown";
65)         }
66)             
67)     } else {
68)        print "Our request failed, we had no result.\n";
69)        return "Unknown";
70)     }

71) 
72)     return "Unknown";

73) }
74) 
75) # This is the list of all known Tor mirrors
76) # Add new mirrors to the bottom!
77) my %m = ( 
78)        mirror000 => {
79)             orgName => "cypherpunks.at",
80)             isoCC => "AT",
81)             subRegion => "",
82)             region => "Europe",
83)             ipv4 => "True",
84)             ipv6 => "False",
85)             loadBalanced => "Unknown",
86)             httpWebsiteMirror => "http://tor.cypherpunks.at/",
87)             rsyncWebsiteMirror => "rsync://tor.cypherpunks.at/tor",
88)             httpDistMirror => "http://tor.cypherpunks.at/dist/",
89)             rsyncDistMirror => "rsync: tor.cypherpunks.at::tor/dist/",

90)             updateDate => "Unknown",

91)         },
92) 
93)        mirror001 => {
94)             orgName => "depthstrike.com",
95)             isoCC => "CA",
96)             subRegion => "NS",
97)             region => "North America",
98)             ipv4 => "True",
99)             ipv6 => "False",
100)             loadBalanced => "Unknown",
101)             httpWebsiteMirror => "http://tor.depthstrike.com/",
102)             ftpWebsiteMirror => "",
103)             rsyncWebsiteMirror => "",
104)             httpDistMirror => "http://tor.depthstrike.com/dist/",
105)             rsyncDistMirror => "",

106)             updateDate => "Unknown",

107)         },
108) 
109)        mirror002 => {
110)             orgName => "hermetix.org",
111)             isoCC => "CA",
112)             subRegion => "QC",
113)             region => "North America",
114)             ipv4 => "True",
115)             ipv6 => "False",
116)             loadBalanced => "Unknown",
117)             httpWebsiteMirror => "http://tor.hermetix.org/",
118)             rsyncWebsiteMirror => "",
119)             httpDistMirror => "http://tor.hermetix.org/dist/",
120)             rsyncDistMirror => "",

121)             updateDate => "Unknown",

122)         },
123) 
124)        mirror003 => {
125)             orgName => "Boinc.ch",
126)             isoCC => "CH",
127)             subRegion => "",
128)             region => "Europe",
129)             ipv4 => "True",
130)             ipv6 => "False",
131)             loadBalanced => "Unknown",
132)             httpWebsiteMirror => "http://tor.boinc.ch/",
133)             ftpWebsiteMirror => "",
134)             rsyncWebsiteMirror => "",
135)             httpDistMirror => "http://tor.boinc.ch/dist/",
136)             rsyncDistMirror => "",

137)             updateDate => "Unknown",

138)         },
139) 
140)        mirror004 => {
141)             orgName => "anonymity.cn",
142)             isoCC => "CN",
143)             subRegion => "",
144)             region => "Asia",
145)             ipv4 => "True",
146)             ipv6 => "False",
147)             loadBalanced => "Unknown",
148)             httpWebsiteMirror => "http://tor.anonymity.cn/",
149)             ftpWebsiteMirror => "",
150)             rsyncWebsiteMirror => "",
151)             httpDistMirror => "http://tor.anonymity.cn/dist/",
152)             rsyncDistMirror => "",

153)             updateDate => "Unknown",

154)         },
155) 
156)        mirror004 => {
157)             orgName => "bbs",
158)             isoCC => "DE",
159)             subRegion => "",
160)             region => "Europe",
161)             ipv4 => "True",
162)             ipv6 => "False",
163)             loadBalanced => "Unknown",
164)             httpWebsiteMirror => "http://tor.blingblingsquad.net/",
165)             ftpWebsiteMirror => "",
166)             rsyncWebsiteMirror => "",
167)             httpDistMirror => "http://tor.blingblingsquad.net/dist/",
168)             rsyncDistMirror => "",

169)             updateDate => "Unknown",

170)         },
171) 
172)        mirror005 => {
173)             orgName => "Berapla",
174)             isoCC => "DE",
175)             subRegion => "",
176)             region => "Europe",
177)             ipv4 => "True",
178)             ipv6 => "False",
179)             loadBalanced => "Unknown",
180)             httpWebsiteMirror => "http://download.berapla.de/mirrors/tor/",
181)             ftpWebsiteMirror => "",
182)             rsyncWebsiteMirror => "",
183)             httpDistMirror => "",
184)             rsyncDistMirror => "",

185)             updateDate => "Unknown",

186)         },
187) 
188)        mirror006 => {
189)             orgName => "cybermirror",
190)             isoCC => "DE",
191)             subRegion => "",
192)             region => "Europe",
193)             ipv4 => "True",
194)             ipv6 => "False",
195)             loadBalanced => "Unknown",
196)             httpWebsiteMirror => "http://tor.cybermirror.org/",
197)             ftpWebsiteMirror => "",
198)             rsyncWebsiteMirror => "",
199)             httpDistMirror => "http://tor.cybermirror.org/dist/",
200)             rsyncDistMirror => "",

201)             updateDate => "Unknown",

202)         },
203) 
204)        mirror007 => {
205)             orgName => "Spline",
206)             isoCC => "DE",
207)             subRegion => "FU",
208)             region => "Europe",
209)             ipv4 => "True",
210)             ipv6 => "False",
211)             loadBalanced => "Unknown",
212)             httpWebsiteMirror => "http://rem.spline.de/tor/",
213)             ftpWebsiteMirror => "",
214)             rsyncWebsiteMirror => "",
215)             httpDistMirror => "",
216)             rsyncDistMirror => "",

217)             updateDate => "Unknown",

218)         },
219) 
220)        mirror008 => {
221)             orgName => "mirror.bsdhost.eu",
222)             isoCC => "DE",
223)             subRegion => "",
224)             region => "Europe",
225)             ipv4 => "True",
226)             ipv6 => "False",
227)             loadBalanced => "Unknown",
228)             httpWebsiteMirror => "http://mirror.bsdhost.eu/www.torproject.org/",
229)             ftpWebsiteMirror => "",
230)             rsyncWebsiteMirror => "",
231)             httpDistMirror => "http://mirror.bsdhost.eu/www.torproject.org/dist/",
232)             rsyncDistMirror => "",

233)             updateDate => "Unknown",

234)         },
235) 
236)        mirror009 => {
237)             orgName => "onionland",
238)             isoCC => "DE",
239)             subRegion => "",
240)             region => "Europe",
241)             ipv4 => "True",
242)             ipv6 => "False",
243)             loadBalanced => "Unknown",
244)             httpWebsiteMirror => "http://mirror.onionland.org/",
245)             ftpWebsiteMirror => "",
246)             rsyncWebsiteMirror => "rsync: mirror.onionland.org::tor/",
247)             httpDistMirror => "http://mirror.onionland.org/dist/",
248)             rsyncDistMirror => "rsync: mirror.onionland.org::tor/dist/",

249)             updateDate => "Unknown",

250)         },
251) 
252)        mirror010 => {
253)             orgName => "plentyfact",
254)             isoCC => "DE",
255)             subRegion => "",
256)             region => "Europe",
257)             ipv4 => "True",
258)             ipv6 => "False",
259)             loadBalanced => "Unknown",
260)             httpWebsiteMirror => "http://tor.plentyfact.net/",
261)             ftpWebsiteMirror => "",
262)             httpsWebsiteMirror => "https://tor.plentyfact.net/",
263)             rsyncWebsiteMirror => "",
264)             httpDistMirror => "http://tor.plentyfact.net/dist/",
265)             rsyncDistMirror => "",

266)             updateDate => "Unknown",

267)         },
268) 
269)        mirror011 => {
270)             orgName => "loxal.net",
271)             isoCC => "DE",
272)             subRegion => "",
273)             region => "Europe",
274)             ipv4 => "True",
275)             ipv6 => "False",
276)             loadBalanced => "Unknown",
277)             httpWebsiteMirror => "http://tor-anonymizer.mirror.loxal.net/",
278)             ftpWebsiteMirror => "",
279)             rsyncWebsiteMirror => "",
280)             httpDistMirror => "http://tor-anonymizer.mirror.loxal.net/dist/",
281)             rsyncDistMirror => "",

282)             updateDate => "Unknown",

283)         },
284) 
285)        mirror012 => {
286)             orgName => "centervenus.com",
287)             isoCC => "DE",
288)             subRegion => "",
289)             region => "Europe",
290)             ipv4 => "True",
291)             ipv6 => "False",
292)             loadBalanced => "Unknown",
293)             httpWebsiteMirror => "",
294)             ftpWebsiteMirror => "",
295)             rsyncWebsiteMirror => "",
296)             httpDistMirror => "http://www.centervenus.com/mirrors/tor/dist/",
297)             rsyncDistMirror => "",

298)             updateDate => "Unknown",

299)         },
300) 
301)        mirror013 => {
302)             orgName => "zdg-gmbh.eu",
303)             isoCC => "DK",
304)             subRegion => "",
305)             region => "Europe",
306)             ipv4 => "True",
307)             ipv6 => "False",
308)             loadBalanced => "Unknown",
309)             httpWebsiteMirror => "http://tor.zdg-gmbh.eu/", 
310)             ftpWebsiteMirror => "",
311)             rsyncWebsiteMirror => "", 
312)             httpDistMirror => "http://tor.zdg-gmbh.eu/dist/",
313)             rsyncDistMirror => "",

314)             updateDate => "Unknown",

315)         },
316) 
317)        mirror014 => {
318)             orgName => "CRAN",
319)             isoCC => "FR",
320)             subRegion => "",
321)             region => "Europe",
322)             ipv4 => "True",
323)             ipv6 => "False",
324)             loadBalanced => "Unknown",
325)             httpWebsiteMirror => "http://tor.miroir-francais.fr/",
326)             rsyncWebsiteMirror => "rsync: miroir-francais.fr::tor", 
327)             ftpWebsiteMirror => "ftp://miroir-francais.fr/pub/tor/",
328)             httpDistMirror => "",
329)             rsyncDistMirror => "",

330)             updateDate => "Unknown",

331)         },
332) 
333)        mirror015 => {
334)             orgName => "tor.newworldorder.com.es",
335)             isoCC => "HU",
336)             subRegion => "",
337)             region => "Europe",
338)             ipv4 => "True",
339)             ipv6 => "False",
340)             loadBalanced => "Unknown",
341)             httpWebsiteMirror => "http://tor.newworldorder.com.es/",
342)             rsyncWebsiteMirror => "", 
343)             ftpWebsiteMirror => "",
344)             httpDistMirror => "",
345)             rsyncDistMirror => "",

346)             updateDate => "Unknown",

347)         },
348) 
349)        mirror016 => {
350)             orgName => "amorphis.eu",
351)             isoCC => "NL",
352)             subRegion => "",
353)             region => "Europe",
354)             ipv4 => "True",
355)             ipv6 => "False",
356)             loadBalanced => "Unknown",
357)             httpWebsiteMirror => "http://tor.amorphis.eu/",
358)             rsyncWebsiteMirror => "", 
359)             ftpWebsiteMirror => "",
360)             httpDistMirror => "http://tor.amorphis.eu/dist/",
361)             rsyncDistMirror => "",

362)             updateDate => "Unknown",

363)         },
364) 
365)        mirror017 => {
366)             orgName => "BIT BV",
367)             isoCC => "NL",
368)             subRegion => "",
369)             region => "Europe",
370)             ipv4 => "True",
371)             ipv6 => "False",
372)             loadBalanced => "Unknown",
373)             httpWebsiteMirror => "http://ftp.bit.nl/mirror/tor/",
374)             rsyncWebsiteMirror => "", 
375)             ftpWebsiteMirror => "ftp://ftp.bit.nl/mirror/tor/",
376)             httpDistMirror => "",
377)             rsyncDistMirror => "",

378)             updateDate => "Unknown",

379)         },
380) 
381)        mirror018 => {
382)             orgName => "CCC",
383)             isoCC => "NL",
384)             subRegion => "",
385)             region => "Europe",
386)             ipv4 => "True",
387)             ipv6 => "False",
388)             loadBalanced => "Unknown",
389)             httpWebsiteMirror => "http://tor.ccc.de/",
390)             rsyncWebsiteMirror => "", 
391)             ftpWebsiteMirror => "",
392)             httpDistMirror => "http://tor.ccc.de/dist/",
393)             rsyncDistMirror => "",

394)             updateDate => "Unknown",

395)         },
396) 
397)        mirror018 => {
398)             orgName => "kamagurka.org",
399)             isoCC => "NL",
400)             subRegion => "Haarlem",
401)             region => "Europe",
402)             ipv4 => "True",
403)             ipv6 => "False",
404)             loadBalanced => "Unknown",
405)             httpWebsiteMirror => "http://tor.kamagurka.org/",
406)             rsyncWebsiteMirror => "", 
407)             ftpWebsiteMirror => "",
408)             httpDistMirror => "http://tor.kamagurka.org/dist/",
409)             rsyncDistMirror => "",

410)             updateDate => "Unknown",

411)         },
412) 
413)        mirror019 => {
414)             orgName => "OS Mirror",
415)             isoCC => "NL",
416)             subRegion => "",
417)             region => "Europe",
418)             ipv4 => "True",
419)             ipv6 => "False",
420)             loadBalanced => "Unknown",
421)             httpWebsiteMirror => "http://tor.osmirror.nl/",
422)             rsyncWebsiteMirror => "rsync: rsync.osmirror.nl::tor/", 
423)             ftpWebsiteMirror => "ftp://ftp.osmirror.nl/pub/tor/",
424)             httpDistMirror => "",
425)             rsyncDistMirror => "",

426)             updateDate => "Unknown",

427)         },
428) 
429) 
430)        mirror020 => {
431)             orgName => "Meulie.net",
432)             isoCC => "NO",
433)             subRegion => "",
434)             region => "Europe",
435)             ipv4 => "True",
436)             ipv6 => "False",
437)             loadBalanced => "Unknown",
438)             httpWebsiteMirror => "http://tor.meulie.net/",
439)             rsyncWebsiteMirror => "", 
440)             ftpWebsiteMirror => "",
441)             httpDistMirror => "",
442)             rsyncDistMirror => "",

443)             updateDate => "Unknown",

444)         },
445) 
446)        mirror021 => {
447)             orgName => "Swedish Linux Society",
448)             isoCC => "SE",
449)             subRegion => "",
450)             region => "Europe",
451)             ipv4 => "True",
452)             ipv6 => "False",
453)             loadBalanced => "Unknown",
454)             httpWebsiteMirror => "http://ftp.se.linux.org/crypto/tor/",
455)             rsyncWebsiteMirror => "", 
456)             ftpWebsiteMirror => "ftp://ftp.se.linux.org/pub/crypto/tor/",
457)             httpDistMirror => "",
458)             rsyncDistMirror => "",

459)             updateDate => "Unknown",

460)         },
461) 
462)        mirror022 => {
463)             orgName => "Ghirai.com",
464)             isoCC => "UK",
465)             subRegion => "London",
466)             region => "Europe",
467)             ipv4 => "True",
468)             ipv6 => "False",
469)             loadBalanced => "Unknown",
470)             httpWebsiteMirror => "http://www.ghirai.com/tor/",
471)             rsyncWebsiteMirror => "", 
472)             ftpWebsiteMirror => "",
473)             httpDistMirror => "",
474)             rsyncDistMirror => "",

475)             updateDate => "Unknown",

476)         },
477) 
478)        mirror023 => {
479)             orgName => "BJWOnline.com",
480)             isoCC => "US",
481)             subRegion => "California",
482)             region => "North America",
483)             ipv4 => "True",
484)             ipv6 => "False",
485)             loadBalanced => "Unknown",
486)             httpWebsiteMirror => "http://mirror.bjwonline.com/tor/",
487)             rsyncWebsiteMirror => "", 
488)             ftpWebsiteMirror => "",
489)             httpDistMirror => "",
490)             rsyncDistMirror => "",

491)             updateDate => "Unknown",

492)         },
493) 
494)        mirror024 => {
495)             orgName => "Libertarian Action Network",

496)             isoCC => "US",

497)             subRegion => "",
498)             region => "North America",
499)             ipv4 => "True",
500)             ipv6 => "False",
501)             loadBalanced => "Unknown",
502)             httpWebsiteMirror => "",
503)             rsyncWebsiteMirror => "", 
504)             ftpWebsiteMirror => "ftp://libertarianactivism.com/tor.eff.org/dist/",
505)             httpDistMirror => "",
506)             rsyncDistMirror => "",

507)             updateDate => "Unknown",

508)         },
509) 
510)        mirror025 => {
511)             orgName => "TheOnionRouter.com",
512)             isoCC => "US",
513)             subRegion => "Texas",
514)             region => "North America",
515)             ipv4 => "True",
516)             ipv6 => "False",
517)             loadBalanced => "Unknown",
518)             httpWebsiteMirror => "http://www.theonionrouter.com/",
519)             rsyncWebsiteMirror => "", 
520)             ftpWebsiteMirror => "",
521)             httpDistMirror => "http://www.theonionrouter.com/dist/",
522)             rsyncDistMirror => "",

523)             updateDate => "Unknown",

524)         },
525) 
526)        mirror026 => {
527)             orgName => "Site2nd.org",
528)             isoCC => "USA",
529)             subRegion => "Texas",
530)             region => "North America",
531)             ipv4 => "True",
532)             ipv6 => "False",
533)             loadBalanced => "Unknown",
534)             httpWebsiteMirror => "http://tor.site2nd.org",
535)             rsyncWebsiteMirror => "", 
536)             ftpWebsiteMirror => "",
537)             httpDistMirror => "",
538)             rsyncDistMirror => "",

539)             updateDate => "Unknown",

540)         },
541) 
542)        mirror027 => {
543)             adminContact => "jeroen\@unfix.org",
544)             orgName => "unfix",
545)             isoCC => "CH",
546)             subRegion => "",
547)             region => "Europe",
548)             ipv4 => "True",
549)             ipv6 => "True",
550)             loadBalanced => "Unknown",
551)             httpWebsiteMirror => "http://tor.unfix.org/",
552)             rsyncWebsiteMirror => "", 
553)             ftpWebsiteMirror => "",
554)             httpDistMirror => "",
555)             rsyncDistMirror => "",

556)             updateDate => "Unknown",

557)         },
558) 
559)        mirror028 => {
560)             adminContact => "jeroen\@unfix.org",
561)             orgName => "sixx",

562)             isoCC => "CH",

563)             subRegion => "",
564)             region => "Europe",
565)             ipv4 => "True",
566)             ipv6 => "True",
567)             loadBalanced => "Unknown",
568)             httpWebsiteMirror => "http://tor.sixxs.net/",
569)             rsyncWebsiteMirror => "", 
570)             ftpWebsiteMirror => "",
571)             httpDistMirror => "",
572)             rsyncDistMirror => "",

573)             updateDate => "Unknown",

574)         },
575) );
576) 
577) my $count = values %m;
578) print "We have a total of $count mirrors\n";
579) print "Fetching the last updated date for each mirror.\n";
580) 
581) foreach my $server ( keys %m ) {
582) 
583)     print "Attempting to fetch from $m{$server}{'orgName'}\n";
584) 
585)     if ($m{$server}{'httpWebsiteMirror'}) {
586)         print "Attempt to fetch via HTTP.\n";
587)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpWebsiteMirror'}");
588)     } elsif ($m{$server}{'httpsWebsiteMirror'}) {
589)         print "Attempt to fetch via HTTPS.\n";
590)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpsWebsiteMirror'}");
591)     } elsif ($m{$server}{'ftpWebsiteMirror'}) {
592)         print "Attempt to fetch via FTP.\n";
593)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'ftpWebsiteMirror'}");
594)     } else {
595)         print "We were unable to fetch or store anything. We still have the following: $m{$server}{'updateDate'}\n";
596)     }
597) 
598)     print "We fetched and stored the following: $m{$server}{'updateDate'}\n";
599) 
600)  }
601) 
602) 
603) print "We sorted the following mirrors by their date of last update: \n";
604) foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
605) 
606)      print "\n";
607)      print "Mirror $m{$server}{'orgName'}: \n";
608) 
609)      foreach my $attrib ( sort keys %{$m{$server}} ) {
610)         print "$attrib = $m{$server}{$attrib}";
611)         print "\n";
612)      };
613) }
614) 
615) my $outFile = "include/mirrors-table.wmi";
616) my $html;
617) open(OUT, "> $outFile") or die "Can't open $outFile: $!";
618) 
619) # Here's where we open a file and print some wml include goodness 
620) # This is storted from last known recent update to unknown update times
621) foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
622) 

623)      my $time;
624)      if( "$m{$server}{'updateDate'}" ne "Unknown") {
625)         $time = ctime($m{$server}{'updateDate'});
626)         chomp($time);
627)      } else { $time = "Unknown"; }
628) 

629) print OUT <<"END";
630)      \n<tr>\n
631)          <td>$m{$server}{'isoCC'}</td>\n
632)          <td>$m{$server}{'orgName'}</td>\n
633)          <td>$time</td>\n
634) END
635) 
636)      my %prettyNames = (
637)                         httpWebsiteMirror => "http",
638)                         httpsWebsiteMirror => "https",
639)                         ftpWebsiteMirror => "ftp",
640)                         rsyncWebsiteMirror => "rsync",
641)                         httpDistMirror => "http",
642)                         httpsDistMirror => "https",
643)                         rsyncDistMirrors => "rsync", );
644) 
645)      foreach my $precious ( sort keys %prettyNames )

646)      {
647)         if ($m{$server}{"$precious"}) {
648)             print OUT "    <td><a href=\"" . $m{$server}{$precious} . "\">" .

649)                       "$prettyNames{$precious}</a></td>\n";