tor-webwml.git

1) #!/usr/bin/perl -w
2) use warnings;
3) use strict;
4) use LWP::Simple;
5) use LWP;
6) use Date::Parse;

7) use Date::Format;

8) 

9) #
10) # A quick hack by Jacob Appelbaum <jacob@appelbaum.net>
11) # LWP suggestions by Leigh Honeywell 
12) # This is Free Software (GPLv3)
13) # http://www.gnu.org/licenses/gpl-3.0.txt
14) #
15) 

16) print "Creating LWP agent ($LWP::VERSION)...\n";
17) my $lua = LWP::UserAgent->new(
18)     keep_alive => 1,
19)     timeout => 15, 
20)     agent => "Tor MirrorCheck Agent"
21) );
22) 
23) sub sanitize {
24)     my $taintedData = shift;
25)     my $cleanedData;
26)     my $whitelist = '-a-zA-Z0-9: +';
27) 
28)     # clean the data, return cleaned data
29)     $taintedData =~ s/[^$whitelist]//go;
30)     $cleanedData = $taintedData;
31) 
32)     return $cleanedData;
33) }
34) 
35) sub FetchDate {
36)     my $url = shift; # Base url for mirror
37)     my $trace = "project/trace/www.torproject.org"; # Location of recent update info
38)     $url = "$url$trace";
39) 
40)     print "Fetching possible date from: $url\n";
41) 
42)     my $request = new HTTP::Request GET => "$url";
43)     my $result = $lua->request($request);
44) 
45)     if ($result->is_success){
46)        my $taint = $result->content;
47)        my $content = sanitize($taint);
48)        if ($content) {
49)             
50)             my $date = str2time($content);
51) 
52)             if ($date) {
53)                 print "We've fetched a date $date.\n";
54)                 return $date;
55)             } else {
56)                 print "We've haven't fetched a date.\n";
57)                 return "Unknown";
58)             }
59) 
60)         } else {
61)             print "Unable to fetch date, empty content returned.\n";
62)             return "Unknown";
63)         }
64)             
65)     } else {
66)        print "Our request failed, we had no result.\n";
67)        return "Unknown";
68)     }
69) }
70) 
71) # This is the list of all known Tor mirrors
72) # Add new mirrors to the bottom!
73) my %m = ( 
74)        mirror000 => {
75)             orgName => "cypherpunks.at",
76)             isoCC => "AT",
77)             subRegion => "",
78)             region => "Europe",
79)             ipv4 => "True",
80)             ipv6 => "False",
81)             loadBalanced => "Unknown",
82)             httpWebsiteMirror => "http://tor.cypherpunks.at/",
83)             rsyncWebsiteMirror => "rsync://tor.cypherpunks.at/tor",
84)             httpDistMirror => "http://tor.cypherpunks.at/dist/",
85)             rsyncDistMirror => "rsync: tor.cypherpunks.at::tor/dist/",
86)             updateDate => "",
87)         },
88) 
89)        mirror001 => {
90)             orgName => "depthstrike.com",
91)             isoCC => "CA",
92)             subRegion => "NS",
93)             region => "North America",
94)             ipv4 => "True",
95)             ipv6 => "False",
96)             loadBalanced => "Unknown",
97)             httpWebsiteMirror => "http://tor.depthstrike.com/",
98)             ftpWebsiteMirror => "",
99)             rsyncWebsiteMirror => "",
100)             httpDistMirror => "http://tor.depthstrike.com/dist/",
101)             rsyncDistMirror => "",
102)             updateDate => "",
103)         },
104) 
105)        mirror002 => {
106)             orgName => "hermetix.org",
107)             isoCC => "CA",
108)             subRegion => "QC",
109)             region => "North America",
110)             ipv4 => "True",
111)             ipv6 => "False",
112)             loadBalanced => "Unknown",
113)             httpWebsiteMirror => "http://tor.hermetix.org/",
114)             rsyncWebsiteMirror => "",
115)             httpDistMirror => "http://tor.hermetix.org/dist/",
116)             rsyncDistMirror => "",
117)             updateDate => "",
118)         },
119) 
120)        mirror003 => {
121)             orgName => "Boinc.ch",
122)             isoCC => "CH",
123)             subRegion => "",
124)             region => "Europe",
125)             ipv4 => "True",
126)             ipv6 => "False",
127)             loadBalanced => "Unknown",
128)             httpWebsiteMirror => "http://tor.boinc.ch/",
129)             ftpWebsiteMirror => "",
130)             rsyncWebsiteMirror => "",
131)             httpDistMirror => "http://tor.boinc.ch/dist/",
132)             rsyncDistMirror => "",
133)             updateDate => "",
134)         },
135) 
136)        mirror004 => {
137)             orgName => "anonymity.cn",
138)             isoCC => "CN",
139)             subRegion => "",
140)             region => "Asia",
141)             ipv4 => "True",
142)             ipv6 => "False",
143)             loadBalanced => "Unknown",
144)             httpWebsiteMirror => "http://tor.anonymity.cn/",
145)             ftpWebsiteMirror => "",
146)             rsyncWebsiteMirror => "",
147)             httpDistMirror => "http://tor.anonymity.cn/dist/",
148)             rsyncDistMirror => "",
149)             updateDate => "",
150)         },
151) 
152)        mirror004 => {
153)             orgName => "bbs",
154)             isoCC => "DE",
155)             subRegion => "",
156)             region => "Europe",
157)             ipv4 => "True",
158)             ipv6 => "False",
159)             loadBalanced => "Unknown",
160)             httpWebsiteMirror => "http://tor.blingblingsquad.net/",
161)             ftpWebsiteMirror => "",
162)             rsyncWebsiteMirror => "",
163)             httpDistMirror => "http://tor.blingblingsquad.net/dist/",
164)             rsyncDistMirror => "",
165)             updateDate => "",
166)         },
167) 
168)        mirror005 => {
169)             orgName => "Berapla",
170)             isoCC => "DE",
171)             subRegion => "",
172)             region => "Europe",
173)             ipv4 => "True",
174)             ipv6 => "False",
175)             loadBalanced => "Unknown",
176)             httpWebsiteMirror => "http://download.berapla.de/mirrors/tor/",
177)             ftpWebsiteMirror => "",
178)             rsyncWebsiteMirror => "",
179)             httpDistMirror => "",
180)             rsyncDistMirror => "",
181)             updateDate => "",
182)         },
183) 
184)        mirror006 => {
185)             orgName => "cybermirror",
186)             isoCC => "DE",
187)             subRegion => "",
188)             region => "Europe",
189)             ipv4 => "True",
190)             ipv6 => "False",
191)             loadBalanced => "Unknown",
192)             httpWebsiteMirror => "http://tor.cybermirror.org/",
193)             ftpWebsiteMirror => "",
194)             rsyncWebsiteMirror => "",
195)             httpDistMirror => "http://tor.cybermirror.org/dist/",
196)             rsyncDistMirror => "",
197)             updateDate => "",
198)         },
199) 
200)        mirror007 => {
201)             orgName => "Spline",
202)             isoCC => "DE",
203)             subRegion => "FU",
204)             region => "Europe",
205)             ipv4 => "True",
206)             ipv6 => "False",
207)             loadBalanced => "Unknown",
208)             httpWebsiteMirror => "http://rem.spline.de/tor/",
209)             ftpWebsiteMirror => "",
210)             rsyncWebsiteMirror => "",
211)             httpDistMirror => "",
212)             rsyncDistMirror => "",
213)             updateDate => "",
214)         },
215) 
216)        mirror008 => {
217)             orgName => "mirror.bsdhost.eu",
218)             isoCC => "DE",
219)             subRegion => "",
220)             region => "Europe",
221)             ipv4 => "True",
222)             ipv6 => "False",
223)             loadBalanced => "Unknown",
224)             httpWebsiteMirror => "http://mirror.bsdhost.eu/www.torproject.org/",
225)             ftpWebsiteMirror => "",
226)             rsyncWebsiteMirror => "",
227)             httpDistMirror => "http://mirror.bsdhost.eu/www.torproject.org/dist/",
228)             rsyncDistMirror => "",
229)             updateDate => "",
230)         },
231) 
232)        mirror009 => {
233)             orgName => "onionland",
234)             isoCC => "DE",
235)             subRegion => "",
236)             region => "Europe",
237)             ipv4 => "True",
238)             ipv6 => "False",
239)             loadBalanced => "Unknown",
240)             httpWebsiteMirror => "http://mirror.onionland.org/",
241)             ftpWebsiteMirror => "",
242)             rsyncWebsiteMirror => "rsync: mirror.onionland.org::tor/",
243)             httpDistMirror => "http://mirror.onionland.org/dist/",
244)             rsyncDistMirror => "rsync: mirror.onionland.org::tor/dist/",
245)             updateDate => "",
246)         },
247) 
248)        mirror010 => {
249)             orgName => "plentyfact",
250)             isoCC => "DE",
251)             subRegion => "",
252)             region => "Europe",
253)             ipv4 => "True",
254)             ipv6 => "False",
255)             loadBalanced => "Unknown",
256)             httpWebsiteMirror => "http://tor.plentyfact.net/",
257)             ftpWebsiteMirror => "",
258)             httpsWebsiteMirror => "https://tor.plentyfact.net/",
259)             rsyncWebsiteMirror => "",
260)             httpDistMirror => "http://tor.plentyfact.net/dist/",
261)             rsyncDistMirror => "",
262)             updateDate => "",
263)         },
264) 
265)        mirror011 => {
266)             orgName => "loxal.net",
267)             isoCC => "DE",
268)             subRegion => "",
269)             region => "Europe",
270)             ipv4 => "True",
271)             ipv6 => "False",
272)             loadBalanced => "Unknown",
273)             httpWebsiteMirror => "http://tor-anonymizer.mirror.loxal.net/",
274)             ftpWebsiteMirror => "",
275)             rsyncWebsiteMirror => "",
276)             httpDistMirror => "http://tor-anonymizer.mirror.loxal.net/dist/",
277)             rsyncDistMirror => "",
278)             updateDate => "",
279)         },
280) 
281)        mirror012 => {
282)             orgName => "centervenus.com",
283)             isoCC => "DE",
284)             subRegion => "",
285)             region => "Europe",
286)             ipv4 => "True",
287)             ipv6 => "False",
288)             loadBalanced => "Unknown",
289)             httpWebsiteMirror => "",
290)             ftpWebsiteMirror => "",
291)             rsyncWebsiteMirror => "",
292)             httpDistMirror => "http://www.centervenus.com/mirrors/tor/dist/",
293)             rsyncDistMirror => "",
294)             updateDate => "",
295)         },
296) 
297)        mirror013 => {
298)             orgName => "zdg-gmbh.eu",
299)             isoCC => "DK",
300)             subRegion => "",
301)             region => "Europe",
302)             ipv4 => "True",
303)             ipv6 => "False",
304)             loadBalanced => "Unknown",
305)             httpWebsiteMirror => "http://tor.zdg-gmbh.eu/", 
306)             ftpWebsiteMirror => "",
307)             rsyncWebsiteMirror => "", 
308)             httpDistMirror => "http://tor.zdg-gmbh.eu/dist/",
309)             rsyncDistMirror => "",
310)             updateDate => "",
311)         },
312) 
313)        mirror014 => {
314)             orgName => "CRAN",
315)             isoCC => "FR",
316)             subRegion => "",
317)             region => "Europe",
318)             ipv4 => "True",
319)             ipv6 => "False",
320)             loadBalanced => "Unknown",
321)             httpWebsiteMirror => "http://tor.miroir-francais.fr/",
322)             rsyncWebsiteMirror => "rsync: miroir-francais.fr::tor", 
323)             ftpWebsiteMirror => "ftp://miroir-francais.fr/pub/tor/",
324)             httpDistMirror => "",
325)             rsyncDistMirror => "",
326)             updateDate => "",
327)         },
328) 
329)        mirror015 => {
330)             orgName => "tor.newworldorder.com.es",
331)             isoCC => "HU",
332)             subRegion => "",
333)             region => "Europe",
334)             ipv4 => "True",
335)             ipv6 => "False",
336)             loadBalanced => "Unknown",
337)             httpWebsiteMirror => "http://tor.newworldorder.com.es/",
338)             rsyncWebsiteMirror => "", 
339)             ftpWebsiteMirror => "",
340)             httpDistMirror => "",
341)             rsyncDistMirror => "",
342)             updateDate => "",
343)         },
344) 
345)        mirror016 => {
346)             orgName => "amorphis.eu",
347)             isoCC => "NL",
348)             subRegion => "",
349)             region => "Europe",
350)             ipv4 => "True",
351)             ipv6 => "False",
352)             loadBalanced => "Unknown",
353)             httpWebsiteMirror => "http://tor.amorphis.eu/",
354)             rsyncWebsiteMirror => "", 
355)             ftpWebsiteMirror => "",
356)             httpDistMirror => "http://tor.amorphis.eu/dist/",
357)             rsyncDistMirror => "",
358)             updateDate => "",
359)         },
360) 
361)        mirror017 => {
362)             orgName => "BIT BV",
363)             isoCC => "NL",
364)             subRegion => "",
365)             region => "Europe",
366)             ipv4 => "True",
367)             ipv6 => "False",
368)             loadBalanced => "Unknown",
369)             httpWebsiteMirror => "http://ftp.bit.nl/mirror/tor/",
370)             rsyncWebsiteMirror => "", 
371)             ftpWebsiteMirror => "ftp://ftp.bit.nl/mirror/tor/",
372)             httpDistMirror => "",
373)             rsyncDistMirror => "",
374)             updateDate => "",
375)         },
376) 
377)        mirror018 => {
378)             orgName => "CCC",
379)             isoCC => "NL",
380)             subRegion => "",
381)             region => "Europe",
382)             ipv4 => "True",
383)             ipv6 => "False",
384)             loadBalanced => "Unknown",
385)             httpWebsiteMirror => "http://tor.ccc.de/",
386)             rsyncWebsiteMirror => "", 
387)             ftpWebsiteMirror => "",
388)             httpDistMirror => "http://tor.ccc.de/dist/",
389)             rsyncDistMirror => "",
390)             updateDate => "",
391)         },
392) 
393)        mirror018 => {
394)             orgName => "kamagurka.org",
395)             isoCC => "NL",
396)             subRegion => "Haarlem",
397)             region => "Europe",
398)             ipv4 => "True",
399)             ipv6 => "False",
400)             loadBalanced => "Unknown",
401)             httpWebsiteMirror => "http://tor.kamagurka.org/",
402)             rsyncWebsiteMirror => "", 
403)             ftpWebsiteMirror => "",
404)             httpDistMirror => "http://tor.kamagurka.org/dist/",
405)             rsyncDistMirror => "",
406)             updateDate => "",
407)         },
408) 
409)        mirror019 => {
410)             orgName => "OS Mirror",
411)             isoCC => "NL",
412)             subRegion => "",
413)             region => "Europe",
414)             ipv4 => "True",
415)             ipv6 => "False",
416)             loadBalanced => "Unknown",
417)             httpWebsiteMirror => "http://tor.osmirror.nl/",
418)             rsyncWebsiteMirror => "rsync: rsync.osmirror.nl::tor/", 
419)             ftpWebsiteMirror => "ftp://ftp.osmirror.nl/pub/tor/",
420)             httpDistMirror => "",
421)             rsyncDistMirror => "",
422)             updateDate => "",
423)         },
424) 
425) 
426)        mirror020 => {
427)             orgName => "Meulie.net",
428)             isoCC => "NO",
429)             subRegion => "",
430)             region => "Europe",
431)             ipv4 => "True",
432)             ipv6 => "False",
433)             loadBalanced => "Unknown",
434)             httpWebsiteMirror => "http://tor.meulie.net/",
435)             rsyncWebsiteMirror => "", 
436)             ftpWebsiteMirror => "",
437)             httpDistMirror => "",
438)             rsyncDistMirror => "",
439)             updateDate => "",
440)         },
441) 
442)        mirror021 => {
443)             orgName => "Swedish Linux Society",
444)             isoCC => "SE",
445)             subRegion => "",
446)             region => "Europe",
447)             ipv4 => "True",
448)             ipv6 => "False",
449)             loadBalanced => "Unknown",
450)             httpWebsiteMirror => "http://ftp.se.linux.org/crypto/tor/",
451)             rsyncWebsiteMirror => "", 
452)             ftpWebsiteMirror => "ftp://ftp.se.linux.org/pub/crypto/tor/",
453)             httpDistMirror => "",
454)             rsyncDistMirror => "",
455)             updateDate => "",
456)         },
457) 
458)        mirror022 => {
459)             orgName => "Ghirai.com",
460)             isoCC => "UK",
461)             subRegion => "London",
462)             region => "Europe",
463)             ipv4 => "True",
464)             ipv6 => "False",
465)             loadBalanced => "Unknown",
466)             httpWebsiteMirror => "http://www.ghirai.com/tor/",
467)             rsyncWebsiteMirror => "", 
468)             ftpWebsiteMirror => "",
469)             httpDistMirror => "",
470)             rsyncDistMirror => "",
471)             updateDate => "",
472)         },
473) 
474)        mirror023 => {
475)             orgName => "BJWOnline.com",
476)             isoCC => "US",
477)             subRegion => "California",
478)             region => "North America",
479)             ipv4 => "True",
480)             ipv6 => "False",
481)             loadBalanced => "Unknown",
482)             httpWebsiteMirror => "http://mirror.bjwonline.com/tor/",
483)             rsyncWebsiteMirror => "", 
484)             ftpWebsiteMirror => "",
485)             httpDistMirror => "",
486)             rsyncDistMirror => "",
487)             updateDate => "",
488)         },
489) 
490)        mirror024 => {
491)             orgName => "Libertarian Action Network",

492)             isoCC => "US",

493)             subRegion => "",
494)             region => "North America",
495)             ipv4 => "True",
496)             ipv6 => "False",
497)             loadBalanced => "Unknown",
498)             httpWebsiteMirror => "",
499)             rsyncWebsiteMirror => "", 
500)             ftpWebsiteMirror => "ftp://libertarianactivism.com/tor.eff.org/dist/",
501)             httpDistMirror => "",
502)             rsyncDistMirror => "",
503)             updateDate => "",
504)         },
505) 
506)        mirror025 => {
507)             orgName => "TheOnionRouter.com",
508)             isoCC => "US",
509)             subRegion => "Texas",
510)             region => "North America",
511)             ipv4 => "True",
512)             ipv6 => "False",
513)             loadBalanced => "Unknown",
514)             httpWebsiteMirror => "http://www.theonionrouter.com/",
515)             rsyncWebsiteMirror => "", 
516)             ftpWebsiteMirror => "",
517)             httpDistMirror => "http://www.theonionrouter.com/dist/",
518)             rsyncDistMirror => "",
519)             updateDate => "",
520)         },
521) 
522)        mirror026 => {
523)             orgName => "Site2nd.org",
524)             isoCC => "USA",
525)             subRegion => "Texas",
526)             region => "North America",
527)             ipv4 => "True",
528)             ipv6 => "False",
529)             loadBalanced => "Unknown",
530)             httpWebsiteMirror => "http://tor.site2nd.org",
531)             rsyncWebsiteMirror => "", 
532)             ftpWebsiteMirror => "",
533)             httpDistMirror => "",
534)             rsyncDistMirror => "",
535)             updateDate => "",
536)         },
537) 
538)        mirror027 => {
539)             adminContact => "jeroen\@unfix.org",
540)             orgName => "unfix",
541)             isoCC => "CH",
542)             subRegion => "",
543)             region => "Europe",
544)             ipv4 => "True",
545)             ipv6 => "True",
546)             loadBalanced => "Unknown",
547)             httpWebsiteMirror => "http://tor.unfix.org/",
548)             rsyncWebsiteMirror => "", 
549)             ftpWebsiteMirror => "",
550)             httpDistMirror => "",
551)             rsyncDistMirror => "",
552)             updateDate => "",
553)         },
554) 
555)        mirror028 => {
556)             adminContact => "jeroen\@unfix.org",
557)             orgName => "sixx",

558)             isoCC => "CH",

559)             subRegion => "",
560)             region => "Europe",
561)             ipv4 => "True",
562)             ipv6 => "True",
563)             loadBalanced => "Unknown",
564)             httpWebsiteMirror => "http://tor.sixxs.net/",
565)             rsyncWebsiteMirror => "", 
566)             ftpWebsiteMirror => "",
567)             httpDistMirror => "",
568)             rsyncDistMirror => "",
569)             updateDate => "",
570)         },
571) );
572) 
573) my $count = values %m;
574) print "We have a total of $count mirrors\n";
575) print "Fetching the last updated date for each mirror.\n";
576) 
577) foreach my $server ( keys %m ) {
578) 
579)     print "Attempting to fetch from $m{$server}{'orgName'}\n";
580) 
581)     if ($m{$server}{'httpWebsiteMirror'}) {
582)         print "Attempt to fetch via HTTP.\n";
583)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpWebsiteMirror'}");
584)     } elsif ($m{$server}{'httpsWebsiteMirror'}) {
585)         print "Attempt to fetch via HTTPS.\n";
586)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpsWebsiteMirror'}");
587)     } elsif ($m{$server}{'ftpWebsiteMirror'}) {
588)         print "Attempt to fetch via FTP.\n";
589)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'ftpWebsiteMirror'}");
590)     } else {
591)         print "We were unable to fetch or store anything. We still have the following: $m{$server}{'updateDate'}\n";
592)     }
593) 
594)     print "We fetched and stored the following: $m{$server}{'updateDate'}\n";
595) 
596)  }
597) 
598) 
599) print "We sorted the following mirrors by their date of last update: \n";
600) foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
601) 
602)      print "\n";
603)      print "Mirror $m{$server}{'orgName'}: \n";
604) 
605)      foreach my $attrib ( sort keys %{$m{$server}} ) {
606)         print "$attrib = $m{$server}{$attrib}";
607)         print "\n";
608)      };
609) }
610) 
611) my $outFile = "include/mirrors-table.wmi";
612) my $html;
613) open(OUT, "> $outFile") or die "Can't open $outFile: $!";
614) 
615) # Here's where we open a file and print some wml include goodness 
616) # This is storted from last known recent update to unknown update times
617) foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
618) 

619)      my $time = ctime($m{$server}{'updateDate'});
620)      chomp($time);
621) print OUT <<"END";
622)      \n<tr>\n
623)          <td>$m{$server}{'isoCC'}</td>\n
624)          <td>$m{$server}{'orgName'}</td>\n
625)          <td>$time</td>\n
626) END
627) 
628)      my %prettyNames = (
629)                         httpWebsiteMirror => "http",
630)                         httpsWebsiteMirror => "https",
631)                         ftpWebsiteMirror => "ftp",
632)                         rsyncWebsiteMirror => "rsync",
633)                         httpDistMirror => "http",
634)                         httpsDistMirror => "https",
635)                         rsyncDistMirrors => "rsync", );
636) 
637)      foreach my $precious ( sort keys %prettyNames )

638)      {
639)         if ($m{$server}{"$precious"}) {
640)             print OUT "    <td><a href=\"" . $m{$server}{$precious} . "\">" .

641)                       "$prettyNames{$precious}</a></td>\n";