tor-webwml.git

1) #!/usr/bin/perl -w
2) use warnings;
3) use strict;
4) use LWP::Simple;
5) use LWP;
6) use Date::Parse;

7) use Date::Format;

8) 
9) print "Creating LWP agent ($LWP::VERSION)...\n";
10) my $lua = LWP::UserAgent->new(
11)     keep_alive => 1,
12)     timeout => 15, 
13)     agent => "Tor MirrorCheck Agent"
14) );
15) 
16) sub sanitize {
17)     my $taintedData = shift;
18)     my $cleanedData;
19)     my $whitelist = '-a-zA-Z0-9: +';
20) 
21)     # clean the data, return cleaned data
22)     $taintedData =~ s/[^$whitelist]//go;
23)     $cleanedData = $taintedData;
24) 
25)     return $cleanedData;
26) }
27) 
28) sub FetchDate {
29)     my $url = shift; # Base url for mirror
30)     my $trace = "project/trace/www.torproject.org"; # Location of recent update info
31)     $url = "$url$trace";
32) 
33)     print "Fetching possible date from: $url\n";
34) 
35)     my $request = new HTTP::Request GET => "$url";
36)     my $result = $lua->request($request);
37) 
38)     if ($result->is_success){
39)        my $taint = $result->content;
40)        my $content = sanitize($taint);
41)        if ($content) {
42)             
43)             my $date = str2time($content);
44) 
45)             if ($date) {
46)                 print "We've fetched a date $date.\n";
47)                 return $date;
48)             } else {
49)                 print "We've haven't fetched a date.\n";
50)                 return "Unknown";
51)             }
52) 
53)         } else {
54)             print "Unable to fetch date, empty content returned.\n";
55)             return "Unknown";
56)         }
57)             
58)     } else {
59)        print "Our request failed, we had no result.\n";
60)        return "Unknown";
61)     }
62) }
63) 
64) # This is the list of all known Tor mirrors
65) # Add new mirrors to the bottom!
66) my %m = ( 
67)        mirror000 => {
68)             orgName => "cypherpunks.at",
69)             isoCC => "AT",
70)             subRegion => "",
71)             region => "Europe",
72)             ipv4 => "True",
73)             ipv6 => "False",
74)             loadBalanced => "Unknown",
75)             httpWebsiteMirror => "http://tor.cypherpunks.at/",
76)             rsyncWebsiteMirror => "rsync://tor.cypherpunks.at/tor",
77)             httpDistMirror => "http://tor.cypherpunks.at/dist/",
78)             rsyncDistMirror => "rsync: tor.cypherpunks.at::tor/dist/",
79)             updateDate => "",
80)         },
81) 
82)        mirror001 => {
83)             orgName => "depthstrike.com",
84)             isoCC => "CA",
85)             subRegion => "NS",
86)             region => "North America",
87)             ipv4 => "True",
88)             ipv6 => "False",
89)             loadBalanced => "Unknown",
90)             httpWebsiteMirror => "http://tor.depthstrike.com/",
91)             ftpWebsiteMirror => "",
92)             rsyncWebsiteMirror => "",
93)             httpDistMirror => "http://tor.depthstrike.com/dist/",
94)             rsyncDistMirror => "",
95)             updateDate => "",
96)         },
97) 
98)        mirror002 => {
99)             orgName => "hermetix.org",
100)             isoCC => "CA",
101)             subRegion => "QC",
102)             region => "North America",
103)             ipv4 => "True",
104)             ipv6 => "False",
105)             loadBalanced => "Unknown",
106)             httpWebsiteMirror => "http://tor.hermetix.org/",
107)             rsyncWebsiteMirror => "",
108)             httpDistMirror => "http://tor.hermetix.org/dist/",
109)             rsyncDistMirror => "",
110)             updateDate => "",
111)         },
112) 
113)        mirror003 => {
114)             orgName => "Boinc.ch",
115)             isoCC => "CH",
116)             subRegion => "",
117)             region => "Europe",
118)             ipv4 => "True",
119)             ipv6 => "False",
120)             loadBalanced => "Unknown",
121)             httpWebsiteMirror => "http://tor.boinc.ch/",
122)             ftpWebsiteMirror => "",
123)             rsyncWebsiteMirror => "",
124)             httpDistMirror => "http://tor.boinc.ch/dist/",
125)             rsyncDistMirror => "",
126)             updateDate => "",
127)         },
128) 
129)        mirror004 => {
130)             orgName => "anonymity.cn",
131)             isoCC => "CN",
132)             subRegion => "",
133)             region => "Asia",
134)             ipv4 => "True",
135)             ipv6 => "False",
136)             loadBalanced => "Unknown",
137)             httpWebsiteMirror => "http://tor.anonymity.cn/",
138)             ftpWebsiteMirror => "",
139)             rsyncWebsiteMirror => "",
140)             httpDistMirror => "http://tor.anonymity.cn/dist/",
141)             rsyncDistMirror => "",
142)             updateDate => "",
143)         },
144) 
145)        mirror004 => {
146)             orgName => "bbs",
147)             isoCC => "DE",
148)             subRegion => "",
149)             region => "Europe",
150)             ipv4 => "True",
151)             ipv6 => "False",
152)             loadBalanced => "Unknown",
153)             httpWebsiteMirror => "http://tor.blingblingsquad.net/",
154)             ftpWebsiteMirror => "",
155)             rsyncWebsiteMirror => "",
156)             httpDistMirror => "http://tor.blingblingsquad.net/dist/",
157)             rsyncDistMirror => "",
158)             updateDate => "",
159)         },
160) 
161)        mirror005 => {
162)             orgName => "Berapla",
163)             isoCC => "DE",
164)             subRegion => "",
165)             region => "Europe",
166)             ipv4 => "True",
167)             ipv6 => "False",
168)             loadBalanced => "Unknown",
169)             httpWebsiteMirror => "http://download.berapla.de/mirrors/tor/",
170)             ftpWebsiteMirror => "",
171)             rsyncWebsiteMirror => "",
172)             httpDistMirror => "",
173)             rsyncDistMirror => "",
174)             updateDate => "",
175)         },
176) 
177)        mirror006 => {
178)             orgName => "cybermirror",
179)             isoCC => "DE",
180)             subRegion => "",
181)             region => "Europe",
182)             ipv4 => "True",
183)             ipv6 => "False",
184)             loadBalanced => "Unknown",
185)             httpWebsiteMirror => "http://tor.cybermirror.org/",
186)             ftpWebsiteMirror => "",
187)             rsyncWebsiteMirror => "",
188)             httpDistMirror => "http://tor.cybermirror.org/dist/",
189)             rsyncDistMirror => "",
190)             updateDate => "",
191)         },
192) 
193)        mirror007 => {
194)             orgName => "Spline",
195)             isoCC => "DE",
196)             subRegion => "FU",
197)             region => "Europe",
198)             ipv4 => "True",
199)             ipv6 => "False",
200)             loadBalanced => "Unknown",
201)             httpWebsiteMirror => "http://rem.spline.de/tor/",
202)             ftpWebsiteMirror => "",
203)             rsyncWebsiteMirror => "",
204)             httpDistMirror => "",
205)             rsyncDistMirror => "",
206)             updateDate => "",
207)         },
208) 
209)        mirror008 => {
210)             orgName => "mirror.bsdhost.eu",
211)             isoCC => "DE",
212)             subRegion => "",
213)             region => "Europe",
214)             ipv4 => "True",
215)             ipv6 => "False",
216)             loadBalanced => "Unknown",
217)             httpWebsiteMirror => "http://mirror.bsdhost.eu/www.torproject.org/",
218)             ftpWebsiteMirror => "",
219)             rsyncWebsiteMirror => "",
220)             httpDistMirror => "http://mirror.bsdhost.eu/www.torproject.org/dist/",
221)             rsyncDistMirror => "",
222)             updateDate => "",
223)         },
224) 
225)        mirror009 => {
226)             orgName => "onionland",
227)             isoCC => "DE",
228)             subRegion => "",
229)             region => "Europe",
230)             ipv4 => "True",
231)             ipv6 => "False",
232)             loadBalanced => "Unknown",
233)             httpWebsiteMirror => "http://mirror.onionland.org/",
234)             ftpWebsiteMirror => "",
235)             rsyncWebsiteMirror => "rsync: mirror.onionland.org::tor/",
236)             httpDistMirror => "http://mirror.onionland.org/dist/",
237)             rsyncDistMirror => "rsync: mirror.onionland.org::tor/dist/",
238)             updateDate => "",
239)         },
240) 
241)        mirror010 => {
242)             orgName => "plentyfact",
243)             isoCC => "DE",
244)             subRegion => "",
245)             region => "Europe",
246)             ipv4 => "True",
247)             ipv6 => "False",
248)             loadBalanced => "Unknown",
249)             httpWebsiteMirror => "http://tor.plentyfact.net/",
250)             ftpWebsiteMirror => "",
251)             httpsWebsiteMirror => "https://tor.plentyfact.net/",
252)             rsyncWebsiteMirror => "",
253)             httpDistMirror => "http://tor.plentyfact.net/dist/",
254)             rsyncDistMirror => "",
255)             updateDate => "",
256)         },
257) 
258)        mirror011 => {
259)             orgName => "loxal.net",
260)             isoCC => "DE",
261)             subRegion => "",
262)             region => "Europe",
263)             ipv4 => "True",
264)             ipv6 => "False",
265)             loadBalanced => "Unknown",
266)             httpWebsiteMirror => "http://tor-anonymizer.mirror.loxal.net/",
267)             ftpWebsiteMirror => "",
268)             rsyncWebsiteMirror => "",
269)             httpDistMirror => "http://tor-anonymizer.mirror.loxal.net/dist/",
270)             rsyncDistMirror => "",
271)             updateDate => "",
272)         },
273) 
274)        mirror012 => {
275)             orgName => "centervenus.com",
276)             isoCC => "DE",
277)             subRegion => "",
278)             region => "Europe",
279)             ipv4 => "True",
280)             ipv6 => "False",
281)             loadBalanced => "Unknown",
282)             httpWebsiteMirror => "",
283)             ftpWebsiteMirror => "",
284)             rsyncWebsiteMirror => "",
285)             httpDistMirror => "http://www.centervenus.com/mirrors/tor/dist/",
286)             rsyncDistMirror => "",
287)             updateDate => "",
288)         },
289) 
290)        mirror013 => {
291)             orgName => "zdg-gmbh.eu",
292)             isoCC => "DK",
293)             subRegion => "",
294)             region => "Europe",
295)             ipv4 => "True",
296)             ipv6 => "False",
297)             loadBalanced => "Unknown",
298)             httpWebsiteMirror => "http://tor.zdg-gmbh.eu/", 
299)             ftpWebsiteMirror => "",
300)             rsyncWebsiteMirror => "", 
301)             httpDistMirror => "http://tor.zdg-gmbh.eu/dist/",
302)             rsyncDistMirror => "",
303)             updateDate => "",
304)         },
305) 
306)        mirror014 => {
307)             orgName => "CRAN",
308)             isoCC => "FR",
309)             subRegion => "",
310)             region => "Europe",
311)             ipv4 => "True",
312)             ipv6 => "False",
313)             loadBalanced => "Unknown",
314)             httpWebsiteMirror => "http://tor.miroir-francais.fr/",
315)             rsyncWebsiteMirror => "rsync: miroir-francais.fr::tor", 
316)             ftpWebsiteMirror => "ftp://miroir-francais.fr/pub/tor/",
317)             httpDistMirror => "",
318)             rsyncDistMirror => "",
319)             updateDate => "",
320)         },
321) 
322)        mirror015 => {
323)             orgName => "tor.newworldorder.com.es",
324)             isoCC => "HU",
325)             subRegion => "",
326)             region => "Europe",
327)             ipv4 => "True",
328)             ipv6 => "False",
329)             loadBalanced => "Unknown",
330)             httpWebsiteMirror => "http://tor.newworldorder.com.es/",
331)             rsyncWebsiteMirror => "", 
332)             ftpWebsiteMirror => "",
333)             httpDistMirror => "",
334)             rsyncDistMirror => "",
335)             updateDate => "",
336)         },
337) 
338)        mirror016 => {
339)             orgName => "amorphis.eu",
340)             isoCC => "NL",
341)             subRegion => "",
342)             region => "Europe",
343)             ipv4 => "True",
344)             ipv6 => "False",
345)             loadBalanced => "Unknown",
346)             httpWebsiteMirror => "http://tor.amorphis.eu/",
347)             rsyncWebsiteMirror => "", 
348)             ftpWebsiteMirror => "",
349)             httpDistMirror => "http://tor.amorphis.eu/dist/",
350)             rsyncDistMirror => "",
351)             updateDate => "",
352)         },
353) 
354)        mirror017 => {
355)             orgName => "BIT BV",
356)             isoCC => "NL",
357)             subRegion => "",
358)             region => "Europe",
359)             ipv4 => "True",
360)             ipv6 => "False",
361)             loadBalanced => "Unknown",
362)             httpWebsiteMirror => "http://ftp.bit.nl/mirror/tor/",
363)             rsyncWebsiteMirror => "", 
364)             ftpWebsiteMirror => "ftp://ftp.bit.nl/mirror/tor/",
365)             httpDistMirror => "",
366)             rsyncDistMirror => "",
367)             updateDate => "",
368)         },
369) 
370)        mirror018 => {
371)             orgName => "CCC",
372)             isoCC => "NL",
373)             subRegion => "",
374)             region => "Europe",
375)             ipv4 => "True",
376)             ipv6 => "False",
377)             loadBalanced => "Unknown",
378)             httpWebsiteMirror => "http://tor.ccc.de/",
379)             rsyncWebsiteMirror => "", 
380)             ftpWebsiteMirror => "",
381)             httpDistMirror => "http://tor.ccc.de/dist/",
382)             rsyncDistMirror => "",
383)             updateDate => "",
384)         },
385) 
386)        mirror018 => {
387)             orgName => "kamagurka.org",
388)             isoCC => "NL",
389)             subRegion => "Haarlem",
390)             region => "Europe",
391)             ipv4 => "True",
392)             ipv6 => "False",
393)             loadBalanced => "Unknown",
394)             httpWebsiteMirror => "http://tor.kamagurka.org/",
395)             rsyncWebsiteMirror => "", 
396)             ftpWebsiteMirror => "",
397)             httpDistMirror => "http://tor.kamagurka.org/dist/",
398)             rsyncDistMirror => "",
399)             updateDate => "",
400)         },
401) 
402)        mirror019 => {
403)             orgName => "OS Mirror",
404)             isoCC => "NL",
405)             subRegion => "",
406)             region => "Europe",
407)             ipv4 => "True",
408)             ipv6 => "False",
409)             loadBalanced => "Unknown",
410)             httpWebsiteMirror => "http://tor.osmirror.nl/",
411)             rsyncWebsiteMirror => "rsync: rsync.osmirror.nl::tor/", 
412)             ftpWebsiteMirror => "ftp://ftp.osmirror.nl/pub/tor/",
413)             httpDistMirror => "",
414)             rsyncDistMirror => "",
415)             updateDate => "",
416)         },
417) 
418) 
419)        mirror020 => {
420)             orgName => "Meulie.net",
421)             isoCC => "NO",
422)             subRegion => "",
423)             region => "Europe",
424)             ipv4 => "True",
425)             ipv6 => "False",
426)             loadBalanced => "Unknown",
427)             httpWebsiteMirror => "http://tor.meulie.net/",
428)             rsyncWebsiteMirror => "", 
429)             ftpWebsiteMirror => "",
430)             httpDistMirror => "",
431)             rsyncDistMirror => "",
432)             updateDate => "",
433)         },
434) 
435)        mirror021 => {
436)             orgName => "Swedish Linux Society",
437)             isoCC => "SE",
438)             subRegion => "",
439)             region => "Europe",
440)             ipv4 => "True",
441)             ipv6 => "False",
442)             loadBalanced => "Unknown",
443)             httpWebsiteMirror => "http://ftp.se.linux.org/crypto/tor/",
444)             rsyncWebsiteMirror => "", 
445)             ftpWebsiteMirror => "ftp://ftp.se.linux.org/pub/crypto/tor/",
446)             httpDistMirror => "",
447)             rsyncDistMirror => "",
448)             updateDate => "",
449)         },
450) 
451)        mirror022 => {
452)             orgName => "Ghirai.com",
453)             isoCC => "UK",
454)             subRegion => "London",
455)             region => "Europe",
456)             ipv4 => "True",
457)             ipv6 => "False",
458)             loadBalanced => "Unknown",
459)             httpWebsiteMirror => "http://www.ghirai.com/tor/",
460)             rsyncWebsiteMirror => "", 
461)             ftpWebsiteMirror => "",
462)             httpDistMirror => "",
463)             rsyncDistMirror => "",
464)             updateDate => "",
465)         },
466) 
467)        mirror023 => {
468)             orgName => "BJWOnline.com",
469)             isoCC => "US",
470)             subRegion => "California",
471)             region => "North America",
472)             ipv4 => "True",
473)             ipv6 => "False",
474)             loadBalanced => "Unknown",
475)             httpWebsiteMirror => "http://mirror.bjwonline.com/tor/",
476)             rsyncWebsiteMirror => "", 
477)             ftpWebsiteMirror => "",
478)             httpDistMirror => "",
479)             rsyncDistMirror => "",
480)             updateDate => "",
481)         },
482) 
483)        mirror024 => {
484)             orgName => "Libertarian Action Network",
485)             isoCC => "",
486)             subRegion => "",
487)             region => "North America",
488)             ipv4 => "True",
489)             ipv6 => "False",
490)             loadBalanced => "Unknown",
491)             httpWebsiteMirror => "",
492)             rsyncWebsiteMirror => "", 
493)             ftpWebsiteMirror => "ftp://libertarianactivism.com/tor.eff.org/dist/",
494)             httpDistMirror => "",
495)             rsyncDistMirror => "",
496)             updateDate => "",
497)         },
498) 
499)        mirror025 => {
500)             orgName => "TheOnionRouter.com",
501)             isoCC => "US",
502)             subRegion => "Texas",
503)             region => "North America",
504)             ipv4 => "True",
505)             ipv6 => "False",
506)             loadBalanced => "Unknown",
507)             httpWebsiteMirror => "http://www.theonionrouter.com/",
508)             rsyncWebsiteMirror => "", 
509)             ftpWebsiteMirror => "",
510)             httpDistMirror => "http://www.theonionrouter.com/dist/",
511)             rsyncDistMirror => "",
512)             updateDate => "",
513)         },
514) 
515)        mirror026 => {
516)             orgName => "Site2nd.org",
517)             isoCC => "USA",
518)             subRegion => "Texas",
519)             region => "North America",
520)             ipv4 => "True",
521)             ipv6 => "False",
522)             loadBalanced => "Unknown",
523)             httpWebsiteMirror => "http://tor.site2nd.org",
524)             rsyncWebsiteMirror => "", 
525)             ftpWebsiteMirror => "",
526)             httpDistMirror => "",
527)             rsyncDistMirror => "",
528)             updateDate => "",
529)         },
530) 
531)        mirror027 => {
532)             adminContact => "jeroen\@unfix.org",
533)             orgName => "unfix",
534)             isoCC => "CH",
535)             subRegion => "",
536)             region => "Europe",
537)             ipv4 => "True",
538)             ipv6 => "True",
539)             loadBalanced => "Unknown",
540)             httpWebsiteMirror => "http://tor.unfix.org/",
541)             rsyncWebsiteMirror => "", 
542)             ftpWebsiteMirror => "",
543)             httpDistMirror => "",
544)             rsyncDistMirror => "",
545)             updateDate => "",
546)         },
547) 
548)        mirror028 => {
549)             adminContact => "jeroen\@unfix.org",
550)             orgName => "sixx",

551)             isoCC => "CH",

552)             subRegion => "",
553)             region => "Europe",
554)             ipv4 => "True",
555)             ipv6 => "True",
556)             loadBalanced => "Unknown",
557)             httpWebsiteMirror => "http://tor.sixxs.net/",
558)             rsyncWebsiteMirror => "", 
559)             ftpWebsiteMirror => "",
560)             httpDistMirror => "",
561)             rsyncDistMirror => "",
562)             updateDate => "",
563)         },
564) );
565) 
566) my $count = values %m;
567) print "We have a total of $count mirrors\n";
568) print "Fetching the last updated date for each mirror.\n";
569) 
570) foreach my $server ( keys %m ) {
571) 
572)     print "Attempting to fetch from $m{$server}{'orgName'}\n";
573) 
574)     if ($m{$server}{'httpWebsiteMirror'}) {
575)         print "Attempt to fetch via HTTP.\n";
576)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpWebsiteMirror'}");
577)     } elsif ($m{$server}{'httpsWebsiteMirror'}) {
578)         print "Attempt to fetch via HTTPS.\n";
579)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpsWebsiteMirror'}");
580)     } elsif ($m{$server}{'ftpWebsiteMirror'}) {
581)         print "Attempt to fetch via FTP.\n";
582)         $m{$server}{"updateDate"} = FetchDate("$m{$server}{'ftpWebsiteMirror'}");
583)     } else {
584)         print "We were unable to fetch or store anything. We still have the following: $m{$server}{'updateDate'}\n";
585)     }
586) 
587)     print "We fetched and stored the following: $m{$server}{'updateDate'}\n";
588) 
589)  }
590) 
591) 
592) print "We sorted the following mirrors by their date of last update: \n";
593) foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
594) 
595)      print "\n";
596)      print "Mirror $m{$server}{'orgName'}: \n";
597) 
598)      foreach my $attrib ( sort keys %{$m{$server}} ) {
599)         print "$attrib = $m{$server}{$attrib}";
600)         print "\n";
601)      };
602) }
603) 
604) my $outFile = "include/mirrors-table.wmi";
605) my $html;
606) open(OUT, "> $outFile") or die "Can't open $outFile: $!";
607) 
608) # Here's where we open a file and print some wml include goodness 
609) # This is storted from last known recent update to unknown update times
610) foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
611) 

612)      my $time = ctime($m{$server}{'updateDate'});
613)      chomp($time);
614) print OUT <<"END";
615)      \n<tr>\n
616)          <td>$m{$server}{'isoCC'}</td>\n
617)          <td>$m{$server}{'orgName'}</td>\n
618)          <td>$time</td>\n
619) END
620) 
621)      my %prettyNames = (
622)                         httpWebsiteMirror => "http",
623)                         httpsWebsiteMirror => "https",
624)                         ftpWebsiteMirror => "ftp",
625)                         rsyncWebsiteMirror => "rsync",
626)                         httpDistMirror => "http",
627)                         httpsDistMirror => "https",
628)                         rsyncDistMirrors => "rsync", );
629) 
630)      foreach my $precious ( sort keys %prettyNames )

631)      {
632)         if ($m{$server}{"$precious"}) {
633)             print OUT "    <td><a href=\"" . $m{$server}{$precious} . "\">" .

634)                       "$prettyNames{$precious}</a></td>\n";