Browse code

update mirror script again, run it and update mirrors table.

Andrew Lewman authored on06/12/2013 04:49:41
Showing2 changed files
... ...
@@ -16,6 +16,23 @@
16 16
     <td> - </td>
17 17
 </tr>
18 18
      
19
+<tr>
20
+
21
+         <td>DK</td>
22
+
23
+         <td>Zentrum der Gesundheit</td>
24
+
25
+         <td>Up to date</td>
26
+
27
+    <td> - </td>
28
+    <td><a href="http://tor.idnr.ws/dist/">http</a></td>
29
+    <td><a href="http://tor.idnr.ws/">http</a></td>
30
+    <td> - </td>
31
+    <td> - </td>
32
+    <td> - </td>
33
+    <td> - </td>
34
+</tr>
35
+     
19 36
 <tr>
20 37
 
21 38
          <td>NO</td>
... ...
@@ -305,23 +322,6 @@
305 322
     <td> - </td>
306 323
 </tr>
307 324
      
308
-<tr>
309
-
310
-         <td>US</td>
311
-
312
-         <td>Xpdm</td>
313
-
314
-         <td>Up to date</td>
315
-
316
-    <td> - </td>
317
-    <td><a href="http://torproj.xpdm.us/dist/">http</a></td>
318
-    <td><a href="http://torproj.xpdm.us/">http</a></td>
319
-    <td><a href="https://torproj.xpdm.us/dist/">https</a></td>
320
-    <td><a href="https://torproj.xpdm.us/">https</a></td>
321
-    <td> - </td>
322
-    <td> - </td>
323
-</tr>
324
-     
325 325
 <tr>
326 326
 
327 327
          <td>DE</td>
... ...
@@ -358,34 +358,17 @@
358 358
      
359 359
 <tr>
360 360
 
361
-         <td>LT</td>
361
+         <td>MX</td>
362 362
 
363 363
          <td></td>
364 364
 
365 365
          <td>Up to date</td>
366 366
 
367 367
     <td> - </td>
368
-    <td><a href="http://tor.vesta.nu/dist/">http</a></td>
369
-    <td><a href="http://tor.vesta.nu/">http</a></td>
370
-    <td> - </td>
371
-    <td> - </td>
372
-    <td> - </td>
373
-    <td> - </td>
374
-</tr>
375
-     
376
-<tr>
377
-
378
-         <td>US</td>
379
-
380
-         <td>AskApache</td>
381
-
382
-         <td>Up to date</td>
383
-
384
-    <td> - </td>
385
-    <td><a href="http://tor.askapache.com/dist/">http</a></td>
386
-    <td><a href="http://tor.askapache.com/">http</a></td>
387
-    <td> - </td>
388
-    <td> - </td>
368
+    <td><a href="http://fbnaia.homelinux.net/torproject/dist/">http</a></td>
369
+    <td><a href="http://fbnaia.homelinux.net/torproject/">http</a></td>
370
+    <td><a href="https://fbnaia.homelinux.net/torproject//dist/">https</a></td>
371
+    <td><a href="https://fbnaia.homelinux.net/torproject/">https</a></td>
389 372
     <td> - </td>
390 373
     <td> - </td>
391 374
 </tr>
... ...
@@ -492,6 +475,23 @@
492 475
     <td> - </td>
493 476
 </tr>
494 477
      
478
+<tr>
479
+
480
+         <td>NL</td>
481
+
482
+         <td>Amorphis</td>
483
+
484
+         <td>Up to date</td>
485
+
486
+    <td> - </td>
487
+    <td><a href="http://tor.amorphis.eu/dist/">http</a></td>
488
+    <td><a href="http://tor.amorphis.eu/">http</a></td>
489
+    <td> - </td>
490
+    <td> - </td>
491
+    <td> - </td>
492
+    <td> - </td>
493
+</tr>
494
+     
495 495
 <tr>
496 496
 
497 497
          <td>US</td>
... ...
@@ -662,23 +662,6 @@
662 662
     <td><a href="rsync://creep.im/tor">rsync</a></td>
663 663
 </tr>
664 664
      
665
-<tr>
666
-
667
-         <td>US</td>
668
-
669
-         <td></td>
670
-
671
-         <td>Up to date</td>
672
-
673
-    <td><a href="ftp://mirrors.go-parts.com/tor/">ftp</a></td>
674
-    <td><a href="http://mirrors.go-parts.com/tor/dist/">http</a></td>
675
-    <td> - </td>
676
-    <td> - </td>
677
-    <td> - </td>
678
-    <td><a href="rsync://mirrors.go-parts.com/mirrors/tor/">rsync</a></td>
679
-    <td> - </td>
680
-</tr>
681
-     
682 665
 <tr>
683 666
 
684 667
          <td>DE</td>
... ...
@@ -749,32 +732,15 @@
749 732
      
750 733
 <tr>
751 734
 
752
-         <td>IS</td>
753
-
754
-         <td>myRL.net</td>
755
-
756
-         <td>Up to date</td>
757
-
758
-    <td> - </td>
759
-    <td><a href="http://tor.myrl.net/dist/">http</a></td>
760
-    <td><a href="http://tor.myrl.net/">http</a></td>
761
-    <td><a href="https://tor.myrl.net/dist/">https</a></td>
762
-    <td><a href="https://tor.myrl.net/">https</a></td>
763
-    <td> - </td>
764
-    <td> - </td>
765
-</tr>
766
-     
767
-<tr>
768
-
769
-         <td>NL</td>
735
+         <td>SE</td>
770 736
 
771
-         <td>Amorphis</td>
737
+         <td>homosu</td>
772 738
 
773 739
          <td>Up to date</td>
774 740
 
775 741
     <td> - </td>
776
-    <td><a href="http://tor.amorphis.eu/dist/">http</a></td>
777
-    <td><a href="http://tor.amorphis.eu/">http</a></td>
742
+    <td><a href="http://tor.homosu.net/dist/">http</a></td>
743
+    <td><a href="http://tor.homosu.net/">http</a></td>
778 744
     <td> - </td>
779 745
     <td> - </td>
780 746
     <td> - </td>
... ...
@@ -783,17 +749,17 @@
783 749
      
784 750
 <tr>
785 751
 
786
-         <td>DK</td>
752
+         <td>IS</td>
787 753
 
788
-         <td>Zentrum der Gesundheit</td>
754
+         <td>myRL.net</td>
789 755
 
790 756
          <td>Up to date</td>
791 757
 
792 758
     <td> - </td>
793
-    <td><a href="http://tor.idnr.ws/dist/">http</a></td>
794
-    <td><a href="http://tor.idnr.ws/">http</a></td>
795
-    <td> - </td>
796
-    <td> - </td>
759
+    <td><a href="http://tor.myrl.net/dist/">http</a></td>
760
+    <td><a href="http://tor.myrl.net/">http</a></td>
761
+    <td><a href="https://tor.myrl.net/dist/">https</a></td>
762
+    <td><a href="https://tor.myrl.net/">https</a></td>
797 763
     <td> - </td>
798 764
     <td> - </td>
799 765
 </tr>
... ...
@@ -817,32 +783,15 @@
817 783
      
818 784
 <tr>
819 785
 
820
-         <td>SE</td>
786
+         <td>LT</td>
821 787
 
822
-         <td>homosu</td>
788
+         <td></td>
823 789
 
824 790
          <td>Up to date</td>
825 791
 
826 792
     <td> - </td>
827
-    <td><a href="http://tor.homosu.net/dist/">http</a></td>
828
-    <td><a href="http://tor.homosu.net/">http</a></td>
829
-    <td> - </td>
830
-    <td> - </td>
831
-    <td> - </td>
832
-    <td> - </td>
833
-</tr>
834
-     
835
-<tr>
836
-
837
-         <td>IN</td>
838
-
839
-         <td>India Tor Fans</td>
840
-
841
-         <td>DO NOT USE. Out of date.</td>
842
-
843
-    <td> - </td>
844
-    <td><a href="http://www.torproject.org.in/dist/">http</a></td>
845
-    <td><a href="http://www.torproject.org.in/">http</a></td>
793
+    <td><a href="http://tor.vesta.nu/dist/">http</a></td>
794
+    <td><a href="http://tor.vesta.nu/">http</a></td>
846 795
     <td> - </td>
847 796
     <td> - </td>
848 797
     <td> - </td>
... ...
@@ -851,17 +800,17 @@
851 800
      
852 801
 <tr>
853 802
 
854
-         <td>TN</td>
803
+         <td>US</td>
855 804
 
856
-         <td></td>
805
+         <td>AskApache</td>
857 806
 
858
-         <td>DO NOT USE. Out of date.</td>
807
+         <td>Up to date</td>
859 808
 
860 809
     <td> - </td>
810
+    <td><a href="http://tor.askapache.com/dist/">http</a></td>
811
+    <td><a href="http://tor.askapache.com/">http</a></td>
861 812
     <td> - </td>
862
-    <td><a href="http://torproject.antagonism.org/">http</a></td>
863 813
     <td> - </td>
864
-    <td><a href="https://torproject.antagonism.org/">https</a></td>
865 814
     <td> - </td>
866 815
     <td> - </td>
867 816
 </tr>
... ...
@@ -900,23 +849,6 @@
900 849
     <td><a href="rsync://mirrors.whitedholdings.org/tor">rsync</a></td>
901 850
 </tr>
902 851
      
903
-<tr>
904
-
905
-         <td>US</td>
906
-
907
-         <td>searchprivate</td>
908
-
909
-         <td>DO NOT USE. Out of date.</td>
910
-
911
-    <td> - </td>
912
-    <td><a href="http://tor.searchprivate.com/dist/">http</a></td>
913
-    <td><a href="http://tor.searchprivate.com/">http</a></td>
914
-    <td> - </td>
915
-    <td> - </td>
916
-    <td> - </td>
917
-    <td> - </td>
918
-</tr>
919
-     
920 852
 <tr>
921 853
 
922 854
          <td>DE</td>
... ...
@@ -951,23 +883,6 @@
951 883
     <td> - </td>
952 884
 </tr>
953 885
      
954
-<tr>
955
-
956
-         <td>DE</td>
957
-
958
-         <td>Netzkonstrukt Berlin</td>
959
-
960
-         <td>DO NOT USE. Out of date.</td>
961
-
962
-    <td> - </td>
963
-    <td><a href="http://mirror.ntzk.de/torproject.org/dist/">http</a></td>
964
-    <td><a href="http://mirror.ntzk.de/torproject.org/">http</a></td>
965
-    <td> - </td>
966
-    <td> - </td>
967
-    <td> - </td>
968
-    <td> - </td>
969
-</tr>
970
-     
971 886
 <tr>
972 887
 
973 888
          <td>UA</td>
... ...
@@ -985,23 +900,6 @@
985 900
     <td> - </td>
986 901
 </tr>
987 902
      
988
-<tr>
989
-
990
-         <td>MX</td>
991
-
992
-         <td></td>
993
-
994
-         <td>Unknown</td>
995
-
996
-    <td> - </td>
997
-    <td><a href="http://fbnaia.homelinux.net/torproject/dist/">http</a></td>
998
-    <td><a href="http://fbnaia.homelinux.net/torproject/">http</a></td>
999
-    <td><a href="https://fbnaia.homelinux.net/torproject//dist/">https</a></td>
1000
-    <td><a href="https://fbnaia.homelinux.net/torproject/">https</a></td>
1001
-    <td> - </td>
1002
-    <td> - </td>
1003
-</tr>
1004
-     
1005 903
 <tr>
1006 904
 
1007 905
          <td>FR</td>
... ...
@@ -1171,3 +1069,37 @@
1171 1069
     <td> - </td>
1172 1070
     <td> - </td>
1173 1071
 </tr>
1072
+     
1073
+<tr>
1074
+
1075
+         <td>US</td>
1076
+
1077
+         <td>Xpdm</td>
1078
+
1079
+         <td>Failed signature check</td>
1080
+
1081
+    <td> - </td>
1082
+    <td><a href="http://torproj.xpdm.us/dist/">http</a></td>
1083
+    <td><a href="http://torproj.xpdm.us/">http</a></td>
1084
+    <td><a href="https://torproj.xpdm.us/dist/">https</a></td>
1085
+    <td><a href="https://torproj.xpdm.us/">https</a></td>
1086
+    <td> - </td>
1087
+    <td> - </td>
1088
+</tr>
1089
+     
1090
+<tr>
1091
+
1092
+         <td>US</td>
1093
+
1094
+         <td></td>
1095
+
1096
+         <td>Failed signature check</td>
1097
+
1098
+    <td><a href="ftp://mirrors.go-parts.com/tor/">ftp</a></td>
1099
+    <td><a href="http://mirrors.go-parts.com/tor/dist/">http</a></td>
1100
+    <td> - </td>
1101
+    <td> - </td>
1102
+    <td> - </td>
1103
+    <td><a href="rsync://mirrors.go-parts.com/mirrors/tor/">rsync</a></td>
1104
+    <td> - </td>
1105
+</tr>
... ...
@@ -59,26 +59,31 @@ sub ExtractDate {
59 59
     my $date    = str2time($content);
60 60
 
61 61
     if ($date) {
62
-    	print "ExtractDate($content) = $date\n";
62
+    	print "\tExtractDate($content) = $date\n";
63 63
         return $date;
64 64
     } else {
65
-    	print "ExtractDate($content) = ?\n";
65
+    	print "\tExtractDate($content) = ?\n";
66 66
 	return undef;
67 67
     }
68 68
 }
69 69
 
70 70
 sub ExtractSig {
71
-    my $content = shift;
72
-    return sha256_hex($content); 
71
+    my $content = shift; 
72
+    my $url     = shift;
73
+    my $sig = sha256_hex($content);
74
+    print "\tExtractSig($url) = $sig\n";
75
+    return $sig;
73 76
 }
74 77
 
75 78
 sub Fetch {
76 79
     my ($url, $sub) = @_; # Base url for mirror
80
+    $|++; # unbuffer stdout to show progress
77 81
 
82
+    print "\nGET $url: ";
78 83
     my $request = new HTTP::Request GET => "$url";
79 84
     my $result = $lua->request($request);
80 85
     my $code = $result->code();
81
-    print "\nGET $url: $code\n";
86
+    print "$code\n";
82 87
 
83 88
     if ($result->is_success && $code eq "200"){
84 89
        my $content = $result->content;
... ...
@@ -146,8 +151,9 @@ for (1 .. 1)
146 151
 
147 152
 print "Using these files for sig matching:\n";
148 153
 print join("\n", keys %randomtorfiles);
154
+print "\n";
149 155
 
150
-# Adjust official Tor time by out-of-date offset: number of days * seconds per day
156
+# Adjust offical Tor time by out-of-date offset: number of days * seconds per day
151 157
 $tortime -= 1 * 172800;
152 158
 print "The official time for Tor is $tortime. \n";
153 159
 
... ...
@@ -163,12 +169,10 @@ for(my $server = 0; $server < scalar(@m); $server++) {
163 169
                 foreach my $randomtorfile(keys %randomtorfiles) {
164 170
                     my $sig = Fetch("$m[$server]->{$serverType}/$randomtorfile", \&ExtractSig);
165 171
             	    if (!$sig) {
166
-		        print STDERR "Unreadable $randomtorfile on $m[$server]->{$serverType}";
167 172
 			$m[$server]->{sigMatched} = 0;
168 173
             	    	last;
169 174
 		    } elsif ($sig ne $randomtorfiles{$randomtorfile}) {
170 175
 			$m[$server]->{sigMatched} = 0;
171
-		        print STDERR "Sig mismatch of $randomtorfile on $m[$server]->{$serverType}";
172 176
             	    	last;
173 177
             	    }
174 178
 		}
... ...
@@ -181,15 +185,13 @@ for(my $server = 0; $server < scalar(@m); $server++) {
181 185
 sub PrintServer {
182 186
      my $server = shift;
183 187
      my $time;
184
-     if ( $server->{'updateDate'} ) {
185
-	  if ( $server->{'updateDate'} > $tortime ) {
186
-	    $time = "Up to date";
187
-	  } else { $time = "DO NOT USE. Out of date."; }
188
-     } else { $time = "Unknown"; }
188
+     if ( $server->{updateDate} && $server->{sigMatched} ) { $time = "Up to date"; } 
189
+     elsif (!$server->{updateDate}) 			   { $time = "Unknown"; }
190
+     else 			    			   { $time = "Failed signature check"; }
189 191
 print OUT <<"END";
190 192
      \n<tr>\n
191
-         <td>$server->{'isoCC'}</td>\n
192
-         <td>$server->{'orgName'}</td>\n
193
+         <td>$server->{isoCC}</td>\n
194
+         <td>$server->{orgName}</td>\n
193 195
          <td>$time</td>\n
194 196
 END
195 197
 
... ...
@@ -204,7 +206,7 @@ END
204 206
 
205 207
      foreach my $precious ( sort keys %prettyNames )
206 208
      {
207
-        if ($server->{"$precious"}) {
209
+        if ($server->{$precious}) {
208 210
             print OUT "    <td><a href=\"" . $server->{$precious} . "\">" .
209 211
                       "$prettyNames{$precious}</a></td>\n";
210 212
         } else { print OUT "    <td> - </td>\n"; }
... ...
@@ -219,12 +221,16 @@ open(OUT, "> $outFile") or die "Can't open $outFile: $!";
219 221
 
220 222
 # Here's where we open a file and print some wml include goodness
221 223
 # This is sorted from last known recent update to unknown update times
222
-foreach my $server ( sort { $b->{'updateDate'} <=> $a->{'updateDate'}} grep {$_->{updateDate} && $_->{sigMatched}} @m ) {
224
+foreach my $server ( sort { $b->{updateDate} <=> $a->{updateDate} } grep {$_->{updateDate} && $_->{updateDate} > $tortime && $_->{sigMatched}} @m ) {
225
+    PrintServer($server);
226
+}
227
+foreach my $server ( grep {!$_->{updateDate}}  @m ) {
223 228
     PrintServer($server);
224 229
 }
225
-foreach my $server ( grep {!$_->{updateDate} || !$_->{sigMatched}} @m ) {
230
+foreach my $server ( grep {!$_->{sigMatched} && $_->{updateDate} && $_->{updateDate} > $tortime} @m ) {
226 231
     PrintServer($server);
227 232
 }
233
+# That leaves those servers whose updateDate < torTime. Check the csv for those.
228 234
 
229 235
 DumpMirrors(@m);
230 236