Browse code

new update mirrors script, see ticket 10269, update mirrors table with the new version.

Andrew Lewman authored on04/12/2013 23:12:29
Showing2 changed files
... ...
@@ -1,21 +1,4 @@
1 1
      
2
-<tr>
3
-
4
-         <td>INT</td>
5
-
6
-         <td>CoralCDN</td>
7
-
8
-         <td>Up to date</td>
9
-
10
-    <td> - </td>
11
-    <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td>
12
-    <td><a href="http://www.torproject.org.nyud.net/">http</a></td>
13
-    <td> - </td>
14
-    <td> - </td>
15
-    <td> - </td>
16
-    <td> - </td>
17
-</tr>
18
-     
19 2
 <tr>
20 3
 
21 4
          <td>CZ</td>
... ...
@@ -118,23 +101,6 @@
118 101
     <td> - </td>
119 102
 </tr>
120 103
      
121
-<tr>
122
-
123
-         <td>LT</td>
124
-
125
-         <td></td>
126
-
127
-         <td>Up to date</td>
128
-
129
-    <td> - </td>
130
-    <td><a href="http://tor.vesta.nu/dist/">http</a></td>
131
-    <td><a href="http://tor.vesta.nu/">http</a></td>
132
-    <td> - </td>
133
-    <td> - </td>
134
-    <td> - </td>
135
-    <td> - </td>
136
-</tr>
137
-     
138 104
 <tr>
139 105
 
140 106
          <td>DE</td>
... ...
@@ -339,40 +305,6 @@
339 305
     <td> - </td>
340 306
 </tr>
341 307
      
342
-<tr>
343
-
344
-         <td>US</td>
345
-
346
-         <td>AskApache</td>
347
-
348
-         <td>Up to date</td>
349
-
350
-    <td> - </td>
351
-    <td><a href="http://tor.askapache.com/dist/">http</a></td>
352
-    <td><a href="http://tor.askapache.com/">http</a></td>
353
-    <td> - </td>
354
-    <td> - </td>
355
-    <td> - </td>
356
-    <td> - </td>
357
-</tr>
358
-     
359
-<tr>
360
-
361
-         <td>DE</td>
362
-
363
-         <td>Netzkonstrukt Berlin</td>
364
-
365
-         <td>Up to date</td>
366
-
367
-    <td> - </td>
368
-    <td><a href="http://mirror.ntzk.de/torproject.org/dist/">http</a></td>
369
-    <td><a href="http://mirror.ntzk.de/torproject.org/">http</a></td>
370
-    <td> - </td>
371
-    <td> - </td>
372
-    <td> - </td>
373
-    <td> - </td>
374
-</tr>
375
-     
376 308
 <tr>
377 309
 
378 310
          <td>RU</td>
... ...
@@ -443,34 +375,34 @@
443 375
      
444 376
 <tr>
445 377
 
446
-         <td>AT</td>
378
+         <td>INT</td>
447 379
 
448
-         <td>TechAsk.IT</td>
380
+         <td>CoralCDN</td>
449 381
 
450 382
          <td>Up to date</td>
451 383
 
452 384
     <td> - </td>
453
-    <td><a href="http://www.unicorncloud.org/public/torproject.org/dist">http</a></td>
454
-    <td><a href="http://www.unicorncloud.org/public/torproject.org/">http</a></td>
455
-    <td><a href="https://www.unicorncloud.org/public/torproject.org/dist">https</a></td>
456
-    <td><a href="https://www.unicorncloud.org/public/torproject.org/">https</a></td>
385
+    <td><a href="http://www.torproject.org.nyud.net/dist/">http</a></td>
386
+    <td><a href="http://www.torproject.org.nyud.net/">http</a></td>
387
+    <td> - </td>
388
+    <td> - </td>
457 389
     <td> - </td>
458 390
     <td> - </td>
459 391
 </tr>
460 392
      
461 393
 <tr>
462 394
 
463
-         <td>NL</td>
395
+         <td>AT</td>
464 396
 
465
-         <td>Amorphis</td>
397
+         <td>TechAsk.IT</td>
466 398
 
467 399
          <td>Up to date</td>
468 400
 
469 401
     <td> - </td>
470
-    <td><a href="http://tor.amorphis.eu/dist/">http</a></td>
471
-    <td><a href="http://tor.amorphis.eu/">http</a></td>
472
-    <td> - </td>
473
-    <td> - </td>
402
+    <td><a href="http://www.unicorncloud.org/public/torproject.org/dist">http</a></td>
403
+    <td><a href="http://www.unicorncloud.org/public/torproject.org/">http</a></td>
404
+    <td><a href="https://www.unicorncloud.org/public/torproject.org/dist">https</a></td>
405
+    <td><a href="https://www.unicorncloud.org/public/torproject.org/">https</a></td>
474 406
     <td> - </td>
475 407
     <td> - </td>
476 408
 </tr>
... ...
@@ -628,6 +560,23 @@
628 560
     <td> - </td>
629 561
 </tr>
630 562
      
563
+<tr>
564
+
565
+         <td>MX</td>
566
+
567
+         <td></td>
568
+
569
+         <td>Up to date</td>
570
+
571
+    <td> - </td>
572
+    <td><a href="http://fbnaia.homelinux.net/torproject/dist/">http</a></td>
573
+    <td><a href="http://fbnaia.homelinux.net/torproject/">http</a></td>
574
+    <td><a href="https://fbnaia.homelinux.net/torproject//dist/">https</a></td>
575
+    <td><a href="https://fbnaia.homelinux.net/torproject/">https</a></td>
576
+    <td> - </td>
577
+    <td> - </td>
578
+</tr>
579
+     
631 580
 <tr>
632 581
 
633 582
          <td>NL</td>
... ...
@@ -834,103 +783,18 @@
834 783
      
835 784
 <tr>
836 785
 
837
-         <td>IN</td>
838
-
839
-         <td>India Tor Fans</td>
840
-
841
-         <td>DO NOT USE. Out of date.</td>
842
-
843
-    <td> - </td>
844
-    <td><a href="http://www.torproject.org.in/dist/">http</a></td>
845
-    <td><a href="http://www.torproject.org.in/">http</a></td>
846
-    <td> - </td>
847
-    <td> - </td>
848
-    <td> - </td>
849
-    <td> - </td>
850
-</tr>
851
-     
852
-<tr>
853
-
854
-         <td>TN</td>
855
-
856
-         <td></td>
857
-
858
-         <td>DO NOT USE. Out of date.</td>
859
-
860
-    <td> - </td>
861
-    <td> - </td>
862
-    <td><a href="http://torproject.antagonism.org/">http</a></td>
863
-    <td> - </td>
864
-    <td><a href="https://torproject.antagonism.org/">https</a></td>
865
-    <td> - </td>
866
-    <td> - </td>
867
-</tr>
868
-     
869
-<tr>
870
-
871
-         <td>US</td>
872
-
873
-         <td>searchprivate</td>
874
-
875
-         <td>DO NOT USE. Out of date.</td>
876
-
877
-    <td> - </td>
878
-    <td><a href="http://tor.searchprivate.com/dist/">http</a></td>
879
-    <td><a href="http://tor.searchprivate.com/">http</a></td>
880
-    <td> - </td>
881
-    <td> - </td>
882
-    <td> - </td>
883
-    <td> - </td>
884
-</tr>
885
-     
886
-<tr>
887
-
888
-         <td>DE</td>
889
-
890
-         <td></td>
891
-
892
-         <td>Unknown</td>
893
-
894
-    <td> - </td>
895
-    <td><a href="http://tor.taskserver.de/dist/">http</a></td>
896
-    <td><a href="http://tor.taskserver.de/">http</a></td>
897
-    <td> - </td>
898
-    <td> - </td>
899
-    <td> - </td>
900
-    <td> - </td>
901
-</tr>
902
-     
903
-<tr>
904
-
905
-         <td>US</td>
786
+         <td>LT</td>
906 787
 
907 788
          <td></td>
908 789
 
909
-         <td>Unknown</td>
790
+         <td>Up to date</td>
910 791
 
911 792
     <td> - </td>
912
-    <td><a href="http://tor.mirrors.whitedholdings.org/dist/">http</a></td>
913
-    <td><a href="http://tor.mirrors.whitedholdings.org/">http</a></td>
914
-    <td><a href="https://tor.mirrors.whitedholdings.org/dist/">https</a></td>
915
-    <td><a href="https://tor.mirrors.whitedholdings.org/">https</a></td>
916
-    <td><a href="rsync://tor.mirrors.whitedholdings.org/tor-dist">rsync</a></td>
917
-    <td><a href="rsync://mirrors.whitedholdings.org/tor">rsync</a></td>
918
-</tr>
919
-     
920
-<tr>
921
-
922
-         <td>US</td>
923
-
924
-         <td></td>
925
-
926
-         <td>Unknown</td>
927
-
928
-    <td><a href="ftp://mirrors.go-parts.com/tor/">ftp</a></td>
929
-    <td><a href="http://mirrors.go-parts.com/tor/dist/">http</a></td>
793
+    <td><a href="http://tor.vesta.nu/dist/">http</a></td>
794
+    <td><a href="http://tor.vesta.nu/">http</a></td>
930 795
     <td> - </td>
931 796
     <td> - </td>
932 797
     <td> - </td>
933
-    <td><a href="rsync://mirrors.go-parts.com/mirrors/tor/">rsync</a></td>
934 798
     <td> - </td>
935 799
 </tr>
936 800
      
... ...
@@ -938,236 +802,49 @@
938 802
 
939 803
          <td>US</td>
940 804
 
941
-         <td></td>
805
+         <td>AskApache</td>
942 806
 
943
-         <td>Unknown</td>
807
+         <td>Up to date</td>
944 808
 
945 809
     <td> - </td>
946
-    <td><a href="http://tor.loritsu.com/dist/">http</a></td>
947
-    <td><a href="http://tor.loritsu.com/">http</a></td>
948
-    <td> - </td>
949
-    <td> - </td>
950
-    <td> - </td>
810
+    <td><a href="http://tor.askapache.com/dist/">http</a></td>
811
+    <td><a href="http://tor.askapache.com/">http</a></td>
951 812
     <td> - </td>
952
-</tr>
953
-     
954
-<tr>
955
-
956
-         <td>US</td>
957
-
958
-         <td>NW Linux</td>
959
-
960
-         <td>Unknown</td>
961
-
962 813
     <td> - </td>
963
-    <td><a href="http://torproject.nwlinux.us/dist/">http</a></td>
964
-    <td><a href="http://torproject.nwlinux.us/">http</a></td>
965 814
     <td> - </td>
966 815
     <td> - </td>
967
-    <td><a href="rsync://nwlinux.us/tor-dist">rsync</a></td>
968
-    <td><a href="rsync://nwlinux.us/tor-web">rsync</a></td>
969 816
 </tr>
970 817
      
971 818
 <tr>
972 819
 
973 820
          <td>NL</td>
974 821
 
975
-         <td></td>
976
-
977
-         <td>DO NOT USE. Out of date.</td>
978
-
979
-    <td> - </td>
980
-    <td> - </td>
981
-    <td> - </td>
982
-    <td><a href="https://www.coevoet.nl/tor/dist/">https</a></td>
983
-    <td> - </td>
984
-    <td> - </td>
985
-    <td> - </td>
986
-</tr>
987
-     
988
-<tr>
989
-
990
-         <td>LU</td>
991
-
992
-         <td></td>
993
-
994
-         <td>Unknown</td>
995
-
996
-    <td> - </td>
997
-    <td><a href="http://torproject.adamas.ai/dist/">http</a></td>
998
-    <td><a href="http://torproject.adamas.ai/">http</a></td>
999
-    <td> - </td>
1000
-    <td> - </td>
1001
-    <td> - </td>
1002
-    <td> - </td>
1003
-</tr>
1004
-     
1005
-<tr>
1006
-
1007
-         <td>FR</td>
1008
-
1009
-         <td>LazyTiger</td>
822
+         <td>Amorphis</td>
1010 823
 
1011
-         <td>Unknown</td>
824
+         <td>Up to date</td>
1012 825
 
1013 826
     <td> - </td>
1014
-    <td><a href="http://tor.taiga-san.net/dist/">http</a></td>
1015
-    <td><a href="http://tor.taiga-san.net/">http</a></td>
1016
-    <td> - </td>
1017
-    <td> - </td>
1018
-    <td> - </td>
827
+    <td><a href="http://tor.amorphis.eu/dist/">http</a></td>
828
+    <td><a href="http://tor.amorphis.eu/">http</a></td>
1019 829
     <td> - </td>
1020
-</tr>
1021
-     
1022
-<tr>
1023
-
1024
-         <td>DE</td>
1025
-
1026
-         <td>[[:bbs:]]</td>
1027
-
1028
-         <td>Unknown</td>
1029
-
1030 830
     <td> - </td>
1031
-    <td><a href="http://tor.blingblingsquad.net/dist/">http</a></td>
1032
-    <td><a href="http://tor.blingblingsquad.net/">http</a></td>
1033
-    <td><a href="https://tor.blingblingsquad.net/dist/">https</a></td>
1034
-    <td><a href="https://tor.blingblingsquad.net/">https</a></td>
1035 831
     <td> - </td>
1036 832
     <td> - </td>
1037 833
 </tr>
1038 834
      
1039 835
 <tr>
1040 836
 
1041
-         <td>US</td>
837
+         <td>IN</td>
1042 838
 
1043
-         <td></td>
839
+         <td>India Tor Fans</td>
1044 840
 
1045 841
          <td>DO NOT USE. Out of date.</td>
1046 842
 
1047 843
     <td> - </td>
1048
-    <td><a href="http://www.netgull.com/torproject/">http</a></td>
1049
-    <td> - </td>
1050
-    <td> - </td>
1051
-    <td> - </td>
1052
-    <td> - </td>
1053
-    <td> - </td>
1054
-</tr>
1055
-     
1056
-<tr>
1057
-
1058
-         <td>US</td>
1059
-
1060
-         <td></td>
1061
-
1062
-         <td>Unknown</td>
1063
-
1064
-    <td> - </td>
1065
-    <td><a href="http://tor.minibofh.org/dist/">http</a></td>
1066
-    <td><a href="http://tor.minibofh.org/">http</a></td>
1067
-    <td> - </td>
1068
-    <td> - </td>
1069
-    <td> - </td>
1070
-    <td> - </td>
1071
-</tr>
1072
-     
1073
-<tr>
1074
-
1075
-         <td>FR</td>
1076
-
1077
-         <td></td>
1078
-
1079
-         <td>Unknown</td>
1080
-
1081
-    <td> - </td>
1082
-    <td><a href="http://37.187.0.127/tormirror/dist/">http</a></td>
1083
-    <td><a href="http://37.187.0.127/tormirror/">http</a></td>
1084
-    <td> - </td>
1085
-    <td> - </td>
1086
-    <td> - </td>
1087
-    <td> - </td>
1088
-</tr>
1089
-     
1090
-<tr>
1091
-
1092
-         <td>MX</td>
1093
-
1094
-         <td></td>
1095
-
1096
-         <td>Unknown</td>
1097
-
1098
-    <td> - </td>
1099
-    <td><a href="http://fbnaia.homelinux.net/torproject/dist/">http</a></td>
1100
-    <td><a href="http://fbnaia.homelinux.net/torproject/">http</a></td>
1101
-    <td><a href="https://fbnaia.homelinux.net/torproject//dist/">https</a></td>
1102
-    <td><a href="https://fbnaia.homelinux.net/torproject/">https</a></td>
1103
-    <td> - </td>
1104
-    <td> - </td>
1105
-</tr>
1106
-     
1107
-<tr>
1108
-
1109
-         <td>EE</td>
1110
-
1111
-         <td></td>
1112
-
1113
-         <td>Unknown</td>
1114
-
1115
-    <td> - </td>
1116
-    <td><a href="http://tor.li/dist/">http</a></td>
1117
-    <td><a href="http://tor.li/">http</a></td>
1118
-    <td><a href="https://tor.li/dist/">https</a></td>
1119
-    <td><a href="https://tor.li/">https</a></td>
1120
-    <td> - </td>
1121
-    <td> - </td>
1122
-</tr>
1123
-     
1124
-<tr>
1125
-
1126
-         <td>UA</td>
1127
-
1128
-         <td></td>
1129
-
1130
-         <td>Unknown</td>
1131
-
1132
-    <td> - </td>
1133
-    <td><a href="http://torua.reactor-xg.kiev.ua/dist/">http</a></td>
1134
-    <td><a href="http://torua.reactor-xg.kiev.ua/">http</a></td>
1135
-    <td> - </td>
1136
-    <td> - </td>
1137
-    <td> - </td>
1138
-    <td> - </td>
1139
-</tr>
1140
-     
1141
-<tr>
1142
-
1143
-         <td>UK</td>
1144
-
1145
-         <td></td>
1146
-
1147
-         <td>Unknown</td>
1148
-
1149
-    <td> - </td>
1150
-    <td><a href="http://tor.mage.me.uk/dist/">http</a></td>
1151
-    <td><a href="http://tor.mage.me.uk/">http</a></td>
1152
-    <td> - </td>
1153
-    <td> - </td>
1154
-    <td> - </td>
844
+    <td><a href="http://www.torproject.org.in/dist/">http</a></td>
845
+    <td><a href="http://www.torproject.org.in/">http</a></td>
1155 846
     <td> - </td>
1156
-</tr>
1157
-     
1158
-<tr>
1159
-
1160
-         <td>DE</td>
1161
-
1162
-         <td>tor-mirror.de</td>
1163
-
1164
-         <td>Unknown</td>
1165
-
1166 847
     <td> - </td>
1167
-    <td><a href="http://tor-mirror.de/dist/">http</a></td>
1168
-    <td><a href="http://tor-mirror.de/">http</a></td>
1169
-    <td><a href="https://tor-mirror.de/dist/">https</a></td>
1170
-    <td><a href="https://tor-mirror.de/">https</a></td>
1171 848
     <td> - </td>
1172 849
     <td> - </td>
1173 850
 </tr>
... ...
@@ -1,23 +1,16 @@
1 1
 #!/usr/bin/perl -w
2 2
 use warnings;
3 3
 use strict;
4
+use Data::Dumper;
4 5
 use LWP::Simple;
6
+use HTML::LinkExtor;
5 7
 use LWP;
6 8
 use Date::Parse;
7 9
 use Date::Format;
10
+use Digest::SHA qw(sha256_hex);
8 11
 
9
-#
10
-# A quick hack by Jacob Appelbaum <jacob@appelbaum.net>
11
-# LWP suggestions by Leigh Honeywell
12 12
 # This is Free Software (GPLv3)
13 13
 # http://www.gnu.org/licenses/gpl-3.0.txt
14
-#
15
-# CHANGELOG
16
-# 20091003 Code changes to elimiate the need for a trailing slash in addresses for script runtime
17
-# 20091004 Code changes to increase out of date tolerance to 48 hours
18
-# 20091028 Code changes to increase timout to 30 seconds (attempting to # resolve "unknown" status')
19
-# 20091028 Code changes to change user agent of script
20
-# 20100807 Remove dead mirrors.
21 14
 
22 15
 print "Creating LWP agent ($LWP::VERSION)...\n";
23 16
 my $lua = LWP::UserAgent->new(
... ...
@@ -37,45 +30,71 @@ sub sanitize {
37 30
 
38 31
     return $cleanedData;
39 32
 }
33
+sub ExtractLinks {
34
+    my $content = shift; 
35
+    my $url     = shift;
36
+    my @links;
40 37
 
41
-sub FetchDate {
42
-    my $url = shift; # Base url for mirror
43
-    my $trace = "project/trace/www-master.torproject.org"; # this file should always exist
44
-    $url = "$url/$trace";
38
+    my $parser = HTML::LinkExtor->new(undef, $url);
39
+    $parser->parse($content);
40
+    foreach my $linkarray($parser->links)
41
+    {
42
+         my ($elt_type, $attr_name, $attr_value) = @$linkarray;
43
+         if ($elt_type eq 'a' && $attr_name eq 'href' && $attr_value =~ /\/$/ && $attr_value =~ /^$url/)
44
+         {
45
+         	push @links, Fetch($attr_value, \&ExtractLinks);
46
+         }
47
+	 elsif ($attr_value =~ /\.(xpi|dmg|exe|tar\.gz)$/)
48
+	 #elsif ($attr_value =~ /\.(asc)$/)
49
+         {
50
+         	push @links, $attr_value;
51
+         }
52
+    }
53
+    return @links;
54
+}
55
+
56
+sub ExtractDate {
57
+    my $content = shift;  
58
+    $content    = sanitize($content);
59
+    my $date    = str2time($content);
60
+
61
+    print "Extracting possible date from: $content\n";
62
+    if ($date) {
63
+        print "We've fetched a date $date.\n";
64
+        return $date;
65
+    } else {
66
+        print "We haven't fetched a date.\n";
67
+	return undef;
68
+    }
69
+}
45 70
 
46
-    print "Fetching possible date from: $url\n";
71
+sub ExtractSig {
72
+    my $content = shift;
73
+    return sha256_hex($content); 
74
+}
75
+
76
+sub Fetch {
77
+    my ($url, $sub) = @_; # Base url for mirror
78
+    print "Fetch $url\n";
47 79
 
48 80
     my $request = new HTTP::Request GET => "$url";
49 81
     my $result = $lua->request($request);
50 82
     my $code = $result->code();
51
-    print "Result code $code\n";
83
+    print "\tResult code $code\n";
52 84
 
53 85
     if ($result->is_success && $code eq "200"){
54
-       my $taint = $result->content;
55
-       my $content = sanitize($taint);
86
+       my $content = $result->content;
56 87
        if ($content) {
57
-
58
-            my $date = str2time($content);
59
-
60
-            if ($date) {
61
-                print "We've fetched a date $date.\n";
62
-                return $date;
63
-            } else {
64
-                print "We've haven't fetched a date.\n";
65
-                return "Unknown";
66
-            }
67
-
88
+	    return $sub->($content, $url);
68 89
         } else {
69
-            print "Unable to fetch date, empty content returned.\n";
70
-            return "Unknown";
90
+            print "Unable to fetch $url, empty content returned.\n";
71 91
         }
72 92
 
73 93
     } else {
74 94
        print "Our request failed, we had no result.\n";
75
-       return "Unknown";
76 95
     }
77 96
 
78
-    return "Unknown";
97
+    return undef;
79 98
 }
80 99
 
81 100
 # This is the list of all known Tor mirrors
... ...
@@ -1349,33 +1368,73 @@ my $count = values %m;
1349 1368
 print "We have a total of $count mirrors\n";
1350 1369
 print "Fetching the last updated date for each mirror.\n";
1351 1370
 
1352
-my $tortime;
1353
-$tortime = FetchDate("https://www.torproject.org/");
1371
+my $tortime = Fetch("https://www.torproject.org/project/trace/www-master.torproject.org", \&ExtractDate);
1372
+my @torfiles = Fetch("https://www.torproject.org/dist/", \&ExtractLinks); 
1373
+my %randomtorfiles;
1374
+
1375
+for (1 .. 1)
1376
+{
1377
+	my $r = int(rand(scalar(@torfiles)));
1378
+	my $suffix = $torfiles[$r];
1379
+	$suffix =~ s/^https:\/\/www.torproject.org//;
1380
+	$randomtorfiles{$suffix} = Fetch($torfiles[$r], \&ExtractSig);
1381
+}
1382
+
1383
+print "Using these files for sig matching:\n";
1384
+print join("\n", keys %randomtorfiles);
1385
+
1354 1386
 # Adjust offical Tor time by out-of-date offset: number of days * seconds per day
1355 1387
 $tortime -= 1 * 172800;
1356 1388
 print "The official time for Tor is $tortime. \n";
1389
+my %todelete;
1357 1390
 
1358 1391
 foreach my $server ( keys %m ) {
1359 1392
 
1360
-    print "Attempting to fetch from $m{$server}{'orgName'}\n";
1393
+    print "Attempting to fetch from $server: $m{$server}{'orgName'}\n";
1361 1394
 
1362
-    if ($m{$server}{'httpWebsiteMirror'}) {
1363
-        print "Attempt to fetch via HTTP.\n";
1364
-        $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpWebsiteMirror'}");
1365
-    } elsif ($m{$server}{'httpsWebsiteMirror'}) {
1366
-        print "Attempt to fetch via HTTPS.\n";
1367
-        $m{$server}{"updateDate"} = FetchDate("$m{$server}{'httpsWebsiteMirror'}");
1368
-    } elsif ($m{$server}{'ftpWebsiteMirror'}) {
1369
-        print "Attempt to fetch via FTP.\n";
1370
-        $m{$server}{"updateDate"} = FetchDate("$m{$server}{'ftpWebsiteMirror'}");
1371
-    } else {
1372
-        print "We were unable to fetch or store anything. We still have the following: $m{$server}{'updateDate'}\n";
1395
+    foreach my $serverType('httpWebsiteMirror', 'httpsWebsiteMirror', 'ftpWebsiteMirror')
1396
+    {
1397
+        if ($m{$server}{$serverType}) {
1398
+            print "Attempt to fetch via $serverType.\n";
1399
+            $m{$server}{"updateDate"} = Fetch("$m{$server}{$serverType}/project/trace/www-master.torproject.org", \&ExtractDate);
1400
+            if ($m{$server}{updateDate}) {
1401
+                foreach my $randomtorfile(keys %randomtorfiles) {
1402
+                	my $sig = Fetch("$m{$server}{$serverType}/$randomtorfile", \&ExtractSig);
1403
+            		if (!$sig) {
1404
+			    $todelete{$server} = "Unreadable $randomtorfile";
1405
+		        } elsif ($sig ne $randomtorfiles{$randomtorfile}) {
1406
+			    $todelete{$server} = "Sig mismatch on $randomtorfile";
1407
+            		} else {
1408
+        	    	    print "Sig $sig matches for $randomtorfile on $m{$server}{$serverType}\n";
1409
+			}
1410
+            	        last;
1411
+		}
1412
+            } else {
1413
+		delete $m{$server}{updateDate};
1414
+		$todelete{$server} = "Unreadable date";
1415
+            }
1416
+	    last;
1417
+        } 
1373 1418
     }
1374
-
1375
-    print "We fetched and stored the following: $m{$server}{'updateDate'}\n";
1376
-
1419
+    if (exists $m{$server}{updateDate}) {
1420
+        print "We fetched and stored the following: $m{$server}{'updateDate'}\n";
1421
+    } else {
1422
+	$todelete{$server} = "Unreadable date";
1423
+    } 
1377 1424
  }
1378 1425
 
1426
+foreach my $outdated(keys %todelete)
1427
+{
1428
+    print "Deleted $outdated due to $todelete{$outdated}\n";
1429
+    print Dumper($m{$outdated});
1430
+    delete  $m{$outdated};
1431
+}
1432
+
1433
+foreach my $k(keys %m)
1434
+{
1435
+    print "$k\n";
1436
+    print "$m{$k}{updateDate}\n";
1437
+}
1379 1438
 
1380 1439
 print "We sorted the following mirrors by their date of last update: \n";
1381 1440
 foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
... ...
@@ -1394,7 +1453,7 @@ my $html;
1394 1453
 open(OUT, "> $outFile") or die "Can't open $outFile: $!";
1395 1454
 
1396 1455
 # Here's where we open a file and print some wml include goodness
1397
-# This is storted from last known recent update to unknown update times
1456
+# This is sorted from last known recent update to unknown update times
1398 1457
 foreach my $server ( sort { $m{$b}{'updateDate'} <=> $m{$a}{'updateDate'}} keys %m ) {
1399 1458
 
1400 1459
      my $time;