update README, upload analysis scripts

README.md

@@ -1,2 +1,9 @@
-# censorwatch
-Website for Censorwatch, CIS' project to map net neutrality violations and web censorship in India
+# CensorWatch
+
+This repository hosts the website for CensorWatch, CIS' project to map net neutrality violations and web censorship in India, the data collected through the project and the scripts used to analyse the data
+
+To download the data, visit the Releases section of this repository.
+
+To import the data (using Mongo DB), extract the zip archive and inside the directory run `mongorestore --db log_database`.
+
+To analyse the data (using R), see the `analysis_scripts` directory. `connect.R` and `query1.R` are useful starting points.

analysis_scripts/confirm_DNS_blocks.R

@@ -0,0 +1,32 @@
+censorious_DNS_servers <- c("203.109.71.154", "123.176.40.68", "106.51.113.17", "123.176.40.69", "49.207.46.38", "123.176.40.67", "49.207.46.62", "202.83.21.15", "49.205.75.6", "202.83.24.75", "202.83.21.14", "218.248.112.60")
+
+for (server in censorious_DNS_servers) {
+  query_with_parameters <- paste('{ "result.test_result.resolved_ip": "', server, '"}', sep = "")
+  responses <- dnsprobe$find(query = query_with_parameters, fields='{"_id": 1}')
+
+  for (response in responses$'_id') {
+    query_with_parameters <- paste('{"_id": { "$oid" : "', response, '" } }', sep = "")
+    updates <- dnsprobe$update(query = query_with_parameters, update = '{ "$set" : { "confirmed_block" : "True"} }')
+  }
+}
+
+
+mongo$update(
+  query = paste0('{"_id": { "$oid" : "', mongoID, '" } }'),
+  update = '{ "$set" : { "confirmed_block" : "True"} }'
+)
+
+
+bad_measurements <- c('AS133997','AS132559','AS132976', 'AS133287',  'AS134177',  'AS134293', 'AS134674' , 'AS135690', 'AS136305',  'AS138277', 'AS139567', 'AS45235', 'AS58965', 'AS133720')
+for (asn in bad_measurements) {
+  query_with_parameters <- paste('{ "ip_info.asn.asn": "', asn, '"}', sep = "")
+  responses <- tlsprobe$find(query = query_with_parameters, fields='{"_id": 1}')
+  if (length(responses) == 0) {
+    next
+  }
+
+  for (response in responses$'_id') {
+    query_with_parameters <- paste('{"_id": { "$oid" : "', response, '" } }', sep = "")
+    updates <- tlsprobe$update(query = query_with_parameters, update = '{ "$set" : { "invalid" : "True"} }')
+  }
+}

analysis_scripts/connect.R

@@ -0,0 +1,4 @@
+connection_string = 'mongodb://localhost:27017/?readPreference=primary&appname=MongoDB%20Compass&ssl=false'
+dnsprobe = mongo(collection="dnsprobe", db="log_database", url=connection_string)
+tlsprobe = mongo(collection="tlsprobe", db="log_database", url=connection_string)
+httpprobe = mongo(collection="http2probe", db="log_database", url=connection_string)

analysis_scripts/final_table.r

@@ -0,0 +1,151 @@
+httpHostsBlockedPerASN <- httpprobe$aggregate('[{"$group":{"_id": {"host":"$hostname", "asn_number": "$ip_info.asn.asn"}, "measurements": { "$sum": 1 }, "Num_error": { "$sum": { "$cond": [ {"$ifNull": ["$error", false]}, 1, 0 ] }}, "Num_blocked": { "$sum": { "$cond": [ { "$eq": [ "$result.test_result.censored", "True" ] }, 1, 0 ] }} }}]')
+dnsHostsBlockedPerASN <- dnsprobe$aggregate('[{"$group":{"_id": {"host":"$hostname", "asn_number": "$ip_info.asn.asn"}, "measurements": { "$sum": 1 }, "Num_error": { "$sum": { "$cond": [ {"$ifNull": ["$sourceIp", false]}, 1, 0 ] }}, "Num_unknown": { "$sum": { "$cond": [ { "$eq": [ "$result.test_result.censored", "Unknown" ] }, 1, 0 ] }} ,"Num_blocked": { "$sum": { "$cond": [ { "$eq": [ "$confirmed_block", "True" ] }, 1, 0 ] }} }}]')
+tlsHostsBlockedPerASN <- tlsprobe$aggregate('[{"$group":{"_id": {"host":"$sniHostname", "asn_number": "$ip_info.asn.asn"}, "measurements": { "$sum": 1 }, "Num_error": { "$sum": { "$cond": [ {"$ifNull": ["$error1", false]}, 1, 0 ] }}, "Num_blocked": { "$sum": { "$cond": [ { "$eq": [ "$result.test_result.censored", "True" ] }, 1, 0 ] }} }}]')
+
+dnsprobe_unique_AS <- dnsprobe$distinct("ip_info.asn.asn")
+httpprobe_unique_AS <- httpprobe$distinct("ip_info.asn.asn")
+tlsprobe_unique_AS <- tlsprobe$distinct("ip_info.asn.asn")
+
+httpprobe_hosts <- httpprobe$distinct("hostname")
+
+httpprobe_AS_number_name_map <- httpprobe$aggregate('[{"$group":{"_id":"$ip_info.asn.asn", "AS_name": { "$first" : "$ip_info.asn.name" }}}]')
+
+unique_ASNs <- union( union(httpprobe_unique_AS, dnsprobe_unique_AS), tlsprobe_unique_AS)
+ASNs_to_consider <- unique_ASNs[!unique_ASNs %in% bad_measurements]
+
+Hosts_to_consider <- httpprobe_hosts[!httpprobe_hosts %in% bad_websites]
+Hosts_to_consider <- Hosts_to_consider[!Hosts_to_consider %in% high_error_websites]
+
+empty_list <- rep(0, length(ASNs_to_consider))
+results3 <- data.frame(asn = ASNs_to_consider, asn_name = empty_list, measurements = empty_list, number_of_blocked_sites = empty_list, number_of_inconclusive = empty_list, number_of_unmeasured_sites = empty_list)
+
+#for (host in Hosts_to_consider) {
+#  if (is.null(results3[[host]])) {
+#    results3[[host]] = data.frame(measurements = empty_list, errors = empty_list, blanks = empty_list, blocklist = empty_list)
+#  }
+#}
+net_blocklist <- c()
+
+for(x in 1:length(ASNs_to_consider)) {
+  asn <- ASNs_to_consider[x]
+
+  blocklist <- c()
+  dns_hosts_measured <- c()
+  tls_hosts_measured <- c()
+  http_hosts_measured <- c()
+
+  tls_hosts_error <- c()
+  dns_hosts_error <- c()
+  http_hosts_error <- c()
+
+  dns_blocks <- 0
+  tls_blocks <- 0
+  http_blocks <- 0
+
+  dns_measurements <- 0
+  tls_measurements <- 0
+  http_measurements <- 0
+
+  dns_errors <- c()
+  tls_errors <- c()
+  http_errors <- c()
+
+  dns_blanks <- 0
+  tls_blanks <- 0
+  http_blanks <- 0
+
+  indexes <-  which(dnsHostsBlockedPerASN$"_id"$asn_number == asn)
+  #cat("here ", length(indexes), "\n")
+  for(index in indexes) {
+    host <- dnsHostsBlockedPerASN$"_id"$host[index]
+    #cat("here2 ", host, "\n")
+
+    if(!(host %in% Hosts_to_consider)) {
+      next
+    }
+
+    if(dnsHostsBlockedPerASN$measurements[index] != dnsHostsBlockedPerASN$Num_unknown[index]) {
+      dns_hosts_measured <- append(dns_hosts_measured, host)
+    }
+
+    if(dnsHostsBlockedPerASN$measurements[index] == dnsHostsBlockedPerASN$Num_error[index]) {
+      dns_hosts_error <- append(dns_hosts_error, host)
+    }
+
+    dns_measurements <- dns_measurements + dnsHostsBlockedPerASN$measurements[index]
+
+    if (dnsHostsBlockedPerASN$Num_blocked[index] >= 1) {
+        blocklist <- append(blocklist, host)
+    }
+
+    #cat("here2 measure: ", dnsHostsBlockedPerASN$measurements[index], " blocks ", dnsHostsBlockedPerASN$Num_blocked[index], "\n")
+
+  }
+  dns_blanks <- setdiff(Hosts_to_consider, dns_hosts_measured)
+  #dns_errors <- setdiff(Hosts_to_consider, dns_hosts_error)
+  #dns_hosts_error <- dns_hosts_error[!dns_hosts_error %in% dns_hosts_measured]
+
+
+  indexes <-  which(tlsHostsBlockedPerASN$"_id"$asn_number == asn)
+  for(index in indexes) {
+    host <- tlsHostsBlockedPerASN$"_id"$host[index]
+
+    if(!(host %in% Hosts_to_consider)) {
+      next
+    }  
+
+    tls_hosts_measured <- append(tls_hosts_measured, host)
+
+    if(tlsHostsBlockedPerASN$measurements[index] == tlsHostsBlockedPerASN$Num_error[index]) {
+      tls_hosts_error <- append(tls_hosts_error, host)
+    }
+
+    tls_measurements <- tls_measurements + tlsHostsBlockedPerASN$measurements[index]
+
+    if (tlsHostsBlockedPerASN$Num_blocked[index] >= 1) {
+      blocklist <- append(blocklist, host)
+    }
+  }
+  tls_blanks <- setdiff(Hosts_to_consider, tls_hosts_measured)
+
+  indexes <-  which(httpHostsBlockedPerASN$"_id"$asn_number == asn)
+  for(index in indexes) {
+    host <- httpHostsBlockedPerASN$"_id"$host[index]
+
+    if(!(host %in% Hosts_to_consider)) {
+      next
+    } 
+
+    http_hosts_measured <- append(http_hosts_measured, host)
+
+    if(httpHostsBlockedPerASN$measurements[index] == httpHostsBlockedPerASN$Num_error[index]) {
+      http_hosts_error <- append(http_hosts_error, host)
+    }
+
+    http_measurements <- http_measurements + httpHostsBlockedPerASN$measurements[index]
+
+    if (httpHostsBlockedPerASN$Num_blocked[index] >= 1) {
+      blocklist <- append(blocklist, host)
+    }
+  }
+  http_blanks <- setdiff(Hosts_to_consider, http_hosts_measured)
+  #http_hosts_error <- http_hosts_error[!http_hosts_error %in% http_hosts_measured]
+  #http_errors <- setdiff(Hosts_to_consider, http_hosts_error)
+
+ # cat("asn:", asn, " measurements: ", (dns_measurements + tls_measurements + http_measurements), " dns_blocks ", dns_blocks, " tls_blocks ", tls_blocks, " http_blocks ", http_blocks, "for host ", host, "\n")
+  blocklist <- unique(blocklist)
+  results3$number_of_blocked_sites[x] <- length(blocklist)
+
+  list_of_inconclusive <- union( union(dns_hosts_error, tls_hosts_error), http_hosts_error) 
+  list_of_inconclusive <- list_of_inconclusive[!list_of_inconclusive %in% blocklist]
+  results3$number_of_inconclusive[x] <- length(list_of_inconclusive)
+
+  list_of_unmeasured <-  union( union(dns_blanks, tls_blanks), http_blanks)
+  list_of_unmeasured <- list_of_unmeasured[!list_of_unmeasured %in% blocklist]
+  results3$number_of_unmeasured_sites[x] <- length(list_of_unmeasured)
+
+  results3$measurements[x] <- (dns_measurements + tls_measurements + http_measurements) / (length(Hosts_to_consider) * 3)
+  results3$asn_name[x] <- httpprobe_AS_number_name_map$AS_name[which(httpprobe_AS_number_name_map$"_id" == asn)]
+
+  net_blocklist <- union(net_blocklist, blocklist)
+}  

analysis_scripts/final_table_regions.r

@@ -0,0 +1,144 @@
+httpHostsBlockedPerState <- httpprobe$aggregate('[{"$match": {"invalid": {"$ne": "True"}}} , {"$group":{"_id": {"host":"$hostname", "region": "$state"}, "measurements": { "$sum": 1 }, "Num_error": { "$sum": { "$cond": [ {"$ifNull": ["$error", false]}, 1, 0 ] }}, "Num_blocked": { "$sum": { "$cond": [ { "$eq": [ "$result.test_result.censored", "True" ] }, 1, 0 ] }} }}]')
+dnsHostsBlockedPerState <- dnsprobe$aggregate('[{"$match": {"invalid": {"$ne": "True"}}} , {"$group":{"_id": {"host":"$hostname", "region": "$state"}, "measurements": { "$sum": 1 }, "Num_error": { "$sum": { "$cond": [ {"$ifNull": ["$error", false]}, 1, 0 ] }}, "Num_unknown": { "$sum": { "$cond": [ { "$eq": [ "$result.test_result.censored", "Unknown" ] }, 1, 0 ] }} , "Num_blocked": { "$sum": { "$cond": [ { "$eq": [ "$confirmed_block", "True" ] }, 1, 0 ] }} }}]')
+tlsHostsBlockedPerState <- tlsprobe$aggregate('[{"$match": {"invalid": {"$ne": "True"}}} , {"$group":{"_id": {"host":"$sniHostname", "region": "$state"}, "measurements": { "$sum": 1 }, "Num_error": { "$sum": { "$cond": [ {"$ifNull": ["$error1", false]}, 1, 0 ] }}, "Num_blocked": { "$sum": { "$cond": [ { "$eq": [ "$result.test_result.censored", "True" ] }, 1, 0 ] }} }}]')
+
+dnsprobe_unique_state <- dnsprobe$distinct("state")
+httpprobe_unique_state <- httpprobe$distinct("state")
+httpprobe_hosts <- httpprobe$distinct("hostname")
+
+unique_ASNs <- union(httpprobe_unique_AS, dnsprobe_unique_AS)
+ASNs_to_consider <- unique_ASNs[!unique_ASNs %in% bad_measurements]
+
+unique_states <- union(httpprobe_unique_state, dnsprobe_unique_state)
+
+Hosts_to_consider <- httpprobe_hosts[!httpprobe_hosts %in% bad_websites]
+Hosts_to_consider <- Hosts_to_consider[!Hosts_to_consider %in% high_error_websites]
+
+empty_list <- rep(0, length(unique_states))
+results4 <- data.frame(state = unique_states, measurements = empty_list, number_of_blocked_sites = empty_list, number_of_inconclusive = empty_list, number_of_unmeasured_sites = empty_list)
+
+#for (host in Hosts_to_consider) {
+#  if (is.null(results4[[host]])) {
+#    results4[[host]] = data.frame(measurements = empty_list, errors = empty_list, blanks = empty_list, blocklist = empty_list)
+#  }
+#}
+
+for(x in 1:length(unique_states)) {
+  state <- unique_states[x]
+
+  blocklist <- c()
+  dns_hosts_measured <- c()
+  tls_hosts_measured <- c()
+  http_hosts_measured <- c()
+
+  tls_hosts_error <- c()
+  dns_hosts_error <- c()
+  http_hosts_error <- c()
+
+  dns_blocks <- 0
+  tls_blocks <- 0
+  http_blocks <- 0
+
+  dns_measurements <- 0
+  tls_measurements <- 0
+  http_measurements <- 0
+
+  dns_blanks <- 0
+  tls_blanks <- 0
+  http_blanks <- 0
+
+  indexes <-  which(dnsHostsBlockedPerState$"_id"$region == state)
+  #cat("here ", length(indexes), "\n")
+  for(index in indexes) {
+    host <- dnsHostsBlockedPerState$"_id"$host[index]
+    #cat("here2 ", host, "\n")
+
+    if(!(host %in% Hosts_to_consider)) {
+      next
+    }
+
+    if(dnsHostsBlockedPerState$measurements[index] != dnsHostsBlockedPerState$Num_unknown[index]) {
+      dns_hosts_measured <- append(dns_hosts_measured, host)
+    }
+
+    if(dnsHostsBlockedPerState$measurements[index] == dnsHostsBlockedPerState$Num_error[index]) {
+      dns_hosts_error <- append(dns_hosts_error, host)
+    }
+
+    dns_measurements <- dns_measurements + dnsHostsBlockedPerState$measurements[index]
+
+    if (dnsHostsBlockedPerState$Num_blocked[index] >= 1) {
+      blocklist <- append(blocklist, host)
+    }
+
+    #cat("here2 measure: ", dnsHostsBlockedPerASN$measurements[index], " blocks ", dnsHostsBlockedPerASN$Num_blocked[index], "\n")
+
+  }
+  dns_blanks <- setdiff(Hosts_to_consider, dns_hosts_measured)
+
+  indexes <-  which(tlsHostsBlockedPerState$"_id"$region == state)
+  for(index in indexes) {
+    host <- tlsHostsBlockedPerState$"_id"$host[index]
+
+    if(!(host %in% Hosts_to_consider)) {
+      next
+    }  
+
+    tls_hosts_measured <- append(tls_hosts_measured, host)
+
+    if(tlsHostsBlockedPerState$measurements[index] == tlsHostsBlockedPerState$Num_error[index]) {
+      tls_hosts_error <- append(tls_hosts_error, host)
+    }
+
+    tls_measurements <- tls_measurements + tlsHostsBlockedPerState$measurements[index]
+
+    if (tlsHostsBlockedPerState$Num_blocked[index] >= 1) {
+      blocklist <- append(blocklist, host)
+    }
+  }
+  tls_blanks <- setdiff(Hosts_to_consider, tls_hosts_measured)
+
+
+  indexes <-  which(httpHostsBlockedPerState$"_id"$region == state)
+  for(index in indexes) {
+    host <- httpHostsBlockedPerState$"_id"$host[index]
+
+    if(!(host %in% Hosts_to_consider)) {
+      next
+    } 
+
+    http_hosts_measured <- append(http_hosts_measured, host)
+
+    if(httpHostsBlockedPerState$measurements[index] == httpHostsBlockedPerState$Num_error[index]) {
+      http_hosts_error <- append(http_hosts_error, host)
+    }
+
+    http_measurements <- http_measurements + httpHostsBlockedPerState$measurements[index]
+
+    if (httpHostsBlockedPerState$Num_blocked[index] >= 1) {
+      blocklist <- append(blocklist, host)
+    }
+  }
+  http_blanks <- setdiff(Hosts_to_consider, http_hosts_measured)
+
+
+  # cat("asn:", asn, " measurements: ", (dns_measurements + tls_measurements + http_measurements), " dns_blocks ", dns_blocks, " tls_blocks ", tls_blocks, " http_blocks ", http_blocks, "for host ", host, "\n")
+  blocklist <- unique(blocklist)
+  results4$number_of_blocked_sites[x] <- length(blocklist)
+
+  list_of_inconclusive <- union( union(dns_hosts_error, tls_hosts_error), http_hosts_error) 
+  list_of_inconclusive <- list_of_inconclusive[!list_of_inconclusive %in% blocklist]
+  results4$number_of_inconclusive[x] <- length(list_of_inconclusive)
+
+  list_of_unmeasured <-  union( union(dns_blanks, tls_blanks), http_blanks)
+  list_of_unmeasured <- list_of_unmeasured[!list_of_unmeasured %in% blocklist]
+  results4$number_of_unmeasured_sites[x] <- length(list_of_unmeasured)
+
+  results4$measurements[x] <- (dns_measurements + tls_measurements + http_measurements) / (length(Hosts_to_consider) * 3)
+}  
+
+results4 <- results4[order(results4$number_of_unmeasured_sites),]
+colnames(results4) <- c("Region", "Readings", "Number of Confirmed\nBlocked Sites", "Number of Sites With\nInconclusive Readings")
+results4$Readings <- round(results4$Readings, digits = 1)
+rownames(results4) <- 1:nrow(results4)
+write.csv(results4, "BlocksByRegion.csv", row.names = F)