From 3220fd8a861bf2982734482c6077a01fb88a61f9 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Wed, 18 Jun 2014 22:42:30 -0400 Subject: [PATCH 1/2] change the list of TLDs to be a Set --- lib/swot.rb | 251 ++++++++++++++++++++++++++-------------------------- 1 file changed, 126 insertions(+), 125 deletions(-) diff --git a/lib/swot.rb b/lib/swot.rb index d8bef27faf..2de4535ffc 100644 --- a/lib/swot.rb +++ b/lib/swot.rb @@ -1,134 +1,135 @@ require 'public_suffix' +require 'set' module Swot VERSION = "0.3.0" # These top-level domains are guaranteed to be academic institutions. - ACADEMIC_TLDS = { - 'ac.ae' => 1, - 'ac.at' => 1, - 'ac.bd' => 1, - 'ac.be' => 1, - 'ac.cr' => 1, - 'ac.cy' => 1, - 'ac.fj' => 1, - 'ac.id' => 1, - 'ac.il' => 1, - 'ac.ir' => 1, - 'ac.jp' => 1, - 'ac.ke' => 1, - 'ac.kr' => 1, - 'ac.ma' => 1, - 'ac.mu' => 1, - 'ac.mw' => 1, - 'ac.mz' => 1, - 'ac.nz' => 1, - 'ac.pa' => 1, - 'ac.pg' => 1, - 'ac.rs' => 1, - 'ac.ru' => 1, - 'ac.rw' => 1, - 'ac.th' => 1, - 'ac.tz' => 1, - 'ac.ug' => 1, - 'ac.uk' => 1, - 'ac.yu' => 1, - 'ac.za' => 1, - 'ac.zm' => 1, - 'ac.zw' => 1, - 'edu' => 1, - 'edu.af' => 1, - 'edu.al' => 1, - 'edu.ar' => 1, - 'edu.au' => 1, - 'edu.az' => 1, - 'edu.ba' => 1, - 'edu.bb' => 1, - 'edu.bd' => 1, - 'edu.bh' => 1, - 'edu.bi' => 1, - 'edu.bn' => 1, - 'edu.bo' => 1, - 'edu.br' => 1, - 'edu.bs' => 1, - 'edu.bt' => 1, - 'edu.bz' => 1, - 'edu.co' => 1, - 'edu.cu' => 1, - 'edu.do' => 1, - 'edu.dz' => 1, - 'edu.ec' => 1, - 'edu.ee' => 1, - 'edu.eg' => 1, - 'edu.er' => 1, - 'edu.es' => 1, - 'edu.et' => 1, - 'edu.ge' => 1, - 'edu.gh' => 1, - 'edu.gr' => 1, - 'edu.gt' => 1, - 'edu.hk' => 1, - 'edu.hn' => 1, - 'edu.ht' => 1, - 'edu.iq' => 1, - 'edu.jm' => 1, - 'edu.jo' => 1, - 'edu.kg' => 1, - 'edu.kh' => 1, - 'edu.kn' => 1, - 'edu.kw' => 1, - 'edu.ky' => 1, - 'edu.kz' => 1, - 'edu.la' => 1, - 'edu.lb' => 1, - 'edu.lv' => 1, - 'edu.ly' => 1, - 'edu.mk' => 1, - 'edu.mm' => 1, - 'edu.mn' => 1, - 'edu.mo' => 1, - 'edu.mt' => 1, - 'edu.mx' => 1, - 'edu.my' => 1, - 'edu.ni' => 1, - 'edu.np' => 1, - 'edu.om' => 1, - 'edu.pa' => 1, - 'edu.pe' => 1, - 'edu.ph' => 1, - 'edu.pk' => 1, - 'edu.pl' => 1, - 'edu.pr' => 1, - 'edu.ps' => 1, - 'edu.pt' => 1, - 'edu.py' => 1, - 'edu.qa' => 1, - 'edu.rs' => 1, - 'edu.ru' => 1, - 'edu.sa' => 1, - 'edu.sd' => 1, - 'edu.sg' => 1, - 'edu.sv' => 1, - 'edu.sy' => 1, - 'edu.tr' => 1, - 'edu.tt' => 1, - 'edu.tw' => 1, - 'edu.ua' => 1, - 'edu.uy' => 1, - 'edu.ve' => 1, - 'edu.vn' => 1, - 'edu.ws' => 1, - 'edu.ye' => 1, - 'edu.zm' => 1, - 'vic.edu.au' => 1, - } + ACADEMIC_TLDS = %w( + ac.ae + ac.at + ac.bd + ac.be + ac.cr + ac.cy + ac.fj + ac.id + ac.il + ac.ir + ac.jp + ac.ke + ac.kr + ac.ma + ac.mu + ac.mw + ac.mz + ac.nz + ac.pa + ac.pg + ac.rs + ac.ru + ac.rw + ac.th + ac.tz + ac.ug + ac.uk + ac.yu + ac.za + ac.zm + ac.zw + edu + edu.af + edu.al + edu.ar + edu.au + edu.az + edu.ba + edu.bb + edu.bd + edu.bh + edu.bi + edu.bn + edu.bo + edu.br + edu.bs + edu.bt + edu.bz + edu.co + edu.cu + edu.do + edu.dz + edu.ec + edu.ee + edu.eg + edu.er + edu.es + edu.et + edu.ge + edu.gh + edu.gr + edu.gt + edu.hk + edu.hn + edu.ht + edu.iq + edu.jm + edu.jo + edu.kg + edu.kh + edu.kn + edu.kw + edu.ky + edu.kz + edu.la + edu.lb + edu.lv + edu.ly + edu.mk + edu.mm + edu.mn + edu.mo + edu.mt + edu.mx + edu.my + edu.ni + edu.np + edu.om + edu.pa + edu.pe + edu.ph + edu.pk + edu.pl + edu.pr + edu.ps + edu.pt + edu.py + edu.qa + edu.rs + edu.ru + edu.sa + edu.sd + edu.sg + edu.sv + edu.sy + edu.tr + edu.tt + edu.tw + edu.ua + edu.uy + edu.ve + edu.vn + edu.ws + edu.ye + edu.zm + vic.edu.au + ).to_set.freeze # These are domains that snuck into the edu registry, # but don't pass the education sniff test # Note: validated domain must not end with the blacklisted string - BLACKLIST = [ - 'si.edu', - ] + BLACKLIST = %w( + si.edu + ).freeze class << self @@ -140,11 +141,11 @@ module Swot return false if text.nil? begin domain = get_domain(text) - return false if domain.nil? - - if BLACKLIST.any? { |d| domain.name =~ /(\A|\.)#{Regexp.escape(d)}\z/ } + if domain.nil? false - elsif ACADEMIC_TLDS[domain.tld] + elsif BLACKLIST.any? { |d| domain.name =~ /(\A|\.)#{Regexp.escape(d)}\z/ } + false + elsif ACADEMIC_TLDS.include?(domain.tld) true elsif match_academic_domain?(domain) true From 292c4afa6f91b7689b7b8ce36921767fdcb9d954 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Wed, 18 Jun 2014 22:46:04 -0400 Subject: [PATCH 2/2] move the academic TLDs into their own file --- lib/swot.rb | 124 +------------------------------------- lib/swot/academic_tlds.rb | 123 +++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 122 deletions(-) create mode 100644 lib/swot/academic_tlds.rb diff --git a/lib/swot.rb b/lib/swot.rb index 2de4535ffc..0ef89b5657 100644 --- a/lib/swot.rb +++ b/lib/swot.rb @@ -1,129 +1,9 @@ -require 'public_suffix' -require 'set' +require "public_suffix" +require File.join(File.dirname(__FILE__), "swot", "academic_tlds.rb") module Swot VERSION = "0.3.0" - # These top-level domains are guaranteed to be academic institutions. - ACADEMIC_TLDS = %w( - ac.ae - ac.at - ac.bd - ac.be - ac.cr - ac.cy - ac.fj - ac.id - ac.il - ac.ir - ac.jp - ac.ke - ac.kr - ac.ma - ac.mu - ac.mw - ac.mz - ac.nz - ac.pa - ac.pg - ac.rs - ac.ru - ac.rw - ac.th - ac.tz - ac.ug - ac.uk - ac.yu - ac.za - ac.zm - ac.zw - edu - edu.af - edu.al - edu.ar - edu.au - edu.az - edu.ba - edu.bb - edu.bd - edu.bh - edu.bi - edu.bn - edu.bo - edu.br - edu.bs - edu.bt - edu.bz - edu.co - edu.cu - edu.do - edu.dz - edu.ec - edu.ee - edu.eg - edu.er - edu.es - edu.et - edu.ge - edu.gh - edu.gr - edu.gt - edu.hk - edu.hn - edu.ht - edu.iq - edu.jm - edu.jo - edu.kg - edu.kh - edu.kn - edu.kw - edu.ky - edu.kz - edu.la - edu.lb - edu.lv - edu.ly - edu.mk - edu.mm - edu.mn - edu.mo - edu.mt - edu.mx - edu.my - edu.ni - edu.np - edu.om - edu.pa - edu.pe - edu.ph - edu.pk - edu.pl - edu.pr - edu.ps - edu.pt - edu.py - edu.qa - edu.rs - edu.ru - edu.sa - edu.sd - edu.sg - edu.sv - edu.sy - edu.tr - edu.tt - edu.tw - edu.ua - edu.uy - edu.ve - edu.vn - edu.ws - edu.ye - edu.zm - vic.edu.au - ).to_set.freeze - # These are domains that snuck into the edu registry, # but don't pass the education sniff test # Note: validated domain must not end with the blacklisted string diff --git a/lib/swot/academic_tlds.rb b/lib/swot/academic_tlds.rb new file mode 100644 index 0000000000..7f3582d4b5 --- /dev/null +++ b/lib/swot/academic_tlds.rb @@ -0,0 +1,123 @@ +require 'set' + +module Swot + # These top-level domains are guaranteed to be academic institutions. + ACADEMIC_TLDS = %w( + ac.ae + ac.at + ac.bd + ac.be + ac.cr + ac.cy + ac.fj + ac.id + ac.il + ac.ir + ac.jp + ac.ke + ac.kr + ac.ma + ac.mu + ac.mw + ac.mz + ac.nz + ac.pa + ac.pg + ac.rs + ac.ru + ac.rw + ac.th + ac.tz + ac.ug + ac.uk + ac.yu + ac.za + ac.zm + ac.zw + edu + edu.af + edu.al + edu.ar + edu.au + edu.az + edu.ba + edu.bb + edu.bd + edu.bh + edu.bi + edu.bn + edu.bo + edu.br + edu.bs + edu.bt + edu.bz + edu.co + edu.cu + edu.do + edu.dz + edu.ec + edu.ee + edu.eg + edu.er + edu.es + edu.et + edu.ge + edu.gh + edu.gr + edu.gt + edu.hk + edu.hn + edu.ht + edu.iq + edu.jm + edu.jo + edu.kg + edu.kh + edu.kn + edu.kw + edu.ky + edu.kz + edu.la + edu.lb + edu.lv + edu.ly + edu.mk + edu.mm + edu.mn + edu.mo + edu.mt + edu.mx + edu.my + edu.ni + edu.np + edu.om + edu.pa + edu.pe + edu.ph + edu.pk + edu.pl + edu.pr + edu.ps + edu.pt + edu.py + edu.qa + edu.rs + edu.ru + edu.sa + edu.sd + edu.sg + edu.sv + edu.sy + edu.tr + edu.tt + edu.tw + edu.ua + edu.uy + edu.ve + edu.vn + edu.ws + edu.ye + edu.zm + vic.edu.au + ).to_set.freeze +end