diff --git a/Rakefile b/Rakefile index 40d0db3c06..da74325853 100644 --- a/Rakefile +++ b/Rakefile @@ -68,23 +68,29 @@ task :quackit_import do require 'open-uri' require_relative File.join('lib', 'swot', 'academic_tlds') - domains = Set.new + new_domains = Set.new doc = Nokogiri::HTML(open('http://www.quackit.com/domain-names/country_domain_extensions.cfm')) doc.css('#content li').each do |li| - desc = li.content.split(' - ')[1] + desc = li.content.split(/\s+-\s+/)[1] if desc =~ /academic|education|school/i domain_el = li.at_css('b') - domain = domain_el.content.sub(/\A\./, '') - if domain =~ /\A(\w+\.)*\w+\z/ && !Swot::ACADEMIC_TLDS.include?(domain) - puts "#{domain} - #{desc.strip.gsub(/\s+/, ' ')}" - domains << domain + # some lines have more than one domain listed + domains = domain_el.content.split(/\s*\/\s*/) + domains.each do |domain| + # remove leading space + domain = domain.sub(/\A\./, '') + unless Swot::ACADEMIC_TLDS.include?(domain) + # print out for manual review + puts "#{domain} - #{desc.strip.gsub(/\s+/, ' ')}" + new_domains << domain + end end end end - puts "\nNEW DOMAINS:\n\n" + puts "\nNEW DOMAINS (#{new_domains.size}):\n\n" - domains.sort.each do |domain| + new_domains.sort.each do |domain| puts domain end end