From 2f6d7b2f130966a58830204994861f8c82b9b4a6 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Wed, 18 Jun 2014 22:57:00 -0400 Subject: [PATCH 1/6] add indian academic SLD http://www.registry.ernet.in/Policy.asp --- lib/swot/academic_tlds.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/swot/academic_tlds.rb b/lib/swot/academic_tlds.rb index 7f3582d4b5..61e1634be6 100644 --- a/lib/swot/academic_tlds.rb +++ b/lib/swot/academic_tlds.rb @@ -12,6 +12,7 @@ module Swot ac.fj ac.id ac.il + ac.in ac.ir ac.jp ac.ke From df0ec20ac6bfe2b07329889d58d2ec5aee77137e Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Wed, 18 Jun 2014 22:57:50 -0400 Subject: [PATCH 2/6] add chinese SLD https://en.wikipedia.org/wiki/.ac_(second-level_domain) --- lib/swot/academic_tlds.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/swot/academic_tlds.rb b/lib/swot/academic_tlds.rb index 61e1634be6..afa81b946d 100644 --- a/lib/swot/academic_tlds.rb +++ b/lib/swot/academic_tlds.rb @@ -52,6 +52,7 @@ module Swot edu.bs edu.bt edu.bz + edu.cn edu.co edu.cu edu.do From 317ea1413f0bf56b50129104f48b9426ecbf5d13 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 19 Jun 2014 00:58:55 -0400 Subject: [PATCH 3/6] add script for importing domains from quackit --- Gemfile | 1 + Rakefile | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 9c4b589a23..19527e33d9 100644 --- a/Gemfile +++ b/Gemfile @@ -3,6 +3,7 @@ source "http://rubygems.org" gem 'public_suffix' group :development do + gem "nokogiri", "~> 1.5" gem "rake", "~> 10.0" gem "rdoc", "~> 3.12" gem "bundler", "~> 1.5" diff --git a/Rakefile b/Rakefile index 9fdf50fee7..40d0db3c06 100644 --- a/Rakefile +++ b/Rakefile @@ -61,4 +61,30 @@ task :add, :sld, :tld, :name do |t, args| puts "commit failed" end end -end \ No newline at end of file +end + +task :quackit_import do + require 'nokogiri' + require 'open-uri' + require_relative File.join('lib', 'swot', 'academic_tlds') + + domains = Set.new + doc = Nokogiri::HTML(open('http://www.quackit.com/domain-names/country_domain_extensions.cfm')) + doc.css('#content li').each do |li| + desc = li.content.split(' - ')[1] + if desc =~ /academic|education|school/i + domain_el = li.at_css('b') + domain = domain_el.content.sub(/\A\./, '') + if domain =~ /\A(\w+\.)*\w+\z/ && !Swot::ACADEMIC_TLDS.include?(domain) + puts "#{domain} - #{desc.strip.gsub(/\s+/, ' ')}" + domains << domain + end + end + end + + puts "\nNEW DOMAINS:\n\n" + + domains.sort.each do |domain| + puts domain + end +end From 23d57c9c54419fff112176ab383c98248641dc17 Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 19 Jun 2014 01:02:44 -0400 Subject: [PATCH 4/6] improve quackit script --- Rakefile | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Rakefile b/Rakefile index 40d0db3c06..da74325853 100644 --- a/Rakefile +++ b/Rakefile @@ -68,23 +68,29 @@ task :quackit_import do require 'open-uri' require_relative File.join('lib', 'swot', 'academic_tlds') - domains = Set.new + new_domains = Set.new doc = Nokogiri::HTML(open('http://www.quackit.com/domain-names/country_domain_extensions.cfm')) doc.css('#content li').each do |li| - desc = li.content.split(' - ')[1] + desc = li.content.split(/\s+-\s+/)[1] if desc =~ /academic|education|school/i domain_el = li.at_css('b') - domain = domain_el.content.sub(/\A\./, '') - if domain =~ /\A(\w+\.)*\w+\z/ && !Swot::ACADEMIC_TLDS.include?(domain) - puts "#{domain} - #{desc.strip.gsub(/\s+/, ' ')}" - domains << domain + # some lines have more than one domain listed + domains = domain_el.content.split(/\s*\/\s*/) + domains.each do |domain| + # remove leading space + domain = domain.sub(/\A\./, '') + unless Swot::ACADEMIC_TLDS.include?(domain) + # print out for manual review + puts "#{domain} - #{desc.strip.gsub(/\s+/, ' ')}" + new_domains << domain + end end end end - puts "\nNEW DOMAINS:\n\n" + puts "\nNEW DOMAINS (#{new_domains.size}):\n\n" - domains.sort.each do |domain| + new_domains.sort.each do |domain| puts domain end end From 03da6a79fae68db26ee3ebe498ac2ab53ccbf0ea Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 19 Jun 2014 01:14:35 -0400 Subject: [PATCH 5/6] add a bunch of SLDs, c/o quackit --- Rakefile | 4 ++-- lib/swot/academic_tlds.rb | 44 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/Rakefile b/Rakefile index da74325853..348e620580 100644 --- a/Rakefile +++ b/Rakefile @@ -78,7 +78,7 @@ task :quackit_import do domains = domain_el.content.split(/\s*\/\s*/) domains.each do |domain| # remove leading space - domain = domain.sub(/\A\./, '') + domain = domain.strip.sub(/\A\./, '') unless Swot::ACADEMIC_TLDS.include?(domain) # print out for manual review puts "#{domain} - #{desc.strip.gsub(/\s+/, ' ')}" @@ -90,7 +90,7 @@ task :quackit_import do puts "\nNEW DOMAINS (#{new_domains.size}):\n\n" - new_domains.sort.each do |domain| + new_domains.each do |domain| puts domain end end diff --git a/lib/swot/academic_tlds.rb b/lib/swot/academic_tlds.rb index afa81b946d..219edf1975 100644 --- a/lib/swot/academic_tlds.rb +++ b/lib/swot/academic_tlds.rb @@ -7,9 +7,12 @@ module Swot ac.at ac.bd ac.be + ac.cn ac.cr ac.cy ac.fj + ac.gg + ac.gn ac.id ac.il ac.in @@ -18,15 +21,20 @@ module Swot ac.ke ac.kr ac.ma + ac.me ac.mu ac.mw ac.mz + ac.ni ac.nz + ac.om ac.pa ac.pg + ac.pr ac.rs ac.ru ac.rw + ac.sz ac.th ac.tz ac.ug @@ -35,6 +43,9 @@ module Swot ac.za ac.zm ac.zw + ed.ao + ed.cr + ed.jp edu edu.af edu.al @@ -52,6 +63,7 @@ module Swot edu.bs edu.bt edu.bz + edu.ck edu.cn edu.co edu.cu @@ -70,6 +82,7 @@ module Swot edu.hk edu.hn edu.ht + edu.in edu.iq edu.jm edu.jo @@ -81,13 +94,19 @@ module Swot edu.kz edu.la edu.lb + edu.lr edu.lv edu.ly + edu.me + edu.mg edu.mk + edu.ml edu.mm edu.mn edu.mo edu.mt + edu.mv + edu.mw edu.mx edu.my edu.ni @@ -101,13 +120,17 @@ module Swot edu.pr edu.ps edu.pt + edu.pw edu.py edu.qa edu.rs edu.ru edu.sa + edu.sc edu.sd edu.sg + edu.sh + edu.sl edu.sv edu.sy edu.tr @@ -120,6 +143,27 @@ module Swot edu.ws edu.ye edu.zm + es.kr + g12.br + hs.kr + ms.kr + sc.kr + sc.ug + sch.ae + sch.gg + sch.id + sch.ir + sch.je + sch.jo + sch.lk + sch.ly + sch.my + sch.om + sch.ps + sch.sa + sch.uk + school.nz + school.za vic.edu.au ).to_set.freeze end From 78814f7e0996f3af8747522a647d218265a6886c Mon Sep 17 00:00:00 2001 From: Aidan Feldman Date: Thu, 19 Jun 2014 01:21:32 -0400 Subject: [PATCH 6/6] better domain/desc splitting in quackit script --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 348e620580..35e9aba557 100644 --- a/Rakefile +++ b/Rakefile @@ -71,7 +71,7 @@ task :quackit_import do new_domains = Set.new doc = Nokogiri::HTML(open('http://www.quackit.com/domain-names/country_domain_extensions.cfm')) doc.css('#content li').each do |li| - desc = li.content.split(/\s+-\s+/)[1] + desc = li.content.split(/\s+-\s+/, 2)[1] if desc =~ /academic|education|school/i domain_el = li.at_css('b') # some lines have more than one domain listed