Build compact swot.txt for faster isAcademic()
This commit is contained in:
Maxim Shafirov
parent
da95a73a34
commit
4d56bc5110
23
.idea/runConfigurations/Build_Swot.xml
generated
Normal file
23
.idea/runConfigurations/Build_Swot.xml
generated
Normal file
@ -0,0 +1,23 @@
|
||||
<component name="ProjectRunConfigurationManager">
|
||||
<configuration default="false" name="Build Swot" type="JetRunConfigurationType" factoryName="Kotlin">
|
||||
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
|
||||
<option name="MAIN_CLASS_NAME" value="swot.SwotPackage" />
|
||||
<option name="VM_PARAMETERS" value="" />
|
||||
<option name="PROGRAM_PARAMETERS" value="" />
|
||||
<option name="WORKING_DIRECTORY" value="" />
|
||||
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
|
||||
<option name="ALTERNATIVE_JRE_PATH" />
|
||||
<option name="PASS_PARENT_ENVS" value="true" />
|
||||
<module name="swot" />
|
||||
<envs />
|
||||
<RunnerSettings RunnerId="Debug">
|
||||
<option name="DEBUG_PORT" value="" />
|
||||
<option name="TRANSPORT" value="0" />
|
||||
<option name="LOCAL" value="true" />
|
||||
</RunnerSettings>
|
||||
<RunnerSettings RunnerId="Run" />
|
||||
<ConfigurationWrapper RunnerId="Debug" />
|
||||
<ConfigurationWrapper RunnerId="Run" />
|
||||
<method />
|
||||
</configuration>
|
||||
</component>
|
||||
@ -1 +0,0 @@
|
||||
2.0.0
|
||||
@ -1,4 +0,0 @@
|
||||
language: ruby
|
||||
rvm:
|
||||
- 1.9.3
|
||||
- 2.0.0
|
||||
12
Gemfile
12
Gemfile
@ -1,12 +0,0 @@
|
||||
source "http://rubygems.org"
|
||||
|
||||
gem 'public_suffix'
|
||||
gem 'naughty_or_nice'
|
||||
|
||||
group :development do
|
||||
gem "nokogiri", "~> 1.5"
|
||||
gem "rake", "~> 10.0"
|
||||
gem "rdoc", "~> 3.12"
|
||||
gem "bundler", "~> 1.5"
|
||||
gem "jeweler", "~> 1.8"
|
||||
end
|
||||
96
Rakefile
96
Rakefile
@ -1,96 +0,0 @@
|
||||
# encoding: utf-8
|
||||
|
||||
require 'rubygems'
|
||||
require 'bundler'
|
||||
begin
|
||||
Bundler.setup(:default, :development)
|
||||
rescue Bundler::BundlerError => e
|
||||
$stderr.puts e.message
|
||||
$stderr.puts "Run `bundle install` to install missing gems"
|
||||
exit e.status_code
|
||||
end
|
||||
require 'rake'
|
||||
|
||||
require 'jeweler'
|
||||
Jeweler::Tasks.new do |gem|
|
||||
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
||||
gem.name = "swot"
|
||||
gem.homepage = "http://github.com/leereilly/swot"
|
||||
gem.license = "MIT"
|
||||
gem.summary = %Q{email helpers}
|
||||
gem.description = %Q{email helpers}
|
||||
gem.email = "lee@leereilly.net"
|
||||
gem.authors = ["Lee Reilly"]
|
||||
# dependencies defined in Gemfile
|
||||
end
|
||||
Jeweler::RubygemsDotOrgTasks.new
|
||||
|
||||
require 'rake/testtask'
|
||||
Rake::TestTask.new(:test) do |test|
|
||||
test.libs << 'lib' << 'test'
|
||||
test.pattern = 'test/**/test_*.rb'
|
||||
test.verbose = true
|
||||
end
|
||||
|
||||
task :default => :test
|
||||
|
||||
require 'rdoc/task'
|
||||
Rake::RDocTask.new do |rdoc|
|
||||
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
||||
|
||||
rdoc.rdoc_dir = 'rdoc'
|
||||
rdoc.title = "swot #{version}"
|
||||
rdoc.rdoc_files.include('README*')
|
||||
rdoc.rdoc_files.include('lib/**/*.rb')
|
||||
end
|
||||
|
||||
task :add, :sld, :tld, :name do |t, args|
|
||||
file_path = "#{File.expand_path(__FILE__+'/..')}/lib/domains/#{args.tld}/#{args.sld}"
|
||||
if File.exists?(file_path)
|
||||
puts "already exists"
|
||||
elsif
|
||||
begin
|
||||
FileUtils.mkdir_p(File.dirname(file_path))
|
||||
File.open( file_path, "w" ) do |contents|
|
||||
contents.print args.name
|
||||
end
|
||||
`git add #{file_path}`
|
||||
`git commit -m "Add #{args.name}"`
|
||||
puts "commit successful"
|
||||
rescue
|
||||
puts "commit failed"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
task :quackit_import do
|
||||
require 'nokogiri'
|
||||
require 'open-uri'
|
||||
require_relative File.join('lib', 'swot', 'academic_tlds')
|
||||
|
||||
new_domains = Set.new
|
||||
doc = Nokogiri::HTML(open('http://www.quackit.com/domain-names/country_domain_extensions.cfm'))
|
||||
doc.css('#content li').each do |li|
|
||||
desc = li.content.split(/\s+-\s+/, 2)[1]
|
||||
if desc =~ /academic|education|school/i
|
||||
domain_el = li.at_css('b')
|
||||
# some lines have more than one domain listed
|
||||
domains = domain_el.content.split(/\s*\/\s*/)
|
||||
domains.each do |domain|
|
||||
# remove leading space
|
||||
domain = domain.strip.sub(/\A\./, '')
|
||||
unless Swot::ACADEMIC_TLDS.include?(domain)
|
||||
# print out for manual review
|
||||
puts "#{domain} - #{desc.strip.gsub(/\s+/, ' ')}"
|
||||
new_domains << domain
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
puts "\nNEW DOMAINS (#{new_domains.size}):\n\n"
|
||||
|
||||
new_domains.each do |domain|
|
||||
puts domain
|
||||
end
|
||||
end
|
||||
27
src/swot/Compiler.kt
Normal file
27
src/swot/Compiler.kt
Normal file
@ -0,0 +1,27 @@
|
||||
package swot
|
||||
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* @author max
|
||||
*/
|
||||
|
||||
object CompilationState {
|
||||
val blacklist = File("lib/domains/blacklist.txt").readLines().toHashSet()
|
||||
val domains = File("lib/domains/tlds.txt").readLines().toHashSet()
|
||||
}
|
||||
|
||||
fun main(args: Array<String>) {
|
||||
val root = File("lib/domains")
|
||||
root.recurse {
|
||||
if (it.isFile()) {
|
||||
val parts = root.relativePath(it).trimTrailing(".txt").split('/').toList()
|
||||
if (!checkSet(CompilationState.blacklist, parts) && !checkSet(CompilationState.domains, parts)) {
|
||||
CompilationState.domains.add(parts.reverse().join("."))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val blacklist = CompilationState.blacklist.map { "-$it" }.sort().join("\n")
|
||||
File("out/artifacts/swot.txt").writeText(blacklist + "\n" + CompilationState.domains.sort().join("\n"))
|
||||
}
|
||||
7748
swot.gemspec
7748
swot.gemspec
File diff suppressed because it is too large
Load Diff
@ -1,14 +0,0 @@
|
||||
require 'rubygems'
|
||||
require 'bundler'
|
||||
begin
|
||||
Bundler.setup(:default, :development)
|
||||
rescue Bundler::BundlerError => e
|
||||
$stderr.puts e.message
|
||||
$stderr.puts "Run `bundle install` to install missing gems"
|
||||
exit e.status_code
|
||||
end
|
||||
require 'minitest/autorun'
|
||||
|
||||
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
||||
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
||||
require 'swot'
|
||||
@ -1,78 +0,0 @@
|
||||
# encoding: UTF-8
|
||||
require 'helper'
|
||||
|
||||
describe Swot do
|
||||
it "recognizes academic email addresses and domains" do
|
||||
assert_equal Swot::is_academic?('lreilly@stanford.edu'), true
|
||||
assert_equal Swot::is_academic?('LREILLY@STANFORD.EDU'), true
|
||||
assert_equal Swot::is_academic?('Lreilly@Stanford.Edu'), true
|
||||
assert_equal Swot::is_academic?('lreilly@slac.stanford.edu'), true
|
||||
assert_equal Swot::is_academic?('lreilly@strath.ac.uk'), true
|
||||
assert_equal Swot::is_academic?('lreilly@soft-eng.strath.ac.uk'), true
|
||||
assert_equal Swot::is_academic?('lee@ugr.es'), true
|
||||
assert_equal Swot::is_academic?('lee@uottawa.ca'), true
|
||||
assert_equal Swot::is_academic?('lee@mother.edu.ru'), true
|
||||
assert_equal Swot::is_academic?('lee@ucy.ac.cy'), true
|
||||
|
||||
assert_equal Swot::is_academic?('lee@leerilly.net'), false
|
||||
assert_equal Swot::is_academic?('lee@gmail.com'), false
|
||||
assert_equal Swot::is_academic?('lee@stanford.edu.com'), false
|
||||
assert_equal Swot::is_academic?('lee@strath.ac.uk.com'), false
|
||||
|
||||
assert_equal Swot::is_academic?('stanford.edu'), true
|
||||
assert_equal Swot::is_academic?('slac.stanford.edu'), true
|
||||
assert_equal Swot::is_academic?('www.stanford.edu'), true
|
||||
assert_equal Swot::is_academic?('http://www.stanford.edu'), true
|
||||
assert_equal Swot::is_academic?('http://www.stanford.edu:9393'), true
|
||||
assert_equal Swot::is_academic?('strath.ac.uk'), true
|
||||
assert_equal Swot::is_academic?('soft-eng.strath.ac.uk'), true
|
||||
assert_equal Swot::is_academic?('ugr.es'), true
|
||||
assert_equal Swot::is_academic?('uottawa.ca'), true
|
||||
assert_equal Swot::is_academic?('mother.edu.ru'), true
|
||||
assert_equal Swot::is_academic?('ucy.ac.cy'), true
|
||||
|
||||
assert_equal Swot::is_academic?('leerilly.net'), false
|
||||
assert_equal Swot::is_academic?('gmail.com'), false
|
||||
assert_equal Swot::is_academic?('stanford.edu.com'), false
|
||||
assert_equal Swot::is_academic?('strath.ac.uk.com'), false
|
||||
|
||||
assert_equal Swot::is_academic?(nil), false
|
||||
assert_equal Swot::is_academic?(''), false
|
||||
assert_equal Swot::is_academic?('the'), false
|
||||
|
||||
assert_equal Swot::is_academic?(' stanford.edu'), true
|
||||
assert_equal Swot::is_academic?('lee@strath.ac.uk '), true
|
||||
assert_equal Swot::is_academic?(' gmail.com '), false
|
||||
|
||||
assert_equal Swot::is_academic?('lee@stud.uni-corvinus.hu'), true
|
||||
|
||||
# overkill
|
||||
assert_equal Swot::is_academic?('lee@harvard.edu'), true
|
||||
assert_equal Swot::is_academic?('lee@mail.harvard.edu'), true
|
||||
end
|
||||
|
||||
it "returns name of valid institution" do
|
||||
assert_equal Swot::get_institution_name('lreilly@cs.strath.ac.uk'), "University of Strathclyde"
|
||||
assert_equal Swot::get_institution_name('lreilly@fadi.at'), "BRG Fadingerstraße Linz, Austria"
|
||||
end
|
||||
|
||||
it "returns nil when institution invalid" do
|
||||
assert_equal Swot::get_institution_name('foo@shop.com'), nil
|
||||
end
|
||||
|
||||
it "test aliased methods" do
|
||||
assert_equal Swot::academic?('stanford.edu'), true
|
||||
assert_equal Swot::school_name('lreilly@cs.strath.ac.uk'), "University of Strathclyde"
|
||||
end
|
||||
|
||||
it "fail blacklisted domains" do
|
||||
["si.edu", " si.edu ", "imposter@si.edu", "foo.si.edu", "america.edu"].each do |domain|
|
||||
assert_equal false, Swot::is_academic?(domain), "#{domain} should be denied"
|
||||
end
|
||||
end
|
||||
|
||||
it "not err on tld-only domains" do
|
||||
Swot::is_academic? ".com"
|
||||
assert_equal false, Swot::is_academic?(".com")
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue
Block a user