From e31977ac2914504c22b3ec35569de4d7f8fdcc13 Mon Sep 17 00:00:00 2001 From: evazion Date: Mon, 17 Oct 2022 20:13:59 -0500 Subject: [PATCH] emails: move EmailValidator into Danbooru::EmailAddress. --- app/logical/danbooru/email_address.rb | 327 ++++++++++++++++++++++- app/logical/email_validator.rb | 325 ---------------------- app/models/email_address.rb | 16 +- script/fixes/117_fix_invalid_emails.rb | 2 +- script/fixes/119_renormalize_emails.rb | 2 +- script/fixes/121_fix_typo_emails.rb | 4 +- test/unit/danbooru_email_address_test.rb | 27 ++ test/unit/email_address_test.rb | 13 + test/unit/email_validator_test.rb | 27 -- 9 files changed, 363 insertions(+), 380 deletions(-) delete mode 100644 app/logical/email_validator.rb create mode 100644 test/unit/danbooru_email_address_test.rb delete mode 100644 test/unit/email_validator_test.rb diff --git a/app/logical/danbooru/email_address.rb b/app/logical/danbooru/email_address.rb index b0708e63d..1b8b01831 100644 --- a/app/logical/danbooru/email_address.rb +++ b/app/logical/danbooru/email_address.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true # A utility class that represents an email address. A wrapper around Mail::Address -# that adds extra utility methods for normalizing and validating email addresses. +# that adds extra methods for validating email addresses, correcting addresses +# containing typos, and canonicalizing multiple forms of the same address. # # @see https://www.rubydoc.info/gems/mail/Mail/Address # @see app/logical/email_address_type.rb @@ -13,13 +14,226 @@ module Danbooru # https://www.regular-expressions.info/email.html EMAIL_REGEX = /\A[a-z0-9._%+-]+@(?:[a-z0-9][a-z0-9-]{0,61}\.)+[a-z]{2,}\z/i + # Sites that ignore dots in email addresses, e.g. where `foo.bar@gmail.com` is the same as `foobar@gmail.com`. + IGNORE_DOTS = %w[gmail.com] + + # Sites that allow plus addressing, e.g. `test+nospam@gmail.com`. + # @see https://en.wikipedia.org/wiki/Email_address#Subaddressing + PLUS_ADDRESSING = %w[gmail.com hotmail.com outlook.com live.com] + MINUS_ADDRESSING = %w[yahoo.com] + + # Sites that have multiple domains mapping to the same logical email address. + CANONICAL_DOMAINS = { + "googlemail.com" => "gmail.com", + "hotmail.com.ar" => "outlook.com", + "hotmail.com.au" => "outlook.com", + "hotmail.com.br" => "outlook.com", + "hotmail.com.hk" => "outlook.com", + "hotmail.com.tw" => "outlook.com", + "hotmail.co.jp" => "outlook.com", + "hotmail.co.nz" => "outlook.com", + "hotmail.co.th" => "outlook.com", + "hotmail.co.uk" => "outlook.com", + "hotmail.com" => "outlook.com", + "hotmail.be" => "outlook.com", + "hotmail.ca" => "outlook.com", + "hotmail.cl" => "outlook.com", + "hotmail.de" => "outlook.com", + "hotmail.dk" => "outlook.com", + "hotmail.es" => "outlook.com", + "hotmail.fi" => "outlook.com", + "hotmail.fr" => "outlook.com", + "hotmail.hu" => "outlook.com", + "hotmail.it" => "outlook.com", + "hotmail.my" => "outlook.com", + "hotmail.nl" => "outlook.com", + "hotmail.no" => "outlook.com", + "hotmail.ru" => "outlook.com", + "hotmail.sg" => "outlook.com", + "hotmail.se" => "outlook.com", + "live.com.au" => "outlook.com", + "live.com.ar" => "outlook.com", + "live.com.mx" => "outlook.com", + "live.com.pt" => "outlook.com", + "live.co.uk" => "outlook.com", + "live.com" => "outlook.com", + "live.at" => "outlook.com", + "live.be" => "outlook.com", + "live.ca" => "outlook.com", + "live.cl" => "outlook.com", + "live.cn" => "outlook.com", + "live.de" => "outlook.com", + "live.dk" => "outlook.com", + "live.fr" => "outlook.com", + "live.hk" => "outlook.com", + "live.ie" => "outlook.com", + "live.it" => "outlook.com", + "live.jp" => "outlook.com", + "live.nl" => "outlook.com", + "live.no" => "outlook.com", + "live.ru" => "outlook.com", + "live.se" => "outlook.com", + "msn.com" => "outlook.com", + "outlook.com.ar" => "outlook.com", + "outlook.com.au" => "outlook.com", + "outlook.com.br" => "outlook.com", + "outlook.co.id" => "outlook.com", + "outlook.co.uk" => "outlook.com", + "outlook.co.jp" => "outlook.com", + "outlook.co.nz" => "outlook.com", + "outlook.co.th" => "outlook.com", + "outlook.at" => "outlook.com", + "outlook.be" => "outlook.com", + "outlook.ca" => "outlook.com", + "outlook.cl" => "outlook.com", + "outlook.cn" => "outlook.com", + "outlook.de" => "outlook.com", + "outlook.dk" => "outlook.com", + "outlook.es" => "outlook.com", + "outlook.fr" => "outlook.com", + "outlook.ie" => "outlook.com", + "outlook.it" => "outlook.com", + "outlook.kr" => "outlook.com", + "outlook.jp" => "outlook.com", + "outlook.nl" => "outlook.com", + "outlook.pt" => "outlook.com", + "outlook.ru" => "outlook.com", + "outlook.sa" => "outlook.com", + "outlook.se" => "outlook.com", + "yahoo.com.au" => "yahoo.com", + "yahoo.com.ar" => "yahoo.com", + "yahoo.com.br" => "yahoo.com", + "yahoo.com.cn" => "yahoo.com", + "yahoo.com.hk" => "yahoo.com", + "yahoo.com.mx" => "yahoo.com", + "yahoo.com.my" => "yahoo.com", + "yahoo.com.ph" => "yahoo.com", + "yahoo.com.sg" => "yahoo.com", + "yahoo.com.tw" => "yahoo.com", + "yahoo.com.vn" => "yahoo.com", + "yahoo.co.id" => "yahoo.com", + "yahoo.co.kr" => "yahoo.com", + "yahoo.co.jp" => "yahoo.com", + "yahoo.co.nz" => "yahoo.com", + "yahoo.co.uk" => "yahoo.com", + "yahoo.co.th" => "yahoo.com", + "yahoo.ne.jp" => "yahoo.com", + "yahoo.ca" => "yahoo.com", + "yahoo.cn" => "yahoo.com", + "yahoo.de" => "yahoo.com", + "yahoo.dk" => "yahoo.com", + "yahoo.es" => "yahoo.com", + "yahoo.fr" => "yahoo.com", + "yahoo.ie" => "yahoo.com", + "yahoo.in" => "yahoo.com", + "yahoo.it" => "yahoo.com", + "yahoo.no" => "yahoo.com", + "yahoo.se" => "yahoo.com", + "ymail.com" => "yahoo.com", + "126.com" => "163.com", + "aim.com" => "aol.com", + "gmx.com" => "gmx.net", + "gmx.at" => "gmx.net", + "gmx.ch" => "gmx.net", + "gmx.de" => "gmx.net", + "gmx.fr" => "gmx.net", + "gmx.us" => "gmx.net", + "pm.me" => "protonmail.com", + "protonmail.ch" => "protonmail.com", + "proton.me" => "protonmail.com", + "tuta.io" => "tutanota.com", + "email.com" => "mail.com", + "me.com" => "icloud.com", + "ya.ru" => "yandex.ru", + "yandex.com" => "yandex.ru", + "yandex.by" => "yandex.ru", + "yandex.ua" => "yandex.ru", + "yandex.kz" => "yandex.ru", + "inbox.ru" => "mail.ru", + "bk.ru" => "mail.ru", + "list.ru" => "mail.ru", + "internet.ru" => "mail.ru", + "hanmail.net" => "daum.net", + } + + # A list of domains known not to be disposable. + # + # @see https://www.mailboxvalidator.com/domain + NONDISPOSABLE_DOMAINS = %w[ + gmail.com + outlook.com + yahoo.com + aol.com + comcast.net + att.net + bellsouth.net + cox.net + sbcglobal.net + verizon.net + icloud.com + rocketmail.com + windowslive.com + qq.com + vip.qq.com + sina.com + naver.com + 163.com + daum.net + mail.goo.ne.jp + nate.com + mail.com + protonmail.com + gmx.net + web.de + freenet.de + o2.pl + op.pl + wp.pl + interia.pl + mail.ru + yandex.ru + rambler.ru + abv.bg + seznam.cz + libero.it + laposte.net + free.fr + orange.fr + citromail.hu + ukr.net + t-online.de + inbox.lv + luukku.com + lycos.com + tlen.pl + infoseek.jp + excite.co.jp + mac.com + wanadoo.fr + ezweb.ne.jp + arcor.de + docomo.ne.jp + earthlink.net + charter.net + hushmail.com + inbox.com + juno.com + shaw.ca + walla.com + tutanota.com + foxmail.com + vivaldi.net + fastmail.com + relay.firefox.com + ] + # @return [String] The original email address as a string. attr_reader :address # @return [Mail::Address] The parsed email address. attr_reader :parsed_address - delegate :local, to: :parsed_address + delegate :local, :domain, to: :parsed_address alias_method :name, :local alias_method :to_s, :address @@ -27,10 +241,10 @@ module Danbooru # # @param string [String, Danbooru::EmailAddress] def initialize(string) - raise Error, "#{string} is not a valid email address" if !string.match?(EMAIL_REGEX) + raise Error, "#{string} is not a valid email address" if !self.class.is_valid?(string) @address = string.to_s - @parsed_address = Mail::Address.new(parsed_address) + @parsed_address = Mail::Address.new(address) end # Parse a string into an email address, or return nil if the string is not a syntactically valid email address. @@ -44,11 +258,11 @@ module Danbooru end # Parse a string into an email address while attempting to fix common typos and mistakes, or return - # nil if the string can't be normalized into a valid email address. + # nil if the string can't be corrected into a valid email address. # # @param address [String] # @return [Danbooru::EmailAddress] - def self.normalize(address) + def self.correct(address) address = address.gsub(/[[:space:]]+/, " ").strip address = address.gsub(/[\\\/]$/, '') # @qq.com\ -> @qq.com, @web.de/ -> @web.de @@ -100,16 +314,101 @@ module Danbooru parse(address) end - # @return [Danbooru::EmailAddress] The email address, normalized to fix typos. - def normalized_address - Danbooru::EmailAddress.normalize(address) + # Returns true if the string is a syntactically valid email address. + # + # @param address [String] The email address. + # @return [Boolean] True if the email address is syntactically valid. + def self.is_valid?(address) + address.to_s.match?(EMAIL_REGEX) end - # @return [PublicSuffix::Domain] The domain part of the email address. - def domain - @domain ||= PublicSuffix.parse(parsed_address.domain) - rescue PublicSuffix::DomainNotAllowed - nil + concerning :DeliverableMethods do + # Returns true if the email address can't receive mail. Checks that the domain exists, that it has a valid MX record, + # that the mail server exists, and that it responds successfully to the RCPT TO command for the given address. + # + # @param from_address [String] The from address to use when connecting to the mail server. + # @param timeout [Integer] The network timeout when connecting to the mail server. + # @param allow_smtp [Boolean] If true, check if the mail server responds to the RCPT TO command. Disabled by default because many ISPs and server providers block port 25. + # @return [Boolean] True if the email address is definitely undeliverable. False if the address is eligible for delivery. Delivery could + # still fail if the mailbox doesn't exist and the server lied to the RCPT TO command. + def undeliverable?(from_address: Danbooru.config.contact_email, timeout: 3, allow_smtp: false) + mail_server = mx_domain(timeout: timeout) + return true if mail_server.nil? + + return false if !allow_smtp + smtp = Net::SMTP.new(mail_server) + smtp.read_timeout = timeout + smtp.open_timeout = timeout + + from_domain = Danbooru::EmailAddress.new(from_address).domain.to_s + smtp.start(from_domain) do |conn| + conn.mailfrom(from_address) + + # Net::SMTPFatalError is raised if RCPT TO returns a 5xx error. + response = conn.rcptto(to_address) rescue $! + return response.is_a?(Net::SMTPFatalError) + end + rescue + false + end + + # Perform a DNS MX record lookup of the domain and return the name of the mail server, if it exists. + # + # @param timeout [Integer] The network timeout when resolving the domain. + # @return [String] The DNS name of the mail server. + def mx_domain(timeout: nil) + dns = Resolv::DNS.new + dns.timeouts = timeout + response = dns.getresource(domain.to_s, Resolv::DNS::Resource::IN::MX) + + response.exchange.to_s + rescue Resolv::ResolvError + nil + end + end + + # Returns true if the email address is not a disposable or throwaway address (it comes from a well-known email provider). + # Returns false if the address is potentially disposable (it comes from an unknown email provider, or a personal domain). + def is_nondisposable? + domain.to_s.in?(NONDISPOSABLE_DOMAINS) + end + + # @return [Danbooru::EmailAddress] The email address with typos corrected, e.g. "foo@gamil.com" => "foo@gmail.com". + def corrected_address + Danbooru::EmailAddress.correct(address) + end + + # @return [Danbooru::EmailAddress] The email address converted into canonical form, e.g. "Foo.Bar+nospam@googlemail.com" => "foobar@gmail.com". + def canonicalized_address + Danbooru::EmailAddress.new("#{canonical_name}@#{canonical_domain}") + end + + # @return [String] The name with the subaddress and periods removed, e.g. "Foo.Bar+nospam@gmail.com" => "foobar". + def canonical_name + name = name_and_subaddress.first + name = name.delete(".") if canonical_domain.in?(IGNORE_DOTS) + name.downcase + end + + # @return [String, nil] The part of the name after the `+` or `-`, e.g. "foo+nospam@gmail.com" => "nospam". + def subaddress + name_and_subaddress.second + end + + # @return [Array] The address split into the name and the subaddress, e.g. "foo+nospam@gmail.com" => ["foo", "nospam"] + def name_and_subaddress + if canonical_domain.in?(PLUS_ADDRESSING) + name.split("+") + elsif canonical_domain.in?(MINUS_ADDRESSING) + name.split("-") + else + [name, nil] + end + end + + # @return [String] The primary domain for the site, if the site has multiple domains, e.g. "googlemail.com" => "gmail.com". + def canonical_domain + @canonical_domain ||= CANONICAL_DOMAINS.fetch(domain.to_s, domain.to_s) end def as_json diff --git a/app/logical/email_validator.rb b/app/logical/email_validator.rb deleted file mode 100644 index 2585f2424..000000000 --- a/app/logical/email_validator.rb +++ /dev/null @@ -1,325 +0,0 @@ -# frozen_string_literal: true - -require 'resolv' - -# Validates that an email address is well-formed, is deliverable, and is not a -# disposable or throwaway email address. Also normalizes equivalent addresses to -# a single canonical form, so that users can't use different forms of the same -# address to register multiple accounts. -module EmailValidator - module_function - - # https://www.regular-expressions.info/email.html - EMAIL_REGEX = /\A[a-z0-9._%+-]+@(?:[a-z0-9][a-z0-9-]{0,61}\.)+[a-z]{2,}\z/i - - # Sites that ignore dots in email addresses, e.g. where `te.st@gmail.com` is - # the same as `test@gmail.com`. - IGNORE_DOTS = %w[gmail.com] - - # Sites that allow plus addressing, e.g. `test+nospam@gmail.com`. - # @see https://en.wikipedia.org/wiki/Email_address#Subaddressing - IGNORE_PLUS_ADDRESSING = %w[gmail.com hotmail.com outlook.com live.com] - IGNORE_MINUS_ADDRESSING = %w[yahoo.com] - - # Sites that have multiple domains mapping to the same logical email address. - CANONICAL_DOMAINS = { - "googlemail.com" => "gmail.com", - "hotmail.com.ar" => "outlook.com", - "hotmail.com.au" => "outlook.com", - "hotmail.com.br" => "outlook.com", - "hotmail.com.hk" => "outlook.com", - "hotmail.com.tw" => "outlook.com", - "hotmail.co.jp" => "outlook.com", - "hotmail.co.nz" => "outlook.com", - "hotmail.co.th" => "outlook.com", - "hotmail.co.uk" => "outlook.com", - "hotmail.com" => "outlook.com", - "hotmail.be" => "outlook.com", - "hotmail.ca" => "outlook.com", - "hotmail.cl" => "outlook.com", - "hotmail.de" => "outlook.com", - "hotmail.dk" => "outlook.com", - "hotmail.es" => "outlook.com", - "hotmail.fi" => "outlook.com", - "hotmail.fr" => "outlook.com", - "hotmail.hu" => "outlook.com", - "hotmail.it" => "outlook.com", - "hotmail.my" => "outlook.com", - "hotmail.nl" => "outlook.com", - "hotmail.no" => "outlook.com", - "hotmail.ru" => "outlook.com", - "hotmail.sg" => "outlook.com", - "hotmail.se" => "outlook.com", - "live.com.au" => "outlook.com", - "live.com.ar" => "outlook.com", - "live.com.mx" => "outlook.com", - "live.com.pt" => "outlook.com", - "live.co.uk" => "outlook.com", - "live.com" => "outlook.com", - "live.at" => "outlook.com", - "live.be" => "outlook.com", - "live.ca" => "outlook.com", - "live.cl" => "outlook.com", - "live.cn" => "outlook.com", - "live.de" => "outlook.com", - "live.dk" => "outlook.com", - "live.fr" => "outlook.com", - "live.hk" => "outlook.com", - "live.ie" => "outlook.com", - "live.it" => "outlook.com", - "live.jp" => "outlook.com", - "live.nl" => "outlook.com", - "live.no" => "outlook.com", - "live.ru" => "outlook.com", - "live.se" => "outlook.com", - "msn.com" => "outlook.com", - "outlook.com.ar" => "outlook.com", - "outlook.com.au" => "outlook.com", - "outlook.com.br" => "outlook.com", - "outlook.co.id" => "outlook.com", - "outlook.co.uk" => "outlook.com", - "outlook.co.jp" => "outlook.com", - "outlook.co.nz" => "outlook.com", - "outlook.co.th" => "outlook.com", - "outlook.at" => "outlook.com", - "outlook.be" => "outlook.com", - "outlook.ca" => "outlook.com", - "outlook.cl" => "outlook.com", - "outlook.cn" => "outlook.com", - "outlook.de" => "outlook.com", - "outlook.dk" => "outlook.com", - "outlook.es" => "outlook.com", - "outlook.fr" => "outlook.com", - "outlook.ie" => "outlook.com", - "outlook.it" => "outlook.com", - "outlook.kr" => "outlook.com", - "outlook.jp" => "outlook.com", - "outlook.nl" => "outlook.com", - "outlook.pt" => "outlook.com", - "outlook.ru" => "outlook.com", - "outlook.sa" => "outlook.com", - "outlook.se" => "outlook.com", - "yahoo.com.au" => "yahoo.com", - "yahoo.com.ar" => "yahoo.com", - "yahoo.com.br" => "yahoo.com", - "yahoo.com.cn" => "yahoo.com", - "yahoo.com.hk" => "yahoo.com", - "yahoo.com.mx" => "yahoo.com", - "yahoo.com.my" => "yahoo.com", - "yahoo.com.ph" => "yahoo.com", - "yahoo.com.sg" => "yahoo.com", - "yahoo.com.tw" => "yahoo.com", - "yahoo.com.vn" => "yahoo.com", - "yahoo.co.id" => "yahoo.com", - "yahoo.co.kr" => "yahoo.com", - "yahoo.co.jp" => "yahoo.com", - "yahoo.co.nz" => "yahoo.com", - "yahoo.co.uk" => "yahoo.com", - "yahoo.co.th" => "yahoo.com", - "yahoo.ne.jp" => "yahoo.com", - "yahoo.ca" => "yahoo.com", - "yahoo.cn" => "yahoo.com", - "yahoo.de" => "yahoo.com", - "yahoo.dk" => "yahoo.com", - "yahoo.es" => "yahoo.com", - "yahoo.fr" => "yahoo.com", - "yahoo.ie" => "yahoo.com", - "yahoo.in" => "yahoo.com", - "yahoo.it" => "yahoo.com", - "yahoo.no" => "yahoo.com", - "yahoo.se" => "yahoo.com", - "ymail.com" => "yahoo.com", - "126.com" => "163.com", - "aim.com" => "aol.com", - "gmx.com" => "gmx.net", - "gmx.at" => "gmx.net", - "gmx.ch" => "gmx.net", - "gmx.de" => "gmx.net", - "gmx.fr" => "gmx.net", - "gmx.us" => "gmx.net", - "pm.me" => "protonmail.com", - "protonmail.ch" => "protonmail.com", - "proton.me" => "protonmail.com", - "tuta.io" => "tutanota.com", - "email.com" => "mail.com", - "me.com" => "icloud.com", - "ya.ru" => "yandex.ru", - "yandex.com" => "yandex.ru", - "yandex.by" => "yandex.ru", - "yandex.ua" => "yandex.ru", - "yandex.kz" => "yandex.ru", - "inbox.ru" => "mail.ru", - "bk.ru" => "mail.ru", - "list.ru" => "mail.ru", - "internet.ru" => "mail.ru", - "hanmail.net" => "daum.net", - } - - # A list of domains known not to be disposable. A user's email must be on - # this list to unrestrict their account. If a user is Restricted and their - # email is not in this list, then it's assumed to be disposable and can't be - # used to unrestrict their account even if they verify their email address. - # - # https://www.mailboxvalidator.com/domain - NONDISPOSABLE_DOMAINS = %w[ - gmail.com - outlook.com - yahoo.com - aol.com - comcast.net - att.net - bellsouth.net - cox.net - sbcglobal.net - verizon.net - icloud.com - rocketmail.com - windowslive.com - qq.com - vip.qq.com - sina.com - naver.com - 163.com - daum.net - mail.goo.ne.jp - nate.com - mail.com - protonmail.com - gmx.net - web.de - freenet.de - o2.pl - op.pl - wp.pl - interia.pl - mail.ru - yandex.ru - rambler.ru - abv.bg - seznam.cz - libero.it - laposte.net - free.fr - orange.fr - citromail.hu - ukr.net - t-online.de - inbox.lv - luukku.com - lycos.com - tlen.pl - infoseek.jp - excite.co.jp - mac.com - wanadoo.fr - ezweb.ne.jp - arcor.de - docomo.ne.jp - earthlink.net - charter.net - hushmail.com - inbox.com - juno.com - shaw.ca - walla.com - tutanota.com - foxmail.com - vivaldi.net - fastmail.com - relay.firefox.com - ] - - # Returns true if it's okay to connect to port 25. Disabled outside of - # production because many home ISPs blackhole port 25. - def smtp_enabled? - Rails.env.production? - end - - # Normalize an email address by stripping out plus addressing and dots, if - # applicable, and rewriting the domain to a canonical domain. - # @param address [String] the email address to normalize - # @return [String] the normalized address - def normalize(address) - return nil unless address.count("@") == 1 - - name, domain = address.downcase.split("@") - - domain = CANONICAL_DOMAINS.fetch(domain, domain) - name = name.delete(".") if domain.in?(IGNORE_DOTS) - name = name.gsub(/\+.*\z/, "") if domain.in?(IGNORE_PLUS_ADDRESSING) - name = name.gsub(/-.*\z/, "") if domain.in?(IGNORE_MINUS_ADDRESSING) - - "#{name}@#{domain}" - end - - # Returns true if the email address is correctly formatted. - # @param [String] the email address - # @return [Boolean] - def is_valid?(address) - address.match?(EMAIL_REGEX) - end - - # Returns true if the email is a throwaway or disposable email address. - # @param [String] the email address - # @return [Boolean] - def is_restricted?(address) - domain = Mail::Address.new(address).domain - !domain.in?(NONDISPOSABLE_DOMAINS) - rescue Mail::Field::IncompleteParseError - true - end - - # Returns true if the email can't be delivered. Checks if the domain has an MX - # record and responds to the RCPT TO command. - # @param to_address [String] the email address to check - # @param from_address [String] the email address to check from - # @return [Boolean] - def undeliverable?(to_address, from_address: Danbooru.config.contact_email, timeout: 3) - mail_server = mx_domain(to_address, timeout: timeout) - mail_server.nil? || rcpt_to_failed?(to_address, from_address, mail_server, timeout: timeout) - rescue - false - end - - # Returns true if the email can't be delivered. Sends a RCPT TO command over - # port 25 to check if the mailbox exists. - # @param to_address [String] the email address to check - # @param from_address [String] the email address to check from - # @param mail_server [String] the DNS name of the SMTP server - # @param timeout [Integer] the network timeout - # @return [Boolean] - def rcpt_to_failed?(to_address, from_address, mail_server, timeout: nil) - return false unless smtp_enabled? - - from_domain = Mail::Address.new(from_address).domain - - smtp = Net::SMTP.new(mail_server) - smtp.read_timeout = timeout - smtp.open_timeout = timeout - - smtp.start(from_domain) do |conn| - conn.mailfrom(from_address) - - # Net::SMTPFatalError is raised if RCPT TO returns a 5xx error. - response = conn.rcptto(to_address) rescue $! - return response.is_a?(Net::SMTPFatalError) - end - end - - # Does a DNS MX record lookup of the domain in the email address and returns the - # name of the mail server, if it exists. - # @param to_address [String] the email address to check - # @param timeout [Integer] the network timeout - # @return [String] the DNS name of the mail server - def mx_domain(to_address, timeout: nil) - domain = Mail::Address.new(to_address).domain - - dns = Resolv::DNS.new - dns.timeouts = timeout - response = dns.getresource(domain, Resolv::DNS::Resource::IN::MX) - - response.exchange.to_s - rescue Resolv::ResolvError - nil - end -end diff --git a/app/models/email_address.rb b/app/models/email_address.rb index dcd670b62..d73c3558c 100644 --- a/app/models/email_address.rb +++ b/app/models/email_address.rb @@ -6,7 +6,7 @@ class EmailAddress < ApplicationRecord attribute :address attribute :normalized_address - validates :address, presence: true, format: { message: "is invalid", with: EmailValidator::EMAIL_REGEX } + validates :address, presence: true, format: { message: "is invalid", with: Danbooru::EmailAddress::EMAIL_REGEX } validates :normalized_address, presence: true, uniqueness: true validates :user_id, uniqueness: true validate :validate_deliverable, on: :deliverable @@ -20,25 +20,21 @@ class EmailAddress < ApplicationRecord end def address=(value) - value = Danbooru::EmailAddress.normalize(value)&.to_s || value - self.normalized_address = EmailValidator.normalize(value) || address + value = Danbooru::EmailAddress.correct(value)&.to_s || value + self.normalized_address = Danbooru::EmailAddress.parse(value)&.canonicalized_address&.to_s || value super end def is_restricted? - EmailValidator.is_restricted?(normalized_address) + !Danbooru::EmailAddress.new(normalized_address).is_nondisposable? end def is_normalized? address == normalized_address end - def is_valid? - EmailValidator.is_valid?(address) - end - def self.restricted(restricted = true) - domains = EmailValidator::NONDISPOSABLE_DOMAINS + domains = Danbooru::EmailAddress::NONDISPOSABLE_DOMAINS domain_regex = domains.map { |domain| Regexp.escape(domain) }.join("|") if restricted.to_s.truthy? @@ -59,7 +55,7 @@ class EmailAddress < ApplicationRecord end def validate_deliverable - if EmailValidator.undeliverable?(address) + if Danbooru::EmailAddress.new(address).undeliverable?(allow_smtp: Rails.env.production?) errors.add(:address, "is invalid or does not exist") end end diff --git a/script/fixes/117_fix_invalid_emails.rb b/script/fixes/117_fix_invalid_emails.rb index c7b33464f..f42e5a1d8 100755 --- a/script/fixes/117_fix_invalid_emails.rb +++ b/script/fixes/117_fix_invalid_emails.rb @@ -77,7 +77,7 @@ with_confirmation do address = address.downcase.gsub(/^(.*)\1$/i, '\1') if address.downcase.match?(/^(.*)\1$/i) # Foo@gmail.comfoo@gmail.com -> foo@gmail.com address = address.downcase.gsub(/^(.*)@\1@[a-zA-Z]+\.com$/i, '\1') if address.downcase.match?(/^(.*)@\1@[a-zA-Z]+\.com$/i) # foo@foo@gmail.com -> foo@gmail.com - normalized_address = EmailValidator.normalize(address) + normalized_address = Danbooru::EmailAddress.parse(address)&.canonicalized_address&.to_s dupe_emails = EmailAddress.where(normalized_address: normalized_address).excluding(email) if dupe_emails.present? puts "#{old_address.ljust(40, " ")} DELETE (#{dupe_emails.map { "#{_1.user.name}##{_1.user.id}" }.join(", ")}, #{email.user.name}##{email.user.id})" diff --git a/script/fixes/119_renormalize_emails.rb b/script/fixes/119_renormalize_emails.rb index 05b2f9b2d..ac694fd58 100755 --- a/script/fixes/119_renormalize_emails.rb +++ b/script/fixes/119_renormalize_emails.rb @@ -4,7 +4,7 @@ require_relative "base" with_confirmation do emails = EmailAddress.find_each do |email| - normalized_address = EmailValidator.normalize(email.address) + normalized_address = Danbooru::EmailAddress.new(email.address).canonicalized_address.to_s if email.normalized_address != normalized_address dupe_emails = EmailAddress.where(normalized_address: normalized_address).joins(:user).to_a diff --git a/script/fixes/121_fix_typo_emails.rb b/script/fixes/121_fix_typo_emails.rb index 16f401911..0679e2904 100755 --- a/script/fixes/121_fix_typo_emails.rb +++ b/script/fixes/121_fix_typo_emails.rb @@ -5,8 +5,8 @@ require_relative "base" with_confirmation do EmailAddress.find_each do |email| old_address = email.address.to_s - fixed_address = Danbooru::EmailAddress.normalize(old_address).to_s - normalized_address = EmailValidator.normalize(fixed_address) + fixed_address = Danbooru::EmailAddress.correct(old_address).to_s + normalized_address = Danbooru::EmailAddress.new(fixed_address).canonicalized_address.to_s next if old_address == fixed_address diff --git a/test/unit/danbooru_email_address_test.rb b/test/unit/danbooru_email_address_test.rb new file mode 100644 index 000000000..378733fbe --- /dev/null +++ b/test/unit/danbooru_email_address_test.rb @@ -0,0 +1,27 @@ +require 'test_helper' + +class DanbooruEmailAddressTest < ActiveSupport::TestCase + def assert_undeliverable(expected, address) + assert_equal(false, Danbooru::EmailAddress.new("webmaster@danbooru.donmai.us").undeliverable?(allow_smtp: true)) + end + + context "Danbooru::EmailAddress" do + context "#undeliverable?" do + should "return good addresses as deliverable" do + assert_undeliverable(false, "webmaster@danbooru.donmai.us") + assert_undeliverable(false, "noizave+spam@gmail.com") + end + + should "return nonexistent domains as undeliverable" do + assert_undeliverable(true, "nobody@does.not.exist.donmai.us") + end + + # XXX these tests are known to fail if your network blocks port 25. + should_eventually "return nonexistent addresses as undeliverable" do + assert_undeliverable(true, "does.not.exist.13yoigo34iy@gmail.com") + assert_undeliverable(true, "does.not.exist.13yoigo34iy@outlook.com") + assert_undeliverable(true, "does.not.exist.13yoigo34iy@hotmail.com") + end + end + end +end diff --git a/test/unit/email_address_test.rb b/test/unit/email_address_test.rb index cc9914661..265a8f7b2 100644 --- a/test/unit/email_address_test.rb +++ b/test/unit/email_address_test.rb @@ -11,6 +11,8 @@ class EmailAddressTest < ActiveSupport::TestCase should allow_value("foo_bar@gmail.com").for(:address) should allow_value("foo+bar@gmail.com").for(:address) should allow_value("foo@foo.bar.com").for(:address) + should allow_value("foo@iki.fi").for(:address) + should allow_value("foo@ne.jp").for(:address) should_not allow_value("foo@example").for(:address) should_not allow_value("fooqq@.com").for(:address) @@ -21,6 +23,17 @@ class EmailAddressTest < ActiveSupport::TestCase should_not allow_value("foo@localhost").for(:address) end + context "normalization" do + should "normalize email addresses" do + assert_equal("foo@gmail.com", EmailAddress.new(address: "FOO@GMAIL.com").normalized_address.to_s) + assert_equal("foo@gmail.com", EmailAddress.new(address: "foo@googlemail.com").normalized_address.to_s) + assert_equal("foobar@gmail.com", EmailAddress.new(address: "foo.bar@googlemail.com").normalized_address.to_s) + assert_equal("foobar@gmail.com", EmailAddress.new(address: "foo.bar+nospam@googlemail.com").normalized_address.to_s) + assert_equal("foobar@gmail.com", EmailAddress.new(address: "Foo.Bar+nospam@Googlemail.com").normalized_address.to_s) + assert_equal("foo.bar@yahoo.com", EmailAddress.new(address: "Foo.Bar-nospam@yahoo.com").normalized_address.to_s) + end + end + should "fix typos" do assert_equal("foo@gmail.com", EmailAddress.new(address: "foo@gmail.com ").address.to_s) assert_equal("foo@gmail.com", EmailAddress.new(address: " foo@gmail.com").address.to_s) diff --git a/test/unit/email_validator_test.rb b/test/unit/email_validator_test.rb deleted file mode 100644 index 77392c08c..000000000 --- a/test/unit/email_validator_test.rb +++ /dev/null @@ -1,27 +0,0 @@ -require 'test_helper' - -class EmailValidatorTest < ActiveSupport::TestCase - context "EmailValidator" do - setup do - EmailValidator.stubs(:smtp_enabled?).returns(true) - end - - context "#undeliverable?" do - should "return good addresses as deliverable" do - assert_equal(false, EmailValidator.undeliverable?("webmaster@danbooru.donmai.us")) - assert_equal(false, EmailValidator.undeliverable?("noizave+spam@gmail.com")) - end - - should "return nonexistent domains as undeliverable" do - assert_equal(true, EmailValidator.undeliverable?("nobody@does.not.exist.donmai.us")) - end - - # XXX these tests are known to fail if your network blocks port 25. - should_eventually "return nonexistent addresses as undeliverable" do - assert_equal(true, EmailValidator.undeliverable?("does.not.exist.13yoigo34iy@gmail.com")) - assert_equal(true, EmailValidator.undeliverable?("does.not.exist.13yoigo34iy@outlook.com")) - assert_equal(true, EmailValidator.undeliverable?("does.not.exist.13yoigo34iy@hotmail.com")) - end - end - end -end