diff --git a/Gemfile b/Gemfile index b90676312..7c02b449e 100644 --- a/Gemfile +++ b/Gemfile @@ -56,6 +56,7 @@ gem 'highline' gem 'dtext_rb', :git => "https://github.com/r888888888/dtext_rb.git", :require => "dtext" gem 'google-api-client' gem 'cityhash' +gem 'bigquery', :git => "https://github.com/abronte/BigQuery.git", :ref => "b92b4e0b54574e3fde7ad910f39a67538ed387ad" # needed for looser jpeg header compat gem 'ruby-imagespec', :require => "image_spec", :git => "https://github.com/r888888888/ruby-imagespec.git", :branch => "exif-fixes" diff --git a/Gemfile.lock b/Gemfile.lock index 54d15ecf4..1342742dd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,12 @@ +GIT + remote: https://github.com/abronte/BigQuery.git + revision: b92b4e0b54574e3fde7ad910f39a67538ed387ad + ref: b92b4e0b54574e3fde7ad910f39a67538ed387ad + specs: + bigquery (0.9.0) + google-api-client (~> 0.9.3) + googleauth (~> 0.5.0) + GIT remote: https://github.com/halostatue/diff-lcs.git revision: ff796c262db5a22f14b4765c09655b4faccc132a @@ -412,6 +421,7 @@ DEPENDENCIES awesome_print aws-sdk (~> 2) bcrypt-ruby + bigquery! byebug capistrano (~> 3.4.0) capistrano-rails diff --git a/app/logical/big_query/base.rb b/app/logical/big_query/base.rb new file mode 100644 index 000000000..d26f193f5 --- /dev/null +++ b/app/logical/big_query/base.rb @@ -0,0 +1,32 @@ +require "big_query" + +module BigQuery + class Base + def query(q) + client.query(q) + end + + def escape(s) + Regexp.escape(s).gsub(/\\/, '\0\0').gsub(/['"]/, '\\\\\0') + end + + def client + @_client ||= BigQuery::Client.new( + "json_key" => client_options[:google_key_path], + "project_id" => google_config["project_id"], + "dataset" => client_options[:google_data_set] + ) + end + + def client_options + @_client_options ||= { + google_key_path: "/var/www/danbooru/shared/google-key.json", + google_data_set: "danbooru_#{Rails.env}" + } + end + + def google_config + @_google_config ||= JSON.parse(File.read(client_options[:google_key_path])) + end + end +end diff --git a/app/logical/big_query/post_version.rb b/app/logical/big_query/post_version.rb new file mode 100644 index 000000000..0a53e698b --- /dev/null +++ b/app/logical/big_query/post_version.rb @@ -0,0 +1,15 @@ +module BigQuery + class PostVersion < Base + def find_removed(tag, limit = 1_000) + tag = escape(tag) + limit = limit.to_i + query("select id, post_id, updated_at, updater_id, updater_ip_addr, tags, added_tags, removed_tags, parent_id, rating, source from [danbooru_#{Rails.env}.post_versions] where regexp_match(removed_tags, \"(?:^| )#{tag}(?:$| )\") order by updated_at desc limit #{limit}") + end + + def find_added(tag, limit = 1_000) + tag = escape(tag) + limit = limit.to_i + query("select id, post_id, updated_at, updater_id, updater_ip_addr, tags, added_tags, removed_tags, parent_id, rating, source from [danbooru_#{Rails.env}.post_versions] where regexp_match(added_tags, \"(?:^| )#{tag}(?:$| )\") order by updated_at desc limit #{limit}") + end + end +end