add mock recommender service for development, add user-context recommended posts

Gemfile

@@ -65,6 +65,10 @@ group :production do
   gem 'capistrano-deploytags', '~> 1.0.0', require: false
 end
 
+group :development do
+  gem 'sinatra'
+end
+
 group :development, :test do
   gem 'awesome_print'
   gem 'pry-byebug'

Gemfile.lock

@@ -237,6 +237,7 @@ GEM
     multi_json (1.13.1)
     multi_xml (0.6.0)
     multipart-post (2.0.0)
+    mustermann (1.0.2)
     naught (1.1.0)
     net-http-digest_auth (1.4.1)
     net-http-persistent (2.9.4)
@@ -274,6 +275,8 @@ GEM
       win32-file (>= 0.7.0)
     public_suffix (3.0.2)
     rack (2.0.5)
+    rack-protection (2.0.3)
+      rack
     rack-test (1.0.0)
       rack (>= 1.0, < 3)
     radix62 (1.0.1)
@@ -354,6 +357,11 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    sinatra (2.0.3)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.3)
+      tilt (~> 2.0)
     sprockets (3.7.1)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -485,6 +493,7 @@ DEPENDENCIES
   shoulda-matchers
   simple_form
   simplecov
+  sinatra
   sprockets-rails
   statistics2
   streamio-ffmpeg

Procfile

@@ -1,2 +1,6 @@
-unicorn: bundle exec rails server
+unicorn: bin/rails server -p 3000
-jobs: bundle exec rake jobs:work
+jobs: bin/rake jobs:work
+recommender: bundle exec ruby script/mock_services/recommender.rb
+iqdbs: bundle exec ruby script/mock_services/iqdbs.rb
+reportbooru: bundle exec ruby script/mock_services/reportbooru.rb
+listbooru: bundle exec ruby script/mock_services/listbooru.rb

app/assets/javascripts/posts.js.erb

@@ -452,6 +452,9 @@
         $("#edit").hide();
         $("#share").hide();
         $("#recommended").show();
+        $.get("/recommended_posts", {context: "post", post_id: Danbooru.meta("post-id")}, function(data) {
+          $("#recommended").html(data);
+        });
       } else {
         $("#edit").hide();
         $("#comments").hide();

app/controllers/recommended_posts_controller.rb

@@ -0,0 +1,23 @@
+class RecommendedPostsController < ApplicationController
+  before_action :member_only
+  respond_to :html
+
+  def show
+    @posts = load_posts()
+
+    if request.xhr?
+      render partial: "show", layout: false
+    end
+  end
+
+private
+
+  def load_posts
+    if params[:context] == "post"
+      @posts = RecommenderService.recommend(post_id: params[:post_id])
+
+    elsif params[:context] == "user"
+      @posts = RecommenderService.recommend(user_id: CurrentUser.id)
+    end
+  end
+end

app/logical/post_sets/recommended.rb

@@ -1,16 +1,10 @@
 module PostSets
   class Recommended < PostSets::Post
-    def initialize(post)
+    attr_reader :posts
+    
+    def initialize(posts)
       super("")
-      @post = post
+      @posts = posts
-    end
-
-    def posts
-      @posts ||= begin
-        response = RecommenderService.similar(@post)
-        post_ids = response.reject {|x| x[0] == @post.id}.slice(0, 6).map {|x| x[0]}
-        ::Post.find(post_ids)
-      end
     end
 
     def presenter

app/models/recommender_service.rb

@@ -1,25 +1,26 @@
 module RecommenderService
   extend self
 
-  SCORE_THRESHOLD = 10
+  SCORE_THRESHOLD = 5
 
   def enabled?
     Danbooru.config.recommender_server.present?
   end
 
-  def available?(post)
+  def available_for_post?(post)
     return true if Rails.env.development?
+
     enabled? && CurrentUser.enable_recommended_posts? && post.created_at > Date.civil(2018, 1, 1) && post.score >= SCORE_THRESHOLD
   end
 
-  def similar(post)
+  def available_for_user?
-    if Danbooru.config.recommender_server == "development"
+    enabled? && CurrentUser.is_gold?
-      return Post.order("random()").limit(6).map {|x| [x.id, "1.000"]}
+  end
-    end
 
-    Cache.get("rss:#{post.id}", 1.day) do
+  def recommend_for_user(user_id)
+    ids = Cache.get("rsu:#{user_id}", 1.day) do
       resp = HTTParty.get(
-        "#{Danbooru.config.recommender_server}/similar/#{post.id}", 
+        "#{Danbooru.config.recommender_server}/recommend/#{user_id}", 
         Danbooru.config.httparty_options.merge(
           basic_auth: {
             username: "danbooru", 
@@ -29,5 +30,30 @@ module RecommenderService
       )
       JSON.parse(resp.body)
     end
+    Post.find(ids.map(&:first))
+  end
+
+  def recommend_for_post(post_id)
+    ids = Cache.get("rss:#{post_id}", 1.day) do
+      resp = HTTParty.get(
+        "#{Danbooru.config.recommender_server}/similar/#{post_id}", 
+        Danbooru.config.httparty_options.merge(
+          basic_auth: {
+            username: "danbooru", 
+            password: Danbooru.config.recommender_key
+          }
+        )
+      )
+      JSON.parse(resp.body)
+    end
+    Post.find(ids.reject {|x| x[0] == post_id}.map(&:first))
+  end
+
+  def recommend(post_id: nil, user_id: nil)
+    if post_id
+      recommend_for_post(post_id)
+    elsif user_id
+      recommend_for_user(user_id)
+    end
   end
 end

app/views/posts/partials/common/_secondary_links.html.erb

@@ -3,6 +3,9 @@
     <li><%= link_to "Listing", posts_path %></li>
     <li id="secondary-links-posts-upload" class="nonessential"><%= link_to "Upload", new_upload_path %></li>
     <li id="secondary-links-posts-hot"><%= link_to "Hot", posts_path(:tags => "order:rank", :d => "1") %></li>
+    <% if RecommenderService.available_for_user? %>
+      <li><%= link_to "Recommended", recommended_posts_path(context: "user") %></li>
+    <% end %>
     <% unless CurrentUser.is_anonymous? %>
       <li id="secondary-links-posts-favorites"><%= link_to "Favorites", favorites_path %></li>
       <li id="secondary-links-posts-favorite-groups"><%= link_to "Fav groups", favorite_groups_path %></li>

app/views/posts/show.html.erb

@@ -89,7 +89,7 @@
       <menu id="post-sections">
         <li><a href="#comments">Comments</a></li>
 
-        <% if RecommenderService.enabled? %>
+        <% if RecommenderService.available_for_post?(@post) %>
           <li><a href="#recommended">Recommended</a></li>
         <% end %>
 
@@ -100,13 +100,11 @@
         <li><a href="#share">Share</a></li>
       </menu>
 
-      <section id="recommended" data-available="<%= RecommenderService.available?(@post) %>">
+      <% if RecommenderService.available_for_post?(@post) %>
-        <% if RecommenderService.available?(@post) %>
+        <section id="recommended">
-          <%= render "posts/partials/index/recommended", post: @post %>
+          <p><em>Loading...</em></p>
-        <% else %>
+        </section>
-          <p><em>Not enough data available</em></p>
+      <% end %>
-        <% end %>
-      </section>
 
       <section id="comments">
         <% if !CurrentUser.user.is_builder? %>

app/views/recommended_posts/_show.html.erb

@@ -0,0 +1,3 @@
+<section class="recommended-posts user-disable-cropped-<%= Danbooru.config.enable_image_cropping && CurrentUser.user.disable_cropped_thumbnails? %>">
+  <%= PostSets::Recommended.new(@posts).presenter.post_previews_html(self) %>
+</section>

app/views/recommended_posts/show.html.erb

@@ -0,0 +1,15 @@
+<div id="c-posts">
+  <div id="a-index">
+    <h1>Recommended Posts</h1>
+
+    <p>Based on your voting history, you may enjoy these posts. Vote more to get more accurate results. These recommendations update every hour.</p>
+
+    <%= render partial: "show" %> 
+  </div>
+</div>
+
+<%= render "posts/partials/common/secondary_links" %>
+
+<% content_for(:page_title) do %>
+  Recommended Posts - <%= Danbooru.config.app_name %>
+<% end %>

config/routes.rb

@@ -251,6 +251,7 @@ Rails.application.routes.draw do
   post "reports/post_versions_create" => "reports#post_versions_create"
   get "reports/down_voting_post" => "reports#down_voting_post"
   post "reports/down_voting_post_create" => "reports#down_voting_post_create"
+  resource :recommended_posts, only: [:show]
   resources :saved_searches, :except => [:show] do
     collection do
       get :labels

script/mock_services/README.md

@@ -0,0 +1,7 @@
+These are mocked services to be used for development purposes.
+
+- danbooru: port 3000
+- recommender: port 3001
+- iqdbs: port 3002
+- reportbooru: port 3003
+- listbooru: port 3004

script/mock_services/iqdbs.rb

@@ -0,0 +1,14 @@
+require 'sinatra'
+require 'json'
+require_relative './mock_service_helper'
+
+set :port, 3002
+
+configure do
+  POST_IDS = MockServiceHelper.fetch_post_ids()
+end
+
+get '/similar' do
+  content_type :json
+  POST_IDS[0..10].map {|x| {post_id: x}}.to_json
+end

script/mock_services/listbooru.rb

@@ -0,0 +1,8 @@
+require 'sinatra'
+require 'json'
+
+set :port, 3004
+
+post '/v2/search' do
+  # todo
+end

script/mock_services/mock_service_helper.rb

@@ -0,0 +1,22 @@
+require 'socket'
+require 'timeout'
+require 'httparty'
+
+module MockServiceHelper
+  extend self
+
+  DANBOORU_PORT = 3000
+
+  def fetch_post_ids()
+    begin
+      s = TCPSocket.new("localhost", DANBOORU_PORT)
+      s.close
+    rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+      sleep 1
+      retry
+    end
+
+    json = HTTParty.get("http://localhost:#{DANBOORU_PORT}/posts.json?random=true&limit=10").body
+    return JSON.parse(json).map {|x| x["id"]}
+  end
+end

script/mock_services/recommender.rb

@@ -0,0 +1,19 @@
+require 'sinatra'
+require 'json'
+require_relative './mock_service_helper'
+
+set :port, 3001
+
+configure do
+  POST_IDS = MockServiceHelper.fetch_post_ids()
+end
+
+get '/recommend/:user_id' do
+  content_type :json
+  POST_IDS[0..10].map {|x| [x, "1.000"]}.to_json
+end
+
+get '/similar/:post_id' do
+  content_type :json
+  POST_IDS[0..6].map {|x| [x, "1.000"]}.to_json  
+end

script/mock_services/reportbooru.rb

@@ -0,0 +1,26 @@
+require 'sinatra'
+require 'json'
+
+set :port, 3003
+
+get '/missed_searches' do
+  content_type :text
+  return "abcdefg 10.0\nblahblahblah 20.0\n"
+end
+
+get '/post_searches/rank' do
+  content_type :json
+  return [["abc", 100], ["def", 200]].to_json
+end
+
+get '/reports/user_similarity' do
+  # todo
+end
+
+get '/reports/uploads' do
+  # todo
+end
+
+post '/post_views' do
+  # todo
+end

test/functional/posts_controller_test.rb

@@ -120,15 +120,12 @@ class PostsControllerTest < ActionDispatch::IntegrationTest
         setup do
           @post2 = create(:post)
           RecommenderService.stubs(:enabled?).returns(true)
-          RecommenderService.stubs(:available?).returns(true)
+          RecommenderService.stubs(:available_for_post?).returns(true)
-          RecommenderService.stubs(:similar).returns([[@post.id, "1.0"], [@post2.id, "0.01"]])
         end
 
-        should "render a section for similar posts" do
+        should "not error out" do
           get_auth post_path(@post), @user
           assert_response :success
-          assert_select ".similar-posts"
-          assert_select ".similar-posts #post_#{@post2.id}"
         end
       end
     end

test/functional/recommended_posts_controller_test.rb

@@ -0,0 +1,41 @@
+require "test_helper"
+
+class RecommendedPostsControllerTest < ActionDispatch::IntegrationTest
+  context "The recommended posts controller" do
+    setup do
+      @user = travel_to(1.month.ago) {create(:user)}
+      as_user do
+        @post = create(:post, :tag_string => "aaaa")
+      end
+      RecommenderService.stubs(:enabled?).returns(true)
+    end
+
+    context "post context" do
+      setup do
+        RecommenderService.stubs(:available_for_post?).returns(true)
+        RecommenderService.stubs(:recommend_for_post).returns([@post])
+      end
+
+      should "render" do
+        get_auth recommended_posts_path, @user, xhr: true, params: {context: "post", post_id: @post.id}
+        assert_response :success
+        assert_select ".recommended-posts"
+        assert_select ".recommended-posts #post_#{@post.id}"
+      end
+    end
+
+    context "user context" do
+      setup do
+        RecommenderService.stubs(:available_for_user?).returns(true)
+        RecommenderService.stubs(:recommend_for_user).returns([@post])
+      end
+      
+      should "render" do
+        get_auth recommended_posts_path, @user, params: {context: "user"}
+        assert_response :success
+        assert_select ".recommended-posts"
+        assert_select ".recommended-posts #post_#{@post.id}"
+      end
+    end
+  end
+end