From 79ea6f7e6c37b5f017c4208b6bfce5ce8dcf3ec5 Mon Sep 17 00:00:00 2001 From: Albert Yi Date: Tue, 1 May 2018 15:24:24 -0700 Subject: [PATCH] Add Docker and Travis config files to enable CI tests Also fixes some Rails 6.0 deprecation warnings --- .travis.yml | 51 +++++----------- Gemfile.lock | 4 +- .../moderator/dashboard/queries/artist.rb | 2 +- .../moderator/dashboard/queries/comment.rb | 2 +- .../moderator/dashboard/queries/note.rb | 2 +- .../dashboard/queries/post_appeal.rb | 2 +- .../moderator/dashboard/queries/upload.rb | 2 +- .../moderator/dashboard/queries/wiki_page.rb | 2 +- app/logical/post_query_builder.rb | 14 ++--- app/models/bulk_update_request.rb | 2 +- app/models/pool.rb | 2 +- app/models/saved_search.rb | 2 +- app/models/tag_relationship.rb | 2 +- config/docker/compose.yml | 59 +++++++++++++++++++ config/docker/danbooru-base | 31 ++++++++++ config/docker/danbooru-ci | 9 +++ test/test_helper.rb | 2 + test/unit/upload_test.rb | 6 +- 18 files changed, 138 insertions(+), 58 deletions(-) create mode 100644 config/docker/compose.yml create mode 100644 config/docker/danbooru-base create mode 100644 config/docker/danbooru-ci diff --git a/.travis.yml b/.travis.yml index 27268f97f..84491defd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,46 +1,25 @@ -sudo: enabled +sudo: required +dist: trusty language: bash env: global: - - DISTRO=ubuntu:16.04 - - BUILD_DIR=/build - - PACKAGES="ruby2.5 ruby2.5-dev ruby-switch bundler libglib2.0-dev zlib1g-dev git sudo postgresql-9.5 postgresql-server-dev-9.5 libvips-dev libvips-tools ffmpeg mkvtoolnix" - - DATABASE_URL="postgresql:///danbooru_test?template=template0" - - SECRET_TOKEN="xxx" - - SESSION_SECRET_KEY="xxx" - -services: - - docker + - DOCKER_COMPOSE_VERSION=1.21.1 before_install: - - docker run -d --name test-container -v $(pwd):$BUILD_DIR $DISTRO sleep 1d + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin install: - - docker exec -t test-container bash -c " - cd $BUILD_DIR; - apt-get update; - apt-get install -y tzdata software-properties-common; - cp /usr/share/zoneinfo/America/Los_Angeles /etc/localtime; - apt-add-repository -y ppa:brightbox/ruby-ng; - apt-get update; - apt-get install -y $PACKAGES; - - service postgresql start; - sudo -u postgres createuser --echo --superuser root; - createdb --echo --template=template0 danbooru_test; - - cp script/install/danbooru_local_config.rb.templ config/danbooru_local_config.rb; - cp script/install/database.yml.templ config/database.yml; - ruby-switch --set ruby2.5 - - git clone https://github.com/r888888888/test_parser.git $BUILD_DIR/test_parser; - cd $BUILD_DIR/test_parser; - make install; - mkdir ~/.danbooru; - " + - env | egrep 'DANBOORU|RAILS|DATABASE_URL|SECRET' > .env.test + - docker-compose -f config/docker/compose.yml up -d web + - docker-compose -f config/docker/compose.yml exec archives dockerize -wait tcp://db:5432 bash -l -c 'cd /app ; bundle exec rake db:create ; bundle exec rake db:migrate' + - docker-compose -f config/docker/compose.yml exec web dockerize -wait tcp://db:5432 bash -l -c 'cd /app ; bin/rake db:create ; bin/rake db:migrate' script: - - docker exec -t test-container bash -c " - cd $BUILD_DIR; - bundle install && bin/rake" \ No newline at end of file + - docker-compose -f config/docker/compose.yml exec web bin/rake test + +after_script: + - docker-compose -f config/docker/compose.yml down diff --git a/Gemfile.lock b/Gemfile.lock index 1db086539..01c3ca80d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,9 +9,9 @@ GIT GIT remote: https://github.com/r888888888/dtext_rb.git - revision: 47f238ec378be0933fcd7eccf1904fbd1f8d8613 + revision: 4ef9a39688225bbe6b3d6da14d2e945cfbfb02ce specs: - dtext_rb (1.7.0) + dtext_rb (1.8.0) nokogiri (~> 1.8) GIT diff --git a/app/logical/moderator/dashboard/queries/artist.rb b/app/logical/moderator/dashboard/queries/artist.rb index 326dfce31..e2c545e1a 100644 --- a/app/logical/moderator/dashboard/queries/artist.rb +++ b/app/logical/moderator/dashboard/queries/artist.rb @@ -7,7 +7,7 @@ module Moderator .where("artist_versions.created_at > ?", min_date) .where("users.level <= ?", max_level) .group(:updater) - .order("count(*) desc") + .order(Arel.sql("count(*) desc")) .limit(10) .count .map { |user, count| new(user, count) } diff --git a/app/logical/moderator/dashboard/queries/comment.rb b/app/logical/moderator/dashboard/queries/comment.rb index 9a9a06e90..526469045 100644 --- a/app/logical/moderator/dashboard/queries/comment.rb +++ b/app/logical/moderator/dashboard/queries/comment.rb @@ -9,7 +9,7 @@ module Moderator .where("users.level <= ?", max_level) .group(:comment) .having("count(*) >= 3") - .order("count(*) desc") + .order(Arel.sql("count(*) desc")) .limit(10) .count .map { |comment, count| new(comment, count) } diff --git a/app/logical/moderator/dashboard/queries/note.rb b/app/logical/moderator/dashboard/queries/note.rb index f9129694a..666b09ffc 100644 --- a/app/logical/moderator/dashboard/queries/note.rb +++ b/app/logical/moderator/dashboard/queries/note.rb @@ -7,7 +7,7 @@ module Moderator .where("note_versions.created_at > ?", min_date) .where("users.level <= ?", max_level) .group(:updater) - .order("count(*) desc") + .order(Arel.sql("count(*) desc")) .limit(10) .count .map { |user, count| new(user, count) } diff --git a/app/logical/moderator/dashboard/queries/post_appeal.rb b/app/logical/moderator/dashboard/queries/post_appeal.rb index f6ae5937e..dd251c8e4 100644 --- a/app/logical/moderator/dashboard/queries/post_appeal.rb +++ b/app/logical/moderator/dashboard/queries/post_appeal.rb @@ -7,7 +7,7 @@ module Moderator .deleted .where("post_appeals.created_at > ?", min_date) .group(:id) - .order("count(*) desc") + .order(Arel.sql("count(*) desc")) .limit(10) end end diff --git a/app/logical/moderator/dashboard/queries/upload.rb b/app/logical/moderator/dashboard/queries/upload.rb index 372878566..0399a7848 100644 --- a/app/logical/moderator/dashboard/queries/upload.rb +++ b/app/logical/moderator/dashboard/queries/upload.rb @@ -7,7 +7,7 @@ module Moderator .where("posts.created_at > ?", min_date) .where("users.level <= ?", max_level) .group(:uploader) - .order("count(*) desc") + .order(Arel.sql("count(*) desc")) .limit(10) .count .map { |user, count| new(user, count) } diff --git a/app/logical/moderator/dashboard/queries/wiki_page.rb b/app/logical/moderator/dashboard/queries/wiki_page.rb index e32b0ed75..7a9932dc7 100644 --- a/app/logical/moderator/dashboard/queries/wiki_page.rb +++ b/app/logical/moderator/dashboard/queries/wiki_page.rb @@ -7,7 +7,7 @@ module Moderator .where("wiki_page_versions.created_at > ?", min_date) .where("users.level <= ?", max_level) .group(:updater) - .order("count(*) desc") + .order(Arel.sql("count(*) desc")) .limit(10) .count .map { |user, count| new(user, count) } diff --git a/app/logical/post_query_builder.rb b/app/logical/post_query_builder.rb index 1bf302a1c..f92bb834f 100644 --- a/app/logical/post_query_builder.rb +++ b/app/logical/post_query_builder.rb @@ -363,7 +363,7 @@ class PostQueryBuilder if q[:ordpool].present? pool_id = q[:ordpool].to_i - relation = relation.order("position(' '||posts.id||' ' in ' '||(select post_ids from pools where id = #{pool_id})||' ')") + relation = relation.order(Arel.sql("position(' '||posts.id||' ' in ' '||(select post_ids from pools where id = #{pool_id})||' ')")) end if q[:favgroups_neg].present? @@ -479,20 +479,20 @@ class PostQueryBuilder relation = relation.order("artist_commentaries.updated_at ASC") when "mpixels", "mpixels_desc" - relation = relation.where("posts.image_width is not null and posts.image_height is not null") + relation = relation.where(Arel.sql("posts.image_width is not null and posts.image_height is not null")) # Use "w*h/1000000", even though "w*h" would give the same result, so this can use # the posts_mpixels index. - relation = relation.order("posts.image_width * posts.image_height / 1000000.0 DESC") + relation = relation.order(Arel.sql("posts.image_width * posts.image_height / 1000000.0 DESC")) when "mpixels_asc" relation = relation.where("posts.image_width is not null and posts.image_height is not null") - relation = relation.order("posts.image_width * posts.image_height / 1000000.0 ASC") + relation = relation.order(Arel.sql("posts.image_width * posts.image_height / 1000000.0 ASC")) when "portrait" - relation = relation.order("1.0 * posts.image_width / GREATEST(1, posts.image_height) ASC") + relation = relation.order(Arel.sql("1.0 * posts.image_width / GREATEST(1, posts.image_height) ASC")) when "landscape" - relation = relation.order("1.0 * posts.image_width / GREATEST(1, posts.image_height) DESC") + relation = relation.order(Arel.sql("1.0 * posts.image_width / GREATEST(1, posts.image_height) DESC")) when "filesize", "filesize_desc" relation = relation.order("posts.file_size DESC") @@ -513,7 +513,7 @@ class PostQueryBuilder relation = relation.order("posts.tag_count_#{TagCategory.short_name_mapping[$1]} ASC") when "rank" - relation = relation.order("log(3, posts.score) + (extract(epoch from posts.created_at) - extract(epoch from timestamp '2005-05-24')) / 35000 DESC") + relation = relation.order(Arel.sql("log(3, posts.score) + (extract(epoch from posts.created_at) - extract(epoch from timestamp '2005-05-24')) / 35000 DESC")) when "custom" if q[:post_id].present? && q[:post_id][0] == :in diff --git a/app/models/bulk_update_request.rb b/app/models/bulk_update_request.rb index 60af20c07..d9a78b0f6 100644 --- a/app/models/bulk_update_request.rb +++ b/app/models/bulk_update_request.rb @@ -17,7 +17,7 @@ class BulkUpdateRequest < ApplicationRecord before_validation :normalize_text after_create :create_forum_topic - scope :pending_first, lambda { order("(case status when 'pending' then 0 when 'approved' then 1 else 2 end)") } + scope :pending_first, lambda { order(Arel.sql("(case status when 'pending' then 0 when 'approved' then 1 else 2 end)")) } scope :pending, ->{where(status: "pending")} scope :expired, ->{where("created_at < ?", TagRelationship::EXPIRY.days.ago)} scope :old, ->{where("created_at between ? and ?", TagRelationship::EXPIRY.days.ago, TagRelationship::EXPIRY_WARNING.days.ago)} diff --git a/app/models/pool.rb b/app/models/pool.rb index 086c17663..22432acde 100644 --- a/app/models/pool.rb +++ b/app/models/pool.rb @@ -38,7 +38,7 @@ class Pool < ApplicationRecord end def series_first - order("(case pools.category when 'series' then 0 else 1 end), pools.name") + order(Arel.sql("(case pools.category when 'series' then 0 else 1 end), pools.name")) end def name_matches(name) diff --git a/app/models/saved_search.rb b/app/models/saved_search.rb index 23006c721..872d3dd12 100644 --- a/app/models/saved_search.rb +++ b/app/models/saved_search.rb @@ -73,7 +73,7 @@ class SavedSearch < ApplicationRecord def self.labels_for(user_id) Cache.get(cache_key(user_id)) do - SavedSearch.where(user_id: user_id).order("label").pluck("distinct unnest(labels) as label") + SavedSearch.where(user_id: user_id).order("label").pluck(Arel.sql("distinct unnest(labels) as label")) end end diff --git a/app/models/tag_relationship.rb b/app/models/tag_relationship.rb index 3a9391d95..7e84d4b41 100644 --- a/app/models/tag_relationship.rb +++ b/app/models/tag_relationship.rb @@ -70,7 +70,7 @@ class TagRelationship < ApplicationRecord end def pending_first - order("(case status when 'pending' then 1 when 'queued' then 2 when 'active' then 3 else 0 end), antecedent_name, consequent_name") + order(Arel.sql("(case status when 'pending' then 1 when 'queued' then 2 when 'active' then 3 else 0 end), antecedent_name, consequent_name")) end def active diff --git a/config/docker/compose.yml b/config/docker/compose.yml new file mode 100644 index 000000000..6759a0e30 --- /dev/null +++ b/config/docker/compose.yml @@ -0,0 +1,59 @@ +version: '3' +services: + db: + image: r888888888/postgres + ports: + - "5432:5432" + environment: + - POSTGRES_PASSWORD=73e1ac2e91eb52a9981d09fca561aac3 + memcached: + image: memcached:alpine + ports: + - "11211:11211" + command: memcached + archives: + image: r888888888/archives + command: sleep 1d + environment: + - POSTGRES_HOST=db + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=73e1ac2e91eb52a9981d09fca561aac3 + - RAILS_ENV + - AMAZON_KEY + - AMAZON_SQS_REGION + - AMAZON_SECRET + - SQS_ARCHIVES_URL + - RUN + depends_on: + - db + web: + environment: + - SECRET_TOKEN=73e1ac2e91eb52a9981d09fca561aac3 + - SESSION_SECRET_KEY=73e1ac2e91eb52a9981d09fca561aac3 + - RAILS_ENV + - DATABASE_URL + - RO_DATABASE_URL + - ARCHIVE_DATABASE_URL + - DANBOORU_MEMCACHED_SERVERS + - DANBOORU_AWS_SQS_ARCHIVE_URL + - DANBOORU_PIXIV_LOGIN + - DANBOORU_PIXIV_PASSWORD + - DANBOORU_TWITTER_API_KEY + - DANBOORU_TWITTER_API_SECRET + - DANBOORU_AWS_ACCESS_KEY_ID + - DANBOORU_AWS_SECRET_ACCESS_KEY + - DANBOORU_AWS_SQS_REGION + - DANBOORU_NIJIE_LOGIN + - DANBOORU_NIJIE_PASSWORD + - DANBOORU_NICO_SEIGA_LOGIN + - DANBOORU_NICO_SEIGA_PASSWORD + build: + context: ../.. + dockerfile: config/docker/danbooru-ci + command: sleep 1d + ports: + - "3000:3000" + depends_on: + - db + - memcached + - archives diff --git a/config/docker/danbooru-base b/config/docker/danbooru-base new file mode 100644 index 000000000..a31787a8e --- /dev/null +++ b/config/docker/danbooru-base @@ -0,0 +1,31 @@ +# This sets up a docker container suitable for use with Travis CI + +FROM ruby:2.5.1-slim-stretch + +RUN apt-get update +ENV DOCKERIZE_VERSION v0.6.1 +RUN apt-get -y install wget +RUN wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz +RUN apt-get -y install -qq apt-utils build-essential automake libssl-dev libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel memcached libmemcached-dev git curl libcurl4-openssl-dev imagemagick libmagickcore-dev libmagickwand-dev sendmail-bin sendmail postgresql-client libpq-dev nginx ssh coreutils libavcodec-dev mkvtoolnix emacs24-nox telnet libvips42 libvips-tools libvips-dev > /dev/null +RUN useradd -ms /bin/bash danbooru -u 1000 +RUN mkdir /app +RUN mkdir -p /var/www/danbooru/shared/data +RUN mkdir -p /var/www/danbooru/shared/data/preview +RUN mkdir -p /var/www/danbooru/shared/data/sample +COPY . /app +RUN chown -R danbooru:danbooru /app +RUN chown -R danbooru:danbooru /var/www/danbooru +RUN mkdir /var/run/danbooru +RUN chown danbooru:danbooru /var/run/danbooru +EXPOSE 3000 +USER danbooru +RUN echo 'gem: --no-document' > ~/.gemrc +RUN gem install bundler --quiet +WORKDIR /app +RUN bundle install > /dev/null +COPY script/install/database.yml.templ /app/config/database.yml +COPY script/install/danbooru_local_config.rb.templ /app/config/danbooru_local_config.rb +EXPOSE 3000 +CMD sleep 1d \ No newline at end of file diff --git a/config/docker/danbooru-ci b/config/docker/danbooru-ci new file mode 100644 index 000000000..4f4c014e7 --- /dev/null +++ b/config/docker/danbooru-ci @@ -0,0 +1,9 @@ +# This sets up a docker container suitable for use with Travis CI + +FROM r888888888/danbooru-base + +USER root +COPY . /app +RUN chown -R danbooru:danbooru . +USER danbooru +RUN bundle install > /dev/null diff --git a/test/test_helper.rb b/test/test_helper.rb index 93532d75f..192c92a22 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -58,6 +58,7 @@ class ActiveSupport::TestCase include TestHelpers setup do + Socket.stubs(:gethostname).returns("www.example.com") mock_popular_search_service! mock_missed_search_service! WebMock.allow_net_connect! @@ -104,6 +105,7 @@ class ActionDispatch::IntegrationTest def setup super + Socket.stubs(:gethostname).returns("www.example.com") Danbooru.config.stubs(:enable_sock_puppet_validation?).returns(false) end diff --git a/test/unit/upload_test.rb b/test/unit/upload_test.rb index b5e278ed1..5bb86a3eb 100644 --- a/test/unit/upload_test.rb +++ b/test/unit/upload_test.rb @@ -96,17 +96,17 @@ class UploadTest < ActiveSupport::TestCase context "determining if a file is downloadable" do should "classify HTTP sources as downloadable" do - @upload = FactoryBot.create(:source_upload, :source => "http://www.example.com/1.jpg") + @upload = FactoryBot.create(:source_upload, :source => "http://www.google.com/1.jpg") assert_not_nil(@upload.is_downloadable?) end should "classify HTTPS sources as downloadable" do - @upload = FactoryBot.create(:source_upload, :source => "https://www.example.com/1.jpg") + @upload = FactoryBot.create(:source_upload, :source => "https://www.google.com/1.jpg") assert_not_nil(@upload.is_downloadable?) end should "classify non-HTTP/HTTPS sources as not downloadable" do - @upload = FactoryBot.create(:source_upload, :source => "ftp://www.example.com/1.jpg") + @upload = FactoryBot.create(:source_upload, :source => "ftp://www.google.com/1.jpg") assert_nil(@upload.is_downloadable?) end end