Update docker test configuration.

.circleci/config.yml

@@ -1,37 +0,0 @@
-version: 2
-
-defaults: &defaults
-  working_directory: ~/app
-  docker:
-    - image: circleci/buildpack-deps:trusty
-      environment:
-        DOCKER_VERSION: 18.03.1-ce
-        DOCKER_COMPOSE_VERSION: 1.21.1
-
-jobs:
-  build:
-    <<: *defaults
-    parallelism: 4
-    steps:
-      - checkout
-      - setup_remote_docker:
-          docker_layer_caching: true
-      - run: 
-          name: Create .env.test file
-          command: env | egrep 'DEBUG|DANBOORU|RAILS|DATABASE_URL|SECRET' > .env.test
-      - run:
-          name: Build and Start Docker containers
-          command: |
-            docker-compose -f config/docker/compose.yml up -d web
-      - run:
-          name: Run database migrations
-          command: |
-            docker-compose -f config/docker/compose.yml exec archives dockerize -wait tcp://$PGHOST:$PGPORT bash -l -c 'cd /app ; bundle exec rake db:create ; bundle exec rake db:migrate'
-            docker-compose -f config/docker/compose.yml exec web dockerize -wait tcp://$PGHOST:$PGPORT bash -l -c 'cd /app ; bin/rake db:create ; bin/rake db:migrate'
-      - run:
-          name: Run tests
-          command: |
-            circleci tests glob test/**/*_test.rb | circleci tests split --split-by=timings | xargs -I{} docker-compose -f config/docker/compose.yml exec -T web bash -l -c 'cd /app ; bin/rails test --no-ci-clean {}'
-            docker cp docker_web_1:/app/test/reports /tmp
-      - store_test_results:
-          path: /tmp/reports

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.gitignore

@@ -1,16 +1,17 @@
 .env.*
-.bundle
+.yarn-integrity
+.gitconfig
+.git/
+.bundle/
 config/database.yml
 config/danbooru_local_config.rb
 config/deploy/*.rb
 config/newrelic.yml
+coverage/
+node_modules/
 log/
 tmp/
-public/data
+public/data/
-coverage
+public/packs-test/
-.gitconfig
+public/packs/
-test/reports
+test/reports/
-public/packs-test
-public/packs
-node_modules
-.yarn-integrity

Dockerfile

@@ -0,0 +1 @@
+config/docker/Dockerfile.danbooru

config/docker/Dockerfile.danbooru

@@ -0,0 +1,36 @@
+FROM ubuntu:20.04
+
+WORKDIR /app
+
+# prevent apt-get from asking questions about our timezone or locale.
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN \
+  apt-get update && \
+  apt-get -y install --no-install-recommends \
+    build-essential \
+    ruby \
+    ruby-dev \
+    git \
+    nodejs \
+    yarnpkg \
+    webpack \
+    ffmpeg \
+    mkvtoolnix \
+    libvips-dev \
+    libxml2-dev \
+    postgresql-client \
+    postgresql-server-dev-all
+
+# webpacker expects the binary to be called `yarn`, but debian/ubuntu installs it as `yarnpkg`.
+RUN ln -sf /usr/bin/yarnpkg /usr/bin/yarn
+
+COPY Gemfile Gemfile.lock ./
+RUN gem install bundler && bundle install
+
+COPY package.json yarn.lock ./
+RUN yarn install
+
+COPY . .
+
+CMD sleep 1d

config/docker/Dockerfile.postgres

@@ -0,0 +1,7 @@
+FROM postgres:latest
+
+RUN apt-get update && apt-get -y install build-essential git postgresql-server-dev-all
+
+RUN rm -rf /test_parser
+WORKDIR /test_parser
+RUN git clone https://github.com/r888888888/test_parser . && make install

config/docker/compose.yml

@@ -1,62 +0,0 @@
-version: '3'
-services:
-  db:
-    image: r888888888/postgres
-    ports:
-      - "5432:5432"
-    environment:
-      - POSTGRES_PASSWORD
-  redis:
-    image: redis:latest
-    ports:
-      - "6379:6379"
-  archives:
-    image: r888888888/archives
-    command: sleep 1d
-    environment:
-      - POSTGRES_HOST
-      - POSTGRES_USER
-      - POSTGRES_PASSWORD
-      - RAILS_ENV
-      - AMAZON_KEY
-      - AMAZON_SQS_REGION
-      - AMAZON_SECRET
-      - SQS_ARCHIVES_URL
-      - RUN
-    depends_on:
-      - db
-  web:
-    environment:
-      - SECRET_TOKEN
-      - SESSION_SECRET_KEY
-      - RAILS_ENV
-      - DATABASE_URL
-      - DEBUG
-      - ARCHIVE_DATABASE_URL
-      - DANBOORU_AWS_SQS_ARCHIVE_URL
-      - DANBOORU_PIXIV_LOGIN
-      - DANBOORU_PIXIV_PASSWORD
-      - DANBOORU_TWITTER_API_KEY
-      - DANBOORU_TWITTER_API_SECRET
-      - DANBOORU_AWS_ACCESS_KEY_ID
-      - DANBOORU_AWS_SECRET_ACCESS_KEY
-      - DANBOORU_AWS_SQS_REGION
-      - DANBOORU_NIJIE_LOGIN
-      - DANBOORU_NIJIE_PASSWORD
-      - DANBOORU_NICO_SEIGA_LOGIN
-      - DANBOORU_NICO_SEIGA_PASSWORD
-      - DANBOORU_PERSIST_PIXIV_SESSION
-      - DANBOORU_TUMBLR_CONSUMER_KEY
-      - CIRCLE_NODE_TOTAL
-      - CIRCLE_NODE_INDEX
-      - CIRCLE_BUILD_IMAGE
-    build: 
-      context: ../..
-      dockerfile: config/docker/danbooru-ci
-    command: sleep 1d
-    ports:
-      - "3000:3000"
-    depends_on:
-      - db
-      - archives
-      - redis

config/docker/danbooru-base

@@ -1,49 +0,0 @@
-FROM ruby:2.5.1-slim-stretch
-
-ENV DOCKERIZE_VERSION v0.6.1
-ENV VIPS_VERSION 8.7.0
-
-RUN apt-get update && \
-  apt-get -y install wget && \
-  wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
-  tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz  && \
-  rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
-  apt-get -y install -qq curl gnupg apt-transport-https apt-utils build-essential automake libssl-dev libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel memcached libmemcached-dev git libcurl4-openssl-dev sendmail-bin sendmail postgresql-client libpq-dev nginx ssh coreutils ffmpeg mkvtoolnix emacs24-nox telnet libgif-dev && \
-  curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
-  echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \
-  curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash - && \
-  apt-get update && \
-  apt-get -y install nodejs yarn && \
-  apt-get remove cmdtest && \
-  apt-get clean && \
-  cd /tmp && \
-  wget -q https://github.com/libvips/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz && \
-  tar xzf vips-$VIPS_VERSION.tar.gz && \
-  cd vips-$VIPS_VERSION && \
-  ./configure --prefix=/usr && \
-  make install && \
-  ldconfig && \
-  cd /tmp && \
-  rm -rf vips-$VIPS_VERSION && \
-  useradd -ms /bin/bash danbooru -u 1000 && \
-  mkdir /app && \
-  mkdir -p /var/www/danbooru/shared/data && \
-  mkdir -p /var/www/danbooru/shared/data/preview && \
-  mkdir -p /var/www/danbooru/shared/data/crop && \
-  mkdir -p /var/www/danbooru/shared/data/sample
-COPY . /app
-RUN chown -R danbooru:danbooru /app && \
-  chown -R danbooru:danbooru /var/www/danbooru && \
-  mkdir /var/run/danbooru && \
-  chown danbooru:danbooru /var/run/danbooru
-EXPOSE 3000
-USER danbooru
-RUN echo 'gem: --no-document' > ~/.gemrc && \
-  gem install bundler --quiet
-WORKDIR /app
-RUN bundle install
-RUN yarn install
-COPY script/install/database.yml.templ /app/config/database.yml
-COPY script/install/danbooru_local_config.rb.templ /app/config/danbooru_local_config.rb
-EXPOSE 3000
-CMD sleep 1d

config/docker/danbooru-ci

@@ -1,30 +0,0 @@
-# This sets up a docker container suitable for use with Travis CI
-
-FROM r888888888/danbooru-base
-
-ENV PATH $PATH:/usr/local/bin
-
-# Work around a bug in the ruby:2.5.1-slim-stretch base image. The base image
-# sets BUNDLER_VERSION=1.16.4, which forces `bundle install` to use an outdated
-# version even when we install a newer version. Remove this after updating the
-# base image.
-#
-# https://discuss.circleci.com/t/using-bundler-2-0-during-ci-fails/27411
-# https://github.com/bundler/bundler/issues/6782
-# https://github.com/docker-library/ruby/issues/246
-ENV BUNDLER_VERSION 2.0.2
-
-USER root
-COPY . /app
-RUN chown -R danbooru:danbooru /app
-
-# install circleci cli tools
-RUN curl -q -o /usr/local/bin/circleci https://circle-downloads.s3.amazonaws.com/releases/build_agent_wrapper/circleci
-RUN chmod +x /usr/local/bin/circleci
-RUN sed -i'' -e 's/run -it/run -i/' /usr/local/bin/circleci
-
-USER danbooru
-RUN git reset --hard master
-RUN git clean --force
-RUN gem install bundler --version $BUNDLER_VERSION
-RUN bundle install > /dev/null

config/docker/docker-compose-test.yaml

@@ -0,0 +1,43 @@
+#
+# Usage: dotenv -f .env.test docker-compose -f config/docker/docker-compose-test.yaml -p danbooru up -d
+#
+# Docker Compose file used to create an instance of danbooru along with a test database to run the test suite.
+#
+# https://docs.docker.com/compose/compose-file/
+#
+
+version: '3'
+services:
+  danbooru:
+    build:
+      context: ../..
+      dockerfile: config/docker/Dockerfile.danbooru
+    environment:
+      - DATABASE_URL=postgresql://danbooru:danbooru@localhost/danbooru
+      - ARCHIVE_DATABASE_URL=postgresql://danbooru:danbooru@localhost/danbooru
+      - DANBOORU_SECRET_KEY_BASE=1234
+      - DANBOORU_AWS_SQS_ENABLED
+      - DANBOORU_TWITTER_API_KEY
+      - DANBOORU_TWITTER_API_SECRET
+      - DANBOORU_PIXIV_LOGIN
+      - DANBOORU_PIXIV_PASSWORD
+      - DANBOORU_NIJIE_LOGIN
+      - DANBOORU_NIJIE_PASSWORD
+      - DANBOORU_NICO_SEIGA_LOGIN
+      - DANBOORU_NICO_SEIGA_PASSWORD
+      - DANBOORU_TUMBLR_CONSUMER_KEY
+      - DANBOORU_DEVIANTART_CLIENT_ID
+      - DANBOORU_DEVIANTART_CLIENT_SECRET
+      - DANBOORU_RAKISMET_KEY
+      - DANBOORU_RAKISMET_URL
+      - DANBOORU_IP_REGISTRY_API_KEY
+      - DANBOORU_GOOGLE_MAPS_API_KEY
+    depends_on:
+      - postgres
+    command: /app/config/docker/run-tests.sh
+  postgres:
+    build:
+      dockerfile: Dockerfile.postgres
+    environment:
+      POSTGRES_USER: danbooru
+      POSTGRES_PASSWORD: danbooru

config/docker/run-tests.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/bash -eu
+# Used as an entrypoint by the Docker image to prepare the test database before running the test suite.
+
+setup_database() {
+  cd /app
+  RAILS_ENV=test bin/rails db:test:prepare
+}
+
+# create the post_versions and pool_versions tables needed by the test suite.
+setup_archives() {
+  mkdir /archives
+  cd /archives
+  git clone https://github.com/evazion/archives .
+  gem install bundler -v 1.13.3
+  bundle install --binstubs
+  RAILS_ENV=test bin/rake db:migrate
+}
+
+run_tests() {
+  cd /app
+  bin/rails test
+}
+
+setup_database
+setup_archives
+run_tests