diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 912ab12a8..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 - -defaults: &defaults - working_directory: ~/app - docker: - - image: circleci/buildpack-deps:trusty - environment: - DOCKER_VERSION: 18.03.1-ce - DOCKER_COMPOSE_VERSION: 1.21.1 - -jobs: - build: - <<: *defaults - parallelism: 4 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Create .env.test file - command: env | egrep 'DEBUG|DANBOORU|RAILS|DATABASE_URL|SECRET' > .env.test - - run: - name: Build and Start Docker containers - command: | - docker-compose -f config/docker/compose.yml up -d web - - run: - name: Run database migrations - command: | - docker-compose -f config/docker/compose.yml exec archives dockerize -wait tcp://$PGHOST:$PGPORT bash -l -c 'cd /app ; bundle exec rake db:create ; bundle exec rake db:migrate' - docker-compose -f config/docker/compose.yml exec web dockerize -wait tcp://$PGHOST:$PGPORT bash -l -c 'cd /app ; bin/rake db:create ; bin/rake db:migrate' - - run: - name: Run tests - command: | - circleci tests glob test/**/*_test.rb | circleci tests split --split-by=timings | xargs -I{} docker-compose -f config/docker/compose.yml exec -T web bash -l -c 'cd /app ; bin/rails test --no-ci-clean {}' - docker cp docker_web_1:/app/test/reports /tmp - - store_test_results: - path: /tmp/reports diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 000000000..3e4e48b0b --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index b6215d29d..8f4292733 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,17 @@ .env.* -.bundle +.yarn-integrity +.gitconfig +.git/ +.bundle/ config/database.yml config/danbooru_local_config.rb config/deploy/*.rb config/newrelic.yml +coverage/ +node_modules/ log/ tmp/ -public/data -coverage -.gitconfig -test/reports -public/packs-test -public/packs -node_modules -.yarn-integrity +public/data/ +public/packs-test/ +public/packs/ +test/reports/ diff --git a/Dockerfile b/Dockerfile new file mode 120000 index 000000000..339648b63 --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +config/docker/Dockerfile.danbooru \ No newline at end of file diff --git a/config/docker/Dockerfile.danbooru b/config/docker/Dockerfile.danbooru new file mode 100644 index 000000000..ce295b729 --- /dev/null +++ b/config/docker/Dockerfile.danbooru @@ -0,0 +1,36 @@ +FROM ubuntu:20.04 + +WORKDIR /app + +# prevent apt-get from asking questions about our timezone or locale. +ARG DEBIAN_FRONTEND=noninteractive + +RUN \ + apt-get update && \ + apt-get -y install --no-install-recommends \ + build-essential \ + ruby \ + ruby-dev \ + git \ + nodejs \ + yarnpkg \ + webpack \ + ffmpeg \ + mkvtoolnix \ + libvips-dev \ + libxml2-dev \ + postgresql-client \ + postgresql-server-dev-all + +# webpacker expects the binary to be called `yarn`, but debian/ubuntu installs it as `yarnpkg`. +RUN ln -sf /usr/bin/yarnpkg /usr/bin/yarn + +COPY Gemfile Gemfile.lock ./ +RUN gem install bundler && bundle install + +COPY package.json yarn.lock ./ +RUN yarn install + +COPY . . + +CMD sleep 1d diff --git a/config/docker/Dockerfile.postgres b/config/docker/Dockerfile.postgres new file mode 100644 index 000000000..4dfa1f9c7 --- /dev/null +++ b/config/docker/Dockerfile.postgres @@ -0,0 +1,7 @@ +FROM postgres:latest + +RUN apt-get update && apt-get -y install build-essential git postgresql-server-dev-all + +RUN rm -rf /test_parser +WORKDIR /test_parser +RUN git clone https://github.com/r888888888/test_parser . && make install diff --git a/config/docker/compose.yml b/config/docker/compose.yml deleted file mode 100644 index f912d6165..000000000 --- a/config/docker/compose.yml +++ /dev/null @@ -1,62 +0,0 @@ -version: '3' -services: - db: - image: r888888888/postgres - ports: - - "5432:5432" - environment: - - POSTGRES_PASSWORD - redis: - image: redis:latest - ports: - - "6379:6379" - archives: - image: r888888888/archives - command: sleep 1d - environment: - - POSTGRES_HOST - - POSTGRES_USER - - POSTGRES_PASSWORD - - RAILS_ENV - - AMAZON_KEY - - AMAZON_SQS_REGION - - AMAZON_SECRET - - SQS_ARCHIVES_URL - - RUN - depends_on: - - db - web: - environment: - - SECRET_TOKEN - - SESSION_SECRET_KEY - - RAILS_ENV - - DATABASE_URL - - DEBUG - - ARCHIVE_DATABASE_URL - - DANBOORU_AWS_SQS_ARCHIVE_URL - - DANBOORU_PIXIV_LOGIN - - DANBOORU_PIXIV_PASSWORD - - DANBOORU_TWITTER_API_KEY - - DANBOORU_TWITTER_API_SECRET - - DANBOORU_AWS_ACCESS_KEY_ID - - DANBOORU_AWS_SECRET_ACCESS_KEY - - DANBOORU_AWS_SQS_REGION - - DANBOORU_NIJIE_LOGIN - - DANBOORU_NIJIE_PASSWORD - - DANBOORU_NICO_SEIGA_LOGIN - - DANBOORU_NICO_SEIGA_PASSWORD - - DANBOORU_PERSIST_PIXIV_SESSION - - DANBOORU_TUMBLR_CONSUMER_KEY - - CIRCLE_NODE_TOTAL - - CIRCLE_NODE_INDEX - - CIRCLE_BUILD_IMAGE - build: - context: ../.. - dockerfile: config/docker/danbooru-ci - command: sleep 1d - ports: - - "3000:3000" - depends_on: - - db - - archives - - redis diff --git a/config/docker/danbooru-base b/config/docker/danbooru-base deleted file mode 100644 index bab4b0794..000000000 --- a/config/docker/danbooru-base +++ /dev/null @@ -1,49 +0,0 @@ -FROM ruby:2.5.1-slim-stretch - -ENV DOCKERIZE_VERSION v0.6.1 -ENV VIPS_VERSION 8.7.0 - -RUN apt-get update && \ - apt-get -y install wget && \ - wget -q https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ - tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ - rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \ - apt-get -y install -qq curl gnupg apt-transport-https apt-utils build-essential automake libssl-dev libxml2-dev libxslt-dev ncurses-dev sudo libreadline-dev flex bison ragel memcached libmemcached-dev git libcurl4-openssl-dev sendmail-bin sendmail postgresql-client libpq-dev nginx ssh coreutils ffmpeg mkvtoolnix emacs24-nox telnet libgif-dev && \ - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ - echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ - curl -sSL https://deb.nodesource.com/setup_10.x | sudo -E bash - && \ - apt-get update && \ - apt-get -y install nodejs yarn && \ - apt-get remove cmdtest && \ - apt-get clean && \ - cd /tmp && \ - wget -q https://github.com/libvips/libvips/releases/download/v$VIPS_VERSION/vips-$VIPS_VERSION.tar.gz && \ - tar xzf vips-$VIPS_VERSION.tar.gz && \ - cd vips-$VIPS_VERSION && \ - ./configure --prefix=/usr && \ - make install && \ - ldconfig && \ - cd /tmp && \ - rm -rf vips-$VIPS_VERSION && \ - useradd -ms /bin/bash danbooru -u 1000 && \ - mkdir /app && \ - mkdir -p /var/www/danbooru/shared/data && \ - mkdir -p /var/www/danbooru/shared/data/preview && \ - mkdir -p /var/www/danbooru/shared/data/crop && \ - mkdir -p /var/www/danbooru/shared/data/sample -COPY . /app -RUN chown -R danbooru:danbooru /app && \ - chown -R danbooru:danbooru /var/www/danbooru && \ - mkdir /var/run/danbooru && \ - chown danbooru:danbooru /var/run/danbooru -EXPOSE 3000 -USER danbooru -RUN echo 'gem: --no-document' > ~/.gemrc && \ - gem install bundler --quiet -WORKDIR /app -RUN bundle install -RUN yarn install -COPY script/install/database.yml.templ /app/config/database.yml -COPY script/install/danbooru_local_config.rb.templ /app/config/danbooru_local_config.rb -EXPOSE 3000 -CMD sleep 1d diff --git a/config/docker/danbooru-ci b/config/docker/danbooru-ci deleted file mode 100644 index 26c64479e..000000000 --- a/config/docker/danbooru-ci +++ /dev/null @@ -1,30 +0,0 @@ -# This sets up a docker container suitable for use with Travis CI - -FROM r888888888/danbooru-base - -ENV PATH $PATH:/usr/local/bin - -# Work around a bug in the ruby:2.5.1-slim-stretch base image. The base image -# sets BUNDLER_VERSION=1.16.4, which forces `bundle install` to use an outdated -# version even when we install a newer version. Remove this after updating the -# base image. -# -# https://discuss.circleci.com/t/using-bundler-2-0-during-ci-fails/27411 -# https://github.com/bundler/bundler/issues/6782 -# https://github.com/docker-library/ruby/issues/246 -ENV BUNDLER_VERSION 2.0.2 - -USER root -COPY . /app -RUN chown -R danbooru:danbooru /app - -# install circleci cli tools -RUN curl -q -o /usr/local/bin/circleci https://circle-downloads.s3.amazonaws.com/releases/build_agent_wrapper/circleci -RUN chmod +x /usr/local/bin/circleci -RUN sed -i'' -e 's/run -it/run -i/' /usr/local/bin/circleci - -USER danbooru -RUN git reset --hard master -RUN git clean --force -RUN gem install bundler --version $BUNDLER_VERSION -RUN bundle install > /dev/null diff --git a/config/docker/docker-compose-test.yaml b/config/docker/docker-compose-test.yaml new file mode 100644 index 000000000..0ebf4693a --- /dev/null +++ b/config/docker/docker-compose-test.yaml @@ -0,0 +1,43 @@ +# +# Usage: dotenv -f .env.test docker-compose -f config/docker/docker-compose-test.yaml -p danbooru up -d +# +# Docker Compose file used to create an instance of danbooru along with a test database to run the test suite. +# +# https://docs.docker.com/compose/compose-file/ +# + +version: '3' +services: + danbooru: + build: + context: ../.. + dockerfile: config/docker/Dockerfile.danbooru + environment: + - DATABASE_URL=postgresql://danbooru:danbooru@localhost/danbooru + - ARCHIVE_DATABASE_URL=postgresql://danbooru:danbooru@localhost/danbooru + - DANBOORU_SECRET_KEY_BASE=1234 + - DANBOORU_AWS_SQS_ENABLED + - DANBOORU_TWITTER_API_KEY + - DANBOORU_TWITTER_API_SECRET + - DANBOORU_PIXIV_LOGIN + - DANBOORU_PIXIV_PASSWORD + - DANBOORU_NIJIE_LOGIN + - DANBOORU_NIJIE_PASSWORD + - DANBOORU_NICO_SEIGA_LOGIN + - DANBOORU_NICO_SEIGA_PASSWORD + - DANBOORU_TUMBLR_CONSUMER_KEY + - DANBOORU_DEVIANTART_CLIENT_ID + - DANBOORU_DEVIANTART_CLIENT_SECRET + - DANBOORU_RAKISMET_KEY + - DANBOORU_RAKISMET_URL + - DANBOORU_IP_REGISTRY_API_KEY + - DANBOORU_GOOGLE_MAPS_API_KEY + depends_on: + - postgres + command: /app/config/docker/run-tests.sh + postgres: + build: + dockerfile: Dockerfile.postgres + environment: + POSTGRES_USER: danbooru + POSTGRES_PASSWORD: danbooru diff --git a/config/docker/run-tests.sh b/config/docker/run-tests.sh new file mode 100755 index 000000000..289afdc8c --- /dev/null +++ b/config/docker/run-tests.sh @@ -0,0 +1,26 @@ +#!/usr/bin/bash -eu +# Used as an entrypoint by the Docker image to prepare the test database before running the test suite. + +setup_database() { + cd /app + RAILS_ENV=test bin/rails db:test:prepare +} + +# create the post_versions and pool_versions tables needed by the test suite. +setup_archives() { + mkdir /archives + cd /archives + git clone https://github.com/evazion/archives . + gem install bundler -v 1.13.3 + bundle install --binstubs + RAILS_ENV=test bin/rake db:migrate +} + +run_tests() { + cd /app + bin/rails test +} + +setup_database +setup_archives +run_tests