Adding localstack to docker and S3 file storage testing (#26)

* Adding localstack to docker

* Unit tests for localstack S3 working

* Github flow

* Github flow fix s3 tests

* Playing with github actions, tests working on dev via localhost and docker

* Try using IP

* Try using github services in CI

* Try without volumes on services

* Try something else..'

* Use seperate docker compose yaml for CI

* Specify docker compose ym;l file to use for test

* Fix -f option

* volume mounting

* try mount again

* try use permissions

* Update dir for permissioning

* Update create bucket script to output commands

* Try to create bucket

* Try using awscli not awslocal

* Add region

* Add connection timeout

* Add overwrite

* Add debug

* More debug

* Use conftest to create s3 bucket instead

* Adding health check for localstack service

* Try netwrok mode bridge for tests

* Try some other stuff

* Ignore when failing create bucket if exists

* Make sure github actions is using the localhost as the ip for selenium

* Try setting values from the docker compose for diff envs

* Try using network mode host

* Remove ports

* Use container name instead of docker-compose run

* Clean up host variables

* Clean up code

Co-authored-by: Thu Trang Pham <thu@joinmodernhealth.com>

.github/workflows/test.yml

@@ -16,6 +16,34 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    services:
+      selenium:
+        image: selenium/standalone-firefox:latest
+        ports:
+          - "4444:4444" # Selenium
+          - "5900:5900" # VNC
+      localstack:
+        image: localstack/localstack:latest
+        env:
+          SERVICES: s3
+          DEFAULT_REGION: us-west-1
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+          # enable persistance
+          DATA_DIR: /tmp/localstack/data
+          LAMBDA_EXECUTOR: local
+          DOCKER_HOST: unix:///var/run/docker.sock
+          DEBUG: true
+        volumes:
+          # It doesn't seem like the scripts in entrypoint are being ran... or they are not copied over since
+          # the checkout action happens after init services on Github Actions
+          # - "${{ github.workspace }}/docker-entrypoint-initaws.d:/docker-entrypoint-initaws.d"
+          - "${{ github.workspace }}/tmp/localstack:/tmp/localstack"
+          - "/var/run/docker.sock:/var/run/docker.sock"
+        ports:
+          - 4566:4566
+          - 4571:4571
+        options: --health-cmd="curl http://localhost:4566/health?reload" --health-interval=10s --health-timeout=5s --health-retries=3
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
@@ -24,22 +52,36 @@ jobs:
       DJANGO_VERSION: ${{ matrix.django-version }}
       PYTHON_VERSION: ${{ matrix.python-version }}
       COMPOSE_INTERACTIVE_NO_CLI: 1
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+      AWS_DEFAULT_REGION: us-west-1
     steps:
+    - name: Update Permissions
+      run: |
+        sudo chown -R $USER:$USER ${{ github.workspace }}
+      # required because actions/checkout@2 wants to delete the /tmp/localstack folder
     - uses: actions/checkout@v2
     - name: Build Docker for Python 3.6
       if: ${{ matrix.python-version == 3.6 }}
       run: |
         export SELENIUM_VERSION=3.141.0
-        docker-compose build
+        docker-compose -f docker-compose.ci.yml up -d --build
     - name: Build Docker for other Python versions
       if: ${{ matrix.python-version != 3.6 }}
       run: |
         export SELENIUM_VERSION=4.0.0a7
-        docker-compose build
+        docker-compose -f docker-compose.ci.yml up -d --build
-    - name: Start Docker
+    - name: Attempt to connect to localstack and create bucket
-      run: docker-compose up -d
+      run: |
+        curl -X GET http://localhost:4566/health
+        aws --endpoint-url http://localhost:4566 s3 mb s3://mybucket 2> /dev/null || true
+      # Since docker-entrypoint-initaws.d can't be used to create the s3 bucket on CI
     - name: Integration Test
-      run: docker-compose run web make test-all
+      run: |
+        docker exec -t web_main make test-all
+      # Known Issue: docker-compose cannot run/exec in container via service name when in host network_mode.
+      # See: https://github.com/docker/compose/issues/4548
+      # IE: this doesn't work: docker-compose -f docker-compose.ci.yml run web make test-all
     - name: Coveralls
       uses: AndreMiras/coveralls-python-action@develop
       with:

Dockerfile

@@ -5,8 +5,10 @@ ENV USE_DOCKER=true
 WORKDIR /code
 COPY . /code/
 ARG DJANGO_VERSION="3.1.7"
+RUN echo "Installing Django Version: ${DJANGO_VERSION}"
 RUN pip install django==${DJANGO_VERSION}
 RUN pip install -r requirements.txt
 RUN pip install -e .
 ARG SELENIUM_VERSION="4.0.0a7"
+RUN echo "Installing Selenium Version: ${SELENIUM_VERSION}"
 RUN pip install selenium~=${SELENIUM_VERSION}

admin_confirm/tests/helpers.py

@@ -4,6 +4,8 @@ from django.core.cache import cache
 from django.test import TestCase, RequestFactory
 from django.contrib.auth.models import User
 from django.test import LiveServerTestCase
+from tests.test_project.settings import SELENIUM_HOST
+
 from selenium import webdriver
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.support.ui import Select
@@ -76,7 +78,7 @@ class AdminConfirmIntegrationTestCase(LiveServerTestCase):
     def setUpClass(cls):
         cls.host = socket.gethostbyname(socket.gethostname())
         cls.selenium = webdriver.Remote(
-            command_executor="http://selenium:4444/wd/hub",
+            command_executor=f"http://{SELENIUM_HOST}:4444/wd/hub",
             desired_capabilities=DesiredCapabilities.FIREFOX,
         )
         super().setUpClass()

admin_confirm/tests/integration/test_with_form_input_types.py

@@ -12,7 +12,6 @@ from tests.market.admin import item_admin, shoppingmall_admin
 
 from django.contrib.admin import VERTICAL
 from admin_confirm.constants import CONFIRM_ADD, CONFIRM_CHANGE
-from selenium.webdriver.support.ui import Select
 from selenium.webdriver.common.by import By
 
 

admin_confirm/tests/integration/test_with_s3_storage.py

@@ -0,0 +1,200 @@
+"""
+Tests confirmation of add/change
+on ModelAdmin that utilize caches
+and S3 as a storage backend
+"""
+import os
+
+import pytest
+import pkg_resources
+import localstack_client.session
+
+from importlib import reload
+from selenium.webdriver.remote.file_detector import LocalFileDetector
+from selenium.webdriver.common.by import By
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.conf import settings
+
+from tests.market.models import Item
+
+from admin_confirm.tests.helpers import AdminConfirmIntegrationTestCase
+from tests.market.admin import shoppingmall_admin
+
+from admin_confirm.constants import CONFIRM_CHANGE
+
+
+class ConfirmWithS3StorageTests(AdminConfirmIntegrationTestCase):
+    def setUp(self):
+        self.selenium.file_detector = LocalFileDetector()
+        session = localstack_client.session.Session(region_name="us-west-1")
+        self.s3 = session.resource("s3")
+        self.bucket = self.s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        # Delete all current files
+        for obj in self.bucket.objects.all():
+            obj.delete()
+        super().setUp()
+
+    def tearDown(self):
+        reload(shoppingmall_admin)
+        # Delete all current files
+        for obj in self.bucket.objects.all():
+            obj.delete()
+        super().tearDown()
+
+    def test_s3_is_being_used(self):
+        self.assertTrue(settings.USE_S3)
+        self.assertIsNotNone(settings.AWS_ACCESS_KEY_ID)
+        self.assertEqual(
+            settings.DEFAULT_FILE_STORAGE,
+            "tests.storage_backends.PublicMediaStorage",
+        )
+
+    def test_should_save_file_additions(self):
+        selenium_version = pkg_resources.get_distribution("selenium").parsed_version
+        if selenium_version.major < 4:
+            pytest.skip(
+                "Known issue `https://github.com/SeleniumHQ/selenium/issues/8762` with this selenium version."
+            )
+
+        item = Item.objects.create(
+            name="item", price=1, currency=Item.VALID_CURRENCIES[0][0]
+        )
+
+        self.selenium.get(
+            self.live_server_url + f"/admin/market/item/{item.id}/change/"
+        )
+        self.assertIn(CONFIRM_CHANGE, self.selenium.page_source)
+
+        # Make a change to trigger confirmation page
+        price = self.selenium.find_element(By.NAME, "price")
+        price.send_keys(2)
+
+        # Upload a new file
+        self.selenium.find_element(By.ID, "id_file").send_keys(
+            os.getcwd() + "/screenshot.png"
+        )
+
+        self.selenium.find_element(By.NAME, "_continue").click()
+
+        # Should have hidden form containing the updated price
+        self.assertIn("Confirm", self.selenium.page_source)
+        hidden_form = self.selenium.find_element(By.ID, "hidden-form")
+        price = hidden_form.find_element(By.NAME, "price")
+        self.assertEqual("21.00", price.get_attribute("value"))
+
+        self.selenium.find_element(By.NAME, "_confirmation_received")
+        self.selenium.find_element(By.NAME, "_continue").click()
+
+        item.refresh_from_db()
+        self.assertRegex(item.file.name, r"screenshot.*\.png$")
+        self.assertEqual(21, int(item.price))
+
+        # Check S3 for the file
+        objects = [obj for obj in self.bucket.objects.all()]
+        self.assertEqual(len(objects), 1)
+        self.assertRegex(objects[0].key, r"screenshot.*\.png$")
+
+    def test_should_save_file_changes(self):
+        selenium_version = pkg_resources.get_distribution("selenium").parsed_version
+        if selenium_version.major < 4:
+            pytest.skip(
+                "Known issue `https://github.com/SeleniumHQ/selenium/issues/8762` with this selenium version."
+            )
+
+        file = SimpleUploadedFile(
+            name="old_file.jpg",
+            content=open("screenshot.png", "rb").read(),
+            content_type="image/jpeg",
+        )
+        item = Item.objects.create(
+            name="item", price=1, currency=Item.VALID_CURRENCIES[0][0], file=file
+        )
+
+        self.selenium.get(
+            self.live_server_url + f"/admin/market/item/{item.id}/change/"
+        )
+        self.assertIn(CONFIRM_CHANGE, self.selenium.page_source)
+
+        # Make a change to trigger confirmation page
+        price = self.selenium.find_element(By.NAME, "price")
+        price.send_keys(2)
+
+        # Upload a new file
+        self.selenium.find_element(By.ID, "id_file").send_keys(
+            os.getcwd() + "/screenshot.png"
+        )
+
+        self.selenium.find_element(By.NAME, "_continue").click()
+
+        # Should have hidden form containing the updated price
+        self.assertIn("Confirm", self.selenium.page_source)
+        hidden_form = self.selenium.find_element(By.ID, "hidden-form")
+        price = hidden_form.find_element(By.NAME, "price")
+        self.assertEqual("21.00", price.get_attribute("value"))
+
+        self.selenium.find_element(By.NAME, "_confirmation_received")
+        self.selenium.find_element(By.NAME, "_continue").click()
+
+        item.refresh_from_db()
+        self.assertEqual(21, int(item.price))
+        self.assertRegex(item.file.name, r"screenshot.*\.png$")
+
+        # Check S3 for the file
+        objects = [obj for obj in self.bucket.objects.all()]
+        self.assertEqual(len(objects), 2)
+        get_last_modified = lambda obj: int(obj.last_modified.strftime("%s"))
+        objects_by_last_modified = [
+            obj for obj in sorted(objects, key=get_last_modified)
+        ]
+        self.assertRegex(objects_by_last_modified[-1].key, r"screenshot.*\.png$")
+        self.assertRegex(objects_by_last_modified[0].key, r"old_file.*\.jpg$")
+
+    def test_should_remove_file_if_clear_selected(self):
+        file = SimpleUploadedFile(
+            name="old_file.jpg",
+            content=open("screenshot.png", "rb").read(),
+            content_type="image/jpeg",
+        )
+        item = Item.objects.create(
+            name="item", price=1, currency=Item.VALID_CURRENCIES[0][0], file=file
+        )
+
+        self.selenium.get(
+            self.live_server_url + f"/admin/market/item/{item.id}/change/"
+        )
+        self.assertIn(CONFIRM_CHANGE, self.selenium.page_source)
+
+        # Make a change to trigger confirmation page
+        price = self.selenium.find_element(By.NAME, "price")
+        price.send_keys(2)
+
+        # Choose to clear the existing file
+        self.selenium.find_element(By.ID, "file-clear_id").click()
+        self.assertTrue(
+            self.selenium.find_element(
+                By.XPATH, ".//*[@id='file-clear_id']"
+            ).get_attribute("checked")
+        )
+
+        self.selenium.find_element(By.NAME, "_continue").click()
+
+        # Should have hidden form containing the updated price
+        self.assertIn("Confirm", self.selenium.page_source)
+        hidden_form = self.selenium.find_element(By.ID, "hidden-form")
+        price = hidden_form.find_element(By.NAME, "price")
+        self.assertEqual("21.00", price.get_attribute("value"))
+
+        self.selenium.find_element(By.NAME, "_confirmation_received")
+        self.selenium.find_element(By.NAME, "_continue").click()
+
+        item.refresh_from_db()
+        self.assertEqual(21, int(item.price))
+        # Should have cleared `file` since clear was selected
+        self.assertFalse(item.file)
+
+        # Check S3 for the file
+        # Since deleting from model instance in Django does not automatically
+        #   delete from storage, the old file should still be in S3
+        objects = [obj for obj in self.bucket.objects.all()]
+        self.assertEqual(len(objects), 1)
+        self.assertRegex(objects[0].key, r"old_file.*\.jpg$")

docker-compose.ci.yml

@@ -12,15 +12,5 @@ services:
     command: python tests/manage.py runserver 0.0.0.0:8000
     volumes:
       - .:/code
-    ports:
+    network_mode: host
-      - "8000:8000"
+    container_name: web_main
-    depends_on:
-      - selenium
-  selenium:
-    # image: selenium/standalone-firefox
-    image: selenium/standalone-firefox-debug:latest
-    ports:
-      - "4444:4444"   # Selenium
-      - "5900:5900"   # VNC
-    volumes:
-      - .:/code

docker-compose.dev.yml

@@ -0,0 +1,51 @@
+version: "3.9"
+
+services:
+  web:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        PYTHON_VERSION: "$PYTHON_VERSION"
+        DJANGO_VERSION: "$DJANGO_VERSION"
+        SELENIUM_VERSION: "$SELENIUM_VERSION"
+    command: python tests/manage.py runserver 0.0.0.0:8000
+    volumes:
+      - .:/code
+    ports:
+      - "8000:8000"
+    depends_on:
+      - selenium
+      - localstack
+    environment:
+      - SELENIUM_HOST=selenium
+      # Used for localstack_client as well as our project
+      - LOCALSTACK_HOST=host.docker.internal
+
+  selenium:
+    image: selenium/standalone-firefox
+    ports:
+      - "4444:4444" # Selenium
+      - "5900:5900" # VNC
+    volumes:
+      - .:/code
+
+  localstack:
+    image: localstack/localstack
+    container_name: localstack_main
+    network_mode: bridge
+    ports:
+      - "4566:4566"
+      - "4571:4571"
+    environment:
+      - SERVICES=s3
+      - DEBUG=true
+      # enable persistance
+      - DATA_DIR=/tmp/localstack/data
+      - LAMBDA_EXECUTOR=docker
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - HOSTNAME_EXTERNAL=localstack
+    volumes:
+      - "./docker-entrypoint-initaws.d:/docker-entrypoint-initaws.d"
+      - "./tmp/localstack:/tmp/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"

docker-entrypoint-initaws.d/create_bucket.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -x
+awslocal s3 mb s3://mybucket
+set +x

docs/testing_notes.md

@@ -17,7 +17,7 @@ These are some areas which might/probably have issues that are not currently tes
 ## Save Options
 
 - [x] Save
-- [x] Conitnue
+- [x] Continue
 - [x] Save As New
 - [x] Add another
 

requirements.txt

@@ -8,9 +8,17 @@ twine~=3.3.0
 coveralls~=3.0.0
 Pillow~=8.1.0 # For ImageField
 
+### SELENIUM ###
 # Known issue: https://github.com/SeleniumHQ/selenium/issues/8762
 # Python 3.6 should use because selenium 4 doesn't work with py3.6
 # selenium~=3.141.0
 
 # Others should use
 selenium~=4.0.0.a5
+### END SELENIUM ###
+
+### S3 ###
+localstack~=0.12.9.1 # For testing with S3
+django-storages~=1.11.1
+boto3~=1.17.47
+### END S3 ###

setup.cfg

@@ -25,4 +25,4 @@ branch = True
 [tool:pytest]
 DJANGO_SETTINGS_MODULE=tests.test_project.settings.test
 addopts = --doctest-modules -ra -l --tb=short --show-capture=all --color=yes
-testpaths = admin_confirm
+testpaths = admin_confirm/tests

tests/storage_backends.py

@@ -0,0 +1,12 @@
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+class StaticStorage(S3Boto3Storage):
+    location = "static"
+    default_acl = "public-read"
+
+
+class PublicMediaStorage(S3Boto3Storage):
+    location = "media"
+    default_acl = "public-read"
+    file_overwrite = False

tests/test_project/settings/base.py

@@ -43,6 +43,7 @@ INSTALLED_APPS = [
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
+    "storages",
 ]
 
 MIDDLEWARE = [
@@ -114,8 +115,43 @@ USE_L10N = True
 
 USE_TZ = True
 
+# Setting the hostnames of the services which we are running
+# On github actions, services can be configured on the jobs themselves
+# and can be accessed at localhost. See: https://docs.github.com/en/actions/guides/about-service-containers#communicating-with-service-containers
+
+LOCALSTACK_HOST = os.getenv("LOCALSTACK_HOST", "localhost")
+SELENIUM_HOST = os.getenv("SELENIUM_HOST", "localhost")
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.0/howto/static-files/
 
-STATIC_URL = "/static/"
+# STATIC_URL = "/static/"
+
+# S3 Storage settings
+
+USE_S3 = os.getenv("USE_S3", "true").lower() == "true"
+
+if USE_S3:
+    # aws settings
+    AWS_S3_ENDPOINT_URL = f"http://{LOCALSTACK_HOST}:4566"
+    AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "test")
+    AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "test")
+    AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "mybucket")
+    AWS_DEFAULT_ACL = None
+    AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
+    AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
+    # s3 static settings
+    STATIC_LOCATION = "static"
+    STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/"
+    STATICFILES_STORAGE = "tests.storage_backends.StaticStorage"
+    # s3 public media settings
+    PUBLIC_MEDIA_LOCATION = "media"
+    MEDIA_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{PUBLIC_MEDIA_LOCATION}/"
+    DEFAULT_FILE_STORAGE = "tests.storage_backends.PublicMediaStorage"
+else:
+    STATIC_URL = "/staticfiles/"
+    STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles")
+    MEDIA_URL = "/mediafiles/"
+    MEDIA_ROOT = os.path.join(BASE_DIR, "mediafiles")
+
+STATICFILES_DIRS = (os.path.join(BASE_DIR, "static"),)

tests/test_project/settings/local.py

@@ -3,3 +3,5 @@ from .base import *
 INSTALLED_APPS = INSTALLED_APPS + ["market"]
 WSGI_APPLICATION = "test_project.wsgi.application"
 ROOT_URLCONF = "test_project.urls"
+
+USE_S3 = "True"

tests/test_project/settings/test.py

@@ -3,3 +3,5 @@ from .base import *
 INSTALLED_APPS = INSTALLED_APPS + ["tests.market"]
 WSGI_APPLICATION = "tests.test_project.wsgi.application"
 ROOT_URLCONF = "tests.test_project.urls"
+
+USE_S3 = "True"