Adding localstack to docker and S3 file storage testing (#26)

* Adding localstack to docker

* Unit tests for localstack S3 working

* Github flow

* Github flow fix s3 tests

* Playing with github actions, tests working on dev via localhost and docker

* Try using IP

* Try using github services in CI

* Try without volumes on services

* Try something else..'

* Use seperate docker compose yaml for CI

* Specify docker compose ym;l file to use for test

* Fix -f option

* volume mounting

* try mount again

* try use permissions

* Update dir for permissioning

* Update create bucket script to output commands

* Try to create bucket

* Try using awscli not awslocal

* Add region

* Add connection timeout

* Add overwrite

* Add debug

* More debug

* Use conftest to create s3 bucket instead

* Adding health check for localstack service

* Try netwrok mode bridge for tests

* Try some other stuff

* Ignore when failing create bucket if exists

* Make sure github actions is using the localhost as the ip for selenium

* Try setting values from the docker compose for diff envs

* Try using network mode host

* Remove ports

* Use container name instead of docker-compose run

* Clean up host variables

* Clean up code

Co-authored-by: Thu Trang Pham <thu@joinmodernhealth.com>

tests/test_project/settings/base.py

@@ -43,6 +43,7 @@ INSTALLED_APPS = [
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
+    "storages",
 ]
 
 MIDDLEWARE = [
@@ -114,8 +115,43 @@ USE_L10N = True
 
 USE_TZ = True
 
+# Setting the hostnames of the services which we are running
+# On github actions, services can be configured on the jobs themselves
+# and can be accessed at localhost. See: https://docs.github.com/en/actions/guides/about-service-containers#communicating-with-service-containers
+
+LOCALSTACK_HOST = os.getenv("LOCALSTACK_HOST", "localhost")
+SELENIUM_HOST = os.getenv("SELENIUM_HOST", "localhost")
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.0/howto/static-files/
 
-STATIC_URL = "/static/"
+# STATIC_URL = "/static/"
+
+# S3 Storage settings
+
+USE_S3 = os.getenv("USE_S3", "true").lower() == "true"
+
+if USE_S3:
+    # aws settings
+    AWS_S3_ENDPOINT_URL = f"http://{LOCALSTACK_HOST}:4566"
+    AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", "test")
+    AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", "test")
+    AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "mybucket")
+    AWS_DEFAULT_ACL = None
+    AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
+    AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
+    # s3 static settings
+    STATIC_LOCATION = "static"
+    STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{STATIC_LOCATION}/"
+    STATICFILES_STORAGE = "tests.storage_backends.StaticStorage"
+    # s3 public media settings
+    PUBLIC_MEDIA_LOCATION = "media"
+    MEDIA_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{PUBLIC_MEDIA_LOCATION}/"
+    DEFAULT_FILE_STORAGE = "tests.storage_backends.PublicMediaStorage"
+else:
+    STATIC_URL = "/staticfiles/"
+    STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles")
+    MEDIA_URL = "/mediafiles/"
+    MEDIA_ROOT = os.path.join(BASE_DIR, "mediafiles")
+
+STATICFILES_DIRS = (os.path.join(BASE_DIR, "static"),)

tests/test_project/settings/local.py

@@ -3,3 +3,5 @@ from .base import *
 INSTALLED_APPS = INSTALLED_APPS + ["market"]
 WSGI_APPLICATION = "test_project.wsgi.application"
 ROOT_URLCONF = "test_project.urls"
+
+USE_S3 = "True"

tests/test_project/settings/test.py

@@ -3,3 +3,5 @@ from .base import *
 INSTALLED_APPS = INSTALLED_APPS + ["tests.market"]
 WSGI_APPLICATION = "tests.test_project.wsgi.application"
 ROOT_URLCONF = "tests.test_project.urls"
+
+USE_S3 = "True"