diff --git a/authentik/root/settings.py b/authentik/root/settings.py
index 76aa4429c..195c1c38b 100644
--- a/authentik/root/settings.py
+++ b/authentik/root/settings.py
@@ -55,10 +55,6 @@ MEDIA_ROOT = BASE_DIR + "/media"
 
 DEBUG = CONFIG.y_bool("debug")
 SECRET_KEY = CONFIG.y("secret_key")
-if DEBUG:
-    SECRET_KEY = (
-        "9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s"  # noqa Debug # nosec
-    )
 
 INTERNAL_IPS = ["127.0.0.1"]
 ALLOWED_HOSTS = ["*"]
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 090b21af0..dc767ff58 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -43,7 +43,9 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run pylint authentik tests lifecycle
+              script: |
+                python -m scripts.generate_ci_config
+                pipenv run pylint authentik tests lifecycle
       - job: black
         pool:
           vmImage: 'ubuntu-latest'
@@ -140,7 +142,9 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run ./manage.py migrate
+              script: |
+                python -m scripts.generate_ci_config
+                pipenv run ./manage.py migrate
       - job: migrations_from_previous_release
         pool:
           vmImage: 'ubuntu-latest'
@@ -171,8 +175,9 @@ stages:
           - task: CmdLine@2
             displayName: Migrate to last tagged release
             inputs:
-              script:
-                pipenv run ./manage.py migrate
+              script: |
+                python -m scripts.generate_ci_config
+                pipenv run python -m lifecycle.migrate
           - task: CmdLine@2
             displayName: Install current branch
             inputs:
@@ -185,7 +190,6 @@ stages:
             inputs:
               script: |
                 pipenv run python -m lifecycle.migrate
-                pipenv run ./manage.py migrate
       - job: coverage_unittest
         pool:
           vmImage: 'ubuntu-latest'
@@ -210,6 +214,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
+                python -m scripts.generate_ci_config
                 pipenv run make test
           - task: CmdLine@2
             inputs:
@@ -253,6 +258,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
+                python -m scripts.generate_ci_config
                 pipenv run make test-integration
           - task: CmdLine@2
             inputs:
@@ -307,6 +313,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
+                python -m scripts.generate_ci_config
                 pipenv run make test-e2e
           - task: CmdLine@2
             condition: always()
diff --git a/scripts/agent-install.sh b/scripts/agent-install.sh
deleted file mode 100644
index 592cfe869..000000000
--- a/scripts/agent-install.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash -xe
-wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
-
-VERSION=3.9.0
-
-wget https://www.python.org/ftp/python/$VERSION/Python-$VERSION.tgz
-tar xvzf Python-$VERSION.tgz
-cd Python-$VERSION/
-
-./configure --prefix=$HOME/_work/_tool/Python/$VERSION/x64/ --enable-optimizations --with-ensurepip=install
-make -j 8
-sudo make altinstall
-touch $HOME/_work/_tool/Python/$VERSION/x64.complete
-
-ln -s $HOME/_work/_tool/Python/3.9.5/x64 $HOME/_work/_tool/Python/3/x64
-ln -s $HOME/_work/_tool/Python/3.9.5/x64 $HOME/_work/_tool/Python/3.9/x64
diff --git a/scripts/generate_ci_config.py b/scripts/generate_ci_config.py
new file mode 100644
index 000000000..55c9fd5b1
--- /dev/null
+++ b/scripts/generate_ci_config.py
@@ -0,0 +1,8 @@
+"""Utility script to generate a config for CI runs"""
+from authentik.providers.oauth2.generators import generate_client_id
+from yaml import safe_dump
+
+with open("local.env.yml", "w") as _config:
+    safe_dump({
+        "secret_key": generate_client_id()
+    }, _config, default_flow_style=False)
diff --git a/website/developer-docs/local-dev-environment.md b/website/developer-docs/local-dev-environment.md
index b3969eafc..7bb909c01 100644
--- a/website/developer-docs/local-dev-environment.md
+++ b/website/developer-docs/local-dev-environment.md
@@ -22,6 +22,7 @@ postgresql:
   user: postgres
 
 log_level: debug
+secret_key: "A long key you can generate with `pwgen 40 1` for example"
 ```
 
 Afterwards, you can start authentik by running `./manage.py runserver`. Generally speaking, authentik is a Django application.
diff --git a/website/docs/installation/docker-compose.md b/website/docs/installation/docker-compose.md
index 312d02919..06bc126f1 100644
--- a/website/docs/installation/docker-compose.md
+++ b/website/docs/installation/docker-compose.md
@@ -12,7 +12,7 @@ This installation method is for test-setups and small-scale productive setups.
 
 ## Preparation
 
-Download the latest `docker-compose.yml` from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.4/docker-compose.yml). Place it in a directory of your choice.
+Download the latest `docker-compose.yml` from [here](https://raw.githubusercontent.com/goauthentik/authentik/version/2021.5.1/docker-compose.yml). Place it in a directory of your choice.
 
 To optionally enable error-reporting, run `echo AUTHENTIK_ERROR_REPORTING__ENABLED=true >> .env`
 
diff --git a/website/docs/releases/v0.14.md b/website/docs/releases/v0.14.md
index a367680df..78a20c70f 100644
--- a/website/docs/releases/v0.14.md
+++ b/website/docs/releases/v0.14.md
@@ -51,7 +51,7 @@ This release does not introduce any new requirements.
 
 ### docker-compose
 
-Download the latest docker-compose file from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-0.14/docker-compose.yml). Afterwards, simply run `docker-compose up -d` and then the standard upgrade command of `docker-compose run --rm server migrate`.
+Download the docker-compose file for 0.14 from  [here](https://raw.githubusercontent.com/goauthentik/authentik/version-0.14/docker-compose.yml). Afterwards, simply run `docker-compose up -d` and then the standard upgrade command of `docker-compose run --rm server migrate`.
 
 ### Kubernetes
 
diff --git a/website/docs/releases/v2021.1.md b/website/docs/releases/v2021.1.md
index 304d76519..98cfafb36 100644
--- a/website/docs/releases/v2021.1.md
+++ b/website/docs/releases/v2021.1.md
@@ -60,7 +60,7 @@ This release does not introduce any new requirements.
 
 ### docker-compose
 
-Download the latest docker-compose file from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.1/docker-compose.yml). Afterwards, simply run `docker-compose up -d` and then the standard upgrade command of `docker-compose run --rm server migrate`.
+Download the docker-compose file for 2021.1 from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.1/docker-compose.yml). Afterwards, simply run `docker-compose up -d` and then the standard upgrade command of `docker-compose run --rm server migrate`.
 
 ### Kubernetes
 
diff --git a/website/docs/releases/v2021.2.md b/website/docs/releases/v2021.2.md
index 8f35157cf..44e6ec65c 100644
--- a/website/docs/releases/v2021.2.md
+++ b/website/docs/releases/v2021.2.md
@@ -124,7 +124,7 @@ The integrations affected are:
 
 ### docker-compose
 
-Download the latest docker-compose file from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.2/docker-compose.yml). Afterwards, simply run `docker-compose up -d` and then the standard upgrade command of `docker-compose run --rm server migrate`.
+Download the docker-compose file for 2021.2 from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.2/docker-compose.yml). Afterwards, simply run `docker-compose up -d` and then the standard upgrade command of `docker-compose run --rm server migrate`.
 
 ### Kubernetes
 
diff --git a/website/docs/releases/v2021.3.md b/website/docs/releases/v2021.3.md
index fc4ceaab2..402511925 100644
--- a/website/docs/releases/v2021.3.md
+++ b/website/docs/releases/v2021.3.md
@@ -87,7 +87,7 @@ This release does not introduce any new requirements.
 
 ### docker-compose
 
-Download the latest docker-compose file from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.3/docker-compose.yml). Afterwards, simply run `docker-compose up -d` and then the standard upgrade command of `docker-compose run --rm server migrate`.
+Download the docker-compose file for 2021.3 from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.3/docker-compose.yml). Afterwards, simply run `docker-compose up -d` and then the standard upgrade command of `docker-compose run --rm server migrate`.
 
 ### Kubernetes
 
diff --git a/website/docs/releases/v2021.4.md b/website/docs/releases/v2021.4.md
index 120691f6c..05828f216 100644
--- a/website/docs/releases/v2021.4.md
+++ b/website/docs/releases/v2021.4.md
@@ -133,7 +133,7 @@ This release does not introduce any new requirements.
 
 ### docker-compose
 
-Download the latest docker-compose file from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.4/docker-compose.yml). Afterwards, simply run `docker-compose up -d`.
+Download the docker-compose file for 2021.4 from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.4/docker-compose.yml). Afterwards, simply run `docker-compose up -d`.
 
 ### Kubernetes
 
diff --git a/website/docs/releases/v2021.5.md b/website/docs/releases/v2021.5.md
index dd7467e56..3885da3af 100644
--- a/website/docs/releases/v2021.5.md
+++ b/website/docs/releases/v2021.5.md
@@ -62,16 +62,19 @@ This feature is still in technical preview, so please report any Bugs you run in
 
 - core: fix application's slug field not being set to unique
 - flows: fix error when using cancel flow
+- lib: Fix config loading of secrets from files (#887)
 - lib: fix parsing of remote IP header when behind multiple reverse proxies
 - lifecycle: check if group of docker socket exists
 - lifecycle: fix error when worker is not running as root
 - outposts: fix error when controller loads from cache but cache has expired
 - outposts: fix missing default for OutpostState.for_channel
 - outposts: fix reload notification not working due to wrong ID being cached
+- outposts/ldap: fix AUTHENTIK_INSECURE not being respected for API client during bind
 - outposts/proxy: fix error redeeming code when using non-standard ports
 - outposts/proxy: fix insecure TLS Skip
 - providers/ldap: use username instead of name for user dn (#883)
 - providers/proxy: connect ingress to https instead of http
+- root: only load debug secret key when debug is enabled
 - web: fix chunks overwriting each other
 - web/admin: add notice for LDAP Provider's group selection
 - web/admin: fix PropertyMappings not loading correctly
@@ -83,7 +86,7 @@ This release does not introduce any new requirements.
 
 ### docker-compose
 
-Download the latest docker-compose file from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.5/docker-compose.yml). Afterwards, simply run `docker-compose up -d`.
+Download the docker-compose file for 2021.5 from [here](https://raw.githubusercontent.com/goauthentik/authentik/version-2021.5/docker-compose.yml). Afterwards, simply run `docker-compose up -d`.
 
 :::warning
 The public port of the compose stack has been changed from 443 to 9000 and 9443 to prevent port contention.