diff --git a/authentik/root/settings.py b/authentik/root/settings.py
index 1ceb77add..a10276c06 100644
--- a/authentik/root/settings.py
+++ b/authentik/root/settings.py
@@ -55,10 +55,6 @@ MEDIA_ROOT = BASE_DIR + "/media"
 
 DEBUG = CONFIG.y_bool("debug")
 SECRET_KEY = CONFIG.y("secret_key")
-if DEBUG:
-    SECRET_KEY = (
-        "9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s"  # noqa Debug # nosec
-    )
 
 INTERNAL_IPS = ["127.0.0.1"]
 ALLOWED_HOSTS = ["*"]
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 971ef6db2..816136c7f 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -43,7 +43,9 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run pylint authentik tests lifecycle
+              script: |
+                python -m scripts.generate_ci_config
+                pipenv run pylint authentik tests lifecycle
       - job: black
         pool:
           vmImage: 'ubuntu-latest'
@@ -140,7 +142,9 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run ./manage.py migrate
+              script: |
+                python -m scripts.generate_ci_config
+                pipenv run ./manage.py migrate
       - job: migrations_from_previous_release
         pool:
           vmImage: 'ubuntu-latest'
@@ -171,8 +175,9 @@ stages:
           - task: CmdLine@2
             displayName: Migrate to last tagged release
             inputs:
-              script:
-                pipenv run ./manage.py migrate
+              script: |
+                python -m scripts.generate_ci_config
+                pipenv run python -m lifecycle.migrate
           - task: CmdLine@2
             displayName: Install current branch
             inputs:
@@ -185,7 +190,6 @@ stages:
             inputs:
               script: |
                 pipenv run python -m lifecycle.migrate
-                pipenv run ./manage.py migrate
       - job: coverage_unittest
         pool:
           vmImage: 'ubuntu-latest'
@@ -210,6 +214,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
+                python -m scripts.generate_ci_config
                 pipenv run make test
           - task: CmdLine@2
             inputs:
@@ -253,6 +258,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
+                python -m scripts.generate_ci_config
                 pipenv run make test-integration
           - task: CmdLine@2
             inputs:
@@ -308,6 +314,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
+                python -m scripts.generate_ci_config
                 pipenv run make test-e2e
           - task: CmdLine@2
             condition: always()
diff --git a/scripts/generate_ci_config.py b/scripts/generate_ci_config.py
new file mode 100644
index 000000000..55c9fd5b1
--- /dev/null
+++ b/scripts/generate_ci_config.py
@@ -0,0 +1,8 @@
+"""Utility script to generate a config for CI runs"""
+from authentik.providers.oauth2.generators import generate_client_id
+from yaml import safe_dump
+
+with open("local.env.yml", "w") as _config:
+    safe_dump({
+        "secret_key": generate_client_id()
+    }, _config, default_flow_style=False)
diff --git a/website/developer-docs/local-dev-environment.md b/website/developer-docs/local-dev-environment.md
index 771570e65..e44bcdb77 100644
--- a/website/developer-docs/local-dev-environment.md
+++ b/website/developer-docs/local-dev-environment.md
@@ -22,6 +22,7 @@ postgresql:
   user: postgres
 
 log_level: debug
+secret_key: "A long key you can generate with `pwgen 40 1` for example"
 ```
 
 Afterwards, you can start authentik by running `./manage.py runserver`. Generally speaking, authentik is a Django application.