From aeee3ad7f9f630a1cff9bf4dd8ed489eb006a201 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Tue, 20 Oct 2020 18:42:26 +0200 Subject: [PATCH] e2e: add @retry decorator to make e2e tests more reliable --- e2e/test_flows_enroll.py | 4 +++- e2e/test_flows_login.py | 3 ++- e2e/test_flows_otp.py | 5 +++- e2e/test_flows_stage_setup.py | 3 ++- e2e/test_provider_oauth2_github.py | 5 +++- e2e/test_provider_oauth2_grafana.py | 7 +++++- e2e/test_provider_oauth2_oidc.py | 6 ++++- e2e/test_provider_proxy.py | 4 +++- e2e/test_provider_saml.py | 6 ++++- e2e/test_source_oauth.py | 6 ++++- e2e/test_source_saml.py | 5 +++- e2e/utils.py | 37 ++++++++++++++++++++++++++++- swagger.yaml | 2 +- 13 files changed, 80 insertions(+), 13 deletions(-) diff --git a/e2e/test_flows_enroll.py b/e2e/test_flows_enroll.py index 2bb947f28..0d5be0e50 100644 --- a/e2e/test_flows_enroll.py +++ b/e2e/test_flows_enroll.py @@ -8,7 +8,7 @@ from docker.types import Healthcheck from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ec -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding from passbook.stages.email.models import EmailStage, EmailTemplates from passbook.stages.identification.models import IdentificationStage @@ -34,6 +34,7 @@ class TestFlowsEnroll(SeleniumTestCase): ), } + @retry() def test_enroll_2_step(self): """Test 2-step enroll flow""" # First stage fields @@ -119,6 +120,7 @@ class TestFlowsEnroll(SeleniumTestCase): "foo@bar.baz", ) + @retry() @override_settings(EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend") def test_enroll_email(self): """Test enroll with Email verification""" diff --git a/e2e/test_flows_login.py b/e2e/test_flows_login.py index 7d549123b..c75e853ca 100644 --- a/e2e/test_flows_login.py +++ b/e2e/test_flows_login.py @@ -5,13 +5,14 @@ from unittest.case import skipUnless from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry @skipUnless(platform.startswith("linux"), "requires local docker") class TestFlowsLogin(SeleniumTestCase): """test default login flow""" + @retry() def test_login(self): """test default login flow""" self.driver.get(f"{self.live_server_url}/flows/default-authentication-flow/") diff --git a/e2e/test_flows_otp.py b/e2e/test_flows_otp.py index 556aaa17f..eeade2d86 100644 --- a/e2e/test_flows_otp.py +++ b/e2e/test_flows_otp.py @@ -12,7 +12,7 @@ from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry from passbook.flows.models import Flow, FlowStageBinding from passbook.stages.otp_validate.models import OTPValidateStage @@ -21,6 +21,7 @@ from passbook.stages.otp_validate.models import OTPValidateStage class TestFlowsOTP(SeleniumTestCase): """test flow with otp stages""" + @retry() def test_otp_validate(self): """test flow with otp stages""" sleep(1) @@ -52,6 +53,7 @@ class TestFlowsOTP(SeleniumTestCase): USER().username, ) + @retry() def test_otp_totp_setup(self): """test TOTP Setup stage""" flow: Flow = Flow.objects.get(slug="default-authentication-flow") @@ -98,6 +100,7 @@ class TestFlowsOTP(SeleniumTestCase): self.assertTrue(TOTPDevice.objects.filter(user=USER(), confirmed=True).exists()) + @retry() def test_otp_static_setup(self): """test Static OTP Setup stage""" flow: Flow = Flow.objects.get(slug="default-authentication-flow") diff --git a/e2e/test_flows_stage_setup.py b/e2e/test_flows_stage_setup.py index 355ab1600..de3c85ef8 100644 --- a/e2e/test_flows_stage_setup.py +++ b/e2e/test_flows_stage_setup.py @@ -5,7 +5,7 @@ from unittest.case import skipUnless from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry from passbook.core.models import User from passbook.flows.models import Flow, FlowDesignation from passbook.providers.oauth2.generators import generate_client_secret @@ -16,6 +16,7 @@ from passbook.stages.password.models import PasswordStage class TestFlowsStageSetup(SeleniumTestCase): """test stage setup flows""" + @retry() def test_password_change(self): """test password change flow""" # Ensure that password stage has change_flow set diff --git a/e2e/test_provider_oauth2_github.py b/e2e/test_provider_oauth2_github.py index ce755bb5d..3e2a75a58 100644 --- a/e2e/test_provider_oauth2_github.py +++ b/e2e/test_provider_oauth2_github.py @@ -9,7 +9,7 @@ from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry from passbook.core.models import Application from passbook.flows.models import Flow from passbook.policies.expression.models import ExpressionPolicy @@ -61,6 +61,7 @@ class TestProviderOAuth2Github(SeleniumTestCase): }, } + @retry() def test_authorization_consent_implied(self): """test OAuth Provider flow (default authorization flow with implied consent)""" # Bootstrap all needed objects @@ -115,6 +116,7 @@ class TestProviderOAuth2Github(SeleniumTestCase): USER().username, ) + @retry() def test_authorization_consent_explicit(self): """test OAuth Provider flow (default authorization flow with explicit consent)""" # Bootstrap all needed objects @@ -184,6 +186,7 @@ class TestProviderOAuth2Github(SeleniumTestCase): USER().username, ) + @retry() def test_denied(self): """test OAuth Provider flow (default authorization flow, denied)""" # Bootstrap all needed objects diff --git a/e2e/test_provider_oauth2_grafana.py b/e2e/test_provider_oauth2_grafana.py index a201cf24c..8d9270ea1 100644 --- a/e2e/test_provider_oauth2_grafana.py +++ b/e2e/test_provider_oauth2_grafana.py @@ -10,7 +10,7 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry from passbook.core.models import Application from passbook.crypto.models import CertificateKeyPair from passbook.flows.models import Flow @@ -80,6 +80,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase): }, } + @retry() def test_redirect_uri_error(self): """test OpenID Provider flow (invalid redirect URI, check error message)""" sleep(1) @@ -122,6 +123,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase): "Redirect URI Error", ) + @retry() def test_authorization_consent_implied(self): """test OpenID Provider flow (default authorization flow with implied consent)""" sleep(1) @@ -183,6 +185,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase): USER().email, ) + @retry() def test_authorization_logout(self): """test OpenID Provider flow with logout""" sleep(1) @@ -252,6 +255,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase): ) self.driver.find_element(By.ID, "logout").click() + @retry() def test_authorization_consent_explicit(self): """test OpenID Provider flow (default authorization flow with explicit consent)""" sleep(1) @@ -325,6 +329,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase): USER().email, ) + @retry() def test_authorization_denied(self): """test OpenID Provider flow (default authorization with access deny)""" sleep(1) diff --git a/e2e/test_provider_oauth2_oidc.py b/e2e/test_provider_oauth2_oidc.py index 42d74d1da..0a98d8303 100644 --- a/e2e/test_provider_oauth2_oidc.py +++ b/e2e/test_provider_oauth2_oidc.py @@ -12,7 +12,7 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry from passbook.core.models import Application from passbook.crypto.models import CertificateKeyPair from passbook.flows.models import Flow @@ -76,6 +76,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase): LOGGER.info("Container failed healthcheck") sleep(1) + @retry() def test_redirect_uri_error(self): """test OpenID Provider flow (invalid redirect URI, check error message)""" sleep(1) @@ -119,6 +120,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase): "Redirect URI Error", ) + @retry() def test_authorization_consent_implied(self): """test OpenID Provider flow (default authorization flow with implied consent)""" sleep(1) @@ -169,6 +171,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase): self.assertEqual(body["IDTokenClaims"]["email"], USER().email) self.assertEqual(body["UserInfo"]["email"], USER().email) + @retry() def test_authorization_consent_explicit(self): """test OpenID Provider flow (default authorization flow with explicit consent)""" sleep(1) @@ -229,6 +232,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase): self.assertEqual(body["IDTokenClaims"]["email"], USER().email) self.assertEqual(body["UserInfo"]["email"], USER().email) + @retry() def test_authorization_denied(self): """test OpenID Provider flow (default authorization with access deny)""" sleep(1) diff --git a/e2e/test_provider_proxy.py b/e2e/test_provider_proxy.py index 10918d5b0..ff1823369 100644 --- a/e2e/test_provider_proxy.py +++ b/e2e/test_provider_proxy.py @@ -11,7 +11,7 @@ from docker.models.containers import Container from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry from passbook import __version__ from passbook.core.models import Application from passbook.flows.models import Flow @@ -57,6 +57,7 @@ class TestProviderProxy(SeleniumTestCase): ) return container + @retry() def test_proxy_simple(self): """Test simple outpost setup with single provider""" proxy: ProxyProvider = ProxyProvider.objects.create( @@ -110,6 +111,7 @@ class TestProviderProxy(SeleniumTestCase): class TestProviderProxyConnect(ChannelsLiveServerTestCase): """Test Proxy connectivity over websockets""" + @retry() def test_proxy_connectivity(self): """Test proxy connectivity over websocket""" SeleniumTestCase().apply_default_data() diff --git a/e2e/test_provider_saml.py b/e2e/test_provider_saml.py index 6645e66f4..e3620bbd8 100644 --- a/e2e/test_provider_saml.py +++ b/e2e/test_provider_saml.py @@ -12,7 +12,7 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger -from e2e.utils import USER, SeleniumTestCase +from e2e.utils import USER, SeleniumTestCase, retry from passbook.core.models import Application from passbook.crypto.models import CertificateKeyPair from passbook.flows.models import Flow @@ -66,6 +66,7 @@ class TestProviderSAML(SeleniumTestCase): LOGGER.info("Container failed healthcheck") sleep(1) + @retry() def test_sp_initiated_implicit(self): """test SAML Provider flow SP-initiated flow (implicit consent)""" # Bootstrap all needed objects @@ -105,6 +106,7 @@ class TestProviderSAML(SeleniumTestCase): self.assertEqual(body["attr"]["mail"], [USER().email]) self.assertEqual(body["attr"]["uid"], [str(USER().pk)]) + @retry() def test_sp_initiated_explicit(self): """test SAML Provider flow SP-initiated flow (explicit consent)""" # Bootstrap all needed objects @@ -150,6 +152,7 @@ class TestProviderSAML(SeleniumTestCase): self.assertEqual(body["attr"]["mail"], [USER().email]) self.assertEqual(body["attr"]["uid"], [str(USER().pk)]) + @retry() def test_idp_initiated_implicit(self): """test SAML Provider flow IdP-initiated flow (implicit consent)""" # Bootstrap all needed objects @@ -195,6 +198,7 @@ class TestProviderSAML(SeleniumTestCase): self.assertEqual(body["attr"]["mail"], [USER().email]) self.assertEqual(body["attr"]["uid"], [str(USER().pk)]) + @retry() def test_sp_initiated_denied(self): """test SAML Provider flow SP-initiated flow (Policy denies access)""" # Bootstrap all needed objects diff --git a/e2e/test_source_oauth.py b/e2e/test_source_oauth.py index 3684b2219..d14d2d2ba 100644 --- a/e2e/test_source_oauth.py +++ b/e2e/test_source_oauth.py @@ -14,7 +14,7 @@ from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger from yaml import safe_dump -from e2e.utils import SeleniumTestCase +from e2e.utils import SeleniumTestCase, retry from passbook.flows.models import Flow from passbook.providers.oauth2.generators import ( generate_client_id, @@ -106,6 +106,7 @@ class TestSourceOAuth2(SeleniumTestCase): consumer_secret=self.client_secret, ) + @retry() def test_oauth_enroll(self): """test OAuth Source With With OIDC""" self.create_objects() @@ -159,6 +160,7 @@ class TestSourceOAuth2(SeleniumTestCase): "admin@example.com", ) + @retry() @override_settings(SESSION_COOKIE_SAMESITE="strict") def test_oauth_samesite_strict(self): """test OAuth Source With SameSite set to strict @@ -195,6 +197,7 @@ class TestSourceOAuth2(SeleniumTestCase): "Authentication Failed.", ) + @retry() def test_oauth_enroll_auth(self): """test OAuth Source With With OIDC (enroll and authenticate again)""" self.test_oauth_enroll() @@ -291,6 +294,7 @@ class TestSourceOAuth1(SeleniumTestCase): consumer_secret=self.client_secret, ) + @retry() def test_oauth_enroll(self): """test OAuth Source With With OIDC""" self.create_objects() diff --git a/e2e/test_source_saml.py b/e2e/test_source_saml.py index 03425f9d0..59df63c01 100644 --- a/e2e/test_source_saml.py +++ b/e2e/test_source_saml.py @@ -10,7 +10,7 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from structlog import get_logger -from e2e.utils import SeleniumTestCase +from e2e.utils import SeleniumTestCase, retry from passbook.crypto.models import CertificateKeyPair from passbook.flows.models import Flow from passbook.sources.saml.models import SAMLBindingTypes, SAMLSource @@ -92,6 +92,7 @@ class TestSourceSAML(SeleniumTestCase): }, } + @retry() def test_idp_redirect(self): """test SAML Source With redirect binding""" # Bootstrap all needed objects @@ -141,6 +142,7 @@ class TestSourceSAML(SeleniumTestCase): self.driver.find_element(By.ID, "id_username").get_attribute("value"), "" ) + @retry() def test_idp_post(self): """test SAML Source With post binding""" # Bootstrap all needed objects @@ -192,6 +194,7 @@ class TestSourceSAML(SeleniumTestCase): self.driver.find_element(By.ID, "id_username").get_attribute("value"), "" ) + @retry() def test_idp_post_auto(self): """test SAML Source With post binding (auto redirect)""" # Bootstrap all needed objects diff --git a/e2e/utils.py b/e2e/utils.py index 8e956b592..76680dbad 100644 --- a/e2e/utils.py +++ b/e2e/utils.py @@ -1,19 +1,22 @@ """passbook e2e testing utilities""" +from functools import wraps from glob import glob from importlib.util import module_from_spec, spec_from_file_location from inspect import getmembers, isfunction from os import environ, makedirs from time import sleep, time -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional from django.apps import apps from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.db import connection, transaction from django.db.utils import IntegrityError from django.shortcuts import reverse +from django.test.testcases import TestCase from docker import DockerClient, from_env from docker.models.containers import Container from selenium import webdriver +from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.remote.webdriver import WebDriver from selenium.webdriver.support.ui import WebDriverWait @@ -123,3 +126,35 @@ class SeleniumTestCase(StaticLiveServerTestCase): func(apps, schema_editor) except IntegrityError: pass + + +def retry(max_retires=3, exceptions=None): + """Retry test multiple times. Default to catching Selenium Timeout Exception""" + + if not exceptions: + exceptions = [TimeoutException] + + def retry_actual(func: Callable): + """Retry test multiple times""" + count = 1 + + @wraps(func) + def wrapper(self: TestCase, *args, **kwargs): + """Run test again if we're below max_retries, including tearDown and + setUp. Otherwise raise the error""" + nonlocal count + try: + return func(self, *args, **kwargs) + # pylint: disable=catching-non-exception + except tuple(exceptions) as exc: + count += 1 + if count > max_retires: + # pylint: disable=raising-non-exception + raise exc + self.tearDown() + self.setUp() + return wrapper(self, *args, **kwargs) + + return wrapper + + return retry_actual diff --git a/swagger.yaml b/swagger.yaml index 6ae3d39c7..794981199 100755 --- a/swagger.yaml +++ b/swagger.yaml @@ -6348,7 +6348,7 @@ definitions: for input-based policies. type: boolean re_evaluate_policies: - title: Evaluate on call + title: Re evaluate policies description: Evaluate policies when the Stage is present to the user. type: boolean order: