From 2c25d6d71ae51a855a8a0aeb3c0e450ee819a03c Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Tue, 3 Oct 2023 10:13:37 -0500 Subject: [PATCH 1/2] Run pyupgrade --py38-plus to remove pre-3.8 idioms --- lemur/__about__.py | 4 +- lemur/__init__.py | 41 ++++----- lemur/api_keys/cli.py | 7 +- lemur/api_keys/views.py | 25 +++-- lemur/auth/ldap.py | 12 +-- lemur/auth/permissions.py | 20 ++-- lemur/auth/service.py | 3 +- lemur/auth/views.py | 19 ++-- lemur/authorities/models.py | 11 +-- lemur/authorities/service.py | 20 ++-- lemur/authorities/views.py | 20 ++-- lemur/authorizations/models.py | 7 +- lemur/certificates/cli.py | 46 +++++----- lemur/certificates/models.py | 42 ++++----- lemur/certificates/schemas.py | 9 +- lemur/certificates/service.py | 17 ++-- lemur/certificates/verify.py | 24 ++--- lemur/certificates/views.py | 53 +++++------ lemur/common/celery.py | 10 +- lemur/common/defaults.py | 17 ++-- lemur/common/fields.py | 19 ++-- lemur/common/managers.py | 6 +- lemur/common/utils.py | 12 +-- lemur/common/validators.py | 15 ++- lemur/database.py | 10 +- lemur/destinations/models.py | 7 +- lemur/destinations/views.py | 17 ++-- lemur/dns_providers/cli.py | 9 +- lemur/dns_providers/models.py | 9 +- lemur/dns_providers/util.py | 8 +- lemur/dns_providers/views.py | 4 +- lemur/domains/models.py | 5 +- lemur/domains/views.py | 13 +-- lemur/endpoints/models.py | 15 ++- lemur/endpoints/service.py | 10 +- lemur/endpoints/views.py | 11 +-- lemur/exceptions.py | 6 +- lemur/factory.py | 13 ++- lemur/logs/service.py | 7 +- lemur/logs/views.py | 10 +- lemur/manage.py | 91 +++++++++---------- lemur/metrics.py | 2 +- lemur/migrations/env.py | 7 +- lemur/migrations/versions/29d8c8455c86_.py | 5 +- lemur/migrations/versions/3307381f3b88_.py | 8 +- lemur/migrations/versions/5770674184de_.py | 6 +- lemur/migrations/versions/c301c59688d2_.py | 13 ++- lemur/notifications/messaging.py | 11 +-- lemur/notifications/models.py | 11 +-- lemur/notifications/service.py | 9 +- lemur/notifications/views.py | 17 ++-- lemur/pending_certificates/cli.py | 7 +- lemur/pending_certificates/models.py | 31 +++---- lemur/pending_certificates/schemas.py | 9 +- lemur/pending_certificates/service.py | 19 ++-- lemur/pending_certificates/views.py | 21 ++--- lemur/plugins/__init__.py | 2 - lemur/plugins/base/__init__.py | 1 - lemur/plugins/base/manager.py | 6 +- lemur/plugins/lemur_acme/acme_handlers.py | 35 ++++--- lemur/plugins/lemur_acme/challenge_types.py | 20 ++-- lemur/plugins/lemur_acme/cloudflare.py | 8 +- lemur/plugins/lemur_acme/dyn.py | 25 +++-- lemur/plugins/lemur_acme/plugin.py | 12 +-- lemur/plugins/lemur_acme/route53.py | 8 +- lemur/plugins/lemur_acme/ultradns.py | 10 +- lemur/plugins/lemur_adcs/plugin.py | 20 ++-- lemur/plugins/lemur_atlas/plugin.py | 7 +- lemur/plugins/lemur_aws/cloudfront.py | 5 +- lemur/plugins/lemur_aws/plugin.py | 12 +-- lemur/plugins/lemur_aws/s3.py | 2 +- lemur/plugins/lemur_aws/sns.py | 3 +- lemur/plugins/lemur_azure_dest/plugin.py | 17 ++-- lemur/plugins/lemur_cfssl/plugin.py | 25 +++-- lemur/plugins/lemur_cryptography/plugin.py | 13 +-- lemur/plugins/lemur_csr/plugin.py | 6 +- lemur/plugins/lemur_digicert/plugin.py | 42 ++++----- .../lemur_digicert/tests/test_digicert.py | 8 +- lemur/plugins/lemur_email/plugin.py | 14 ++- lemur/plugins/lemur_entrust/plugin.py | 20 ++-- .../lemur_entrust/tests/test_entrust.py | 4 +- lemur/plugins/lemur_jks/plugin.py | 2 +- lemur/plugins/lemur_kubernetes/plugin.py | 17 ++-- lemur/plugins/lemur_openssl/plugin.py | 18 ++-- lemur/plugins/lemur_sftp/plugin.py | 31 +++---- lemur/plugins/lemur_slack/tests/test_slack.py | 5 +- .../lemur_statsd/lemur_statsd/plugin.py | 5 +- lemur/plugins/lemur_statsd/setup.py | 1 - lemur/plugins/lemur_vault_dest/plugin.py | 23 +++-- lemur/plugins/lemur_verisign/plugin.py | 18 ++-- lemur/plugins/views.py | 8 +- lemur/roles/models.py | 9 +- lemur/roles/views.py | 19 ++-- lemur/schemas.py | 25 +++-- lemur/sources/cli.py | 27 +++--- lemur/sources/models.py | 9 +- lemur/sources/service.py | 40 ++++---- lemur/sources/views.py | 17 ++-- lemur/tests/conftest.py | 9 +- lemur/tests/factories.py | 34 +++---- lemur/tests/plugins/destination_plugin.py | 4 +- lemur/tests/plugins/issuer_plugin.py | 11 +-- lemur/tests/plugins/notification_plugin.py | 2 +- lemur/tests/plugins/source_plugin.py | 2 +- lemur/tests/test_certificates.py | 11 +-- lemur/tests/test_ldap.py | 7 +- lemur/users/models.py | 13 +-- lemur/users/views.py | 10 +- lemur/utils.py | 6 +- 109 files changed, 719 insertions(+), 869 deletions(-) diff --git a/lemur/__about__.py b/lemur/__about__.py index 395fecf206..fda30c213a 100644 --- a/lemur/__about__.py +++ b/lemur/__about__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - __all__ = [ "__title__", "__summary__", @@ -21,4 +19,4 @@ __email__ = "security@netflix.com" __license__ = "Apache License, Version 2.0" -__copyright__ = "Copyright 2018 {0}".format(__author__) +__copyright__ = f"Copyright 2018 {__author__}" diff --git a/lemur/__init__.py b/lemur/__init__.py index f766f61103..3ec38a4677 100644 --- a/lemur/__init__.py +++ b/lemur/__init__.py @@ -12,28 +12,9 @@ import socket import time import urllib.parse -from flask import g, request +from flask import g, request from lemur import factory -from lemur.extensions import metrics - -from lemur.users.views import mod as users_bp -from lemur.roles.views import mod as roles_bp -from lemur.auth.views import mod as auth_bp -from lemur.domains.views import mod as domains_bp -from lemur.destinations.views import mod as destinations_bp -from lemur.authorities.views import mod as authorities_bp -from lemur.certificates.views import mod as certificates_bp -from lemur.defaults.views import mod as defaults_bp -from lemur.plugins.views import mod as plugins_bp -from lemur.notifications.views import mod as notifications_bp -from lemur.sources.views import mod as sources_bp -from lemur.endpoints.views import mod as endpoints_bp -from lemur.logs.views import mod as logs_bp -from lemur.api_keys.views import mod as api_key_bp -from lemur.pending_certificates.views import mod as pending_certificates_bp -from lemur.dns_providers.views import mod as dns_providers_bp - from lemur.__about__ import ( __author__, __copyright__, @@ -44,7 +25,23 @@ __uri__, __version__, ) - +from lemur.api_keys.views import mod as api_key_bp +from lemur.auth.views import mod as auth_bp +from lemur.authorities.views import mod as authorities_bp +from lemur.certificates.views import mod as certificates_bp +from lemur.defaults.views import mod as defaults_bp +from lemur.destinations.views import mod as destinations_bp +from lemur.dns_providers.views import mod as dns_providers_bp +from lemur.domains.views import mod as domains_bp +from lemur.endpoints.views import mod as endpoints_bp +from lemur.extensions import metrics +from lemur.logs.views import mod as logs_bp +from lemur.notifications.views import mod as notifications_bp +from lemur.pending_certificates.views import mod as pending_certificates_bp +from lemur.plugins.views import mod as plugins_bp +from lemur.roles.views import mod as roles_bp +from lemur.sources.views import mod as sources_bp +from lemur.users.views import mod as users_bp __all__ = [ "__title__", @@ -184,5 +181,5 @@ def sanitize_path(*, path: str) -> str: # Record our response time metric metrics.send("response_time", "TIMER", elapsed, metric_tags=tags) - metrics.send("status_code_{}".format(response.status_code), "counter", 1) + metrics.send(f"status_code_{response.status_code}", "counter", 1) return response diff --git a/lemur/api_keys/cli.py b/lemur/api_keys/cli.py index ef23573813..6069d5b3b6 100644 --- a/lemur/api_keys/cli.py +++ b/lemur/api_keys/cli.py @@ -5,13 +5,12 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Eric Coan """ -import click +from datetime import datetime +import click from lemur.api_keys import service as api_key_service from lemur.auth.service import create_token -from datetime import datetime - @click.group(name="api_keys", help="Handles all api key related tasks.") def cli(): @@ -45,7 +44,7 @@ def create(uid, name, ttl): ) click.echo("[+] Successfully created a new api key. Generating a JWT...") jwt = create_token(uid, key.id, key.ttl) - click.echo("[+] Your JWT is: {jwt}".format(jwt=jwt)) + click.echo(f"[+] Your JWT is: {jwt}") @cli.command("revoke") diff --git a/lemur/api_keys/views.py b/lemur/api_keys/views.py index 367feb07d6..c72f1d0ac7 100644 --- a/lemur/api_keys/views.py +++ b/lemur/api_keys/views.py @@ -11,14 +11,7 @@ from flask import Blueprint, g from flask_restful import reqparse, Api - from lemur.api_keys import service -from lemur.auth.service import AuthenticatedResource, create_token -from lemur.auth.permissions import ApiKeyCreatorPermission - -from lemur.common.schema import validate_schema -from lemur.common.utils import paginated_parser - from lemur.api_keys.schemas import ( api_key_input_schema, api_key_revoke_schema, @@ -27,6 +20,10 @@ api_key_described_output_schema, user_api_key_input_schema, ) +from lemur.auth.permissions import ApiKeyCreatorPermission +from lemur.auth.service import AuthenticatedResource, create_token +from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser mod = Blueprint("api_keys", __name__) api = Api(mod) @@ -36,7 +33,7 @@ class ApiKeyList(AuthenticatedResource): """ Defines the 'api_keys' endpoint """ def __init__(self): - super(ApiKeyList, self).__init__() + super().__init__() @validate_schema(None, api_keys_output_schema) def get(self): @@ -134,7 +131,7 @@ def post(self, data=None): if data["user"]["id"] != g.current_user.id: return ( dict( - message="You are not authorized to create tokens for: {0}".format( + message="You are not authorized to create tokens for: {}".format( data["user"]["username"] ) ), @@ -157,7 +154,7 @@ class ApiKeyUserList(AuthenticatedResource): """ Defines the 'keys' endpoint on the 'users' endpoint. """ def __init__(self): - super(ApiKeyUserList, self).__init__() + super().__init__() @validate_schema(None, api_keys_output_schema) def get(self, user_id): @@ -254,7 +251,7 @@ def post(self, user_id, data=None): if user_id != g.current_user.id: return ( dict( - message="You are not authorized to create tokens for: {0}".format( + message="You are not authorized to create tokens for: {}".format( user_id ) ), @@ -276,7 +273,7 @@ def post(self, user_id, data=None): class ApiKeys(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(ApiKeys, self).__init__() + super().__init__() @validate_schema(None, api_key_output_schema) def get(self, aid): @@ -416,7 +413,7 @@ def delete(self, aid): class UserApiKeys(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(UserApiKeys, self).__init__() + super().__init__() @validate_schema(None, api_key_output_schema) def get(self, uid, aid): @@ -565,7 +562,7 @@ def delete(self, uid, aid): class ApiKeysDescribed(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(ApiKeysDescribed, self).__init__() + super().__init__() @validate_schema(None, api_key_described_output_schema) def get(self, aid): diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index 846431d55b..4c1733f262 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -29,7 +29,7 @@ def __init__(self, args): self.ldap_principal = args["username"] self.ldap_email_domain = current_app.config.get("LDAP_EMAIL_DOMAIN", None) if "@" not in self.ldap_principal: - self.ldap_principal = "%s@%s" % ( + self.ldap_principal = "{}@{}".format( self.ldap_principal, self.ldap_email_domain, ) @@ -110,7 +110,7 @@ def _authorize(self): # update their 'roles' role = role_service.get_by_name(self.ldap_principal) if not role: - description = "auto generated role based on owner: {0}".format( + description = "auto generated role based on owner: {}".format( self.ldap_principal ) role = role_service.create( @@ -127,7 +127,7 @@ def _authorize(self): if role: if ldap_group_name in self.ldap_groups: current_app.logger.debug( - "assigning role {0} to ldap user {1}".format( + "assigning role {} to ldap user {}".format( self.ldap_principal, role ) ) @@ -156,7 +156,7 @@ def _bind(self): raise an exception on error. """ if "@" not in self.ldap_principal: - self.ldap_principal = "%s@%s" % ( + self.ldap_principal = "{}@{}".format( self.ldap_principal, self.ldap_email_domain, ) @@ -186,7 +186,7 @@ def _bind(self): except ldap.SERVER_DOWN: raise Exception("ldap server unavailable") except ldap.LDAPError as e: - raise Exception("ldap error: {0}".format(e)) + raise Exception(f"ldap error: {e}") if self.ldap_is_active_directory: # Lookup user DN, needed to search for group membership @@ -198,7 +198,7 @@ def _bind(self): )[0][1]["distinguishedName"][0] userdn = userdn.decode("utf-8") # Search all groups that have the userDN as a member - groupfilter = "(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))".format( + groupfilter = "(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={}))".format( userdn ) lgroups = self.ldap_client.search_s( diff --git a/lemur/auth/permissions.py b/lemur/auth/permissions.py index 933cc1e9b0..bb7741f888 100644 --- a/lemur/auth/permissions.py +++ b/lemur/auth/permissions.py @@ -6,8 +6,8 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from functools import partial from collections import namedtuple +from functools import partial from flask import current_app from flask_principal import Permission, RoleNeed @@ -29,7 +29,7 @@ def __init__(self): for role in sensitive_domain_roles: needs.append(RoleNeed(role)) - super(SensitiveDomainPermission, self).__init__(*needs) + super().__init__(*needs) class CertificatePermission(Permission): @@ -41,12 +41,12 @@ def __init__(self, owner, roles): if str(r) != str(r).lower(): needs.append(CertificateOwnerNeed(str(r).lower())) - super(CertificatePermission, self).__init__(*needs) + super().__init__(*needs) class ApiKeyCreatorPermission(Permission): def __init__(self): - super(ApiKeyCreatorPermission, self).__init__(RoleNeed("admin")) + super().__init__(RoleNeed("admin")) RoleMember = namedtuple("RoleMember", ["method", "value"]) @@ -56,16 +56,16 @@ def __init__(self): class RoleMemberPermission(Permission): def __init__(self, role_id): needs = [RoleNeed("admin"), RoleMemberNeed(role_id)] - super(RoleMemberPermission, self).__init__(*needs) + super().__init__(*needs) class AuthorityCreatorPermission(Permission): def __init__(self): requires_admin = current_app.config.get("ADMIN_ONLY_AUTHORITY_CREATION", False) if requires_admin: - super(AuthorityCreatorPermission, self).__init__(RoleNeed("admin")) + super().__init__(RoleNeed("admin")) else: - super(AuthorityCreatorPermission, self).__init__() + super().__init__() AuthorityCreator = namedtuple("AuthorityCreator", ["method", "value"]) @@ -81,7 +81,7 @@ def __init__(self, authority_id, roles): for r in roles: needs.append(AuthorityOwnerNeed(str(r))) - super(AuthorityPermission, self).__init__(*needs) + super().__init__(*needs) class StrictRolePermission(Permission): @@ -89,6 +89,6 @@ def __init__(self): strict_role_enforcement = current_app.config.get("LEMUR_STRICT_ROLE_ENFORCEMENT", False) if strict_role_enforcement: needs = [RoleNeed("admin"), RoleNeed("operator")] - super(StrictRolePermission, self).__init__(*needs) + super().__init__(*needs) else: - super(StrictRolePermission, self).__init__() + super().__init__() diff --git a/lemur/auth/service.py b/lemur/auth/service.py index 6f2b83b391..e082f18f39 100644 --- a/lemur/auth/service.py +++ b/lemur/auth/service.py @@ -21,7 +21,6 @@ from flask_principal import Identity, identity_changed from flask_principal import identity_loaded, RoleNeed, UserNeed from flask_restful import Resource - from lemur.api_keys import service as api_key_service from lemur.auth.permissions import AuthorityCreatorNeed, RoleMemberNeed from lemur.users import service as user_service @@ -209,4 +208,4 @@ class AuthenticatedResource(Resource): method_decorators = [login_required] def __init__(self): - super(AuthenticatedResource, self).__init__() + super().__init__() diff --git a/lemur/auth/views.py b/lemur/auth/views.py index 4ea5ce439a..9a1eefa9e0 100644 --- a/lemur/auth/views.py +++ b/lemur/auth/views.py @@ -21,7 +21,6 @@ from flask_limiter.util import get_remote_address from flask_principal import Identity, identity_changed from flask_restful import reqparse, Resource, Api - from lemur.auth import ldap from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key from lemur.common.utils import get_psuedo_random_string, get_state_token_secret @@ -63,7 +62,7 @@ def exchange_for_access_token( } # the secret and cliendId will be given to you when you signup for the provider - token = "{0}:{1}".format(client_id, secret) + token = f"{client_id}:{secret}" basic = base64.b64encode(bytes(token, "utf-8")) headers = { @@ -71,9 +70,9 @@ def exchange_for_access_token( } if current_app.config.get("TOKEN_AUTH_HEADER_CASE_SENSITIVE"): - headers["Authorization"] = "Basic {0}".format(basic.decode("utf-8")) + headers["Authorization"] = "Basic {}".format(basic.decode("utf-8")) else: - headers["authorization"] = "basic {0}".format(basic.decode("utf-8")) + headers["authorization"] = "basic {}".format(basic.decode("utf-8")) # exchange authorization code for access token. r = requests.post( @@ -400,7 +399,7 @@ class Login(Resource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Login, self).__init__() + super().__init__() @limiter.limit("10/5minute") def post(self): @@ -480,7 +479,7 @@ def post(self): ) return dict(token=create_token(user)) except Exception as e: - current_app.logger.error("ldap error: {0}".format(e)) + current_app.logger.error(f"ldap error: {e}") ldap_message = "ldap error: %s" % e metrics.send( "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS} @@ -507,7 +506,7 @@ class Ping(Resource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Ping, self).__init__() + super().__init__() def get(self): return "Redirecting..." @@ -567,7 +566,7 @@ def post(self): class OAuth2(Resource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(OAuth2, self).__init__() + super().__init__() def get(self): return "Redirecting..." @@ -630,7 +629,7 @@ def post(self): class Google(Resource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Google, self).__init__() + super().__init__() def post(self): access_token_url = "https://accounts.google.com/o/oauth2/token" @@ -658,7 +657,7 @@ def post(self): token = r.json() # Step 2. Retrieve information about the current user - headers = {"Authorization": "Bearer {0}".format(token["access_token"])} + headers = {"Authorization": "Bearer {}".format(token["access_token"])} r = requests.get(user_info_url, headers=headers) profile = r.json() diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index 224d3b4847..6502bb22d3 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -9,7 +9,9 @@ import json from flask import current_app -from sqlalchemy.orm import relationship +from lemur.database import BaseModel, db +from lemur.models import roles_authorities +from lemur.plugins.base import plugins from sqlalchemy import ( Column, Integer, @@ -22,10 +24,7 @@ Boolean, ) from sqlalchemy.dialects.postgresql import JSON - -from lemur.database import BaseModel, db -from lemur.plugins.base import plugins -from lemur.models import roles_authorities +from sqlalchemy.orm import relationship class Authority(BaseModel): @@ -122,7 +121,7 @@ def default_validity_days(self): return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365) # 1 year default def __repr__(self): - return "Authority(name={name})".format(name=self.name) + return f"Authority(name={self.name})" @property def is_cn_optional(self): diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index 144953f374..d87878f9a4 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -12,16 +12,14 @@ import json from flask import current_app - from lemur import database -from lemur.common.utils import truthiness, data_encrypt -from lemur.extensions import metrics from lemur.authorities.models import Authority from lemur.certificates.models import Certificate -from lemur.roles import service as role_service -from lemur.logs import service as log_service - from lemur.certificates.service import upload +from lemur.common.utils import truthiness, data_encrypt +from lemur.extensions import metrics +from lemur.logs import service as log_service +from lemur.roles import service as role_service def update(authority_id, description, owner, active, roles): @@ -100,7 +98,7 @@ def create_authority_roles(roles, owner, plugin_title, creator): role = role_service.create( r["name"], password=r["password"], - description="Auto generated role for {0}".format(plugin_title), + description=f"Auto generated role for {plugin_title}", username=r["username"], ) @@ -110,7 +108,7 @@ def create_authority_roles(roles, owner, plugin_title, creator): owner_role = role_service.get_by_name(owner) if not owner_role: owner_role = role_service.create( - owner, description="Auto generated role based on owner: {0}".format(owner) + owner, description=f"Auto generated role based on owner: {owner}" ) role_objs.append(owner_role) @@ -211,8 +209,8 @@ def get_authority_role(ca_name, creator=None): """ if creator: if creator.is_admin: - return role_service.get_by_name("{0}_admin".format(ca_name)) - return role_service.get_by_name("{0}_operator".format(ca_name)) + return role_service.get_by_name(f"{ca_name}_admin") + return role_service.get_by_name(f"{ca_name}_operator") def render(args): @@ -230,7 +228,7 @@ def render(args): if "active" in filt: query = query.filter(Authority.active == truthiness(terms[1])) elif "cn" in filt: - term = "%{0}%".format(terms[1]) + term = f"%{terms[1]}%" sub_query = ( database.session_query(Certificate.root_authority_id) .filter(Certificate.cn.ilike(term)) diff --git a/lemur/authorities/views.py b/lemur/authorities/views.py index 4a723f183f..bf238410c4 100644 --- a/lemur/authorities/views.py +++ b/lemur/authorities/views.py @@ -7,15 +7,8 @@ """ from flask import Blueprint, g from flask_restful import reqparse, Api - -from lemur.common import validators -from lemur.common.utils import paginated_parser -from lemur.common.schema import validate_schema -from lemur.auth.service import AuthenticatedResource from lemur.auth.permissions import AuthorityCreatorPermission, AuthorityPermission, StrictRolePermission - -from lemur.certificates import service as certificate_service - +from lemur.auth.service import AuthenticatedResource from lemur.authorities import service from lemur.authorities.schemas import ( authority_input_schema, @@ -23,7 +16,10 @@ authorities_output_schema, authority_update_schema, ) - +from lemur.certificates import service as certificate_service +from lemur.common import validators +from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser mod = Blueprint("authorities", __name__) api = Api(mod) @@ -34,7 +30,7 @@ class AuthoritiesList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(AuthoritiesList, self).__init__() + super().__init__() @validate_schema(None, authorities_output_schema) def get(self): @@ -243,7 +239,7 @@ def post(self, data=None): class Authorities(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Authorities, self).__init__() + super().__init__() @validate_schema(None, authority_output_schema) def get(self, authority_id): @@ -425,7 +421,7 @@ def put(self, authority_id, data=None): class CertificateAuthority(AuthenticatedResource): def __init__(self): - super(CertificateAuthority, self).__init__() + super().__init__() @validate_schema(None, authority_output_schema) def get(self, certificate_id): diff --git a/lemur/authorizations/models.py b/lemur/authorizations/models.py index f727228ae3..773d31dfb6 100644 --- a/lemur/authorizations/models.py +++ b/lemur/authorizations/models.py @@ -5,11 +5,10 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Netflix Secops """ -from sqlalchemy import Column, Integer, String -from sqlalchemy_utils import JSONType from lemur.database import BaseModel - from lemur.plugins.base import plugins +from sqlalchemy import Column, Integer, String +from sqlalchemy_utils import JSONType class Authorization(BaseModel): @@ -25,7 +24,7 @@ def plugin(self): return plugins.get(self.plugin_name) def __repr__(self): - return "Authorization(id={id})".format(id=self.id) + return f"Authorization(id={self.id})" def __init__(self, account_number, domains, dns_provider_type, options=None): self.account_number = account_number diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index f7fea793bd..491ed1cd02 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -5,17 +5,13 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import arrow import sys -import click +from time import sleep +import arrow +import click from flask import current_app from flask_principal import Identity, identity_changed -from sqlalchemy import or_ -from tabulate import tabulate -from time import sleep -from sentry_sdk import capture_exception - from lemur import database from lemur.authorities.models import Authority from lemur.authorities.service import get as authorities_get_by_id @@ -46,7 +42,10 @@ from lemur.notifications.messaging import send_rotation_notification, send_reissue_no_endpoints_notification, \ send_reissue_failed_notification from lemur.plugins.base import plugins +from sentry_sdk import capture_exception +from sqlalchemy import or_ from sqlalchemy.orm.exc import MultipleResultsFound +from tabulate import tabulate @click.group(name="certificates", help="Handles all certificate related tasks.") @@ -90,7 +89,7 @@ def validate_certificate(certificate_name): cert = get_by_name(certificate_name) if not cert: - click.echo("[-] No certificate found with name: {0}".format(certificate_name)) + click.echo(f"[-] No certificate found with name: {certificate_name}") sys.exit(1) return cert @@ -106,7 +105,7 @@ def validate_endpoint(endpoint_name): endpoint = endpoint_service.get_by_name(endpoint_name) if not endpoint: - click.echo("[-] No endpoint found with name: {0}".format(endpoint_name)) + click.echo(f"[-] No endpoint found with name: {endpoint_name}") sys.exit(1) return endpoint @@ -123,7 +122,7 @@ def validate_endpoint_from_source(endpoint_name, source): endpoint = endpoint_service.get_by_name_and_source(endpoint_name, source) if not endpoint: - click.echo("[-] No endpoint found from source {0} with name: {1}".format(source, endpoint_name)) + click.echo(f"[-] No endpoint found from source {source} with name: {endpoint_name}") sys.exit(1) return endpoint @@ -156,7 +155,7 @@ def request_rotation(endpoint, certificate, message, commit): f"Error rotating certificate: {certificate.name}", exc_info=True ) click.echo( - "[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format( + "[!] Failed to rotate endpoint {} to certificate {} reason: {}".format( endpoint.name, certificate.name, e ) ) @@ -177,7 +176,7 @@ def request_reissue(certificate, notify, commit): status = FAILURE_METRIC_STATUS notify = notify and certificate.notify try: - click.echo("[+] {0} is eligible for re-issuance".format(certificate.name)) + click.echo(f"[+] {certificate.name} is eligible for re-issuance") # set the lemur identity for all cli commands identity_changed.send(current_app._get_current_object(), identity=Identity(1)) @@ -187,7 +186,7 @@ def request_reissue(certificate, notify, commit): if commit: new_cert = reissue_certificate(certificate, notify=notify, replace=True) - click.echo("[+] New certificate named: {0}".format(new_cert.name)) + click.echo(f"[+] New certificate named: {new_cert.name}") if notify and isinstance(new_cert, Certificate): # let celery handle PendingCertificates send_reissue_no_endpoints_notification(certificate, new_cert) @@ -279,7 +278,8 @@ def rotate(endpoint_name, source, new_certificate_name, old_certificate_name, me try: endpoint = validate_endpoint(endpoint_name) except MultipleResultsFound as e: - click.echo("[!] Multiple endpoints found with name {0}, try narrowing the search down to an endpoint from a specific source by re-running this command with the --source flag.".format(endpoint_name)) + click.echo( + f"[!] Multiple endpoints found with name {endpoint_name}, try narrowing the search down to an endpoint from a specific source by re-running this command with the --source flag.") log_data["message"] = "Multiple endpoints found with same name, unable to perform rotation" log_data["duplicated_endpoint_name"] = endpoint_name current_app.logger.info(log_data) @@ -581,7 +581,7 @@ def reissue(old_certificate_name, notify, commit): except Exception as e: capture_exception() current_app.logger.exception("Error reissuing certificate.", exc_info=True) - click.echo("[!] Failed to reissue certificates. Reason: {}".format(e)) + click.echo(f"[!] Failed to reissue certificates. Reason: {e}") metrics.send( "certificate_reissue_job", "counter", 1, metric_tags={"status": status} @@ -617,18 +617,18 @@ def query(fqdns, issuer, owner, expired): if issuer: sub_query = ( database.session_query(Authority.id) - .filter(Authority.name.ilike("%{0}%".format(issuer))) + .filter(Authority.name.ilike(f"%{issuer}%")) .subquery() ) q = q.filter( or_( - Certificate.issuer.ilike("%{0}%".format(issuer)), + Certificate.issuer.ilike(f"%{issuer}%"), Certificate.authority_id.in_(sub_query), ) ) if owner: - q = q.filter(Certificate.owner.ilike("%{0}%".format(owner))) + q = q.filter(Certificate.owner.ilike(f"%{owner}%")) if not expired: q = q.filter(Certificate.expired == False) # noqa @@ -637,8 +637,8 @@ def query(fqdns, issuer, owner, expired): for f in fqdns.split(","): q = q.filter( or_( - Certificate.cn.ilike("%{0}%".format(f)), - Certificate.domains.any(Domain.name.ilike("%{0}%".format(f))), + Certificate.cn.ilike(f"%{f}%"), + Certificate.domains.any(Domain.name.ilike(f"%{f}%")), ) ) @@ -653,7 +653,7 @@ def worker(data, commit, reason): try: cert = get(int(parts[0].strip())) - click.echo("[+] Revoking certificate. Id: {0} Name: {1}".format(cert.id, cert.name)) + click.echo(f"[+] Revoking certificate. Id: {cert.id} Name: {cert.name}") if commit: revoke_certificate(cert, reason) @@ -672,7 +672,7 @@ def worker(data, commit, reason): 1, metric_tags={"status": FAILURE_METRIC_STATUS}, ) - click.echo("[!] Failed to revoke certificates. Reason: {}".format(e)) + click.echo(f"[!] Failed to revoke certificates. Reason: {e}") @cli.command("clear_pending") @@ -729,7 +729,7 @@ def revoke(path, cert_id, reason, message, commit): if cert_id: worker(cert_id, commit, comments) else: - with open(path, "r") as f: + with open(path) as f: for x in f.readlines()[2:]: worker(x, commit, comments) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 22b899e17f..e7be34e5c1 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -11,6 +11,22 @@ from cryptography import x509 from flask import current_app from idna.core import InvalidCodepoint +from lemur.common import defaults, utils, validators +from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS +from lemur.database import BaseModel +from lemur.domains.models import Domain +from lemur.extensions import metrics +from lemur.models import ( + certificate_source_associations, + certificate_destination_associations, + certificate_notification_associations, + certificate_replacement_associations, + roles_certificates, + pending_cert_replacement_associations, +) +from lemur.plugins.base import plugins +from lemur.policies.models import RotationPolicy +from lemur.utils import Vault from sentry_sdk import capture_exception from sqlalchemy import ( event, @@ -33,24 +49,6 @@ from werkzeug.utils import cached_property -from lemur.common import defaults, utils, validators -from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS -from lemur.database import BaseModel -from lemur.domains.models import Domain -from lemur.extensions import metrics -from lemur.models import ( - certificate_source_associations, - certificate_destination_associations, - certificate_notification_associations, - certificate_replacement_associations, - roles_certificates, - pending_cert_replacement_associations, -) -from lemur.plugins.base import plugins -from lemur.policies.models import RotationPolicy -from lemur.utils import Vault - - def get_sequence(name): if "-" not in name: return name, None @@ -77,14 +75,14 @@ def get_or_increase_name(name, serial): if not certificates: return name - serial_name = "{0}-{1}".format(name, hex(int(serial))[2:].upper()) + serial_name = f"{name}-{hex(int(serial))[2:].upper()}" certificates = Certificate.query.filter(Certificate.name == serial_name).all() if not certificates: return serial_name certificates = Certificate.query.filter( - Certificate.name.ilike("{0}%".format(serial_name)) + Certificate.name.ilike(f"{serial_name}%") ).all() ends = [0] @@ -94,7 +92,7 @@ def get_or_increase_name(name, serial): if end: ends.append(end) - return "{0}-{1}".format(root, max(ends) + 1) + return f"{root}-{max(ends) + 1}" class Certificate(BaseModel): @@ -436,7 +434,7 @@ def extensions(self): return return_extensions def __repr__(self): - return "Certificate(name={name})".format(name=self.name) + return f"Certificate(name={self.name})" class CertificateAssociation(BaseModel): diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index b76b5e683b..e643ca3817 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -8,9 +8,6 @@ from flask import current_app from flask_restful import inputs from flask_restful.reqparse import RequestParser -from marshmallow import fields, validate, validates_schema, post_load, pre_load, post_dump -from marshmallow.exceptions import ValidationError - from lemur.authorities.schemas import AuthorityNestedOutputSchema from lemur.certificates import utils as cert_utils from lemur.common import missing, utils, validators @@ -38,6 +35,8 @@ AssociatedRotationPolicySchema, ) from lemur.users.schemas import UserNestedOutputSchema +from marshmallow import fields, validate, validates_schema, post_load, pre_load, post_dump +from marshmallow.exceptions import ValidationError class CertificateSchema(LemurInputSchema): @@ -52,7 +51,7 @@ def default_notification(self, data): data[ "notifications" ] += notification_service.create_default_expiration_notifications( - "DEFAULT_{0}".format(data["owner"].split("@")[0].upper()), + "DEFAULT_{}".format(data["owner"].split("@")[0].upper()), [data["owner"]], ) @@ -228,7 +227,7 @@ def enforce_notifications(self, data): :return: """ if data.get("owner"): - notification_name = "DEFAULT_{0}".format( + notification_name = "DEFAULT_{}".format( data["owner"].split("@")[0].upper() ) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 7cd3a6c26a..ce58e85750 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -15,10 +15,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from flask import current_app -from sentry_sdk import capture_exception -from sqlalchemy import and_, func, or_, not_, cast, Integer -from sqlalchemy.sql.expression import false, true - from lemur import database from lemur.authorities.models import Authority from lemur.certificates.models import Certificate, CertificateAssociation @@ -37,6 +33,9 @@ from lemur.plugins.utils import get_plugin_option from lemur.roles import service as role_service from lemur.roles.models import Role +from sentry_sdk import capture_exception +from sqlalchemy import and_, func, or_, not_, cast, Integer +from sqlalchemy.sql.expression import false, true csr_created = signals.signal("csr_created", "CSR generated") csr_imported = signals.signal("csr_imported", "CSR imported from external source") @@ -598,7 +597,7 @@ def render(args): if filt: terms = filt.split(";") - term = "%{0}%".format(terms[1]) + term = f"%{terms[1]}%" # Exact matches for quotes. Only applies to name, issuer, and cn if terms[1].startswith('"') and terms[1].endswith('"'): term = terms[1][1:-1] @@ -804,7 +803,7 @@ def create_csr(**csr_config): if v: if k in critical_extensions: current_app.logger.debug( - "Adding Critical Extension: {0} {1}".format(k, v) + f"Adding Critical Extension: {k} {v}" ) if k == "sub_alt_names": if v["names"]: @@ -813,7 +812,7 @@ def create_csr(**csr_config): builder = builder.add_extension(v, critical=True) if k in noncritical_extensions: - current_app.logger.debug("Adding Extension: {0} {1}".format(k, v)) + current_app.logger.debug(f"Adding Extension: {k} {v}") builder = builder.add_extension(v, critical=False) ski = extensions.get("subject_key_identifier", {}) @@ -1185,13 +1184,13 @@ def get_certs_for_expiring_deployed_cert_check(exclude_domains, exclude_owners): exclude_conditions = [] if exclude_domains: for e in exclude_domains: - exclude_conditions.append(~Certificate.name.ilike("%{}%".format(e))) + exclude_conditions.append(~Certificate.name.ilike(f"%{e}%")) q = q.filter(and_(*exclude_conditions)) if exclude_owners: for e in exclude_owners: - exclude_conditions.append(~Certificate.owner.ilike("{}".format(e))) + exclude_conditions.append(~Certificate.owner.ilike(f"{e}")) q = q.filter(and_(*exclude_conditions)) diff --git a/lemur/certificates/verify.py b/lemur/certificates/verify.py index 4c1e0d24e5..186ee35f96 100644 --- a/lemur/certificates/verify.py +++ b/lemur/certificates/verify.py @@ -5,18 +5,18 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import requests import subprocess -from flask import current_app -from requests.exceptions import ConnectionError, InvalidSchema, Timeout -from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from sentry_sdk import capture_exception from subprocess import TimeoutExpired -from lemur.utils import mktempfile +import requests +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from flask import current_app from lemur.common.utils import parse_certificate from lemur.extensions import metrics +from lemur.utils import mktempfile +from requests.exceptions import ConnectionError, InvalidSchema, Timeout +from sentry_sdk import capture_exception crl_cache = {} @@ -38,7 +38,7 @@ def ocsp_verify(cert, cert_path, issuer_chain_path): if not url: current_app.logger.debug( - "No OCSP URL in certificate {}".format(cert.serial_number) + f"No OCSP URL in certificate {cert.serial_number}" ) return None @@ -110,7 +110,7 @@ def crl_verify(cert, cert_path): ).value except x509.ExtensionNotFound: current_app.logger.debug( - "No CRLDP extension in certificate {}".format(cert.serial_number) + f"No CRLDP extension in certificate {cert.serial_number}" ) return None @@ -134,7 +134,7 @@ def crl_verify(cert, cert_path): response.content, backend=default_backend() ) else: - current_app.logger.debug("CRL point is cached {}".format(point)) + current_app.logger.debug(f"CRL point is cached {point}") for r in crl_cache[point]: if cert.serial_number == r.serial_number: @@ -165,7 +165,7 @@ def verify(cert_path, issuer_chain_path): :param issuer_chain_path: :return: True if valid, False otherwise """ - with open(cert_path, "rt") as c: + with open(cert_path) as c: try: cert = parse_certificate(c.read()) except ValueError as e: @@ -193,7 +193,7 @@ def verify(cert_path, issuer_chain_path): crl_err = 1 if verify_result is None: - current_app.logger.warning("Failed to verify {}".format(cert.serial_number)) + current_app.logger.warning(f"Failed to verify {cert.serial_number}") return verify_result, ocsp_err, crl_err diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index e2c029b7eb..be284e53a5 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -6,22 +6,11 @@ .. moduleauthor:: Kevin Glisson """ import base64 -from builtins import str from flask import Blueprint, make_response, jsonify, g, current_app from flask_restful import reqparse, Api, inputs - -from lemur.certificates.service import validate_no_duplicate_destinations -from lemur.common import validators -from lemur.plugins.bases.authorization import UnauthorizedError -from sentry_sdk import capture_exception - -from lemur.common.schema import validate_schema -from lemur.common.utils import paginated_parser - -from lemur.auth.service import AuthenticatedResource from lemur.auth.permissions import AuthorityPermission, CertificatePermission, StrictRolePermission - +from lemur.auth.service import AuthenticatedResource from lemur.certificates import service from lemur.certificates.models import Certificate from lemur.certificates.schemas import ( @@ -34,10 +23,14 @@ certificates_list_output_schema_factory, certificate_revoke_schema, ) - -from lemur.roles import service as role_service +from lemur.certificates.service import validate_no_duplicate_destinations +from lemur.common import validators +from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser from lemur.logs import service as log_service - +from lemur.plugins.bases.authorization import UnauthorizedError +from lemur.roles import service as role_service +from sentry_sdk import capture_exception mod = Blueprint("certificates", __name__) api = Api(mod) @@ -48,7 +41,7 @@ class CertificatesListValid(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificatesListValid, self).__init__() + super().__init__() @validate_schema(None, certificates_output_schema) def get(self): @@ -152,7 +145,7 @@ class CertificatesNameQuery(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificatesNameQuery, self).__init__() + super().__init__() @validate_schema(None, certificates_output_schema) def get(self, certificate_name): @@ -262,7 +255,7 @@ class CertificatesList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificatesList, self).__init__() + super().__init__() @validate_schema(None, certificates_list_output_schema_factory) def get(self): @@ -542,7 +535,7 @@ class CertificatesUpload(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificatesUpload, self).__init__() + super().__init__() @validate_schema(certificate_upload_input_schema, certificate_output_schema) def post(self, data=None): @@ -657,7 +650,7 @@ class CertificatesStats(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificatesStats, self).__init__() + super().__init__() def get(self): self.reqparse.add_argument("metric", type=str, location="args") @@ -680,7 +673,7 @@ def get(self): class CertificatePrivateKey(AuthenticatedResource): def __init__(self): - super(CertificatePrivateKey, self).__init__() + super().__init__() def get(self, certificate_id): """ @@ -737,7 +730,7 @@ def get(self, certificate_id): class Certificates(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Certificates, self).__init__() + super().__init__() @validate_schema(None, certificate_output_schema) def get(self, certificate_id): @@ -939,7 +932,7 @@ def put(self, certificate_id, data=None): if not cert.private_key: return ( dict( - message="Unable to add destination: {0}. Certificate does not have required private key.".format( + message="Unable to add destination: {}. Certificate does not have required private key.".format( destination.label ) ), @@ -1136,7 +1129,7 @@ def delete(self, certificate_id, data=None): class CertificateUpdateOwner(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificateUpdateOwner, self).__init__() + super().__init__() @validate_schema(certificate_edit_input_schema, certificate_output_schema) def post(self, certificate_id, data=None): @@ -1255,7 +1248,7 @@ class NotificationCertificatesList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(NotificationCertificatesList, self).__init__() + super().__init__() @validate_schema(None, certificates_output_schema) def get(self, notification_id): @@ -1367,7 +1360,7 @@ def get(self, notification_id): class CertificatesReplacementsList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificatesReplacementsList, self).__init__() + super().__init__() @validate_schema(None, certificates_output_schema) def get(self, certificate_id): @@ -1459,7 +1452,7 @@ def get(self, certificate_id): class CertificateExport(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificateExport, self).__init__() + super().__init__() @validate_schema(certificate_export_input_schema, None) def post(self, certificate_id, data=None): @@ -1540,7 +1533,7 @@ def post(self, certificate_id, data=None): if not cert.private_key: return ( dict( - message="Unable to export certificate, plugin: {0} requires a private key but no key was found.".format( + message="Unable to export certificate, plugin: {} requires a private key but no key was found.".format( plugin.slug ) ), @@ -1586,7 +1579,7 @@ def post(self, certificate_id, data=None): class CertificateRevoke(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificateRevoke, self).__init__() + super().__init__() @validate_schema(certificate_revoke_schema, None) def put(self, certificate_id, data=None): @@ -1673,7 +1666,7 @@ def put(self, certificate_id, data=None): class CertificateDeactivate(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificateDeactivate, self).__init__() + super().__init__() def put(self, certificate_id): """ diff --git a/lemur/common/celery.py b/lemur/common/celery.py index a199ef39be..603f377d2c 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -10,14 +10,13 @@ import copy import sys import time +from datetime import datetime, timezone, timedelta + from celery import Celery from celery.app.task import Context from celery.exceptions import SoftTimeLimitExceeded from celery.signals import task_failure, task_received, task_revoked, task_success -from datetime import datetime, timezone, timedelta from flask import current_app -from sentry_sdk import capture_exception - from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate from lemur.certificates import service as certificate_service @@ -35,6 +34,7 @@ from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources +from sentry_sdk import capture_exception if current_app: flask_app = current_app @@ -245,7 +245,7 @@ def fetch_acme_cert(id, notify_reissue_cert_id=None): function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, - "message": "Resolving pending certificate {}".format(id), + "message": f"Resolving pending certificate {id}", "task_id": task_id, "id": id, } @@ -373,7 +373,7 @@ def fetch_all_pending_acme_certs(): cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == "acme-issuer": if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5): - log_data["message"] = "Triggering job for cert {}".format(cert.name) + log_data["message"] = f"Triggering job for cert {cert.name}" log_data["cert_name"] = cert.name log_data["cert_id"] = cert.id current_app.logger.debug(log_data) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index 39ea8bc31d..f291fba380 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -1,13 +1,12 @@ import re -import unicodedata +import unicodedata from cryptography import x509 from cryptography.hazmat.primitives.serialization import Encoding from flask import current_app -from sentry_sdk import capture_exception - from lemur.common.utils import is_selfsigned from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE +from sentry_sdk import capture_exception def text_to_slug(value, joiner="-"): @@ -108,7 +107,7 @@ def organization(cert): return o[0].value.strip() except Exception as e: capture_exception() - current_app.logger.error("Unable to get organization! {0}".format(e)) + current_app.logger.error(f"Unable to get organization! {e}") def organizational_unit(cert): @@ -125,7 +124,7 @@ def organizational_unit(cert): return ou[0].value.strip() except Exception as e: capture_exception() - current_app.logger.error("Unable to get organizational unit! {0}".format(e)) + current_app.logger.error(f"Unable to get organizational unit! {e}") def country(cert): @@ -142,7 +141,7 @@ def country(cert): return c[0].value.strip() except Exception as e: capture_exception() - current_app.logger.error("Unable to get country! {0}".format(e)) + current_app.logger.error(f"Unable to get country! {e}") def state(cert): @@ -159,7 +158,7 @@ def state(cert): return s[0].value.strip() except Exception as e: capture_exception() - current_app.logger.error("Unable to get state! {0}".format(e)) + current_app.logger.error(f"Unable to get state! {e}") def location(cert): @@ -176,7 +175,7 @@ def location(cert): return loc[0].value.strip() except Exception as e: capture_exception() - current_app.logger.error("Unable to get location! {0}".format(e)) + current_app.logger.error(f"Unable to get location! {e}") def domains(cert): @@ -277,7 +276,7 @@ def issuer(cert): ) or cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME) if not attrs: current_app.logger.error( - "Unable to get issuer! Cert serial {:x}".format(cert.serial_number) + f"Unable to get issuer! Cert serial {cert.serial_number:x}" ) return "" diff --git a/lemur/common/fields.py b/lemur/common/fields.py index 156318321c..e0922c1a47 100644 --- a/lemur/common/fields.py +++ b/lemur/common/fields.py @@ -5,20 +5,17 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import arrow -import warnings import ipaddress - -from flask import current_app +import warnings from datetime import datetime as dt +import arrow from cryptography import x509 - +from flask import current_app +from lemur.common import validators from marshmallow import utils -from marshmallow.fields import Field from marshmallow.exceptions import ValidationError - -from lemur.common import validators +from marshmallow.fields import Field class Hex(Field): @@ -71,14 +68,14 @@ class ArrowDateTime(Field): } def __init__(self, format=None, **kwargs): - super(ArrowDateTime, self).__init__(**kwargs) + super().__init__(**kwargs) # Allow this to be None. It may be set later in the ``_serialize`` # or ``_desrialize`` methods This allows a Schema to dynamically set the # dateformat, e.g. from a Meta option self.dateformat = format def _add_to_schema(self, field_name, schema): - super(ArrowDateTime, self)._add_to_schema(field_name, schema) + super()._add_to_schema(field_name, schema) self.dateformat = self.dateformat or schema.opts.dateformat def _serialize(self, value, attr, obj): @@ -366,7 +363,7 @@ def _serialize(self, value, attr, obj): value = value.dotted_string else: current_app.logger.warning( - "Unknown SubAltName type: {name}".format(name=name) + f"Unknown SubAltName type: {name}" ) continue diff --git a/lemur/common/managers.py b/lemur/common/managers.py index 6ce2608ff1..5a9fcfd73d 100644 --- a/lemur/common/managers.py +++ b/lemur/common/managers.py @@ -12,7 +12,7 @@ # inspired by https://github.com/getsentry/sentry -class InstanceManager(object): +class InstanceManager: def __init__(self, class_list=None, instances=True): if class_list is None: class_list = [] @@ -63,12 +63,12 @@ def all(self): except InvalidConfiguration as e: current_app.logger.warning( - "Plugin '{0}' may not work correctly. {1}".format(class_name, e) + f"Plugin '{class_name}' may not work correctly. {e}" ) except Exception as e: current_app.logger.exception( - "Unable to import {0}. Reason: {1}".format(cls_path, e) + f"Unable to import {cls_path}. Reason: {e}" ) continue diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 86aa4d42e3..2ac58a2879 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -15,8 +15,10 @@ import string import OpenSSL +import josepy as jose import pem import sqlalchemy +from certbot.crypto_util import CERT_PEM_REGEX from cryptography import x509 from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend @@ -24,13 +26,10 @@ from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding from cryptography.hazmat.primitives.serialization import load_pem_private_key, Encoding, pkcs7 from flask_restful.reqparse import RequestParser -from sqlalchemy import and_, func -import josepy as jose - -from certbot.crypto_util import CERT_PEM_REGEX from lemur.constants import CERTIFICATE_KEY_TYPES from lemur.exceptions import InvalidConfiguration from lemur.utils import Vault +from sqlalchemy import and_, func from sqlalchemy.dialects.postgresql import TEXT paginated_parser = RequestParser() @@ -335,7 +334,7 @@ def validate_conf(app, required_vars): for var in required_vars: if var not in app.config: raise InvalidConfiguration( - "Required variable '{var}' is not set in Lemur's conf.".format(var=var) + f"Required variable '{var}' is not set in Lemur's conf." ) @@ -400,8 +399,7 @@ def windowed_query(q, column, windowsize): """"Break a Query into windows on a given column.""" for whereclause in column_windows(q.session, column, windowsize): - for row in q.filter(whereclause).order_by(column): - yield row + yield from q.filter(whereclause).order_by(column) def truthiness(s): diff --git a/lemur/common/validators.py b/lemur/common/validators.py index 6bc95703f8..798961d91a 100644 --- a/lemur/common/validators.py +++ b/lemur/common/validators.py @@ -5,11 +5,10 @@ from cryptography.hazmat.backends import default_backend from cryptography.x509 import NameOID from flask import current_app -from marshmallow.exceptions import ValidationError - from lemur.auth.permissions import SensitiveDomainPermission from lemur.common.utils import check_cert_signature, is_weekend from lemur.plugins.base import plugins +from marshmallow.exceptions import ValidationError def common_name(value): @@ -35,7 +34,7 @@ def sensitive_domain(domain): allowlist = current_app.config.get("LEMUR_ALLOWED_DOMAINS", []) if allowlist and not any(re.match(pattern, domain) for pattern in allowlist): raise ValidationError( - "Domain {0} does not match allowed domain patterns. " + "Domain {} does not match allowed domain patterns. " "Contact an administrator to issue the certificate.".format(domain) ) @@ -44,7 +43,7 @@ def sensitive_domain(domain): if domain_service.is_domain_sensitive(domain): raise ValidationError( - "Domain {0} has been marked as sensitive. " + "Domain {} has been marked as sensitive. " "Contact an administrator to issue the certificate.".format(domain) ) @@ -58,7 +57,7 @@ def encoding(oid_encoding): valid_types = ["b64asn1", "string", "ia5string"] if oid_encoding.lower() not in [o_type.lower() for o_type in valid_types]: raise ValidationError( - "Invalid Oid Encoding: {0} choose from {1}".format( + "Invalid Oid Encoding: {} choose from {}".format( oid_encoding, ",".join(valid_types) ) ) @@ -83,7 +82,7 @@ def sub_alt_type(alt_type): ] if alt_type.lower() not in [a_type.lower() for a_type in valid_types]: raise ValidationError( - "Invalid SubAltName Type: {0} choose from {1}".format( + "Invalid SubAltName Type: {} choose from {}".format( type, ",".join(valid_types) ) ) @@ -140,7 +139,7 @@ def dates(data): < data["authority"].authority_certificate.not_before.date() ): raise ValidationError( - "Validity start must not be before {0}".format( + "Validity start must not be before {}".format( data["authority"].authority_certificate.not_before ) ) @@ -150,7 +149,7 @@ def dates(data): > data["authority"].authority_certificate.not_after.date() ): raise ValidationError( - "Validity end must not be after {0}".format( + "Validity end must not be after {}".format( data["authority"].authority_certificate.not_after ) ) diff --git a/lemur/database.py b/lemur/database.py index 544de6e6c9..c1f5ea5a56 100644 --- a/lemur/database.py +++ b/lemur/database.py @@ -9,16 +9,14 @@ .. moduleauthor:: Kevin Glisson """ -from inflection import underscore from flask_sqlalchemy.model import DefaultMeta +from inflection import underscore +from lemur.exceptions import AttrNotFound, DuplicateError +from lemur.extensions import db from sqlalchemy import exc, func, distinct from sqlalchemy.orm import make_transient, lazyload from sqlalchemy.sql import and_, or_ -from lemur.exceptions import AttrNotFound, DuplicateError -from lemur.extensions import db - - BaseModel: DefaultMeta = db.Model @@ -211,7 +209,7 @@ def filter(query, model, terms): :return: """ column = get_model_column(model, underscore(terms[0])) - return query.filter(column.ilike("%{}%".format(terms[1]))) + return query.filter(column.ilike(f"%{terms[1]}%")) def sort(query, model, field, direction): diff --git a/lemur/destinations/models.py b/lemur/destinations/models.py index b67084fe25..f631e64119 100644 --- a/lemur/destinations/models.py +++ b/lemur/destinations/models.py @@ -5,12 +5,11 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +from lemur.database import BaseModel +from lemur.plugins.base import plugins from sqlalchemy import Column, Integer, String, Text from sqlalchemy.orm import validates from sqlalchemy_utils import JSONType -from lemur.database import BaseModel - -from lemur.plugins.base import plugins class Destination(BaseModel): @@ -32,4 +31,4 @@ def validate_label(self, key, label): return label def __repr__(self): - return "Destination(label={label})".format(label=self.label) + return f"Destination(label={self.label})" diff --git a/lemur/destinations/views.py b/lemur/destinations/views.py index 785006ec20..b333f9d085 100644 --- a/lemur/destinations/views.py +++ b/lemur/destinations/views.py @@ -8,20 +8,17 @@ """ from flask import Blueprint from flask_restful import Api, reqparse -from lemur.destinations import service - -from lemur.auth.service import AuthenticatedResource from lemur.auth.permissions import admin_permission -from lemur.common.utils import paginated_parser - +from lemur.auth.service import AuthenticatedResource from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser +from lemur.destinations import service from lemur.destinations.schemas import ( destinations_output_schema, destination_input_schema, destination_output_schema, ) - mod = Blueprint("destinations", __name__) api = Api(mod) @@ -31,7 +28,7 @@ class DestinationsList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(DestinationsList, self).__init__() + super().__init__() @validate_schema(None, destinations_output_schema) def get(self): @@ -193,7 +190,7 @@ def post(self, data=None): class Destinations(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Destinations, self).__init__() + super().__init__() @validate_schema(None, destination_output_schema) def get(self, destination_id): @@ -355,7 +352,7 @@ class CertificateDestinations(AuthenticatedResource): """ Defines the 'certificate/ """ -from sqlalchemy import Column, Integer, String, Boolean - from lemur.database import BaseModel +from sqlalchemy import Column, Integer, String, Boolean class Domain(BaseModel): @@ -20,4 +19,4 @@ class Domain(BaseModel): sensitive = Column(Boolean, default=False) def __repr__(self): - return "Domain(name={name})".format(name=self.name) + return f"Domain(name={self.name})" diff --git a/lemur/domains/views.py b/lemur/domains/views.py index b7c74edcc6..e35c996f44 100644 --- a/lemur/domains/views.py +++ b/lemur/domains/views.py @@ -9,14 +9,11 @@ """ from flask import Blueprint from flask_restful import reqparse, Api - -from lemur.domains import service -from lemur.auth.service import AuthenticatedResource from lemur.auth.permissions import SensitiveDomainPermission, StrictRolePermission - +from lemur.auth.service import AuthenticatedResource from lemur.common.schema import validate_schema from lemur.common.utils import paginated_parser - +from lemur.domains import service from lemur.domains.schemas import ( domain_input_schema, domain_output_schema, @@ -31,7 +28,7 @@ class DomainsList(AuthenticatedResource): """ Defines the 'domains' endpoint """ def __init__(self): - super(DomainsList, self).__init__() + super().__init__() @validate_schema(None, domains_output_schema) def get(self): @@ -136,7 +133,7 @@ def post(self, data=None): class Domains(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Domains, self).__init__() + super().__init__() @validate_schema(None, domain_output_schema) def get(self, domain_id): @@ -220,7 +217,7 @@ class CertificateDomains(AuthenticatedResource): """ Defines the 'domains' endpoint """ def __init__(self): - super(CertificateDomains, self).__init__() + super().__init__() @validate_schema(None, domains_output_schema) def get(self, certificate_id): diff --git a/lemur/endpoints/models.py b/lemur/endpoints/models.py index 2b3bcf385f..adb9721f88 100644 --- a/lemur/endpoints/models.py +++ b/lemur/endpoints/models.py @@ -7,18 +7,15 @@ .. moduleauthor:: Kevin Glisson """ import arrow -from sqlalchemy.orm import relationship -from sqlalchemy.ext.associationproxy import association_proxy +from lemur.database import BaseModel +from lemur.models import policies_ciphers from sqlalchemy import Column, Integer, String, Boolean, ForeignKey +from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import case - from sqlalchemy_utils import ArrowType -from lemur.database import BaseModel - -from lemur.models import policies_ciphers - BAD_CIPHERS = ["Protocol-SSLv3", "Protocol-SSLv2", "Protocol-TLSv1"] @@ -95,7 +92,7 @@ def issues(self): issues.append( { "name": "deprecated cipher", - "value": "{0} has been deprecated consider removing it.".format( + "value": "{} has been deprecated consider removing it.".format( cipher.name ), } @@ -120,4 +117,4 @@ def issues(self): return issues def __repr__(self): - return "Endpoint(name={name})".format(name=self.name) + return f"Endpoint(name={self.name})" diff --git a/lemur/endpoints/service.py b/lemur/endpoints/service.py index e402c4ced6..d2e9c4f453 100644 --- a/lemur/endpoints/service.py +++ b/lemur/endpoints/service.py @@ -9,15 +9,13 @@ """ import arrow - -from sqlalchemy import func -from sqlalchemy.orm import joinedload - from lemur import database from lemur.common.utils import truthiness from lemur.endpoints.models import Endpoint, EndpointDnsAlias, Policy, Cipher -from lemur.sources.models import Source from lemur.extensions import metrics +from lemur.sources.models import Source +from sqlalchemy import func +from sqlalchemy.orm import joinedload def get_all(): @@ -187,7 +185,7 @@ def render(args): if filt: terms = filt.split(";") - term = "%{0}%".format(terms[1]) + term = f"%{terms[1]}%" if "active" in filt: # this is really weird but strcmp seems to not work here?? query = query.filter(Endpoint.active == truthiness(terms[1])) elif "port" in filt: diff --git a/lemur/endpoints/views.py b/lemur/endpoints/views.py index 9f469a6bf1..8071157372 100644 --- a/lemur/endpoints/views.py +++ b/lemur/endpoints/views.py @@ -7,15 +7,12 @@ """ from flask import Blueprint, g from flask_restful import reqparse, Api - -from lemur.common.utils import paginated_parser -from lemur.common.schema import validate_schema from lemur.auth.service import AuthenticatedResource - +from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser from lemur.endpoints import service from lemur.endpoints.schemas import endpoint_output_schema, endpoints_output_schema - mod = Blueprint("endpoints", __name__) api = Api(mod) @@ -25,7 +22,7 @@ class EndpointsList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(EndpointsList, self).__init__() + super().__init__() @validate_schema(None, endpoints_output_schema) def get(self): @@ -71,7 +68,7 @@ def get(self): class Endpoints(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Endpoints, self).__init__() + super().__init__() @validate_schema(None, endpoint_output_schema) def get(self, endpoint_id): diff --git a/lemur/exceptions.py b/lemur/exceptions.py index 3fdcb2180a..78538aed73 100644 --- a/lemur/exceptions.py +++ b/lemur/exceptions.py @@ -16,7 +16,7 @@ def __init__(self, key): self.key = key def __str__(self): - return repr("Duplicate found! Could not create: {0}".format(self.key)) + return repr(f"Duplicate found! Could not create: {self.key}") class InvalidListener(LemurException): @@ -32,7 +32,7 @@ def __init__(self, field): def __str__(self): return repr( - "Invalid distribution {0}, must use IAM certificates".format(self.field) + f"Invalid distribution {self.field}, must use IAM certificates" ) @@ -50,7 +50,7 @@ def __init__(self, field): self.field = field def __str__(self): - return repr("The field '{0}' is not sortable or filterable".format(self.field)) + return repr(f"The field '{self.field}' is not sortable or filterable") class InvalidConfiguration(Exception): diff --git a/lemur/factory.py b/lemur/factory.py index 62e6519385..7e2d8a66c2 100644 --- a/lemur/factory.py +++ b/lemur/factory.py @@ -23,15 +23,14 @@ from click import get_current_context from flask import Flask, current_app from flask_replicated import FlaskReplicated +from lemur.certificates.hooks import activate_debug_dump +from lemur.common.health import mod as health +from lemur.extensions import db, migrate, principal, smtp_mail, metrics, cors from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.flask import FlaskIntegration from sentry_sdk.integrations.redis import RedisIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration -from lemur.certificates.hooks import activate_debug_dump -from lemur.common.health import mod as health -from lemur.extensions import db, migrate, principal, smtp_mail, metrics, cors - DEFAULT_BLUEPRINTS = (health,) API_VERSION = 1 @@ -104,7 +103,7 @@ def from_file(file_path, silent=False): exec( # nosec: config file safe compile(config_file.read(), file_path, "exec"), d.__dict__ ) - except IOError as e: + except OSError as e: if silent and e.errno in (errno.ENOENT, errno.EISDIR): return False e.strerror = "Unable to load configuration file (%s)" % e.strerror @@ -200,7 +199,7 @@ def configure_blueprints(app, blueprints): :param blueprints: """ for blueprint in blueprints: - app.register_blueprint(blueprint, url_prefix="/api/{0}".format(API_VERSION)) + app.register_blueprint(blueprint, url_prefix=f"/api/{API_VERSION}") def configure_database(app): @@ -268,7 +267,7 @@ def install_plugins(app): import traceback app.logger.error( - "Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc()) + f"Failed to load plugin {ep.name!r}:\n{traceback.format_exc()}\n" ) else: register(plugin) diff --git a/lemur/logs/service.py b/lemur/logs/service.py index 41e277e3b0..5b0492f20b 100644 --- a/lemur/logs/service.py +++ b/lemur/logs/service.py @@ -8,11 +8,10 @@ .. moduleauthor:: Kevin Glisson """ from flask import current_app, g - from lemur import database +from lemur.certificates.models import Certificate from lemur.logs.models import Log from lemur.users.models import User -from lemur.certificates.models import Certificate def create(user, type, certificate=None): @@ -88,14 +87,14 @@ def render(args): if "certificate.name" in terms: sub_query = database.session_query(Certificate.id).filter( - Certificate.name.ilike("%{0}%".format(terms[1])) + Certificate.name.ilike(f"%{terms[1]}%") ) query = query.filter(Log.certificate_id.in_(sub_query)) elif "user.email" in terms: sub_query = database.session_query(User.id).filter( - User.email.ilike("%{0}%".format(terms[1])) + User.email.ilike(f"%{terms[1]}%") ) query = query.filter(Log.user_id.in_(sub_query)) diff --git a/lemur/logs/views.py b/lemur/logs/views.py index 57c588ed61..6d6fea23b5 100644 --- a/lemur/logs/views.py +++ b/lemur/logs/views.py @@ -7,15 +7,11 @@ """ from flask import Blueprint from flask_restful import reqparse, Api - +from lemur.auth.service import AuthenticatedResource from lemur.common.schema import validate_schema from lemur.common.utils import paginated_parser - -from lemur.auth.service import AuthenticatedResource -from lemur.logs.schemas import logs_output_schema - from lemur.logs import service - +from lemur.logs.schemas import logs_output_schema mod = Blueprint("logs", __name__) api = Api(mod) @@ -26,7 +22,7 @@ class LogsList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(LogsList, self).__init__() + super().__init__() @validate_schema(None, logs_output_schema) def get(self): diff --git a/lemur/manage.py b/lemur/manage.py index 5bcda982ef..152b812dbd 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -1,55 +1,46 @@ #!/usr/bin/env python -from __future__ import unicode_literals # at top of module -import click -import os -import sys import base64 -import requests import json +import os +import sys +import click +import requests from cryptography.fernet import Fernet - from flask import current_app from flask.cli import FlaskGroup, pass_script_info -from flask_migrate.cli import db from flask_migrate import stamp - -from lemur.dns_providers.cli import cli as dns_provider_cli -from lemur.acme_providers.cli import cli as acme_cli -from lemur.sources.cli import cli as source_cli -from lemur.policies.cli import cli as policy_cli -from lemur.reporting.cli import cli as report_cli -from lemur.certificates.cli import cli as certificate_cli -from lemur.notifications.cli import cli as notification_cli -from lemur.pending_certificates.cli import cli as pending_certificate_cli - - -from lemur import database -from lemur.users import service as user_service -from lemur.roles import service as role_service -from lemur.policies import service as policy_service -from lemur.notifications import service as notification_service - -from lemur.common.utils import validate_conf - +from flask_migrate.cli import db from lemur import create_app - -# Needed to be imported so that SQLAlchemy create_all can find our models -from lemur.users.models import User # noqa -from lemur.roles.models import Role # noqa +from lemur import database +from lemur.acme_providers.cli import cli as acme_cli from lemur.authorities.models import Authority # noqa +from lemur.certificates.cli import cli as certificate_cli from lemur.certificates.models import Certificate # noqa +from lemur.common.utils import validate_conf from lemur.destinations.models import Destination # noqa +from lemur.dns_providers.cli import cli as dns_provider_cli +from lemur.dns_providers.models import DnsProvider # noqa from lemur.domains.models import Domain # noqa -from lemur.notifications.models import Notification # noqa -from lemur.sources.models import Source # noqa -from lemur.logs.models import Log # noqa from lemur.endpoints.models import Endpoint # noqa -from lemur.policies.models import RotationPolicy # noqa +from lemur.logs.models import Log # noqa +from lemur.notifications import service as notification_service +from lemur.notifications.cli import cli as notification_cli +from lemur.notifications.models import Notification # noqa +from lemur.pending_certificates.cli import cli as pending_certificate_cli from lemur.pending_certificates.models import PendingCertificate # noqa -from lemur.dns_providers.models import DnsProvider # noqa - +from lemur.policies import service as policy_service +from lemur.policies.cli import cli as policy_cli +from lemur.policies.models import RotationPolicy # noqa +from lemur.reporting.cli import cli as report_cli +from lemur.roles import service as role_service +from lemur.roles.models import Role # noqa +from lemur.sources.cli import cli as source_cli +from lemur.sources.models import Source # noqa +from lemur.users import service as user_service +# Needed to be imported so that SQLAlchemy create_all can find our models +from lemur.users.models import User # noqa from sqlalchemy.sql import text @@ -294,7 +285,7 @@ def initialize_app(password): recipients = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL") click.echo("[+] Creating expiration email notifications!") click.echo( - "[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications".format( + "[!] Using {} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications".format( recipients ) ) @@ -339,7 +330,7 @@ def create_user(username, email, active, roles, password): if role_obj: role_objs.append(role_obj) else: - click.echo("[!] Cannot find role {0}".format(r)) + click.echo(f"[!] Cannot find role {r}") sys.exit(1) if not password: @@ -352,7 +343,7 @@ def create_user(username, email, active, roles, password): sys.exit(1) user_service.create(username, password, email, active, None, role_objs) - click.echo("[+] Created new user: {0}".format(username)) + click.echo(f"[+] Created new user: {username}") @cli.command("reset_password") @@ -364,10 +355,10 @@ def reset_password(username): user = user_service.get_by_username(username) if not user: - click.echo("[!] No user found for username: {0}".format(username)) + click.echo(f"[!] No user found for username: {username}") sys.exit(1) - click.echo("[+] Resetting password for {0}".format(username)) + click.echo(f"[+] Resetting password for {username}") password1 = click.prompt("Password", hide_input=True) password2 = click.prompt("Confirm Password", hide_input=True) @@ -394,10 +385,10 @@ def create_role(name, users, description): if user_obj: user_objs.append(user_obj) else: - click.echo("[!] Cannot find user {0}".format(u)) + click.echo(f"[!] Cannot find user {u}") sys.exit(1) role_service.create(name, description=description, users=users) - click.echo("[+] Created new role: {0}".format(name)) + click.echo(f"[+] Created new role: {name}") @cli.command("start", context_settings=dict(ignore_unknown_options=True, allow_extra_args=True)) @@ -446,7 +437,7 @@ def create_config(config_path): with open(config_path, "w") as f: f.write(config) - click.echo("[+] Created a new configuration file {0}".format(config_path)) + click.echo(f"[+] Created a new configuration file {config_path}") @cli.command("lock") @@ -473,7 +464,7 @@ def lock(path=None): key = Fernet.generate_key() if not os.path.exists(dest_dir): - click.echo("[+] Creating encryption directory: {0}".format(dest_dir)) + click.echo(f"[+] Creating encryption directory: {dest_dir}") os.makedirs(dest_dir) for root, dirs, files in os.walk(os.path.join(path, "decrypted")): @@ -485,10 +476,10 @@ def lock(path=None): data = f.encrypt(in_file.read()) out_file.write(data) click.echo( - "[+] Writing file: {0} Source: {1}".format(dest, source) + f"[+] Writing file: {dest} Source: {source}" ) - click.echo("[+] Keys have been encrypted with key {0}".format(key)) + click.echo(f"[+] Keys have been encrypted with key {key}") @cli.command("unlock") @@ -512,7 +503,7 @@ def unlock(path=None): source_dir = os.path.join(path, "encrypted") if not os.path.exists(dest_dir): - click.echo("[+] Creating decryption directory: {0}".format(dest_dir)) + click.echo(f"[+] Creating decryption directory: {dest_dir}") os.makedirs(dest_dir) for root, dirs, files in os.walk(source_dir): @@ -524,7 +515,7 @@ def unlock(path=None): data = f.decrypt(in_file.read()) out_file.write(data) click.echo( - "[+] Writing file: {0} Source: {1}".format(dest, source) + f"[+] Writing file: {dest} Source: {source}" ) click.echo("[+] Keys have been unencrypted!") @@ -554,7 +545,7 @@ def publish_verisign_units(): { "timestamp": 1321351651, "type": "GAUGE", - "name": "Symantec {0} Unit Count".format(name), + "name": f"Symantec {name} Unit Count", "tags": {}, "value": value, } diff --git a/lemur/metrics.py b/lemur/metrics.py index 4981e04631..da777c8756 100644 --- a/lemur/metrics.py +++ b/lemur/metrics.py @@ -9,7 +9,7 @@ from lemur.plugins.base import plugins -class Metrics(object): +class Metrics: """ :param app: The Flask application object. Defaults to None. """ diff --git a/lemur/migrations/env.py b/lemur/migrations/env.py index d783ee3911..8331b1a4ed 100644 --- a/lemur/migrations/env.py +++ b/lemur/migrations/env.py @@ -1,10 +1,7 @@ -from __future__ import with_statement -from alembic import context -from sqlalchemy import engine_from_config, pool from logging.config import fileConfig -import alembic_autogenerate_enums - +from alembic import context +from sqlalchemy import engine_from_config, pool # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/lemur/migrations/versions/29d8c8455c86_.py b/lemur/migrations/versions/29d8c8455c86_.py index 3a0e871708..5f9ed43145 100644 --- a/lemur/migrations/versions/29d8c8455c86_.py +++ b/lemur/migrations/versions/29d8c8455c86_.py @@ -10,9 +10,8 @@ revision = "29d8c8455c86" down_revision = "3307381f3b88" -from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql +from alembic import op def upgrade(): @@ -54,7 +53,7 @@ def upgrade(): sa.Column( "date_created", sa.DateTime(), - server_default=sa.text(u"now()"), + server_default=sa.text("now()"), nullable=False, ), sa.Column("policy_id", sa.Integer(), nullable=True), diff --git a/lemur/migrations/versions/3307381f3b88_.py b/lemur/migrations/versions/3307381f3b88_.py index 2af0448bfc..ff68dfd351 100644 --- a/lemur/migrations/versions/3307381f3b88_.py +++ b/lemur/migrations/versions/3307381f3b88_.py @@ -15,10 +15,10 @@ revision = "3307381f3b88" down_revision = "412b22cb656a" -from alembic import op import sqlalchemy as sa -from sqlalchemy.sql import text +from alembic import op from sqlalchemy.dialects import postgresql +from sqlalchemy.sql import text def upgrade(): @@ -38,7 +38,7 @@ def upgrade(): "certificates", "owner", existing_type=sa.VARCHAR(length=128), nullable=True ) op.drop_constraint( - u"certificates_authority_id_fkey", "certificates", type_="foreignkey" + "certificates_authority_id_fkey", "certificates", type_="foreignkey" ) op.create_foreign_key( None, @@ -150,7 +150,7 @@ def downgrade(): op.drop_constraint(None, "certificates", type_="foreignkey") op.drop_constraint(None, "certificates", type_="foreignkey") op.create_foreign_key( - u"certificates_authority_id_fkey", + "certificates_authority_id_fkey", "certificates", "authorities", ["authority_id"], diff --git a/lemur/migrations/versions/5770674184de_.py b/lemur/migrations/versions/5770674184de_.py index 49d8936789..2f8db7691f 100644 --- a/lemur/migrations/versions/5770674184de_.py +++ b/lemur/migrations/versions/5770674184de_.py @@ -30,13 +30,13 @@ def upgrade(): # Iterate through all entries and mark as seen for each certificate_id and notification_id pair for x in results: # If we've seen a pair already, delete the duplicates - if seen.get("{}-{}".format(x.certificate_id, x.notification_id)): - print("Deleting duplicate: {}".format(x)) + if seen.get(f"{x.certificate_id}-{x.notification_id}"): + print(f"Deleting duplicate: {x}") d = session.query(certificate_notification_associations).filter( certificate_notification_associations.c.id == x.id ) d.delete(synchronize_session=False) - seen["{}-{}".format(x.certificate_id, x.notification_id)] = True + seen[f"{x.certificate_id}-{x.notification_id}"] = True db.session.commit() db.session.flush() diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index c4213f491c..b33a7ea647 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -30,15 +30,14 @@ revision = 'c301c59688d2' down_revision = '434c29e40511' -from alembic import op -from sqlalchemy.sql import text -import time import datetime -from flask import current_app - +import time from logging import Formatter, FileHandler, getLogger +from alembic import op +from flask import current_app from lemur.common import utils +from sqlalchemy.sql import text log = getLogger(__name__) handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log")) @@ -109,9 +108,9 @@ def update_key_type(): try: cert_key_type = utils.get_key_type_from_certificate(body) except ValueError as e: - log.error("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) + log.error("Error in processing certificate - ID: {} Error: {} \n".format(cert_id, str(e))) else: - log.info("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) + log.info("Processing certificate - ID: {} key_type: {}\n".format(cert_id, cert_key_type)) stmt = text( "update certificates set key_type=:key_type where id=:id" ) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index b723a2cf98..6607110db3 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -15,10 +15,6 @@ import arrow from flask import current_app -from sentry_sdk import capture_exception -from sqlalchemy import and_ -from sqlalchemy.sql.expression import false, true - from lemur import database from lemur.certificates import service as certificates_service from lemur.certificates.models import Certificate @@ -29,6 +25,9 @@ from lemur.pending_certificates.schemas import pending_certificate_output_schema from lemur.plugins import plugins from lemur.plugins.utils import get_plugin_option +from sentry_sdk import capture_exception +from sqlalchemy import and_ +from sqlalchemy.sql.expression import false, true def get_certificates(exclude=None): @@ -51,7 +50,7 @@ def get_certificates(exclude=None): exclude_conditions = [] if exclude: for e in exclude: - exclude_conditions.append(~Certificate.name.ilike("%{}%".format(e))) + exclude_conditions.append(~Certificate.name.ilike(f"%{e}%")) q = q.filter(and_(*exclude_conditions)) @@ -85,7 +84,7 @@ def get_certificates_for_security_summary_email(exclude=None): exclude_conditions = [] if exclude: for e in exclude: - exclude_conditions.append(~Certificate.name.ilike("%{}%".format(e))) + exclude_conditions.append(~Certificate.name.ilike(f"%{e}%")) q = q.filter(and_(*exclude_conditions)) diff --git a/lemur/notifications/models.py b/lemur/notifications/models.py index c7a30891a1..0ce3a7d43c 100644 --- a/lemur/notifications/models.py +++ b/lemur/notifications/models.py @@ -5,16 +5,15 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from sqlalchemy.orm import relationship -from sqlalchemy import Integer, String, Column, Boolean, Text -from sqlalchemy_utils import JSONType - from lemur.database import BaseModel -from lemur.plugins.base import plugins from lemur.models import ( certificate_notification_associations, pending_cert_notification_associations, ) +from lemur.plugins.base import plugins +from sqlalchemy import Integer, String, Column, Boolean, Text +from sqlalchemy.orm import relationship +from sqlalchemy_utils import JSONType class Notification(BaseModel): @@ -45,4 +44,4 @@ def plugin(self): return plugins.get(self.plugin_name) def __repr__(self): - return "Notification(label={label})".format(label=self.label) + return f"Notification(label={self.label})" diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index f9a5bd6d54..d8be187b10 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -9,13 +9,12 @@ """ from flask import current_app - from lemur import database -from lemur.constants import EMAIL_RE, EMAIL_RE_HELP from lemur.certificates.models import Certificate from lemur.common.utils import truthiness, check_validation -from lemur.notifications.models import Notification +from lemur.constants import EMAIL_RE, EMAIL_RE_HELP from lemur.logs import service as log_service +from lemur.notifications.models import Notification def create_default_expiration_notifications(name, recipients, intervals=None): @@ -58,7 +57,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None): notifications = [] for i in intervals: - n = get_by_label("{name}_{interval}_DAY".format(name=name, interval=i)) + n = get_by_label(f"{name}_{i}_DAY") if not n: inter = [ { @@ -72,7 +71,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None): ] inter.extend(options) n = create( - label="{name}_{interval}_DAY".format(name=name, interval=i), + label=f"{name}_{i}_DAY", plugin_name=current_app.config.get( "LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification" ), diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index 20e0575215..71f469ca8f 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -8,6 +8,10 @@ """ from flask import Blueprint from flask_restful import Api, reqparse, inputs +from lemur.auth.permissions import StrictRolePermission +from lemur.auth.service import AuthenticatedResource +from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser from lemur.notifications import service from lemur.notifications.schemas import ( notification_input_schema, @@ -15,13 +19,6 @@ notifications_output_schema, ) -from lemur.auth.service import AuthenticatedResource -from lemur.common.utils import paginated_parser -from lemur.auth.permissions import StrictRolePermission - -from lemur.common.schema import validate_schema - - mod = Blueprint("notifications", __name__) api = Api(mod) @@ -31,7 +28,7 @@ class NotificationsList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(NotificationsList, self).__init__() + super().__init__() @validate_schema(None, notifications_output_schema) def get(self): @@ -238,7 +235,7 @@ def post(self, data=None): class Notifications(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Notifications, self).__init__() + super().__init__() @validate_schema(None, notification_output_schema) def get(self, notification_id): @@ -391,7 +388,7 @@ class CertificateNotifications(AuthenticatedResource): """ Defines the 'certificate/ """ -import click import copy import sys +import click from flask import current_app - from lemur.authorities.service import get as get_authority from lemur.constants import ACME_ADDITIONAL_ATTEMPTS from lemur.notifications.messaging import send_pending_failure_notification @@ -64,7 +63,7 @@ def fetch(ids): pending_certificate_service.increment_attempt(cert) failed += 1 click.echo( - "[+] Certificates: New: {new} Failed: {failed}".format(new=new, failed=failed) + f"[+] Certificates: New: {new} Failed: {failed}" ) @@ -80,7 +79,7 @@ def fetch_all_acme(): certificates. """ - log_data = {"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)} + log_data = {"function": f"{__name__}.{sys._getframe().f_code.co_name}"} pending_certs = pending_certificate_service.get_unresolved_pending_certs() new = 0 failed = 0 diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index e151ec467c..a7ef8085f8 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -5,6 +5,17 @@ """ from datetime import datetime as dt +from lemur.certificates.models import get_sequence +from lemur.common import defaults, utils +from lemur.database import BaseModel +from lemur.models import ( + pending_cert_source_associations, + pending_cert_destination_associations, + pending_cert_notification_associations, + pending_cert_replacement_associations, + pending_cert_role_associations, +) +from lemur.utils import Vault from sqlalchemy import ( Integer, ForeignKey, @@ -19,30 +30,18 @@ from sqlalchemy_utils import JSONType from sqlalchemy_utils.types.arrow import ArrowType -from lemur.certificates.models import get_sequence -from lemur.common import defaults, utils -from lemur.database import BaseModel -from lemur.models import ( - pending_cert_source_associations, - pending_cert_destination_associations, - pending_cert_notification_associations, - pending_cert_replacement_associations, - pending_cert_role_associations, -) -from lemur.utils import Vault - def get_or_increase_name(name, serial): certificates = PendingCertificate.query.filter( - PendingCertificate.name.ilike("{0}%".format(name)) + PendingCertificate.name.ilike(f"{name}%") ).all() if not certificates: return name - serial_name = "{0}-{1}".format(name, hex(int(serial))[2:].upper()) + serial_name = f"{name}-{hex(int(serial))[2:].upper()}" certificates = PendingCertificate.query.filter( - PendingCertificate.name.ilike("{0}%".format(serial_name)) + PendingCertificate.name.ilike(f"{serial_name}%") ).all() if not certificates: @@ -55,7 +54,7 @@ def get_or_increase_name(name, serial): if end: ends.append(end) - return "{0}-{1}".format(root, max(ends) + 1) + return f"{root}-{max(ends) + 1}" class PendingCertificate(BaseModel): diff --git a/lemur/pending_certificates/schemas.py b/lemur/pending_certificates/schemas.py index 68f22b4aa9..0d6edb9342 100644 --- a/lemur/pending_certificates/schemas.py +++ b/lemur/pending_certificates/schemas.py @@ -1,9 +1,6 @@ -from marshmallow import fields, validates_schema, post_load -from marshmallow.exceptions import ValidationError - -from lemur.common import utils, validators from lemur.authorities.schemas import AuthorityNestedOutputSchema from lemur.certificates.schemas import CertificateNestedOutputSchema +from lemur.common import utils, validators from lemur.common.schema import LemurInputSchema, LemurOutputSchema from lemur.destinations.schemas import DestinationNestedOutputSchema from lemur.domains.schemas import DomainNestedOutputSchema @@ -20,6 +17,8 @@ ExtensionSchema, ) from lemur.users.schemas import UserNestedOutputSchema +from marshmallow import fields, validates_schema, post_load +from marshmallow.exceptions import ValidationError class PendingCertificateSchema(LemurInputSchema): @@ -92,7 +91,7 @@ def enforce_notifications(self, data): :return: """ if data["owner"]: - notification_name = "DEFAULT_{0}".format( + notification_name = "DEFAULT_{}".format( data["owner"].split("@")[0].upper() ) data[ diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index 70f3d9e3bc..8085930d33 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -4,25 +4,24 @@ .. moduleauthor:: James Chuong """ import arrow -from sqlalchemy import or_, cast, Integer from flask import current_app - from lemur import database -from lemur.authorities.models import Authority from lemur.authorities import service as authorities_service +from lemur.authorities.models import Authority from lemur.certificates import service as certificate_service from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common.utils import truthiness, parse_cert_chain, parse_certificate from lemur.common import validators +from lemur.common.utils import truthiness, parse_cert_chain, parse_certificate from lemur.destinations.models import Destination from lemur.domains.models import Domain from lemur.extensions import metrics +from lemur.logs import service as log_service from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate from lemur.plugins.base import plugins from lemur.roles.models import Role from lemur.users import service as user_service -from lemur.logs import service as log_service +from sqlalchemy import or_, cast, Integer def get(pending_cert_id): @@ -106,7 +105,7 @@ def create_certificate(pending_certificate, certificate, user): data, errors = CertificateUploadInputSchema().load(certificate) if errors: raise Exception( - "Unable to create certificate: {reasons}".format(reasons=errors) + f"Unable to create certificate: {errors}" ) data.update(vars(pending_certificate)) @@ -212,13 +211,13 @@ def render(args): # we can't rely on issuer being correct in the cert directly so we combine queries sub_query = ( database.session_query(Authority.id) - .filter(Authority.name.ilike("%{0}%".format(terms[1]))) + .filter(Authority.name.ilike(f"%{terms[1]}%")) .subquery() ) query = query.filter( or_( - PendingCertificate.issuer.ilike("%{0}%".format(terms[1])), + PendingCertificate.issuer.ilike(f"%{terms[1]}%"), PendingCertificate.authority_id.in_(sub_query), ) ) @@ -234,9 +233,9 @@ def render(args): elif "cn" in terms: query = query.filter( or_( - PendingCertificate.cn.ilike("%{0}%".format(terms[1])), + PendingCertificate.cn.ilike(f"%{terms[1]}%"), PendingCertificate.domains.any( - Domain.name.ilike("%{0}%".format(terms[1])) + Domain.name.ilike(f"%{terms[1]}%") ), ) ) diff --git a/lemur/pending_certificates/views.py b/lemur/pending_certificates/views.py index f91005af1c..cfbdaa44fe 100644 --- a/lemur/pending_certificates/views.py +++ b/lemur/pending_certificates/views.py @@ -6,24 +6,19 @@ """ from flask import Blueprint, g, make_response, jsonify from flask_restful import Api, reqparse, inputs - -from lemur.auth.service import AuthenticatedResource from lemur.auth.permissions import CertificatePermission, StrictRolePermission - +from lemur.auth.service import AuthenticatedResource from lemur.common.schema import validate_schema from lemur.common.utils import paginated_parser - -from lemur.pending_certificates import service -from lemur.roles import service as role_service from lemur.logs import service as log_service - - +from lemur.pending_certificates import service from lemur.pending_certificates.schemas import ( pending_certificate_output_schema, pending_certificate_edit_input_schema, pending_certificate_cancel_schema, pending_certificate_upload_input_schema, ) +from lemur.roles import service as role_service mod = Blueprint("pending_certificates", __name__) api = Api(mod) @@ -32,7 +27,7 @@ class PendingCertificatesList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(PendingCertificatesList, self).__init__() + super().__init__() @validate_schema(None, pending_certificate_output_schema) def get(self): @@ -129,7 +124,7 @@ def get(self): class PendingCertificates(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(PendingCertificates, self).__init__() + super().__init__() @validate_schema(None, pending_certificate_output_schema) def get(self, pending_certificate_id): @@ -319,7 +314,7 @@ def put(self, pending_certificate_id, data=None): if not pending_cert.private_key: return ( dict( - message="Unable to add destination: {0}. Certificate does not have required private key.".format( + message="Unable to add destination: {}. Certificate does not have required private key.".format( destination.label ) ), @@ -395,7 +390,7 @@ def delete(self, pending_certificate_id, data=None): class PendingCertificatePrivateKey(AuthenticatedResource): def __init__(self): - super(PendingCertificatePrivateKey, self).__init__() + super().__init__() def get(self, pending_certificate_id): """ @@ -453,7 +448,7 @@ class PendingCertificatesUpload(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(PendingCertificatesUpload, self).__init__() + super().__init__() @validate_schema( pending_certificate_upload_input_schema, pending_certificate_output_schema diff --git a/lemur/plugins/__init__.py b/lemur/plugins/__init__.py index 1645006425..d39c621c19 100644 --- a/lemur/plugins/__init__.py +++ b/lemur/plugins/__init__.py @@ -1,4 +1,2 @@ -from __future__ import absolute_import - from lemur.plugins.base import * # noqa from lemur.plugins.bases import * # noqa diff --git a/lemur/plugins/base/__init__.py b/lemur/plugins/base/__init__.py index 4e9f1e8325..3c3c6229ae 100644 --- a/lemur/plugins/base/__init__.py +++ b/lemur/plugins/base/__init__.py @@ -6,7 +6,6 @@ .. moduleauthor:: Kevin Glisson """ -from __future__ import absolute_import, print_function from lemur.plugins.base.manager import PluginManager from lemur.plugins.base.v1 import * # noqa diff --git a/lemur/plugins/base/manager.py b/lemur/plugins/base/manager.py index 117700a668..42dde291e2 100644 --- a/lemur/plugins/base/manager.py +++ b/lemur/plugins/base/manager.py @@ -19,7 +19,7 @@ def __len__(self): def all(self, version=1, plugin_type=None): for plugin in sorted( - super(PluginManager, self).all(), key=lambda x: x.get_title() + super().all(), key=lambda x: x.get_title() ): if not plugin.type == plugin_type and plugin_type: continue @@ -63,9 +63,9 @@ def first(self, func_name, *args, **kwargs): return result def register(self, cls): - self.add("%s.%s" % (cls.__module__, cls.__name__)) + self.add(f"{cls.__module__}.{cls.__name__}") return cls def unregister(self, cls): - self.remove("%s.%s" % (cls.__module__, cls.__name__)) + self.remove(f"{cls.__module__}.{cls.__name__}") return cls diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index cd87f2d0c2..495e7a0f29 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -12,34 +12,31 @@ .. moduleauthor:: Curtis Castrapel .. moduleauthor:: Mathias Petermann """ -from datetime import datetime, timezone, timedelta import json import time +from datetime import datetime, timezone, timedelta import OpenSSL.crypto -import josepy as jose import dns.resolver +import josepy as jose from acme import challenges, errors, messages from acme.client import ClientV2, ClientNetwork from acme.errors import TimeoutError from acme.messages import Error as AcmeError, STATUS_VALID from certbot import crypto_util as acme_crypto_util from flask import current_app -from sentry_sdk import capture_exception - +from lemur.authorities import service as authorities_service +from lemur.common.utils import data_encrypt, data_decrypt, is_json from lemur.common.utils import generate_private_key, key_to_alg from lemur.dns_providers import service as dns_provider_service from lemur.exceptions import InvalidAuthority, UnknownProvider, InvalidConfiguration from lemur.extensions import metrics - from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns, nsone -from lemur.authorities import service as authorities_service from retrying import retry - -from lemur.common.utils import data_encrypt, data_decrypt, is_json +from sentry_sdk import capture_exception -class AuthorizationRecord(object): +class AuthorizationRecord: def __init__(self, domain, target_domain, authz, dns_challenge, change_id, cname_delegation): self.domain = domain self.target_domain = target_domain @@ -49,7 +46,7 @@ def __init__(self, domain, target_domain, authz, dns_challenge, change_id, cname self.cname_delegation = cname_delegation -class AcmeHandler(object): +class AcmeHandler: def reuse_account(self, authority): if not authority.options: @@ -123,7 +120,7 @@ def request_certificate(self, acme_client, authorizations, order): self.log_remaining_validation(orderr.authorizations, acme_client.net.account.uri.replace('https://', '')) current_app.logger.debug( - "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) + f"{type(pem_certificate)} {type(pem_certificate_chain)}" ) return pem_certificate, pem_certificate_chain @@ -181,7 +178,7 @@ def setup_acme_client(self, authority): key = jose.JWK.json_loads(existing_key) regr = messages.RegistrationResource.json_loads(existing_regr) current_app.logger.debug( - "Connecting with directory at {0}".format(directory_url) + f"Connecting with directory at {directory_url}" ) net = ClientNetwork(key, account=regr, alg=key_to_alg(key)) directory = ClientV2.get_directory(directory_url, net) @@ -193,7 +190,7 @@ def setup_acme_client(self, authority): current_app.logger.debug("Creating a new ACME account") current_app.logger.debug( - "Connecting with directory at {0}".format(directory_url) + f"Connecting with directory at {directory_url}" ) net = ClientNetwork(key, account=None, timeout=3600, alg=key_to_alg(key)) @@ -235,7 +232,7 @@ def setup_acme_client(self, authority): authorities_service.update_options(authority.id, options=json.dumps(new_options)) - current_app.logger.debug("Connected: {0}".format(registration.uri)) + current_app.logger.debug(f"Connected: {registration.uri}") return client, registration @@ -255,7 +252,7 @@ def get_domains(self, options): if dns_name.value not in domains: domains.append(dns_name.value) - current_app.logger.debug("Got these domains: {0}".format(domains)) + current_app.logger.debug(f"Got these domains: {domains}") return domains def revoke_certificate(self, certificate, crl_reason=0): @@ -351,7 +348,7 @@ def get_dns_provider(self, type): } provider = provider_types.get(type) if not provider: - raise UnknownProvider("No such DNS provider: {}".format(type)) + raise UnknownProvider(f"No such DNS provider: {type}") return provider def start_dns_challenge( @@ -398,7 +395,7 @@ def start_dns_challenge( def complete_dns_challenge(self, acme_client, authz_record): current_app.logger.debug( - "Finalizing DNS challenge for {0}".format( + "Finalizing DNS challenge for {}".format( authz_record.authz[0].body.identifier.value ) ) @@ -406,7 +403,7 @@ def complete_dns_challenge(self, acme_client, authz_record): if not dns_providers: metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) raise Exception( - "No DNS providers found for domain: {}".format(authz_record.target_domain) + f"No DNS providers found for domain: {authz_record.target_domain}" ) for dns_provider in dns_providers: @@ -474,7 +471,7 @@ def get_authorizations(self, acme_client, order, order_info): metrics.send( "get_authorizations_no_dns_provider_for_domain", "counter", 1 ) - raise Exception("No DNS providers found for domain: {}".format(target_domain)) + raise Exception(f"No DNS providers found for domain: {target_domain}") for dns_provider in self.dns_providers_for_domain[target_domain]: dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index 08961fbac9..6a19902c48 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -7,33 +7,31 @@ .. moduleauthor:: Mathias Petermann """ -from datetime import datetime, timedelta import json +from datetime import datetime, timedelta from acme import challenges from acme.errors import WildcardUnsupportedError from acme.messages import errors, STATUS_VALID, ERROR_CODES from botocore.exceptions import ClientError from flask import current_app -from sentry_sdk import capture_exception - from lemur.authorizations import service as authorization_service -from lemur.constants import ACME_ADDITIONAL_ATTEMPTS from lemur.common.utils import drop_last_cert_from_chain +from lemur.constants import ACME_ADDITIONAL_ATTEMPTS +from lemur.destinations import service as destination_service from lemur.exceptions import LemurException, InvalidConfiguration from lemur.extensions import metrics from lemur.plugins.base import plugins -from lemur.destinations import service as destination_service from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler - from retrying import retry +from sentry_sdk import capture_exception class AcmeChallengeMissmatchError(LemurException): pass -class AcmeChallenge(object): +class AcmeChallenge: """ This is the base class, all ACME challenges will need to extend, allowing for future extendability """ @@ -106,7 +104,7 @@ def create_certificate(self, csr, issuer_options): current_app.logger.info(log_data) if len(chall) == 0 and not all_pre_validated: - raise Exception('HTTP-01 challenge was not offered by the CA server at {}'.format(orderr.uri)) + raise Exception(f'HTTP-01 challenge was not offered by the CA server at {orderr.uri}') elif not all_pre_validated: validation_target = None for option in json.loads(issuer_options["authority"].options): @@ -174,7 +172,7 @@ def deploy(self, challenge, acme_client, validation_target): if destination is None: raise Exception( - 'Couldn\'t find the destination with name {}. Cant complete HTTP01 challenge'.format(validation_target)) + f'Couldn\'t find the destination with name {validation_target}. Cant complete HTTP01 challenge') destination_plugin = plugins.get(destination.plugin_name) @@ -190,7 +188,7 @@ def cleanup(self, token_path, validation_target): if destination is None: current_app.logger.info( - 'Couldn\'t find the destination with name {}, won\'t cleanup the challenge'.format(validation_target)) + f'Couldn\'t find the destination with name {validation_target}, won\'t cleanup the challenge') destination_plugin = plugins.get(destination.plugin_name) @@ -224,7 +222,7 @@ def create_certificate(self, csr, issuer_options): credentials = json.loads(dns_provider.credentials) current_app.logger.debug( - "Using DNS provider: {0}".format(dns_provider.provider_type) + f"Using DNS provider: {dns_provider.provider_type}" ) account_number = credentials.get("account_id") provider_type = dns_provider.provider_type diff --git a/lemur/plugins/lemur_acme/cloudflare.py b/lemur/plugins/lemur_acme/cloudflare.py index a19495f8fc..af13867e1e 100644 --- a/lemur/plugins/lemur_acme/cloudflare.py +++ b/lemur/plugins/lemur_acme/cloudflare.py @@ -19,7 +19,7 @@ def find_zone_id(host): while n < 5: n = n + 1 domain = ".".join(elements[-n:]) - current_app.logger.debug("Trying to get ID for zone {0}".format(domain)) + current_app.logger.debug(f"Trying to get ID for zone {domain}") try: zone = cf.zones.get(params={"name": domain, "per_page": 1}) @@ -58,14 +58,14 @@ def create_txt_record(host, value, account_number): txt_record = {"name": host, "type": "TXT", "content": value} current_app.logger.debug( - "Creating TXT record {0} with value {1}".format(host, value) + f"Creating TXT record {host} with value {value}" ) try: r = cf.zones.dns_records.post(zone_id, data=txt_record) except Exception as e: current_app.logger.error( - "/zones.dns_records.post %s: %s" % (txt_record["name"], e) + "/zones.dns_records.post {}: {}".format(txt_record["name"], e) ) return zone_id, r["id"] @@ -74,7 +74,7 @@ def delete_txt_record(change_ids, account_number, host, value): cf = cf_api_call() for change_id in change_ids: zone_id, record_id = change_id - current_app.logger.debug("Removing record with id {0}".format(record_id)) + current_app.logger.debug(f"Removing record with id {record_id}") try: cf.zones.dns_records.delete(zone_id, record_id) except Exception as e: diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index e76b9f4dea..31a5a96b3c 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -14,9 +14,8 @@ from dyn.tm.session import DynectSession from dyn.tm.zones import Node, Zone, get_all_zones from flask import current_app -from sentry_sdk import capture_exception - from lemur.extensions import metrics +from sentry_sdk import capture_exception def get_dynect_session(): @@ -60,7 +59,7 @@ def wait_for_dns_change(change_id, account_number=None): number_of_attempts = 20 for attempts in range(0, number_of_attempts): status = _has_dns_propagated(fqdn, token) - current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) + current_app.logger.debug(f"Record status for fqdn: {fqdn}: {status}") if status: metrics.send("wait_for_dns_change_success", "counter", 1, metric_tags={"dns": fqdn}) break @@ -92,7 +91,7 @@ def get_zone_name(domain): zone_name = z.name if not zone_name: metrics.send("dyn_no_zone_name", "counter", 1) - raise Exception("No Dyn zone found for domain: {}".format(domain)) + raise Exception(f"No Dyn zone found for domain: {domain}") return zone_name @@ -110,16 +109,16 @@ def create_txt_record(domain, token, account_number): zone_name = get_zone_name(domain) zone_parts = len(zone_name.split(".")) node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = "{0}.{1}".format(node_name, zone_name) + fqdn = f"{node_name}.{zone_name}" zone = Zone(zone_name) try: zone.add_record( - node_name, record_type="TXT", txtdata='"{}"'.format(token), ttl=5 + node_name, record_type="TXT", txtdata=f'"{token}"', ttl=5 ) zone.publish() current_app.logger.debug( - "TXT record created: {0}, token: {1}".format(fqdn, token) + f"TXT record created: {fqdn}, token: {token}" ) except (DynectCreateError, DynectUpdateError) as e: if "Cannot duplicate existing record data" in e.message: @@ -146,7 +145,7 @@ def delete_txt_record(change_id, account_number, domain, token): zone_name = get_zone_name(domain) zone_parts = len(zone_name.split(".")) node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = "{0}.{1}".format(node_name, zone_name) + fqdn = f"{node_name}.{zone_name}" zone = Zone(zone_name) node = Node(zone_name, fqdn) @@ -158,8 +157,8 @@ def delete_txt_record(change_id, account_number, domain, token): # No Text Records remain or host is not in the zone anymore because all records have been deleted. return for txt_record in all_txt_records: - if txt_record.txtdata == ("{}".format(token)): - current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn)) + if txt_record.txtdata == (f"{token}"): + current_app.logger.debug(f"Deleting TXT record name: {fqdn}") try: txt_record.delete() except DynectDeleteError: @@ -215,14 +214,14 @@ def delete_acme_txt_records(domain): zone_name = get_zone_name(domain) zone_parts = len(zone_name.split(".")) node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = "{0}.{1}".format(node_name, zone_name) + fqdn = f"{node_name}.{zone_name}" zone = Zone(zone_name) node = Node(zone_name, fqdn) all_txt_records = node.get_all_records_by_type("TXT") for txt_record in all_txt_records: - current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn)) + current_app.logger.debug(f"Deleting TXT record name: {fqdn}") try: txt_record.delete() except DynectDeleteError: @@ -255,7 +254,7 @@ def get_authoritative_nameserver(domain): while not last: s = n.split(depth) - last = s[0].to_unicode() == u"@" + last = s[0].to_unicode() == "@" sub = s[1] query = dns.message.make_query(sub, dns.rdatatype.NS) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f779e88ed7..504ea216bc 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -16,19 +16,17 @@ from acme.messages import Error as AcmeError from botocore.exceptions import ClientError from flask import current_app -from sentry_sdk import capture_exception - from lemur.authorizations import service as authorization_service from lemur.common.utils import check_validation, drop_last_cert_from_chain from lemur.constants import CRLReason, EMAIL_RE from lemur.dns_providers import service as dns_provider_service from lemur.exceptions import InvalidConfiguration from lemur.extensions import metrics - from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler from lemur.plugins.lemur_acme.challenge_types import AcmeHttpChallenge, AcmeDnsChallenge +from sentry_sdk import capture_exception class ACMEIssuerPlugin(IssuerPlugin): @@ -113,7 +111,7 @@ class ACMEIssuerPlugin(IssuerPlugin): ] def __init__(self, *args, **kwargs): - super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def get_ordered_certificate(self, pending_cert): self.acme = AcmeDnsHandler() @@ -302,7 +300,7 @@ def create_authority(options): plugin_options = options.get("plugin", {}).get("plugin_options") if not plugin_options: - error = "Invalid options for lemur_acme plugin: {}".format(options) + error = f"Invalid options for lemur_acme plugin: {options}" current_app.logger.error(error) raise InvalidConfiguration(error) # Define static acme_root based off configuration variable by default. However, if user has passed a @@ -416,7 +414,7 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): ] def __init__(self, *args, **kwargs): - super(ACMEHttpIssuerPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def create_certificate(self, csr, issuer_options): """ @@ -444,7 +442,7 @@ def create_authority(options): plugin_options = options.get("plugin", {}).get("plugin_options") if not plugin_options: - error = "Invalid options for lemur_acme plugin: {}".format(options) + error = f"Invalid options for lemur_acme plugin: {options}" current_app.logger.error(error) raise InvalidConfiguration(error) # Define static acme_root based off configuration variable by default. However, if user has passed a diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py index 7c653c7e6a..87b1c51ace 100644 --- a/lemur/plugins/lemur_acme/route53.py +++ b/lemur/plugins/lemur_acme/route53.py @@ -34,7 +34,7 @@ def _find_zone_id(domain, client=None): chosen_zone = (zone["Name"], zone["Id"]) if chosen_zone is None: - raise ValueError("Unable to find a Route53 hosted zone for {}".format(domain)) + raise ValueError(f"Unable to find a Route53 hosted zone for {domain}") return chosen_zone[1] # Return the chosen zone ID @@ -75,10 +75,10 @@ def change_txt_record(action, zone_id, domain, value, client=None): seen = False for record in current_txt_records: for k, v in record.items(): - if '"{}"'.format(value) == v: + if f'"{value}"' == v: seen = True if not seen: - current_txt_records.append({"Value": '"{}"'.format(value)}) + current_txt_records.append({"Value": f'"{value}"'}) if action == "DELETE" and len(current_txt_records) > 1: # If we want to delete one record out of many, we'll update the record to not include the deleted value instead. @@ -86,7 +86,7 @@ def change_txt_record(action, zone_id, domain, value, client=None): current_txt_records = [ record for record in current_txt_records - if not (record.get("Value") == '"{}"'.format(value)) + if not (record.get("Value") == f'"{value}"') ] action = "UPSERT" diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index fae5ffa784..c4d2ef424c 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -1,18 +1,16 @@ -import time -import requests import json import sys +import time import dns import dns.exception import dns.name import dns.query import dns.resolver - +import requests from flask import current_app -from sentry_sdk import capture_exception - from lemur.extensions import metrics +from sentry_sdk import capture_exception class Record: @@ -413,7 +411,7 @@ def get_authoritative_nameserver(domain): while not last: s = n.split(depth) - last = s[0].to_unicode() == u"@" + last = s[0].to_unicode() == "@" sub = s[1] query = dns.message.make_query(sub, dns.rdatatype.NS) diff --git a/lemur/plugins/lemur_adcs/plugin.py b/lemur/plugins/lemur_adcs/plugin.py index c4007aafc1..d9cbb4a57d 100644 --- a/lemur/plugins/lemur_adcs/plugin.py +++ b/lemur/plugins/lemur_adcs/plugin.py @@ -1,9 +1,9 @@ -from lemur.plugins.bases import IssuerPlugin, SourcePlugin import requests -from lemur.plugins import lemur_adcs as ADCS -from certsrv import Certsrv from OpenSSL import crypto +from certsrv import Certsrv from flask import current_app +from lemur.plugins import lemur_adcs as ADCS +from lemur.plugins.bases import IssuerPlugin, SourcePlugin class ADCSIssuerPlugin(IssuerPlugin): @@ -18,7 +18,7 @@ class ADCSIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): """Initialize the issuer with the appropriate details.""" self.session = requests.Session() - super(ADCSIssuerPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) @staticmethod def create_authority(options): @@ -44,12 +44,12 @@ def create_certificate(self, csr, issuer_options): # if there is a config variable ADCS_TEMPLATE_ take the value as Cert template # else default to ADCS_TEMPLATE to be compatible with former versions authority = issuer_options.get("authority").name.upper() - adcs_template = current_app.config.get("ADCS_TEMPLATE_{0}".format(authority), current_app.config.get("ADCS_TEMPLATE")) + adcs_template = current_app.config.get(f"ADCS_TEMPLATE_{authority}", current_app.config.get("ADCS_TEMPLATE")) ca_server = Certsrv( adcs_server, adcs_user, adcs_pwd, auth_method=adcs_auth_method ) - current_app.logger.info("Requesting CSR: {0}".format(csr)) - current_app.logger.info("Issuer options: {0}".format(issuer_options)) + current_app.logger.info(f"Requesting CSR: {csr}") + current_app.logger.info(f"Issuer options: {issuer_options}") cert = ( ca_server.get_cert(csr, adcs_template, encoding="b64") .decode("utf-8") @@ -98,19 +98,19 @@ def get_certificates(self, options, **kwargs): .replace("\r\n", "\n") ) except Exception as err: - if "{0}".format(err).find("CERTSRV_E_PROPERTY_EMPTY"): + if f"{err}".find("CERTSRV_E_PROPERTY_EMPTY"): # this error indicates end of certificate list(?), so we stop break else: # We do nothing in case there is no certificate returned for other reasons - current_app.logger.info("Error with id {0}: {1}".format(id, err)) + current_app.logger.info(f"Error with id {id}: {err}") else: # we have a certificate pubkey = crypto.load_certificate(crypto.FILETYPE_PEM, cert) # loop through extensions to see if we find "TLS Web Server Authentication" for e_id in range(0, pubkey.get_extension_count() - 1): try: - extension = "{0}".format(pubkey.get_extension(e_id)) + extension = f"{pubkey.get_extension(e_id)}" except Exception: extensionn = "" if extension.find("TLS Web Server Authentication") != -1: diff --git a/lemur/plugins/lemur_atlas/plugin.py b/lemur/plugins/lemur_atlas/plugin.py index 6811d8d995..5ab4d89d77 100644 --- a/lemur/plugins/lemur_atlas/plugin.py +++ b/lemur/plugins/lemur_atlas/plugin.py @@ -7,15 +7,14 @@ .. moduleauthor:: Kevin Glisson """ import json +from datetime import datetime from typing import Any, Dict import requests -from requests.exceptions import ConnectionError -from datetime import datetime - from flask import current_app from lemur.plugins import lemur_atlas as atlas from lemur.plugins.bases.metric import MetricPlugin +from requests.exceptions import ConnectionError def millis_since_epoch(): @@ -99,7 +98,7 @@ def submit( if res.status_code != 200: current_app.logger.warning( - "Failed to publish altas metric. {0}".format(res.content) + f"Failed to publish altas metric. {res.content}" ) except ConnectionError: diff --git a/lemur/plugins/lemur_aws/cloudfront.py b/lemur/plugins/lemur_aws/cloudfront.py index 77dbf56ac2..be19f67ee9 100644 --- a/lemur/plugins/lemur_aws/cloudfront.py +++ b/lemur/plugins/lemur_aws/cloudfront.py @@ -6,11 +6,10 @@ """ from flask import current_app -from sentry_sdk import capture_exception - from lemur.exceptions import InvalidDistribution from lemur.extensions import metrics from lemur.plugins.lemur_aws.sts import sts_client +from sentry_sdk import capture_exception def get_all_distributions(**kwargs): @@ -108,7 +107,7 @@ def attach_certificate(distribution_id, iam_cert_id, **kwargs): raise InvalidDistribution(distribution_id) if iam_cert_id == viewer_cert["IAMCertificateId"]: current_app.logger.warning( - "distribution {0} already assigned to IAM certificate {1}, not updated".format( + "distribution {} already assigned to IAM certificate {}, not updated".format( distribution_id, iam_cert_id)) return viewer_cert["IAMCertificateId"] = iam_cert_id diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index ef0af5619a..c5d0297078 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -32,17 +32,17 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Harm Weites """ -from os.path import join import sys +from os.path import join + from acme.errors import ClientError from flask import current_app -from sentry_sdk import capture_exception - from lemur.common.utils import check_validation from lemur.extensions import metrics from lemur.plugins import lemur_aws as aws, ExpirationNotificationPlugin from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns, cloudfront +from sentry_sdk import capture_exception def get_region_from_dns(dns): @@ -126,7 +126,7 @@ def get_elb_endpoints(account_number, region, elb_dict): ) endpoint["policy"] = format_elb_cipher_policy(policy) - current_app.logger.debug("Found new endpoint. Endpoint: {}".format(endpoint)) + current_app.logger.debug(f"Found new endpoint. Endpoint: {endpoint}") endpoints.append(endpoint) @@ -434,7 +434,7 @@ def get_certificate_by_name(self, certificate_name, options): ) except ClientError: current_app.logger.warning( - "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) + f"get_elb_certificate_failed: Unable to get certificate for {certificate_name}") capture_exception() metrics.send( "get_elb_certificate_failed", "counter", 1, @@ -584,7 +584,7 @@ class S3DestinationPlugin(ExportDestinationPlugin): ] def __init__(self, *args, **kwargs): - super(S3DestinationPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def upload(self, name, body, private_key, chain, options, **kwargs): files = self.export(body, private_key, chain, options) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index fb198aaf67..cc75c884d2 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -20,7 +20,7 @@ def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): """ bucket = kwargs["resource"].Bucket(bucket_name) current_app.logger.debug( - "Persisting data to S3. Bucket: {0} Prefix: {1}".format(bucket_name, prefix) + f"Persisting data to S3. Bucket: {bucket_name} Prefix: {prefix}" ) # get data ready for writing diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py index 710b4714f2..d440079618 100644 --- a/lemur/plugins/lemur_aws/sns.py +++ b/lemur/plugins/lemur_aws/sns.py @@ -10,14 +10,13 @@ import arrow import boto3 from flask import current_app - from lemur.plugins.utils import get_plugin_option def publish(topic_arn, certificates, notification_type, options, **kwargs): sns_client = boto3.client("sns", **kwargs) message_ids = {} - subject = "Lemur: {0} Notification".format(notification_type.capitalize()) + subject = f"Lemur: {notification_type.capitalize()} Notification" for certificate in certificates: message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject, options) diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py index 34cb4711bd..f989fed810 100755 --- a/lemur/plugins/lemur_azure_dest/plugin.py +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -9,19 +9,18 @@ .. moduleauthor:: sirferl """ -from flask import current_app +import base64 +import json +import sys +import requests +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs12 +from flask import current_app from lemur.common.defaults import common_name, bitstrength from lemur.common.utils import parse_certificate, parse_private_key, check_validation from lemur.plugins.bases import DestinationPlugin -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.serialization import pkcs12 -import requests -import json -import sys -import base64 - def handle_response(my_response): """ @@ -129,7 +128,7 @@ class AzureDestinationPlugin(DestinationPlugin): def __init__(self, *args, **kwargs): self.session = requests.Session() - super(AzureDestinationPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def upload(self, name, body, private_key, cert_chain, options, **kwargs): """ diff --git a/lemur/plugins/lemur_cfssl/plugin.py b/lemur/plugins/lemur_cfssl/plugin.py index dba67e7f1c..185ffdc261 100644 --- a/lemur/plugins/lemur_cfssl/plugin.py +++ b/lemur/plugins/lemur_cfssl/plugin.py @@ -8,20 +8,19 @@ .. moduleauthor:: Charles Hendrie """ -import json -import requests import base64 -import hmac import hashlib +import hmac +import json +import requests from flask import current_app - -from lemur.common.utils import parse_certificate from lemur.common.utils import get_authority_key +from lemur.common.utils import parse_certificate from lemur.constants import CRLReason -from lemur.plugins.bases import IssuerPlugin -from lemur.plugins import lemur_cfssl as cfssl from lemur.extensions import metrics +from lemur.plugins import lemur_cfssl as cfssl +from lemur.plugins.bases import IssuerPlugin class CfsslIssuerPlugin(IssuerPlugin): @@ -35,7 +34,7 @@ class CfsslIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): self.session = requests.Session() - super(CfsslIssuerPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def create_certificate(self, csr, issuer_options): """ @@ -46,10 +45,10 @@ def create_certificate(self, csr, issuer_options): :return: """ current_app.logger.info( - "Requesting a new cfssl certificate with csr: {0}".format(csr) + f"Requesting a new cfssl certificate with csr: {csr}" ) - url = "{0}{1}".format(current_app.config.get("CFSSL_URL"), "/api/v1/cfssl/sign") + url = "{}{}".format(current_app.config.get("CFSSL_URL"), "/api/v1/cfssl/sign") data = {"certificate_request": csr} data = json.dumps(data) @@ -72,7 +71,7 @@ def create_certificate(self, csr, issuer_options): {"token": token.decode("utf-8"), "request": data.decode("utf-8")} ) - url = "{0}{1}".format( + url = "{}{}".format( current_app.config.get("CFSSL_URL"), "/api/v1/cfssl/authsign" ) response = self.session.post( @@ -107,7 +106,7 @@ def create_authority(options): def revoke_certificate(self, certificate, reason): """Revoke a CFSSL certificate.""" base_url = current_app.config.get("CFSSL_URL") - create_url = "{0}/api/v1/cfssl/revoke".format(base_url) + create_url = f"{base_url}/api/v1/cfssl/revoke" crl_reason = CRLReason.unspecified if "crl_reason" in reason: @@ -122,7 +121,7 @@ def revoke_certificate(self, certificate, reason): + crl_reason.name + '"}' ) - current_app.logger.debug("Revoking cert: {0}".format(data)) + current_app.logger.debug(f"Revoking cert: {data}") response = self.session.post( create_url, data=data.encode(encoding="utf_8", errors="strict") ) diff --git a/lemur/plugins/lemur_cryptography/plugin.py b/lemur/plugins/lemur_cryptography/plugin.py index 1cf60fbaaa..f7692c8c8d 100644 --- a/lemur/plugins/lemur_cryptography/plugin.py +++ b/lemur/plugins/lemur_cryptography/plugin.py @@ -8,17 +8,14 @@ """ import uuid -from flask import current_app - from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization - +from flask import current_app +from lemur.certificates.service import create_csr from lemur.common.utils import parse_private_key -from lemur.plugins.bases import IssuerPlugin from lemur.plugins import lemur_cryptography as cryptography_issuer - -from lemur.certificates.service import create_csr +from lemur.plugins.bases import IssuerPlugin def build_certificate_authority(options): @@ -213,7 +210,7 @@ def create_certificate(self, csr, options): :return: :raise Exception: """ current_app.logger.debug( - "Issuing new cryptography certificate with options: {0}".format(options) + f"Issuing new cryptography certificate with options: {options}" ) cert_pem, chain_cert_pem = issue_certificate(csr, options) return cert_pem, chain_cert_pem, None @@ -228,7 +225,7 @@ def create_authority(options): :return: """ current_app.logger.debug( - "Issuing new cryptography authority with options: {0}".format(options) + f"Issuing new cryptography authority with options: {options}" ) cert_pem, private_key, chain_cert_pem = build_certificate_authority(options) roles = [ diff --git a/lemur/plugins/lemur_csr/plugin.py b/lemur/plugins/lemur_csr/plugin.py index 776dfce5aa..01bffdeb01 100644 --- a/lemur/plugins/lemur_csr/plugin.py +++ b/lemur/plugins/lemur_csr/plugin.py @@ -3,14 +3,12 @@ An export plugin that exports CSR from a private key and certificate. """ -from io import open import subprocess from flask import current_app - -from lemur.utils import mktempfile, mktemppath -from lemur.plugins.bases import ExportPlugin from lemur.plugins import lemur_csr as csr +from lemur.plugins.bases import ExportPlugin +from lemur.utils import mktempfile, mktemppath def run_process(command): diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 946c08b5fc..4886eb3e2f 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -16,20 +16,20 @@ import copy import ipaddress import json +import sys from typing import Any, Dict, List import arrow import pem import requests -import sys from cryptography import x509 from flask import current_app, g from lemur.common.utils import validate_conf, convert_pkcs7_bytes_to_pem from lemur.extensions import metrics from lemur.plugins import lemur_digicert as digicert from lemur.plugins.bases import IssuerPlugin, SourcePlugin -from retrying import retry from requests.packages.urllib3.util.retry import Retry +from retrying import retry def log_status_code(r, *args, **kwargs): @@ -46,7 +46,7 @@ def log_status_code(r, *args, **kwargs): "status_code": r.status_code, "url": (r.url if r.url else ""), } - metrics.send("digicert_status_code_{}".format(r.status_code), "counter", 1) + metrics.send(f"digicert_status_code_{r.status_code}", "counter", 1) current_app.logger.info(log_data) @@ -260,7 +260,7 @@ def handle_cis_response(session, response): @retry(stop_max_attempt_number=10, wait_fixed=1000) def get_certificate_id(session, base_url, order_id): """Retrieve certificate order id from Digicert API.""" - order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id) + order_url = f"{base_url}/services/v2/order/certificate/{order_id}" response_data = handle_response(session.get(order_url)) if response_data["status"] != "issued": raise Exception("Order not in issued state.") @@ -271,7 +271,7 @@ def get_certificate_id(session, base_url, order_id): @retry(stop_max_attempt_number=10, wait_fixed=1000) def get_cis_certificate(session, base_url, order_id): """Retrieve certificate order id from Digicert API, including the chain""" - certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id) + certificate_url = f"{base_url}/platform/cis/certificate/{order_id}/download" session.headers.update({"Accept": "application/x-pkcs7-certificates"}) response = session.get(certificate_url) session.headers.pop("Accept") @@ -314,7 +314,7 @@ def __init__(self, *args, **kwargs): self.session.hooks = dict(response=log_status_code) - super(DigiCertSourcePlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def get_certificates(self): pass @@ -360,7 +360,7 @@ def __init__(self, *args, **kwargs): self.session.hooks = dict(response=log_status_code) - super(DigiCertIssuerPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def create_certificate(self, csr, issuer_options): """Create a DigiCert certificate. @@ -373,7 +373,7 @@ def create_certificate(self, csr, issuer_options): cert_type = current_app.config.get("DIGICERT_ORDER_TYPE") # make certificate request - determinator_url = "{0}/services/v2/order/certificate/{1}".format( + determinator_url = "{}/services/v2/order/certificate/{}".format( base_url, cert_type ) data = map_fields(issuer_options, csr) @@ -387,7 +387,7 @@ def create_certificate(self, csr, issuer_options): certificate_id = get_certificate_id(self.session, base_url, order_id) # retrieve certificate - certificate_url = "{0}/services/v2/certificate/{1}/download/format/pem_all".format( + certificate_url = "{}/services/v2/certificate/{}/download/format/pem_all".format( base_url, certificate_id ) end_entity, intermediate, root = pem.parse( @@ -404,7 +404,7 @@ def revoke_certificate(self, certificate, reason): base_url = current_app.config.get("DIGICERT_URL") # make certificate revoke request - create_url = "{0}/services/v2/certificate/{1}/revoke".format( + create_url = "{}/services/v2/certificate/{}/revoke".format( base_url, certificate.external_id ) @@ -424,7 +424,7 @@ def get_ordered_certificate(self, pending_cert): certificate_id = get_certificate_id(self.session, base_url, order_id) except Exception as ex: return None - certificate_url = "{0}/services/v2/certificate/{1}/download/format/pem_all".format( + certificate_url = "{}/services/v2/certificate/{}/download/format/pem_all".format( base_url, certificate_id ) end_entity, intermediate, root = pem.parse( @@ -440,7 +440,7 @@ def get_ordered_certificate(self, pending_cert): def cancel_ordered_certificate(self, pending_cert, **kwargs): """ Set the certificate order to canceled """ base_url = current_app.config.get("DIGICERT_URL") - api_url = "{0}/services/v2/order/certificate/{1}/status".format( + api_url = "{}/services/v2/order/certificate/{}/status".format( base_url, pending_cert.external_id ) payload = {"status": "CANCELED", "note": kwargs.get("note")} @@ -450,16 +450,16 @@ def cancel_ordered_certificate(self, pending_cert, **kwargs): # don't own that order (someone else's order id!). Either way, we can just ignore it # and have it removed from Lemur current_app.logger.warning( - "Digicert Plugin tried to cancel pending certificate {0} but it does not exist!".format( + "Digicert Plugin tried to cancel pending certificate {} but it does not exist!".format( pending_cert.name ) ) elif response.status_code != 204: current_app.logger.debug( - "{0} code {1}".format(response.status_code, response.content) + f"{response.status_code} code {response.content}" ) raise Exception( - "Failed to cancel pending certificate {0}".format(pending_cert.name) + f"Failed to cancel pending certificate {pending_cert.name}" ) @staticmethod @@ -518,14 +518,14 @@ def __init__(self, *args, **kwargs): adapter = requests.adapters.HTTPAdapter(max_retries=retry_strategy) self.session.mount("https://", adapter) - super(DigiCertCISSourcePlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def get_certificates(self, options, **kwargs): """Fetch all Digicert certificates.""" base_url = current_app.config.get("DIGICERT_CIS_URL") # make request - search_url = "{0}/platform/cis/certificate/search".format(base_url) + search_url = f"{base_url}/platform/cis/certificate/search" certs = [] page = 1 @@ -537,7 +537,7 @@ def get_certificates(self, options, **kwargs): data = handle_cis_response(self.session, response) for c in data["certificates"]: - download_url = "{0}/platform/cis/certificate/{1}".format( + download_url = "{}/platform/cis/certificate/{}".format( base_url, c["id"] ) certificate = self.session.get(download_url) @@ -597,14 +597,14 @@ def __init__(self, *args, **kwargs): adapter = requests.adapters.HTTPAdapter(max_retries=retry_strategy) self.session.mount("https://", adapter) - super(DigiCertCISIssuerPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def create_certificate(self, csr, issuer_options): """Create a DigiCert certificate.""" base_url = current_app.config.get("DIGICERT_CIS_URL") # make certificate request - create_url = "{0}/platform/cis/certificate".format(base_url) + create_url = f"{base_url}/platform/cis/certificate" data = map_cis_fields(issuer_options, csr) response = self.session.post(create_url, data=json.dumps(data)) @@ -627,7 +627,7 @@ def revoke_certificate(self, certificate, reason): base_url = current_app.config.get("DIGICERT_CIS_URL") # make certificate revoke request - revoke_url = "{0}/platform/cis/certificate/{1}/revoke".format( + revoke_url = "{}/platform/cis/certificate/{}/revoke".format( base_url, certificate.external_id ) metrics.send("digicert_revoke_certificate_success", "counter", 1) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index ca15a939fd..89b5e8448f 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -47,7 +47,7 @@ def test_map_fields_with_validity_years_and_ip_addr(mock_current_app): with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: mock_signature_hash.return_value = "sha256" - names = [u"one.example.com", u"two.example.com", u"three.example.com"] + names = ["one.example.com", "two.example.com", "three.example.com"] ip_addr_names = ["1.2.3.4", "2001:db8:85a3::8a2e:370:7334"] options = { "common_name": "example.com", @@ -77,7 +77,7 @@ def test_map_fields_with_validity_end_and_start(mock_current_app): with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: mock_signature_hash.return_value = "sha256" - names = [u"one.example.com", u"two.example.com", u"three.example.com"] + names = ["one.example.com", "two.example.com", "three.example.com"] options = { "common_name": "example.com", "owner": "bob@example.com", @@ -109,7 +109,7 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority): with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: mock_signature_hash.return_value = "sha256" - names = [u"one.example.com", u"two.example.com", u"three.example.com"] + names = ["one.example.com", "two.example.com", "three.example.com"] options = { "common_name": "example.com", "owner": "bob@example.com", @@ -144,7 +144,7 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: mock_signature_hash.return_value = "sha256" - names = [u"one.example.com", u"two.example.com", u"three.example.com"] + names = ["one.example.com", "two.example.com", "three.example.com"] options = { "common_name": "example.com", "owner": "bob@example.com", diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 1ccac0fdc0..b27d6b767a 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -6,21 +6,19 @@ .. moduleauthor:: Kevin Glisson """ -import boto3 from html.parser import HTMLParser + +import boto3 from flask import current_app from flask_mail import Message -from sentry_sdk import capture_exception - from lemur.constants import EMAIL_RE, EMAIL_RE_HELP -from lemur.extensions import smtp_mail from lemur.exceptions import InvalidConfiguration - -from lemur.plugins.bases import ExpirationNotificationPlugin +from lemur.extensions import smtp_mail from lemur.plugins import lemur_email as email - +from lemur.plugins.bases import ExpirationNotificationPlugin from lemur.plugins.lemur_email.templates.config import env from lemur.plugins.utils import get_plugin_option +from sentry_sdk import capture_exception def render_html(template_name, options, certificates): @@ -33,7 +31,7 @@ def render_html(template_name, options, certificates): :return: """ message = {"options": options, "certificates": certificates} - template = env.get_template("{}.html".format(template_name)) + template = env.get_template(f"{template_name}.html") return template.render( dict(message=message, hostname=current_app.config.get("LEMUR_HOSTNAME")) ) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 7267ff42ba..15b8718d3b 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -1,16 +1,16 @@ -import arrow -import requests import json import sys -from flask import current_app -from retrying import retry -from requests.packages.urllib3.util.retry import Retry +import arrow +import requests +from flask import current_app +from lemur.common.utils import validate_conf, get_key_type_from_certificate from lemur.constants import CRLReason +from lemur.extensions import metrics from lemur.plugins import lemur_entrust as entrust from lemur.plugins.bases import IssuerPlugin, SourcePlugin -from lemur.extensions import metrics -from lemur.common.utils import validate_conf, get_key_type_from_certificate +from requests.packages.urllib3.util.retry import Retry +from retrying import retry def log_status_code(r, *args, **kwargs): @@ -242,7 +242,7 @@ def __init__(self, *args, **kwargs): adapter = requests.adapters.HTTPAdapter(max_retries=retry_strategy) self.session.mount("https://", adapter) - super(EntrustIssuerPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def create_certificate(self, csr, issuer_options): """ @@ -417,7 +417,7 @@ def __init__(self, *args, **kwargs): adapter = requests.adapters.HTTPAdapter(max_retries=retry_strategy) self.session.mount("https://", adapter) - super(EntrustSourcePlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def get_certificates(self, options, **kwargs): """ Fetch all Entrust certificates """ @@ -446,7 +446,7 @@ def get_certificates(self, options, **kwargs): if status_code > 399: raise Exception(f"ENTRUST error: {status_code}\n{data['errors']}") for c in data["certificates"]: - download_url = "{0}{1}".format( + download_url = "{}{}".format( host, c["uri"] ) cert_response = self.session.get(download_url) diff --git a/lemur/plugins/lemur_entrust/tests/test_entrust.py b/lemur/plugins/lemur_entrust/tests/test_entrust.py index f448aad492..416461b7ec 100644 --- a/lemur/plugins/lemur_entrust/tests/test_entrust.py +++ b/lemur/plugins/lemur_entrust/tests/test_entrust.py @@ -2,8 +2,8 @@ import arrow from cryptography import x509 -from lemur.plugins.lemur_entrust import plugin from freezegun import freeze_time +from lemur.plugins.lemur_entrust import plugin def config_mock(*args): @@ -35,7 +35,7 @@ def test_process_options(mock_current_app, authority): mock_current_app.config.get = Mock(side_effect=config_mock) plugin.determine_end_date = Mock(return_value=arrow.get(2017, 11, 5).format('YYYY-MM-DD')) authority.name = "Entrust" - names = [u"one.example.com", u"two.example.com", u"three.example.com"] + names = ["one.example.com", "two.example.com", "three.example.com"] options = { "common_name": "example.com", "owner": "bob@example.com", diff --git a/lemur/plugins/lemur_jks/plugin.py b/lemur/plugins/lemur_jks/plugin.py index c0501fe404..f5942e8dc5 100644 --- a/lemur/plugins/lemur_jks/plugin.py +++ b/lemur/plugins/lemur_jks/plugin.py @@ -31,7 +31,7 @@ def create_truststore(cert, chain, alias, passphrase): entries = [] for idx, cert_bytes in enumerate(cert_chain_as_der(cert, chain)): # The original cert gets name _cert, first chain element is _cert_1, etc. - cert_alias = alias + "_cert" + ("_{}".format(idx) if idx else "") + cert_alias = alias + "_cert" + (f"_{idx}" if idx else "") entries.append(TrustedCertEntry.new(cert_alias, cert_bytes)) return KeyStore.new("jks", entries).saves(passphrase) diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index c6674d2091..2ee6211f3c 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -15,7 +15,6 @@ import requests from flask import current_app - from lemur.common.defaults import common_name from lemur.common.utils import parse_certificate, base64encode, check_validation from lemur.plugins.bases import DestinationPlugin @@ -81,7 +80,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain): } if secret_format == "Full": secret["data"] = { - "combined.pem": base64encode("%s\n%s" % (body, private_key)), + "combined.pem": base64encode(f"{body}\n{private_key}"), "ca.crt": base64encode(cert_chain), "service.key": base64encode(private_key), "service.crt": base64encode(body), @@ -89,7 +88,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain): if secret_format == "TLS": secret["type"] = "kubernetes.io/tls" secret["data"] = { - "tls.crt": base64encode("%s\n%s" % (body, cert_chain)), + "tls.crt": base64encode(f"{body}\n{cert_chain}"), "tls.key": base64encode(private_key), } if secret_format == "Certificate": @@ -184,7 +183,7 @@ class KubernetesDestinationPlugin(DestinationPlugin): ] def __init__(self, *args, **kwargs): - super(KubernetesDestinationPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def upload(self, name, body, private_key, cert_chain, options, **kwargs): @@ -209,7 +208,7 @@ def upload(self, name, body, private_key, cert_chain, options, **kwargs): except Exception as e: current_app.logger.exception( - "Exception in upload: {}".format(e), exc_info=True + f"Exception in upload: {e}", exc_info=True ) raise @@ -221,7 +220,7 @@ def k8s_bearer(self, options): bearer = self.get_option("kubernetesAuthToken", options) if not bearer: bearer_file = self.get_option("kubernetesAuthTokenFile", options) - with open(bearer_file, "r") as file: + with open(bearer_file) as file: bearer = file.readline() if bearer: current_app.logger.debug("Using token read from %s", bearer_file) @@ -251,7 +250,7 @@ def k8s_namespace(self, options): namespace = self.get_option("kubernetesNamespace", options) if not namespace: namespace_file = self.get_option("kubernetesNamespaceFile", options) - with open(namespace_file, "r") as file: + with open(namespace_file) as file: namespace = file.readline() if namespace: current_app.logger.debug( @@ -268,7 +267,7 @@ def k8s_namespace(self, options): class K8sSession(requests.Session): def __init__(self, bearer, cert_file): - super(K8sSession, self).__init__() + super().__init__() self.headers.update({"Authorization": "Bearer %s" % bearer}) @@ -296,7 +295,7 @@ def request( """ This method overrides the default timeout to be 10s. """ - return super(K8sSession, self).request( + return super().request( method, url, params, diff --git a/lemur/plugins/lemur_openssl/plugin.py b/lemur/plugins/lemur_openssl/plugin.py index f6527eca7d..e0726b6cc5 100644 --- a/lemur/plugins/lemur_openssl/plugin.py +++ b/lemur/plugins/lemur_openssl/plugin.py @@ -6,16 +6,14 @@ .. moduleauthor:: Kevin Glisson """ -from io import open import subprocess from flask import current_app - -from lemur.utils import mktempfile, mktemppath -from lemur.plugins.bases import ExportPlugin -from lemur.plugins import lemur_openssl as openssl -from lemur.common.utils import get_psuedo_random_string, parse_certificate, check_validation from lemur.common.defaults import common_name +from lemur.common.utils import get_psuedo_random_string, parse_certificate, check_validation +from lemur.plugins import lemur_openssl as openssl +from lemur.plugins.bases import ExportPlugin +from lemur.utils import mktempfile, mktemppath def run_process(command): @@ -93,7 +91,7 @@ def create_pkcs12(cert, chain, p12_tmp, key, alias, passphrase, legacy: bool = F "-out", p12_tmp, "-password", - "pass:{}".format(passphrase), + f"pass:{passphrase}", ] if legacy: @@ -163,19 +161,19 @@ def export(self, body, chain, key, options, **kwargs): with mktemppath() as output_tmp: if type == "PKCS12 (.p12)": if not key: - raise Exception("Private Key required by {0}".format(type)) + raise Exception(f"Private Key required by {type}") create_pkcs12(body, chain, output_tmp, key, alias, passphrase) extension = "p12" elif type == "legacy PKCS12 (.p12)": if not key: - raise Exception("Private Key required by {0}".format(type)) + raise Exception(f"Private Key required by {type}") create_pkcs12(body, chain, output_tmp, key, alias, passphrase, legacy=True) extension = "p12" else: - raise Exception("Unable to export, unsupported type: {0}".format(type)) + raise Exception(f"Unable to export, unsupported type: {type}") with open(output_tmp, "rb") as f: raw = f.read() diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 947bf9d469..697496a7bd 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -19,13 +19,12 @@ from os import path import paramiko -from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError - from flask import current_app -from lemur.plugins import lemur_sftp from lemur.common.defaults import common_name from lemur.common.utils import parse_certificate, check_validation +from lemur.plugins import lemur_sftp from lemur.plugins.bases import DestinationPlugin +from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError class SFTPDestinationPlugin(DestinationPlugin): @@ -108,7 +107,7 @@ def open_sftp_connection(self, options): # delete files try: current_app.logger.debug( - "Connecting to {0}@{1}:{2}".format(user, host, port) + f"Connecting to {user}@{host}:{port}" ) ssh = paramiko.SSHClient() @@ -135,10 +134,10 @@ def open_sftp_connection(self, options): return ssh.open_sftp(), ssh except AuthenticationException as e: - current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + current_app.logger.error(f"ERROR in {e.__class__}: {e}") raise AuthenticationException("Couldn't connect to {0}, due to an Authentication exception.") except NoValidConnectionsError as e: - current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + current_app.logger.error(f"ERROR in {e.__class__}: {e}") raise NoValidConnectionsError("Couldn't connect to {0}, possible timeout or invalid hostname") # this is called when using this as a default destination plugin @@ -200,14 +199,14 @@ def delete_file(self, dst_path, files, options): # delete files for filename, _ in files.items(): current_app.logger.debug( - "Deleting {0} from {1}".format(filename, dst_path) + f"Deleting {filename} from {dst_path}" ) try: sftp.remove(path.join(dst_path, filename)) except PermissionError as permerror: if permerror.errno == 13: current_app.logger.debug( - "Deleting {0} from {1} returned Permission Denied Error, making file writable and retrying".format( + "Deleting {} from {} returned Permission Denied Error, making file writable and retrying".format( filename, dst_path) ) sftp.chmod(path.join(dst_path, filename), 0o600) @@ -217,7 +216,7 @@ def delete_file(self, dst_path, files, options): except (AuthenticationException, NoValidConnectionsError) as e: raise e except Exception as e: - current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + current_app.logger.error(f"ERROR in {e.__class__}: {e}") try: ssh.close() except BaseException: @@ -252,18 +251,18 @@ def upload_file(self, dst_path, files, options): if part != "/" and part != "": remote_path = path.join(remote_path, part) sftp.stat(remote_path) - except IOError: - current_app.logger.debug("{0} doesn't exist, trying to create it".format(remote_path)) + except OSError: + current_app.logger.debug(f"{remote_path} doesn't exist, trying to create it") try: sftp.mkdir(remote_path) - except IOError as ioerror: + except OSError as ioerror: current_app.logger.debug( - "Couldn't create {0}, error message: {1}".format(remote_path, ioerror)) + f"Couldn't create {remote_path}, error message: {ioerror}") # upload certificate files to the sftp destination for filename, data in files.items(): current_app.logger.debug( - "Uploading {0} to {1}".format(filename, dst_path) + f"Uploading {filename} to {dst_path}" ) try: with sftp.open(path.join(dst_path, filename), "w") as f: @@ -271,7 +270,7 @@ def upload_file(self, dst_path, files, options): except PermissionError as permerror: if permerror.errno == 13: current_app.logger.debug( - "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format( + "Uploading {} to {} returned Permission Denied Error, making file writable and retrying".format( filename, dst_path) ) sftp.chmod(path.join(dst_path, filename), 0o600) @@ -285,7 +284,7 @@ def upload_file(self, dst_path, files, options): except (AuthenticationException, NoValidConnectionsError) as e: raise e except Exception as e: - current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + current_app.logger.error(f"ERROR in {e.__class__}: {e}") try: ssh.close() except BaseException: diff --git a/lemur/plugins/lemur_slack/tests/test_slack.py b/lemur/plugins/lemur_slack/tests/test_slack.py index eb69593189..813bbe12d5 100644 --- a/lemur/plugins/lemur_slack/tests/test_slack.py +++ b/lemur/plugins/lemur_slack/tests/test_slack.py @@ -1,10 +1,9 @@ from datetime import timedelta import arrow -from moto import mock_ses - from lemur.tests.factories import NotificationFactory, CertificateFactory from lemur.tests.test_messaging import verify_sender_email +from moto import mock_ses def test_formatting(certificate): @@ -18,7 +17,7 @@ def test_formatting(certificate): "color": "danger", "fields": [ {"short": True, "value": "joe@example.com", "title": "Owner"}, - {"short": True, "value": u"Tuesday, December 31, 2047", "title": "Expires"}, + {"short": True, "value": "Tuesday, December 31, 2047", "title": "Expires"}, {"short": True, "value": 0, "title": "Endpoints Detected"}, ], "title_link": "https://lemur.example.com/#/certificates/{name}".format( diff --git a/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py b/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py index 822b1db658..9d74971763 100644 --- a/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py +++ b/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py @@ -1,8 +1,7 @@ import lemur_statsd as plug - +from datadog import DogStatsd from flask import current_app from lemur.plugins.bases.metric import MetricPlugin -from datadog import DogStatsd class StatsdMetricPlugin(MetricPlugin): @@ -37,7 +36,7 @@ def submit( "Invalid Metric Tags for Statsd: Tags must be in dict format" ) else: - tags = list(map(lambda e: "{0}:{1}".format(*e), metric_tags.items())) + tags = list(map(lambda e: "{}:{}".format(*e), metric_tags.items())) if metric_type.upper() == "COUNTER": self.statsd.increment(metric_name, metric_value, tags) diff --git a/lemur/plugins/lemur_statsd/setup.py b/lemur/plugins/lemur_statsd/setup.py index 9b3c5f5205..a6a5c8ac8b 100644 --- a/lemur/plugins/lemur_statsd/setup.py +++ b/lemur/plugins/lemur_statsd/setup.py @@ -1,5 +1,4 @@ """Basic package information""" -from __future__ import absolute_import from setuptools import setup, find_packages install_requires = ["lemur", "datadog"] diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 0869909f61..186709a542 100755 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -16,12 +16,11 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from flask import current_app -from validators.url import url - from lemur.common.defaults import common_name, country, state, location, organizational_unit, organization from lemur.common.utils import parse_certificate, check_validation from lemur.plugins.bases import DestinationPlugin from lemur.plugins.bases import SourcePlugin +from validators.url import url class VaultSourcePlugin(SourcePlugin): @@ -105,19 +104,19 @@ def get_certificates(self, options, **kwargs): client = hvac.Client(url=url) if auth_method == 'token': - with open(auth_key, "r") as tfile: + with open(auth_key) as tfile: token = tfile.readline().rstrip("\n") client.token = token if auth_method == 'kubernetes': token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' - with open(token_path, 'r') as f: + with open(token_path) as f: jwt = f.read() client.auth_kubernetes(auth_key, jwt) client.secrets.kv.default_kv_version = api_version - path = "{0}/{1}".format(path, obj_name) + path = f"{path}/{obj_name}" secret = get_secret(client, mount, path) for cname in secret["data"]: @@ -232,7 +231,7 @@ class VaultDestinationPlugin(DestinationPlugin): ] def __init__(self, *args, **kwargs): - super(VaultDestinationPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def upload(self, name, body, private_key, cert_chain, options, **kwargs): """ @@ -275,13 +274,13 @@ def upload(self, name, body, private_key, cert_chain, options, **kwargs): client = hvac.Client(url=url) if auth_method == 'token': - with open(auth_key, "r") as tfile: + with open(auth_key) as tfile: token = tfile.readline().rstrip("\n") client.token = token if auth_method == 'kubernetes': token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' - with open(token_path, 'r') as f: + with open(token_path) as f: jwt = f.read() client.auth_kubernetes(auth_key, jwt) @@ -307,7 +306,7 @@ def upload(self, name, body, private_key, cert_chain, options, **kwargs): C=country(cert) ) - path = "{0}/{1}".format(t_path, f_obj_name) + path = f"{t_path}/{f_obj_name}" secret = get_secret(client, mount, path) secret["data"][cname] = {} @@ -318,14 +317,14 @@ def upload(self, name, body, private_key, cert_chain, options, **kwargs): chain = cert_chain if bundle == "Nginx": - secret["data"][cname]["crt"] = "{0}\n{1}".format(body, chain) + secret["data"][cname]["crt"] = f"{body}\n{chain}" secret["data"][cname]["key"] = private_key elif bundle == "Apache": secret["data"][cname]["crt"] = body secret["data"][cname]["chain"] = chain secret["data"][cname]["key"] = private_key elif bundle == "PEM": - secret["data"][cname]["pem"] = "{0}\n{1}\n{2}".format( + secret["data"][cname]["pem"] = "{}\n{}\n{}".format( body, chain, private_key ) else: @@ -339,7 +338,7 @@ def upload(self, name, body, private_key, cert_chain, options, **kwargs): ) except ConnectionError as err: current_app.logger.exception( - "Exception uploading secret to vault: {0}".format(err), exc_info=True + f"Exception uploading secret to vault: {err}", exc_info=True ) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 7f6d84c7fd..2c423a714e 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -12,12 +12,11 @@ import xmltodict from cryptography import x509 from flask import current_app -from sentry_sdk import capture_exception - from lemur.common.utils import get_psuedo_random_string from lemur.extensions import metrics from lemur.plugins import lemur_verisign as verisign from lemur.plugins.bases import IssuerPlugin, SourcePlugin +from sentry_sdk import capture_exception # https://support.venafi.com/entries/66445046-Info-VeriSign-Error-Codes VERISIGN_ERRORS = { @@ -72,7 +71,7 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ - metrics.send("symantec_status_code_{}".format(r.status_code), "counter", 1) + metrics.send(f"symantec_status_code_{r.status_code}", "counter", 1) def get_additional_names(options): @@ -102,7 +101,7 @@ def process_options(options): # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type # else default to "Server", to be compatoible with former versions authority = options.get("authority").name.upper() - product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority), "Server") + product_type = current_app.config.get(f"VERISIGN_PRODUCT_{authority}", "Server") data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", @@ -189,7 +188,7 @@ def __init__(self, *args, **kwargs): self.session = requests.Session() self.session.cert = current_app.config.get("VERISIGN_PEM_PATH") self.session.hooks = dict(response=log_status_code) - super(VerisignIssuerPlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def create_certificate(self, csr, issuer_options): """ @@ -205,7 +204,7 @@ def create_certificate(self, csr, issuer_options): data["csr"] = csr current_app.logger.info( - "Requesting a new verisign certificate: {0}".format(data) + f"Requesting a new verisign certificate: {data}" ) response = self.session.post(url, data=data) @@ -227,7 +226,8 @@ def create_certificate(self, csr, issuer_options): external_id = None if 'Transaction_ID' in response_dict['Response'].keys(): external_id = response_dict['Response']['Transaction_ID'] - chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) + chain = current_app.config.get(f"VERISIGN_INTERMEDIATE_{authority}", + current_app.config.get("VERISIGN_INTERMEDIATE")) return cert, chain, external_id @staticmethod @@ -281,7 +281,7 @@ def clear_pending_certificates(self): response = self.session.get(url, params={"transaction_id": order_id}) if response.status_code == 200: - print("Rejecting certificate. TransactionId: {}".format(order_id)) + print(f"Rejecting certificate. TransactionId: {order_id}") class VerisignSourcePlugin(SourcePlugin): @@ -298,7 +298,7 @@ class VerisignSourcePlugin(SourcePlugin): def __init__(self, *args, **kwargs): self.session = requests.Session() self.session.cert = current_app.config.get("VERISIGN_PEM_PATH") - super(VerisignSourcePlugin, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def get_certificates(self): url = current_app.config.get("VERISIGN_URL") + "/reportingws" diff --git a/lemur/plugins/views.py b/lemur/plugins/views.py index 605b234a21..9dc5f23876 100644 --- a/lemur/plugins/views.py +++ b/lemur/plugins/views.py @@ -9,11 +9,9 @@ from flask import Blueprint from flask_restful import Api, reqparse from lemur.auth.service import AuthenticatedResource - - -from lemur.schemas import plugins_output_schema, plugin_output_schema from lemur.common.schema import validate_schema from lemur.plugins.base import plugins +from lemur.schemas import plugins_output_schema, plugin_output_schema mod = Blueprint("plugins", __name__) api = Api(mod) @@ -24,7 +22,7 @@ class PluginsList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(PluginsList, self).__init__() + super().__init__() @validate_schema(None, plugins_output_schema) def get(self): @@ -83,7 +81,7 @@ class Plugins(AuthenticatedResource): """ Defines the 'plugins' endpoint """ def __init__(self): - super(Plugins, self).__init__() + super().__init__() @validate_schema(None, plugin_output_schema) def get(self, name): diff --git a/lemur/roles/models.py b/lemur/roles/models.py index 5c9e7f7a38..e11a1c2936 100644 --- a/lemur/roles/models.py +++ b/lemur/roles/models.py @@ -9,17 +9,16 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy.orm import relationship -from sqlalchemy import Boolean, Column, Integer, String, Text, ForeignKey - from lemur.database import BaseModel -from lemur.utils import Vault from lemur.models import ( roles_users, roles_authorities, roles_certificates, pending_cert_role_associations, ) +from lemur.utils import Vault +from sqlalchemy import Boolean, Column, Integer, String, Text, ForeignKey +from sqlalchemy.orm import relationship class Role(BaseModel): @@ -52,4 +51,4 @@ class Role(BaseModel): sensitive_fields = ("password",) def __repr__(self): - return "Role(name={name})".format(name=self.name) + return f"Role(name={self.name})" diff --git a/lemur/roles/views.py b/lemur/roles/views.py index 787e03b342..afe0a7a0bb 100644 --- a/lemur/roles/views.py +++ b/lemur/roles/views.py @@ -10,21 +10,18 @@ from flask import Blueprint, g from flask import make_response, jsonify from flask_restful import reqparse, Api - -from lemur.roles import service -from lemur.auth.service import AuthenticatedResource from lemur.auth.permissions import RoleMemberPermission, admin_permission +from lemur.auth.service import AuthenticatedResource +from lemur.common.schema import validate_schema from lemur.common.utils import paginated_parser from lemur.logs import service as log_service - -from lemur.common.schema import validate_schema +from lemur.roles import service from lemur.roles.schemas import ( role_input_schema, role_output_schema, roles_output_schema, ) - mod = Blueprint("roles", __name__) api = Api(mod) @@ -34,7 +31,7 @@ class RolesList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(RolesList, self).__init__() + super().__init__() @validate_schema(None, roles_output_schema) def get(self): @@ -153,7 +150,7 @@ def post(self, data=None): class RoleViewCredentials(AuthenticatedResource): def __init__(self): - super(RoleViewCredentials, self).__init__() + super().__init__() def get(self, role_id): """ @@ -209,7 +206,7 @@ def get(self, role_id): class Roles(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Roles, self).__init__() + super().__init__() @validate_schema(None, role_output_schema) def get(self, role_id): @@ -342,7 +339,7 @@ class UserRolesList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(UserRolesList, self).__init__() + super().__init__() @validate_schema(None, roles_output_schema) def get(self, user_id): @@ -402,7 +399,7 @@ class AuthorityRolesList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(AuthorityRolesList, self).__init__() + super().__init__() @validate_schema(None, roles_output_schema) def get(self, authority_id): diff --git a/lemur/schemas.py b/lemur/schemas.py index b7266a52bb..9f52ae2ef5 100644 --- a/lemur/schemas.py +++ b/lemur/schemas.py @@ -7,30 +7,27 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy.orm.exc import NoResultFound - -from marshmallow import fields, post_load, pre_load, post_dump -from marshmallow.exceptions import ValidationError - +from lemur.authorities.models import Authority +from lemur.certificates.models import Certificate from lemur.common import validators -from lemur.common.schema import LemurSchema, LemurInputSchema, LemurOutputSchema from lemur.common.fields import ( KeyUsageExtension, ExtendedKeyUsageExtension, BasicConstraintsExtension, SubjectAlternativeNameExtension, ) - +from lemur.common.schema import LemurSchema, LemurInputSchema, LemurOutputSchema +from lemur.destinations.models import Destination +from lemur.dns_providers.models import DnsProvider +from lemur.notifications.models import Notification from lemur.plugins import plugins from lemur.plugins.utils import get_plugin_option +from lemur.policies.models import RotationPolicy from lemur.roles.models import Role from lemur.users.models import User -from lemur.authorities.models import Authority -from lemur.dns_providers.models import DnsProvider -from lemur.policies.models import RotationPolicy -from lemur.certificates.models import Certificate -from lemur.destinations.models import Destination -from lemur.notifications.models import Notification +from marshmallow import fields, post_load, pre_load, post_dump +from marshmallow.exceptions import ValidationError +from sqlalchemy.orm.exc import NoResultFound def validate_options(options): @@ -190,7 +187,7 @@ def get_object(self, data, many=False): data["plugin_object"] = plugins.get(data["slug"]) except KeyError as e: raise ValidationError( - "Unable to find plugin. Slug: {0} Reason: {1}".format(data["slug"], e) + "Unable to find plugin. Slug: {} Reason: {}".format(data["slug"], e) ) plugin_options_validated = [] diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index d4d0ca050c..db3b48df95 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -5,25 +5,22 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from copy import deepcopy -import click import sys import time +from copy import deepcopy -from tabulate import tabulate +import click from flask import current_app -from sentry_sdk import capture_exception - +from lemur.certificates import service as certificate_service from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS - +from lemur.destinations import service as dest_service from lemur.extensions import metrics from lemur.plugins.base import plugins from lemur.plugins.utils import get_plugin_option, set_plugin_option - -from lemur.destinations import service as dest_service from lemur.sources import service as source_service from lemur.users import service as user_service -from lemur.certificates import service as certificate_service +from sentry_sdk import capture_exception +from tabulate import tabulate @click.group(name="source", help="Handles all source related tasks.") @@ -50,7 +47,7 @@ def validate_sources(source_strings): if not source: click.echo( - "Unable to find specified source with label: {0}".format(source_str) + f"Unable to find specified source with label: {source_str}" ) sys.exit(1) @@ -77,7 +74,7 @@ def validate_destinations(destination_strings): if not dest: click.echo( - "Unable to find specified destination with label: {0}".format(label) + f"Unable to find specified destination with label: {label}" ) sys.exit(1) @@ -155,7 +152,7 @@ def sync(source_strings, ttl): except Exception as e: current_app.logger.exception(e) - click.echo("[X] Failed syncing source {label}!\n".format(label=source.label)) + click.echo(f"[X] Failed syncing source {source.label}!\n") capture_exception() metrics.send( @@ -206,7 +203,7 @@ def clean(source_strings, commit): start_time = time.time() - click.echo("[+] Staring to clean source: {label}!\n".format(label=source.label)) + click.echo(f"[+] Staring to clean source: {source.label}!\n") cleaned = 0 certificates = certificate_service.get_all_pending_cleaning_expired(source) @@ -272,7 +269,7 @@ def clean_unused_and_expiring_within_days(source_strings, days_to_expire, commit start_time = time.time() - click.echo("[+] Staring to clean source: {label}!\n".format(label=source.label)) + click.echo(f"[+] Staring to clean source: {source.label}!\n") cleaned = 0 certificates = certificate_service.get_all_pending_cleaning_expiring_in_days(source, days_to_expire) @@ -338,7 +335,7 @@ def clean_unused_and_issued_since_days(source_strings, days_since_issuance, comm start_time = time.time() - click.echo("[+] Staring to clean source: {label}!\n".format(label=source.label)) + click.echo(f"[+] Staring to clean source: {source.label}!\n") cleaned = 0 certificates = certificate_service.get_all_pending_cleaning_issued_since_days(source, days_since_issuance) diff --git a/lemur/sources/models.py b/lemur/sources/models.py index bf514cde71..24c69bd4fb 100644 --- a/lemur/sources/models.py +++ b/lemur/sources/models.py @@ -5,13 +5,12 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from sqlalchemy.orm import relationship -from sqlalchemy import Column, Integer, String, Text, Boolean -from sqlalchemy_utils import JSONType from lemur.database import BaseModel - from lemur.plugins.base import plugins +from sqlalchemy import Column, Integer, String, Text, Boolean +from sqlalchemy.orm import relationship from sqlalchemy_utils import ArrowType +from sqlalchemy_utils import JSONType class Source(BaseModel): @@ -30,4 +29,4 @@ def plugin(self): return plugins.get(self.plugin_name) def __repr__(self): - return "Source(label={label})".format(label=self.label) + return f"Source(label={self.label})" diff --git a/lemur/sources/service.py b/lemur/sources/service.py index ac6c884e9e..ced4ffb840 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -5,31 +5,29 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import arrow -from datetime import timedelta import copy +from datetime import timedelta +import arrow from flask import current_app -from sqlalchemy.exc import OperationalError -from sentry_sdk import capture_exception -from sqlalchemy import cast -from sqlalchemy_utils import ArrowType - from lemur import database -from lemur.sources.models import Source -from lemur.certificates.models import Certificate from lemur.certificates import service as certificate_service +from lemur.certificates.models import Certificate +from lemur.certificates.schemas import CertificateUploadInputSchema +from lemur.common.defaults import serial +from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate +from lemur.destinations import service as destination_service from lemur.endpoints import service as endpoint_service from lemur.endpoints.models import Endpoint from lemur.extensions import metrics -from lemur.destinations import service as destination_service - -from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate -from lemur.common.defaults import serial from lemur.logs import service as log_service from lemur.plugins.base import plugins from lemur.plugins.utils import get_plugin_option, set_plugin_option +from lemur.sources.models import Source +from sentry_sdk import capture_exception +from sqlalchemy import cast +from sqlalchemy.exc import OperationalError +from sqlalchemy_utils import ArrowType def certificate_create(certificate, source): @@ -37,7 +35,7 @@ def certificate_create(certificate, source): if errors: raise Exception( - "Unable to import certificate: {reasons}".format(reasons=errors) + f"Unable to import certificate: {errors}" ) data["creator"] = certificate["creator"] @@ -73,14 +71,14 @@ def sync_update_destination(certificate, source): def sync_endpoints(source): new, updated, updated_by_hash = 0, 0, 0 - current_app.logger.debug("Retrieving endpoints from {0}".format(source.label)) + current_app.logger.debug(f"Retrieving endpoints from {source.label}") s = plugins.get(source.plugin_name) try: endpoints = s.get_endpoints(source.options) except NotImplementedError: current_app.logger.warning( - "Unable to sync endpoints for source {0} plugin has not implemented 'get_endpoints'".format( + "Unable to sync endpoints for source {} plugin has not implemented 'get_endpoints'".format( source.label ) ) @@ -117,7 +115,7 @@ def sync_endpoints(source): certificate_attached_to_endpoint = s.get_certificate_by_name(certificate_name, source.options) except NotImplementedError: current_app.logger.warning( - "Unable to describe server certificate for endpoints in source {0}:" + "Unable to describe server certificate for endpoints in source {}:" " plugin has not implemented 'get_certificate_by_name'".format( source.label ) @@ -133,7 +131,7 @@ def sync_endpoints(source): if len(lemur_matching_cert) > 1: current_app.logger.error( - "Too Many Certificates Found{0}. Name: {1} Endpoint: {2}".format( + "Too Many Certificates Found{}. Name: {} Endpoint: {}".format( len(lemur_matching_cert), certificate_name, endpoint["name"] ) ) @@ -176,7 +174,7 @@ def sync_endpoints(source): new += 1 else: - current_app.logger.debug("Endpoint Updated: {}".format(endpoint)) + current_app.logger.debug(f"Endpoint Updated: {endpoint}") endpoint_service.update(exists.id, **endpoint) updated += 1 @@ -230,7 +228,7 @@ def find_cert(certificate): def sync_certificates(source, user): new, updated, updated_by_hash, unlinked = 0, 0, 0, 0 - current_app.logger.debug("Retrieving certificates from {0}".format(source.label)) + current_app.logger.debug(f"Retrieving certificates from {source.label}") s = plugins.get(source.plugin_name) certificates = s.get_certificates(source.options) diff --git a/lemur/sources/views.py b/lemur/sources/views.py index e48a539b18..6229d9b889 100644 --- a/lemur/sources/views.py +++ b/lemur/sources/views.py @@ -8,20 +8,17 @@ """ from flask import Blueprint from flask_restful import Api, reqparse -from lemur.sources import service - +from lemur.auth.permissions import admin_permission +from lemur.auth.service import AuthenticatedResource from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser +from lemur.sources import service from lemur.sources.schemas import ( source_input_schema, source_output_schema, sources_output_schema, ) -from lemur.auth.service import AuthenticatedResource -from lemur.auth.permissions import admin_permission -from lemur.common.utils import paginated_parser - - mod = Blueprint("sources", __name__) api = Api(mod) @@ -31,7 +28,7 @@ class SourcesList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(SourcesList, self).__init__() + super().__init__() @validate_schema(None, sources_output_schema) def get(self): @@ -175,7 +172,7 @@ def post(self, data=None): class Sources(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Sources, self).__init__() + super().__init__() @validate_schema(None, source_output_schema) def get(self, source_id): @@ -308,7 +305,7 @@ class CertificateSources(AuthenticatedResource): """ Defines the 'certificate/ """ -from sqlalchemy.orm import relationship +from lemur.database import BaseModel, db +from lemur.extensions import bcrypt +from lemur.models import roles_users from sqlalchemy import Integer, String, Column, Boolean from sqlalchemy.event import listen - +from sqlalchemy.orm import relationship from sqlalchemy_utils.types.arrow import ArrowType -from lemur.database import BaseModel, db -from lemur.models import roles_users - -from lemur.extensions import bcrypt - def hash_password(mapper, connect, target): """ @@ -105,7 +102,7 @@ def is_admin_or_global_cert_issuer(self): return True def __repr__(self): - return "User(username={username})".format(username=self.username) + return f"User(username={self.username})" listen(User, "before_insert", hash_password) diff --git a/lemur/users/views.py b/lemur/users/views.py index 29aa3d48c7..62e454d114 100644 --- a/lemur/users/views.py +++ b/lemur/users/views.py @@ -30,7 +30,7 @@ class UsersList(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(UsersList, self).__init__() + super().__init__() @validate_schema(None, users_output_schema) def get(self): @@ -170,7 +170,7 @@ def post(self, data=None): class Users(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(Users, self).__init__() + super().__init__() @validate_schema(None, user_output_schema) def get(self, user_id): @@ -285,7 +285,7 @@ def put(self, user_id, data=None): class CertificateUsers(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(CertificateUsers, self).__init__() + super().__init__() @validate_schema(None, user_output_schema) def get(self, certificate_id): @@ -327,7 +327,7 @@ def get(self, certificate_id): class RoleUsers(AuthenticatedResource): def __init__(self): self.reqparse = reqparse.RequestParser() - super(RoleUsers, self).__init__() + super().__init__() @validate_schema(None, users_output_schema) def get(self, role_id): @@ -380,7 +380,7 @@ def get(self, role_id): class Me(AuthenticatedResource): def __init__(self): - super(Me, self).__init__() + super().__init__() @validate_schema(None, user_output_schema) def get(self): diff --git a/lemur/utils.py b/lemur/utils.py index 8413ff731e..ba702d2eac 100644 --- a/lemur/utils.py +++ b/lemur/utils.py @@ -10,9 +10,9 @@ from contextlib import contextmanager import cryptography.fernet -from sqlalchemy import types from cryptography.fernet import Fernet, MultiFernet from flask import current_app +from sqlalchemy import types @contextmanager @@ -26,7 +26,7 @@ def mktempfile(): try: os.unlink(name) except OSError as e: - current_app.logger.debug("No file {0}".format(name)) + current_app.logger.debug(f"No file {name}") @contextmanager @@ -40,7 +40,7 @@ def mktemppath(): try: os.unlink(path) except OSError as e: - current_app.logger.debug("No file {0}".format(path)) + current_app.logger.debug(f"No file {path}") def get_keys(): From cdd24ccf9d573e37908b3dacab18af8d9ab064d4 Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Thu, 19 Oct 2023 16:25:01 -0500 Subject: [PATCH 2/2] rebase --- .github/workflows/ci.yml | 88 +- .github/workflows/codeql.yml | 4 +- .readthedocs.yml | 2 +- Makefile | 5 +- README.rst | 4 +- docs/administration.rst | 2 +- docs/developer/index.rst | 2 +- docs/security.rst | 4 +- lemur/__init__.py | 39 +- lemur/api_keys/cli.py | 5 +- lemur/api_keys/views.py | 11 +- lemur/auth/permissions.py | 2 +- lemur/auth/service.py | 1 + lemur/auth/views.py | 2 + lemur/authorities/models.py | 9 +- lemur/authorities/schemas.py | 13 +- lemur/authorities/service.py | 14 +- lemur/authorities/views.py | 14 +- lemur/authorizations/models.py | 5 +- lemur/certificates/cli.py | 16 +- lemur/certificates/models.py | 34 +- lemur/certificates/schemas.py | 5 +- lemur/certificates/service.py | 7 +- lemur/certificates/verify.py | 14 +- lemur/certificates/views.py | 22 +- lemur/common/celery.py | 6 +- lemur/common/defaults.py | 5 +- lemur/common/fields.py | 3 +- lemur/common/utils.py | 7 +- lemur/common/validators.py | 3 +- lemur/database.py | 8 +- lemur/destinations/models.py | 5 +- lemur/destinations/views.py | 9 +- lemur/dns_providers/cli.py | 7 +- lemur/dns_providers/models.py | 7 +- lemur/dns_providers/util.py | 6 +- lemur/domains/models.py | 3 +- lemur/domains/schemas.py | 2 + lemur/domains/views.py | 7 +- lemur/endpoints/models.py | 11 +- lemur/endpoints/service.py | 8 +- lemur/endpoints/views.py | 7 +- lemur/factory.py | 18 +- lemur/logs/service.py | 3 +- lemur/logs/views.py | 8 +- lemur/manage.py | 68 +- lemur/migrations/env.py | 6 +- lemur/migrations/versions/29d8c8455c86_.py | 3 +- lemur/migrations/versions/3307381f3b88_.py | 4 +- lemur/migrations/versions/c301c59688d2_.py | 9 +- lemur/notifications/messaging.py | 7 +- lemur/notifications/models.py | 9 +- lemur/notifications/schemas.py | 1 + lemur/notifications/service.py | 5 +- lemur/notifications/views.py | 11 +- lemur/pending_certificates/cli.py | 3 +- lemur/pending_certificates/models.py | 23 +- lemur/pending_certificates/schemas.py | 5 +- lemur/pending_certificates/service.py | 9 +- lemur/pending_certificates/views.py | 11 +- lemur/plugins/base/manager.py | 4 +- lemur/plugins/lemur_acme/acme_handlers.py | 8 +- lemur/plugins/lemur_acme/challenge_types.py | 10 +- lemur/plugins/lemur_acme/dyn.py | 3 +- lemur/plugins/lemur_acme/plugin.py | 4 +- .../plugins/lemur_acme/tests/test_acme_dns.py | 8 +- .../lemur_acme/tests/test_acme_handler.py | 6 +- lemur/plugins/lemur_acme/ultradns.py | 8 +- lemur/plugins/lemur_adcs/plugin.py | 6 +- lemur/plugins/lemur_atlas/plugin.py | 5 +- lemur/plugins/lemur_aws/cloudfront.py | 3 +- lemur/plugins/lemur_aws/plugin.py | 6 +- lemur/plugins/lemur_aws/sns.py | 1 + lemur/plugins/lemur_aws/tests/test_sns.py | 2 +- lemur/plugins/lemur_azure_dest/plugin.py | 15 +- lemur/plugins/lemur_cfssl/plugin.py | 13 +- lemur/plugins/lemur_cryptography/plugin.py | 9 +- lemur/plugins/lemur_csr/plugin.py | 5 +- lemur/plugins/lemur_digicert/plugin.py | 5 +- lemur/plugins/lemur_email/plugin.py | 12 +- lemur/plugins/lemur_entrust/plugin.py | 5 +- .../lemur_entrust/tests/test_entrust.py | 2 +- lemur/plugins/lemur_kubernetes/plugin.py | 5 +- lemur/plugins/lemur_openssl/plugin.py | 9 +- lemur/plugins/lemur_sftp/plugin.py | 5 +- lemur/plugins/lemur_slack/tests/test_slack.py | 3 +- .../lemur_statsd/lemur_statsd/plugin.py | 3 +- lemur/plugins/lemur_vault_dest/plugin.py | 3 +- lemur/plugins/lemur_verisign/plugin.py | 6 +- lemur/plugins/views.py | 4 +- lemur/roles/models.py | 7 +- lemur/roles/views.py | 9 +- lemur/schemas.py | 7 +- lemur/sources/cli.py | 15 +- lemur/sources/models.py | 7 +- lemur/sources/service.py | 26 +- lemur/sources/views.py | 11 +- lemur/tests/conftest.py | 4 +- lemur/tests/factories.py | 2 +- lemur/tests/plugins/issuer_plugin.py | 7 +- lemur/tests/test_certificates.py | 5 +- lemur/tests/test_dns_providers.py | 1 + lemur/tests/test_ldap.py | 3 +- lemur/tests/test_verify.py | 37 +- lemur/users/models.py | 11 +- lemur/utils.py | 2 +- requirements-dev.in | 6 +- requirements-dev.txt | 797 +++++++++++++++++- requirements-docs.in | 12 +- requirements-docs.txt | 343 ++++++-- requirements-tests.in | 29 +- requirements-tests.txt | 483 +++++++++-- requirements.in | 12 +- requirements.txt | 49 +- setup.py | 2 +- tox.ini | 2 +- 116 files changed, 2050 insertions(+), 648 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2ed1c0a76..bcc351c329 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,21 +8,19 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: [ "3.9", "3.10", "3.11" ] postgres-version: [12, 15] os: [ubuntu-20.04, ubuntu-22.04] include: - - python-version: "3.8" - toxenv: py38 - python-version: "3.9" toxenv: py39 - python-version: "3.10" toxenv: py310 + - python-version: "3.11" + toxenv: py311 exclude: - os: ubuntu-20.04 # python 3.10 isn't working on 20.04 due to missing "libldap-2.5.so.0" python-version: "3.10" - - os: ubuntu-22.04 # python 3.8 isn't working on 22.04 due to missing "libldap_r-2.4.so.2" - python-version: "3.8" - os: ubuntu-22.04 # python 3.9 isn't working on 22.04 due to missing "libldap_r-2.4.so.2" python-version: "3.9" fail-fast: false # run all matrix jobs even if one fails, so we know if the problem is version-specific @@ -56,49 +54,43 @@ jobs: FORCE_COLOR: 1 steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - name: Set up Node.js 16 - uses: actions/setup-node@v3 - with: - node-version: 16 - - name: Install dependencies - run: | - python -m venv ~/env - source ~/env/bin/activate - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - pip install mypy - pip install bandit - sudo apt-get update - sudo apt-get install libsasl2-dev libldap2-dev xvfb - - name: Test - run: | - python -m venv ~/env - source ~/env/bin/activate - make test - - name: mypy - run: | - python -m venv ~/env - source ~/env/bin/activate - mypy --install-types --non-interactive . - - name: Test JS - run: | - python -m venv ~/env - source ~/env/bin/activate - bandit -r . -ll -ii -x lemur/tests/,docs - xvfb-run make test-js - - name: Coveralls GitHub Action - uses: coverallsapp/github-action@v2 - with: - parallel: true - flag-name: Python ${{ matrix.python-version }} Postgres ${{ matrix.postgres-version }} OS ${{ matrix.os }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Set up Node.js 16 + uses: actions/setup-node@v3 + with: + node-version: 16 + - name: Install dependencies + run: | + python -m venv ~/env + source ~/env/bin/activate + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + pip install bandit + sudo apt-get update + sudo apt-get install libsasl2-dev libldap2-dev xvfb + - name: Test + run: | + python -m venv ~/env + source ~/env/bin/activate + make test + - name: Test JS + run: | + python -m venv ~/env + source ~/env/bin/activate + bandit -r . -ll -ii -x lemur/tests/,docs + xvfb-run make test-js + - name: Coveralls GitHub Action + uses: coverallsapp/github-action@v2 + with: + parallel: true + flag-name: Python ${{ matrix.python-version }} Postgres ${{ matrix.postgres-version }} OS ${{ matrix.os }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} coveralls: name: Indicate completion to coveralls.io needs: build diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f405357c9e..1658d0310b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,10 +13,10 @@ name: "CodeQL" on: push: - branches: [ "master" ] + branches: [ "main" ] pull_request: # The branches below must be a subset of the branches above - branches: [ "master" ] + branches: [ "main" ] schedule: - cron: '15 16 * * 2' diff --git a/.readthedocs.yml b/.readthedocs.yml index 505205ac80..01867230de 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -17,7 +17,7 @@ formats: all build: os: ubuntu-22.04 tools: - python: "3.10" + python: "3.11" python: install: - requirements: requirements-docs.txt diff --git a/Makefile b/Makefile index a6fb093178..8d7ff75f2b 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ ifeq ($(USER), root) else npm install endif - pip install "setuptools>=0.9.8" + pip install setuptools # order matters here, base package must install first pip install -e . pip install -e "file://`pwd`#egg=lemur[dev]" @@ -28,7 +28,7 @@ ifeq ($(USER), root) else npm install endif - pip install "setuptools>=0.9.8" + pip install setuptools # order matters here, base package must install first pip install -e . node_modules/.bin/gulp build @@ -94,6 +94,7 @@ lint: lint-python lint-js lint-python: @echo "--> Linting Python files" PYFLAKES_NODOCTEST=1 flake8 lemur + mypy . @echo "" lint-js: diff --git a/README.rst b/README.rst index 3cdc687dbe..51fc12e6b0 100644 --- a/README.rst +++ b/README.rst @@ -11,8 +11,8 @@ Lemur .. image:: https://img.shields.io/badge/NetflixOSS-active-brightgreen.svg -.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=master - :target: https://coveralls.io/github/Netflix/lemur?branch=master +.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=main + :target: https://coveralls.io/github/Netflix/lemur?branch=main diff --git a/docs/administration.rst b/docs/administration.rst index 6324479d6e..0cf4e40c65 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -2044,7 +2044,7 @@ To get the latest code from github run .. note:: It's important to grab the latest release by specifying the release tag. This tags denote stable versions of Lemur. - If you want to try the bleeding edge version of Lemur you can by using the master branch. + If you want to try the bleeding edge version of Lemur you can by using the main branch. After you have the latest version of the Lemur code base you must run any needed database migrations. To run migrations diff --git a/docs/developer/index.rst b/docs/developer/index.rst index 1a08ffb57a..4e75bfde56 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -227,7 +227,7 @@ Uncertain about how to write tests? Take a look at some existing tests that are You can see a list of open pull requests (pending changes) by visiting https://github.com/netflix/lemur/pulls -Pull requests should be against **master** and pass all TravisCI checks +Pull requests should be against **main** and pass all TravisCI checks Writing a Plugin diff --git a/docs/security.rst b/docs/security.rst index 6c31c88680..38bed19083 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -22,7 +22,7 @@ further follow-up. Supported Versions ------------------ -At any given time, we will provide security support for the `master`_ branch +At any given time, we will provide security support for the `main`_ branch as well as the most recent release. Disclosure Process @@ -76,6 +76,6 @@ On the day of disclosure, we will take the following steps: 2. Issue an updated release 3. Make the security advisory public -.. _`master`: https://github.com/Netflix/lemur +.. _`main`: https://github.com/Netflix/lemur .. _GitHub Security Advisory: https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/about-repository-security-advisories .. _private vulnerability reporting process: https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability \ No newline at end of file diff --git a/lemur/__init__.py b/lemur/__init__.py index 3ec38a4677..0059abb228 100644 --- a/lemur/__init__.py +++ b/lemur/__init__.py @@ -12,9 +12,28 @@ import socket import time import urllib.parse - from flask import g, request + from lemur import factory +from lemur.extensions import metrics + +from lemur.users.views import mod as users_bp +from lemur.roles.views import mod as roles_bp +from lemur.auth.views import mod as auth_bp +from lemur.domains.views import mod as domains_bp +from lemur.destinations.views import mod as destinations_bp +from lemur.authorities.views import mod as authorities_bp +from lemur.certificates.views import mod as certificates_bp +from lemur.defaults.views import mod as defaults_bp +from lemur.plugins.views import mod as plugins_bp +from lemur.notifications.views import mod as notifications_bp +from lemur.sources.views import mod as sources_bp +from lemur.endpoints.views import mod as endpoints_bp +from lemur.logs.views import mod as logs_bp +from lemur.api_keys.views import mod as api_key_bp +from lemur.pending_certificates.views import mod as pending_certificates_bp +from lemur.dns_providers.views import mod as dns_providers_bp + from lemur.__about__ import ( __author__, __copyright__, @@ -25,23 +44,7 @@ __uri__, __version__, ) -from lemur.api_keys.views import mod as api_key_bp -from lemur.auth.views import mod as auth_bp -from lemur.authorities.views import mod as authorities_bp -from lemur.certificates.views import mod as certificates_bp -from lemur.defaults.views import mod as defaults_bp -from lemur.destinations.views import mod as destinations_bp -from lemur.dns_providers.views import mod as dns_providers_bp -from lemur.domains.views import mod as domains_bp -from lemur.endpoints.views import mod as endpoints_bp -from lemur.extensions import metrics -from lemur.logs.views import mod as logs_bp -from lemur.notifications.views import mod as notifications_bp -from lemur.pending_certificates.views import mod as pending_certificates_bp -from lemur.plugins.views import mod as plugins_bp -from lemur.roles.views import mod as roles_bp -from lemur.sources.views import mod as sources_bp -from lemur.users.views import mod as users_bp + __all__ = [ "__title__", diff --git a/lemur/api_keys/cli.py b/lemur/api_keys/cli.py index 6069d5b3b6..ab4738f4b3 100644 --- a/lemur/api_keys/cli.py +++ b/lemur/api_keys/cli.py @@ -5,12 +5,13 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Eric Coan """ -from datetime import datetime - import click + from lemur.api_keys import service as api_key_service from lemur.auth.service import create_token +from datetime import datetime + @click.group(name="api_keys", help="Handles all api key related tasks.") def cli(): diff --git a/lemur/api_keys/views.py b/lemur/api_keys/views.py index c72f1d0ac7..4b2d3b747e 100644 --- a/lemur/api_keys/views.py +++ b/lemur/api_keys/views.py @@ -11,7 +11,14 @@ from flask import Blueprint, g from flask_restful import reqparse, Api + from lemur.api_keys import service +from lemur.auth.service import AuthenticatedResource, create_token +from lemur.auth.permissions import ApiKeyCreatorPermission + +from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser + from lemur.api_keys.schemas import ( api_key_input_schema, api_key_revoke_schema, @@ -20,10 +27,6 @@ api_key_described_output_schema, user_api_key_input_schema, ) -from lemur.auth.permissions import ApiKeyCreatorPermission -from lemur.auth.service import AuthenticatedResource, create_token -from lemur.common.schema import validate_schema -from lemur.common.utils import paginated_parser mod = Blueprint("api_keys", __name__) api = Api(mod) diff --git a/lemur/auth/permissions.py b/lemur/auth/permissions.py index bb7741f888..8319d58186 100644 --- a/lemur/auth/permissions.py +++ b/lemur/auth/permissions.py @@ -6,8 +6,8 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from collections import namedtuple from functools import partial +from collections import namedtuple from flask import current_app from flask_principal import Permission, RoleNeed diff --git a/lemur/auth/service.py b/lemur/auth/service.py index e082f18f39..a92154971d 100644 --- a/lemur/auth/service.py +++ b/lemur/auth/service.py @@ -21,6 +21,7 @@ from flask_principal import Identity, identity_changed from flask_principal import identity_loaded, RoleNeed, UserNeed from flask_restful import Resource + from lemur.api_keys import service as api_key_service from lemur.auth.permissions import AuthorityCreatorNeed, RoleMemberNeed from lemur.users import service as user_service diff --git a/lemur/auth/views.py b/lemur/auth/views.py index 9a1eefa9e0..08337d809e 100644 --- a/lemur/auth/views.py +++ b/lemur/auth/views.py @@ -21,6 +21,7 @@ from flask_limiter.util import get_remote_address from flask_principal import Identity, identity_changed from flask_restful import reqparse, Resource, Api + from lemur.auth import ldap from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key from lemur.common.utils import get_psuedo_random_string, get_state_token_secret @@ -277,6 +278,7 @@ def create_user_roles(profile: dict) -> list[str]: current_app.config["LEMUR_DEFAULT_ROLE"], description="This is the default Lemur role.", ) + roles.append(default) # Dedupe the roles roles = list(set(roles)) diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index 6502bb22d3..d7d332ef6f 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -9,9 +9,7 @@ import json from flask import current_app -from lemur.database import BaseModel, db -from lemur.models import roles_authorities -from lemur.plugins.base import plugins +from sqlalchemy.orm import relationship from sqlalchemy import ( Column, Integer, @@ -24,7 +22,10 @@ Boolean, ) from sqlalchemy.dialects.postgresql import JSON -from sqlalchemy.orm import relationship + +from lemur.database import BaseModel, db +from lemur.plugins.base import plugins +from lemur.models import roles_authorities class Authority(BaseModel): diff --git a/lemur/authorities/schemas.py b/lemur/authorities/schemas.py index 50f6cad061..3e79db5334 100644 --- a/lemur/authorities/schemas.py +++ b/lemur/authorities/schemas.py @@ -6,11 +6,14 @@ .. moduleauthor:: Kevin Glisson """ from flask import current_app - from marshmallow import fields, validates_schema, pre_load from marshmallow import validate from marshmallow.exceptions import ValidationError +from lemur.common import validators, missing +from lemur.common.fields import ArrowDateTime +from lemur.common.schema import LemurInputSchema, LemurOutputSchema +from lemur.constants import CERTIFICATE_KEY_TYPES from lemur.schemas import ( PluginInputSchema, PluginOutputSchema, @@ -19,11 +22,6 @@ AssociatedRoleSchema, ) from lemur.users.schemas import UserNestedOutputSchema -from lemur.common.schema import LemurInputSchema, LemurOutputSchema -from lemur.common import validators, missing - -from lemur.common.fields import ArrowDateTime -from lemur.constants import CERTIFICATE_KEY_TYPES class AuthorityInputSchema(LemurInputSchema): @@ -60,7 +58,8 @@ class AuthorityInputSchema(LemurInputSchema): parent = fields.Nested(AssociatedAuthoritySchema) signing_algorithm = fields.String( validate=validate.OneOf(["sha256WithRSA", "sha1WithRSA", - "sha256WithECDSA", "SHA384withECDSA", "SHA512withECDSA", "sha384WithECDSA", "sha512WithECDSA"]), + "sha256WithECDSA", "SHA384withECDSA", "SHA512withECDSA", "sha384WithECDSA", + "sha512WithECDSA"]), missing="sha256WithRSA", ) key_type = fields.String( diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index d87878f9a4..fd6017d752 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -12,14 +12,16 @@ import json from flask import current_app + from lemur import database -from lemur.authorities.models import Authority -from lemur.certificates.models import Certificate -from lemur.certificates.service import upload from lemur.common.utils import truthiness, data_encrypt from lemur.extensions import metrics -from lemur.logs import service as log_service +from lemur.authorities.models import Authority +from lemur.certificates.models import Certificate from lemur.roles import service as role_service +from lemur.logs import service as log_service + +from lemur.certificates.service import upload def update(authority_id, description, owner, active, roles): @@ -64,6 +66,10 @@ def mint(**kwargs): issuer = kwargs["plugin"]["plugin_object"] values = issuer.create_authority(kwargs) + # Check body for root cert + if values[0] is None: + raise ValueError(f"Plugin '{issuer.get_title()}' provided no root certification. Check plugin configuration.") + # support older plugins if len(values) == 3: body, chain, roles = values diff --git a/lemur/authorities/views.py b/lemur/authorities/views.py index bf238410c4..651a33cdb8 100644 --- a/lemur/authorities/views.py +++ b/lemur/authorities/views.py @@ -7,8 +7,15 @@ """ from flask import Blueprint, g from flask_restful import reqparse, Api -from lemur.auth.permissions import AuthorityCreatorPermission, AuthorityPermission, StrictRolePermission + +from lemur.common import validators +from lemur.common.utils import paginated_parser +from lemur.common.schema import validate_schema from lemur.auth.service import AuthenticatedResource +from lemur.auth.permissions import AuthorityCreatorPermission, AuthorityPermission, StrictRolePermission + +from lemur.certificates import service as certificate_service + from lemur.authorities import service from lemur.authorities.schemas import ( authority_input_schema, @@ -16,10 +23,7 @@ authorities_output_schema, authority_update_schema, ) -from lemur.certificates import service as certificate_service -from lemur.common import validators -from lemur.common.schema import validate_schema -from lemur.common.utils import paginated_parser + mod = Blueprint("authorities", __name__) api = Api(mod) diff --git a/lemur/authorizations/models.py b/lemur/authorizations/models.py index 773d31dfb6..ecbc529141 100644 --- a/lemur/authorizations/models.py +++ b/lemur/authorizations/models.py @@ -5,10 +5,11 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Netflix Secops """ -from lemur.database import BaseModel -from lemur.plugins.base import plugins from sqlalchemy import Column, Integer, String from sqlalchemy_utils import JSONType +from lemur.database import BaseModel + +from lemur.plugins.base import plugins class Authorization(BaseModel): diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 491ed1cd02..32ab738c40 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -5,13 +5,17 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import sys -from time import sleep - import arrow +import sys import click + from flask import current_app from flask_principal import Identity, identity_changed +from sqlalchemy import or_ +from tabulate import tabulate +from time import sleep +from sentry_sdk import capture_exception + from lemur import database from lemur.authorities.models import Authority from lemur.authorities.service import get as authorities_get_by_id @@ -42,10 +46,7 @@ from lemur.notifications.messaging import send_rotation_notification, send_reissue_no_endpoints_notification, \ send_reissue_failed_notification from lemur.plugins.base import plugins -from sentry_sdk import capture_exception -from sqlalchemy import or_ from sqlalchemy.orm.exc import MultipleResultsFound -from tabulate import tabulate @click.group(name="certificates", help="Handles all certificate related tasks.") @@ -278,8 +279,7 @@ def rotate(endpoint_name, source, new_certificate_name, old_certificate_name, me try: endpoint = validate_endpoint(endpoint_name) except MultipleResultsFound as e: - click.echo( - f"[!] Multiple endpoints found with name {endpoint_name}, try narrowing the search down to an endpoint from a specific source by re-running this command with the --source flag.") + click.echo(f"[!] Multiple endpoints found with name {endpoint_name}, try narrowing the search down to an endpoint from a specific source by re-running this command with the --source flag.") log_data["message"] = "Multiple endpoints found with same name, unable to perform rotation" log_data["duplicated_endpoint_name"] = endpoint_name current_app.logger.info(log_data) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index e7be34e5c1..c5c7989f3c 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -11,22 +11,6 @@ from cryptography import x509 from flask import current_app from idna.core import InvalidCodepoint -from lemur.common import defaults, utils, validators -from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS -from lemur.database import BaseModel -from lemur.domains.models import Domain -from lemur.extensions import metrics -from lemur.models import ( - certificate_source_associations, - certificate_destination_associations, - certificate_notification_associations, - certificate_replacement_associations, - roles_certificates, - pending_cert_replacement_associations, -) -from lemur.plugins.base import plugins -from lemur.policies.models import RotationPolicy -from lemur.utils import Vault from sentry_sdk import capture_exception from sqlalchemy import ( event, @@ -49,6 +33,24 @@ from werkzeug.utils import cached_property +from lemur.common import defaults, utils, validators +from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS +from lemur.database import BaseModel +from lemur.domains.models import Domain +from lemur.extensions import metrics +from lemur.models import ( + certificate_source_associations, + certificate_destination_associations, + certificate_notification_associations, + certificate_replacement_associations, + roles_certificates, + pending_cert_replacement_associations, +) +from lemur.plugins.base import plugins +from lemur.policies.models import RotationPolicy +from lemur.utils import Vault + + def get_sequence(name): if "-" not in name: return name, None diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index e643ca3817..f6a7a0b958 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -8,6 +8,9 @@ from flask import current_app from flask_restful import inputs from flask_restful.reqparse import RequestParser +from marshmallow import fields, validate, validates_schema, post_load, pre_load, post_dump +from marshmallow.exceptions import ValidationError + from lemur.authorities.schemas import AuthorityNestedOutputSchema from lemur.certificates import utils as cert_utils from lemur.common import missing, utils, validators @@ -35,8 +38,6 @@ AssociatedRotationPolicySchema, ) from lemur.users.schemas import UserNestedOutputSchema -from marshmallow import fields, validate, validates_schema, post_load, pre_load, post_dump -from marshmallow.exceptions import ValidationError class CertificateSchema(LemurInputSchema): diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index ce58e85750..acb3dfb76e 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -15,6 +15,10 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from flask import current_app +from sentry_sdk import capture_exception +from sqlalchemy import and_, func, or_, not_, cast, Integer +from sqlalchemy.sql.expression import false, true + from lemur import database from lemur.authorities.models import Authority from lemur.certificates.models import Certificate, CertificateAssociation @@ -33,9 +37,6 @@ from lemur.plugins.utils import get_plugin_option from lemur.roles import service as role_service from lemur.roles.models import Role -from sentry_sdk import capture_exception -from sqlalchemy import and_, func, or_, not_, cast, Integer -from sqlalchemy.sql.expression import false, true csr_created = signals.signal("csr_created", "CSR generated") csr_imported = signals.signal("csr_imported", "CSR imported from external source") diff --git a/lemur/certificates/verify.py b/lemur/certificates/verify.py index 186ee35f96..be91249f51 100644 --- a/lemur/certificates/verify.py +++ b/lemur/certificates/verify.py @@ -5,18 +5,18 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import subprocess -from subprocess import TimeoutExpired - import requests +import subprocess +from flask import current_app +from requests.exceptions import ConnectionError, InvalidSchema, Timeout from cryptography import x509 from cryptography.hazmat.backends import default_backend -from flask import current_app +from sentry_sdk import capture_exception +from subprocess import TimeoutExpired + +from lemur.utils import mktempfile from lemur.common.utils import parse_certificate from lemur.extensions import metrics -from lemur.utils import mktempfile -from requests.exceptions import ConnectionError, InvalidSchema, Timeout -from sentry_sdk import capture_exception crl_cache = {} diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index be284e53a5..d3e8f39e03 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -9,8 +9,18 @@ from flask import Blueprint, make_response, jsonify, g, current_app from flask_restful import reqparse, Api, inputs -from lemur.auth.permissions import AuthorityPermission, CertificatePermission, StrictRolePermission + +from lemur.certificates.service import validate_no_duplicate_destinations +from lemur.common import validators +from lemur.plugins.bases.authorization import UnauthorizedError +from sentry_sdk import capture_exception + +from lemur.common.schema import validate_schema +from lemur.common.utils import paginated_parser + from lemur.auth.service import AuthenticatedResource +from lemur.auth.permissions import AuthorityPermission, CertificatePermission, StrictRolePermission + from lemur.certificates import service from lemur.certificates.models import Certificate from lemur.certificates.schemas import ( @@ -23,14 +33,10 @@ certificates_list_output_schema_factory, certificate_revoke_schema, ) -from lemur.certificates.service import validate_no_duplicate_destinations -from lemur.common import validators -from lemur.common.schema import validate_schema -from lemur.common.utils import paginated_parser -from lemur.logs import service as log_service -from lemur.plugins.bases.authorization import UnauthorizedError + from lemur.roles import service as role_service -from sentry_sdk import capture_exception +from lemur.logs import service as log_service + mod = Blueprint("certificates", __name__) api = Api(mod) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 603f377d2c..d66841a138 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -10,13 +10,14 @@ import copy import sys import time -from datetime import datetime, timezone, timedelta - from celery import Celery from celery.app.task import Context from celery.exceptions import SoftTimeLimitExceeded from celery.signals import task_failure, task_received, task_revoked, task_success +from datetime import datetime, timezone, timedelta from flask import current_app +from sentry_sdk import capture_exception + from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate from lemur.certificates import service as certificate_service @@ -34,7 +35,6 @@ from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources -from sentry_sdk import capture_exception if current_app: flask_app = current_app diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index f291fba380..674546591f 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -1,12 +1,13 @@ import re - import unicodedata + from cryptography import x509 from cryptography.hazmat.primitives.serialization import Encoding from flask import current_app +from sentry_sdk import capture_exception + from lemur.common.utils import is_selfsigned from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE -from sentry_sdk import capture_exception def text_to_slug(value, joiner="-"): diff --git a/lemur/common/fields.py b/lemur/common/fields.py index e0922c1a47..5f99ef341d 100644 --- a/lemur/common/fields.py +++ b/lemur/common/fields.py @@ -12,11 +12,12 @@ import arrow from cryptography import x509 from flask import current_app -from lemur.common import validators from marshmallow import utils from marshmallow.exceptions import ValidationError from marshmallow.fields import Field +from lemur.common import validators + class Hex(Field): """ diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 2ac58a2879..5be7ccea18 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -15,10 +15,8 @@ import string import OpenSSL -import josepy as jose import pem import sqlalchemy -from certbot.crypto_util import CERT_PEM_REGEX from cryptography import x509 from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend @@ -26,10 +24,13 @@ from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding from cryptography.hazmat.primitives.serialization import load_pem_private_key, Encoding, pkcs7 from flask_restful.reqparse import RequestParser +from sqlalchemy import and_, func +import josepy as jose + +from certbot.crypto_util import CERT_PEM_REGEX from lemur.constants import CERTIFICATE_KEY_TYPES from lemur.exceptions import InvalidConfiguration from lemur.utils import Vault -from sqlalchemy import and_, func from sqlalchemy.dialects.postgresql import TEXT paginated_parser = RequestParser() diff --git a/lemur/common/validators.py b/lemur/common/validators.py index 798961d91a..7b3913d24e 100644 --- a/lemur/common/validators.py +++ b/lemur/common/validators.py @@ -5,10 +5,11 @@ from cryptography.hazmat.backends import default_backend from cryptography.x509 import NameOID from flask import current_app +from marshmallow.exceptions import ValidationError + from lemur.auth.permissions import SensitiveDomainPermission from lemur.common.utils import check_cert_signature, is_weekend from lemur.plugins.base import plugins -from marshmallow.exceptions import ValidationError def common_name(value): diff --git a/lemur/database.py b/lemur/database.py index c1f5ea5a56..e99272f62a 100644 --- a/lemur/database.py +++ b/lemur/database.py @@ -9,14 +9,16 @@ .. moduleauthor:: Kevin Glisson """ -from flask_sqlalchemy.model import DefaultMeta from inflection import underscore -from lemur.exceptions import AttrNotFound, DuplicateError -from lemur.extensions import db +from flask_sqlalchemy.model import DefaultMeta from sqlalchemy import exc, func, distinct from sqlalchemy.orm import make_transient, lazyload from sqlalchemy.sql import and_, or_ +from lemur.exceptions import AttrNotFound, DuplicateError +from lemur.extensions import db + + BaseModel: DefaultMeta = db.Model diff --git a/lemur/destinations/models.py b/lemur/destinations/models.py index f631e64119..caa3288ccd 100644 --- a/lemur/destinations/models.py +++ b/lemur/destinations/models.py @@ -5,11 +5,12 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from lemur.database import BaseModel -from lemur.plugins.base import plugins from sqlalchemy import Column, Integer, String, Text from sqlalchemy.orm import validates from sqlalchemy_utils import JSONType +from lemur.database import BaseModel + +from lemur.plugins.base import plugins class Destination(BaseModel): diff --git a/lemur/destinations/views.py b/lemur/destinations/views.py index b333f9d085..02dc8922ce 100644 --- a/lemur/destinations/views.py +++ b/lemur/destinations/views.py @@ -8,17 +8,20 @@ """ from flask import Blueprint from flask_restful import Api, reqparse -from lemur.auth.permissions import admin_permission +from lemur.destinations import service + from lemur.auth.service import AuthenticatedResource -from lemur.common.schema import validate_schema +from lemur.auth.permissions import admin_permission from lemur.common.utils import paginated_parser -from lemur.destinations import service + +from lemur.common.schema import validate_schema from lemur.destinations.schemas import ( destinations_output_schema, destination_input_schema, destination_output_schema, ) + mod = Blueprint("destinations", __name__) api = Api(mod) diff --git a/lemur/dns_providers/cli.py b/lemur/dns_providers/cli.py index cfbe508a92..6b4165e39d 100644 --- a/lemur/dns_providers/cli.py +++ b/lemur/dns_providers/cli.py @@ -1,11 +1,12 @@ import sys - import click + +from sentry_sdk import capture_exception + from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.plugins.lemur_acme.acme_handlers import AcmeDnsHandler from lemur.dns_providers.service import get_all_dns_providers, set_domains from lemur.extensions import metrics -from lemur.plugins.lemur_acme.acme_handlers import AcmeDnsHandler -from sentry_sdk import capture_exception @click.group(name="dns_providers", help="Iterates through all DNS providers and sets DNS zones in the database.") diff --git a/lemur/dns_providers/models.py b/lemur/dns_providers/models.py index bfc914155d..dfa4e71328 100644 --- a/lemur/dns_providers/models.py +++ b/lemur/dns_providers/models.py @@ -1,11 +1,12 @@ -from lemur.database import BaseModel -from lemur.plugins.base import plugins -from lemur.utils import Vault from sqlalchemy import Column, Integer, String, text from sqlalchemy.dialects.postgresql import JSON from sqlalchemy.orm import relationship from sqlalchemy_utils import ArrowType +from lemur.database import BaseModel +from lemur.plugins.base import plugins +from lemur.utils import Vault + class DnsProvider(BaseModel): __tablename__ = "dns_providers" diff --git a/lemur/dns_providers/util.py b/lemur/dns_providers/util.py index a3cff8d2c9..56bfe097ed 100644 --- a/lemur/dns_providers/util.py +++ b/lemur/dns_providers/util.py @@ -1,14 +1,14 @@ -import re import sys - import dns import dns.exception import dns.name import dns.query import dns.resolver -from lemur.extensions import metrics +import re from sentry_sdk import capture_exception +from lemur.extensions import metrics + class DNSError(Exception): """Base class for DNS Exceptions.""" diff --git a/lemur/domains/models.py b/lemur/domains/models.py index 8a0ace7f20..ffed79bec3 100644 --- a/lemur/domains/models.py +++ b/lemur/domains/models.py @@ -7,9 +7,10 @@ .. moduleauthor:: Kevin Glisson """ -from lemur.database import BaseModel from sqlalchemy import Column, Integer, String, Boolean +from lemur.database import BaseModel + class Domain(BaseModel): __tablename__ = "domains" diff --git a/lemur/domains/schemas.py b/lemur/domains/schemas.py index 6cf7fd31ce..9272f9a04d 100644 --- a/lemur/domains/schemas.py +++ b/lemur/domains/schemas.py @@ -6,9 +6,11 @@ .. moduleauthor:: Kevin Glisson """ from marshmallow import fields + from lemur.common.schema import LemurInputSchema, LemurOutputSchema from lemur.schemas import AssociatedCertificateSchema + # from lemur.certificates.schemas import CertificateNestedOutputSchema diff --git a/lemur/domains/views.py b/lemur/domains/views.py index e35c996f44..7a6d72d2f2 100644 --- a/lemur/domains/views.py +++ b/lemur/domains/views.py @@ -9,11 +9,14 @@ """ from flask import Blueprint from flask_restful import reqparse, Api -from lemur.auth.permissions import SensitiveDomainPermission, StrictRolePermission + +from lemur.domains import service from lemur.auth.service import AuthenticatedResource +from lemur.auth.permissions import SensitiveDomainPermission, StrictRolePermission + from lemur.common.schema import validate_schema from lemur.common.utils import paginated_parser -from lemur.domains import service + from lemur.domains.schemas import ( domain_input_schema, domain_output_schema, diff --git a/lemur/endpoints/models.py b/lemur/endpoints/models.py index adb9721f88..5528b0e460 100644 --- a/lemur/endpoints/models.py +++ b/lemur/endpoints/models.py @@ -7,15 +7,18 @@ .. moduleauthor:: Kevin Glisson """ import arrow -from lemur.database import BaseModel -from lemur.models import policies_ciphers -from sqlalchemy import Column, Integer, String, Boolean, ForeignKey +from sqlalchemy.orm import relationship from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy import Column, Integer, String, Boolean, ForeignKey from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import case + from sqlalchemy_utils import ArrowType +from lemur.database import BaseModel + +from lemur.models import policies_ciphers + BAD_CIPHERS = ["Protocol-SSLv3", "Protocol-SSLv2", "Protocol-TLSv1"] diff --git a/lemur/endpoints/service.py b/lemur/endpoints/service.py index d2e9c4f453..3d31950188 100644 --- a/lemur/endpoints/service.py +++ b/lemur/endpoints/service.py @@ -9,13 +9,15 @@ """ import arrow + +from sqlalchemy import func +from sqlalchemy.orm import joinedload + from lemur import database from lemur.common.utils import truthiness from lemur.endpoints.models import Endpoint, EndpointDnsAlias, Policy, Cipher -from lemur.extensions import metrics from lemur.sources.models import Source -from sqlalchemy import func -from sqlalchemy.orm import joinedload +from lemur.extensions import metrics def get_all(): diff --git a/lemur/endpoints/views.py b/lemur/endpoints/views.py index 8071157372..aa9b22cfc8 100644 --- a/lemur/endpoints/views.py +++ b/lemur/endpoints/views.py @@ -7,12 +7,15 @@ """ from flask import Blueprint, g from flask_restful import reqparse, Api -from lemur.auth.service import AuthenticatedResource -from lemur.common.schema import validate_schema + from lemur.common.utils import paginated_parser +from lemur.common.schema import validate_schema +from lemur.auth.service import AuthenticatedResource + from lemur.endpoints import service from lemur.endpoints.schemas import endpoint_output_schema, endpoints_output_schema + mod = Blueprint("endpoints", __name__) api = Api(mod) diff --git a/lemur/factory.py b/lemur/factory.py index 7e2d8a66c2..15dd3f278c 100644 --- a/lemur/factory.py +++ b/lemur/factory.py @@ -14,23 +14,24 @@ import os import socket import stat +from importlib.metadata import entry_points from logging import Formatter, StreamHandler from logging.handlers import RotatingFileHandler import logmatic -import pkg_resources import sentry_sdk from click import get_current_context from flask import Flask, current_app from flask_replicated import FlaskReplicated -from lemur.certificates.hooks import activate_debug_dump -from lemur.common.health import mod as health -from lemur.extensions import db, migrate, principal, smtp_mail, metrics, cors from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.flask import FlaskIntegration from sentry_sdk.integrations.redis import RedisIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration +from lemur.certificates.hooks import activate_debug_dump +from lemur.common.health import mod as health +from lemur.extensions import db, migrate, principal, smtp_mail, metrics, cors + DEFAULT_BLUEPRINTS = (health,) API_VERSION = 1 @@ -255,19 +256,14 @@ def install_plugins(app): from lemur.plugins import plugins from lemur.plugins.base import register - # entry_points={ - # 'lemur.plugins': [ - # 'verisign = lemur_verisign.plugin:VerisignPlugin' - # ], - # }, - for ep in pkg_resources.iter_entry_points("lemur.plugins"): + for ep in entry_points().get("lemur.plugins", []): try: plugin = ep.load() except Exception: import traceback app.logger.error( - f"Failed to load plugin {ep.name!r}:\n{traceback.format_exc()}\n" + "Failed to load plugin {!r}:\n{}\n".format(ep.name, traceback.format_exc()) ) else: register(plugin) diff --git a/lemur/logs/service.py b/lemur/logs/service.py index 5b0492f20b..6d53ecc335 100644 --- a/lemur/logs/service.py +++ b/lemur/logs/service.py @@ -8,10 +8,11 @@ .. moduleauthor:: Kevin Glisson """ from flask import current_app, g + from lemur import database -from lemur.certificates.models import Certificate from lemur.logs.models import Log from lemur.users.models import User +from lemur.certificates.models import Certificate def create(user, type, certificate=None): diff --git a/lemur/logs/views.py b/lemur/logs/views.py index 6d6fea23b5..5175bff4e7 100644 --- a/lemur/logs/views.py +++ b/lemur/logs/views.py @@ -7,12 +7,16 @@ """ from flask import Blueprint from flask_restful import reqparse, Api -from lemur.auth.service import AuthenticatedResource + from lemur.common.schema import validate_schema from lemur.common.utils import paginated_parser -from lemur.logs import service + +from lemur.auth.service import AuthenticatedResource from lemur.logs.schemas import logs_output_schema +from lemur.logs import service + + mod = Blueprint("logs", __name__) api = Api(mod) diff --git a/lemur/manage.py b/lemur/manage.py index 152b812dbd..db4b910fca 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -1,46 +1,54 @@ #!/usr/bin/env python -import base64 -import json +import click import os import sys - -import click +import base64 import requests +import json + from cryptography.fernet import Fernet + from flask import current_app from flask.cli import FlaskGroup, pass_script_info -from flask_migrate import stamp from flask_migrate.cli import db -from lemur import create_app -from lemur import database +from flask_migrate import stamp + +from lemur.dns_providers.cli import cli as dns_provider_cli from lemur.acme_providers.cli import cli as acme_cli -from lemur.authorities.models import Authority # noqa +from lemur.sources.cli import cli as source_cli +from lemur.policies.cli import cli as policy_cli +from lemur.reporting.cli import cli as report_cli from lemur.certificates.cli import cli as certificate_cli -from lemur.certificates.models import Certificate # noqa -from lemur.common.utils import validate_conf -from lemur.destinations.models import Destination # noqa -from lemur.dns_providers.cli import cli as dns_provider_cli -from lemur.dns_providers.models import DnsProvider # noqa -from lemur.domains.models import Domain # noqa -from lemur.endpoints.models import Endpoint # noqa -from lemur.logs.models import Log # noqa -from lemur.notifications import service as notification_service from lemur.notifications.cli import cli as notification_cli -from lemur.notifications.models import Notification # noqa from lemur.pending_certificates.cli import cli as pending_certificate_cli -from lemur.pending_certificates.models import PendingCertificate # noqa -from lemur.policies import service as policy_service -from lemur.policies.cli import cli as policy_cli -from lemur.policies.models import RotationPolicy # noqa -from lemur.reporting.cli import cli as report_cli -from lemur.roles import service as role_service -from lemur.roles.models import Role # noqa -from lemur.sources.cli import cli as source_cli -from lemur.sources.models import Source # noqa + + +from lemur import database from lemur.users import service as user_service +from lemur.roles import service as role_service +from lemur.policies import service as policy_service +from lemur.notifications import service as notification_service + +from lemur.common.utils import validate_conf + +from lemur import create_app + # Needed to be imported so that SQLAlchemy create_all can find our models from lemur.users.models import User # noqa +from lemur.roles.models import Role # noqa +from lemur.authorities.models import Authority # noqa +from lemur.certificates.models import Certificate # noqa +from lemur.destinations.models import Destination # noqa +from lemur.domains.models import Domain # noqa +from lemur.notifications.models import Notification # noqa +from lemur.sources.models import Source # noqa +from lemur.logs.models import Log # noqa +from lemur.endpoints.models import Endpoint # noqa +from lemur.policies.models import RotationPolicy # noqa +from lemur.pending_certificates.models import PendingCertificate # noqa +from lemur.dns_providers.models import DnsProvider # noqa + from sqlalchemy.sql import text @@ -148,12 +156,12 @@ def cli(script_info, config): # IDP_ROLES_PREFIX = "PREFIX-" # prefix for all IDP-defined roles, used to match naming conventions # IDP_ROLES_SUFFIX = "_SUFFIX" # suffix for all IDP-defined roles, used to match naming conventions # IDP_ROLES_DESCRIPTION = "Automatically generated role" # Description to attach to automatically generated roles -# IDP_ROLES_MAPPING = {} # Dictionary that matches the IDP group name to the Lemur role. The Lemur role must exist. -# Example: IDP_ROLES_MAPPING = {"security": "admin", "engineering": "operator", "jane_from_accounting": "read-only"} +# IDP_ROLES_MAPPING = {{}} # Dictionary that matches the IDP group name to the Lemur role. The Lemur role must exist. +# Example: IDP_ROLES_MAPPING = {{"security": "admin", "engineering": "operator", "jane_from_accounting": "read-only"}} IDP_ASSIGN_ROLES_FROM_USER_GROUPS = True # Assigns a Lemur role for each group found attached to the user IDP_CREATE_ROLES_FROM_USER_GROUPS = True # Creates a Lemur role for each group found attached to the user if missing # Protects the built-in groups and prevents dynamically assigning users to them. Prevents IDP admin from becoming -# Lemur admin. Use IDP_ROLES_MAPPING to create a mapping to assign these groups if desired. eg {"admin": "admin"} +# Lemur admin. Use IDP_ROLES_MAPPING to create a mapping to assign these groups if desired. eg {{"admin": "admin"}} IDP_PROTECT_BUILTINS = True IDP_CREATE_PER_USER_ROLE = True # Generates Lemur role for each user (allows cert assignment to a single user) diff --git a/lemur/migrations/env.py b/lemur/migrations/env.py index 8331b1a4ed..123ab1f3c3 100644 --- a/lemur/migrations/env.py +++ b/lemur/migrations/env.py @@ -1,7 +1,9 @@ -from logging.config import fileConfig - from alembic import context from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +import alembic_autogenerate_enums + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/lemur/migrations/versions/29d8c8455c86_.py b/lemur/migrations/versions/29d8c8455c86_.py index 5f9ed43145..cc95c17509 100644 --- a/lemur/migrations/versions/29d8c8455c86_.py +++ b/lemur/migrations/versions/29d8c8455c86_.py @@ -10,8 +10,9 @@ revision = "29d8c8455c86" down_revision = "3307381f3b88" -import sqlalchemy as sa from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql def upgrade(): diff --git a/lemur/migrations/versions/3307381f3b88_.py b/lemur/migrations/versions/3307381f3b88_.py index ff68dfd351..61f65dbc23 100644 --- a/lemur/migrations/versions/3307381f3b88_.py +++ b/lemur/migrations/versions/3307381f3b88_.py @@ -15,10 +15,10 @@ revision = "3307381f3b88" down_revision = "412b22cb656a" -import sqlalchemy as sa from alembic import op -from sqlalchemy.dialects import postgresql +import sqlalchemy as sa from sqlalchemy.sql import text +from sqlalchemy.dialects import postgresql def upgrade(): diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index b33a7ea647..63a890a16f 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -30,14 +30,15 @@ revision = 'c301c59688d2' down_revision = '434c29e40511' -import datetime +from alembic import op +from sqlalchemy.sql import text import time +import datetime +from flask import current_app + from logging import Formatter, FileHandler, getLogger -from alembic import op -from flask import current_app from lemur.common import utils -from sqlalchemy.sql import text log = getLogger(__name__) handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log")) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 6607110db3..8c4604bf00 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -15,6 +15,10 @@ import arrow from flask import current_app +from sentry_sdk import capture_exception +from sqlalchemy import and_ +from sqlalchemy.sql.expression import false, true + from lemur import database from lemur.certificates import service as certificates_service from lemur.certificates.models import Certificate @@ -25,9 +29,6 @@ from lemur.pending_certificates.schemas import pending_certificate_output_schema from lemur.plugins import plugins from lemur.plugins.utils import get_plugin_option -from sentry_sdk import capture_exception -from sqlalchemy import and_ -from sqlalchemy.sql.expression import false, true def get_certificates(exclude=None): diff --git a/lemur/notifications/models.py b/lemur/notifications/models.py index 0ce3a7d43c..2ae55ea8ae 100644 --- a/lemur/notifications/models.py +++ b/lemur/notifications/models.py @@ -5,15 +5,16 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +from sqlalchemy.orm import relationship +from sqlalchemy import Integer, String, Column, Boolean, Text +from sqlalchemy_utils import JSONType + from lemur.database import BaseModel +from lemur.plugins.base import plugins from lemur.models import ( certificate_notification_associations, pending_cert_notification_associations, ) -from lemur.plugins.base import plugins -from sqlalchemy import Integer, String, Column, Boolean, Text -from sqlalchemy.orm import relationship -from sqlalchemy_utils import JSONType class Notification(BaseModel): diff --git a/lemur/notifications/schemas.py b/lemur/notifications/schemas.py index 6ef5c506ba..b34d531609 100644 --- a/lemur/notifications/schemas.py +++ b/lemur/notifications/schemas.py @@ -6,6 +6,7 @@ .. moduleauthor:: Kevin Glisson """ from marshmallow import fields, post_dump + from lemur.common.schema import LemurInputSchema, LemurOutputSchema from lemur.schemas import ( PluginInputSchema, diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index d8be187b10..6c67bab471 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -9,12 +9,13 @@ """ from flask import current_app + from lemur import database +from lemur.constants import EMAIL_RE, EMAIL_RE_HELP from lemur.certificates.models import Certificate from lemur.common.utils import truthiness, check_validation -from lemur.constants import EMAIL_RE, EMAIL_RE_HELP -from lemur.logs import service as log_service from lemur.notifications.models import Notification +from lemur.logs import service as log_service def create_default_expiration_notifications(name, recipients, intervals=None): diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index 71f469ca8f..10a3905fb5 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -8,10 +8,6 @@ """ from flask import Blueprint from flask_restful import Api, reqparse, inputs -from lemur.auth.permissions import StrictRolePermission -from lemur.auth.service import AuthenticatedResource -from lemur.common.schema import validate_schema -from lemur.common.utils import paginated_parser from lemur.notifications import service from lemur.notifications.schemas import ( notification_input_schema, @@ -19,6 +15,13 @@ notifications_output_schema, ) +from lemur.auth.service import AuthenticatedResource +from lemur.common.utils import paginated_parser +from lemur.auth.permissions import StrictRolePermission + +from lemur.common.schema import validate_schema + + mod = Blueprint("notifications", __name__) api = Api(mod) diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index d5b8d0a360..ac70b6b6dc 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -5,11 +5,12 @@ .. moduleauthor:: Curtis Castrapel """ +import click import copy import sys -import click from flask import current_app + from lemur.authorities.service import get as get_authority from lemur.constants import ACME_ADDITIONAL_ATTEMPTS from lemur.notifications.messaging import send_pending_failure_notification diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index a7ef8085f8..4bbbcf906b 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -5,17 +5,6 @@ """ from datetime import datetime as dt -from lemur.certificates.models import get_sequence -from lemur.common import defaults, utils -from lemur.database import BaseModel -from lemur.models import ( - pending_cert_source_associations, - pending_cert_destination_associations, - pending_cert_notification_associations, - pending_cert_replacement_associations, - pending_cert_role_associations, -) -from lemur.utils import Vault from sqlalchemy import ( Integer, ForeignKey, @@ -30,6 +19,18 @@ from sqlalchemy_utils import JSONType from sqlalchemy_utils.types.arrow import ArrowType +from lemur.certificates.models import get_sequence +from lemur.common import defaults, utils +from lemur.database import BaseModel +from lemur.models import ( + pending_cert_source_associations, + pending_cert_destination_associations, + pending_cert_notification_associations, + pending_cert_replacement_associations, + pending_cert_role_associations, +) +from lemur.utils import Vault + def get_or_increase_name(name, serial): certificates = PendingCertificate.query.filter( diff --git a/lemur/pending_certificates/schemas.py b/lemur/pending_certificates/schemas.py index 0d6edb9342..7c1a8fcae7 100644 --- a/lemur/pending_certificates/schemas.py +++ b/lemur/pending_certificates/schemas.py @@ -1,3 +1,6 @@ +from marshmallow import fields, validates_schema, post_load +from marshmallow.exceptions import ValidationError + from lemur.authorities.schemas import AuthorityNestedOutputSchema from lemur.certificates.schemas import CertificateNestedOutputSchema from lemur.common import utils, validators @@ -17,8 +20,6 @@ ExtensionSchema, ) from lemur.users.schemas import UserNestedOutputSchema -from marshmallow import fields, validates_schema, post_load -from marshmallow.exceptions import ValidationError class PendingCertificateSchema(LemurInputSchema): diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index 8085930d33..5040029b0d 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -4,24 +4,25 @@ .. moduleauthor:: James Chuong """ import arrow +from sqlalchemy import or_, cast, Integer from flask import current_app + from lemur import database -from lemur.authorities import service as authorities_service from lemur.authorities.models import Authority +from lemur.authorities import service as authorities_service from lemur.certificates import service as certificate_service from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common import validators from lemur.common.utils import truthiness, parse_cert_chain, parse_certificate +from lemur.common import validators from lemur.destinations.models import Destination from lemur.domains.models import Domain from lemur.extensions import metrics -from lemur.logs import service as log_service from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate from lemur.plugins.base import plugins from lemur.roles.models import Role from lemur.users import service as user_service -from sqlalchemy import or_, cast, Integer +from lemur.logs import service as log_service def get(pending_cert_id): diff --git a/lemur/pending_certificates/views.py b/lemur/pending_certificates/views.py index cfbdaa44fe..884ca743fc 100644 --- a/lemur/pending_certificates/views.py +++ b/lemur/pending_certificates/views.py @@ -6,19 +6,24 @@ """ from flask import Blueprint, g, make_response, jsonify from flask_restful import Api, reqparse, inputs -from lemur.auth.permissions import CertificatePermission, StrictRolePermission + from lemur.auth.service import AuthenticatedResource +from lemur.auth.permissions import CertificatePermission, StrictRolePermission + from lemur.common.schema import validate_schema from lemur.common.utils import paginated_parser -from lemur.logs import service as log_service + from lemur.pending_certificates import service +from lemur.roles import service as role_service +from lemur.logs import service as log_service + + from lemur.pending_certificates.schemas import ( pending_certificate_output_schema, pending_certificate_edit_input_schema, pending_certificate_cancel_schema, pending_certificate_upload_input_schema, ) -from lemur.roles import service as role_service mod = Blueprint("pending_certificates", __name__) api = Api(mod) diff --git a/lemur/plugins/base/manager.py b/lemur/plugins/base/manager.py index 42dde291e2..33cd9f9021 100644 --- a/lemur/plugins/base/manager.py +++ b/lemur/plugins/base/manager.py @@ -63,9 +63,9 @@ def first(self, func_name, *args, **kwargs): return result def register(self, cls): - self.add(f"{cls.__module__}.{cls.__name__}") + self.add("{}.{}".format(cls.__module__, cls.__name__)) return cls def unregister(self, cls): - self.remove(f"{cls.__module__}.{cls.__name__}") + self.remove("{}.{}".format(cls.__module__, cls.__name__)) return cls diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index 495e7a0f29..213bee8e9a 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -25,6 +25,9 @@ from acme.messages import Error as AcmeError, STATUS_VALID from certbot import crypto_util as acme_crypto_util from flask import current_app +from retrying import retry +from sentry_sdk import capture_exception + from lemur.authorities import service as authorities_service from lemur.common.utils import data_encrypt, data_decrypt, is_json from lemur.common.utils import generate_private_key, key_to_alg @@ -32,8 +35,6 @@ from lemur.exceptions import InvalidAuthority, UnknownProvider, InvalidConfiguration from lemur.extensions import metrics from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns, nsone -from retrying import retry -from sentry_sdk import capture_exception class AuthorizationRecord: @@ -146,6 +147,9 @@ def extract_cert_and_chain(self, fullchain_pem, alternative_fullchains_pem, pref @retry(stop_max_attempt_number=5, wait_fixed=5000) def setup_acme_client(self, authority): + return self.setup_acme_client_no_retry(authority) + + def setup_acme_client_no_retry(self, authority): if not authority.options: raise InvalidAuthority("Invalid authority. Options not set") options = {} diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index 6a19902c48..0c7e0c9c85 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -7,24 +7,26 @@ .. moduleauthor:: Mathias Petermann """ -import json from datetime import datetime, timedelta +import json from acme import challenges from acme.errors import WildcardUnsupportedError from acme.messages import errors, STATUS_VALID, ERROR_CODES from botocore.exceptions import ClientError from flask import current_app +from sentry_sdk import capture_exception + from lemur.authorizations import service as authorization_service -from lemur.common.utils import drop_last_cert_from_chain from lemur.constants import ACME_ADDITIONAL_ATTEMPTS -from lemur.destinations import service as destination_service +from lemur.common.utils import drop_last_cert_from_chain from lemur.exceptions import LemurException, InvalidConfiguration from lemur.extensions import metrics from lemur.plugins.base import plugins +from lemur.destinations import service as destination_service from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler + from retrying import retry -from sentry_sdk import capture_exception class AcmeChallengeMissmatchError(LemurException): diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 31a5a96b3c..5cf40a95d4 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -14,9 +14,10 @@ from dyn.tm.session import DynectSession from dyn.tm.zones import Node, Zone, get_all_zones from flask import current_app -from lemur.extensions import metrics from sentry_sdk import capture_exception +from lemur.extensions import metrics + def get_dynect_session(): try: diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 504ea216bc..5b0306873b 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -16,17 +16,19 @@ from acme.messages import Error as AcmeError from botocore.exceptions import ClientError from flask import current_app +from sentry_sdk import capture_exception + from lemur.authorizations import service as authorization_service from lemur.common.utils import check_validation, drop_last_cert_from_chain from lemur.constants import CRLReason, EMAIL_RE from lemur.dns_providers import service as dns_provider_service from lemur.exceptions import InvalidConfiguration from lemur.extensions import metrics + from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler from lemur.plugins.lemur_acme.challenge_types import AcmeHttpChallenge, AcmeDnsChallenge -from sentry_sdk import capture_exception class ACMEIssuerPlugin(IssuerPlugin): diff --git a/lemur/plugins/lemur_acme/tests/test_acme_dns.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py index 66b1133bc5..7f4c9cbd10 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_dns.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -1,16 +1,16 @@ import unittest +from unittest.mock import MagicMock from unittest.mock import patch, Mock import josepy as jose - from acme.messages import STATUS_PENDING, STATUS_VALID from cryptography.x509 import DNSName from flask import Flask, current_app + +from lemur.common.utils import generate_private_key from lemur.plugins.lemur_acme import plugin from lemur.plugins.lemur_acme.acme_handlers import AuthorizationRecord -from lemur.common.utils import generate_private_key from lemur.tests.conf import LEMUR_ENCRYPTION_KEYS -from unittest.mock import MagicMock class TestAcmeDns(unittest.TestCase): @@ -214,7 +214,7 @@ def test_setup_acme_client_fail(self): mock_authority = Mock() mock_authority.options = [] with self.assertRaises(Exception): - self.acme.setup_acme_client(mock_authority) + self.acme.setup_acme_client_no_retry(mock_authority) @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWK.json_loads") @patch("lemur.plugins.lemur_acme.acme_handlers.ClientV2") diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index 53297f9594..e1f82d8d24 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -1,10 +1,10 @@ import unittest from unittest.mock import patch, Mock -from flask import Flask from cryptography.x509 import DNSName -from lemur.plugins.lemur_acme import acme_handlers +from flask import Flask +from lemur.plugins.lemur_acme import acme_handlers from lemur.tests.vectors import ( ACME_CHAIN_SHORT_STR, ACME_CHAIN_LONG_STR, @@ -43,7 +43,7 @@ def test_setup_acme_client_fail(self): mock_authority = Mock() mock_authority.options = [] with self.assertRaises(Exception): - self.acme.setup_acme_client(mock_authority) + self.acme.setup_acme_client_no_retry(mock_authority) def test_reuse_account_not_defined(self): mock_authority = Mock() diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index c4d2ef424c..5e78d66355 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -1,17 +1,19 @@ +import time +import requests import json import sys -import time import dns import dns.exception import dns.name import dns.query import dns.resolver -import requests + from flask import current_app -from lemur.extensions import metrics from sentry_sdk import capture_exception +from lemur.extensions import metrics + class Record: """ diff --git a/lemur/plugins/lemur_adcs/plugin.py b/lemur/plugins/lemur_adcs/plugin.py index d9cbb4a57d..41881c9d07 100644 --- a/lemur/plugins/lemur_adcs/plugin.py +++ b/lemur/plugins/lemur_adcs/plugin.py @@ -1,9 +1,9 @@ +from lemur.plugins.bases import IssuerPlugin, SourcePlugin import requests -from OpenSSL import crypto +from lemur.plugins import lemur_adcs as ADCS from certsrv import Certsrv +from OpenSSL import crypto from flask import current_app -from lemur.plugins import lemur_adcs as ADCS -from lemur.plugins.bases import IssuerPlugin, SourcePlugin class ADCSIssuerPlugin(IssuerPlugin): diff --git a/lemur/plugins/lemur_atlas/plugin.py b/lemur/plugins/lemur_atlas/plugin.py index 5ab4d89d77..7fee81c39d 100644 --- a/lemur/plugins/lemur_atlas/plugin.py +++ b/lemur/plugins/lemur_atlas/plugin.py @@ -7,14 +7,15 @@ .. moduleauthor:: Kevin Glisson """ import json -from datetime import datetime from typing import Any, Dict import requests +from requests.exceptions import ConnectionError +from datetime import datetime + from flask import current_app from lemur.plugins import lemur_atlas as atlas from lemur.plugins.bases.metric import MetricPlugin -from requests.exceptions import ConnectionError def millis_since_epoch(): diff --git a/lemur/plugins/lemur_aws/cloudfront.py b/lemur/plugins/lemur_aws/cloudfront.py index be19f67ee9..aced25f334 100644 --- a/lemur/plugins/lemur_aws/cloudfront.py +++ b/lemur/plugins/lemur_aws/cloudfront.py @@ -6,10 +6,11 @@ """ from flask import current_app +from sentry_sdk import capture_exception + from lemur.exceptions import InvalidDistribution from lemur.extensions import metrics from lemur.plugins.lemur_aws.sts import sts_client -from sentry_sdk import capture_exception def get_all_distributions(**kwargs): diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index c5d0297078..1507967072 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -32,17 +32,17 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Harm Weites """ -import sys from os.path import join - +import sys from acme.errors import ClientError from flask import current_app +from sentry_sdk import capture_exception + from lemur.common.utils import check_validation from lemur.extensions import metrics from lemur.plugins import lemur_aws as aws, ExpirationNotificationPlugin from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns, cloudfront -from sentry_sdk import capture_exception def get_region_from_dns(dns): diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py index d440079618..14c4c04c94 100644 --- a/lemur/plugins/lemur_aws/sns.py +++ b/lemur/plugins/lemur_aws/sns.py @@ -10,6 +10,7 @@ import arrow import boto3 from flask import current_app + from lemur.plugins.utils import get_plugin_option diff --git a/lemur/plugins/lemur_aws/tests/test_sns.py b/lemur/plugins/lemur_aws/tests/test_sns.py index 3bb72da8c9..d8176185c2 100644 --- a/lemur/plugins/lemur_aws/tests/test_sns.py +++ b/lemur/plugins/lemur_aws/tests/test_sns.py @@ -5,8 +5,8 @@ import boto3 from moto import mock_sns, mock_sqs, mock_ses -from lemur.notifications import service from lemur.certificates.schemas import certificate_notification_output_schema +from lemur.notifications import service from lemur.plugins.lemur_aws.sns import format_message from lemur.plugins.lemur_aws.sns import publish from lemur.tests.factories import NotificationFactory, CertificateFactory diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py index f989fed810..d7730ceed6 100755 --- a/lemur/plugins/lemur_azure_dest/plugin.py +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -9,18 +9,19 @@ .. moduleauthor:: sirferl """ -import base64 -import json -import sys - -import requests -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.serialization import pkcs12 from flask import current_app + from lemur.common.defaults import common_name, bitstrength from lemur.common.utils import parse_certificate, parse_private_key, check_validation from lemur.plugins.bases import DestinationPlugin +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs12 +import requests +import json +import sys +import base64 + def handle_response(my_response): """ diff --git a/lemur/plugins/lemur_cfssl/plugin.py b/lemur/plugins/lemur_cfssl/plugin.py index 185ffdc261..5fc016594d 100644 --- a/lemur/plugins/lemur_cfssl/plugin.py +++ b/lemur/plugins/lemur_cfssl/plugin.py @@ -8,19 +8,20 @@ .. moduleauthor:: Charles Hendrie """ +import json +import requests import base64 -import hashlib import hmac -import json +import hashlib -import requests from flask import current_app -from lemur.common.utils import get_authority_key + from lemur.common.utils import parse_certificate +from lemur.common.utils import get_authority_key from lemur.constants import CRLReason -from lemur.extensions import metrics -from lemur.plugins import lemur_cfssl as cfssl from lemur.plugins.bases import IssuerPlugin +from lemur.plugins import lemur_cfssl as cfssl +from lemur.extensions import metrics class CfsslIssuerPlugin(IssuerPlugin): diff --git a/lemur/plugins/lemur_cryptography/plugin.py b/lemur/plugins/lemur_cryptography/plugin.py index f7692c8c8d..ecaa55e385 100644 --- a/lemur/plugins/lemur_cryptography/plugin.py +++ b/lemur/plugins/lemur_cryptography/plugin.py @@ -8,14 +8,17 @@ """ import uuid +from flask import current_app + from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization -from flask import current_app -from lemur.certificates.service import create_csr + from lemur.common.utils import parse_private_key -from lemur.plugins import lemur_cryptography as cryptography_issuer from lemur.plugins.bases import IssuerPlugin +from lemur.plugins import lemur_cryptography as cryptography_issuer + +from lemur.certificates.service import create_csr def build_certificate_authority(options): diff --git a/lemur/plugins/lemur_csr/plugin.py b/lemur/plugins/lemur_csr/plugin.py index 01bffdeb01..08827c8b5a 100644 --- a/lemur/plugins/lemur_csr/plugin.py +++ b/lemur/plugins/lemur_csr/plugin.py @@ -6,9 +6,10 @@ import subprocess from flask import current_app -from lemur.plugins import lemur_csr as csr -from lemur.plugins.bases import ExportPlugin + from lemur.utils import mktempfile, mktemppath +from lemur.plugins.bases import ExportPlugin +from lemur.plugins import lemur_csr as csr def run_process(command): diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 4886eb3e2f..f5226615aa 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -24,12 +24,13 @@ import requests from cryptography import x509 from flask import current_app, g +from retrying import retry +from urllib3.util.retry import Retry + from lemur.common.utils import validate_conf, convert_pkcs7_bytes_to_pem from lemur.extensions import metrics from lemur.plugins import lemur_digicert as digicert from lemur.plugins.bases import IssuerPlugin, SourcePlugin -from requests.packages.urllib3.util.retry import Retry -from retrying import retry def log_status_code(r, *args, **kwargs): diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index b27d6b767a..2c7885735b 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -6,19 +6,21 @@ .. moduleauthor:: Kevin Glisson """ -from html.parser import HTMLParser - import boto3 +from html.parser import HTMLParser from flask import current_app from flask_mail import Message +from sentry_sdk import capture_exception + from lemur.constants import EMAIL_RE, EMAIL_RE_HELP -from lemur.exceptions import InvalidConfiguration from lemur.extensions import smtp_mail -from lemur.plugins import lemur_email as email +from lemur.exceptions import InvalidConfiguration + from lemur.plugins.bases import ExpirationNotificationPlugin +from lemur.plugins import lemur_email as email + from lemur.plugins.lemur_email.templates.config import env from lemur.plugins.utils import get_plugin_option -from sentry_sdk import capture_exception def render_html(template_name, options, certificates): diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 15b8718d3b..12a1f4d94d 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -4,13 +4,14 @@ import arrow import requests from flask import current_app +from retrying import retry +from urllib3.util.retry import Retry + from lemur.common.utils import validate_conf, get_key_type_from_certificate from lemur.constants import CRLReason from lemur.extensions import metrics from lemur.plugins import lemur_entrust as entrust from lemur.plugins.bases import IssuerPlugin, SourcePlugin -from requests.packages.urllib3.util.retry import Retry -from retrying import retry def log_status_code(r, *args, **kwargs): diff --git a/lemur/plugins/lemur_entrust/tests/test_entrust.py b/lemur/plugins/lemur_entrust/tests/test_entrust.py index 416461b7ec..441b496e66 100644 --- a/lemur/plugins/lemur_entrust/tests/test_entrust.py +++ b/lemur/plugins/lemur_entrust/tests/test_entrust.py @@ -2,8 +2,8 @@ import arrow from cryptography import x509 -from freezegun import freeze_time from lemur.plugins.lemur_entrust import plugin +from freezegun import freeze_time def config_mock(*args): diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index 2ee6211f3c..560a3a8c8d 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -15,6 +15,7 @@ import requests from flask import current_app + from lemur.common.defaults import common_name from lemur.common.utils import parse_certificate, base64encode, check_validation from lemur.plugins.bases import DestinationPlugin @@ -80,7 +81,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain): } if secret_format == "Full": secret["data"] = { - "combined.pem": base64encode(f"{body}\n{private_key}"), + "combined.pem": base64encode("{}\n{}".format(body, private_key)), "ca.crt": base64encode(cert_chain), "service.key": base64encode(private_key), "service.crt": base64encode(body), @@ -88,7 +89,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain): if secret_format == "TLS": secret["type"] = "kubernetes.io/tls" secret["data"] = { - "tls.crt": base64encode(f"{body}\n{cert_chain}"), + "tls.crt": base64encode("{}\n{}".format(body, cert_chain)), "tls.key": base64encode(private_key), } if secret_format == "Certificate": diff --git a/lemur/plugins/lemur_openssl/plugin.py b/lemur/plugins/lemur_openssl/plugin.py index e0726b6cc5..0c4234d96e 100644 --- a/lemur/plugins/lemur_openssl/plugin.py +++ b/lemur/plugins/lemur_openssl/plugin.py @@ -9,11 +9,12 @@ import subprocess from flask import current_app -from lemur.common.defaults import common_name -from lemur.common.utils import get_psuedo_random_string, parse_certificate, check_validation -from lemur.plugins import lemur_openssl as openssl -from lemur.plugins.bases import ExportPlugin + from lemur.utils import mktempfile, mktemppath +from lemur.plugins.bases import ExportPlugin +from lemur.plugins import lemur_openssl as openssl +from lemur.common.utils import get_psuedo_random_string, parse_certificate, check_validation +from lemur.common.defaults import common_name def run_process(command): diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 697496a7bd..5d25dfb331 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -19,12 +19,13 @@ from os import path import paramiko +from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError + from flask import current_app +from lemur.plugins import lemur_sftp from lemur.common.defaults import common_name from lemur.common.utils import parse_certificate, check_validation -from lemur.plugins import lemur_sftp from lemur.plugins.bases import DestinationPlugin -from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError class SFTPDestinationPlugin(DestinationPlugin): diff --git a/lemur/plugins/lemur_slack/tests/test_slack.py b/lemur/plugins/lemur_slack/tests/test_slack.py index 813bbe12d5..e750463a57 100644 --- a/lemur/plugins/lemur_slack/tests/test_slack.py +++ b/lemur/plugins/lemur_slack/tests/test_slack.py @@ -1,9 +1,10 @@ from datetime import timedelta import arrow +from moto import mock_ses + from lemur.tests.factories import NotificationFactory, CertificateFactory from lemur.tests.test_messaging import verify_sender_email -from moto import mock_ses def test_formatting(certificate): diff --git a/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py b/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py index 9d74971763..008cdc36a3 100644 --- a/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py +++ b/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py @@ -1,7 +1,8 @@ import lemur_statsd as plug -from datadog import DogStatsd + from flask import current_app from lemur.plugins.bases.metric import MetricPlugin +from datadog import DogStatsd class StatsdMetricPlugin(MetricPlugin): diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 186709a542..0a05ba9e6a 100755 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -16,11 +16,12 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from flask import current_app +from validators.url import url + from lemur.common.defaults import common_name, country, state, location, organizational_unit, organization from lemur.common.utils import parse_certificate, check_validation from lemur.plugins.bases import DestinationPlugin from lemur.plugins.bases import SourcePlugin -from validators.url import url class VaultSourcePlugin(SourcePlugin): diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 2c423a714e..da7ff639df 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -12,11 +12,12 @@ import xmltodict from cryptography import x509 from flask import current_app +from sentry_sdk import capture_exception + from lemur.common.utils import get_psuedo_random_string from lemur.extensions import metrics from lemur.plugins import lemur_verisign as verisign from lemur.plugins.bases import IssuerPlugin, SourcePlugin -from sentry_sdk import capture_exception # https://support.venafi.com/entries/66445046-Info-VeriSign-Error-Codes VERISIGN_ERRORS = { @@ -226,8 +227,7 @@ def create_certificate(self, csr, issuer_options): external_id = None if 'Transaction_ID' in response_dict['Response'].keys(): external_id = response_dict['Response']['Transaction_ID'] - chain = current_app.config.get(f"VERISIGN_INTERMEDIATE_{authority}", - current_app.config.get("VERISIGN_INTERMEDIATE")) + chain = current_app.config.get(f"VERISIGN_INTERMEDIATE_{authority}", current_app.config.get("VERISIGN_INTERMEDIATE")) return cert, chain, external_id @staticmethod diff --git a/lemur/plugins/views.py b/lemur/plugins/views.py index 9dc5f23876..050248c17f 100644 --- a/lemur/plugins/views.py +++ b/lemur/plugins/views.py @@ -9,9 +9,11 @@ from flask import Blueprint from flask_restful import Api, reqparse from lemur.auth.service import AuthenticatedResource + + +from lemur.schemas import plugins_output_schema, plugin_output_schema from lemur.common.schema import validate_schema from lemur.plugins.base import plugins -from lemur.schemas import plugins_output_schema, plugin_output_schema mod = Blueprint("plugins", __name__) api = Api(mod) diff --git a/lemur/roles/models.py b/lemur/roles/models.py index e11a1c2936..15899599cf 100644 --- a/lemur/roles/models.py +++ b/lemur/roles/models.py @@ -9,16 +9,17 @@ .. moduleauthor:: Kevin Glisson """ +from sqlalchemy.orm import relationship +from sqlalchemy import Boolean, Column, Integer, String, Text, ForeignKey + from lemur.database import BaseModel +from lemur.utils import Vault from lemur.models import ( roles_users, roles_authorities, roles_certificates, pending_cert_role_associations, ) -from lemur.utils import Vault -from sqlalchemy import Boolean, Column, Integer, String, Text, ForeignKey -from sqlalchemy.orm import relationship class Role(BaseModel): diff --git a/lemur/roles/views.py b/lemur/roles/views.py index afe0a7a0bb..40125b8694 100644 --- a/lemur/roles/views.py +++ b/lemur/roles/views.py @@ -10,18 +10,21 @@ from flask import Blueprint, g from flask import make_response, jsonify from flask_restful import reqparse, Api -from lemur.auth.permissions import RoleMemberPermission, admin_permission + +from lemur.roles import service from lemur.auth.service import AuthenticatedResource -from lemur.common.schema import validate_schema +from lemur.auth.permissions import RoleMemberPermission, admin_permission from lemur.common.utils import paginated_parser from lemur.logs import service as log_service -from lemur.roles import service + +from lemur.common.schema import validate_schema from lemur.roles.schemas import ( role_input_schema, role_output_schema, roles_output_schema, ) + mod = Blueprint("roles", __name__) api = Api(mod) diff --git a/lemur/schemas.py b/lemur/schemas.py index 9f52ae2ef5..5788eaae78 100644 --- a/lemur/schemas.py +++ b/lemur/schemas.py @@ -7,6 +7,10 @@ .. moduleauthor:: Kevin Glisson """ +from marshmallow import fields, post_load, pre_load, post_dump +from marshmallow.exceptions import ValidationError +from sqlalchemy.orm.exc import NoResultFound + from lemur.authorities.models import Authority from lemur.certificates.models import Certificate from lemur.common import validators @@ -25,9 +29,6 @@ from lemur.policies.models import RotationPolicy from lemur.roles.models import Role from lemur.users.models import User -from marshmallow import fields, post_load, pre_load, post_dump -from marshmallow.exceptions import ValidationError -from sqlalchemy.orm.exc import NoResultFound def validate_options(options): diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index db3b48df95..3ea1b9431a 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -5,22 +5,25 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +from copy import deepcopy +import click import sys import time -from copy import deepcopy -import click +from tabulate import tabulate from flask import current_app -from lemur.certificates import service as certificate_service +from sentry_sdk import capture_exception + from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS -from lemur.destinations import service as dest_service + from lemur.extensions import metrics from lemur.plugins.base import plugins from lemur.plugins.utils import get_plugin_option, set_plugin_option + +from lemur.destinations import service as dest_service from lemur.sources import service as source_service from lemur.users import service as user_service -from sentry_sdk import capture_exception -from tabulate import tabulate +from lemur.certificates import service as certificate_service @click.group(name="source", help="Handles all source related tasks.") diff --git a/lemur/sources/models.py b/lemur/sources/models.py index 24c69bd4fb..ca993d8fb0 100644 --- a/lemur/sources/models.py +++ b/lemur/sources/models.py @@ -5,12 +5,13 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +from sqlalchemy.orm import relationship +from sqlalchemy import Column, Integer, String, Text, Boolean +from sqlalchemy_utils import JSONType from lemur.database import BaseModel + from lemur.plugins.base import plugins -from sqlalchemy import Column, Integer, String, Text, Boolean -from sqlalchemy.orm import relationship from sqlalchemy_utils import ArrowType -from sqlalchemy_utils import JSONType class Source(BaseModel): diff --git a/lemur/sources/service.py b/lemur/sources/service.py index ced4ffb840..14dfad3d76 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -5,29 +5,31 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import copy +import arrow from datetime import timedelta +import copy -import arrow from flask import current_app +from sqlalchemy.exc import OperationalError +from sentry_sdk import capture_exception +from sqlalchemy import cast +from sqlalchemy_utils import ArrowType + from lemur import database -from lemur.certificates import service as certificate_service +from lemur.sources.models import Source from lemur.certificates.models import Certificate -from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common.defaults import serial -from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate -from lemur.destinations import service as destination_service +from lemur.certificates import service as certificate_service from lemur.endpoints import service as endpoint_service from lemur.endpoints.models import Endpoint from lemur.extensions import metrics +from lemur.destinations import service as destination_service + +from lemur.certificates.schemas import CertificateUploadInputSchema +from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate +from lemur.common.defaults import serial from lemur.logs import service as log_service from lemur.plugins.base import plugins from lemur.plugins.utils import get_plugin_option, set_plugin_option -from lemur.sources.models import Source -from sentry_sdk import capture_exception -from sqlalchemy import cast -from sqlalchemy.exc import OperationalError -from sqlalchemy_utils import ArrowType def certificate_create(certificate, source): diff --git a/lemur/sources/views.py b/lemur/sources/views.py index 6229d9b889..b104d2d155 100644 --- a/lemur/sources/views.py +++ b/lemur/sources/views.py @@ -8,17 +8,20 @@ """ from flask import Blueprint from flask_restful import Api, reqparse -from lemur.auth.permissions import admin_permission -from lemur.auth.service import AuthenticatedResource -from lemur.common.schema import validate_schema -from lemur.common.utils import paginated_parser from lemur.sources import service + +from lemur.common.schema import validate_schema from lemur.sources.schemas import ( source_input_schema, source_output_schema, sources_output_schema, ) +from lemur.auth.service import AuthenticatedResource +from lemur.auth.permissions import admin_permission +from lemur.common.utils import paginated_parser + + mod = Blueprint("sources", __name__) api = Api(mod) diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index 05508ff2c3..397d74ada3 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -7,6 +7,8 @@ from cryptography.hazmat.primitives import hashes from flask import current_app from flask_principal import identity_changed, Identity +from sqlalchemy.sql import text + from lemur import create_app from lemur.auth.service import create_token from lemur.common.utils import parse_private_key @@ -17,8 +19,6 @@ ROOTCA_CERT_STR, ROOTCA_KEY, ) -from sqlalchemy.sql import text - from .factories import ( ApiKeyFactory, AuthorityFactory, diff --git a/lemur/tests/factories.py b/lemur/tests/factories.py index 89195ff984..dbd478c927 100644 --- a/lemur/tests/factories.py +++ b/lemur/tests/factories.py @@ -5,6 +5,7 @@ from factory import Sequence, post_generation, SubFactory from factory.alchemy import SQLAlchemyModelFactory from factory.fuzzy import FuzzyChoice, FuzzyText, FuzzyDate, FuzzyInteger + from lemur.api_keys.models import ApiKey from lemur.authorities.models import Authority from lemur.certificates.models import Certificate @@ -18,7 +19,6 @@ from lemur.roles.models import Role from lemur.sources.models import Source from lemur.users.models import User - from .vectors import ( SAN_CERT_STR, SAN_CERT_KEY, diff --git a/lemur/tests/plugins/issuer_plugin.py b/lemur/tests/plugins/issuer_plugin.py index 7ac337a243..2a59c81477 100644 --- a/lemur/tests/plugins/issuer_plugin.py +++ b/lemur/tests/plugins/issuer_plugin.py @@ -1,8 +1,9 @@ -from cryptography import x509 -from cryptography.x509.oid import ExtensionOID -from lemur.common.utils import parse_csr from lemur.plugins.bases import IssuerPlugin + from lemur.tests.vectors import SAN_CERT_STR, INTERMEDIATE_CERT_STR, IP_SAN_NO_CN_CERT_STR +from lemur.common.utils import parse_csr +from cryptography import x509 +from cryptography.x509.oid import ExtensionOID class TestIssuerPlugin(IssuerPlugin): diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index c4193af636..69e9330c71 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -12,6 +12,9 @@ from cryptography.hazmat.backends import default_backend from cryptography.x509.oid import ExtensionOID from freezegun import freeze_time +from marshmallow import ValidationError +from sqlalchemy.testing import fail + from lemur.certificates.service import create_csr, identify_and_persist_expiring_deployed_certificates, \ reissue_certificate from lemur.certificates.views import * # noqa @@ -31,8 +34,6 @@ ROOTCA_KEY, ROOTCA_CERT_STR, ) -from marshmallow import ValidationError -from sqlalchemy.testing import fail def test_get_or_increase_name(session, certificate): diff --git a/lemur/tests/test_dns_providers.py b/lemur/tests/test_dns_providers.py index 7bd73037ff..9daae13dd4 100644 --- a/lemur/tests/test_dns_providers.py +++ b/lemur/tests/test_dns_providers.py @@ -1,5 +1,6 @@ import json import unittest + from lemur.dns_providers import util as dnsutil from lemur.dns_providers.schemas import dns_provider_output_schema diff --git a/lemur/tests/test_ldap.py b/lemur/tests/test_ldap.py index 33c4a98995..4f99a4f498 100644 --- a/lemur/tests/test_ldap.py +++ b/lemur/tests/test_ldap.py @@ -1,7 +1,6 @@ -from unittest.mock import patch, MagicMock - import pytest from lemur.auth.ldap import * # noqa +from unittest.mock import patch, MagicMock class LdapPrincipalTester(LdapPrincipal): diff --git a/lemur/tests/test_verify.py b/lemur/tests/test_verify.py index e20a4ad4f3..262a39bb1d 100644 --- a/lemur/tests/test_verify.py +++ b/lemur/tests/test_verify.py @@ -1,4 +1,6 @@ import pytest +import requests +import requests_mock from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization, hashes @@ -6,7 +8,6 @@ from lemur.certificates.verify import verify_string, crl_verify from lemur.utils import mktempfile - from .vectors import INTERMEDIATE_CERT_STR @@ -41,19 +42,21 @@ def test_verify_crl_unknown_scheme(cert_builder, private_key): def test_verify_crl_unreachable(cert_builder, private_key): """Unreachable CRL distribution point results in error.""" ldap_uri = "http://invalid.example.org/crl/foobar.crl" - crl_dp = x509.DistributionPoint( - [UniformResourceIdentifier(ldap_uri)], - relative_name=None, - reasons=None, - crl_issuer=None, - ) - cert = cert_builder.add_extension( - x509.CRLDistributionPoints([crl_dp]), critical=False - ).sign(private_key, hashes.SHA256(), default_backend()) - - with mktempfile() as cert_tmp: - with open(cert_tmp, "wb") as f: - f.write(cert.public_bytes(serialization.Encoding.PEM)) - - with pytest.raises(Exception, match="Unable to retrieve CRL:"): - crl_verify(cert, cert_tmp) + with requests_mock.Mocker() as m: + m.get(ldap_uri, exc=requests.exceptions.Timeout) + crl_dp = x509.DistributionPoint( + [UniformResourceIdentifier(ldap_uri)], + relative_name=None, + reasons=None, + crl_issuer=None, + ) + cert = cert_builder.add_extension( + x509.CRLDistributionPoints([crl_dp]), critical=False + ).sign(private_key, hashes.SHA256(), default_backend()) + + with mktempfile() as cert_tmp: + with open(cert_tmp, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + + with pytest.raises(Exception, match="Unable to retrieve CRL:"): + crl_verify(cert, cert_tmp) diff --git a/lemur/users/models.py b/lemur/users/models.py index d0e38ba3c5..2e42ff8d20 100644 --- a/lemur/users/models.py +++ b/lemur/users/models.py @@ -8,14 +8,17 @@ .. moduleauthor:: Kevin Glisson """ -from lemur.database import BaseModel, db -from lemur.extensions import bcrypt -from lemur.models import roles_users +from sqlalchemy.orm import relationship from sqlalchemy import Integer, String, Column, Boolean from sqlalchemy.event import listen -from sqlalchemy.orm import relationship + from sqlalchemy_utils.types.arrow import ArrowType +from lemur.database import BaseModel, db +from lemur.models import roles_users + +from lemur.extensions import bcrypt + def hash_password(mapper, connect, target): """ diff --git a/lemur/utils.py b/lemur/utils.py index ba702d2eac..c51bce1ca8 100644 --- a/lemur/utils.py +++ b/lemur/utils.py @@ -10,9 +10,9 @@ from contextlib import contextmanager import cryptography.fernet +from sqlalchemy import types from cryptography.fernet import Fernet, MultiFernet from flask import current_app -from sqlalchemy import types @contextmanager diff --git a/requirements-dev.in b/requirements-dev.in index e7ae093474..015ecc6740 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -4,9 +4,5 @@ flake8 pre-commit invoke twine -marshmallow-sqlalchemy == 0.23.1 #related to the marshmallow issue (to avoid conflicts) -marshmallow<2.21.1 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 nodeenv -pyyaml>=4.2b1 -sqlalchemy < 1.4.0 # ImportError: cannot import name '_ColumnEntity' https://github.com/sqlalchemy/sqlalchemy/issues/6226 -urllib3 <= 1.26.16 # limited due to dependency conflict with twine \ No newline at end of file +-r requirements-tests.txt \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 3ac2d1200a..375b328a75 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,102 +1,855 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --no-emit-index-url --output-file=requirements-dev.txt requirements-dev.in # +acme==2.7.1 + # via + # -r requirements-tests.txt + # certbot +alembic==1.12.0 + # via + # -r requirements-tests.txt + # flask-migrate +alembic-autogenerate-enums==0.1.2 + # via -r requirements-tests.txt +amqp==5.1.1 + # via + # -r requirements-tests.txt + # kombu +aniso8601==9.0.1 + # via + # -r requirements-tests.txt + # flask-restful +annotated-types==0.6.0 + # via + # -r requirements-tests.txt + # pydantic +arrow==1.3.0 + # via -r requirements-tests.txt +async-timeout==4.0.3 + # via + # -r requirements-tests.txt + # redis +asyncpool==1.0 + # via -r requirements-tests.txt +attrs==23.1.0 + # via + # -r requirements-tests.txt + # jschema-to-python + # jsonlines + # jsonschema + # referencing + # sarif-om +aws-sam-translator==1.78.0 + # via + # -r requirements-tests.txt + # cfn-lint +aws-xray-sdk==2.12.1 + # via + # -r requirements-tests.txt + # moto +bandit==1.7.5 + # via -r requirements-tests.txt +bcrypt==4.0.1 + # via + # -r requirements-tests.txt + # flask-bcrypt + # paramiko +beautifulsoup4==4.12.2 + # via + # -r requirements-tests.txt + # cloudflare +billiard==4.1.0 + # via + # -r requirements-tests.txt + # celery +black==23.10.0 + # via -r requirements-tests.txt +blinker==1.6.3 + # via + # -r requirements-tests.txt + # flask + # flask-mail + # flask-principal +boto3==1.28.66 + # via + # -r requirements-tests.txt + # aws-sam-translator + # moto +botocore==1.31.66 + # via + # -r requirements-tests.txt + # aws-xray-sdk + # boto3 + # moto + # s3transfer +celery[redis]==5.3.4 + # via + # -r requirements-tests.txt + # celery +certbot==2.7.1 + # via -r requirements-tests.txt certifi==2023.7.22 - # via requests + # via + # -r requirements-tests.txt + # requests + # sentry-sdk +certsrv[ntlm]==2.1.1 + # via + # -r requirements-tests.txt + # certsrv +cffi==1.16.0 + # via + # -r requirements-tests.txt + # cryptography + # pynacl cfgv==3.4.0 # via pre-commit -charset-normalizer==3.2.0 - # via requests +cfn-lint==0.82.2 + # via + # -r requirements-tests.txt + # moto +charset-normalizer==3.3.0 + # via + # -r requirements-tests.txt + # requests +click==8.1.7 + # via + # -r requirements-tests.txt + # black + # celery + # click-didyoumean + # click-plugins + # click-repl + # flask +click-didyoumean==0.3.0 + # via + # -r requirements-tests.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements-tests.txt + # celery +click-repl==0.3.0 + # via + # -r requirements-tests.txt + # celery +cloudflare==2.12.4 + # via -r requirements-tests.txt +configargparse==1.7 + # via + # -r requirements-tests.txt + # certbot +configobj==5.0.8 + # via + # -r requirements-tests.txt + # certbot +coverage==7.3.2 + # via -r requirements-tests.txt +cryptography==41.0.4 + # via + # -r requirements-tests.txt + # acme + # certbot + # josepy + # moto + # paramiko + # pyopenssl + # pyspnego + # python-jose + # requests-ntlm + # sshpubkeys + # types-paramiko + # types-pyopenssl + # types-redis +deprecated==1.2.14 + # via + # -r requirements-tests.txt + # limits distlib==0.3.7 # via virtualenv +distro==1.8.0 + # via + # -r requirements-tests.txt + # certbot +dnspython==1.15.0 + # via + # -r requirements-tests.txt + # dnspython3 +dnspython3==1.15.0 + # via -r requirements-tests.txt +docker==6.1.3 + # via + # -r requirements-tests.txt + # moto docutils==0.20.1 # via readme-renderer +dyn==1.8.6 + # via -r requirements-tests.txt +ecdsa==0.18.0 + # via + # -r requirements-tests.txt + # moto + # python-jose + # sshpubkeys +exceptiongroup==1.1.3 + # via + # -r requirements-tests.txt + # pytest +factory-boy==3.3.0 + # via -r requirements-tests.txt +faker==19.11.0 + # via + # -r requirements-tests.txt + # factory-boy +fakeredis==2.19.0 + # via -r requirements-tests.txt filelock==3.12.4 # via virtualenv flake8==6.1.0 # via -r requirements-dev.in -identify==2.5.29 +flask==2.3.3 + # via + # -r requirements-tests.txt + # flask-bcrypt + # flask-cors + # flask-limiter + # flask-mail + # flask-migrate + # flask-principal + # flask-restful + # flask-sqlalchemy + # pytest-flask +flask-bcrypt==1.0.1 + # via -r requirements-tests.txt +flask-cors==4.0.0 + # via -r requirements-tests.txt +flask-limiter==3.5.0 + # via -r requirements-tests.txt +flask-mail==0.9.1 + # via -r requirements-tests.txt +flask-migrate==4.0.5 + # via -r requirements-tests.txt +flask-principal==0.4.0 + # via -r requirements-tests.txt +flask-replicated==2.1 + # via -r requirements-tests.txt +flask-restful==0.3.10 + # via -r requirements-tests.txt +flask-sqlalchemy==2.5.1 + # via + # -r requirements-tests.txt + # flask-migrate +freezegun==1.2.2 + # via -r requirements-tests.txt +future==0.18.3 + # via -r requirements-tests.txt +gitdb==4.0.10 + # via + # -r requirements-tests.txt + # gitpython +gitpython==3.1.40 + # via + # -r requirements-tests.txt + # bandit +graphql-core==3.2.3 + # via + # -r requirements-tests.txt + # moto +gunicorn==21.2.0 + # via -r requirements-tests.txt +hvac==1.2.1 + # via -r requirements-tests.txt +identify==2.5.30 # via pre-commit idna==3.4 - # via requests + # via + # -r requirements-tests.txt + # requests importlib-metadata==6.8.0 # via + # -r requirements-tests.txt + # certbot + # flask # keyring # twine +importlib-resources==6.1.0 + # via + # -r requirements-tests.txt + # limits +inflection==0.5.1 + # via -r requirements-tests.txt +iniconfig==2.0.0 + # via + # -r requirements-tests.txt + # pytest invoke==2.2.0 # via -r requirements-dev.in +itsdangerous==2.1.2 + # via + # -r requirements-tests.txt + # flask jaraco-classes==3.3.0 # via keyring +javaobj-py3==0.4.3 + # via + # -r requirements-tests.txt + # pyjks +jinja2==3.1.2 + # via + # -r requirements-tests.txt + # flask + # moto +jmespath==1.0.1 + # via + # -r requirements-tests.txt + # boto3 + # botocore +josepy==1.13.0 + # via + # -r requirements-tests.txt + # acme + # certbot +jschema-to-python==1.2.3 + # via + # -r requirements-tests.txt + # cfn-lint +jsondiff==2.0.0 + # via + # -r requirements-tests.txt + # moto +jsonlines==4.0.0 + # via + # -r requirements-tests.txt + # cloudflare +jsonpatch==1.33 + # via + # -r requirements-tests.txt + # cfn-lint +jsonpickle==3.0.2 + # via + # -r requirements-tests.txt + # jschema-to-python +jsonpointer==2.4 + # via + # -r requirements-tests.txt + # jsonpatch +jsonschema==4.19.1 + # via + # -r requirements-tests.txt + # aws-sam-translator + # cfn-lint + # openapi-schema-validator + # openapi-spec-validator +jsonschema-path==0.3.1 + # via + # -r requirements-tests.txt + # openapi-spec-validator +jsonschema-specifications==2023.7.1 + # via + # -r requirements-tests.txt + # jsonschema + # openapi-schema-validator +junit-xml==1.9 + # via + # -r requirements-tests.txt + # cfn-lint keyring==24.2.0 # via twine +kombu==5.3.2 + # via + # -r requirements-tests.txt + # celery +lazy-object-proxy==1.9.0 + # via + # -r requirements-tests.txt + # openapi-spec-validator +limits==3.6.0 + # via + # -r requirements-tests.txt + # flask-limiter +lockfile==0.12.2 + # via -r requirements-tests.txt +logmatic-python==0.1.7 + # via -r requirements-tests.txt +mako==1.2.4 + # via + # -r requirements-tests.txt + # alembic markdown-it-py==3.0.0 - # via rich + # via + # -r requirements-tests.txt + # rich +markupsafe==2.1.3 + # via + # -r requirements-tests.txt + # jinja2 + # mako + # werkzeug marshmallow==2.21.0 # via - # -r requirements-dev.in + # -r requirements-tests.txt # marshmallow-sqlalchemy marshmallow-sqlalchemy==0.23.1 - # via -r requirements-dev.in + # via -r requirements-tests.txt mccabe==0.7.0 # via flake8 mdurl==0.1.2 - # via markdown-it-py + # via + # -r requirements-tests.txt + # markdown-it-py more-itertools==10.1.0 # via jaraco-classes +moto[all]==4.2.6 + # via + # -r requirements-tests.txt + # moto +mpmath==1.3.0 + # via + # -r requirements-tests.txt + # sympy +multipart==0.2.4 + # via + # -r requirements-tests.txt + # moto +mypy==1.6.1 + # via -r requirements-tests.txt +mypy-extensions==1.0.0 + # via + # -r requirements-tests.txt + # black + # mypy +ndg-httpsclient==0.5.1 + # via -r requirements-tests.txt +networkx==3.1 + # via + # -r requirements-tests.txt + # cfn-lint nh3==0.2.14 # via readme-renderer nodeenv==1.8.0 # via # -r requirements-dev.in # pre-commit +nose==1.3.7 + # via -r requirements-tests.txt +openapi-schema-validator==0.6.2 + # via + # -r requirements-tests.txt + # openapi-spec-validator +openapi-spec-validator==0.7.1 + # via + # -r requirements-tests.txt + # moto +ordered-set==4.1.0 + # via + # -r requirements-tests.txt + # flask-limiter +packaging==23.2 + # via + # -r requirements-tests.txt + # black + # docker + # gunicorn + # limits + # pytest +paramiko==3.3.1 + # via -r requirements-tests.txt +parsedatetime==2.6 + # via + # -r requirements-tests.txt + # certbot +pathable==0.4.3 + # via + # -r requirements-tests.txt + # jsonschema-path +pathspec==0.11.2 + # via + # -r requirements-tests.txt + # black +pbr==5.11.1 + # via + # -r requirements-tests.txt + # jschema-to-python + # sarif-om + # stevedore +pem==23.1.0 + # via -r requirements-tests.txt pkginfo==1.9.6 # via twine -platformdirs==3.10.0 - # via virtualenv -pre-commit==3.4.0 +platformdirs==3.11.0 + # via + # -r requirements-tests.txt + # black + # virtualenv +pluggy==1.3.0 + # via + # -r requirements-tests.txt + # pytest +pre-commit==3.5.0 # via -r requirements-dev.in -pycodestyle==2.11.0 +prompt-toolkit==3.0.39 + # via + # -r requirements-tests.txt + # click-repl +psycopg2==2.9.9 + # via -r requirements-tests.txt +py-partiql-parser==0.4.0 + # via + # -r requirements-tests.txt + # moto +pyasn1==0.5.0 + # via + # -r requirements-tests.txt + # ndg-httpsclient + # pyasn1-modules + # pyjks + # python-jose + # python-ldap + # rsa +pyasn1-modules==0.3.0 + # via + # -r requirements-tests.txt + # pyjks + # python-ldap +pycodestyle==2.11.1 # via flake8 +pycparser==2.21 + # via + # -r requirements-tests.txt + # cffi +pycryptodomex==3.19.0 + # via + # -r requirements-tests.txt + # pyjks +pydantic==2.4.2 + # via + # -r requirements-tests.txt + # aws-sam-translator +pydantic-core==2.10.1 + # via + # -r requirements-tests.txt + # pydantic pyflakes==3.1.0 - # via flake8 + # via + # -r requirements-tests.txt + # flake8 pygments==2.16.1 # via + # -r requirements-tests.txt # readme-renderer # rich +pyhcl==0.4.5 + # via + # -r requirements-tests.txt + # hvac +pyjks==20.0.0 + # via -r requirements-tests.txt +pyjwt==2.8.0 + # via -r requirements-tests.txt +pynacl==1.5.0 + # via + # -r requirements-tests.txt + # paramiko +pyopenssl==23.2.0 + # via + # -r requirements-tests.txt + # acme + # josepy + # ndg-httpsclient +pyparsing==3.1.1 + # via + # -r requirements-tests.txt + # moto +pyrfc3339==1.1 + # via + # -r requirements-tests.txt + # acme + # certbot +pyspnego==0.10.2 + # via + # -r requirements-tests.txt + # requests-ntlm +pytest==7.4.2 + # via + # -r requirements-tests.txt + # pytest-flask + # pytest-mock +pytest-flask==1.2.0 + # via -r requirements-tests.txt +pytest-mock==3.11.1 + # via -r requirements-tests.txt +python-dateutil==2.8.2 + # via + # -r requirements-tests.txt + # arrow + # botocore + # celery + # faker + # freezegun + # moto +python-jose[cryptography]==3.3.0 + # via + # -r requirements-tests.txt + # moto + # python-jose +python-json-logger==2.0.7 + # via + # -r requirements-tests.txt + # logmatic-python +python-ldap==3.4.3 + # via -r requirements-tests.txt +pytz==2023.3.post1 + # via + # -r requirements-tests.txt + # acme + # certbot + # flask-restful + # pyrfc3339 pyyaml==6.0.1 # via - # -r requirements-dev.in + # -r requirements-tests.txt + # bandit + # cfn-lint + # cloudflare + # jsonschema-path + # moto # pre-commit + # responses readme-renderer==42.0 # via twine +redis==4.6.0 + # via + # -r requirements-tests.txt + # celery + # fakeredis +referencing==0.30.2 + # via + # -r requirements-tests.txt + # jsonschema + # jsonschema-path + # jsonschema-specifications +regex==2023.10.3 + # via + # -r requirements-tests.txt + # cfn-lint requests==2.31.0 # via + # -r requirements-tests.txt + # acme + # certsrv + # cloudflare + # docker + # hvac + # jsonschema-path + # moto + # requests-mock + # requests-ntlm # requests-toolbelt + # responses # twine +requests-mock==1.11.0 + # via -r requirements-tests.txt +requests-ntlm==1.2.0 + # via + # -r requirements-tests.txt + # certsrv requests-toolbelt==1.0.0 # via twine +responses==0.23.3 + # via + # -r requirements-tests.txt + # moto +retrying==1.3.4 + # via -r requirements-tests.txt +rfc3339-validator==0.1.4 + # via + # -r requirements-tests.txt + # openapi-schema-validator rfc3986==2.0.0 # via twine -rich==13.5.3 - # via twine +rich==13.6.0 + # via + # -r requirements-tests.txt + # bandit + # flask-limiter + # twine +rpds-py==0.10.6 + # via + # -r requirements-tests.txt + # jsonschema + # referencing +rsa==4.9 + # via + # -r requirements-tests.txt + # python-jose +s3transfer==0.7.0 + # via + # -r requirements-tests.txt + # boto3 +sarif-om==1.0.4 + # via + # -r requirements-tests.txt + # cfn-lint +sentry-sdk==1.32.0 + # via -r requirements-tests.txt +six==1.16.0 + # via + # -r requirements-tests.txt + # configobj + # ecdsa + # flask-restful + # junit-xml + # python-dateutil + # requests-mock + # retrying + # rfc3339-validator +smmap==5.0.1 + # via + # -r requirements-tests.txt + # gitdb +sortedcontainers==2.4.0 + # via + # -r requirements-tests.txt + # fakeredis +soupsieve==2.5 + # via + # -r requirements-tests.txt + # beautifulsoup4 sqlalchemy==1.3.24 # via - # -r requirements-dev.in + # -r requirements-tests.txt + # alembic + # flask-sqlalchemy # marshmallow-sqlalchemy + # sqlalchemy-utils +sqlalchemy-utils==0.41.1 + # via -r requirements-tests.txt +sshpubkeys==3.3.1 + # via + # -r requirements-tests.txt + # moto +stevedore==5.1.0 + # via + # -r requirements-tests.txt + # bandit +sympy==1.12 + # via + # -r requirements-tests.txt + # cfn-lint +tabulate==0.9.0 + # via -r requirements-tests.txt +tomli==2.0.1 + # via + # -r requirements-tests.txt + # black + # mypy + # pytest twine==4.0.2 # via -r requirements-dev.in -urllib3==1.26.16 +twofish==0.3.0 # via - # -r requirements-dev.in + # -r requirements-tests.txt + # pyjks +types-deprecated==1.2.9.3 + # via -r requirements-tests.txt +types-paramiko==3.3.0.0 + # via -r requirements-tests.txt +types-pyopenssl==23.2.0.2 + # via + # -r requirements-tests.txt + # types-redis +types-pyrfc3339==1.1.1.5 + # via -r requirements-tests.txt +types-python-dateutil==2.8.19.14 + # via + # -r requirements-tests.txt + # arrow +types-pytz==2023.3.1.1 + # via -r requirements-tests.txt +types-pyyaml==6.0.12.12 + # via + # -r requirements-tests.txt + # responses +types-redis==4.6.0.7 + # via -r requirements-tests.txt +types-requests==2.31.0.6 + # via -r requirements-tests.txt +types-setuptools==68.2.0.0 + # via -r requirements-tests.txt +types-six==1.16.21.9 + # via -r requirements-tests.txt +types-tabulate==0.9.0.3 + # via -r requirements-tests.txt +types-urllib3==1.26.25.14 + # via + # -r requirements-tests.txt + # types-requests +typing-extensions==4.8.0 + # via + # -r requirements-tests.txt + # alembic + # aws-sam-translator + # black + # flask-limiter + # kombu + # limits + # mypy + # pydantic + # pydantic-core +tzdata==2023.3 + # via + # -r requirements-tests.txt + # celery +urllib3==1.26.18 + # via + # -r requirements-tests.txt + # botocore + # docker # requests + # responses + # sentry-sdk # twine +validators==0.22.0 + # via -r requirements-tests.txt +vine==5.0.0 + # via + # -r requirements-tests.txt + # amqp + # celery + # kombu virtualenv==20.24.5 # via pre-commit +wcwidth==0.2.8 + # via + # -r requirements-tests.txt + # prompt-toolkit +websocket-client==1.6.4 + # via + # -r requirements-tests.txt + # docker +werkzeug==3.0.0 + # via + # -r requirements-tests.txt + # flask + # moto + # pytest-flask +wrapt==1.15.0 + # via + # -r requirements-tests.txt + # aws-xray-sdk + # deprecated +xmltodict==0.13.0 + # via + # -r requirements-tests.txt + # moto zipp==3.17.0 - # via importlib-metadata + # via + # -r requirements-tests.txt + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-docs.in b/requirements-docs.in index deb831f313..dcf216ae7c 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -14,7 +14,7 @@ CloudFlare cryptography dnspython3 dyn -Flask +Flask<3 # until https://github.com/pytest-dev/pytest-flask/pull/168 is released Flask-Bcrypt Flask-Cors Flask-Limiter @@ -30,15 +30,15 @@ inflection itsdangerous josepy logmatic-python -marshmallow-sqlalchemy == 0.23.1 #related to the marshmallow issue (to avoid conflicts, as newer versions require marshmallow>=3.0.0) -sqlalchemy < 1.4.0 # ImportError: cannot import name '_ColumnEntity' https://github.com/sqlalchemy/sqlalchemy/issues/6226 +marshmallow-sqlalchemy == 0.23.1 #related to the marshmallow issue (to avoid conflicts) marshmallow<2.21.1 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 +sqlalchemy < 1.4.0 # ImportError: cannot import name '_ColumnEntity' https://github.com/sqlalchemy/sqlalchemy/issues/6226 paramiko # required for the SFTP destination plugin pem -pyjks >= 19 # pyjks < 19 depends on pycryptodome, which conflicts with dyn's usage of pycrypto +pyjks pyjwt pyOpenSSL -redis < 5.1.0 # requires a newer release of fakeredis +redis retrying sentry-sdk SQLAlchemy-Utils @@ -53,4 +53,4 @@ xmltodict sphinx sphinxcontrib-httpdomain sphinx-rtd-theme -docutils <= 0.19 # sphinx-rtd-theme 1.2.2 requires docutils<0.19 +docutils diff --git a/requirements-docs.txt b/requirements-docs.txt index daadd181c5..48e179bc50 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --no-emit-index-url --output-file=requirements-docs.txt requirements-docs.in # -acme==2.6.0 +acme==2.7.1 # via # -r requirements-docs.in # -r requirements-tests.txt @@ -15,18 +15,30 @@ alembic==1.12.0 # via # -r requirements-tests.txt # flask-migrate +alembic-autogenerate-enums==0.1.2 + # via -r requirements-tests.txt amqp==5.1.1 # via # -r requirements-tests.txt # kombu aniso8601==9.0.1 - # via flask-restful -annotated-types==0.5.0 + # via + # -r requirements-tests.txt + # flask-restful +annotated-types==0.6.0 # via # -r requirements-tests.txt # pydantic -arrow==1.2.3 - # via -r requirements-docs.in +arrow==1.3.0 + # via + # -r requirements-docs.in + # -r requirements-tests.txt +async-timeout==4.0.3 + # via + # -r requirements-tests.txt + # redis +asyncpool==1.0 + # via -r requirements-tests.txt attrs==23.1.0 # via # -r requirements-tests.txt @@ -35,43 +47,46 @@ attrs==23.1.0 # jsonschema # referencing # sarif-om -aws-sam-translator==1.75.0 +aws-sam-translator==1.78.0 # via # -r requirements-tests.txt # cfn-lint -aws-xray-sdk==2.12.0 +aws-xray-sdk==2.12.1 # via # -r requirements-tests.txt # moto -babel==2.12.1 +babel==2.13.0 # via sphinx bandit==1.7.5 # via -r requirements-tests.txt bcrypt==4.0.1 # via + # -r requirements-tests.txt # flask-bcrypt # paramiko beautifulsoup4==4.12.2 - # via cloudflare + # via + # -r requirements-tests.txt + # cloudflare billiard==4.1.0 # via # -r requirements-tests.txt # celery -black==23.9.1 +black==23.10.0 # via -r requirements-tests.txt -blinker==1.6.2 +blinker==1.6.3 # via # -r requirements-tests.txt # flask # flask-mail # flask-principal -boto3==1.28.57 +boto3==1.28.66 # via # -r requirements-docs.in # -r requirements-tests.txt # aws-sam-translator # moto -botocore==1.31.57 +botocore==1.31.66 # via # -r requirements-docs.in # -r requirements-tests.txt @@ -83,7 +98,8 @@ celery[redis]==5.3.4 # via # -r requirements-docs.in # -r requirements-tests.txt -certbot==2.6.0 + # celery +certbot==2.7.1 # via # -r requirements-docs.in # -r requirements-tests.txt @@ -92,18 +108,20 @@ certifi==2023.7.22 # -r requirements-tests.txt # requests # sentry-sdk -certsrv==2.1.1 - # via -r requirements-docs.in -cffi==1.15.1 +certsrv[ntlm]==2.1.1 + # via + # -r requirements-docs.in + # -r requirements-tests.txt +cffi==1.16.0 # via # -r requirements-tests.txt # cryptography # pynacl -cfn-lint==0.80.3 +cfn-lint==0.82.2 # via # -r requirements-tests.txt # moto -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 # via # -r requirements-tests.txt # requests @@ -129,7 +147,9 @@ click-repl==0.3.0 # -r requirements-tests.txt # celery cloudflare==2.12.4 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt configargparse==1.7 # via # -r requirements-tests.txt @@ -138,7 +158,7 @@ configobj==5.0.8 # via # -r requirements-tests.txt # certbot -coverage==7.3.1 +coverage==7.3.2 # via -r requirements-tests.txt cryptography==41.0.4 # via @@ -150,18 +170,29 @@ cryptography==41.0.4 # moto # paramiko # pyopenssl + # pyspnego # python-jose + # requests-ntlm # sshpubkeys + # types-paramiko + # types-pyopenssl + # types-redis deprecated==1.2.14 - # via limits + # via + # -r requirements-tests.txt + # limits distro==1.8.0 # via # -r requirements-tests.txt # certbot dnspython==1.15.0 - # via dnspython3 + # via + # -r requirements-tests.txt + # dnspython3 dnspython3==1.15.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt docker==6.1.3 # via # -r requirements-tests.txt @@ -172,16 +203,22 @@ docutils==0.18.1 # sphinx # sphinx-rtd-theme dyn==1.8.6 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt ecdsa==0.18.0 # via # -r requirements-tests.txt # moto # python-jose # sshpubkeys +exceptiongroup==1.1.3 + # via + # -r requirements-tests.txt + # pytest factory-boy==3.3.0 # via -r requirements-tests.txt -faker==19.6.2 +faker==19.11.0 # via # -r requirements-tests.txt # factory-boy @@ -201,23 +238,37 @@ flask==2.3.3 # flask-sqlalchemy # pytest-flask flask-bcrypt==1.0.1 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt flask-cors==4.0.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt flask-limiter==3.5.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt flask-mail==0.9.1 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt flask-migrate==4.0.5 # via # -r requirements-docs.in # -r requirements-tests.txt flask-principal==0.4.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt flask-replicated==2.1 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt flask-restful==0.3.10 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt flask-sqlalchemy==2.5.1 # via # -r requirements-docs.in @@ -225,11 +276,13 @@ flask-sqlalchemy==2.5.1 # flask-migrate freezegun==1.2.2 # via -r requirements-tests.txt +future==0.18.3 + # via -r requirements-tests.txt gitdb==4.0.10 # via # -r requirements-tests.txt # gitpython -gitpython==3.1.37 +gitpython==3.1.40 # via # -r requirements-tests.txt # bandit @@ -238,19 +291,33 @@ graphql-core==3.2.3 # -r requirements-tests.txt # moto gunicorn==21.2.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt hvac==1.2.1 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt idna==3.4 # via # -r requirements-tests.txt # requests imagesize==1.4.1 # via sphinx -importlib-resources==5.13.0 - # via limits +importlib-metadata==6.8.0 + # via + # -r requirements-tests.txt + # certbot + # flask + # sphinx +importlib-resources==6.1.0 + # via + # -r requirements-tests.txt + # limits inflection==0.5.1 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt iniconfig==2.0.0 # via # -r requirements-tests.txt @@ -261,7 +328,9 @@ itsdangerous==2.1.2 # -r requirements-tests.txt # flask javaobj-py3==0.4.3 - # via pyjks + # via + # -r requirements-tests.txt + # pyjks jinja2==3.1.2 # via # -r requirements-tests.txt @@ -288,7 +357,9 @@ jsondiff==2.0.0 # -r requirements-tests.txt # moto jsonlines==4.0.0 - # via cloudflare + # via + # -r requirements-tests.txt + # cloudflare jsonpatch==1.33 # via # -r requirements-tests.txt @@ -308,7 +379,7 @@ jsonschema==4.19.1 # cfn-lint # openapi-schema-validator # openapi-spec-validator -jsonschema-spec==0.2.4 +jsonschema-path==0.3.1 # via # -r requirements-tests.txt # openapi-spec-validator @@ -330,9 +401,15 @@ lazy-object-proxy==1.9.0 # -r requirements-tests.txt # openapi-spec-validator limits==3.6.0 - # via flask-limiter + # via + # -r requirements-tests.txt + # flask-limiter +lockfile==0.12.2 + # via -r requirements-tests.txt logmatic-python==0.1.7 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt mako==1.2.4 # via # -r requirements-tests.txt @@ -360,33 +437,46 @@ mdurl==0.1.2 # via # -r requirements-tests.txt # markdown-it-py -moto[all]==4.2.4 - # via -r requirements-tests.txt +moto[all]==4.2.6 + # via + # -r requirements-tests.txt + # moto mpmath==1.3.0 # via # -r requirements-tests.txt # sympy +multipart==0.2.4 + # via + # -r requirements-tests.txt + # moto +mypy==1.6.1 + # via -r requirements-tests.txt mypy-extensions==1.0.0 # via # -r requirements-tests.txt # black + # mypy +ndg-httpsclient==0.5.1 + # via -r requirements-tests.txt networkx==3.1 # via # -r requirements-tests.txt # cfn-lint nose==1.3.7 # via -r requirements-tests.txt -openapi-schema-validator==0.6.1 +openapi-schema-validator==0.6.2 # via # -r requirements-tests.txt # openapi-spec-validator -openapi-spec-validator==0.6.0 +openapi-spec-validator==0.7.1 # via # -r requirements-tests.txt # moto ordered-set==4.1.0 - # via flask-limiter -packaging==23.1 + # via + # -r requirements-tests.txt + # flask-limiter +packaging==23.2 # via # -r requirements-tests.txt # black @@ -396,7 +486,9 @@ packaging==23.1 # pytest # sphinx paramiko==3.3.1 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt parsedatetime==2.6 # via # -r requirements-tests.txt @@ -404,7 +496,7 @@ parsedatetime==2.6 pathable==0.4.3 # via # -r requirements-tests.txt - # jsonschema-spec + # jsonschema-path pathspec==0.11.2 # via # -r requirements-tests.txt @@ -416,8 +508,10 @@ pbr==5.11.1 # sarif-om # stevedore pem==23.1.0 - # via -r requirements-docs.in -platformdirs==3.10.0 + # via + # -r requirements-docs.in + # -r requirements-tests.txt +platformdirs==3.11.0 # via # -r requirements-tests.txt # black @@ -429,26 +523,33 @@ prompt-toolkit==3.0.39 # via # -r requirements-tests.txt # click-repl -py-partiql-parser==0.3.7 +psycopg2==2.9.9 + # via -r requirements-tests.txt +py-partiql-parser==0.4.0 # via # -r requirements-tests.txt # moto pyasn1==0.5.0 # via # -r requirements-tests.txt + # ndg-httpsclient # pyasn1-modules # pyjks # python-jose # rsa pyasn1-modules==0.3.0 - # via pyjks + # via + # -r requirements-tests.txt + # pyjks pycparser==2.21 # via # -r requirements-tests.txt # cffi pycryptodomex==3.19.0 - # via pyjks -pydantic==2.4.1 + # via + # -r requirements-tests.txt + # pyjks +pydantic==2.4.2 # via # -r requirements-tests.txt # aws-sam-translator @@ -464,19 +565,28 @@ pygments==2.16.1 # rich # sphinx pyhcl==0.4.5 - # via hvac + # via + # -r requirements-tests.txt + # hvac pyjks==20.0.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt pyjwt==2.8.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt pynacl==1.5.0 - # via paramiko + # via + # -r requirements-tests.txt + # paramiko pyopenssl==23.2.0 # via # -r requirements-docs.in # -r requirements-tests.txt # acme # josepy + # ndg-httpsclient pyparsing==3.1.1 # via # -r requirements-tests.txt @@ -486,6 +596,10 @@ pyrfc3339==1.1 # -r requirements-tests.txt # acme # certbot +pyspnego==0.10.2 + # via + # -r requirements-tests.txt + # requests-ntlm pytest==7.4.2 # via # -r requirements-tests.txt @@ -508,8 +622,12 @@ python-jose[cryptography]==3.3.0 # via # -r requirements-tests.txt # moto + # python-jose python-json-logger==2.0.7 - # via logmatic-python + # via + # -r requirements-tests.txt + # logmatic-python + # via -r requirements-tests.txt pytz==2023.3.post1 # via # -r requirements-tests.txt @@ -523,7 +641,7 @@ pyyaml==6.0.1 # bandit # cfn-lint # cloudflare - # jsonschema-spec + # jsonschema-path # moto # responses redis==4.6.0 @@ -536,9 +654,9 @@ referencing==0.30.2 # via # -r requirements-tests.txt # jsonschema - # jsonschema-spec + # jsonschema-path # jsonschema-specifications -regex==2023.8.8 +regex==2023.10.3 # via # -r requirements-tests.txt # cfn-lint @@ -550,29 +668,36 @@ requests==2.31.0 # cloudflare # docker # hvac - # jsonschema-spec + # jsonschema-path # moto # requests-mock + # requests-ntlm # responses # sphinx requests-mock==1.11.0 # via -r requirements-tests.txt +requests-ntlm==1.2.0 + # via + # -r requirements-tests.txt + # certsrv responses==0.23.3 # via # -r requirements-tests.txt # moto retrying==1.3.4 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt rfc3339-validator==0.1.4 # via # -r requirements-tests.txt # openapi-schema-validator -rich==13.5.3 +rich==13.6.0 # via # -r requirements-tests.txt # bandit # flask-limiter -rpds-py==0.10.3 +rpds-py==0.10.6 # via # -r requirements-tests.txt # jsonschema @@ -589,8 +714,10 @@ sarif-om==1.0.4 # via # -r requirements-tests.txt # cfn-lint -sentry-sdk==1.31.0 - # via -r requirements-docs.in +sentry-sdk==1.32.0 + # via + # -r requirements-docs.in + # -r requirements-tests.txt six==1.16.0 # via # -r requirements-tests.txt @@ -614,7 +741,9 @@ sortedcontainers==2.4.0 # -r requirements-tests.txt # fakeredis soupsieve==2.5 - # via beautifulsoup4 + # via + # -r requirements-tests.txt + # beautifulsoup4 sphinx==7.2.6 # via # -r requirements-docs.in @@ -653,7 +782,9 @@ sqlalchemy==1.3.24 # marshmallow-sqlalchemy # sqlalchemy-utils sqlalchemy-utils==0.41.1 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt sshpubkeys==3.3.1 # via # -r requirements-tests.txt @@ -667,27 +798,70 @@ sympy==1.12 # -r requirements-tests.txt # cfn-lint tabulate==0.9.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt +tomli==2.0.1 + # via + # -r requirements-tests.txt + # black + # mypy + # pytest twofish==0.3.0 - # via pyjks + # via + # -r requirements-tests.txt + # pyjks +types-deprecated==1.2.9.3 + # via -r requirements-tests.txt +types-paramiko==3.3.0.0 + # via -r requirements-tests.txt +types-pyopenssl==23.2.0.2 + # via + # -r requirements-tests.txt + # types-redis +types-pyrfc3339==1.1.1.5 + # via -r requirements-tests.txt +types-python-dateutil==2.8.19.14 + # via + # -r requirements-tests.txt + # arrow +types-pytz==2023.3.1.1 + # via -r requirements-tests.txt types-pyyaml==6.0.12.12 # via # -r requirements-tests.txt # responses +types-redis==4.6.0.7 + # via -r requirements-tests.txt +types-requests==2.31.0.6 + # via -r requirements-tests.txt +types-setuptools==68.2.0.0 + # via -r requirements-tests.txt +types-six==1.16.21.9 + # via -r requirements-tests.txt +types-tabulate==0.9.0.3 + # via -r requirements-tests.txt +types-urllib3==1.26.25.14 + # via + # -r requirements-tests.txt + # types-requests typing-extensions==4.8.0 # via # -r requirements-tests.txt # alembic # aws-sam-translator + # black # flask-limiter + # kombu # limits + # mypy # pydantic # pydantic-core tzdata==2023.3 # via # -r requirements-tests.txt # celery -urllib3==1.26.16 +urllib3==1.26.18 # via # -r requirements-tests.txt # botocore @@ -695,6 +869,8 @@ urllib3==1.26.16 # requests # responses # sentry-sdk +validators==0.22.0 + # via -r requirements-tests.txt vine==5.0.0 # via # -r requirements-docs.in @@ -702,15 +878,15 @@ vine==5.0.0 # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.8 # via # -r requirements-tests.txt # prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via # -r requirements-tests.txt # docker -werkzeug==2.3.7 +werkzeug==3.0.0 # via # -r requirements-docs.in # -r requirements-tests.txt @@ -727,6 +903,11 @@ xmltodict==0.13.0 # -r requirements-docs.in # -r requirements-tests.txt # moto +zipp==3.17.0 + # via + # -r requirements-tests.txt + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.in b/requirements-tests.in index 23616713a0..527025ee87 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -1,29 +1,30 @@ # Run `make up-reqs` to update pinned dependencies in requirement text files - +-r requirements.txt bandit black -coverage celery[redis]==5.3.4 -certbot +coverage factory-boy Faker fakeredis -flask -flask-migrate freezegun -itsdangerous -jinja2 -marshmallow-sqlalchemy == 0.23.1 #related to the marshmallow issue (to avoid conflicts) -marshmallow<2.21.1 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 -moto[all] == 4.2.4 # 3.1.2 breaks ELBv2 tests +moto[all] nose openapi-spec-validator pyflakes pytest pytest-flask pytest-mock -redis < 5.1.0 # requires a newer release of fakeredis requests-mock -sqlalchemy < 1.4.0 # ImportError: cannot import name '_ColumnEntity' https://github.com/sqlalchemy/sqlalchemy/issues/6226 -pyyaml>=4.2b1 -werkzeug \ No newline at end of file +mypy +types-tabulate +types-six +types-setuptools +types-pytz +types-pyRFC3339 +types-Deprecated +types-requests +types-pyOpenSSL +types-paramiko +types-redis +types-requests diff --git a/requirements-tests.txt b/requirements-tests.txt index 2cac46544f..67fe51a92f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,59 +1,114 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --no-emit-index-url --output-file=requirements-tests.txt requirements-tests.in # -acme==2.6.0 - # via certbot +acme==2.7.1 + # via + # -r requirements.txt + # certbot alembic==1.12.0 - # via flask-migrate + # via + # -r requirements.txt + # flask-migrate +alembic-autogenerate-enums==0.1.2 + # via -r requirements.txt amqp==5.1.1 - # via kombu -annotated-types==0.5.0 + # via + # -r requirements.txt + # kombu +aniso8601==9.0.1 + # via + # -r requirements.txt + # flask-restful +annotated-types==0.6.0 # via pydantic +arrow==1.3.0 + # via -r requirements.txt +async-timeout==4.0.3 + # via + # -r requirements.txt + # redis +asyncpool==1.0 + # via -r requirements.txt attrs==23.1.0 # via + # -r requirements.txt # jschema-to-python + # jsonlines # jsonschema # referencing # sarif-om -aws-sam-translator==1.75.0 +aws-sam-translator==1.78.0 # via cfn-lint -aws-xray-sdk==2.12.0 +aws-xray-sdk==2.12.1 # via moto bandit==1.7.5 # via -r requirements-tests.in +bcrypt==4.0.1 + # via + # -r requirements.txt + # flask-bcrypt + # paramiko +beautifulsoup4==4.12.2 + # via + # -r requirements.txt + # cloudflare billiard==4.1.0 - # via celery -black==23.9.1 + # via + # -r requirements.txt + # celery +black==23.10.0 # via -r requirements-tests.in -blinker==1.6.2 - # via flask -boto3==1.28.57 +blinker==1.6.3 + # via + # -r requirements.txt + # flask + # flask-mail + # flask-principal +boto3==1.28.66 # via + # -r requirements.txt # aws-sam-translator # moto -botocore==1.31.57 +botocore==1.31.66 # via + # -r requirements.txt # aws-xray-sdk # boto3 # moto # s3transfer celery[redis]==5.3.4 - # via -r requirements-tests.in -certbot==2.6.0 - # via -r requirements-tests.in + # via + # -r requirements-tests.in + # -r requirements.txt + # celery +certbot==2.7.1 + # via -r requirements.txt certifi==2023.7.22 - # via requests -cffi==1.15.1 - # via cryptography -cfn-lint==0.80.3 + # via + # -r requirements.txt + # requests + # sentry-sdk +certsrv[ntlm]==2.1.1 + # via + # -r requirements.txt + # certsrv +cffi==1.16.0 + # via + # -r requirements.txt + # cryptography + # pynacl +cfn-lint==0.82.2 # via moto -charset-normalizer==3.2.0 - # via requests +charset-normalizer==3.3.0 + # via + # -r requirements.txt + # requests click==8.1.7 # via + # -r requirements.txt # black # celery # click-didyoumean @@ -61,38 +116,73 @@ click==8.1.7 # click-repl # flask click-didyoumean==0.3.0 - # via celery + # via + # -r requirements.txt + # celery click-plugins==1.1.1 - # via celery + # via + # -r requirements.txt + # celery click-repl==0.3.0 - # via celery + # via + # -r requirements.txt + # celery +cloudflare==2.12.4 + # via -r requirements.txt configargparse==1.7 - # via certbot + # via + # -r requirements.txt + # certbot configobj==5.0.8 - # via certbot -coverage==7.3.1 + # via + # -r requirements.txt + # certbot +coverage==7.3.2 # via -r requirements-tests.in cryptography==41.0.4 # via + # -r requirements.txt # acme # certbot # josepy # moto + # paramiko # pyopenssl + # pyspnego # python-jose + # requests-ntlm # sshpubkeys + # types-paramiko + # types-pyopenssl + # types-redis +deprecated==1.2.14 + # via + # -r requirements.txt + # limits distro==1.8.0 - # via certbot + # via + # -r requirements.txt + # certbot +dnspython==1.15.0 + # via + # -r requirements.txt + # dnspython3 +dnspython3==1.15.0 + # via -r requirements.txt docker==6.1.3 # via moto +dyn==1.8.6 + # via -r requirements.txt ecdsa==0.18.0 # via # moto # python-jose # sshpubkeys +exceptiongroup==1.1.3 + # via pytest factory-boy==3.3.0 # via -r requirements-tests.in -faker==19.6.2 +faker==19.11.0 # via # -r requirements-tests.in # factory-boy @@ -100,47 +190,98 @@ fakeredis==2.19.0 # via -r requirements-tests.in flask==2.3.3 # via - # -r requirements-tests.in + # -r requirements.txt + # flask-bcrypt + # flask-cors + # flask-limiter + # flask-mail # flask-migrate + # flask-principal + # flask-restful # flask-sqlalchemy # pytest-flask +flask-bcrypt==1.0.1 + # via -r requirements.txt +flask-cors==4.0.0 + # via -r requirements.txt +flask-limiter==3.5.0 + # via -r requirements.txt +flask-mail==0.9.1 + # via -r requirements.txt flask-migrate==4.0.5 - # via -r requirements-tests.in + # via -r requirements.txt +flask-principal==0.4.0 + # via -r requirements.txt +flask-replicated==2.1 + # via -r requirements.txt +flask-restful==0.3.10 + # via -r requirements.txt flask-sqlalchemy==2.5.1 - # via flask-migrate + # via + # -r requirements.txt + # flask-migrate freezegun==1.2.2 # via -r requirements-tests.in +future==0.18.3 + # via -r requirements.txt gitdb==4.0.10 # via gitpython -gitpython==3.1.37 +gitpython==3.1.40 # via bandit graphql-core==3.2.3 # via moto +gunicorn==21.2.0 + # via -r requirements.txt +hvac==1.2.1 + # via -r requirements.txt idna==3.4 - # via requests + # via + # -r requirements.txt + # requests +importlib-metadata==6.8.0 + # via + # -r requirements.txt + # certbot + # flask +importlib-resources==6.1.0 + # via + # -r requirements.txt + # limits +inflection==0.5.1 + # via -r requirements.txt iniconfig==2.0.0 # via pytest itsdangerous==2.1.2 # via - # -r requirements-tests.in + # -r requirements.txt # flask +javaobj-py3==0.4.3 + # via + # -r requirements.txt + # pyjks jinja2==3.1.2 # via - # -r requirements-tests.in + # -r requirements.txt # flask # moto jmespath==1.0.1 # via + # -r requirements.txt # boto3 # botocore josepy==1.13.0 # via + # -r requirements.txt # acme # certbot jschema-to-python==1.2.3 # via cfn-lint jsondiff==2.0.0 # via moto +jsonlines==4.0.0 + # via + # -r requirements.txt + # cloudflare jsonpatch==1.33 # via cfn-lint jsonpickle==3.0.2 @@ -153,7 +294,7 @@ jsonschema==4.19.1 # cfn-lint # openapi-schema-validator # openapi-spec-validator -jsonschema-spec==0.2.4 +jsonschema-path==0.3.1 # via openapi-spec-validator jsonschema-specifications==2023.7.1 # via @@ -162,51 +303,87 @@ jsonschema-specifications==2023.7.1 junit-xml==1.9 # via cfn-lint kombu==5.3.2 - # via celery + # via + # -r requirements.txt + # celery lazy-object-proxy==1.9.0 # via openapi-spec-validator +limits==3.6.0 + # via + # -r requirements.txt + # flask-limiter +lockfile==0.12.2 + # via -r requirements.txt +logmatic-python==0.1.7 + # via -r requirements.txt mako==1.2.4 - # via alembic + # via + # -r requirements.txt + # alembic markdown-it-py==3.0.0 - # via rich + # via + # -r requirements.txt + # rich markupsafe==2.1.3 # via + # -r requirements.txt # jinja2 # mako # werkzeug marshmallow==2.21.0 # via - # -r requirements-tests.in + # -r requirements.txt # marshmallow-sqlalchemy marshmallow-sqlalchemy==0.23.1 - # via -r requirements-tests.in + # via -r requirements.txt mdurl==0.1.2 - # via markdown-it-py -moto[all]==4.2.4 + # via + # -r requirements.txt + # markdown-it-py +moto[all]==4.2.6 # via -r requirements-tests.in mpmath==1.3.0 # via sympy +multipart==0.2.4 + # via moto +mypy==1.6.1 + # via -r requirements-tests.in mypy-extensions==1.0.0 - # via black + # via + # black + # mypy +ndg-httpsclient==0.5.1 + # via -r requirements.txt networkx==3.1 # via cfn-lint nose==1.3.7 # via -r requirements-tests.in -openapi-schema-validator==0.6.1 +openapi-schema-validator==0.6.2 # via openapi-spec-validator -openapi-spec-validator==0.6.0 +openapi-spec-validator==0.7.1 # via # -r requirements-tests.in # moto -packaging==23.1 +ordered-set==4.1.0 + # via + # -r requirements.txt + # flask-limiter +packaging==23.2 # via + # -r requirements.txt # black # docker + # gunicorn + # limits # pytest +paramiko==3.3.1 + # via -r requirements.txt parsedatetime==2.6 - # via certbot + # via + # -r requirements.txt + # certbot pathable==0.4.3 - # via jsonschema-spec + # via jsonschema-path pathspec==0.11.2 # via black pbr==5.11.1 @@ -214,38 +391,81 @@ pbr==5.11.1 # jschema-to-python # sarif-om # stevedore -platformdirs==3.10.0 +pem==23.1.0 + # via -r requirements.txt +platformdirs==3.11.0 # via black pluggy==1.3.0 # via pytest prompt-toolkit==3.0.39 - # via click-repl -py-partiql-parser==0.3.7 + # via + # -r requirements.txt + # click-repl +psycopg2==2.9.9 + # via -r requirements.txt +py-partiql-parser==0.4.0 # via moto pyasn1==0.5.0 # via + # -r requirements.txt + # ndg-httpsclient + # pyasn1-modules + # pyjks # python-jose + # python-ldap # rsa +pyasn1-modules==0.3.0 + # via + # -r requirements.txt + # pyjks + # python-ldap pycparser==2.21 - # via cffi -pydantic==2.4.1 + # via + # -r requirements.txt + # cffi +pycryptodomex==3.19.0 + # via + # -r requirements.txt + # pyjks +pydantic==2.4.2 # via aws-sam-translator pydantic-core==2.10.1 # via pydantic pyflakes==3.1.0 # via -r requirements-tests.in pygments==2.16.1 - # via rich + # via + # -r requirements.txt + # rich +pyhcl==0.4.5 + # via + # -r requirements.txt + # hvac +pyjks==20.0.0 + # via -r requirements.txt +pyjwt==2.8.0 + # via -r requirements.txt +pynacl==1.5.0 + # via + # -r requirements.txt + # paramiko pyopenssl==23.2.0 # via + # -r requirements.txt # acme # josepy + # ndg-httpsclient pyparsing==3.1.1 # via moto pyrfc3339==1.1 # via + # -r requirements.txt # acme # certbot +pyspnego==0.10.2 + # via + # -r requirements.txt + # requests-ntlm pytest==7.4.2 # via # -r requirements-tests.in @@ -257,123 +477,228 @@ pytest-mock==3.11.1 # via -r requirements-tests.in python-dateutil==2.8.2 # via + # -r requirements.txt + # arrow # botocore # celery # faker # freezegun # moto python-jose[cryptography]==3.3.0 - # via moto + # via + # moto + # python-jose +python-json-logger==2.0.7 + # via + # -r requirements.txt + # logmatic-python +python-ldap==3.4.3 + # via -r requirements.txt pytz==2023.3.post1 # via + # -r requirements.txt # acme # certbot + # flask-restful # pyrfc3339 pyyaml==6.0.1 # via - # -r requirements-tests.in + # -r requirements.txt # bandit # cfn-lint - # jsonschema-spec + # cloudflare + # jsonschema-path # moto # responses redis==4.6.0 # via - # -r requirements-tests.in + # -r requirements.txt # celery # fakeredis referencing==0.30.2 # via # jsonschema - # jsonschema-spec + # jsonschema-path # jsonschema-specifications -regex==2023.8.8 +regex==2023.10.3 # via cfn-lint requests==2.31.0 # via + # -r requirements.txt # acme + # certsrv + # cloudflare # docker - # jsonschema-spec + # hvac + # jsonschema-path # moto # requests-mock + # requests-ntlm # responses requests-mock==1.11.0 # via -r requirements-tests.in +requests-ntlm==1.2.0 + # via + # -r requirements.txt + # certsrv responses==0.23.3 # via moto +retrying==1.3.4 + # via -r requirements.txt rfc3339-validator==0.1.4 # via openapi-schema-validator -rich==13.5.3 - # via bandit -rpds-py==0.10.3 +rich==13.6.0 + # via + # -r requirements.txt + # bandit + # flask-limiter +rpds-py==0.10.6 # via # jsonschema # referencing rsa==4.9 # via python-jose s3transfer==0.7.0 - # via boto3 + # via + # -r requirements.txt + # boto3 sarif-om==1.0.4 # via cfn-lint +sentry-sdk==1.32.0 + # via -r requirements.txt six==1.16.0 # via + # -r requirements.txt # configobj # ecdsa + # flask-restful # junit-xml # python-dateutil # requests-mock + # retrying # rfc3339-validator smmap==5.0.1 # via gitdb sortedcontainers==2.4.0 # via fakeredis +soupsieve==2.5 + # via + # -r requirements.txt + # beautifulsoup4 sqlalchemy==1.3.24 # via - # -r requirements-tests.in + # -r requirements.txt # alembic # flask-sqlalchemy # marshmallow-sqlalchemy + # sqlalchemy-utils +sqlalchemy-utils==0.41.1 + # via -r requirements.txt sshpubkeys==3.3.1 # via moto stevedore==5.1.0 # via bandit sympy==1.12 # via cfn-lint +tabulate==0.9.0 + # via -r requirements.txt +tomli==2.0.1 + # via + # black + # mypy + # pytest +twofish==0.3.0 + # via + # -r requirements.txt + # pyjks +types-deprecated==1.2.9.3 + # via -r requirements-tests.in +types-paramiko==3.3.0.0 + # via -r requirements-tests.in +types-pyopenssl==23.2.0.2 + # via + # -r requirements-tests.in + # types-redis +types-pyrfc3339==1.1.1.5 + # via -r requirements-tests.in +types-python-dateutil==2.8.19.14 + # via + # -r requirements.txt + # arrow +types-pytz==2023.3.1.1 + # via -r requirements-tests.in types-pyyaml==6.0.12.12 # via responses +types-redis==4.6.0.7 + # via -r requirements-tests.in +types-requests==2.31.0.6 + # via -r requirements-tests.in +types-setuptools==68.2.0.0 + # via -r requirements-tests.in +types-six==1.16.21.9 + # via -r requirements-tests.in +types-tabulate==0.9.0.3 + # via -r requirements-tests.in +types-urllib3==1.26.25.14 + # via types-requests typing-extensions==4.8.0 # via + # -r requirements.txt # alembic # aws-sam-translator + # black + # flask-limiter + # kombu + # limits + # mypy # pydantic # pydantic-core tzdata==2023.3 - # via celery -urllib3==1.26.16 # via + # -r requirements.txt + # celery +urllib3==1.26.18 + # via + # -r requirements.txt # botocore # docker # requests # responses + # sentry-sdk +validators==0.22.0 + # via -r requirements.txt vine==5.0.0 # via + # -r requirements.txt # amqp # celery # kombu -wcwidth==0.2.6 - # via prompt-toolkit -websocket-client==1.6.3 +wcwidth==0.2.8 + # via + # -r requirements.txt + # prompt-toolkit +websocket-client==1.6.4 # via docker -werkzeug==2.3.7 +werkzeug==3.0.0 # via - # -r requirements-tests.in + # -r requirements.txt # flask # moto # pytest-flask wrapt==1.15.0 - # via aws-xray-sdk + # via + # -r requirements.txt + # aws-xray-sdk + # deprecated xmltodict==0.13.0 - # via moto + # via + # -r requirements.txt + # moto +zipp==3.17.0 + # via + # -r requirements.txt + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.in b/requirements.in index 8d3f17db75..34bd71ae0f 100644 --- a/requirements.in +++ b/requirements.in @@ -21,7 +21,7 @@ Flask-Migrate Flask-Principal Flask-RESTful Flask-SQLAlchemy -Flask +Flask<3 # until https://github.com/pytest-dev/pytest-flask/pull/168 is released Flask-Cors flask_replicated future @@ -38,19 +38,19 @@ ndg-httpsclient paramiko # required for the SFTP destination plugin pem psycopg2 -pyjks >= 19 # pyjks < 19 depends on pycryptodome, which conflicts with dyn's usage of pycrypto +pyjks pyjwt pyOpenSSL -pyyaml>=4.2b1 #high severity alert +pyyaml python_ldap -redis < 5.1.0 # requires a newer release of fakeredis +redis requests retrying sentry-sdk six SQLAlchemy-Utils -sqlalchemy < 1.4.0 # ImportError: cannot import name '_ColumnEntity' https://github.com/sqlalchemy/sqlalchemy/issues/6226 +sqlalchemy < 1.4.0 # ImportError: cannot import name '_ColumnEntity' https://github.com/sqlalchemy/sqlalchemy/issues/6226tabulate tabulate validators werkzeug -xmltodict +xmltodict \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index bf6e145af0..322df18e61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --no-emit-index-url --output-file=requirements.txt requirements.in # -acme==2.6.0 +acme==2.7.1 # via # -r requirements.in # certbot @@ -16,8 +16,10 @@ amqp==5.1.1 # via kombu aniso8601==9.0.1 # via flask-restful -arrow==1.2.3 +arrow==1.3.0 # via -r requirements.in +async-timeout==4.0.3 + # via redis asyncpool==1.0 # via -r requirements.in attrs==23.1.0 @@ -30,21 +32,21 @@ beautifulsoup4==4.12.2 # via cloudflare billiard==4.1.0 # via celery -blinker==1.6.2 +blinker==1.6.3 # via # flask # flask-mail # flask-principal -boto3==1.28.57 +boto3==1.28.66 # via -r requirements.in -botocore==1.31.57 +botocore==1.31.66 # via # -r requirements.in # boto3 # s3transfer celery[redis]==5.3.4 # via -r requirements.in -certbot==2.6.0 +certbot==2.7.1 # via -r requirements.in certifi==2023.7.22 # via @@ -53,11 +55,11 @@ certifi==2023.7.22 # sentry-sdk certsrv[ntlm]==2.1.1 # via -r requirements.in -cffi==1.15.1 +cffi==1.16.0 # via # cryptography # pynacl -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 # via requests click==8.1.7 # via @@ -137,7 +139,11 @@ hvac==1.2.1 # via -r requirements.in idna==3.4 # via requests -importlib-resources==5.13.0 +importlib-metadata==6.8.0 + # via + # certbot + # flask +importlib-resources==6.1.0 # via limits inflection==0.5.1 # via -r requirements.in @@ -190,7 +196,7 @@ ndg-httpsclient==0.5.1 # via -r requirements.in ordered-set==4.1.0 # via flask-limiter -packaging==23.1 +packaging==23.2 # via # gunicorn # limits @@ -202,7 +208,7 @@ pem==23.1.0 # via -r requirements.in prompt-toolkit==3.0.39 # via click-repl -psycopg2==2.9.7 +psycopg2==2.9.9 # via -r requirements.in pyasn1==0.5.0 # via @@ -238,7 +244,7 @@ pyrfc3339==1.1 # via # acme # certbot -pyspnego==0.9.2 +pyspnego==0.10.2 # via requests-ntlm python-dateutil==2.8.2 # via @@ -275,11 +281,11 @@ requests-ntlm==1.2.0 # via certsrv retrying==1.3.4 # via -r requirements.in -rich==13.5.3 +rich==13.6.0 # via flask-limiter s3transfer==0.7.0 # via boto3 -sentry-sdk==1.31.0 +sentry-sdk==1.32.0 # via -r requirements.in six==1.16.0 # via @@ -303,14 +309,17 @@ tabulate==0.9.0 # via -r requirements.in twofish==0.3.0 # via pyjks +types-python-dateutil==2.8.19.14 + # via arrow typing-extensions==4.8.0 # via # alembic # flask-limiter + # kombu # limits tzdata==2023.3 # via celery -urllib3==1.26.16 +urllib3==1.26.18 # via # botocore # requests @@ -322,9 +331,9 @@ vine==5.0.0 # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.8 # via prompt-toolkit -werkzeug==2.3.7 +werkzeug==3.0.0 # via # -r requirements.in # flask @@ -332,6 +341,10 @@ wrapt==1.15.0 # via deprecated xmltodict==0.13.0 # via -r requirements.in +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.py b/setup.py index 970a1d6154..d2294af3ec 100644 --- a/setup.py +++ b/setup.py @@ -168,9 +168,9 @@ def run(self): 'Intended Audience :: System Administrators', 'Operating System :: OS Independent', 'Topic :: Software Development', - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Natural Language :: English", "License :: OSI Approved :: Apache Software License" ] diff --git a/tox.ini b/tox.ini index 34121109c9..c10ac20897 100644 --- a/tox.ini +++ b/tox.ini @@ -1,2 +1,2 @@ [tox] -envlist = py38,py39,py310 +envlist = py39,py310,py311