From a2ead067fa4c5d2edc922345f4c811898b490c2e Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Wed, 15 Jan 2025 16:08:41 -0500 Subject: [PATCH] refactor: Clean up production.py cruft (without changing any settings) TODO more details --- lms/envs/common.py | 28 ++- lms/envs/production.py | 531 +++++++++++------------------------------ 2 files changed, 164 insertions(+), 395 deletions(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index cb7643c3668e..23e0d12b3dde 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -3392,9 +3392,35 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring CSRF_COOKIE_SECURE = False CSRF_TRUSTED_ORIGINS = [] CSRF_TRUSTED_ORIGINS_WITH_SCHEME = [] -CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = '' + +# If setting a cross-domain cookie, it's really important to choose +# a name for the cookie that is DIFFERENT than the cookies used +# by each subdomain. For example, suppose the applications +# at these subdomains are configured to use the following cookie names: +# +# 1) foo.example.com --> "csrftoken" +# 2) baz.example.com --> "csrftoken" +# 3) bar.example.com --> "csrftoken" +# +# For the cross-domain version of the CSRF cookie, you need to choose +# a name DIFFERENT than "csrftoken"; otherwise, the new token configured +# for ".example.com" could conflict with the other cookies, +# non-deterministically causing 403 responses. CROSS_DOMAIN_CSRF_COOKIE_NAME = '' +# When setting the domain for the "cross-domain" version of the CSRF +# cookie, you should choose something like: ".example.com" +# (note the leading dot), where both the referer and the host +# are subdomains of "example.com". +# +# Browser security rules require that +# the cookie domain matches the domain of the server; otherwise +# the cookie won't get set. And once the cookie gets set, the client +# needs to be on a domain that matches the cookie domain, otherwise +# the client won't be able to read the cookie. +CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = '' + + ######################### Django Rest Framework ######################## REST_FRAMEWORK = { diff --git a/lms/envs/production.py b/lms/envs/production.py index 6dc6be634178..e675078eb603 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -44,7 +44,8 @@ def get_env_setting(setting): error_msg = "Set the %s env variable" % setting raise ImproperlyConfigured(error_msg) # lint-amnesty, pylint: disable=raise-missing-from -################################ ALWAYS THE SAME ############################## + +######################### PRODUCTION DEFAULTS ############################## DEBUG = False DEFAULT_TEMPLATE_ENGINE['OPTIONS']['debug'] = False @@ -58,7 +59,69 @@ def get_env_setting(setting): # https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header # for other warnings. SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -################################ END ALWAYS THE SAME ############################## + +###################################### MOVED UP ################################ + +CELERY_RESULT_BACKEND = 'django-cache' +BROKER_HEARTBEAT = 60.0 +BROKER_HEARTBEAT_CHECKRATE = 2 +STATIC_ROOT_BASE = None +STATIC_URL_BASE = ENV_TOKENS.get('STATIC_URL_BASE', None) +EMAIL_FILE_PATH = DATA_DIR / "emails" / "lms" +EMAIL_HOST = 'localhost' +EMAIL_PORT = 25 +EMAIL_USE_TLS = False +SESSION_COOKIE_DOMAIN = None +SESSION_COOKIE_HTTPONLY = True +AWS_SES_REGION_NAME = 'us-east-1' +AWS_SES_REGION_ENDPOINT = 'email.us-east-1.amazonaws.com' +REGISTRATION_EMAIL_PATTERNS_ALLOWED = None +LMS_ROOT_URL = None +CMS_BASE = 'studio.edx.org' +CELERY_EVENT_QUEUE_TTL = None +# COMPREHENSIVE_THEME_LOCALE_PATHS contain the paths to themes locale directories e.g. +# "COMPREHENSIVE_THEME_LOCALE_PATHS" : [ +# "/edx/src/edx-themes/conf/locale" +# ], +COMPREHENSIVE_THEME_LOCALE_PATHS = ENV_TOKENS.get('COMPREHENSIVE_THEME_LOCALE_PATHS', []) +# PREPEND_LOCALE_PATHS contain the paths to locale directories to load first e.g. +# "PREPEND_LOCALE_PATHS" : [ +# "/edx/my-locale" +# ], +PREPEND_LOCALE_PATHS = ENV_TOKENS.get('PREPEND_LOCALE_PATHS', []) +COURSE_LISTINGS = {} +COMMENTS_SERVICE_URL = '' +COMMENTS_SERVICE_KEY = '' +CERT_QUEUE = 'test-pull' +PYTHON_LIB_FILENAME = 'python_lib.zip' +VIDEO_CDN_URL = {} +HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS = {} +AWS_STORAGE_BUCKET_NAME = 'edxuploads' +# Disabling querystring auth instructs Boto to exclude the querystring parameters (e.g. signature, access key) it +# normally appends to every returned URL. +AWS_QUERYSTRING_AUTH = True +AWS_S3_CUSTOM_DOMAIN = 'edxuploads.s3.amazonaws.com' +MONGODB_LOG = {} +ZENDESK_USER = None +ZENDESK_API_KEY = None +EDX_API_KEY = None +CELERY_BROKER_TRANSPORT = "" +CELERY_BROKER_HOSTNAME = "" +CELERY_BROKER_VHOST = "" +CELERY_BROKER_USER = "" +CELERY_BROKER_PASSWORD = "" +BROKER_USE_SSL = False +SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = None +ENABLE_REQUIRE_THIRD_PARTY_AUTH = False + +# TODO make a note about axing these +# SSL external authentication settings +SSL_AUTH_EMAIL_DOMAIN = "MIT.EDU" +SSL_AUTH_DN_FORMAT_STRING = ( + "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}" +) + +########################################################################### # A file path to a YAML file from which to load all the configuration for the edx platform CONFIG_FILE = get_env_setting('LMS_CFG') @@ -71,6 +134,11 @@ def get_env_setting(setting): ENV_TOKENS = __config__ AUTH_TOKENS = __config__ + # _YAML_TOKENS is used strategically to handle YAML config files in a backwards-compatible way. + # Please do not add more references to it. + # See: < TODO TICKET LINK > + _YAML_TOKENS = __config__ + # Add the key/values from config into the global namespace of this module. # But don't override the FEATURES dict because we do that in an additive way. __config_copy__ = copy.deepcopy(__config__) @@ -82,7 +150,6 @@ def get_env_setting(setting): 'JWT_AUTH', 'CELERY_QUEUES', 'MKTG_URL_LINK_MAP', - 'MKTG_URL_OVERRIDES', 'REST_FRAMEWORK', 'EVENT_BUS_PRODUCER_CONFIG', ] @@ -111,20 +178,11 @@ def get_env_setting(setting): BROKER_POOL_LIMIT = 0 BROKER_CONNECTION_TIMEOUT = 1 -# Allow env to configure celery result backend with default set to django-cache -CELERY_RESULT_BACKEND = ENV_TOKENS.get('CELERY_RESULT_BACKEND', 'django-cache') - -# When the broker is behind an ELB, use a heartbeat to refresh the -# connection and to detect if it has been dropped. -BROKER_HEARTBEAT = ENV_TOKENS.get('BROKER_HEARTBEAT', 60.0) -BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2) - # Each worker should only fetch one message at a time CELERYD_PREFETCH_MULTIPLIER = 1 # STATIC_ROOT specifies the directory where static files are # collected -STATIC_ROOT_BASE = ENV_TOKENS.get('STATIC_ROOT_BASE', None) if STATIC_ROOT_BASE: STATIC_ROOT = path(STATIC_ROOT_BASE) WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" @@ -132,84 +190,33 @@ def get_env_setting(setting): # STATIC_URL_BASE specifies the base url to use for static files -STATIC_URL_BASE = ENV_TOKENS.get('STATIC_URL_BASE', None) if STATIC_URL_BASE: STATIC_URL = STATIC_URL_BASE if not STATIC_URL.endswith("/"): STATIC_URL += "/" -# Allow overriding build profile used by RequireJS with one -# contained on a custom theme -REQUIRE_BUILD_PROFILE = ENV_TOKENS.get('REQUIRE_BUILD_PROFILE', REQUIRE_BUILD_PROFILE) - -# The following variables use (or) instead of the default value inside (get). This is to enforce using the Lazy Text -# values when the variable is an empty string. Therefore, setting these variable as empty text in related -# json files will make the system reads their values from django translation files -PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME') or PLATFORM_NAME -PLATFORM_DESCRIPTION = ENV_TOKENS.get('PLATFORM_DESCRIPTION') or PLATFORM_DESCRIPTION - -DATA_DIR = path(ENV_TOKENS.get('DATA_DIR', DATA_DIR)) -CC_MERCHANT_NAME = ENV_TOKENS.get('CC_MERCHANT_NAME', PLATFORM_NAME) -EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', DATA_DIR / "emails" / "lms") -EMAIL_HOST = ENV_TOKENS.get('EMAIL_HOST', 'localhost') # django default is localhost -EMAIL_PORT = ENV_TOKENS.get('EMAIL_PORT', 25) # django default is 25 -EMAIL_USE_TLS = ENV_TOKENS.get('EMAIL_USE_TLS', False) # django default is False -SITE_NAME = ENV_TOKENS.get('SITE_NAME', SITE_NAME) -SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN') -SESSION_COOKIE_HTTPONLY = ENV_TOKENS.get('SESSION_COOKIE_HTTPONLY', True) - -DCS_SESSION_COOKIE_SAMESITE = ENV_TOKENS.get('DCS_SESSION_COOKIE_SAMESITE', DCS_SESSION_COOKIE_SAMESITE) -DCS_SESSION_COOKIE_SAMESITE_FORCE_ALL = ENV_TOKENS.get('DCS_SESSION_COOKIE_SAMESITE_FORCE_ALL', DCS_SESSION_COOKIE_SAMESITE_FORCE_ALL) # lint-amnesty, pylint: disable=line-too-long - -# As django-cookies-samesite package is set to be removed from base requirements when we upgrade to Django 3.2, -# we should follow the settings name provided by Django. -# https://docs.djangoproject.com/en/3.2/ref/settings/#session-cookie-samesite -SESSION_COOKIE_SAMESITE = DCS_SESSION_COOKIE_SAMESITE - -AWS_SES_REGION_NAME = ENV_TOKENS.get('AWS_SES_REGION_NAME', 'us-east-1') -AWS_SES_REGION_ENDPOINT = ENV_TOKENS.get('AWS_SES_REGION_ENDPOINT', 'email.us-east-1.amazonaws.com') +DATA_DIR = path(DATA_DIR) +CC_MERCHANT_NAME = _YAML_TOKENS.get('CC_MERCHANT_NAME', PLATFORM_NAME) -REGISTRATION_EMAIL_PATTERNS_ALLOWED = ENV_TOKENS.get('REGISTRATION_EMAIL_PATTERNS_ALLOWED') - -LMS_ROOT_URL = ENV_TOKENS.get('LMS_ROOT_URL') -LMS_INTERNAL_ROOT_URL = ENV_TOKENS.get('LMS_INTERNAL_ROOT_URL', LMS_ROOT_URL) +# TODO comment about this being for backcompat with yaml +SESSION_COOKIE_SAMESITE = DCS_SESSION_COOKIE_SAMESITE -# List of logout URIs for each IDA that the learner should be logged out of when they logout of the LMS. Only applies to -# IDA for which the social auth flow uses DOT (Django OAuth Toolkit). -IDA_LOGOUT_URI_LIST = ENV_TOKENS.get('IDA_LOGOUT_URI_LIST', []) +LMS_INTERNAL_ROOT_URL = _YAML_TOKENS.get('LMS_INTERNAL_ROOT_URL', LMS_ROOT_URL) -ENV_FEATURES = ENV_TOKENS.get('FEATURES', {}) -for feature, value in ENV_FEATURES.items(): +for feature, value in _YAML_TOKENS.get('FEATURES', {}).items(): FEATURES[feature] = value -CMS_BASE = ENV_TOKENS.get('CMS_BASE', 'studio.edx.org') - ALLOWED_HOSTS = [ # TODO: bbeggs remove this before prod, temp fix to get load testing running "*", - ENV_TOKENS.get('LMS_BASE'), + _YAML_TOKENS.get('LMS_BASE'), FEATURES['PREVIEW_LMS_BASE'], ] -# Sometimes, OAuth2 clients want the user to redirect back to their site after logout. But to determine if the given -# redirect URL/path is safe for redirection, the following variable is used by edX. -LOGIN_REDIRECT_WHITELIST = ENV_TOKENS.get( - 'LOGIN_REDIRECT_WHITELIST', - LOGIN_REDIRECT_WHITELIST -) - -# allow for environments to specify what cookie name our login subsystem should use -# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can -# happen with some browsers (e.g. Firefox) -if ENV_TOKENS.get('SESSION_COOKIE_NAME', None): - # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str() - SESSION_COOKIE_NAME = str(ENV_TOKENS.get('SESSION_COOKIE_NAME')) - # This is the domain that is used to set shared cookies between various sub-domains. # By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN, but we want to make it overrideable. -SHARED_COOKIE_DOMAIN = ENV_TOKENS.get('SHARED_COOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) +SHARED_COOKIE_DOMAIN = _YAML_TOKENS.get('SHARED_COOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) -CACHES = ENV_TOKENS.get('CACHES', CACHES) # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: @@ -225,206 +232,80 @@ def get_env_setting(setting): # we need to run asset collection twice, once for local disk and once for S3. # Once we have migrated to service assets off S3, then we can convert this back to # managed by the yaml file contents -STATICFILES_STORAGE = os.environ.get('STATICFILES_STORAGE', ENV_TOKENS.get('STATICFILES_STORAGE', STATICFILES_STORAGE)) - -# Load all AWS_ prefixed variables to allow an S3Boto3Storage to be configured -_locals = locals() -for key, value in ENV_TOKENS.items(): - if key.startswith('AWS_'): - _locals[key] = value - -# Currency -PAID_COURSE_REGISTRATION_CURRENCY = ENV_TOKENS.get('PAID_COURSE_REGISTRATION_CURRENCY', - PAID_COURSE_REGISTRATION_CURRENCY) # We want Bulk Email running on the high-priority queue, so we define the # routing key that points to it. At the moment, the name is the same. # We have to reset the value here, since we have changed the value of the queue name. -BULK_EMAIL_ROUTING_KEY = ENV_TOKENS.get('BULK_EMAIL_ROUTING_KEY', HIGH_PRIORITY_QUEUE) +BULK_EMAIL_ROUTING_KEY = _YAML_TOKENS.get('BULK_EMAIL_ROUTING_KEY', HIGH_PRIORITY_QUEUE) # We can run smaller jobs on the low priority queue. See note above for why # we have to reset the value here. -BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = ENV_TOKENS.get('BULK_EMAIL_ROUTING_KEY_SMALL_JOBS', DEFAULT_PRIORITY_QUEUE) +BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = _YAML_TOKENS.get('BULK_EMAIL_ROUTING_KEY_SMALL_JOBS', DEFAULT_PRIORITY_QUEUE) # Queue to use for expiring old entitlements -ENTITLEMENTS_EXPIRATION_ROUTING_KEY = ENV_TOKENS.get('ENTITLEMENTS_EXPIRATION_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) - -# Message expiry time in seconds -CELERY_EVENT_QUEUE_TTL = ENV_TOKENS.get('CELERY_EVENT_QUEUE_TTL', None) +ENTITLEMENTS_EXPIRATION_ROUTING_KEY = _YAML_TOKENS.get('ENTITLEMENTS_EXPIRATION_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) # Allow CELERY_QUEUES to be overwritten by ENV_TOKENS, -ENV_CELERY_QUEUES = ENV_TOKENS.get('CELERY_QUEUES', None) -if ENV_CELERY_QUEUES: - CELERY_QUEUES = {queue: {} for queue in ENV_CELERY_QUEUES} +_YAML_CELERY_QUEUES = _YAML_TOKENS.get('CELERY_QUEUES', None) +if _YAML_CELERY_QUEUES: + CELERY_QUEUES = {queue: {} for queue in _YAML_CELERY_QUEUES} # Then add alternate environment queues -ALTERNATE_QUEUE_ENVS = ENV_TOKENS.get('ALTERNATE_WORKER_QUEUES', '').split() +_YAML_ALTERNATE_WORKER_QUEUES = _YAML_TOKENS.get('ALTERNATE_WORKER_QUEUES', '').split() ALTERNATE_QUEUES = [ DEFAULT_PRIORITY_QUEUE.replace(QUEUE_VARIANT, alternate + '.') - for alternate in ALTERNATE_QUEUE_ENVS + for alternate in _YAML_ALTERNATE_WORKER_QUEUES ] + CELERY_QUEUES.update( { alternate: {} for alternate in ALTERNATE_QUEUES - if alternate not in list(CELERY_QUEUES.keys()) + if alternate not in CELERY_QUEUES.keys() } ) -# following setting is for backward compatibility -if ENV_TOKENS.get('COMPREHENSIVE_THEME_DIR', None): - COMPREHENSIVE_THEME_DIR = ENV_TOKENS.get('COMPREHENSIVE_THEME_DIR') - -# COMPREHENSIVE_THEME_LOCALE_PATHS contain the paths to themes locale directories e.g. -# "COMPREHENSIVE_THEME_LOCALE_PATHS" : [ -# "/edx/src/edx-themes/conf/locale" -# ], -COMPREHENSIVE_THEME_LOCALE_PATHS = ENV_TOKENS.get('COMPREHENSIVE_THEME_LOCALE_PATHS', []) - - -# PREPEND_LOCALE_PATHS contain the paths to locale directories to load first e.g. -# "PREPEND_LOCALE_PATHS" : [ -# "/edx/my-locale" -# ], -PREPEND_LOCALE_PATHS = ENV_TOKENS.get('PREPEND_LOCALE_PATHS', []) - - -MKTG_URL_LINK_MAP.update(ENV_TOKENS.get('MKTG_URL_LINK_MAP', {})) -ENTERPRISE_MARKETING_FOOTER_QUERY_PARAMS = ENV_TOKENS.get( - 'ENTERPRISE_MARKETING_FOOTER_QUERY_PARAMS', - ENTERPRISE_MARKETING_FOOTER_QUERY_PARAMS -) -# Marketing link overrides -MKTG_URL_OVERRIDES.update(ENV_TOKENS.get('MKTG_URL_OVERRIDES', MKTG_URL_OVERRIDES)) +MKTG_URL_LINK_MAP.update(_YAML_TOKENS.get('MKTG_URL_LINK_MAP', {})) # Intentional defaults. -ID_VERIFICATION_SUPPORT_LINK = ENV_TOKENS.get('ID_VERIFICATION_SUPPORT_LINK', SUPPORT_SITE_LINK) -PASSWORD_RESET_SUPPORT_LINK = ENV_TOKENS.get('PASSWORD_RESET_SUPPORT_LINK', SUPPORT_SITE_LINK) -ACTIVATION_EMAIL_SUPPORT_LINK = ENV_TOKENS.get('ACTIVATION_EMAIL_SUPPORT_LINK', SUPPORT_SITE_LINK) -LOGIN_ISSUE_SUPPORT_LINK = ENV_TOKENS.get('LOGIN_ISSUE_SUPPORT_LINK', SUPPORT_SITE_LINK) +ID_VERIFICATION_SUPPORT_LINK = YAML_TOKENS.get('ID_VERIFICATION_SUPPORT_LINK', SUPPORT_SITE_LINK) +PASSWORD_RESET_SUPPORT_LINK = _YAML_TOKENS.get('PASSWORD_RESET_SUPPORT_LINK', SUPPORT_SITE_LINK) +ACTIVATION_EMAIL_SUPPORT_LINK = _YAML_TOKENS.get('ACTIVATION_EMAIL_SUPPORT_LINK', SUPPORT_SITE_LINK) +LOGIN_ISSUE_SUPPORT_LINK = _YAML_TOKENS.get('LOGIN_ISSUE_SUPPORT_LINK', SUPPORT_SITE_LINK) # Timezone overrides -TIME_ZONE = ENV_TOKENS.get('CELERY_TIMEZONE', CELERY_TIMEZONE) +TIME_ZONE = CELERY_TIMEZONE # Translation overrides LANGUAGE_DICT = dict(LANGUAGES) -LANGUAGE_COOKIE_NAME = ENV_TOKENS.get('LANGUAGE_COOKIE', None) or ENV_TOKENS.get( - 'LANGUAGE_COOKIE_NAME', LANGUAGE_COOKIE_NAME) +# TODO add a comment about LANGUAGE_COOKIE being deprecated / add to notes +LANGUAGE_COOKIE_NAME = _YAML_TOKENS.get('LANGUAGE_COOKIE') or LANGUAGE_COOKIE_NAME # Additional installed apps -for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): +for app in _YAML_TOKENS.get('ADDL_INSTALLED_APPS', []): INSTALLED_APPS.append(app) - -local_loglevel = ENV_TOKENS.get('LOCAL_LOGLEVEL', 'INFO') -LOG_DIR = ENV_TOKENS.get('LOG_DIR', LOG_DIR) - -LOGGING = get_logger_config(LOG_DIR, - logging_env=ENV_TOKENS.get('LOGGING_ENV', LOGGING_ENV), - local_loglevel=local_loglevel, - service_variant=SERVICE_VARIANT) - -COURSE_LISTINGS = ENV_TOKENS.get('COURSE_LISTINGS', {}) -COMMENTS_SERVICE_URL = ENV_TOKENS.get("COMMENTS_SERVICE_URL", '') -COMMENTS_SERVICE_KEY = ENV_TOKENS.get("COMMENTS_SERVICE_KEY", '') -CERT_QUEUE = ENV_TOKENS.get("CERT_QUEUE", 'test-pull') - -# Python lib settings -PYTHON_LIB_FILENAME = ENV_TOKENS.get('PYTHON_LIB_FILENAME', 'python_lib.zip') - -# Code jail settings -for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): - oldvalue = CODE_JAIL.get(name) - if isinstance(oldvalue, dict): - for subname, subvalue in value.items(): - oldvalue[subname] = subvalue - else: - CODE_JAIL[name] = value - -COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", []) - -# Event Tracking -if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS: - TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS") - -# SSL external authentication settings -SSL_AUTH_EMAIL_DOMAIN = ENV_TOKENS.get("SSL_AUTH_EMAIL_DOMAIN", "MIT.EDU") -SSL_AUTH_DN_FORMAT_STRING = ENV_TOKENS.get( - "SSL_AUTH_DN_FORMAT_STRING", - "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}" +LOGGING = get_logger_config( + LOG_DIR, + logging_env=LOGGING_ENV, + local_loglevel=LOCAL_LOGLEVEL, + service_variant=SERVICE_VARIANT, ) -# Video Caching. Pairing country codes with CDN URLs. -# Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='} -VIDEO_CDN_URL = ENV_TOKENS.get('VIDEO_CDN_URL', {}) - -# Determines whether the CSRF token can be transported on -# unencrypted channels. It is set to False here for backward compatibility, -# but it is highly recommended that this is True for environments accessed -# by end users. -CSRF_COOKIE_SECURE = ENV_TOKENS.get('CSRF_COOKIE_SECURE', False) - # Determines which origins are trusted for unsafe requests eg. POST requests. -CSRF_TRUSTED_ORIGINS = ENV_TOKENS.get('CSRF_TRUSTED_ORIGINS', []) -# values are already updated above with default CSRF_TRUSTED_ORIGINS values but in -# case of new django version these values will override. -if django.VERSION[0] >= 4: # for greater than django 3.2 use schemes. - CSRF_TRUSTED_ORIGINS = ENV_TOKENS.get('CSRF_TRUSTED_ORIGINS_WITH_SCHEME', []) +CSRF_TRUSTED_ORIGINS = _YAML_TOKENS.get('CSRF_TRUSTED_ORIGINS_WITH_SCHEME', []) -############# CORS headers for cross-domain requests ################# - -if FEATURES.get('ENABLE_CORS_HEADERS') or FEATURES.get('ENABLE_CROSS_DOMAIN_CSRF_COOKIE'): +if FEATURES['ENABLE_CORS_HEADERS'] or FEATURES.get('ENABLE_CROSS_DOMAIN_CSRF_COOKIE'): CORS_ALLOW_CREDENTIALS = True - CORS_ORIGIN_WHITELIST = ENV_TOKENS.get('CORS_ORIGIN_WHITELIST', ()) - - CORS_ORIGIN_ALLOW_ALL = ENV_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) - CORS_ALLOW_INSECURE = ENV_TOKENS.get('CORS_ALLOW_INSECURE', False) - - # If setting a cross-domain cookie, it's really important to choose - # a name for the cookie that is DIFFERENT than the cookies used - # by each subdomain. For example, suppose the applications - # at these subdomains are configured to use the following cookie names: - # - # 1) foo.example.com --> "csrftoken" - # 2) baz.example.com --> "csrftoken" - # 3) bar.example.com --> "csrftoken" - # - # For the cross-domain version of the CSRF cookie, you need to choose - # a name DIFFERENT than "csrftoken"; otherwise, the new token configured - # for ".example.com" could conflict with the other cookies, - # non-deterministically causing 403 responses. - # - # Because of the way Django stores cookies, the cookie name MUST - # be a `str`, not unicode. Otherwise there will `TypeError`s will be raised - # when Django tries to call the unicode `translate()` method with the wrong - # number of parameters. - CROSS_DOMAIN_CSRF_COOKIE_NAME = str(ENV_TOKENS.get('CROSS_DOMAIN_CSRF_COOKIE_NAME')) - - # When setting the domain for the "cross-domain" version of the CSRF - # cookie, you should choose something like: ".example.com" - # (note the leading dot), where both the referer and the host - # are subdomains of "example.com". - # - # Browser security rules require that - # the cookie domain matches the domain of the server; otherwise - # the cookie won't get set. And once the cookie gets set, the client - # needs to be on a domain that matches the cookie domain, otherwise - # the client won't be able to read the cookie. + CORS_ORIGIN_WHITELIST = _YAML_TOKENS.get('CORS_ORIGIN_WHITELIST', ()) + CORS_ORIGIN_ALLOW_ALL = YAML_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) + CORS_ALLOW_INSECURE = _YAML_TOKENS.get('CORS_ALLOW_INSECURE', False) CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = ENV_TOKENS.get('CROSS_DOMAIN_CSRF_COOKIE_DOMAIN') +FIELD_OVERRIDE_PROVIDERS = tuple(FIELD_OVERRIDE_PROVIDERS) -# Field overrides. To use the IDDE feature, add -# 'courseware.student_field_overrides.IndividualStudentOverrideProvider'. -FIELD_OVERRIDE_PROVIDERS = tuple(ENV_TOKENS.get('FIELD_OVERRIDE_PROVIDERS', [])) - -############### XBlock filesystem field config ########## -if 'DJFS' in AUTH_TOKENS and AUTH_TOKENS['DJFS'] is not None: - DJFS = AUTH_TOKENS['DJFS'] - -############### Module Store Items ########## -HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS = ENV_TOKENS.get('HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS', {}) # PREVIEW DOMAIN must be present in HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS for the preview to show draft changes if 'PREVIEW_LMS_BASE' in FEATURES and FEATURES['PREVIEW_LMS_BASE'] != '': PREVIEW_DOMAIN = FEATURES['PREVIEW_LMS_BASE'].split(':')[0] @@ -433,26 +314,11 @@ def get_env_setting(setting): PREVIEW_DOMAIN: 'draft-preferred' }) -MODULESTORE_FIELD_OVERRIDE_PROVIDERS = ENV_TOKENS.get( - 'MODULESTORE_FIELD_OVERRIDE_PROVIDERS', - MODULESTORE_FIELD_OVERRIDE_PROVIDERS -) - -XBLOCK_FIELD_DATA_WRAPPERS = ENV_TOKENS.get( - 'XBLOCK_FIELD_DATA_WRAPPERS', - XBLOCK_FIELD_DATA_WRAPPERS -) - ############### Mixed Related(Secure/Not-Secure) Items ########## -LMS_SEGMENT_KEY = AUTH_TOKENS.get('SEGMENT_KEY') - -SECRET_KEY = AUTH_TOKENS['SECRET_KEY'] +LMS_SEGMENT_KEY = _YAML_TOKENS.get('SEGMENT_KEY') -AWS_ACCESS_KEY_ID = AUTH_TOKENS.get("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID) if AWS_ACCESS_KEY_ID == "": AWS_ACCESS_KEY_ID = None - -AWS_SECRET_ACCESS_KEY = AUTH_TOKENS.get("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY) if AWS_SECRET_ACCESS_KEY == "": AWS_SECRET_ACCESS_KEY = None @@ -461,24 +327,10 @@ def get_env_setting(setting): # same with upcoming version setting it to `public-read`. AWS_DEFAULT_ACL = 'public-read' AWS_BUCKET_ACL = AWS_DEFAULT_ACL -AWS_STORAGE_BUCKET_NAME = AUTH_TOKENS.get('AWS_STORAGE_BUCKET_NAME', 'edxuploads') -# Disabling querystring auth instructs Boto to exclude the querystring parameters (e.g. signature, access key) it -# normally appends to every returned URL. -AWS_QUERYSTRING_AUTH = AUTH_TOKENS.get('AWS_QUERYSTRING_AUTH', True) -AWS_S3_CUSTOM_DOMAIN = AUTH_TOKENS.get('AWS_S3_CUSTOM_DOMAIN', 'edxuploads.s3.amazonaws.com') - -if AUTH_TOKENS.get('DEFAULT_FILE_STORAGE'): - DEFAULT_FILE_STORAGE = AUTH_TOKENS.get('DEFAULT_FILE_STORAGE') -elif AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY: +# Change to S3Boto3 if we haven't specified another default storage AND we have specified AWS creds. +if (not _YAML_TOKENS.get('DEFAULT_FILE_STORAGE')) and AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY: DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' -else: - DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' - - -# If there is a database called 'read_replica', you can use the use_read_replica_if_available -# function in util/query.py, which is useful for very large database reads -DATABASES = AUTH_TOKENS.get('DATABASES', DATABASES) # The normal database user does not have enough permissions to run migrations. # Migrations are run with separate credentials, given as DB_MIGRATION_* @@ -494,8 +346,6 @@ def get_env_setting(setting): 'PORT': os.environ.get('DB_MIGRATION_PORT', database['PORT']), }) -XQUEUE_INTERFACE = AUTH_TOKENS.get('XQUEUE_INTERFACE', XQUEUE_INTERFACE) - # Get the MODULESTORE from auth.json, but if it doesn't exist, # use the one from common.py MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE)) @@ -511,137 +361,34 @@ def get_env_setting(setting): if 'OPTIONS' in store and 'fs_root' in store["OPTIONS"]: derived_collection_entry('MODULESTORE', 'default', 'OPTIONS', 'stores', idx, 'OPTIONS', 'fs_root') -MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {}) - -EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '') # django default is '' -EMAIL_HOST_PASSWORD = AUTH_TOKENS.get('EMAIL_HOST_PASSWORD', '') # django default is '' - -# Analytics API -ANALYTICS_API_KEY = AUTH_TOKENS.get("ANALYTICS_API_KEY", ANALYTICS_API_KEY) -ANALYTICS_API_URL = ENV_TOKENS.get("ANALYTICS_API_URL", ANALYTICS_API_URL) - -# Zendesk -ZENDESK_USER = AUTH_TOKENS.get("ZENDESK_USER") -ZENDESK_API_KEY = AUTH_TOKENS.get("ZENDESK_API_KEY") - -# API Key for inbound requests from Notifier service -EDX_API_KEY = AUTH_TOKENS.get("EDX_API_KEY") - -# Celery Broker -CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "") -CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "") -CELERY_BROKER_VHOST = ENV_TOKENS.get("CELERY_BROKER_VHOST", "") -CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "") -CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "") - BROKER_URL = "{}://{}:{}@{}/{}".format(CELERY_BROKER_TRANSPORT, CELERY_BROKER_USER, CELERY_BROKER_PASSWORD, CELERY_BROKER_HOSTNAME, CELERY_BROKER_VHOST) -BROKER_USE_SSL = ENV_TOKENS.get('CELERY_BROKER_USE_SSL', False) - try: BROKER_TRANSPORT_OPTIONS = { 'fanout_patterns': True, 'fanout_prefix': True, - **ENV_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {}) + **_YAML_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {}) } except TypeError as exc: raise ImproperlyConfigured('CELERY_BROKER_TRANSPORT_OPTIONS must be a dict') from exc -# Block Structures - -# upload limits -STUDENT_FILEUPLOAD_MAX_SIZE = ENV_TOKENS.get("STUDENT_FILEUPLOAD_MAX_SIZE", STUDENT_FILEUPLOAD_MAX_SIZE) - # Event tracking -TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {})) -EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) -EVENT_TRACKING_BACKENDS['segmentio']['OPTIONS']['processors'][0]['OPTIONS']['whitelist'].extend( - AUTH_TOKENS.get("EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST", [])) -TRACKING_SEGMENTIO_WEBHOOK_SECRET = AUTH_TOKENS.get( - "TRACKING_SEGMENTIO_WEBHOOK_SECRET", - TRACKING_SEGMENTIO_WEBHOOK_SECRET +TRACKING_BACKENDS.update(_YAML_TOKENS.get("TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update( + _YAML_TOKENS.get("EVENT_TRACKING_BACKENDS", {}) ) -TRACKING_SEGMENTIO_ALLOWED_TYPES = ENV_TOKENS.get("TRACKING_SEGMENTIO_ALLOWED_TYPES", TRACKING_SEGMENTIO_ALLOWED_TYPES) -TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES = ENV_TOKENS.get( - "TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES", - TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES -) -TRACKING_SEGMENTIO_SOURCE_MAP = ENV_TOKENS.get("TRACKING_SEGMENTIO_SOURCE_MAP", TRACKING_SEGMENTIO_SOURCE_MAP) - -# Heartbeat -HEARTBEAT_CELERY_ROUTING_KEY = ENV_TOKENS.get('HEARTBEAT_CELERY_ROUTING_KEY', HEARTBEAT_CELERY_ROUTING_KEY) - -# Student identity verification settings -VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT) -DISABLE_ACCOUNT_ACTIVATION_REQUIREMENT_SWITCH = ENV_TOKENS.get( - "DISABLE_ACCOUNT_ACTIVATION_REQUIREMENT_SWITCH", - DISABLE_ACCOUNT_ACTIVATION_REQUIREMENT_SWITCH +EVENT_TRACKING_BACKENDS['segmentio']['OPTIONS']['processors'][0]['OPTIONS']['whitelist'].extend( + EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST ) # Grades download -GRADES_DOWNLOAD_ROUTING_KEY = ENV_TOKENS.get('GRADES_DOWNLOAD_ROUTING_KEY', HIGH_MEM_QUEUE) - -GRADES_DOWNLOAD = ENV_TOKENS.get("GRADES_DOWNLOAD", GRADES_DOWNLOAD) - -# Rate limit for regrading tasks that a grading policy change can kick off - -# financial reports -FINANCIAL_REPORTS = ENV_TOKENS.get("FINANCIAL_REPORTS", FINANCIAL_REPORTS) - -##### ORA2 ###### -# Prefix for uploads of example-based assessment AI classifiers -# This can be used to separate uploads for different environments -# within the same S3 bucket. -ORA2_FILE_PREFIX = ENV_TOKENS.get("ORA2_FILE_PREFIX", ORA2_FILE_PREFIX) - -##### ACCOUNT LOCKOUT DEFAULT PARAMETERS ##### -MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = ENV_TOKENS.get( - "MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED", MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED -) - -MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = ENV_TOKENS.get( - "MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS", MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS -) - -##### LOGISTRATION RATE LIMIT SETTINGS ##### -LOGISTRATION_RATELIMIT_RATE = ENV_TOKENS.get('LOGISTRATION_RATELIMIT_RATE', LOGISTRATION_RATELIMIT_RATE) -LOGISTRATION_API_RATELIMIT = ENV_TOKENS.get('LOGISTRATION_API_RATELIMIT', LOGISTRATION_API_RATELIMIT) -LOGIN_AND_REGISTER_FORM_RATELIMIT = ENV_TOKENS.get( - 'LOGIN_AND_REGISTER_FORM_RATELIMIT', LOGIN_AND_REGISTER_FORM_RATELIMIT -) -RESET_PASSWORD_TOKEN_VALIDATE_API_RATELIMIT = ENV_TOKENS.get( - 'RESET_PASSWORD_TOKEN_VALIDATE_API_RATELIMIT', RESET_PASSWORD_TOKEN_VALIDATE_API_RATELIMIT -) -RESET_PASSWORD_API_RATELIMIT = ENV_TOKENS.get('RESET_PASSWORD_API_RATELIMIT', RESET_PASSWORD_API_RATELIMIT) - -##### REGISTRATION RATE LIMIT SETTINGS ##### -REGISTRATION_VALIDATION_RATELIMIT = ENV_TOKENS.get( - 'REGISTRATION_VALIDATION_RATELIMIT', REGISTRATION_VALIDATION_RATELIMIT -) - -REGISTRATION_RATELIMIT = ENV_TOKENS.get('REGISTRATION_RATELIMIT', REGISTRATION_RATELIMIT) - -#### PASSWORD POLICY SETTINGS ##### -AUTH_PASSWORD_VALIDATORS = ENV_TOKENS.get("AUTH_PASSWORD_VALIDATORS", AUTH_PASSWORD_VALIDATORS) - -### INACTIVITY SETTINGS #### -SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = AUTH_TOKENS.get("SESSION_INACTIVITY_TIMEOUT_IN_SECONDS") - -##### LMS DEADLINE DISPLAY TIME_ZONE ####### -TIME_ZONE_DISPLAYED_FOR_DEADLINES = ENV_TOKENS.get("TIME_ZONE_DISPLAYED_FOR_DEADLINES", - TIME_ZONE_DISPLAYED_FOR_DEADLINES) - -#### PROCTORED EXAM SETTINGS #### -PROCTORED_EXAM_VIEWABLE_PAST_DUE = ENV_TOKENS.get('PROCTORED_EXAM_VIEWABLE_PAST_DUE', False) - -##### Third-party auth options ################################################ -ENABLE_REQUIRE_THIRD_PARTY_AUTH = ENV_TOKENS.get('ENABLE_REQUIRE_THIRD_PARTY_AUTH', False) +GRADES_DOWNLOAD_ROUTING_KEY = _YAML_TOKENS.get('GRADES_DOWNLOAD_ROUTING_KEY', HIGH_MEM_QUEUE) if FEATURES.get('ENABLE_THIRD_PARTY_AUTH'): - tmp_backends = ENV_TOKENS.get('THIRD_PARTY_AUTH_BACKENDS', [ + AUTHENTICATION_BACKENDS = _YAML_TOKENS.get('THIRD_PARTY_AUTH_BACKENDS', [ 'social_core.backends.google.GoogleOAuth2', 'social_core.backends.linkedin.LinkedinOAuth2', 'social_core.backends.facebook.FacebookOAuth2', @@ -650,39 +397,35 @@ def get_env_setting(setting): 'common.djangoapps.third_party_auth.identityserver3.IdentityServer3', 'common.djangoapps.third_party_auth.saml.SAMLAuthBackend', 'common.djangoapps.third_party_auth.lti.LTIAuthBackend', - ]) - - AUTHENTICATION_BACKENDS = list(tmp_backends) + list(AUTHENTICATION_BACKENDS) - del tmp_backends + ]) + list(AUTHENTICATION_BACKENDS) # The reduced session expiry time during the third party login pipeline. (Value in seconds) - SOCIAL_AUTH_PIPELINE_TIMEOUT = ENV_TOKENS.get('SOCIAL_AUTH_PIPELINE_TIMEOUT', 600) - - # Most provider configuration is done via ConfigurationModels but for a few sensitive values - # we allow configuration via AUTH_TOKENS instead (optionally). - # The SAML private/public key values do not need the delimiter lines (such as - # "-----BEGIN PRIVATE KEY-----", "-----END PRIVATE KEY-----" etc.) but they may be included - # if you want (though it's easier to format the key values as JSON without the delimiters). - SOCIAL_AUTH_SAML_SP_PRIVATE_KEY = AUTH_TOKENS.get('SOCIAL_AUTH_SAML_SP_PRIVATE_KEY', '') - SOCIAL_AUTH_SAML_SP_PUBLIC_CERT = AUTH_TOKENS.get('SOCIAL_AUTH_SAML_SP_PUBLIC_CERT', '') - SOCIAL_AUTH_SAML_SP_PRIVATE_KEY_DICT = AUTH_TOKENS.get('SOCIAL_AUTH_SAML_SP_PRIVATE_KEY_DICT', {}) - SOCIAL_AUTH_SAML_SP_PUBLIC_CERT_DICT = AUTH_TOKENS.get('SOCIAL_AUTH_SAML_SP_PUBLIC_CERT_DICT', {}) - SOCIAL_AUTH_OAUTH_SECRETS = AUTH_TOKENS.get('SOCIAL_AUTH_OAUTH_SECRETS', {}) - SOCIAL_AUTH_LTI_CONSUMER_SECRETS = AUTH_TOKENS.get('SOCIAL_AUTH_LTI_CONSUMER_SECRETS', {}) + SOCIAL_AUTH_PIPELINE_TIMEOUT = _YAML_TOKENS.get('SOCIAL_AUTH_PIPELINE_TIMEOUT', 600) - # third_party_auth config moved to ConfigurationModels. This is for data migration only: - THIRD_PARTY_AUTH_OLD_CONFIG = AUTH_TOKENS.get('THIRD_PARTY_AUTH', None) + # TODO: just define this in common.py + SOCIAL_AUTH_LTI_CONSUMER_SECRETS = _YAML_TOKENS.get('SOCIAL_AUTH_LTI_CONSUMER_SECRETS', {}) - if ENV_TOKENS.get('THIRD_PARTY_AUTH_SAML_FETCH_PERIOD_HOURS', 24) is not None: + # third_party_auth config moved to ConfigurationModels. This is for data migration only: + THIRD_PARTY_AUTH_OLD_CONFIG = _YAML_TOKENS.get('THIRD_PARTY_AUTH', None) + + # TODO: This logic is somewhat insane. We're not sure if it's intentional or not. We've left it + # as-is for strict backwards compatibility, but it's worth revisiting. + if ( + # if we didn't override the value in YAML, + 'THIRD_PARTY_AUTH_SAML_FETCH_PERIOD_HOURS' not in _YAML_TOKENS or + # OR we overrode it to a truthy value, + (hours := _YAML_TOKENS.get('THIRD_PARTY_AUTH_SAML_FETCH_PERDIOD_HOURS', 24)) + ): + # then update CELERYBEAT_SCHEDULE. CELERYBEAT_SCHEDULE['refresh-saml-metadata'] = { 'task': 'common.djangoapps.third_party_auth.fetch_saml_metadata', - 'schedule': datetime.timedelta(hours=ENV_TOKENS.get('THIRD_PARTY_AUTH_SAML_FETCH_PERIOD_HOURS', 24)), + 'schedule': datetime.timedelta(hours=hours), } # The following can be used to integrate a custom login form with third_party_auth. # It should be a dict where the key is a word passed via ?auth_entry=, and the value is a # dict with an arbitrary 'secret_key' and a 'url'. - THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS = AUTH_TOKENS.get('THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS', {}) + THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS = _YAML_TOKENS.get('THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS', {}) ##### OAUTH2 Provider ############## if FEATURES.get('ENABLE_OAUTH2_PROVIDER'):