From d2c4c2cfec3caf38b70919481d86942b7297f719 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 16 Jan 2025 15:35:26 -0500 Subject: [PATCH 1/2] refactor: Use `derived_settings` to lazy load settings. Some of our settings depend on the values of other settings. Rather than explicitly looking up each one in the YAML settings file, we can simply derive them based on the setting in the YAML file after all the YAML settings have been loaded. --- lms/envs/production.py | 164 +++++++++++++++++++++++------------------ 1 file changed, 93 insertions(+), 71 deletions(-) diff --git a/lms/envs/production.py b/lms/envs/production.py index 3ea1c952e47..c19d01b0c9c 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -126,6 +126,99 @@ def get_env_setting(setting): "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}" ) +## Derived Settings +CC_MERCHANT_NAME = lambda settings: settings.PLATFORM_NAME +EMAIL_FILE_PATH = lambda settings: settings.DATA_DIR / "emails" / "lms" +LMS_INTERNAL_ROOT_URL = lambda settings: settings.LMS_ROOT_URL +# This is the domain that is used to set shared cookies between various sub-domains. +# By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN +SHARED_COOKIE_DOMAIN = lambda settings: settings.SESSION_COOKIE_DOMAIN + +# We want Bulk Email running on the high-priority queue, so we define the +# routing key that points to it. At the moment, the name is the same. +# We have to reset the value here, since we have changed the value of the queue name. +BULK_EMAIL_ROUTING_KEY = lambda settings: settings.HIGH_PRIORITY_QUEUE + +# We can run smaller jobs on the low priority queue. See note above for why +# we have to reset the value here. +BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = lambda settings: settings.DEFAULT_PRIORITY_QUEUE + +# Queue to use for expiring old entitlements +ENTITLEMENTS_EXPIRATION_ROUTING_KEY = lambda settings: settings.DEFAULT_PRIORITY_QUEUE + +# Intentional defaults. +ID_VERIFICATION_SUPPORT_LINK = lambda settings: settings.SUPPORT_SITE_LINK +PASSWORD_RESET_SUPPORT_LINK = lambda settings: settings.SUPPORT_SITE_LINK +ACTIVATION_EMAIL_SUPPORT_LINK = lambda settings: settings.SUPPORT_SITE_LINK +LOGIN_ISSUE_SUPPORT_LINK = lambda settings: settings.SUPPORT_SITE_LINK + +# Default queues for various routes +GRADES_DOWNLOAD_ROUTING_KEY = lambda settings: settings.HIGH_MEM_QUEUE +CREDENTIALS_GENERATION_ROUTING_KEY = lambda settings: settings.DEFAULT_PRIORITY_QUEUE +PROGRAM_CERTIFICATES_ROUTING_KEY = lambda settings: settings.DEFAULT_PRIORITY_QUEUE +SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = lambda settings: settings.HIGH_PRIORITY_QUEUE + +############## OPEN EDX ENTERPRISE SERVICE CONFIGURATION ###################### +# The Open edX Enterprise service is currently hosted via the LMS container/process. +# However, for all intents and purposes this service is treated as a standalone IDA. +# These configuration settings are specific to the Enterprise service and you should +# not find references to them within the edx-platform project. + + +# Publicly-accessible enrollment URL, for use on the client side. +def _generate_public_enrollment_api_url(settings): + return (settings.LMS_ROOT_URL or '') + settings.LMS_ENROLLMENT_API_PATH +ENTERPRISE_PUBLIC_ENROLLMENT_API_URL = _generate_public_enrollment_api_url + + +# Enrollment URL used on the server-side. +def _generate_enterprise_enrollment_api_url(settings): + return (settings.LMS_INTERNAL_ROOT_URL or '') + settings.LMS_ENROLLMENT_API_PATH +ENTERPRISE_ENROLLMENT_API_URL = _generate_enterprise_enrollment_api_url + +############## ENTERPRISE SERVICE API CLIENT CONFIGURATION ###################### +# The LMS communicates with the Enterprise service via the requests.Session() client +# The below environmental settings are utilized by the LMS when interacting with +# the service, and override the default parameters which are defined in common.py + + +def _generate_default_enterprise_api_url(settings): + default_enterprise_api_url = None + if settings.LMS_INTERNAL_ROOT_URL is not None: + default_enterprise_api_url = settings.LMS_INTERNAL_ROOT_URL + '/enterprise/api/v1/' + return default_enterprise_api_url +ENTERPRISE_API_URL = _generate_default_enterprise_api_url + + +def _generate_default_enterprise_consent_api_url(settings): + default_enterprise_consent_api_url = None + if settings.LMS_INTERNAL_ROOT_URL is not None: + default_enterprise_consent_api_url = settings.LMS_INTERNAL_ROOT_URL + '/consent/api/v1/' +ENTERPRISE_CONSENT_API_URL = _generate_default_enterprise_consent_api_url + +# Note the order of this matters, don't sort this list. +derived( + 'CC_MERCHANT_NAME', + 'EMAIL_FILE_PATH', + 'LMS_INTERNAL_ROOT_URL', + 'SHARED_COOKIE_DOMAIN', + 'BULK_EMAIL_ROUTING_KEY', + 'BULK_EMAIL_ROUTING_KEY_SMALL_JOBS', + 'ENTITLEMENTS_EXPIRATION_ROUTING_KEY', + 'ID_VERIFICATION_SUPPORT_LINK', + 'PASSWORD_RESET_SUPPORT_LINK', + 'ACTIVATION_EMAIL_SUPPORT_LINK', + 'LOGIN_ISSUE_SUPPORT_LINK', + 'GRADES_DOWNLOAD_ROUTING_KEY', + 'CREDENTIALS_GENERATION_ROUTING_KEY', + 'PROGRAM_CERTIFICATES_ROUTING_KEY', + 'SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY', + 'ENTERPRISE_PUBLIC_ENROLLMENT_API_URL', + 'ENTERPRISE_ENROLLMENT_API_URL', + 'ENTERPRISE_API_URL', + 'ENTERPRISE_CONSENT_API_URL', +) + ####################################################################################################################### # A file path to a YAML file from which to load all the configuration for the edx platform @@ -199,15 +292,11 @@ def get_env_setting(setting): STATIC_URL += "/" DATA_DIR = path(DATA_DIR) -CC_MERCHANT_NAME = _YAML_TOKENS.get('CC_MERCHANT_NAME', PLATFORM_NAME) -EMAIL_FILE_PATH = _YAML_TOKENS.get('EMAIL_FILE_PATH', DATA_DIR / "emails" / "lms") # TODO: This was for backwards compatibility back when installed django-cookie-samesite (not since 2022). # The DCS_ version of the setting can be DEPR'd at this point. SESSION_COOKIE_SAMESITE = DCS_SESSION_COOKIE_SAMESITE -LMS_INTERNAL_ROOT_URL = _YAML_TOKENS.get('LMS_INTERNAL_ROOT_URL', LMS_ROOT_URL) - for feature, value in _YAML_TOKENS.get('FEATURES', {}).items(): FEATURES[feature] = value @@ -217,10 +306,6 @@ def get_env_setting(setting): FEATURES['PREVIEW_LMS_BASE'], ] -# This is the domain that is used to set shared cookies between various sub-domains. -# By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN, but we want to make it overrideable. -SHARED_COOKIE_DOMAIN = _YAML_TOKENS.get('SHARED_kCOOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) - # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: @@ -237,17 +322,6 @@ def get_env_setting(setting): # Once we have migrated to service assets off S3, then we can convert this back to # managed by the yaml file contents -# We want Bulk Email running on the high-priority queue, so we define the -# routing key that points to it. At the moment, the name is the same. -# We have to reset the value here, since we have changed the value of the queue name. -BULK_EMAIL_ROUTING_KEY = _YAML_TOKENS.get('BULK_EMAIL_ROUTING_KEY', HIGH_PRIORITY_QUEUE) - -# We can run smaller jobs on the low priority queue. See note above for why -# we have to reset the value here. -BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = _YAML_TOKENS.get('BULK_EMAIL_ROUTING_KEY_SMALL_JOBS', DEFAULT_PRIORITY_QUEUE) - -# Queue to use for expiring old entitlements -ENTITLEMENTS_EXPIRATION_ROUTING_KEY = _YAML_TOKENS.get('ENTITLEMENTS_EXPIRATION_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) # Build a CELERY_QUEUES dict the way that celery expects, based on a couple lists of queue names from the YAML. _YAML_CELERY_QUEUES = _YAML_TOKENS.get('CELERY_QUEUES', None) @@ -271,12 +345,6 @@ def get_env_setting(setting): MKTG_URL_LINK_MAP.update(_YAML_TOKENS.get('MKTG_URL_LINK_MAP', {})) -# Intentional defaults. -ID_VERIFICATION_SUPPORT_LINK = _YAML_TOKENS.get('ID_VERIFICATION_SUPPORT_LINK', SUPPORT_SITE_LINK) -PASSWORD_RESET_SUPPORT_LINK = _YAML_TOKENS.get('PASSWORD_RESET_SUPPORT_LINK', SUPPORT_SITE_LINK) -ACTIVATION_EMAIL_SUPPORT_LINK = _YAML_TOKENS.get('ACTIVATION_EMAIL_SUPPORT_LINK', SUPPORT_SITE_LINK) -LOGIN_ISSUE_SUPPORT_LINK = _YAML_TOKENS.get('LOGIN_ISSUE_SUPPORT_LINK', SUPPORT_SITE_LINK) - # Timezone overrides TIME_ZONE = CELERY_TIMEZONE @@ -384,9 +452,6 @@ def get_env_setting(setting): EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST ) -# Grades download -GRADES_DOWNLOAD_ROUTING_KEY = _YAML_TOKENS.get('GRADES_DOWNLOAD_ROUTING_KEY', HIGH_MEM_QUEUE) - if FEATURES.get('ENABLE_THIRD_PARTY_AUTH'): AUTHENTICATION_BACKENDS = _YAML_TOKENS.get('THIRD_PARTY_AUTH_BACKENDS', [ 'social_core.backends.google.GoogleOAuth2', @@ -490,49 +555,6 @@ def get_env_setting(setting): #### JWT configuration #### JWT_AUTH.update(_YAML_TOKENS.get('JWT_AUTH', {})) -################################ Settings for Credentials Service ################################ - -CREDENTIALS_GENERATION_ROUTING_KEY = _YAML_TOKENS.get('CREDENTIALS_GENERATION_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) - -PROGRAM_CERTIFICATES_ROUTING_KEY = _YAML_TOKENS.get('PROGRAM_CERTIFICATES_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) - -SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = _YAML_TOKENS.get( - 'SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY', - HIGH_PRIORITY_QUEUE -) - -############## OPEN EDX ENTERPRISE SERVICE CONFIGURATION ###################### -# The Open edX Enterprise service is currently hosted via the LMS container/process. -# However, for all intents and purposes this service is treated as a standalone IDA. -# These configuration settings are specific to the Enterprise service and you should -# not find references to them within the edx-platform project. - -# Publicly-accessible enrollment URL, for use on the client side. -ENTERPRISE_PUBLIC_ENROLLMENT_API_URL = _YAML_TOKENS.get( - 'ENTERPRISE_PUBLIC_ENROLLMENT_API_URL', - (LMS_ROOT_URL or '') + LMS_ENROLLMENT_API_PATH -) - -# Enrollment URL used on the server-side. -ENTERPRISE_ENROLLMENT_API_URL = _YAML_TOKENS.get( - 'ENTERPRISE_ENROLLMENT_API_URL', - (LMS_INTERNAL_ROOT_URL or '') + LMS_ENROLLMENT_API_PATH -) - -############## ENTERPRISE SERVICE API CLIENT CONFIGURATION ###################### -# The LMS communicates with the Enterprise service via the requests.Session() client -# The below environmental settings are utilized by the LMS when interacting with -# the service, and override the default parameters which are defined in common.py - -DEFAULT_ENTERPRISE_API_URL = None -if LMS_INTERNAL_ROOT_URL is not None: - DEFAULT_ENTERPRISE_API_URL = LMS_INTERNAL_ROOT_URL + '/enterprise/api/v1/' -ENTERPRISE_API_URL = _YAML_TOKENS.get('ENTERPRISE_API_URL', DEFAULT_ENTERPRISE_API_URL) - -DEFAULT_ENTERPRISE_CONSENT_API_URL = None -if LMS_INTERNAL_ROOT_URL is not None: - DEFAULT_ENTERPRISE_CONSENT_API_URL = LMS_INTERNAL_ROOT_URL + '/consent/api/v1/' -ENTERPRISE_CONSENT_API_URL = _YAML_TOKENS.get('ENTERPRISE_CONSENT_API_URL', DEFAULT_ENTERPRISE_CONSENT_API_URL) ############## ENTERPRISE SERVICE LMS CONFIGURATION ################################## # The LMS has some features embedded that are related to the Enterprise service, but From 0e6a0a6545662acdb01c4c472a256ee87647be9b Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 16 Jan 2025 15:40:01 -0500 Subject: [PATCH 2/2] docs: Add some more docs for how to use derived_settings. This was a little hard to grok so I added a little example to make it easier to understand. --- openedx/core/lib/derived.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/openedx/core/lib/derived.py b/openedx/core/lib/derived.py index a62731ef543..47e191614fe 100644 --- a/openedx/core/lib/derived.py +++ b/openedx/core/lib/derived.py @@ -3,6 +3,30 @@ via callable methods/lambdas. The derivation time can be controlled to happen after all other settings have been set. The derived setting can also be overridden by setting the derived setting to an actual value. + +Example + +In `lms/envs/common.py`: + +``` +# Double some other value that might get set later. +VALUE = lambda settings: settings.SOME_OTHER_VALUE * 2 +# Register this value as one that needs to be derived later. +derived(VALUE) +``` + +Later in a settings file that depends on common.py + +``` +from lms.envs.common * + +# Set some other value however you want. +SOME_OTHER_VALUE = 4 + +# Derive any settings and pass them this settings file for reference. +# This will update VALUE so that it is the scaler `8` instead of a lambda. +derive_settings(__name__) +``` """ import sys