diff --git a/README.md b/README.md index 8b542f02d5..25a310c32c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Run through those steps **including the addition of `/etc/hosts` aliases and the ### Configure mitxonline and Open edX +**Open edX Tutor** can be used with MITx Online instead of the traditional devstack release. See [Local Open edX Tutor and MITx Online Deployment](docs/source/configuration/tutor.rst) for details. + See [MITx Online Quick Start](docs/source/configuration/quickstart.rst) and [Configure Open edX](docs/source/configuration/open_edx.rst) ### Configuring the CMS @@ -23,13 +25,9 @@ to be usable. You can apply all of those changes by running a management command docker-compose run --rm web ./manage.py configure_wagtail ``` -### Configuring Refine Admin - -See [Configure Refine Admin](docs/source/configuration/refine_admin.rst) - # Running, testing, and administering the app -Running, testing, and administering this app follows the same patterns as our other web apps. +Running, testing, and administering this app follows the same patterns as our other web apps. *Note: for js tests, run the commands in `frontend/public`, or run via `yarn workspaces foreach run ` from the root. diff --git a/docker-compose.yml b/docker-compose.yml index 46de0e4c43..d13fffb521 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ x-extra-hosts: &default-extra-hosts - "edx.odl.local:${OPENEDX_IP:-172.22.0.1}" - "host.docker.internal:host-gateway" + - "local.overhang.io:host-gateway" services: db: @@ -68,7 +69,7 @@ services: links: - db - redis - # these are links instead of `depends_on` + # these are links instead of `depends_on` # because if we just want a shell for `web` we don't want to run these - watch - refine @@ -79,7 +80,7 @@ services: watch: image: node:17.8 - working_dir: /app + working_dir: /app command: ./scripts/run-watch-dev.sh ports: - "8012:8012" @@ -163,9 +164,9 @@ services: varnish: image: varnish:fresh - links: + links: - nginx - ports: + ports: - "8013:80" volumes: - ./config/default.vcl:/etc/varnish/default.vcl:ro @@ -175,4 +176,4 @@ services: volumes: npm-cache: {} django_media: {} - yarn-cache: {} \ No newline at end of file + yarn-cache: {} diff --git a/docs/source/commands/configure_instance.rst b/docs/source/commands/configure_instance.rst index 407b35c137..4acc79cfac 100644 --- a/docs/source/commands/configure_instance.rst +++ b/docs/source/commands/configure_instance.rst @@ -1,12 +1,14 @@ ``configure_instance`` ====================== -Configures a fresh MITx Online instance. For more information, see :doc:`MITx Online Quick Start<../configuration/quickstart>`. +Configures a fresh MITx Online instance. For more information, see :doc:`MITx Online Quick Start<../configuration/quickstart>` and :doc:`Local Open edX Tutor and MITx Online Deployment<../configuration/tutor>`. + +For Tutor deployments, this will use ``local.overhang.io`` for URLs for the edX platform. If you're running a dev deployment, or are using Tutor Nightly, ``--tutor-dev`` will additionally add the proper ports (as Caddy is disabled in these cases). In either case, the two demo courses will still be created but only the Demonstration Course (``course-v1:edX+DemoX+Demo_Course``) will exist in edX, and then only if you import the demo course using the relevant Tutor command. Syntax ------ -``configure_instance [--dont-enroll|-D] [--dont-create-superuser|-S] [--edx-oauth-client ] [--edx-oauth-secret ] [--gateway ]`` +``configure_instance [--dont-enroll|-D] [--dont-create-superuser|-S] [--edx-oauth-client ] [--edx-oauth-secret ] [--gateway ] [--tutor|-T] [--tutor-dev]`` Options ------- @@ -15,5 +17,7 @@ Options * ``--dont-enroll|-D`` - Don't enroll the test learner account in any courses. (Defaults to enrolling the account in ``course-v1:edX+DemoX+Demo_Course``.) * ``--dont-create-superuser|-S`` - Don't create a superuser account. * ``--gateway `` - The Docker gateway IP. Required on Linux. See :doc:`Configure Open edX<../configuration/open_edx>` for more info. -* ``--edx-oauth-client `` - Use the specified client ID for the edX OAuth2 client. (Useful if you're redoing your MITx Online instance and you've already created the corresponding record in edX, since you're not allowed to edit it there.) -* ``--edx-oauth-secret `` - Use the specified client secret for the edX OAuth2 client. (Useful if you're redoing your MITx Online instance and you've already created the corresponding record in edX, since you're not allowed to edit it there.) \ No newline at end of file +* ``--edx-oauth-client `` - Use the specified client ID for the edX OAuth2 client. (Useful if you're redoing your MITx Online instance and you've already created the corresponding record in edX, since you're not allowed to edit it there.) +* ``--edx-oauth-secret `` - Use the specified client secret for the edX OAuth2 client. (Useful if you're redoing your MITx Online instance and you've already created the corresponding record in edX, since you're not allowed to edit it there.) +* ``--tutor|-T`` - Configure the instance for use with a Tutor edX deployment. +* ``--tutor-dev`` - Configure the instnace for use with Tutor dev or nightly. diff --git a/docs/source/configuration/index.rst b/docs/source/configuration/index.rst index c0f6d8df56..f5024eaaf2 100644 --- a/docs/source/configuration/index.rst +++ b/docs/source/configuration/index.rst @@ -5,6 +5,6 @@ Configuration :maxdepth: 2 open_edx + tutor ecommerce - refine_admin - quickstart \ No newline at end of file + quickstart diff --git a/docs/source/configuration/refine_admin.rst b/docs/source/configuration/refine_admin.rst deleted file mode 100644 index e41056c086..0000000000 --- a/docs/source/configuration/refine_admin.rst +++ /dev/null @@ -1,80 +0,0 @@ -Refine Admin -############ - -Some administrative functionality is built out using `Refine `_. The Refine admin is a separate application that uses OAuth2/OIDC for authentication and communicates with MITx Online via the standard REST API. It compliments the Django Admin interface, providing an interface for operations that would otherwise be hard to implement in Django Admin, and for users that don't necessarily need the level of access that Django Admin provides. - -Accessing -********* - -The Docker compose environment builds a ``refine`` container that runs a dev server available at port 8016. - -Set Up -====== - - -Development ------------ - - -You will need to make some adjustments to the MITx Online configuration to allow the Refine admin to work. - - Currently, the application expects to be accessible at ``mitxonline.odl.local``. - -Step 1: Generating the RSA Private Key for OIDC -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following command will generate the key: - -``openssl genrsa 4096 2>/dev/null | sed -e 'H;${x;s/\n/\\n/g;s/^\\n//;p;};d'`` - -This will generate the key and then output it to the terminal in a format that can go into your ``.env`` file. Copy everything including the ``--BEGIN RSA PRIVATE KEY--`` and matching end part into a variable named ``OIDC_RSA_PRIVATE_KEY``. The key will need to be wrapped in quotes. Then, rebuild and restart your environment to pick up the changes. - -The key generation process is from the `Django OAuth Toolkit `_ docs. - -Step 2: Configuring MITx Online -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You will need to add a new Application in the Django Oauth Toolkit section in Django Admin (``/admin/oauth2_provider/application/``). Navigate there and create a new Application. Use these values (overwriting the defaults where necessary): - -* **Client Id**: ``refine-local-client-id`` -* **Redirect uris**: ``http://mitxonline.odl.local:8013/staff-dashboard/oauth2/login/`` -* **Client type**: Public -* **Authorization grants**: Authorization Code -* **Skip authorization**: checked -* **Algorithm**: RSA with SHA-2 256 - -It's also a good idea to provide a Name to differentiate it in the list. You don't need to have a Client Secret for this Application. - -Take the Client Id and put it in the environment variable `MITX_ONLINE_REFINE_OIDC_CONFIG_CLIENT_ID`. - -Once this has been saved, you should be able to log into the Refine admin successfully. - -Deployments (RC, Production, etc) ---------------------------------- - - -**Step 1: Generating the RSA Private Key for OIDC** - -The following command will generate the key: - -``openssl genrsa 4096 2>/dev/null | sed -e 'H;${x;s/\n//g;s/^\\n//;p;};d'`` - -This will generate the key and then output it to the terminal. Copy everything including the ``--BEGIN RSA PRIVATE KEY--`` and matching end part into a variable named ``OIDC_RSA_PRIVATE_KEY``. - -The key generation process is from the `Django OAuth Toolkit `_ docs. - -**Step 2: Configuring MITx Online OAuth Provider** - -You will need to add a new Application in the Django OAuth Toolkit section in Django Admin (``/admin/oauth2_provider/application/``). Navigate there and create a new Application. Use these values (overwriting the defaults where necessary): - -* **Redirect uris**: ``http:///staff-dashboard/oauth2/login/`` -* **Client type**: Public -* **Authorization grants**: Authorization Code -* **Skip authorization**: checked -* **Algorithm**: RSA with SHA-2 256 - -Take the Client Id and put it in the environment variable `MITX_ONLINE_REFINE_OIDC_CONFIG_CLIENT_ID`. - -It's also a good idea to provide a Name to differentiate it in the list. You don't need to have a Client Secret for this Application. - -Once this has been saved, you should be able to log into the Refine admin successfully. diff --git a/docs/source/configuration/tutor.rst b/docs/source/configuration/tutor.rst new file mode 100644 index 0000000000..14483f31b3 --- /dev/null +++ b/docs/source/configuration/tutor.rst @@ -0,0 +1,219 @@ + +Local Open edX Tutor and MITx Online Deployment +=============================================== + +These instructions describe setting up MITx Online and Tutor from scratch on Linux. These instructions should largely apply to macOS users, and they should also apply to users converting from a devstack-based deployment to Tutor. + +.. + + These instructions should work for a Tutor Dev or Tutor Nightly deployment as well. Specify ``--tutor-dev`` instead of ``--tutor`` when running ``configure_instance`` so the URLs have a port on them. + + +At the end of this guide, you should have: + + +* A fully working MITx Online deployment. +* A working Tutor deployment of edX. +* SSO should work from edX to MITx Online. +* MITx Online should have a service worker set up and should be able to perform tasks using the edX api. +* Tutor's included AuthN MFE should be disabled in favor of the MITx Online authentication system. + +Preliminary Steps +----------------- + +``pyenv`` (and ``pyenv-virtualenv``\ ) are highly recommended for managing local Python versions. `Use the instructions on their GitHub page to install if you haven't already installed it. `_ + +You'll want to create at least a virtualenv for Tutor. As of this writing, Tutor uses Python 3.8.12 (in the LMS container at least); I have also successfully used 3.9.16. 3.11 has *not* worked for me. You can optionally create a virtualenv for MITx Online too but that's not strictly necessary. (I have one so I can run black/isort/etc. without having to jump into a container to do it.) + +Tutor Setup, Part One +--------------------- + +.. + + Note that no hosts file changes are needed if you use the default ``local.overhang.io`` domain - that's a real domain with a wildcard subdomain cname that points to 127.0.0.1. + + +To begin, you need to follow the `One-Click Installer `_ instructions provided by Tutor. Do this with your Tutor virtualenv activated. + +.. + + Mac/Arm users should instead follow these instructions: `Running Tutor on ARM-based systems `_ It's mostly the same steps that the quickstart does internally, with some changes to rebuild some of the images and flip some dependencies to use compatible images for Arm. + + +Once Tutor has bootstrapped itself and is available, create a superuser account: + +.. code-block:: + + tutor local do createuser --staff --superuser edx edx@example.org + +Supply a password (the one used by devstack is ``edx`` so use that if you want to be consistent with it). Then, create a service worker account for MITx Online: + +.. code-block:: + + tutor local do createuser --staff mitx_online_serviceworker service@mitxonline.odl.local + +Supply a password (this one doesn't matter for a local deployment, you won't ever actually use the account). + +For best results, create two new courses within edX. The MITx Online ``configure_instance`` command expects a couple of courses to exist in edX (because they come with the devstack package): + +.. list-table:: + :header-rows: 1 + + * - Course ID + - Course Title + * - course-v1:edX+DemoX+Demo_Course + - Demonstration Course + * - course-v1:edX+E2E-101+course + - E2E Test Course + + +If you have a devstack instance handy, you can export these and import them into Tutor. Otherwise, just create them and make sure to set dates for the courses (they default to 2030 otherwise). + +Finally, go here to create an access token for the service worker user: http://local.overhang.io/admin/oauth2_provider/accesstoken/add/ The token can be anything, and the expiration date should just be today plus 10 years. + +MITx Online Setup +----------------- + +To set up MITx Online: + + +#. Get the gateway IP for the ``tutor_local_default`` network: ``docker network inspect tutor_local_default | grep Gateway`` + + * Mac users should instead use the host.docker.internal IP. Specify this by using ``host-internal`` in ``OPENEDX_IP``. + +#. Set up your ``/etc/hosts`` file to point ``mitxonline.odl.local`` to localhost. +#. Clone the repository somewhere. +#. Set up your ``.env`` file. These settings need particular attention: + + * ``OPENEDX_IP``\ : set to the gateway IP from the first step + * ``OPENEDX_API_BASE_URL``\ : set to http://local.overhang.io + * ``OPENEDX_SERVICE_WORKER_USERNAME``\ : set to ``mitx_online_serviceworker`` (unless you changed this) + * ``OPENEDX_SERVICE_WORKER_API_TOKEN``\ : set to the token you generated + +#. Build the app: ``docker compose build`` +#. Run migrations and configure Wagtail:: + + docker compose run --rm web ./manage.py migrate + docker compose run --rm web ./manage.py configure_wagtail + +#. Run the ``configure_instance`` command:: + + docker compose run --rm web ./manage.py configure_instance linux --gateway --tutor + + where ```` is the IP from the first step. (On macOS, specify ``macos`` instead of ``linux``. You can also skip ``--gateway``.) You will need to supply passwords for the MITx Online superuser and test learner accounts. Make a note of the client ID and secret that it will print out at the end. + + +Tutor Setup, Part Two +--------------------- + +Note that some of these steps require editing the main configuration files for the production instance (which is also used for a local deployment). Most of the settings that need to be adjusted to get integration working are overridden by the default Tutor configuration, so you can't update them by setting ``config.yml``. If you're using the development Tutor build, you'll likely need to edit ``development.py`` rather than ``production.py`` as necessary. + +These steps will also disable the AuthN SSO MFE, so from here on you'll get normal edX authentication screens (if you're not being bounced to MITx Online). + + +#. Get the gateway IP of the ``mitxonline_default`` Docker network:: + + docker network inspect mitxonline_default | grep Gateway + +#. Log into to edX using your superuser account, and make sure your session stays open. Sessions are pretty long-lived so this just means not closing the browser that you started the session in. (Part of this process will involve mostly breaking authentication so it's important that you are able to get into the admin.) +#. Stop Tutor: ``tutor local stop`` +#. Change into the configuration root for Tutor:: + + cd "$(tutor config printroot)" + +#. Create a ``env/build/openedx/requirements/private.txt`` with the required extensions:: + + social-auth-mitxpro + mitxpro-openedx-extensions + +#. Edit the ``env/apps/openedx/config/lms.env.yml`` file and add:: + + FEATURES: + SKIP_EMAIL_VALIDATION: true + + to the ``FEATURES`` block (should be at the top). +#. Edit the ``env/apps/openedx/settings/lms/production.py`` and/or ``env/apps/openedx/settings/lms/development.py`` settings file. (The former is used by a local instance, where the latter is used by both dev and nightly instances.) + + * Add to the end of the file: + + * ``THIRD_PARTY_AUTH_BACKENDS = ['social_auth_mitxpro.backends.MITxProOAuth2']`` + * ``AUTHENTICATION_BACKENDS.append('social_auth_mitxpro.backends.MITxProOAuth2')`` + * ``IDA_LOGOUT_URI_LIST.append('http://mitxonline.odl.local:8013/logout/')`` - there's an existing one of these around like 300 in production.py too. + + * Find and update: + + * ``FEATURES['ENABLE_AUTHN_MICROFRONTEND'] = False`` (defaults to True) + * ``REGISTRATION_EXTRA_FIELDS["terms_of_service"] = "hidden"`` (defaults to required) + +#. Build a new ``openedx`` image: ``tutor images build openedx`` (this will take a long time) +#. Run a Docker Compse rebuild: ``tutor local dc build`` (this should be pretty quick - it's likely not required, just doing it here for safety) +#. Restart Tutor: ``tutor local start -d`` (omit ``-d`` if you want to watch the logs) +#. Check your settings. There's a ``print_setting`` command that you can use to verify everything is set properly: + + * ``tutor local run lms ./manage.py lms print_setting REGISTRATION_EXTRA_FIELDS`` + * ``tutor local run lms ./manage.py lms print_setting AUTHENTICATION_BACKENDS`` + * ``tutor local run lms ./manage.py lms print_setting FEATURES`` - will print a lot of stuff + * ``tutor local run lms ./manage.py lms print_setting THIRD_PARTY_AUTH_BACKENDS`` + * If you do have weird errors or settings not showing properly, make sure you edited the right yaml files *and* that they're using the right whitespace (i.e. don't use tabs). + +#. In a separate browser session of some kind (incognito/private browsing/other browser entirely), try to navigate to http://local.overhang.io . It should load but it should give you an error message. In the LMS logs, you should see an error message for "Can't fetch settings for disabled provider." This is proper operation - the OAuth2 settings aren't in place yet. +#. In the superuser session you have open, go to http://local.overhang.io/admin . This should work. If you've been logged out, you should still be able to get in. If you can't (for instance, if you're getting 500 errors), you will need to turn off ``ENABLE_THIRD_PARTY_AUTH`` in ``FEATURES``\ , restart Tutor using ``tutor local stop`` and ``start``\ , not using ``reboot``\ , then try again. +#. Go to http://local.overhang.io/admin/third_party_auth/oauth2providerconfig/add/ and add a provider configuration: + + * Enabled is checked. + * Name: ``mitxonline`` + * Slug: ``mitxpro-oauth2`` + * Site: ``local.overhang.io`` + * Skip hinted login dialog is checked. + * Skip registration form is checked. + * Skip email verification is checked. + * Sync learner profile data is checked. + * Enable sso id verification is checked. + * Backend name: ``mitxpro-oauth2`` + * Client ID and Client Secret: from record created by ``configure_instance`` when you set up MITx Online. + * Other settings: + + { + "AUTHORIZATION_URL": "\http://mitxonline.odl.local:8013/oauth2/authorize/", + "ACCESS_TOKEN_URL": "\http://:8013/oauth2/token/", + "API_ROOT": "\http://:8013/" + } + + where MITXONLINE_GATEWAY_IP is the IP from the ``mitxonline_default`` network from the first step. **Mac users**, use ``host.docker.internal`` for MITXONLINE_GATEWAY_IP. + +#. Configure Tutor for OAuth2 authentication from MITx Online. + + * Go to http://local.overhang.io/admin/oauth2_provider/application/ and either add or edit the ``edx-oauth-app`` entry. + * Ensure these settings are set: + + * Name: ``edx-oauth-app`` + * Redirect uris: ``http://mitxonline.odl.local:8013/login/_private/complete`` + * Client type: ``Confidential`` + * Authorization grant type: ``Confidental`` + * Skip authorization is checked. + + * Save ``Client id`` and ``Client secret``. + +#. Update your MITx Online ``.env`` file. Set ``OPENEDX_API_CLIENT_ID`` and ``OPENEDX_API_CLIENT_SECRET`` to the values from the record you created or updated in the last step. +#. You should now be able to run some MITx Online management commands to ensure the service worker is set up properly: + + * ``sync_courserun --all ALL`` should sync the two test courses (if you made them). + * ``repair_missing_courseware_records`` should also work. + +#. In the separate browser session from step 12, attempt to log in again. This time, you should be able to log in through MITx Online, and you should be able to get to the edX LMS dashboard. If not, then double-check your provider configuration settings and try again. + + * Unlike devstack, the Tutor instance has an Update button for the provider configuration, so you can just update the record you put in. + * If you are still getting "Can't fetch settings" errors, *make sure* your Site is set properly - there are three options by default and only one works. (This was typically the problem I had.) + +#. Optionally, log into the LMS Django Admin and make your MITx Online superuser account a superuser there too. + +Other Notes +----------- + +**Trying to set configuration settings via ``tutor config`` will undo the specialty configuration above.** If you need to make changes to the configuration, either manually edit the ``env/apps/openedx/config/lms.env.yml`` file or the ``env/apps/openedx/settings/lms/production.py`` file and restart your Tutor instance. + +**Make sure your service worker account is active.** It's an easy checkbox to miss. + +**Restarting** If you want to rebuild from scratch, make sure you ``docker image prune``. It's also recommended to remove the Tutor project root folder - ``tutor config printroot`` will tell you where that is. + +**Running Multiple Tutor Instances** If you want to run more than one Tutor instance, it's pretty important to specify the project root explicitly or you may end up with one instance trying to use config files from another and things getting confused from there. `See the Tutor documentation for this. `_ (A suggestion: configure aliases to the ``tutor`` command that run ``tutor --root=`` so you don't have to rely on environment variables, especially if you keep multiple terminal sessions going.) diff --git a/main/management/commands/configure_instance.py b/main/management/commands/configure_instance.py index ae95a548dd..d5b4dbc1a1 100644 --- a/main/management/commands/configure_instance.py +++ b/main/management/commands/configure_instance.py @@ -31,6 +31,9 @@ The product for the courses will both be $999 so they are under the limit for test CyberSource transactions. +If the --tutor/-T option is passed, the command will use the local.overhang.io +address for links to edX rather than edx.odl.local:18000. + This uses other management commands to complete these tasks. So, if you just want to run part of this, use one of these commands: - createsuperuser to create the super user @@ -106,9 +109,36 @@ def add_arguments(self, parser): nargs="?", ) + parser.add_argument( + "--tutor", + "-T", + help="Configure for Tutor.", + action="store_true", + dest="tutor", + ) + + parser.add_argument( + "--tutor-dev", + help="Configure for Tutor Dev/Nightly.", + action="store_true", + dest="tutordev", + ) + + def determine_edx_hostport(self, *args, **kwargs): + """Returns a tuple of the edX host and port depending on what the user's passed in""" + + if kwargs["tutor"]: + return ("local.overhang.io", "") + elif kwargs["tutordev"]: + return ("local.overhang.io", ":8000") + else: + return ("edx.odl.local:18000", ":18000") + def handle(self, *args, **kwargs): """Coordinates the other commands.""" + (edx_host, edx_gateway_port) = self.determine_edx_hostport(**kwargs) + # Step -1: run createsuperuesr if kwargs["superuser"]: self.stdout.write(self.style.SUCCESS("Creating superuser...")) @@ -122,8 +152,8 @@ def handle(self, *args, **kwargs): if kwargs["platform"] == "macos": redirects = "\n".join( [ - "http://edx.odl.local:18000/auth/complete/mitxpro-oauth2/", - "http://host.docker.internal:18000/auth/complete/mitxpro-oauth2/", + f"http://{edx_host}/auth/complete/mitxpro-oauth2/", + f"http://host.docker.internal{edx_gateway_port}/auth/complete/mitxpro-oauth2/", ] ) else: @@ -137,8 +167,8 @@ def handle(self, *args, **kwargs): redirects = "\n".join( [ - "http://edx.odl.local:18000/auth/complete/mitxpro-oauth2/", - f"http://{kwargs['gateway']}:18000/auth/complete/mitxpro-oauth2/", + f"http://{edx_host}/auth/complete/mitxpro-oauth2/", + f"http://{kwargs['gateway']}{edx_gateway_port}/auth/complete/mitxpro-oauth2/", ] ) @@ -200,7 +230,7 @@ def handle(self, *args, **kwargs): "Demonstration Course", live=True, create_run="Demo_Course", - run_url="http://edx.odl.local:18000/courses/course-v1:edX+DemoX+Demo_Course/", + run_url=f"http://{edx_host}/courses/course-v1:edX+DemoX+Demo_Course/", program="program-v1:MITx+DEDP", ) @@ -211,7 +241,7 @@ def handle(self, *args, **kwargs): "E2E Test Course", live=True, create_run="course", - run_url="http://edx.odl.local:18000/courses/course-v1:edX+E2E-101+course/", + run_url=f"http://{edx_host}/courses/course-v1:edX+E2E-101+course/", ) self.stdout.write(self.style.SUCCESS("Syncing course runs (this may fail)..."))