diff --git a/credentials/apps/catalog/management/commands/copy_catalog.py b/credentials/apps/catalog/management/commands/copy_catalog.py index b8ad8dd26..3f40b0738 100644 --- a/credentials/apps/catalog/management/commands/copy_catalog.py +++ b/credentials/apps/catalog/management/commands/copy_catalog.py @@ -3,7 +3,7 @@ import logging from django.contrib.sites.models import Site -from django.core.management import BaseCommand +from django.core.management import BaseCommand, CommandError from credentials.apps.catalog.utils import parse_pathway, parse_program from credentials.apps.core.models import SiteConfiguration @@ -28,8 +28,10 @@ def add_arguments(self, parser): def handle(self, *args, **options): page_size = options.get('page_size') + failure_count = 0 for site in Site.objects.all(): + success = True site_configs = SiteConfiguration.objects.filter(site=site) site_config = site_configs.get() if site_configs.exists() else None @@ -41,10 +43,31 @@ def handle(self, *args, **options): logger.info('Copying catalog data for site {}'.format(site.domain)) client = site_config.catalog_api_client - Command.fetch_programs(site, client, page_size=page_size) - logger.info('Finished copying programs.') - Command.fetch_pathways(site, client, page_size=page_size) - logger.info('Finished copying pathways.') + try: + Command.fetch_programs(site, client, page_size=page_size) + logger.info('Finished copying programs.') + except Exception as error: + success = False + logger.exception('Failed to copy programs for site: {}.'.format(site.domain)) + + try: + Command.fetch_pathways(site, client, page_size=page_size) + logger.info('Finished copying pathways.') + except Exception as error: + success = False + logger.exception('Failed to copy pathways for site: {}.'.format(site.domain)) + + if not success: + failure_count += 1 + + if failure_count: + raise CommandError( + 'Programs / Pathways copying for {failure_for_sites} out of {total_sites} sites have failed. ' + 'Please check the configuration.'.format( + failure_for_sites=failure_count, + total_sites=Site.objects.count(), + ) + ) @staticmethod def fetch_programs(site, client, page_size=None): diff --git a/credentials/apps/catalog/tests/test_commands.py b/credentials/apps/catalog/tests/test_commands.py index 115b128a1..74aa2227f 100644 --- a/credentials/apps/catalog/tests/test_commands.py +++ b/credentials/apps/catalog/tests/test_commands.py @@ -1,10 +1,12 @@ import ddt import responses import slumber -from django.core.management import call_command +from django.core.management import call_command, CommandError from django.test import TestCase from faker import Faker +from testfixtures import LogCapture +from credentials.apps.catalog.management.commands.copy_catalog import logger from credentials.apps.catalog.models import Course, CourseRun, Organization, Pathway, Program from credentials.apps.catalog.tests.factories import OrganizationFactory, ProgramFactory from credentials.apps.core.tests.factories import SiteConfigurationFactory @@ -231,8 +233,16 @@ def test_parse_error(self): self.mock_programs_response({}, 2, 1) self.mock_pathways_response(self.wrap_pathways(self.PATHWAYS)) - with self.assertRaises(KeyError): + with LogCapture(logger.name) as logs, \ + self.assertRaises(CommandError): self.call_command(page_size=1) + logs.check_present( + ( + logger.name, + 'ERROR', + 'Failed to copy programs for site: {}.'.format(self.site) + ) + ) self.assertFirstSaved() @@ -246,8 +256,16 @@ def test_server_error(self): self.mock_pathways_response(self.wrap_pathways(self.PATHWAYS)) self.mock_programs_response({}, 2, 1, status=500) - with self.assertRaises(slumber.exceptions.HttpServerError): + with LogCapture(logger.name) as logs, \ + self.assertRaises(CommandError): self.call_command(page_size=1) + logs.check_present( + ( + logger.name, + 'ERROR', + 'Failed to copy programs for site: {}.'.format(self.site) + ) + ) self.assertFirstSaved()