From 74c95df75f029fc88104b51d165fb4443e54c58a Mon Sep 17 00:00:00 2001 From: "sws-developers@lists.stanford.edu" Date: Wed, 5 Jan 2022 00:24:23 +0000 Subject: [PATCH 1/6] Back to dev --- stanford_migrate.info.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stanford_migrate.info.yml b/stanford_migrate.info.yml index ef0aee7..dfb760b 100755 --- a/stanford_migrate.info.yml +++ b/stanford_migrate.info.yml @@ -3,7 +3,7 @@ description: 'Adds more functionality to migrate and migrate plus modules' type: module core_version_requirement: ^8.8 || ^9 package: 'Stanford' -version: 8.x-1.20 +version: 8.x-1.21-dev dependencies: - drupal:migrate - migrate_plus:migrate_plus From 6123673e21d54dc6c68820a35c8c6253392da3bd Mon Sep 17 00:00:00 2001 From: pookmish Date: Thu, 6 Jan 2022 16:18:01 -0800 Subject: [PATCH 2/6] Deny field access to imported fields on nodes (#44) --- stanford_migrate.module | 92 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/stanford_migrate.module b/stanford_migrate.module index d02f47e..49e66df 100755 --- a/stanford_migrate.module +++ b/stanford_migrate.module @@ -5,16 +5,21 @@ * Contains stanford_migrate.module. */ +use Drupal\Core\Access\AccessResult; use Drupal\Core\Cache\Cache; use Drupal\Core\Entity\EntityInterface; +use Drupal\Core\Field\FieldDefinitionInterface; +use Drupal\Core\Field\FieldItemListInterface; use Drupal\Core\Installer\InstallerKernel; use Drupal\Core\Routing\RouteMatchInterface; +use Drupal\Core\Session\AccountInterface; use Drupal\migrate\Exception\RequirementsException; use Drupal\migrate\MigrateMessage; use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\Plugin\RequirementsInterface; use Drupal\migrate_plus\Entity\Migration; use Drupal\migrate_tools\MigrateExecutable; +use Drupal\node\NodeInterface; use Drupal\ultimate_cron\CronJobInterface; /** @@ -33,6 +38,93 @@ function stanford_migrate_help($route_name, RouteMatchInterface $route_match) { } } +/** + * Get the migration that imported the given node. + * + * @param \Drupal\node\NodeInterface $node + * Node entity. + * + * @return array|\Drupal\migrate_plus\Entity\MigrationInterface|mixed + * Migration entity or null/false if none found. + */ +function stanford_migrate_get_migration(NodeInterface $node) { + // Use a static variable so that it doesn't look up the migrations multiple + // times. + $node_migration = &drupal_static(__FUNCTION__ . $node->id()); + if (!is_null($node_migration)) { + return $node_migration; + } + // Set the static to false and check for null above. If the first attempt to + // find a migration doesn't show anything, we don't want to continue checking. + $node_migration = FALSE; + + $plugin_manager = \Drupal::service('plugin.manager.migration'); + + // Loop through the migration entities, build their migration plugins so that + // we can dig into their source mapping data. + /** @var \Drupal\migrate_plus\Entity\MigrationInterface $migration */ + foreach (Migration::loadMultiple() as $migration) { + + // This migrate entity has methods that allow easy queries on the + // migrate_map tables. + /** @var \Drupal\migrate\Plugin\MigrationInterface $migrate */ + $migrate = $plugin_manager->createInstance($migration->id()); + + // CSV Imported content can be ignored since it's normally a one time thing. + if ($migrate->getSourcePlugin()->getPluginId() == 'csv') { + continue; + } + + $destination_ids = $migrate->getDestinationPlugin()->getIds(); + + // Ignore any migrate plugin that doesn't map to nodes. + if (isset($destination_ids['nid'])) { + + // If the migrate id map returns something, that means this node is tied + // to this migration. Set the static variable for later references and + // get out of here. + if (!empty($migrate->getIdMap()->lookupSourceId(['nid' => $node->id()]))) { + $node_migration = $migration; + return $migration; + } + } + } + +} + +/** + * Implements hook_entity_field_access(). + */ +function stanford_migrate_entity_field_access($operation, FieldDefinitionInterface $field_definition, AccountInterface $account, FieldItemListInterface $items = NULL) { + $route_match = \Drupal::routeMatch(); + // When edit an existing node that was imported via migrate module, mark the + // fields that are mapped from migration as forbidden. + if ( + $operation == 'edit' && + $route_match->getRouteName() == 'entity.node.edit_form' && + $migration = stanford_migrate_get_migration($route_match->getParameter('node')) + ) { + $field_name = $field_definition->getName(); + $columns = $field_definition->getFieldStorageDefinition()->getColumns(); + $processing = !empty($migration->process[$field_name]); + + // This will check if a migrate process is mapped to a specific column on + // the field. + foreach (array_keys($columns) as $column) { + $processing = $processing ?: !empty($migration->process["$field_name/$column"]); + } + + if ($processing) { + \Drupal::messenger() + ->addWarning(t('Some fields can not be edited since they contain imported & synced data. They are not visible here.')); + + return AccessResult::forbidden((string) t('Field is mapped by the importer')); + } + } + + return AccessResult::neutral(); +} + /** * Implements hook_config_readonly_whitelist_patterns(). */ From eb16c0559ec07e9c07975facc23d1dc0667b544c Mon Sep 17 00:00:00 2001 From: pookmish Date: Mon, 10 Jan 2022 11:27:25 -0800 Subject: [PATCH 3/6] Migrate process plugin to adjust date values to 11:59pm (#46) --- .../migrate/process/SmartDateAllDayAdjust.php | 66 +++++++++++++++++ .../process/SmartDateAllDayAdjustTest.php | 74 +++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 src/Plugin/migrate/process/SmartDateAllDayAdjust.php create mode 100644 tests/src/Unit/Plugin/migrate/process/SmartDateAllDayAdjustTest.php diff --git a/src/Plugin/migrate/process/SmartDateAllDayAdjust.php b/src/Plugin/migrate/process/SmartDateAllDayAdjust.php new file mode 100644 index 0000000..e02ca89 --- /dev/null +++ b/src/Plugin/migrate/process/SmartDateAllDayAdjust.php @@ -0,0 +1,66 @@ +configuration['start_time'])) { + return $value; + } + $start = $row->get($this->configuration['start_time']); + if (empty($start)) { + return $value; + } + + if (!is_numeric($start)) { + $start = strtotime($start); + } + + if (!is_numeric($value)) { + $value = strtotime($value); + } + + // If the start and end values are the same & the start is at 12:00 midnight + // then increase the end value to be at the end of the day for the smart + // date module to work correctly. + if ((int) date('Gi', $start) == 0 && $start == $value) { + return $value + (60 * 60 * 24) - 60; + } + + // If either the start or the end date value are not at midnight, don't + // adjust the end value at all since it won't be considered an "All Day" + // date time. + if ((int) date('Gi', $start) || (int) date('Gi', $value)) { + return $value; + } + + // Now that the start and the end must both be at midnight and different + // days, reduce the end value by 1 minute to work correctly with the smart + // date module. + return $value - 60; + } + +} diff --git a/tests/src/Unit/Plugin/migrate/process/SmartDateAllDayAdjustTest.php b/tests/src/Unit/Plugin/migrate/process/SmartDateAllDayAdjustTest.php new file mode 100644 index 0000000..d2be613 --- /dev/null +++ b/tests/src/Unit/Plugin/migrate/process/SmartDateAllDayAdjustTest.php @@ -0,0 +1,74 @@ +createMock(MigrateExecutable::class); + $row = $this->createMock(Row::class); + $start_value = NULL; + $row->method('get')->willReturnReference($start_value); + + // Some regular values and the start time configuration isn't available. + $new_value = $plugin->transform(123456, $migrate_executable, $row, ''); + $this->assertEquals(123456, $new_value); + + // The start time configuration is set, but there's no value for it. + $configuration['start_time'] = 'start'; + $plugin = new SmartDateAllDayAdjust($configuration, 'smartdate_adjust', $definition); + $new_value = $plugin->transform(123456, $migrate_executable, $row, ''); + $this->assertEquals(123456, $new_value); + + // Now the start time exists, but it's not a midnight value. + $start_value = 123; + $new_value = $plugin->transform(123456, $migrate_executable, $row, ''); + $this->assertEquals(123456, $new_value); + + // The start and the end values are the exact same, so the end value should + // be 23 hours, 59 minutes ahead. + $timestamp = strtotime('Jan 10 2020 12:00am'); + $start_value = date('c', $timestamp); + $new_value = $plugin->transform($start_value, $migrate_executable, $row, ''); + $this->assertEquals($timestamp + 60 * 60 * 24 - 60, $new_value); + + // The end value is midnight of the next day, so it should be adjusted to + // 1 minute prior for it to be "all day" + $end_timestamp = strtotime('Jan 11 2020 12:00am'); + $end_value = date('c', $end_timestamp); + $new_value = $plugin->transform($end_value, $migrate_executable, $row, ''); + $this->assertEquals($end_timestamp - 60, $new_value); + + // Start time is midnight, but the end time is at 3PM. + $end_timestamp = strtotime('Jan 11 2020 3:00pm'); + $end_value = date('c', $end_timestamp); + $new_value = $plugin->transform($end_value, $migrate_executable, $row, ''); + $this->assertEquals($end_timestamp, $new_value); + + // Start time is not at midnight, but the end time is. + $start_value = date('c', strtotime('Jan 10 2020 8:00am')); + $end_timestamp = strtotime('Jan 11 2020 12:00am'); + $end_value = date('c', $end_timestamp); + $new_value = $plugin->transform($end_value, $migrate_executable, $row, ''); + $this->assertEquals($end_timestamp, $new_value); + } + +} From 05ab0350a6697a61d0ca3429ffc3d54a2a146b52 Mon Sep 17 00:00:00 2001 From: pookmish Date: Thu, 20 Jan 2022 15:17:14 -0800 Subject: [PATCH 4/6] D8CORE-5193 Add "Forget" orphan action (#47) --- .circleci/config.yml | 18 ------- src/EventSubscriber/EventsSubscriber.php | 29 +++++++++- stanford_migrate.module | 23 +++++++- .../EventSubscriber/EventsSubscriberTest.php | 54 ++++++++++++++++--- 4 files changed, 98 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62692fd..1cd8304 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,21 +47,6 @@ back_to_dev: &back_to_dev composer global require SU-SWS/stanford-caravan:dev-8.x-2.x ~/.composer/vendor/bin/sws-caravan back-to-dev ${CIRCLE_TAG} ${CIRCLE_WORKING_DIRECTORY} -d8_codeception: &d8_codeception - <<: *defaults - steps: - - checkout: - path: /var/www/test - - run: - name: Run Codeception Tests - command: | - composer global require SU-SWS/stanford-caravan:dev-8.x-1.x - ~/.composer/vendor/bin/sws-caravan codeception /var/www/html --extension-dir=/var/www/test - - store_test_results: - path: /var/www/html/artifacts/behat - - store_artifacts: - path: /var/www/html/artifacts - d9_codeception: &d9_codeception <<: *defaults steps: @@ -83,8 +68,6 @@ jobs: <<: *code_coverage run-back-to-dev: <<: *back_to_dev - run-d8-codeception: - <<: *d8_codeception run-d9-codeception: <<: *d9_codeception @@ -104,7 +87,6 @@ workflows: tests: jobs: - run-coverage - - run-d8-codeception - run-d9-codeception # Re-test every sunday in case this code becomes stale. sundays: diff --git a/src/EventSubscriber/EventsSubscriber.php b/src/EventSubscriber/EventsSubscriber.php index 75d09b2..fbeb50f 100644 --- a/src/EventSubscriber/EventsSubscriber.php +++ b/src/EventSubscriber/EventsSubscriber.php @@ -7,7 +7,9 @@ use Drupal\Core\StringTranslation\StringTranslationTrait; use Drupal\migrate\Event\MigrateEvents; use Drupal\migrate\Event\MigrateImportEvent; +use Drupal\migrate\Plugin\MigrateIdMapInterface; use Drupal\migrate\Plugin\MigrationInterface; +use Drupal\migrate\Row; use Symfony\Component\EventDispatcher\EventSubscriberInterface; use Drupal\Core\Entity\EntityTypeManagerInterface; @@ -28,6 +30,11 @@ class EventsSubscriber implements EventSubscriberInterface { */ const ORPHAN_UNPUBLISH = 'unpublish'; + /** + * If the migration is configured to unpublish orphans. + */ + const ORPHAN_FORGET = 'forget'; + /** * Drupal\Core\Entity\EntityTypeManagerInterface definition. * @@ -118,11 +125,20 @@ public function postImport(MigrateImportEvent $event) { $id_exists_in_source = FALSE; // Source key array of the already imported item. $source_id = $id_map->currentSource(); + $row = $id_map->getRowBySource($source_id); + + // The current item is already ignored, lets move on to the next one. This + // is skipped if the migration is ran with `--update` or via the UI with + // the "Update" checkbox checked. + if ($row['source_row_status'] == MigrateIdMapInterface::STATUS_IGNORED) { + $id_map->next(); + continue; + } // Look through the current source to see if we can find a match to the // existing item. foreach ($current_source_ids as $key => $ids) { - if ($ids == $source_id) { + if (md5(json_encode($ids)) == md5(json_encode($source_id))) { // The existing item is in the source, flag it as found and we can // reduce the current source ids to make subsequent lookups faster. unset($current_source_ids[$key]); @@ -169,6 +185,17 @@ public function postImport(MigrateImportEvent $event) { $id_map->delete($id_map->currentSource()); break; + // Tell the migration to ignore the given source ids. + case self::ORPHAN_FORGET: + $this->logger->notice($this->t('Entity since it no longer exists in the source data, it will be now be ignored. Migration: @migration, Entity Type: @entity_type, Label: @label'), [ + '@migration' => $event->getMigration()->label(), + '@entity_type' => $type, + '@label' => $entity->label(), + ]); + $old_row = new Row($id_map->currentSource(), $id_map->currentSource(), TRUE); + $id_map->saveIdMapping($old_row, [], MigrateIdMapInterface::STATUS_IGNORED); + break; + case self::ORPHAN_UNPUBLISH: // Unpublish the orphan only if it is currently published. if ( diff --git a/stanford_migrate.module b/stanford_migrate.module index 49e66df..933658b 100755 --- a/stanford_migrate.module +++ b/stanford_migrate.module @@ -15,6 +15,7 @@ use Drupal\Core\Routing\RouteMatchInterface; use Drupal\Core\Session\AccountInterface; use Drupal\migrate\Exception\RequirementsException; use Drupal\migrate\MigrateMessage; +use Drupal\migrate\Plugin\MigrateIdMapInterface; use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\Plugin\RequirementsInterface; use Drupal\migrate_plus\Entity\Migration; @@ -83,7 +84,8 @@ function stanford_migrate_get_migration(NodeInterface $node) { // If the migrate id map returns something, that means this node is tied // to this migration. Set the static variable for later references and // get out of here. - if (!empty($migrate->getIdMap()->lookupSourceId(['nid' => $node->id()]))) { + $row_data = $migrate->getIdMap()->getRowByDestination(['nid' => $node->id()]); + if (!empty($row_data) && $row_data['source_row_status'] != MigrateIdMapInterface::STATUS_IGNORED) { $node_migration = $migration; return $migration; } @@ -114,6 +116,25 @@ function stanford_migrate_entity_field_access($operation, FieldDefinitionInterfa $processing = $processing ?: !empty($migration->process["$field_name/$column"]); } + // If the migration destination has the `overwrite_properties` configured, + // those fields specifically should be locked, not the other fields that + // are not designated in the original process configuration. + if ($processing && !empty($migration->get('destination')['overwrite_properties'])) { + // If the current field doesn't exist in the overwrite_properties, it + // should not be considered to be processing since it's a one time only + // import. + $processing = FALSE; + + foreach ($migration->get('destination')['overwrite_properties'] as $overwrite_property) { + // If any part of the field is set to overwrite, lock the whole field + // down. + $overwrite_property = strstr($overwrite_property, '/', TRUE) ?: $overwrite_property; + if ($field_name == $overwrite_property) { + $processing = TRUE; + } + } + } + if ($processing) { \Drupal::messenger() ->addWarning(t('Some fields can not be edited since they contain imported & synced data. They are not visible here.')); diff --git a/tests/src/Kernel/EventSubscriber/EventsSubscriberTest.php b/tests/src/Kernel/EventSubscriber/EventsSubscriberTest.php index e98a1df..ed5b8a2 100644 --- a/tests/src/Kernel/EventSubscriber/EventsSubscriberTest.php +++ b/tests/src/Kernel/EventSubscriber/EventsSubscriberTest.php @@ -3,6 +3,7 @@ namespace Drupal\Tests\stanford_migrate\Kernel\EventSubscriber; use Drupal\migrate\MigrateExecutable; +use Drupal\migrate\Plugin\MigrateIdMapInterface; use Drupal\Tests\stanford_migrate\Kernel\StanfordMigrateKernelTestBase; class EventsSubscriberTest extends StanfordMigrateKernelTestBase { @@ -24,10 +25,10 @@ protected function setUp(): void { public function testEventSubscriber() { $migrate = $this->getMigrateExecutable(); $this->assertEquals(1, $migrate->import()); - $this->assertEqual(1, $this->getNodeCount()); + $this->assertEquals(1, $this->getNodeCount()); $migrate->import(); - $this->assertEqual(1, $this->getNodeCount()); + $this->assertEquals(1, $this->getNodeCount()); } /** @@ -36,7 +37,7 @@ public function testEventSubscriber() { public function testDeleteAction() { $migrate = $this->getMigrateExecutable(); $migrate->import(); - $this->assertEqual(1, $this->getNodeCount()); + $this->assertEquals(1, $this->getNodeCount()); \Drupal::configFactory() ->getEditable('migrate_plus.migration.stanford_migrate') ->set('source.urls', []) @@ -47,7 +48,7 @@ public function testDeleteAction() { $migrate = $this->getMigrateExecutable(); $migrate->import(); - $this->assertEqual(0, $this->getNodeCount()); + $this->assertEquals(0, $this->getNodeCount()); } /** @@ -56,7 +57,7 @@ public function testDeleteAction() { public function testUnpublishAction() { $migrate = $this->getMigrateExecutable(); $migrate->import(); - $this->assertEqual(1, $this->getNodeCount()); + $this->assertEquals(1, $this->getNodeCount()); \Drupal::configFactory() ->getEditable('migrate_plus.migration.stanford_migrate') ->set('source.urls', [__DIR__ . '/../test2.xml']) @@ -67,7 +68,7 @@ public function testUnpublishAction() { $migrate = $this->getMigrateExecutable(); $migrate->import(); - $this->assertEqual(2, $this->getNodeCount()); + $this->assertEquals(2, $this->getNodeCount()); $unpublished_nodes = \Drupal::entityTypeManager() ->getStorage('node') @@ -75,6 +76,47 @@ public function testUnpublishAction() { $this->assertCount(1, $unpublished_nodes); } + /** + * Forget Orphan action test. + */ + public function testForgetAction() { + $migrate = $this->getMigrateExecutable(); + $migrate->import(); + $this->assertEquals(1, $this->getNodeCount()); + \Drupal::configFactory() + ->getEditable('migrate_plus.migration.stanford_migrate') + ->set('source.urls', [__DIR__ . '/../test2.xml']) + ->set('source.orphan_action', 'forget') + ->save(); + + drupal_flush_all_caches(); + + $migrate = $this->getMigrateExecutable(); + $migrate->import(); + drupal_flush_all_caches(); + $migrate->import(); + $migrate->import(); + $this->assertEquals(2, $this->getNodeCount()); + + $manager = \Drupal::service('plugin.manager.migration'); + /** @var \Drupal\migrate\Plugin\Migration $migration */ + $migration = $manager->createInstance('stanford_migrate'); + $id_map = $migration->getIdMap(); + $id_map->rewind(); + + $number_ignored = 0; + while ($id_map->current()) { + $row = $id_map->getRowBySource($id_map->currentSource()); + if ($row['source_row_status'] == MigrateIdMapInterface::STATUS_IGNORED) { + $number_ignored++; + } + $id_map->next();; + } + $this->assertGreaterThanOrEqual(2, $id_map->processedCount()); + $this->assertGreaterThanOrEqual(1, $number_ignored); + $this->assertNotEquals($number_ignored, $id_map->processedCount()); + } + /** * Get the migration executable. */ From 1eac4195644439d2cc3bb63c26f7262dbec7aada Mon Sep 17 00:00:00 2001 From: Mike Decker Date: Thu, 27 Jan 2022 11:08:29 -0800 Subject: [PATCH 5/6] Fixed the orphan action to compare string and interger values --- src/EventSubscriber/EventsSubscriber.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/EventSubscriber/EventsSubscriber.php b/src/EventSubscriber/EventsSubscriber.php index fbeb50f..03f3d10 100644 --- a/src/EventSubscriber/EventsSubscriber.php +++ b/src/EventSubscriber/EventsSubscriber.php @@ -138,7 +138,10 @@ public function postImport(MigrateImportEvent $event) { // Look through the current source to see if we can find a match to the // existing item. foreach ($current_source_ids as $key => $ids) { - if (md5(json_encode($ids)) == md5(json_encode($source_id))) { + if ( + (is_array($ids) && is_array($source_id) && empty(array_diff($ids, $source_id))) || + $ids == $source_id + ) { // The existing item is in the source, flag it as found and we can // reduce the current source ids to make subsequent lookups faster. unset($current_source_ids[$key]); @@ -239,7 +242,7 @@ protected function getOrphanAction(MigrationInterface $migration) { // temporary flag that the orphan action has recently occurred. This // will prevent the unnecessary double execution. if ($this->cache->get($cid)) { - return FALSE; +// return FALSE; } $source_config = $migration->getSourceConfiguration(); From 7db6ee8642781831b036e7936f3e6958e84eb551 Mon Sep 17 00:00:00 2001 From: Mike Decker Date: Thu, 27 Jan 2022 16:00:41 -0800 Subject: [PATCH 6/6] 8.1.21 --- CHANGELOG.md | 10 ++++++++++ stanford_migrate.info.yml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36f4003..309af6c 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ # Stanford Migrate +8.x-1.21 +-------------------------------------------------------------------------------- +_Release Date: 2022-01-27_ + +- Fixed the orphan action to compare string and interger values +- D8CORE-5193 Add "Forget" orphan action (#47) +- Migrate process plugin to adjust date values to 11:59pm (#46) +- Deny field access to imported fields on nodes (#44) + + 8.x-1.20 -------------------------------------------------------------------------------- _Release Date: 2022-01-04_ diff --git a/stanford_migrate.info.yml b/stanford_migrate.info.yml index dfb760b..e14115d 100755 --- a/stanford_migrate.info.yml +++ b/stanford_migrate.info.yml @@ -3,7 +3,7 @@ description: 'Adds more functionality to migrate and migrate plus modules' type: module core_version_requirement: ^8.8 || ^9 package: 'Stanford' -version: 8.x-1.21-dev +version: 8.x-1.21 dependencies: - drupal:migrate - migrate_plus:migrate_plus