From 2bed2bceb441f47327314430d0c99aa247148885 Mon Sep 17 00:00:00 2001 From: Seth Shaw Date: Wed, 17 Mar 2021 16:38:04 +0000 Subject: [PATCH 1/6] Increase size of media.field_file_size. --- .../field.storage.media.field_file_size.yml | 4 +- .../islandora_core_feature.install | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 modules/islandora_core_feature/islandora_core_feature.install diff --git a/modules/islandora_core_feature/config/install/field.storage.media.field_file_size.yml b/modules/islandora_core_feature/config/install/field.storage.media.field_file_size.yml index 5c3e67c4..fff956db 100644 --- a/modules/islandora_core_feature/config/install/field.storage.media.field_file_size.yml +++ b/modules/islandora_core_feature/config/install/field.storage.media.field_file_size.yml @@ -11,8 +11,8 @@ field_name: field_file_size entity_type: media type: integer settings: - unsigned: false - size: normal + unsigned: true + size: big module: core locked: false cardinality: 1 diff --git a/modules/islandora_core_feature/islandora_core_feature.install b/modules/islandora_core_feature/islandora_core_feature.install new file mode 100644 index 00000000..6dd19fd0 --- /dev/null +++ b/modules/islandora_core_feature/islandora_core_feature.install @@ -0,0 +1,67 @@ +get($storage_key); + + foreach (["{$entity_type}__{$field_name}", "{$entity_type}_revision__{$field_name}"] as $table) { + // Squirrel away the data. + $data[$table] = $database->select($table, 'n') + ->fields('n') + ->execute() + ->fetchAll(); + + // Clean it out for resizing. + $database->truncate($table)->execute(); + $database->schema()->dropPrimaryKey($table); + + // Resize the main field data table. + $database->query("ALTER TABLE $table MODIFY {$field_name}_value BIGINT(11) unsigned NOT NULL"); + + // Update storage schema. + $field_schema[$table]['fields'][$field_name . '_value']['type'] = "bigint"; + $field_schema[$table]['fields'][$field_name . '_value']['unsigned'] = "true"; + } + $storage_schema->set($storage_key, $field_schema); + + // Update field storage configuration. + $config = \Drupal::configFactory() + ->getEditable("field.storage.{$entity_type}.{$field_name}"); + $config->set('settings.size', 'big'); + $config->set('settings.unsigned', TRUE); + $config->save(TRUE); + + // Make sure the new config persists. + FieldStorageConfig::loadByName($entity_type, $field_name)->save(); + + // Reload the data. + foreach ($data as $table => $rows) { + foreach ($rows as $row) { + $database->insert($table) + ->fields((array) $row) + ->execute(); + } + } + + return t('Length of @entity-type.@field-name updated to an unsigned big int', [ + '@entity-type' => $entity_type, + '@field-name' => $field_name, + ]); +} From 781d0c4e3e12a04ce741d85a96284ffaefa70062 Mon Sep 17 00:00:00 2001 From: Seth Shaw Date: Wed, 17 Mar 2021 19:12:39 +0000 Subject: [PATCH 2/6] coding style --- .../islandora_core_feature/islandora_core_feature.install | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/islandora_core_feature/islandora_core_feature.install b/modules/islandora_core_feature/islandora_core_feature.install index 6dd19fd0..0f6098a7 100644 --- a/modules/islandora_core_feature/islandora_core_feature.install +++ b/modules/islandora_core_feature/islandora_core_feature.install @@ -21,7 +21,10 @@ function islandora_core_feature_update_8001() { $storage_schema = \Drupal::keyValue('entity.storage_schema.sql'); $field_schema = $storage_schema->get($storage_key); - foreach (["{$entity_type}__{$field_name}", "{$entity_type}_revision__{$field_name}"] as $table) { + foreach ([ + "{$entity_type}__{$field_name}", + "{$entity_type}_revision__{$field_name}", + ] as $table) { // Squirrel away the data. $data[$table] = $database->select($table, 'n') ->fields('n') From f0d0d909a474bf340e59d64a48e3a4e8ac7ce76f Mon Sep 17 00:00:00 2001 From: Seth Shaw Date: Wed, 17 Mar 2021 21:36:55 +0000 Subject: [PATCH 3/6] Use temporary files for update and schema::changeField() --- .../islandora_core_feature.install | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/modules/islandora_core_feature/islandora_core_feature.install b/modules/islandora_core_feature/islandora_core_feature.install index 0f6098a7..bad5ccf1 100644 --- a/modules/islandora_core_feature/islandora_core_feature.install +++ b/modules/islandora_core_feature/islandora_core_feature.install @@ -16,33 +16,31 @@ function islandora_core_feature_update_8001() { $field_name = 'field_file_size'; $database = \Drupal::database(); - $data = []; - $storage_key = $entity_type . '.field_schema_data.' . $field_name; - $storage_schema = \Drupal::keyValue('entity.storage_schema.sql'); - $field_schema = $storage_schema->get($storage_key); - - foreach ([ + $tables = [ "{$entity_type}__{$field_name}", "{$entity_type}_revision__{$field_name}", - ] as $table) { + ]; + + foreach ($tables as $table) { + // Squirrel away the data. - $data[$table] = $database->select($table, 'n') - ->fields('n') - ->execute() - ->fetchAll(); + $table_file_path = "public://islandora_core_feature_update_8001_{$table}.json"; + if (!file_exists($table_file_path)) { + $table_data = $database->select($table, 'n')->fields('n')->execute()->fetchAll(); + \Drupal::service('file_system')->saveData(json_encode($table_data), $table_file_path); + } // Clean it out for resizing. $database->truncate($table)->execute(); - $database->schema()->dropPrimaryKey($table); - // Resize the main field data table. - $database->query("ALTER TABLE $table MODIFY {$field_name}_value BIGINT(11) unsigned NOT NULL"); + $database->schema()->changeField($table, $field_name . '_value', $field_name . '_value', [ + 'type' => 'int', + 'size' => 'big', + 'unsigned' => TRUE, + 'not null' => TRUE, + ]); - // Update storage schema. - $field_schema[$table]['fields'][$field_name . '_value']['type'] = "bigint"; - $field_schema[$table]['fields'][$field_name . '_value']['unsigned'] = "true"; } - $storage_schema->set($storage_key, $field_schema); // Update field storage configuration. $config = \Drupal::configFactory() @@ -55,12 +53,18 @@ function islandora_core_feature_update_8001() { FieldStorageConfig::loadByName($entity_type, $field_name)->save(); // Reload the data. - foreach ($data as $table => $rows) { - foreach ($rows as $row) { - $database->insert($table) - ->fields((array) $row) - ->execute(); + foreach ($tables as $table) { + $table_file_path = "public://islandora_core_feature_update_8001_{$table}.json"; + if (file_exists($table_file_path)) { + foreach (json_decode(file_get_contents($table_file_path), TRUE) as $row) { + $database->insert($table) + ->fields((array) $row) + ->execute(); + } + // Clean up. + \Drupal::service('file_system')->delete($table_file_path); } + } return t('Length of @entity-type.@field-name updated to an unsigned big int', [ From a52617e99d9a526e5a01533762083122432f656c Mon Sep 17 00:00:00 2001 From: Seth Shaw Date: Fri, 26 Mar 2021 05:53:50 +0000 Subject: [PATCH 4/6] Rename, recreate, and reload tables instead of using temporary files. --- .../islandora_core_feature.install | 80 +++++++++++++------ 1 file changed, 54 insertions(+), 26 deletions(-) diff --git a/modules/islandora_core_feature/islandora_core_feature.install b/modules/islandora_core_feature/islandora_core_feature.install index bad5ccf1..c7ccde2e 100644 --- a/modules/islandora_core_feature/islandora_core_feature.install +++ b/modules/islandora_core_feature/islandora_core_feature.install @@ -23,23 +23,59 @@ function islandora_core_feature_update_8001() { foreach ($tables as $table) { - // Squirrel away the data. - $table_file_path = "public://islandora_core_feature_update_8001_{$table}.json"; - if (!file_exists($table_file_path)) { - $table_data = $database->select($table, 'n')->fields('n')->execute()->fetchAll(); - \Drupal::service('file_system')->saveData(json_encode($table_data), $table_file_path); - } + // Move the existing data out of the way. + $database->schema()->renameTable($table, $table . '_o'); - // Clean it out for resizing. - $database->truncate($table)->execute(); + // Create replacement tables. + $database->schema()->createTable($table, [ + 'fields' => [ + 'bundle' => [ + 'type' => 'varchar', + 'not null' => TRUE, + 'length' => 128, + 'default' => ' ', + ], + 'deleted' => [ + 'type' => 'int', + 'size' => 'tiny', + 'not null' => TRUE, + 'default' => 0, + ], + 'entity_id' => [ + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'length' => 10, - $database->schema()->changeField($table, $field_name . '_value', $field_name . '_value', [ - 'type' => 'int', - 'size' => 'big', - 'unsigned' => TRUE, - 'not null' => TRUE, + ], + 'revision_id' => [ + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'length' => 10, + ], + 'langcode' => [ + 'type' => 'varchar', + 'not null' => TRUE, + 'length' => 32, + 'default' => ' ', + ], + 'delta' => [ + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'length' => 10, + ], + 'field_file_size_value' => [ + 'type' => 'int', + 'size' => 'big', + 'unsigned' => TRUE, + 'not null' => TRUE, + ], + ], + 'primary key' => ['entity_id', 'deleted', 'delta', 'langcode'], + 'indexes' => ['bundle' => ['bundle'], 'revision_id' => ['revision_id']], ]); - } // Update field storage configuration. @@ -52,19 +88,11 @@ function islandora_core_feature_update_8001() { // Make sure the new config persists. FieldStorageConfig::loadByName($entity_type, $field_name)->save(); - // Reload the data. + // Reload the data and clean up. foreach ($tables as $table) { - $table_file_path = "public://islandora_core_feature_update_8001_{$table}.json"; - if (file_exists($table_file_path)) { - foreach (json_decode(file_get_contents($table_file_path), TRUE) as $row) { - $database->insert($table) - ->fields((array) $row) - ->execute(); - } - // Clean up. - \Drupal::service('file_system')->delete($table_file_path); - } - + $query = $database->select($table . '_o', 'o')->fields('o'); + $database->insert($table)->from($query)->execute(); + $database->schema()->dropTable($table . '_o'); } return t('Length of @entity-type.@field-name updated to an unsigned big int', [ From a8f8e40371a8fc1151af78c2e904f6631f9b2bc4 Mon Sep 17 00:00:00 2001 From: Seth Shaw Date: Fri, 26 Mar 2021 18:18:45 +0000 Subject: [PATCH 5/6] Batch the update --- .../islandora_core_feature.install | 170 +++++++++++------- 1 file changed, 101 insertions(+), 69 deletions(-) diff --git a/modules/islandora_core_feature/islandora_core_feature.install b/modules/islandora_core_feature/islandora_core_feature.install index c7ccde2e..0b0f5c3e 100644 --- a/modules/islandora_core_feature/islandora_core_feature.install +++ b/modules/islandora_core_feature/islandora_core_feature.install @@ -10,9 +10,10 @@ use Drupal\field\Entity\FieldStorageConfig; /** * Updates Media file size field storage for larger files. */ -function islandora_core_feature_update_8001() { +function islandora_core_feature_update_8001(&$sandbox) { $entity_type = 'media'; + $field_name = 'field_file_size'; $database = \Drupal::database(); @@ -21,82 +22,113 @@ function islandora_core_feature_update_8001() { "{$entity_type}_revision__{$field_name}", ]; - foreach ($tables as $table) { + if (!isset($sandbox['progress'])) { + $sandbox['upper_limit'] = 0; + foreach ($tables as $table) { - // Move the existing data out of the way. - $database->schema()->renameTable($table, $table . '_o'); - - // Create replacement tables. - $database->schema()->createTable($table, [ - 'fields' => [ - 'bundle' => [ - 'type' => 'varchar', - 'not null' => TRUE, - 'length' => 128, - 'default' => ' ', - ], - 'deleted' => [ - 'type' => 'int', - 'size' => 'tiny', - 'not null' => TRUE, - 'default' => 0, - ], - 'entity_id' => [ - 'type' => 'int', - 'unsigned' => TRUE, - 'not null' => TRUE, - 'length' => 10, + // Record size of the table for future use. + $sandbox[$table]['size'] = $database->select($table) + ->countQuery() + ->execute() + ->fetchField(); + if ($sandbox[$table]['size'] > $sandbox['upper_limit']) { + $sandbox['upper_limit'] = $sandbox[$table]['size']; + } + // Move the existing data out of the way. + $database->schema()->renameTable($table, $table . '_o'); + // Create replacement tables. + $database->schema()->createTable($table, [ + 'fields' => [ + 'bundle' => [ + 'type' => 'varchar', + 'not null' => TRUE, + 'length' => 128, + 'default' => ' ', + ], + 'deleted' => [ + 'type' => 'int', + 'size' => 'tiny', + 'not null' => TRUE, + 'default' => 0, + ], + 'entity_id' => [ + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'length' => 10, + + ], + 'revision_id' => [ + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'length' => 10, + ], + 'langcode' => [ + 'type' => 'varchar', + 'not null' => TRUE, + 'length' => 32, + 'default' => ' ', + ], + 'delta' => [ + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + 'length' => 10, + ], + 'field_file_size_value' => [ + 'type' => 'int', + 'size' => 'big', + 'unsigned' => TRUE, + 'not null' => TRUE, + ], ], - 'revision_id' => [ - 'type' => 'int', - 'unsigned' => TRUE, - 'not null' => TRUE, - 'length' => 10, - ], - 'langcode' => [ - 'type' => 'varchar', - 'not null' => TRUE, - 'length' => 32, - 'default' => ' ', - ], - 'delta' => [ - 'type' => 'int', - 'unsigned' => TRUE, - 'not null' => TRUE, - 'length' => 10, - ], - 'field_file_size_value' => [ - 'type' => 'int', - 'size' => 'big', - 'unsigned' => TRUE, - 'not null' => TRUE, - ], - ], - 'primary key' => ['entity_id', 'deleted', 'delta', 'langcode'], - 'indexes' => ['bundle' => ['bundle'], 'revision_id' => ['revision_id']], - ]); - } + 'primary key' => ['entity_id', 'deleted', 'delta', 'langcode'], + 'indexes' => ['bundle' => ['bundle'], 'revision_id' => ['revision_id']], + ]); + + } + + // Update field storage configuration. + $config = \Drupal::configFactory() + ->getEditable("field.storage.{$entity_type}.{$field_name}"); + $config->set('settings.size', 'big'); + $config->set('settings.unsigned', TRUE); + $config->save(TRUE); - // Update field storage configuration. - $config = \Drupal::configFactory() - ->getEditable("field.storage.{$entity_type}.{$field_name}"); - $config->set('settings.size', 'big'); - $config->set('settings.unsigned', TRUE); - $config->save(TRUE); + // Make sure the new config persists. + FieldStorageConfig::loadByName($entity_type, $field_name)->save(); - // Make sure the new config persists. - FieldStorageConfig::loadByName($entity_type, $field_name)->save(); + // Track batch reload progress. + $sandbox['progress'] = 0; + } + + // Arbitrary, hopefully reasonable, batch size per table. + $limit = 500; // Reload the data and clean up. foreach ($tables as $table) { - $query = $database->select($table . '_o', 'o')->fields('o'); - $database->insert($table)->from($query)->execute(); - $database->schema()->dropTable($table . '_o'); + if (!isset($sandbox[$table]['done'])) { + $query = $database->select($table . '_o', 'o')->fields('o') + ->orderBy('o.entity_id', 'ASC') + ->orderBy('o.revision_id', 'ASC') + ->orderBy('o.delta', 'ASC') + ->range($sandbox['progress'], $sandbox['progress'] + $limit); + $database->insert($table)->from($query)->execute(); + $database->schema()->dropTable($table . '_o'); + if ($sandbox['progress'] + $limit >= $progress[$table]['size']) { + $sandbox[$table]['done'] = TRUE; + } + } } - return t('Length of @entity-type.@field-name updated to an unsigned big int', [ - '@entity-type' => $entity_type, - '@field-name' => $field_name, - ]); + $sandbox['progress'] = $sandbox['progress'] + $limit; + $sandbox['#finished'] = $sandbox['progress'] >= $sandbox['upper_limit'] ? 1 : $sandbox['progress'] / $sandbox['upper_limit']; + if ($sandbox['#finished'] == 1) { + return t('Length of @entity-type.@field-name updated to an unsigned big int', [ + '@entity-type' => $entity_type, + '@field-name' => $field_name, + ]); + } } From 4a0a47c802f1ec4f591f60cf0c446ad168daee7a Mon Sep 17 00:00:00 2001 From: Seth Shaw Date: Sat, 27 Mar 2021 01:40:40 +0000 Subject: [PATCH 6/6] drop renamed tables *after* we are done copying --- .../islandora_core_feature.install | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/islandora_core_feature/islandora_core_feature.install b/modules/islandora_core_feature/islandora_core_feature.install index 0b0f5c3e..3585dafa 100644 --- a/modules/islandora_core_feature/islandora_core_feature.install +++ b/modules/islandora_core_feature/islandora_core_feature.install @@ -107,7 +107,7 @@ function islandora_core_feature_update_8001(&$sandbox) { // Arbitrary, hopefully reasonable, batch size per table. $limit = 500; - // Reload the data and clean up. + // Reload the data. foreach ($tables as $table) { if (!isset($sandbox[$table]['done'])) { $query = $database->select($table . '_o', 'o')->fields('o') @@ -116,16 +116,19 @@ function islandora_core_feature_update_8001(&$sandbox) { ->orderBy('o.delta', 'ASC') ->range($sandbox['progress'], $sandbox['progress'] + $limit); $database->insert($table)->from($query)->execute(); - $database->schema()->dropTable($table . '_o'); - if ($sandbox['progress'] + $limit >= $progress[$table]['size']) { - $sandbox[$table]['done'] = TRUE; - } } } $sandbox['progress'] = $sandbox['progress'] + $limit; $sandbox['#finished'] = $sandbox['progress'] >= $sandbox['upper_limit'] ? 1 : $sandbox['progress'] / $sandbox['upper_limit']; if ($sandbox['#finished'] == 1) { + // Clean up. + foreach ($tables as $table) { + $database->schema()->dropTable($table . '_o'); + if ($sandbox['progress'] + $limit >= $progress[$table]['size']) { + $sandbox[$table]['done'] = TRUE; + } + } return t('Length of @entity-type.@field-name updated to an unsigned big int', [ '@entity-type' => $entity_type, '@field-name' => $field_name,