Merge branch 'drush'

added queueing annd shards

README.md

@@ -1,6 +1,36 @@
-## INTRODUCTION
+# Islandora In-Place Media
+
+A Drush-driven ingestion tool for Islandora that creates Media entities
+from files already on disk — **without moving them**.
+
+Designed for **large-scale, resumable, idempotent, parallel-safe** ingest
+workflows.
+
+---
+
+## ✨ Features
+
+* ✅ In-place ingestion (no file copying unless paths differ)
+* ✅ Idempotent (safe to re-run; no duplicate Media)
+* ✅ Automatic resume after failure
+* ✅ Drush command interface
+* ✅ Queue-based background processing
+* ✅ Concurrency-safe sharding for parallel workers
+* ✅ Explicit ownership (uid = 1, configurable later)
+* ✅ Works with cron or manual queue runners
+
+---
+
+## 📦 Requirements
+
+* Drupal 10 or 11
+* Drush 10+
+* Islandora Media bundles already configured
+* Files already present on disk (e.g. `public://`, `private://`, or mounted paths)
+
+---
+
 
-The Islandora Inplace Media module allows users to create and attach media from files staged on the server.
 
 ## INSTALLATION
 

drush.services.yml

@@ -0,0 +1,8 @@
+services:
+  islandora_inplace_media.commands:
+    class: Drupal\islandora_inplace_media\Commands\IslandoraInplaceMediaCommands
+    arguments:
+      - '@islandora_inplace_media.processor'
+      - '@state'
+    tags:
+      - { name: drush.command }

islandora_inplace_media.info.yml

@@ -1,7 +1,7 @@
 name: 'Islandora Inplace Media'
 type: module
-description: 'Allows Uploaded files to be attched to exisitng nodes as media'
-package: Custom
+description: 'Allows Uploaded files to be attached to existing nodes as media'
+package: Islandora
 core_version_requirement: ^10 || ^11
 dependencies:
   - islandora:islandora

islandora_inplace_media.services.yml

@@ -1,4 +1,14 @@
 services:
-  islandora_inplace_media.utils:
-    class: Drupal\islandora_inplace_media\Utils
-    arguments: ['@entity_type.manager', '@file_system', '@logger.factory', '@entity_field.manager']
+
+  logger.channel.islandora_inplace_media:
+    class: Drupal\Core\Logger\LoggerChannel
+    factory: logger.factory:get
+    arguments: ['islandora_inplace_media']
+
+  islandora_inplace_media.processor:
+    class: Drupal\islandora_inplace_media\Service\InplaceMediaProcessor
+    arguments:
+      - '@file_system'
+      - '@file.repository'
+      - '@logger.channel.islandora_inplace_media'
+      - '@entity_type.manager'

src/Commands/IslandoraInplaceMediaCommands.php

@@ -0,0 +1,109 @@
+<?php
+
+namespace Drupal\islandora_inplace_media\Commands;
+
+use Drush\Commands\DrushCommands;
+use Drupal\islandora_inplace_media\Service\InplaceMediaProcessor;
+use Drupal\Core\State\StateInterface;
+use Symfony\Component\Console\Helper\ProgressBar;
+
+class IslandoraInplaceMediaCommands extends DrushCommands {
+
+  protected InplaceMediaProcessor $processor;
+  protected StateInterface $state;
+
+  public function __construct(
+    InplaceMediaProcessor $processor,
+    StateInterface $state,
+  ) {
+    parent::__construct();
+    $this->processor = $processor;
+    $this->state = $state;
+  }
+
+  /**
+   * Create Islandora media from files.
+   *
+   * @command islandora:inplace-media
+   * @aliases iim
+   *
+   * @option source_dir
+   *   Source directory containing files.
+   * @option destination_path
+   *   Destination directory (public://, private://, or absolute).
+   * @option media_type
+   *   Media bundle machine name.
+   * @option media_use
+   *   Taxonomy term ID for field_media_use.
+   * @option file_type
+   *   Media file field name.
+   * @option limit
+   *   Maximum number of files to process.
+   * @option offset
+   *   Number of files to skip before processing.
+   * @option queue
+   *   Queue files instead of processing immediately.
+   * @option shards
+   *   Total number of queue shards.
+   */
+
+  public function run(array $options = [
+    'source_dir' => NULL,
+    'destination_path' => NULL,
+    'media_type' => NULL,
+    'media_use' => NULL,
+    'file_type' => 'field_media_file',
+    'limit' => NULL,
+    'offset' => 0,
+    'queue' => FALSE,
+    'shards' => 1,
+  ]) {
+    $files = array_values(array_diff(
+      scandir($options['source_dir']),
+      ['.', '..']
+    ));
+
+    $job_id = hash('sha256', serialize([
+      $options['source_dir'],
+      $options['destination_path'],
+      $options['media_type'],
+      $options['media_use'],
+      $options['file_type'],
+    ]));
+
+    $state_key = "islandora_inplace_media.completed.$job_id";
+    $completed = $this->state->get($state_key, []);
+
+    $files = array_values(array_diff($files, $completed));
+
+    if ($options['queue']) {
+      foreach ($files as $file) {
+        $shard = crc32($file) % (int) $options['shards'];
+        \Drupal::queue('islandora_inplace_media')
+          ->createItem([
+            'file' => $file,
+            'options' => $options,
+            'job_id' => $job_id,
+            'shard' => $shard,
+          ]);
+      }
+      $this->output()->writeln('Files queued with sharding.');
+      return;
+    }
+
+    $progress = new ProgressBar($this->output(), count($files));
+    $progress->start();
+
+    foreach ($files as $file) {
+      if ($this->processor->processFile($file, $options)) {
+        $completed[] = $file;
+        $this->state->set($state_key, $completed);
+      }
+      $progress->advance();
+    }
+
+    $progress->finish();
+    $this->output()->writeln('');
+  }
+
+}

src/Plugin/QueueWorker/InplaceMediaQueueWorker.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace Drupal\islandora_inplace_media\Plugin\QueueWorker;
+
+use Drupal\Core\Queue\QueueWorkerBase;
+
+/**
+ * @QueueWorker(
+ *   id = "islandora_inplace_media",
+ *   title = @Translation("Islandora Inplace Media"),
+ *   cron = {"time" = 60}
+ * )
+ */
+class InplaceMediaQueueWorker extends QueueWorkerBase {
+
+  public function processItem($data) {
+    // Optional shard filtering (used by drush queue:run --shard).
+    if (isset($data['shard']) && isset($_SERVER['DRUSH_SHARD'])) {
+      if ((int) $_SERVER['DRUSH_SHARD'] !== (int) $data['shard']) {
+        return;
+      }
+    }
+
+    $processor = \Drupal::service('islandora_inplace_media.processor');
+    $state = \Drupal::state();
+
+    if ($processor->processFile($data['file'], $data['options'])) {
+      $key = "islandora_inplace_media.completed.{$data['job_id']}";
+      $completed = $state->get($key, []);
+      $completed[] = $data['file'];
+      $state->set($key, $completed);
+    }
+  }
+
+}

src/Service/InplaceMediaProcessor.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace Drupal\islandora_inplace_media\Service;
+
+use Drupal\Core\File\FileSystemInterface;
+use Drupal\file\FileRepositoryInterface;
+use Drupal\Core\File\FileExists;
+use Drupal\file\Entity\File;
+use Drupal\media\Entity\Media;
+use Drupal\Core\Entity\EntityTypeManagerInterface;
+use Psr\Log\LoggerInterface;
+
+class InplaceMediaProcessor {
+
+  protected $mediaStorage;
+
+  public function __construct(
+    FileSystemInterface $fileSystem,
+    FileRepositoryInterface $fileRepository,
+    LoggerInterface $logger,
+    EntityTypeManagerInterface $entityTypeManager,
+  ) {
+    $this->fileSystem = $fileSystem;
+    $this->fileRepository = $fileRepository;
+    $this->logger = $logger;
+    $this->mediaStorage = $entityTypeManager->getStorage('media');
+  }
+
+  protected function mediaExists(string $uri, array $opts): bool {
+    return (bool) $this->mediaStorage->getQuery()
+      ->condition('bundle', $opts['media_type'])
+      ->condition("{$opts['file_type']}.entity.uri", $uri)
+      ->condition('field_media_use.target_id', $opts['media_use'])
+      ->accessCheck(FALSE)
+      ->range(0, 1)
+      ->execute();
+  }
+
+  public function processFile(string $file, array $opts): bool {
+    $source = "{$opts['source_dir']}/{$file}";
+    $dest   = "{$opts['destination_path']}/{$file}";
+
+    if (!file_exists($source)) {
+      $this->logger->warning('Missing file @file', ['@file' => $source]);
+      return FALSE;
+    }
+
+    $uri = ($source === $dest)
+      ? $dest
+      : $this->fileSystem->copy($source, $dest, FileExists::Rename);
+
+    if ($this->mediaExists($uri, $opts)) {
+      return TRUE;
+    }
+
+    $fileEntity = $this->fileRepository->loadByUri($uri)
+      ?? File::create(['uri' => $uri, 'status' => 1]);
+    $fileEntity->save();
+
+    preg_match('/^(\d+)_/', $file, $m);
+    $nid = $m[1] ?? NULL;
+
+    Media::create([
+      'bundle' => $opts['media_type'],
+      'name' => $file,
+      $opts['file_type'] => ['target_id' => $fileEntity->id()],
+      'field_media_use' => ['target_id' => $opts['media_use']],
+      'field_media_of' => $nid,
+    ])->save();
+
+    return TRUE;
+  }
+
+}