The Correct way to setup Queue Workers with Batch Processing for Drupal 8, 9, and 10

If you recently did a google search for setting this up, you likely would have come across this page, https://kushaler2010.medium.com/drupal-9-10-queue-worker-with-batch-processing-and-guzzle-api-4d1b41327550, sadly it seems like this is a ChatGPT generated article. The code is a mix of Drupal 7 code and Drupal 8+ code. It does not function and likely will cause you headaches.  Below I have created an article on the correct way to set this up.  This method is great for situations where you have a lot of items to process and do not want to worry about your internet connection stopping, php errors killing the whole proces, etc.

The big thing this can be used for is something like Queueing notifications on a site to be processed at regular intervals or importing a large CSV file of data.

Lets get Started

1. Define a new Queue Worker in a custom Module.

Create a class for the queue worker. For instance, MyModuleQueueWorker.php:

<?php

namespace Drupal\mymodule\Plugin\QueueWorker;

use Drupal\Core\Queue\QueueWorkerBase;
use Drupal\Core\Queue\QueueWorkerInterface;

/**
 * A Queue Worker that processes items.
 *
 * @QueueWorker(
 *   id = "my_module_queue_worker",
 *   title = @Translation("My Module Queue Worker"),
 *   cron = {"time" = 60}
 * )
 */
class MyModuleQueueWorker extends QueueWorkerBase implements QueueWorkerInterface {

  /**
   * {@inheritdoc}
   */
  public function processItem($data) {
    // Process the queue item.
    \Drupal::logger('mymodule')->info('Processing item: @item', ['@item' => print_r($data, TRUE)]);
    // Add your processing logic here.
  }

}

2. Next Tell Drupal about the new Queue Service

In mymodule.services.yml:

services:
  mymodule.queue:
    class: Drupal\Core\Queue\QueueFactory
    arguments: ['@queue.worker']

3. Add a a Form for Manual Batch Processing

Create a form to manually add items to the queue. For example, MyModuleQueueForm.php:

<?php

namespace Drupal\mymodule\Form;

use Drupal\Core\Form\FormBase;
use Drupal\Core\Form\FormStateInterface;
use Drupal\Core\Queue\QueueFactory;
use Symfony\Component\DependencyInjection\ContainerInterface;

class MyModuleQueueForm extends FormBase {

  /**
   * @var \Drupal\Core\Queue\QueueFactory
   */
  protected $queueFactory;

  public function __construct(QueueFactory $queue_factory) {
    $this->queueFactory = $queue_factory;
  }

  public static function create(ContainerInterface $container) {
    return new static(
      $container->get('queue')
    );
  }

  public function getFormId() {
    return 'mymodule_queue_form';
  }

  public function buildForm(array $form, FormStateInterface $form_state) {
    $form['items'] = [
      '#type' => 'textarea',
      '#title' => $this->t('Items to process'),
      '#description' => $this->t('Enter each item on a new line.'),
    ];

    $form['submit'] = [
      '#type' => 'submit',
      '#value' => $this->t('Add to Queue'),
    ];

    return $form;
  }

  public function submitForm(array &$form, FormStateInterface $form_state) {
    $items = explode(PHP_EOL, $form_state->getValue('items'));
    $queue = $this->queueFactory->get('my_module_queue_worker');

    foreach ($items as $item) {
      $queue->createItem(['data' => trim($item)]);
    }

    $this->messenger()->addMessage($this->t('Items added to queue.'));
  }

}

4. Add Items to a Queue

Below is an example on how to add items programmatically to the queue.

public function submitForm(array &$form, FormStateInterface $form_state) {
  $items = explode(PHP_EOL, $form_state->getValue('items'));
  $queue = $this->queueFactory->get('my_module_queue_worker');

  foreach ($items as $item) {
    $queue->createItem(['data' => trim($item)]);
  }

  $this->messenger()->addMessage($this->t('Items added to queue.'));
}

5. Create a Batch Processing Function 

For those looking to have a manual process along with the automatic processing. This could be included as part of a different form, or in this one.

use Drupal\Core\StringTranslation\StringTranslationTrait;

function mymodule_batch_process(array $items) {
  $batch = [
    'title' => t('Processing items...'),
    'operations' => [],
    'finished' => 'mymodule_batch_finished',
  ];

  foreach ($items as $item) {
    $batch['operations'][] = ['mymodule_batch_process_item', [$item]];
  }

  batch_set($batch);
}

function mymodule_batch_process_item($item, &$context) {
  // Process each item.
  \Drupal::logger('mymodule')->info('Processing batch item: @item', ['@item' => print_r($item, TRUE)]);
  // Add your processing logic here.
}

function mymodule_batch_finished($success, $results, $operations) {
  if ($success) {
    \Drupal::messenger()->addMessage(t('Batch processing complete.'));
  }
  else {
    \Drupal::messenger()->addMessage(t('Batch processing encountered an error.'));
  }
}