Upgrade framework

This commit is contained in:
2023-11-14 16:54:35 +01:00
parent 1648a5cd42
commit 4fcf6fffcc
10548 changed files with 693138 additions and 466698 deletions

View File

@@ -2,10 +2,10 @@
namespace Illuminate\Queue;
use Pheanstalk\Pheanstalk;
use Pheanstalk\Job as PheanstalkJob;
use Illuminate\Queue\Jobs\BeanstalkdJob;
use Illuminate\Contracts\Queue\Queue as QueueContract;
use Illuminate\Queue\Jobs\BeanstalkdJob;
use Pheanstalk\Job as PheanstalkJob;
use Pheanstalk\Pheanstalk;
class BeanstalkdQueue extends Queue implements QueueContract
{
@@ -30,25 +30,40 @@ class BeanstalkdQueue extends Queue implements QueueContract
*/
protected $timeToRun;
/**
* The maximum number of seconds to block for a job.
*
* @var int
*/
protected $blockFor;
/**
* Create a new Beanstalkd queue instance.
*
* @param \Pheanstalk\Pheanstalk $pheanstalk
* @param string $default
* @param int $timeToRun
* @param int $blockFor
* @param bool $dispatchAfterCommit
* @return void
*/
public function __construct(Pheanstalk $pheanstalk, $default, $timeToRun)
public function __construct(Pheanstalk $pheanstalk,
$default,
$timeToRun,
$blockFor = 0,
$dispatchAfterCommit = false)
{
$this->default = $default;
$this->blockFor = $blockFor;
$this->timeToRun = $timeToRun;
$this->pheanstalk = $pheanstalk;
$this->dispatchAfterCommit = $dispatchAfterCommit;
}
/**
* Get the size of the queue.
*
* @param string $queue
* @param string|null $queue
* @return int
*/
public function size($queue = null)
@@ -62,21 +77,29 @@ class BeanstalkdQueue extends Queue implements QueueContract
* Push a new job onto the queue.
*
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function push($job, $data = '', $queue = null)
{
return $this->pushRaw($this->createPayload($job, $data), $queue);
return $this->enqueueUsing(
$job,
$this->createPayload($job, $this->getQueue($queue), $data),
$queue,
null,
function ($payload, $queue) {
return $this->pushRaw($payload, $queue);
}
);
}
/**
* Push a raw payload onto the queue.
*
* @param string $payload
* @param string $queue
* @param array $options
* @param string|null $queue
* @param array $options
* @return mixed
*/
public function pushRaw($payload, $queue = null, array $options = [])
@@ -87,37 +110,62 @@ class BeanstalkdQueue extends Queue implements QueueContract
}
/**
* Push a new job onto the queue after a delay.
* Push a new job onto the queue after (n) seconds.
*
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function later($delay, $job, $data = '', $queue = null)
{
$pheanstalk = $this->pheanstalk->useTube($this->getQueue($queue));
return $pheanstalk->put(
$this->createPayload($job, $data),
Pheanstalk::DEFAULT_PRIORITY,
$this->secondsUntil($delay),
$this->timeToRun
return $this->enqueueUsing(
$job,
$this->createPayload($job, $this->getQueue($queue), $data),
$queue,
$delay,
function ($payload, $queue, $delay) {
return $this->pheanstalk->useTube($this->getQueue($queue))->put(
$payload,
Pheanstalk::DEFAULT_PRIORITY,
$this->secondsUntil($delay),
$this->timeToRun
);
}
);
}
/**
* Push an array of jobs onto the queue.
*
* @param array $jobs
* @param mixed $data
* @param string|null $queue
* @return void
*/
public function bulk($jobs, $data = '', $queue = null)
{
foreach ((array) $jobs as $job) {
if (isset($job->delay)) {
$this->later($job->delay, $job, $data, $queue);
} else {
$this->push($job, $data, $queue);
}
}
}
/**
* Pop the next job off of the queue.
*
* @param string $queue
* @param string|null $queue
* @return \Illuminate\Contracts\Queue\Job|null
*/
public function pop($queue = null)
{
$queue = $this->getQueue($queue);
$job = $this->pheanstalk->watchOnly($queue)->reserve(0);
$job = $this->pheanstalk->watchOnly($queue)->reserveWithTimeout($this->blockFor);
if ($job instanceof PheanstalkJob) {
return new BeanstalkdJob(
@@ -130,7 +178,7 @@ class BeanstalkdQueue extends Queue implements QueueContract
* Delete a message from the Beanstalk queue.
*
* @param string $queue
* @param string $id
* @param string|int $id
* @return void
*/
public function deleteMessage($queue, $id)

View File

@@ -0,0 +1,111 @@
<?php
namespace Illuminate\Queue;
use Closure;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Container\Container;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Laravel\SerializableClosure\SerializableClosure;
use ReflectionFunction;
class CallQueuedClosure implements ShouldQueue
{
use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* The serializable Closure instance.
*
* @var \Laravel\SerializableClosure\SerializableClosure
*/
public $closure;
/**
* The callbacks that should be executed on failure.
*
* @var array
*/
public $failureCallbacks = [];
/**
* Indicate if the job should be deleted when models are missing.
*
* @var bool
*/
public $deleteWhenMissingModels = true;
/**
* Create a new job instance.
*
* @param \Laravel\SerializableClosure\SerializableClosure $closure
* @return void
*/
public function __construct($closure)
{
$this->closure = $closure;
}
/**
* Create a new job instance.
*
* @param \Closure $job
* @return self
*/
public static function create(Closure $job)
{
return new self(new SerializableClosure($job));
}
/**
* Execute the job.
*
* @param \Illuminate\Contracts\Container\Container $container
* @return void
*/
public function handle(Container $container)
{
$container->call($this->closure->getClosure(), ['job' => $this]);
}
/**
* Add a callback to be executed if the job fails.
*
* @param callable $callback
* @return $this
*/
public function onFailure($callback)
{
$this->failureCallbacks[] = $callback instanceof Closure
? new SerializableClosure($callback)
: $callback;
return $this;
}
/**
* Handle a job failure.
*
* @param \Throwable $e
* @return void
*/
public function failed($e)
{
foreach ($this->failureCallbacks as $callback) {
$callback($e);
}
}
/**
* Get the display name for the queued job.
*
* @return string
*/
public function displayName()
{
$reflection = new ReflectionFunction($this->closure->getClosure());
return 'Closure ('.basename($reflection->getFileName()).':'.$reflection->getStartLine().')';
}
}

View File

@@ -2,8 +2,20 @@
namespace Illuminate\Queue;
use Illuminate\Contracts\Queue\Job;
use Exception;
use Illuminate\Bus\Batchable;
use Illuminate\Bus\UniqueLock;
use Illuminate\Contracts\Bus\Dispatcher;
use Illuminate\Contracts\Cache\Repository as Cache;
use Illuminate\Contracts\Container\Container;
use Illuminate\Contracts\Encryption\Encrypter;
use Illuminate\Contracts\Queue\Job;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Database\Eloquent\ModelNotFoundException;
use Illuminate\Pipeline\Pipeline;
use ReflectionClass;
use RuntimeException;
class CallQueuedHandler
{
@@ -14,14 +26,23 @@ class CallQueuedHandler
*/
protected $dispatcher;
/**
* The container instance.
*
* @var \Illuminate\Contracts\Container\Container
*/
protected $container;
/**
* Create a new handler instance.
*
* @param \Illuminate\Contracts\Bus\Dispatcher $dispatcher
* @param \Illuminate\Contracts\Container\Container $container
* @return void
*/
public function __construct(Dispatcher $dispatcher)
public function __construct(Dispatcher $dispatcher, Container $container)
{
$this->container = $container;
$this->dispatcher = $dispatcher;
}
@@ -34,19 +55,77 @@ class CallQueuedHandler
*/
public function call(Job $job, array $data)
{
$command = $this->setJobInstanceIfNecessary(
$job, unserialize($data['command'])
);
try {
$command = $this->setJobInstanceIfNecessary(
$job, $this->getCommand($data)
);
} catch (ModelNotFoundException $e) {
return $this->handleModelNotFound($job, $e);
}
$this->dispatcher->dispatchNow(
$command, $handler = $this->resolveHandler($job, $command)
);
if ($command instanceof ShouldBeUniqueUntilProcessing) {
$this->ensureUniqueJobLockIsReleased($command);
}
$this->dispatchThroughMiddleware($job, $command);
if (! $job->isReleased() && ! $command instanceof ShouldBeUniqueUntilProcessing) {
$this->ensureUniqueJobLockIsReleased($command);
}
if (! $job->hasFailed() && ! $job->isReleased()) {
$this->ensureNextJobInChainIsDispatched($command);
$this->ensureSuccessfulBatchJobIsRecorded($command);
}
if (! $job->isDeletedOrReleased()) {
$job->delete();
}
}
/**
* Get the command from the given payload.
*
* @param array $data
* @return mixed
*
* @throws \RuntimeException
*/
protected function getCommand(array $data)
{
if (str_starts_with($data['command'], 'O:')) {
return unserialize($data['command']);
}
if ($this->container->bound(Encrypter::class)) {
return unserialize($this->container[Encrypter::class]->decrypt($data['command']));
}
throw new RuntimeException('Unable to extract job payload.');
}
/**
* Dispatch the given job / command through its specified middleware.
*
* @param \Illuminate\Contracts\Queue\Job $job
* @param mixed $command
* @return mixed
*/
protected function dispatchThroughMiddleware(Job $job, $command)
{
if ($command instanceof \__PHP_Incomplete_Class) {
throw new Exception('Job is incomplete class: '.json_encode($command));
}
return (new Pipeline($this->container))->send($command)
->through(array_merge(method_exists($command, 'middleware') ? $command->middleware() : [], $command->middleware ?? []))
->then(function ($command) use ($job) {
return $this->dispatcher->dispatchNow(
$command, $this->resolveHandler($job, $command)
);
});
}
/**
* Resolve the handler for the given command.
*
@@ -74,28 +153,145 @@ class CallQueuedHandler
*/
protected function setJobInstanceIfNecessary(Job $job, $instance)
{
if (in_array(InteractsWithQueue::class, class_uses_recursive(get_class($instance)))) {
if (in_array(InteractsWithQueue::class, class_uses_recursive($instance))) {
$instance->setJob($job);
}
return $instance;
}
/**
* Ensure the next job in the chain is dispatched if applicable.
*
* @param mixed $command
* @return void
*/
protected function ensureNextJobInChainIsDispatched($command)
{
if (method_exists($command, 'dispatchNextJobInChain')) {
$command->dispatchNextJobInChain();
}
}
/**
* Ensure the batch is notified of the successful job completion.
*
* @param mixed $command
* @return void
*/
protected function ensureSuccessfulBatchJobIsRecorded($command)
{
$uses = class_uses_recursive($command);
if (! in_array(Batchable::class, $uses) ||
! in_array(InteractsWithQueue::class, $uses)) {
return;
}
if ($batch = $command->batch()) {
$batch->recordSuccessfulJob($command->job->uuid());
}
}
/**
* Ensure the lock for a unique job is released.
*
* @param mixed $command
* @return void
*/
protected function ensureUniqueJobLockIsReleased($command)
{
if ($command instanceof ShouldBeUnique) {
(new UniqueLock($this->container->make(Cache::class)))->release($command);
}
}
/**
* Handle a model not found exception.
*
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Throwable $e
* @return void
*/
protected function handleModelNotFound(Job $job, $e)
{
$class = $job->resolveName();
try {
$shouldDelete = (new ReflectionClass($class))
->getDefaultProperties()['deleteWhenMissingModels'] ?? false;
} catch (Exception $e) {
$shouldDelete = false;
}
if ($shouldDelete) {
return $job->delete();
}
return $job->fail($e);
}
/**
* Call the failed method on the job instance.
*
* The exception that caused the failure will be passed.
*
* @param array $data
* @param \Exception $e
* @param \Throwable|null $e
* @param string $uuid
* @return void
*/
public function failed(array $data, $e)
public function failed(array $data, $e, string $uuid)
{
$command = unserialize($data['command']);
$command = $this->getCommand($data);
if (! $command instanceof ShouldBeUniqueUntilProcessing) {
$this->ensureUniqueJobLockIsReleased($command);
}
if ($command instanceof \__PHP_Incomplete_Class) {
return;
}
$this->ensureFailedBatchJobIsRecorded($uuid, $command, $e);
$this->ensureChainCatchCallbacksAreInvoked($uuid, $command, $e);
if (method_exists($command, 'failed')) {
$command->failed($e);
}
}
/**
* Ensure the batch is notified of the failed job.
*
* @param string $uuid
* @param mixed $command
* @param \Throwable $e
* @return void
*/
protected function ensureFailedBatchJobIsRecorded(string $uuid, $command, $e)
{
if (! in_array(Batchable::class, class_uses_recursive($command))) {
return;
}
if ($batch = $command->batch()) {
$batch->recordFailedJob($uuid, $e);
}
}
/**
* Ensure the chained job catch callbacks are invoked.
*
* @param string $uuid
* @param mixed $command
* @param \Throwable $e
* @return void
*/
protected function ensureChainCatchCallbacksAreInvoked(string $uuid, $command, $e)
{
if (method_exists($command, 'invokeChainCatchCallbacks')) {
$command->invokeChainCatchCallbacks($e);
}
}
}

View File

@@ -2,11 +2,15 @@
namespace Illuminate\Queue\Capsule;
use Illuminate\Queue\QueueManager;
use Illuminate\Container\Container;
use Illuminate\Queue\QueueManager;
use Illuminate\Queue\QueueServiceProvider;
use Illuminate\Support\Traits\CapsuleManagerTrait;
/**
* @mixin \Illuminate\Queue\QueueManager
* @mixin \Illuminate\Contracts\Queue\Queue
*/
class Manager
{
use CapsuleManagerTrait;
@@ -21,16 +25,16 @@ class Manager
/**
* Create a new queue capsule manager.
*
* @param \Illuminate\Container\Container $container
* @param \Illuminate\Container\Container|null $container
* @return void
*/
public function __construct(Container $container = null)
{
$this->setupContainer($container ?: new Container);
// Once we have the container setup, we will setup the default configuration
// options in the container "config" bindings. This just makes this queue
// manager behave correctly since all the correct binding are in place.
// Once we have the container setup, we will set up the default configuration
// options in the container "config" bindings. This'll just make the queue
// manager behave correctly since all the correct bindings are in place.
$this->setupDefaultConfiguration();
$this->setupManager();
@@ -73,7 +77,7 @@ class Manager
/**
* Get a connection instance from the global manager.
*
* @param string $connection
* @param string|null $connection
* @return \Illuminate\Contracts\Queue\Queue
*/
public static function connection($connection = null)
@@ -85,9 +89,9 @@ class Manager
* Push a new job onto the queue.
*
* @param string $job
* @param mixed $data
* @param string $queue
* @param string $connection
* @param mixed $data
* @param string|null $queue
* @param string|null $connection
* @return mixed
*/
public static function push($job, $data = '', $queue = null, $connection = null)
@@ -98,10 +102,10 @@ class Manager
/**
* Push a new an array of jobs onto the queue.
*
* @param array $jobs
* @param mixed $data
* @param string $queue
* @param string $connection
* @param array $jobs
* @param mixed $data
* @param string|null $queue
* @param string|null $connection
* @return mixed
*/
public static function bulk($jobs, $data = '', $queue = null, $connection = null)
@@ -110,13 +114,13 @@ class Manager
}
/**
* Push a new job onto the queue after a delay.
* Push a new job onto the queue after (n) seconds.
*
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param string $job
* @param mixed $data
* @param string $queue
* @param string $connection
* @param mixed $data
* @param string|null $queue
* @param string|null $connection
* @return mixed
*/
public static function later($delay, $job, $data = '', $queue = null, $connection = null)
@@ -127,7 +131,7 @@ class Manager
/**
* Get a registered connection instance.
*
* @param string $name
* @param string|null $name
* @return \Illuminate\Contracts\Queue\Queue
*/
public function getConnection($name = null)
@@ -138,7 +142,7 @@ class Manager
/**
* Register a connection with the manager.
*
* @param array $config
* @param array $config
* @param string $name
* @return void
*/
@@ -173,7 +177,7 @@ class Manager
* Dynamically pass methods to the default connection.
*
* @param string $method
* @param array $parameters
* @param array $parameters
* @return mixed
*/
public static function __callStatic($method, $parameters)

View File

@@ -2,11 +2,9 @@
namespace Illuminate\Queue\Connectors;
use Illuminate\Queue\BeanstalkdQueue;
use Pheanstalk\Connection;
use Pheanstalk\Pheanstalk;
use Illuminate\Support\Arr;
use Pheanstalk\PheanstalkInterface;
use Illuminate\Queue\BeanstalkdQueue;
class BeanstalkdConnector implements ConnectorInterface
{
@@ -18,9 +16,13 @@ class BeanstalkdConnector implements ConnectorInterface
*/
public function connect(array $config)
{
$retryAfter = Arr::get($config, 'retry_after', Pheanstalk::DEFAULT_TTR);
return new BeanstalkdQueue($this->pheanstalk($config), $config['queue'], $retryAfter);
return new BeanstalkdQueue(
$this->pheanstalk($config),
$config['queue'],
$config['retry_after'] ?? Pheanstalk::DEFAULT_TTR,
$config['block_for'] ?? 0,
$config['after_commit'] ?? null
);
}
/**
@@ -31,11 +33,10 @@ class BeanstalkdConnector implements ConnectorInterface
*/
protected function pheanstalk(array $config)
{
return new Pheanstalk(
return Pheanstalk::create(
$config['host'],
Arr::get($config, 'port', PheanstalkInterface::DEFAULT_PORT),
Arr::get($config, 'timeout', Connection::DEFAULT_CONNECT_TIMEOUT),
Arr::get($config, 'persistent', false)
$config['port'] ?? Pheanstalk::DEFAULT_PORT,
$config['timeout'] ?? Connection::DEFAULT_CONNECT_TIMEOUT
);
}
}

View File

@@ -2,9 +2,8 @@
namespace Illuminate\Queue\Connectors;
use Illuminate\Support\Arr;
use Illuminate\Queue\DatabaseQueue;
use Illuminate\Database\ConnectionResolverInterface;
use Illuminate\Queue\DatabaseQueue;
class DatabaseConnector implements ConnectorInterface
{
@@ -35,10 +34,11 @@ class DatabaseConnector implements ConnectorInterface
public function connect(array $config)
{
return new DatabaseQueue(
$this->connections->connection(Arr::get($config, 'connection')),
$this->connections->connection($config['connection'] ?? null),
$config['table'],
$config['queue'],
Arr::get($config, 'retry_after', 60)
$config['retry_after'] ?? 60,
$config['after_commit'] ?? null
);
}
}

View File

@@ -2,9 +2,8 @@
namespace Illuminate\Queue\Connectors;
use Illuminate\Support\Arr;
use Illuminate\Queue\RedisQueue;
use Illuminate\Contracts\Redis\Factory as Redis;
use Illuminate\Queue\RedisQueue;
class RedisConnector implements ConnectorInterface
{
@@ -45,8 +44,11 @@ class RedisConnector implements ConnectorInterface
{
return new RedisQueue(
$this->redis, $config['queue'],
Arr::get($config, 'connection', $this->connection),
Arr::get($config, 'retry_after', 60)
$config['connection'] ?? $this->connection,
$config['retry_after'] ?? 60,
$config['block_for'] ?? null,
$config['after_commit'] ?? null,
$config['migration_batch_size'] ?? -1
);
}
}

View File

@@ -3,8 +3,8 @@
namespace Illuminate\Queue\Connectors;
use Aws\Sqs\SqsClient;
use Illuminate\Support\Arr;
use Illuminate\Queue\SqsQueue;
use Illuminate\Support\Arr;
class SqsConnector implements ConnectorInterface
{
@@ -18,12 +18,18 @@ class SqsConnector implements ConnectorInterface
{
$config = $this->getDefaultConfiguration($config);
if ($config['key'] && $config['secret']) {
$config['credentials'] = Arr::only($config, ['key', 'secret']);
if (! empty($config['key']) && ! empty($config['secret'])) {
$config['credentials'] = Arr::only($config, ['key', 'secret', 'token']);
}
return new SqsQueue(
new SqsClient($config), $config['queue'], Arr::get($config, 'prefix', '')
new SqsClient(
Arr::except($config, ['token'])
),
$config['queue'],
$config['prefix'] ?? '',
$config['suffix'] ?? '',
$config['after_commit'] ?? null
);
}

View File

@@ -0,0 +1,111 @@
<?php
namespace Illuminate\Queue\Console;
use Illuminate\Console\Command;
use Illuminate\Filesystem\Filesystem;
use Illuminate\Support\Composer;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:batches-table')]
class BatchesTableCommand extends Command
{
/**
* The console command name.
*
* @var string
*/
protected $name = 'queue:batches-table';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:batches-table';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Create a migration for the batches database table';
/**
* The filesystem instance.
*
* @var \Illuminate\Filesystem\Filesystem
*/
protected $files;
/**
* @var \Illuminate\Support\Composer
*/
protected $composer;
/**
* Create a new batched queue jobs table command instance.
*
* @param \Illuminate\Filesystem\Filesystem $files
* @param \Illuminate\Support\Composer $composer
* @return void
*/
public function __construct(Filesystem $files, Composer $composer)
{
parent::__construct();
$this->files = $files;
$this->composer = $composer;
}
/**
* Execute the console command.
*
* @return void
*/
public function handle()
{
$table = $this->laravel['config']['queue.batching.table'] ?? 'job_batches';
$this->replaceMigration(
$this->createBaseMigration($table), $table
);
$this->components->info('Migration created successfully.');
$this->composer->dumpAutoloads();
}
/**
* Create a base migration file for the table.
*
* @param string $table
* @return string
*/
protected function createBaseMigration($table = 'job_batches')
{
return $this->laravel['migration.creator']->create(
'create_'.$table.'_table', $this->laravel->databasePath().'/migrations'
);
}
/**
* Replace the generated migration with the batches job table stub.
*
* @param string $path
* @param string $table
* @return void
*/
protected function replaceMigration($path, $table)
{
$stub = str_replace(
'{{table}}', $table, $this->files->get(__DIR__.'/stubs/batches.stub')
);
$this->files->put($path, $stub);
}
}

View File

@@ -0,0 +1,113 @@
<?php
namespace Illuminate\Queue\Console;
use Illuminate\Console\Command;
use Illuminate\Console\ConfirmableTrait;
use Illuminate\Contracts\Queue\ClearableQueue;
use ReflectionClass;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
#[AsCommand(name: 'queue:clear')]
class ClearCommand extends Command
{
use ConfirmableTrait;
/**
* The console command name.
*
* @var string
*/
protected $name = 'queue:clear';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:clear';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Delete all of the jobs from the specified queue';
/**
* Execute the console command.
*
* @return int|null
*/
public function handle()
{
if (! $this->confirmToProceed()) {
return 1;
}
$connection = $this->argument('connection')
?: $this->laravel['config']['queue.default'];
// We need to get the right queue for the connection which is set in the queue
// configuration file for the application. We will pull it based on the set
// connection being run for the queue operation currently being executed.
$queueName = $this->getQueue($connection);
$queue = $this->laravel['queue']->connection($connection);
if ($queue instanceof ClearableQueue) {
$count = $queue->clear($queueName);
$this->components->info('Cleared '.$count.' jobs from the ['.$queueName.'] queue');
} else {
$this->components->error('Clearing queues is not supported on ['.(new ReflectionClass($queue))->getShortName().']');
}
return 0;
}
/**
* Get the queue name to clear.
*
* @param string $connection
* @return string
*/
protected function getQueue($connection)
{
return $this->option('queue') ?: $this->laravel['config']->get(
"queue.connections.{$connection}.queue", 'default'
);
}
/**
* Get the console command arguments.
*
* @return array
*/
protected function getArguments()
{
return [
['connection', InputArgument::OPTIONAL, 'The name of the queue connection to clear'],
];
}
/**
* Get the console command options.
*
* @return array
*/
protected function getOptions()
{
return [
['queue', null, InputOption::VALUE_OPTIONAL, 'The name of the queue to clear'],
['force', null, InputOption::VALUE_NONE, 'Force the operation to run when in production'],
];
}
}

View File

@@ -2,11 +2,12 @@
namespace Illuminate\Queue\Console;
use Illuminate\Support\Str;
use Illuminate\Console\Command;
use Illuminate\Support\Composer;
use Illuminate\Filesystem\Filesystem;
use Illuminate\Support\Composer;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:failed-table')]
class FailedTableCommand extends Command
{
/**
@@ -16,6 +17,17 @@ class FailedTableCommand extends Command
*/
protected $name = 'queue:failed-table';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:failed-table';
/**
* The console command description.
*
@@ -39,7 +51,7 @@ class FailedTableCommand extends Command
* Create a new failed queue jobs table command instance.
*
* @param \Illuminate\Filesystem\Filesystem $files
* @param \Illuminate\Support\Composer $composer
* @param \Illuminate\Support\Composer $composer
* @return void
*/
public function __construct(Filesystem $files, Composer $composer)
@@ -55,15 +67,15 @@ class FailedTableCommand extends Command
*
* @return void
*/
public function fire()
public function handle()
{
$table = $this->laravel['config']['queue.failed.table'];
$this->replaceMigration(
$this->createBaseMigration($table), $table, Str::studly($table)
$this->createBaseMigration($table), $table
);
$this->info('Migration created successfully!');
$this->components->info('Migration created successfully.');
$this->composer->dumpAutoloads();
}
@@ -86,15 +98,12 @@ class FailedTableCommand extends Command
*
* @param string $path
* @param string $table
* @param string $tableClassName
* @return void
*/
protected function replaceMigration($path, $table, $tableClassName)
protected function replaceMigration($path, $table)
{
$stub = str_replace(
['{{table}}', '{{tableClassName}}'],
[$table, $tableClassName],
$this->files->get(__DIR__.'/stubs/failed_jobs.stub')
'{{table}}', $table, $this->files->get(__DIR__.'/stubs/failed_jobs.stub')
);
$this->files->put($path, $stub);

View File

@@ -3,7 +3,9 @@
namespace Illuminate\Queue\Console;
use Illuminate\Console\Command;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:flush')]
class FlushFailedCommand extends Command
{
/**
@@ -11,7 +13,18 @@ class FlushFailedCommand extends Command
*
* @var string
*/
protected $name = 'queue:flush';
protected $signature = 'queue:flush {--hours= : The number of hours to retain failed job data}';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:flush';
/**
* The console command description.
@@ -25,10 +38,16 @@ class FlushFailedCommand extends Command
*
* @return void
*/
public function fire()
public function handle()
{
$this->laravel['queue.failer']->flush();
$this->laravel['queue.failer']->flush($this->option('hours'));
$this->info('All failed jobs deleted successfully!');
if ($this->option('hours')) {
$this->components->info("All jobs that failed more than {$this->option('hours')} hours ago have been deleted successfully.");
return;
}
$this->components->info('All failed jobs deleted successfully.');
}
}

View File

@@ -3,16 +3,28 @@
namespace Illuminate\Queue\Console;
use Illuminate\Console\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:forget')]
class ForgetFailedCommand extends Command
{
/**
* The console command name.
* The console command signature.
*
* @var string
*/
protected $name = 'queue:forget';
protected $signature = 'queue:forget {id : The ID of the failed job}';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:forget';
/**
* The console command description.
@@ -26,24 +38,12 @@ class ForgetFailedCommand extends Command
*
* @return void
*/
public function fire()
public function handle()
{
if ($this->laravel['queue.failer']->forget($this->argument('id'))) {
$this->info('Failed job deleted successfully!');
$this->components->info('Failed job deleted successfully.');
} else {
$this->error('No failed job matches the given ID.');
$this->components->error('No failed job matches the given ID.');
}
}
/**
* Get the console command arguments.
*
* @return array
*/
protected function getArguments()
{
return [
['id', InputArgument::REQUIRED, 'The ID of the failed job'],
];
}
}

View File

@@ -2,9 +2,11 @@
namespace Illuminate\Queue\Console;
use Illuminate\Support\Arr;
use Illuminate\Console\Command;
use Illuminate\Support\Arr;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:failed')]
class ListFailedCommand extends Command
{
/**
@@ -14,6 +16,17 @@ class ListFailedCommand extends Command
*/
protected $name = 'queue:failed';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:failed';
/**
* The console command description.
*
@@ -24,7 +37,7 @@ class ListFailedCommand extends Command
/**
* The table headers for the command.
*
* @var array
* @var string[]
*/
protected $headers = ['ID', 'Connection', 'Queue', 'Class', 'Failed At'];
@@ -33,13 +46,15 @@ class ListFailedCommand extends Command
*
* @return void
*/
public function fire()
public function handle()
{
if (count($jobs = $this->getFailedJobs()) == 0) {
return $this->info('No failed jobs!');
if (count($jobs = $this->getFailedJobs()) === 0) {
return $this->components->info('No failed jobs found.');
}
$this->newLine();
$this->displayFailedJobs($jobs);
$this->newLine();
}
/**
@@ -66,7 +81,7 @@ class ListFailedCommand extends Command
{
$row = array_values(Arr::except($failed, ['payload', 'exception']));
array_splice($row, 3, 0, $this->extractJobName($failed['payload']));
array_splice($row, 3, 0, $this->extractJobName($failed['payload']) ?: '');
return $row;
}
@@ -82,7 +97,7 @@ class ListFailedCommand extends Command
$payload = json_decode($payload, true);
if ($payload && (! isset($payload['data']['command']))) {
return Arr::get($payload, 'job');
return $payload['job'] ?? null;
} elseif ($payload && isset($payload['data']['command'])) {
return $this->matchJobName($payload);
}
@@ -92,17 +107,13 @@ class ListFailedCommand extends Command
* Match the job name from the payload.
*
* @param array $payload
* @return string
* @return string|null
*/
protected function matchJobName($payload)
{
preg_match('/"([^"]+)"/', $payload['data']['command'], $matches);
if (isset($matches[1])) {
return $matches[1];
} else {
return Arr::get($payload, 'job');
}
return $matches[1] ?? $payload['job'] ?? null;
}
/**
@@ -113,6 +124,11 @@ class ListFailedCommand extends Command
*/
protected function displayFailedJobs(array $jobs)
{
$this->table($this->headers, $jobs);
collect($jobs)->each(
fn ($job) => $this->components->twoColumnDetail(
sprintf('<fg=gray>%s</> %s</>', $job[4], $job[0]),
sprintf('<fg=gray>%s@%s</> %s', $job[1], $job[2], $job[3])
),
);
}
}

View File

@@ -2,10 +2,12 @@
namespace Illuminate\Queue\Console;
use Illuminate\Queue\Listener;
use Illuminate\Console\Command;
use Illuminate\Queue\Listener;
use Illuminate\Queue\ListenerOptions;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:listen')]
class ListenCommand extends Command
{
/**
@@ -15,13 +17,27 @@ class ListenCommand extends Command
*/
protected $signature = 'queue:listen
{connection? : The name of connection}
{--delay=0 : Amount of time to delay failed jobs}
{--name=default : The name of the worker}
{--delay=0 : The number of seconds to delay failed jobs (Deprecated)}
{--backoff=0 : The number of seconds to wait before retrying a job that encountered an uncaught exception}
{--force : Force the worker to run even in maintenance mode}
{--memory=128 : The memory limit in megabytes}
{--queue= : The queue to listen on}
{--sleep=3 : Number of seconds to sleep when no job is available}
{--rest=0 : Number of seconds to rest between jobs}
{--timeout=60 : The number of seconds a child process can run}
{--tries=0 : Number of times to attempt a job before logging it failed}';
{--tries=1 : Number of times to attempt a job before logging it failed}';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:listen';
/**
* The console command description.
@@ -55,7 +71,7 @@ class ListenCommand extends Command
*
* @return void
*/
public function fire()
public function handle()
{
// We need to get the right queue for the connection which is set in the queue
// configuration file for the application. We will pull it based on the set
@@ -64,6 +80,8 @@ class ListenCommand extends Command
$connection = $this->input->getArgument('connection')
);
$this->components->info(sprintf('Processing jobs from the [%s] %s.', $queue, str('queue')->plural(explode(',', $queue))));
$this->listener->listen(
$connection, $queue, $this->gatherOptions()
);
@@ -91,11 +109,20 @@ class ListenCommand extends Command
*/
protected function gatherOptions()
{
$backoff = $this->hasOption('backoff')
? $this->option('backoff')
: $this->option('delay');
return new ListenerOptions(
$this->option('env'), $this->option('delay'),
$this->option('memory'), $this->option('timeout'),
$this->option('sleep'), $this->option('tries'),
$this->option('force')
name: $this->option('name'),
environment: $this->option('env'),
backoff: $backoff,
memory: $this->option('memory'),
timeout: $this->option('timeout'),
sleep: $this->option('sleep'),
rest: $this->option('rest'),
maxTries: $this->option('tries'),
force: $this->option('force')
);
}

View File

@@ -0,0 +1,160 @@
<?php
namespace Illuminate\Queue\Console;
use Illuminate\Console\Command;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Contracts\Queue\Factory;
use Illuminate\Queue\Events\QueueBusy;
use Illuminate\Support\Collection;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:monitor')]
class MonitorCommand extends Command
{
/**
* The console command name.
*
* @var string
*/
protected $signature = 'queue:monitor
{queues : The names of the queues to monitor}
{--max=1000 : The maximum number of jobs that can be on the queue before an event is dispatched}';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:monitor';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Monitor the size of the specified queues';
/**
* The queue manager instance.
*
* @var \Illuminate\Contracts\Queue\Factory
*/
protected $manager;
/**
* The events dispatcher instance.
*
* @var \Illuminate\Contracts\Events\Dispatcher
*/
protected $events;
/**
* The table headers for the command.
*
* @var string[]
*/
protected $headers = ['Connection', 'Queue', 'Size', 'Status'];
/**
* Create a new queue monitor command.
*
* @param \Illuminate\Contracts\Queue\Factory $manager
* @param \Illuminate\Contracts\Events\Dispatcher $events
* @return void
*/
public function __construct(Factory $manager, Dispatcher $events)
{
parent::__construct();
$this->manager = $manager;
$this->events = $events;
}
/**
* Execute the console command.
*
* @return void
*/
public function handle()
{
$queues = $this->parseQueues($this->argument('queues'));
$this->displaySizes($queues);
$this->dispatchEvents($queues);
}
/**
* Parse the queues into an array of the connections and queues.
*
* @param string $queues
* @return \Illuminate\Support\Collection
*/
protected function parseQueues($queues)
{
return collect(explode(',', $queues))->map(function ($queue) {
[$connection, $queue] = array_pad(explode(':', $queue, 2), 2, null);
if (! isset($queue)) {
$queue = $connection;
$connection = $this->laravel['config']['queue.default'];
}
return [
'connection' => $connection,
'queue' => $queue,
'size' => $size = $this->manager->connection($connection)->size($queue),
'status' => $size >= $this->option('max') ? '<fg=yellow;options=bold>ALERT</>' : '<fg=green;options=bold>OK</>',
];
});
}
/**
* Display the queue sizes in the console.
*
* @param \Illuminate\Support\Collection $queues
* @return void
*/
protected function displaySizes(Collection $queues)
{
$this->newLine();
$this->components->twoColumnDetail('<fg=gray>Queue name</>', '<fg=gray>Size / Status</>');
$queues->each(function ($queue) {
$status = '['.$queue['size'].'] '.$queue['status'];
$this->components->twoColumnDetail($queue['queue'], $status);
});
$this->newLine();
}
/**
* Fire the monitoring events.
*
* @param \Illuminate\Support\Collection $queues
* @return void
*/
protected function dispatchEvents(Collection $queues)
{
foreach ($queues as $queue) {
if ($queue['status'] == '<fg=green;options=bold>OK</>') {
continue;
}
$this->events->dispatch(
new QueueBusy(
$queue['connection'],
$queue['queue'],
$queue['size'],
)
);
}
}
}

View File

@@ -0,0 +1,80 @@
<?php
namespace Illuminate\Queue\Console;
use Illuminate\Bus\BatchRepository;
use Illuminate\Bus\DatabaseBatchRepository;
use Illuminate\Bus\PrunableBatchRepository;
use Illuminate\Console\Command;
use Illuminate\Support\Carbon;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:prune-batches')]
class PruneBatchesCommand extends Command
{
/**
* The console command signature.
*
* @var string
*/
protected $signature = 'queue:prune-batches
{--hours=24 : The number of hours to retain batch data}
{--unfinished= : The number of hours to retain unfinished batch data }
{--cancelled= : The number of hours to retain cancelled batch data }';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:prune-batches';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Prune stale entries from the batches database';
/**
* Execute the console command.
*
* @return void
*/
public function handle()
{
$repository = $this->laravel[BatchRepository::class];
$count = 0;
if ($repository instanceof PrunableBatchRepository) {
$count = $repository->prune(Carbon::now()->subHours($this->option('hours')));
}
$this->components->info("{$count} entries deleted.");
if ($this->option('unfinished')) {
$count = 0;
if ($repository instanceof DatabaseBatchRepository) {
$count = $repository->pruneUnfinished(Carbon::now()->subHours($this->option('unfinished')));
}
$this->components->info("{$count} unfinished entries deleted.");
}
if ($this->option('cancelled')) {
$count = 0;
if ($repository instanceof DatabaseBatchRepository) {
$count = $repository->pruneCancelled(Carbon::now()->subHours($this->option('cancelled')));
}
$this->components->info("{$count} cancelled entries deleted.");
}
}
}

View File

@@ -0,0 +1,58 @@
<?php
namespace Illuminate\Queue\Console;
use Illuminate\Console\Command;
use Illuminate\Queue\Failed\PrunableFailedJobProvider;
use Illuminate\Support\Carbon;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:prune-failed')]
class PruneFailedJobsCommand extends Command
{
/**
* The console command signature.
*
* @var string
*/
protected $signature = 'queue:prune-failed
{--hours=24 : The number of hours to retain failed jobs data}';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:prune-failed';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Prune stale entries from the failed jobs table';
/**
* Execute the console command.
*
* @return int|null
*/
public function handle()
{
$failer = $this->laravel['queue.failer'];
if ($failer instanceof PrunableFailedJobProvider) {
$count = $failer->prune(Carbon::now()->subHours($this->option('hours')));
} else {
$this->components->error('The ['.class_basename($failer).'] failed job storage driver does not support pruning.');
return 1;
}
$this->components->info("{$count} entries deleted.");
}
}

View File

@@ -2,11 +2,16 @@
namespace Illuminate\Queue\Console;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Contracts\Cache\Repository as Cache;
use Illuminate\Support\InteractsWithTime;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:restart')]
class RestartCommand extends Command
{
use InteractsWithTime;
/**
* The console command name.
*
@@ -14,6 +19,17 @@ class RestartCommand extends Command
*/
protected $name = 'queue:restart';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:restart';
/**
* The console command description.
*
@@ -21,15 +37,35 @@ class RestartCommand extends Command
*/
protected $description = 'Restart queue worker daemons after their current job';
/**
* The cache store implementation.
*
* @var \Illuminate\Contracts\Cache\Repository
*/
protected $cache;
/**
* Create a new queue restart command.
*
* @param \Illuminate\Contracts\Cache\Repository $cache
* @return void
*/
public function __construct(Cache $cache)
{
parent::__construct();
$this->cache = $cache;
}
/**
* Execute the console command.
*
* @return void
*/
public function fire()
public function handle()
{
$this->laravel['cache']->forever('illuminate:queue:restart', Carbon::now()->getTimestamp());
$this->cache->forever('illuminate:queue:restart', $this->currentTime());
$this->info('Broadcasting queue restart signal.');
$this->components->info('Broadcasting queue restart signal.');
}
}

View File

@@ -0,0 +1,64 @@
<?php
namespace Illuminate\Queue\Console;
use Illuminate\Bus\BatchRepository;
use Illuminate\Console\Command;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:retry-batch')]
class RetryBatchCommand extends Command
{
/**
* The console command signature.
*
* @var string
*/
protected $signature = 'queue:retry-batch {id : The ID of the batch whose failed jobs should be retried}';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:retry-batch';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Retry the failed jobs for a batch';
/**
* Execute the console command.
*
* @return int|null
*/
public function handle()
{
$batch = $this->laravel[BatchRepository::class]->find($id = $this->argument('id'));
if (! $batch) {
$this->components->error("Unable to find a batch with ID [{$id}].");
return 1;
} elseif (empty($batch->failedJobIds)) {
$this->components->error('The given batch does not contain any failed jobs.');
return 1;
}
$this->components->info("Pushing failed queue jobs of the batch [$id] back onto the queue.");
foreach ($batch->failedJobIds as $failedJobId) {
$this->components->task($failedJobId, fn () => $this->callSilent('queue:retry', ['id' => $failedJobId]) == 0);
}
$this->newLine();
}
}

View File

@@ -2,18 +2,37 @@
namespace Illuminate\Queue\Console;
use Illuminate\Support\Arr;
use DateTimeInterface;
use Illuminate\Console\Command;
use Symfony\Component\Console\Input\InputArgument;
use Illuminate\Contracts\Encryption\Encrypter;
use Illuminate\Queue\Events\JobRetryRequested;
use Illuminate\Support\Arr;
use RuntimeException;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:retry')]
class RetryCommand extends Command
{
/**
* The console command name.
* The console command signature.
*
* @var string
*/
protected $name = 'queue:retry';
protected $signature = 'queue:retry
{id?* : The ID of the failed job or "all" to retry all jobs}
{--queue= : Retry all of the failed jobs for the specified queue}
{--range=* : Range of job IDs (numeric) to be retried}';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:retry';
/**
* The console command description.
@@ -27,21 +46,29 @@ class RetryCommand extends Command
*
* @return void
*/
public function fire()
public function handle()
{
foreach ($this->getJobIds() as $id) {
$jobsFound = count($ids = $this->getJobIds()) > 0;
if ($jobsFound) {
$this->components->info('Pushing failed queue jobs back onto the queue.');
}
foreach ($ids as $id) {
$job = $this->laravel['queue.failer']->find($id);
if (is_null($job)) {
$this->error("Unable to find failed job with ID [{$id}].");
$this->components->error("Unable to find failed job with ID [{$id}].");
} else {
$this->retryJob($job);
$this->laravel['events']->dispatch(new JobRetryRequested($job));
$this->info("The failed job [{$id}] has been pushed back onto the queue!");
$this->components->task($id, fn () => $this->retryJob($job));
$this->laravel['queue.failer']->forget($id);
}
}
$jobsFound ? $this->newLine() : $this->components->info('No retryable jobs found.');
}
/**
@@ -51,10 +78,57 @@ class RetryCommand extends Command
*/
protected function getJobIds()
{
$ids = $this->argument('id');
$ids = (array) $this->argument('id');
if (count($ids) === 1 && $ids[0] === 'all') {
$ids = Arr::pluck($this->laravel['queue.failer']->all(), 'id');
return Arr::pluck($this->laravel['queue.failer']->all(), 'id');
}
if ($queue = $this->option('queue')) {
return $this->getJobIdsByQueue($queue);
}
if ($ranges = (array) $this->option('range')) {
$ids = array_merge($ids, $this->getJobIdsByRanges($ranges));
}
return array_values(array_filter(array_unique($ids)));
}
/**
* Get the job IDs by queue, if applicable.
*
* @param string $queue
* @return array
*/
protected function getJobIdsByQueue($queue)
{
$ids = collect($this->laravel['queue.failer']->all())
->where('queue', $queue)
->pluck('id')
->toArray();
if (count($ids) === 0) {
$this->components->error("Unable to find failed jobs for queue [{$queue}].");
}
return $ids;
}
/**
* Get the job IDs ranges, if applicable.
*
* @param array $ranges
* @return array
*/
protected function getJobIdsByRanges(array $ranges)
{
$ids = [];
foreach ($ranges as $range) {
if (preg_match('/^[0-9]+\-[0-9]+$/', $range)) {
$ids = array_merge($ids, range(...explode('-', $range)));
}
}
return $ids;
@@ -69,14 +143,14 @@ class RetryCommand extends Command
protected function retryJob($job)
{
$this->laravel['queue']->connection($job->connection)->pushRaw(
$this->resetAttempts($job->payload), $job->queue
$this->refreshRetryUntil($this->resetAttempts($job->payload)), $job->queue
);
}
/**
* Reset the payload attempts.
*
* Applicable to Redis jobs which store attempts in their payload.
* Applicable to Redis and other jobs which store attempts in their payload.
*
* @param string $payload
* @return string
@@ -93,14 +167,39 @@ class RetryCommand extends Command
}
/**
* Get the console command arguments.
* Refresh the "retry until" timestamp for the job.
*
* @return array
* @param string $payload
* @return string
*
* @throws \RuntimeException
*/
protected function getArguments()
protected function refreshRetryUntil($payload)
{
return [
['id', InputArgument::IS_ARRAY, 'The ID of the failed job'],
];
$payload = json_decode($payload, true);
if (! isset($payload['data']['command'])) {
return json_encode($payload);
}
if (str_starts_with($payload['data']['command'], 'O:')) {
$instance = unserialize($payload['data']['command']);
} elseif ($this->laravel->bound(Encrypter::class)) {
$instance = unserialize($this->laravel->make(Encrypter::class)->decrypt($payload['data']['command']));
}
if (! isset($instance)) {
throw new RuntimeException('Unable to extract job payload.');
}
if (is_object($instance) && ! $instance instanceof \__PHP_Incomplete_Class && method_exists($instance, 'retryUntil')) {
$retryUntil = $instance->retryUntil();
$payload['retryUntil'] = $retryUntil instanceof DateTimeInterface
? $retryUntil->getTimestamp()
: $retryUntil;
}
return json_encode($payload);
}
}

View File

@@ -2,11 +2,12 @@
namespace Illuminate\Queue\Console;
use Illuminate\Support\Str;
use Illuminate\Console\Command;
use Illuminate\Support\Composer;
use Illuminate\Filesystem\Filesystem;
use Illuminate\Support\Composer;
use Symfony\Component\Console\Attribute\AsCommand;
#[AsCommand(name: 'queue:table')]
class TableCommand extends Command
{
/**
@@ -16,6 +17,17 @@ class TableCommand extends Command
*/
protected $name = 'queue:table';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:table';
/**
* The console command description.
*
@@ -39,7 +51,7 @@ class TableCommand extends Command
* Create a new queue job table command instance.
*
* @param \Illuminate\Filesystem\Filesystem $files
* @param \Illuminate\Support\Composer $composer
* @param \Illuminate\Support\Composer $composer
* @return void
*/
public function __construct(Filesystem $files, Composer $composer)
@@ -55,15 +67,15 @@ class TableCommand extends Command
*
* @return void
*/
public function fire()
public function handle()
{
$table = $this->laravel['config']['queue.connections.database.table'];
$this->replaceMigration(
$this->createBaseMigration($table), $table, Str::studly($table)
$this->createBaseMigration($table), $table
);
$this->info('Migration created successfully!');
$this->components->info('Migration created successfully.');
$this->composer->dumpAutoloads();
}
@@ -86,15 +98,12 @@ class TableCommand extends Command
*
* @param string $path
* @param string $table
* @param string $tableClassName
* @return void
*/
protected function replaceMigration($path, $table, $tableClassName)
protected function replaceMigration($path, $table)
{
$stub = str_replace(
['{{table}}', '{{tableClassName}}'],
[$table, $tableClassName],
$this->files->get(__DIR__.'/stubs/jobs.stub')
'{{table}}', $table, $this->files->get(__DIR__.'/stubs/jobs.stub')
);
$this->files->put($path, $stub);

View File

@@ -2,15 +2,21 @@
namespace Illuminate\Queue\Console;
use Carbon\Carbon;
use Illuminate\Queue\Worker;
use Illuminate\Console\Command;
use Illuminate\Contracts\Cache\Repository as Cache;
use Illuminate\Contracts\Queue\Job;
use Illuminate\Queue\WorkerOptions;
use Illuminate\Queue\Events\JobFailed;
use Illuminate\Queue\Events\JobProcessed;
use Illuminate\Queue\Events\JobProcessing;
use Illuminate\Queue\Events\JobReleasedAfterException;
use Illuminate\Queue\Worker;
use Illuminate\Queue\WorkerOptions;
use Illuminate\Support\Carbon;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Terminal;
use function Termwind\terminal;
#[AsCommand(name: 'queue:work')]
class WorkCommand extends Command
{
/**
@@ -20,15 +26,32 @@ class WorkCommand extends Command
*/
protected $signature = 'queue:work
{connection? : The name of the queue connection to work}
{--name=default : The name of the worker}
{--queue= : The names of the queues to work}
{--daemon : Run the worker in daemon mode (Deprecated)}
{--once : Only process the next job on the queue}
{--delay=0 : Amount of time to delay failed jobs}
{--stop-when-empty : Stop when the queue is empty}
{--delay=0 : The number of seconds to delay failed jobs (Deprecated)}
{--backoff=0 : The number of seconds to wait before retrying a job that encountered an uncaught exception}
{--max-jobs=0 : The number of jobs to process before stopping}
{--max-time=0 : The maximum number of seconds the worker should run}
{--force : Force the worker to run even in maintenance mode}
{--memory=128 : The memory limit in megabytes}
{--sleep=3 : Number of seconds to sleep when no job is available}
{--rest=0 : Number of seconds to rest between jobs}
{--timeout=60 : The number of seconds a child process can run}
{--tries=0 : Number of times to attempt a job before logging it failed}';
{--tries=1 : Number of times to attempt a job before logging it failed}';
/**
* The name of the console command.
*
* This name is used to identify the command during lazy loading.
*
* @var string|null
*
* @deprecated
*/
protected static $defaultName = 'queue:work';
/**
* The console command description.
@@ -45,24 +68,40 @@ class WorkCommand extends Command
protected $worker;
/**
* Create a new queue listen command.
* The cache store implementation.
*
* @var \Illuminate\Contracts\Cache\Repository
*/
protected $cache;
/**
* Holds the start time of the last processed job, if any.
*
* @var float|null
*/
protected $latestStartedAt;
/**
* Create a new queue work command.
*
* @param \Illuminate\Queue\Worker $worker
* @param \Illuminate\Contracts\Cache\Repository $cache
* @return void
*/
public function __construct(Worker $worker)
public function __construct(Worker $worker, Cache $cache)
{
parent::__construct();
$this->cache = $cache;
$this->worker = $worker;
}
/**
* Execute the console command.
*
* @return void
* @return int|null
*/
public function fire()
public function handle()
{
if ($this->downForMaintenance() && $this->option('once')) {
return $this->worker->sleep($this->option('sleep'));
@@ -81,7 +120,13 @@ class WorkCommand extends Command
// connection being run for the queue operation currently being executed.
$queue = $this->getQueue($connection);
$this->runWorker(
if (Terminal::hasSttyAvailable()) {
$this->components->info(
sprintf('Processing jobs from the [%s] %s.', $queue, str('queue')->plural(explode(',', $queue)))
);
}
return $this->runWorker(
$connection, $queue
);
}
@@ -91,15 +136,16 @@ class WorkCommand extends Command
*
* @param string $connection
* @param string $queue
* @return array
* @return int|null
*/
protected function runWorker($connection, $queue)
{
$this->worker->setCache($this->laravel['cache']->driver());
return $this->worker->{$this->option('once') ? 'runNextJob' : 'daemon'}(
$connection, $queue, $this->gatherWorkerOptions()
);
return $this->worker
->setName($this->option('name'))
->setCache($this->cache)
->{$this->option('once') ? 'runNextJob' : 'daemon'}(
$connection, $queue, $this->gatherWorkerOptions()
);
}
/**
@@ -110,9 +156,17 @@ class WorkCommand extends Command
protected function gatherWorkerOptions()
{
return new WorkerOptions(
$this->option('delay'), $this->option('memory'),
$this->option('timeout'), $this->option('sleep'),
$this->option('tries'), $this->option('force')
$this->option('name'),
max($this->option('backoff'), $this->option('delay')),
$this->option('memory'),
$this->option('timeout'),
$this->option('sleep'),
$this->option('tries'),
$this->option('force'),
$this->option('stop-when-empty'),
$this->option('max-jobs'),
$this->option('max-time'),
$this->option('rest')
);
}
@@ -131,6 +185,10 @@ class WorkCommand extends Command
$this->writeOutput($event->job, 'success');
});
$this->laravel['events']->listen(JobReleasedAfterException::class, function ($event) {
$this->writeOutput($event->job, 'released_after_exception');
});
$this->laravel['events']->listen(JobFailed::class, function ($event) {
$this->writeOutput($event->job, 'failed');
@@ -142,49 +200,61 @@ class WorkCommand extends Command
* Write the status output for the queue worker.
*
* @param \Illuminate\Contracts\Queue\Job $job
* @param string $status
* @param string $status
* @return void
*/
protected function writeOutput(Job $job, $status)
{
switch ($status) {
case 'starting':
return $this->writeStatus($job, 'Processing', 'comment');
case 'success':
return $this->writeStatus($job, 'Processed', 'info');
case 'failed':
return $this->writeStatus($job, 'Failed', 'error');
}
}
/**
* Format the status output for the queue worker.
*
* @param \Illuminate\Contracts\Queue\Job $job
* @param string $status
* @param string $type
* @return void
*/
protected function writeStatus(Job $job, $status, $type)
{
$this->output->writeln(sprintf(
"<{$type}>[%s] %s</{$type}> %s",
$this->output->write(sprintf(
' <fg=gray>%s</> %s%s',
Carbon::now()->format('Y-m-d H:i:s'),
str_pad("{$status}:", 11), $job->resolveName()
$job->resolveName(),
$this->output->isVerbose()
? sprintf(' <fg=gray>%s</>', $job->getJobId())
: ''
));
if ($status == 'starting') {
$this->latestStartedAt = microtime(true);
$dots = max(terminal()->width() - mb_strlen($job->resolveName()) - (
$this->output->isVerbose() ? (mb_strlen($job->getJobId()) + 1) : 0
) - 33, 0);
$this->output->write(' '.str_repeat('<fg=gray>.</>', $dots));
return $this->output->writeln(' <fg=yellow;options=bold>RUNNING</>');
}
$runTime = number_format((microtime(true) - $this->latestStartedAt) * 1000, 2).'ms';
$dots = max(terminal()->width() - mb_strlen($job->resolveName()) - (
$this->output->isVerbose() ? (mb_strlen($job->getJobId()) + 1) : 0
) - mb_strlen($runTime) - 31, 0);
$this->output->write(' '.str_repeat('<fg=gray>.</>', $dots));
$this->output->write(" <fg=gray>$runTime</>");
$this->output->writeln(match ($status) {
'success' => ' <fg=green;options=bold>DONE</>',
'released_after_exception' => ' <fg=yellow;options=bold>FAIL</>',
default => ' <fg=red;options=bold>FAIL</>',
});
}
/**
* Store a failed job event.
*
* @param JobFailed $event
* @param \Illuminate\Queue\Events\JobFailed $event
* @return void
*/
protected function logFailedJob(JobFailed $event)
{
$this->laravel['queue.failer']->log(
$event->connectionName, $event->job->getQueue(),
$event->job->getRawBody(), $event->exception
$event->connectionName,
$event->job->getQueue(),
$event->job->getRawBody(),
$event->exception
);
}

View File

@@ -0,0 +1,39 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('{{table}}', function (Blueprint $table) {
$table->string('id')->primary();
$table->string('name');
$table->integer('total_jobs');
$table->integer('pending_jobs');
$table->integer('failed_jobs');
$table->longText('failed_job_ids');
$table->mediumText('options')->nullable();
$table->integer('cancelled_at')->nullable();
$table->integer('created_at');
$table->integer('finished_at')->nullable();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('{{table}}');
}
};

View File

@@ -1,10 +1,10 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class Create{{tableClassName}}Table extends Migration
return new class extends Migration
{
/**
* Run the migrations.
@@ -14,7 +14,8 @@ class Create{{tableClassName}}Table extends Migration
public function up()
{
Schema::create('{{table}}', function (Blueprint $table) {
$table->bigIncrements('id');
$table->id();
$table->string('uuid')->unique();
$table->text('connection');
$table->text('queue');
$table->longText('payload');
@@ -32,4 +33,4 @@ class Create{{tableClassName}}Table extends Migration
{
Schema::dropIfExists('{{table}}');
}
}
};

View File

@@ -1,10 +1,10 @@
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class Create{{tableClassName}}Table extends Migration
return new class extends Migration
{
/**
* Run the migrations.
@@ -15,14 +15,12 @@ class Create{{tableClassName}}Table extends Migration
{
Schema::create('{{table}}', function (Blueprint $table) {
$table->bigIncrements('id');
$table->string('queue');
$table->string('queue')->index();
$table->longText('payload');
$table->tinyInteger('attempts')->unsigned();
$table->unsignedTinyInteger('attempts');
$table->unsignedInteger('reserved_at')->nullable();
$table->unsignedInteger('available_at');
$table->unsignedInteger('created_at');
$table->index(['queue', 'reserved_at']);
});
}
@@ -35,4 +33,4 @@ class Create{{tableClassName}}Table extends Migration
{
Schema::dropIfExists('{{table}}');
}
}
};

View File

@@ -2,13 +2,16 @@
namespace Illuminate\Queue;
use Carbon\Carbon;
use Illuminate\Contracts\Queue\ClearableQueue;
use Illuminate\Contracts\Queue\Queue as QueueContract;
use Illuminate\Database\Connection;
use Illuminate\Queue\Jobs\DatabaseJob;
use Illuminate\Queue\Jobs\DatabaseJobRecord;
use Illuminate\Contracts\Queue\Queue as QueueContract;
use Illuminate\Support\Carbon;
use Illuminate\Support\Str;
use PDO;
class DatabaseQueue extends Queue implements QueueContract
class DatabaseQueue extends Queue implements QueueContract, ClearableQueue
{
/**
* The database connection instance.
@@ -45,20 +48,26 @@ class DatabaseQueue extends Queue implements QueueContract
* @param string $table
* @param string $default
* @param int $retryAfter
* @param bool $dispatchAfterCommit
* @return void
*/
public function __construct(Connection $database, $table, $default = 'default', $retryAfter = 60)
public function __construct(Connection $database,
$table,
$default = 'default',
$retryAfter = 60,
$dispatchAfterCommit = false)
{
$this->table = $table;
$this->default = $default;
$this->database = $database;
$this->retryAfter = $retryAfter;
$this->dispatchAfterCommit = $dispatchAfterCommit;
}
/**
* Get the size of the queue.
*
* @param string $queue
* @param string|null $queue
* @return int
*/
public function size($queue = null)
@@ -72,21 +81,29 @@ class DatabaseQueue extends Queue implements QueueContract
* Push a new job onto the queue.
*
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function push($job, $data = '', $queue = null)
{
return $this->pushToDatabase($queue, $this->createPayload($job, $data));
return $this->enqueueUsing(
$job,
$this->createPayload($job, $this->getQueue($queue), $data),
$queue,
null,
function ($payload, $queue) {
return $this->pushToDatabase($queue, $payload);
}
);
}
/**
* Push a raw payload onto the queue.
*
* @param string $payload
* @param string $queue
* @param array $options
* @param string|null $queue
* @param array $options
* @return mixed
*/
public function pushRaw($payload, $queue = null, array $options = [])
@@ -95,42 +112,54 @@ class DatabaseQueue extends Queue implements QueueContract
}
/**
* Push a new job onto the queue after a delay.
* Push a new job onto the queue after (n) seconds.
*
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return void
*/
public function later($delay, $job, $data = '', $queue = null)
{
return $this->pushToDatabase($queue, $this->createPayload($job, $data), $delay);
return $this->enqueueUsing(
$job,
$this->createPayload($job, $this->getQueue($queue), $data),
$queue,
$delay,
function ($payload, $queue, $delay) {
return $this->pushToDatabase($queue, $payload, $delay);
}
);
}
/**
* Push an array of jobs onto the queue.
*
* @param array $jobs
* @param mixed $data
* @param string $queue
* @param array $jobs
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function bulk($jobs, $data = '', $queue = null)
{
$queue = $this->getQueue($queue);
$availableAt = $this->availableAt();
$now = $this->availableAt();
return $this->database->table($this->table)->insert(collect((array) $jobs)->map(
function ($job) use ($queue, $data, $availableAt) {
return $this->buildDatabaseRecord($queue, $this->createPayload($job, $data), $availableAt);
function ($job) use ($queue, $data, $now) {
return $this->buildDatabaseRecord(
$queue,
$this->createPayload($job, $this->getQueue($queue), $data),
isset($job->delay) ? $this->availableAt($job->delay) : $now,
);
}
)->all());
}
/**
* Release a reserved job back onto the queue.
* Release a reserved job back onto the queue after (n) seconds.
*
* @param string $queue
* @param \Illuminate\Queue\Jobs\DatabaseJobRecord $job
@@ -143,11 +172,11 @@ class DatabaseQueue extends Queue implements QueueContract
}
/**
* Push a raw payload to the database with a given delay.
* Push a raw payload to the database with a given delay of (n) seconds.
*
* @param string|null $queue
* @param string $payload
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param int $attempts
* @return mixed
*/
@@ -171,31 +200,31 @@ class DatabaseQueue extends Queue implements QueueContract
{
return [
'queue' => $queue,
'payload' => $payload,
'attempts' => $attempts,
'reserved_at' => null,
'available_at' => $availableAt,
'created_at' => $this->currentTime(),
'payload' => $payload,
];
}
/**
* Pop the next job off of the queue.
*
* @param string $queue
* @param string|null $queue
* @return \Illuminate\Contracts\Queue\Job|null
*
* @throws \Throwable
*/
public function pop($queue = null)
{
$queue = $this->getQueue($queue);
$this->database->beginTransaction();
if ($job = $this->getNextAvailableJob($queue)) {
return $this->marshalJob($queue, $job);
}
$this->database->commit();
return $this->database->transaction(function () use ($queue) {
if ($job = $this->getNextAvailableJob($queue)) {
return $this->marshalJob($queue, $job);
}
});
}
/**
@@ -207,7 +236,7 @@ class DatabaseQueue extends Queue implements QueueContract
protected function getNextAvailableJob($queue)
{
$job = $this->database->table($this->table)
->lockForUpdate()
->lock($this->getLockForPopping())
->where('queue', $this->getQueue($queue))
->where(function ($query) {
$this->isAvailable($query);
@@ -219,6 +248,37 @@ class DatabaseQueue extends Queue implements QueueContract
return $job ? new DatabaseJobRecord((object) $job) : null;
}
/**
* Get the lock required for popping the next job.
*
* @return string|bool
*/
protected function getLockForPopping()
{
$databaseEngine = $this->database->getPdo()->getAttribute(PDO::ATTR_DRIVER_NAME);
$databaseVersion = $this->database->getConfig('version') ?? $this->database->getPdo()->getAttribute(PDO::ATTR_SERVER_VERSION);
if (Str::of($databaseVersion)->contains('MariaDB')) {
$databaseEngine = 'mariadb';
$databaseVersion = Str::before(Str::after($databaseVersion, '5.5.5-'), '-');
} elseif (Str::of($databaseVersion)->contains(['vitess', 'PlanetScale'])) {
$databaseEngine = 'vitess';
$databaseVersion = Str::before($databaseVersion, '-');
}
if (($databaseEngine === 'mysql' && version_compare($databaseVersion, '8.0.1', '>=')) ||
($databaseEngine === 'mariadb' && version_compare($databaseVersion, '10.6.0', '>=')) ||
($databaseEngine === 'pgsql' && version_compare($databaseVersion, '9.5', '>='))) {
return 'FOR UPDATE SKIP LOCKED';
}
if ($databaseEngine === 'sqlsrv') {
return 'with(rowlock,updlock,readpast)';
}
return true;
}
/**
* Modify the query to check for available jobs.
*
@@ -259,8 +319,6 @@ class DatabaseQueue extends Queue implements QueueContract
{
$job = $this->markJobAsReserved($job);
$this->database->commit();
return new DatabaseJob(
$this->container, $this, $job, $this->connectionName, $queue
);
@@ -288,16 +346,48 @@ class DatabaseQueue extends Queue implements QueueContract
* @param string $queue
* @param string $id
* @return void
*
* @throws \Throwable
*/
public function deleteReserved($queue, $id)
{
$this->database->beginTransaction();
$this->database->transaction(function () use ($id) {
if ($this->database->table($this->table)->lockForUpdate()->find($id)) {
$this->database->table($this->table)->where('id', $id)->delete();
}
});
}
if ($this->database->table($this->table)->lockForUpdate()->find($id)) {
$this->database->table($this->table)->where('id', $id)->delete();
}
/**
* Delete a reserved job from the reserved queue and release it.
*
* @param string $queue
* @param \Illuminate\Queue\Jobs\DatabaseJob $job
* @param int $delay
* @return void
*/
public function deleteAndRelease($queue, $job, $delay)
{
$this->database->transaction(function () use ($queue, $job, $delay) {
if ($this->database->table($this->table)->lockForUpdate()->find($job->getJobId())) {
$this->database->table($this->table)->where('id', $job->getJobId())->delete();
}
$this->database->commit();
$this->release($queue, $job->getJobRecord(), $delay);
});
}
/**
* Delete all of the jobs from the queue.
*
* @param string $queue
* @return int
*/
public function clear($queue)
{
return $this->database->table($this->table)
->where('queue', $this->getQueue($queue))
->delete();
}
/**
@@ -306,7 +396,7 @@ class DatabaseQueue extends Queue implements QueueContract
* @param string|null $queue
* @return string
*/
protected function getQueue($queue)
public function getQueue($queue)
{
return $queue ?: $this->default;
}

View File

@@ -21,7 +21,7 @@ class JobExceptionOccurred
/**
* The exception instance.
*
* @var \Exception
* @var \Throwable
*/
public $exception;
@@ -30,7 +30,7 @@ class JobExceptionOccurred
*
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Exception $exception
* @param \Throwable $exception
* @return void
*/
public function __construct($connectionName, $job, $exception)

View File

@@ -21,7 +21,7 @@ class JobFailed
/**
* The exception that caused the job to fail.
*
* @var \Exception
* @var \Throwable
*/
public $exception;
@@ -30,7 +30,7 @@ class JobFailed
*
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Exception $exception
* @param \Throwable $exception
* @return void
*/
public function __construct($connectionName, $job, $exception)

View File

@@ -0,0 +1,42 @@
<?php
namespace Illuminate\Queue\Events;
class JobQueued
{
/**
* The connection name.
*
* @var string
*/
public $connectionName;
/**
* The job ID.
*
* @var string|int|null
*/
public $id;
/**
* The job instance.
*
* @var \Closure|string|object
*/
public $job;
/**
* Create a new event instance.
*
* @param string $connectionName
* @param string|int|null $id
* @param \Closure|string|object $job
* @return void
*/
public function __construct($connectionName, $id, $job)
{
$this->connectionName = $connectionName;
$this->id = $id;
$this->job = $job;
}
}

View File

@@ -0,0 +1,33 @@
<?php
namespace Illuminate\Queue\Events;
class JobReleasedAfterException
{
/**
* The connection name.
*
* @var string
*/
public $connectionName;
/**
* The job instance.
*
* @var \Illuminate\Contracts\Queue\Job
*/
public $job;
/**
* Create a new event instance.
*
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @return void
*/
public function __construct($connectionName, $job)
{
$this->job = $job;
$this->connectionName = $connectionName;
}
}

View File

@@ -0,0 +1,45 @@
<?php
namespace Illuminate\Queue\Events;
class JobRetryRequested
{
/**
* The job instance.
*
* @var \stdClass
*/
public $job;
/**
* The decoded job payload.
*
* @var array|null
*/
protected $payload = null;
/**
* Create a new event instance.
*
* @param \stdClass $job
* @return void
*/
public function __construct($job)
{
$this->job = $job;
}
/**
* The job payload.
*
* @return array
*/
public function payload()
{
if (is_null($this->payload)) {
$this->payload = json_decode($this->job->payload, true);
}
return $this->payload;
}
}

View File

@@ -4,5 +4,30 @@ namespace Illuminate\Queue\Events;
class Looping
{
//
/**
* The connection name.
*
* @var string
*/
public $connectionName;
/**
* The queue name.
*
* @var string
*/
public $queue;
/**
* Create a new event instance.
*
* @param string $connectionName
* @param string $queue
* @return void
*/
public function __construct($connectionName, $queue)
{
$this->queue = $queue;
$this->connectionName = $connectionName;
}
}

View File

@@ -0,0 +1,42 @@
<?php
namespace Illuminate\Queue\Events;
class QueueBusy
{
/**
* The connection name.
*
* @var string
*/
public $connection;
/**
* The queue name.
*
* @var string
*/
public $queue;
/**
* The size of the queue.
*
* @var int
*/
public $size;
/**
* Create a new event instance.
*
* @param string $connection
* @param string $queue
* @param int $size
* @return void
*/
public function __construct($connection, $queue, $size)
{
$this->connection = $connection;
$this->queue = $queue;
$this->size = $size;
}
}

View File

@@ -4,5 +4,30 @@ namespace Illuminate\Queue\Events;
class WorkerStopping
{
//
/**
* The worker exit status.
*
* @var int
*/
public $status;
/**
* The worker options.
*
* @var \Illuminate\Queue\WorkerOptions|null
*/
public $workerOptions;
/**
* Create a new event instance.
*
* @param int $status
* @param \Illuminate\Queue\WorkerOptions|null $workerOptions
* @return void
*/
public function __construct($status = 0, $workerOptions = null)
{
$this->status = $status;
$this->workerOptions = $workerOptions;
}
}

View File

@@ -2,10 +2,11 @@
namespace Illuminate\Queue\Failed;
use Carbon\Carbon;
use DateTimeInterface;
use Illuminate\Database\ConnectionResolverInterface;
use Illuminate\Support\Facades\Date;
class DatabaseFailedJobProvider implements FailedJobProviderInterface
class DatabaseFailedJobProvider implements FailedJobProviderInterface, PrunableFailedJobProvider
{
/**
* The connection resolver implementation.
@@ -49,14 +50,14 @@ class DatabaseFailedJobProvider implements FailedJobProviderInterface
* @param string $connection
* @param string $queue
* @param string $payload
* @param \Exception $exception
* @param \Throwable $exception
* @return int|null
*/
public function log($connection, $queue, $payload, $exception)
{
$failed_at = Carbon::now();
$failed_at = Date::now();
$exception = (string) $exception;
$exception = (string) mb_convert_encoding($exception, 'UTF-8');
return $this->getTable()->insertGetId(compact(
'connection', 'queue', 'payload', 'exception', 'failed_at'
@@ -77,7 +78,7 @@ class DatabaseFailedJobProvider implements FailedJobProviderInterface
* Get a single failed job.
*
* @param mixed $id
* @return array
* @return object|null
*/
public function find($id)
{
@@ -98,11 +99,35 @@ class DatabaseFailedJobProvider implements FailedJobProviderInterface
/**
* Flush all of the failed jobs from storage.
*
* @param int|null $hours
* @return void
*/
public function flush()
public function flush($hours = null)
{
$this->getTable()->delete();
$this->getTable()->when($hours, function ($query, $hours) {
$query->where('failed_at', '<=', Date::now()->subHours($hours));
})->delete();
}
/**
* Prune all of the entries older than the given date.
*
* @param \DateTimeInterface $before
* @return int
*/
public function prune(DateTimeInterface $before)
{
$query = $this->getTable()->where('failed_at', '<', $before);
$totalDeleted = 0;
do {
$deleted = $query->take(1000)->delete();
$totalDeleted += $deleted;
} while ($deleted !== 0);
return $totalDeleted;
}
/**

View File

@@ -0,0 +1,155 @@
<?php
namespace Illuminate\Queue\Failed;
use DateTimeInterface;
use Illuminate\Database\ConnectionResolverInterface;
use Illuminate\Support\Facades\Date;
class DatabaseUuidFailedJobProvider implements FailedJobProviderInterface, PrunableFailedJobProvider
{
/**
* The connection resolver implementation.
*
* @var \Illuminate\Database\ConnectionResolverInterface
*/
protected $resolver;
/**
* The database connection name.
*
* @var string
*/
protected $database;
/**
* The database table.
*
* @var string
*/
protected $table;
/**
* Create a new database failed job provider.
*
* @param \Illuminate\Database\ConnectionResolverInterface $resolver
* @param string $database
* @param string $table
* @return void
*/
public function __construct(ConnectionResolverInterface $resolver, $database, $table)
{
$this->table = $table;
$this->resolver = $resolver;
$this->database = $database;
}
/**
* Log a failed job into storage.
*
* @param string $connection
* @param string $queue
* @param string $payload
* @param \Throwable $exception
* @return string|null
*/
public function log($connection, $queue, $payload, $exception)
{
$this->getTable()->insert([
'uuid' => $uuid = json_decode($payload, true)['uuid'],
'connection' => $connection,
'queue' => $queue,
'payload' => $payload,
'exception' => (string) mb_convert_encoding($exception, 'UTF-8'),
'failed_at' => Date::now(),
]);
return $uuid;
}
/**
* Get a list of all of the failed jobs.
*
* @return array
*/
public function all()
{
return $this->getTable()->orderBy('id', 'desc')->get()->map(function ($record) {
$record->id = $record->uuid;
unset($record->uuid);
return $record;
})->all();
}
/**
* Get a single failed job.
*
* @param mixed $id
* @return object|null
*/
public function find($id)
{
if ($record = $this->getTable()->where('uuid', $id)->first()) {
$record->id = $record->uuid;
unset($record->uuid);
}
return $record;
}
/**
* Delete a single failed job from storage.
*
* @param mixed $id
* @return bool
*/
public function forget($id)
{
return $this->getTable()->where('uuid', $id)->delete() > 0;
}
/**
* Flush all of the failed jobs from storage.
*
* @param int|null $hours
* @return void
*/
public function flush($hours = null)
{
$this->getTable()->when($hours, function ($query, $hours) {
$query->where('failed_at', '<=', Date::now()->subHours($hours));
})->delete();
}
/**
* Prune all of the entries older than the given date.
*
* @param \DateTimeInterface $before
* @return int
*/
public function prune(DateTimeInterface $before)
{
$query = $this->getTable()->where('failed_at', '<', $before);
$totalDeleted = 0;
do {
$deleted = $query->take(1000)->delete();
$totalDeleted += $deleted;
} while ($deleted !== 0);
return $totalDeleted;
}
/**
* Get a new query builder instance for the table.
*
* @return \Illuminate\Database\Query\Builder
*/
protected function getTable()
{
return $this->resolver->connection($this->database)->table($this->table);
}
}

View File

@@ -0,0 +1,177 @@
<?php
namespace Illuminate\Queue\Failed;
use Aws\DynamoDb\DynamoDbClient;
use DateTimeInterface;
use Exception;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Date;
class DynamoDbFailedJobProvider implements FailedJobProviderInterface
{
/**
* The DynamoDB client instance.
*
* @var \Aws\DynamoDb\DynamoDbClient
*/
protected $dynamo;
/**
* The application name.
*
* @var string
*/
protected $applicationName;
/**
* The table name.
*
* @var string
*/
protected $table;
/**
* Create a new DynamoDb failed job provider.
*
* @param \Aws\DynamoDb\DynamoDbClient $dynamo
* @param string $applicationName
* @param string $table
* @return void
*/
public function __construct(DynamoDbClient $dynamo, $applicationName, $table)
{
$this->table = $table;
$this->dynamo = $dynamo;
$this->applicationName = $applicationName;
}
/**
* Log a failed job into storage.
*
* @param string $connection
* @param string $queue
* @param string $payload
* @param \Throwable $exception
* @return string|int|null
*/
public function log($connection, $queue, $payload, $exception)
{
$id = json_decode($payload, true)['uuid'];
$failedAt = Date::now();
$this->dynamo->putItem([
'TableName' => $this->table,
'Item' => [
'application' => ['S' => $this->applicationName],
'uuid' => ['S' => $id],
'connection' => ['S' => $connection],
'queue' => ['S' => $queue],
'payload' => ['S' => $payload],
'exception' => ['S' => (string) $exception],
'failed_at' => ['N' => (string) $failedAt->getTimestamp()],
'expires_at' => ['N' => (string) $failedAt->addDays(3)->getTimestamp()],
],
]);
return $id;
}
/**
* Get a list of all of the failed jobs.
*
* @return array
*/
public function all()
{
$results = $this->dynamo->query([
'TableName' => $this->table,
'Select' => 'ALL_ATTRIBUTES',
'KeyConditionExpression' => 'application = :application',
'ExpressionAttributeValues' => [
':application' => ['S' => $this->applicationName],
],
'ScanIndexForward' => false,
]);
return collect($results['Items'])->sortByDesc(function ($result) {
return (int) $result['failed_at']['N'];
})->map(function ($result) {
return (object) [
'id' => $result['uuid']['S'],
'connection' => $result['connection']['S'],
'queue' => $result['queue']['S'],
'payload' => $result['payload']['S'],
'exception' => $result['exception']['S'],
'failed_at' => Carbon::createFromTimestamp(
(int) $result['failed_at']['N']
)->format(DateTimeInterface::ISO8601),
];
})->all();
}
/**
* Get a single failed job.
*
* @param mixed $id
* @return object|null
*/
public function find($id)
{
$result = $this->dynamo->getItem([
'TableName' => $this->table,
'Key' => [
'application' => ['S' => $this->applicationName],
'uuid' => ['S' => $id],
],
]);
if (! isset($result['Item'])) {
return;
}
return (object) [
'id' => $result['Item']['uuid']['S'],
'connection' => $result['Item']['connection']['S'],
'queue' => $result['Item']['queue']['S'],
'payload' => $result['Item']['payload']['S'],
'exception' => $result['Item']['exception']['S'],
'failed_at' => Carbon::createFromTimestamp(
(int) $result['Item']['failed_at']['N']
)->format(DateTimeInterface::ISO8601),
];
}
/**
* Delete a single failed job from storage.
*
* @param mixed $id
* @return bool
*/
public function forget($id)
{
$this->dynamo->deleteItem([
'TableName' => $this->table,
'Key' => [
'application' => ['S' => $this->applicationName],
'uuid' => ['S' => $id],
],
]);
return true;
}
/**
* Flush all of the failed jobs from storage.
*
* @param int|null $hours
* @return void
*
* @throws \Exception
*/
public function flush($hours = null)
{
throw new Exception("DynamoDb failed job storage may not be flushed. Please use DynamoDb's TTL features on your expires_at attribute.");
}
}

View File

@@ -10,8 +10,8 @@ interface FailedJobProviderInterface
* @param string $connection
* @param string $queue
* @param string $payload
* @param \Exception $exception
* @return int|null
* @param \Throwable $exception
* @return string|int|null
*/
public function log($connection, $queue, $payload, $exception);
@@ -26,7 +26,7 @@ interface FailedJobProviderInterface
* Get a single failed job.
*
* @param mixed $id
* @return array
* @return object|null
*/
public function find($id);
@@ -41,7 +41,8 @@ interface FailedJobProviderInterface
/**
* Flush all of the failed jobs from storage.
*
* @param int|null $hours
* @return void
*/
public function flush();
public function flush($hours = null);
}

View File

@@ -10,7 +10,7 @@ class NullFailedJobProvider implements FailedJobProviderInterface
* @param string $connection
* @param string $queue
* @param string $payload
* @param \Exception $exception
* @param \Throwable $exception
* @return int|null
*/
public function log($connection, $queue, $payload, $exception)
@@ -32,7 +32,7 @@ class NullFailedJobProvider implements FailedJobProviderInterface
* Get a single failed job.
*
* @param mixed $id
* @return array
* @return object|null
*/
public function find($id)
{
@@ -53,9 +53,10 @@ class NullFailedJobProvider implements FailedJobProviderInterface
/**
* Flush all of the failed jobs from storage.
*
* @param int|null $hours
* @return void
*/
public function flush()
public function flush($hours = null)
{
//
}

View File

@@ -0,0 +1,16 @@
<?php
namespace Illuminate\Queue\Failed;
use DateTimeInterface;
interface PrunableFailedJobProvider
{
/**
* Prune all of the entries older than the given date.
*
* @param \DateTimeInterface $before
* @return int
*/
public function prune(DateTimeInterface $before);
}

View File

@@ -1,50 +0,0 @@
<?php
namespace Illuminate\Queue;
use Illuminate\Container\Container;
use Illuminate\Queue\Events\JobFailed;
use Illuminate\Contracts\Events\Dispatcher;
class FailingJob
{
/**
* Delete the job, call the "failed" method, and raise the failed job event.
*
* @param string $connectionName
* @param \Illuminate\Queue\Jobs\Job $job
* @param \Exception $e
* @return void
*/
public static function handle($connectionName, $job, $e = null)
{
$job->markAsFailed();
if ($job->isDeleted()) {
return;
}
try {
// If the job has failed, we will delete it, call the "failed" method and then call
// an event indicating the job has failed so it can be logged if needed. This is
// to allow every developer to better keep monitor of their failed queue jobs.
$job->delete();
$job->failed($e);
} finally {
static::events()->fire(new JobFailed(
$connectionName, $job, $e ?: new ManuallyFailedException
));
}
}
/**
* Get the event dispatcher instance.
*
* @return \Illuminate\Contracts\Events\Dispatcher
*/
protected static function events()
{
return Container::getInstance()->make(Dispatcher::class);
}
}

View File

@@ -3,15 +3,17 @@
namespace Illuminate\Queue;
use Illuminate\Contracts\Queue\Job as JobContract;
use InvalidArgumentException;
use Throwable;
trait InteractsWithQueue
{
/**
* The underlying queue job instance.
*
* @var \Illuminate\Contracts\Queue\Job
* @var \Illuminate\Contracts\Queue\Job|null
*/
protected $job;
public $job;
/**
* Get the number of times the job has been attempted.
@@ -38,20 +40,24 @@ trait InteractsWithQueue
/**
* Fail the job from the queue.
*
* @param \Throwable $exception
* @param \Throwable|null $exception
* @return void
*/
public function fail($exception = null)
{
if ($this->job) {
FailingJob::handle($this->job->getConnectionName(), $this->job, $exception);
if ($exception instanceof Throwable || is_null($exception)) {
if ($this->job) {
return $this->job->fail($exception);
}
} else {
throw new InvalidArgumentException('The fail method requires an instance of Throwable.');
}
}
/**
* Release the job back into the queue.
* Release the job back into the queue after (n) seconds.
*
* @param int $delay
* @param int $delay
* @return void
*/
public function release($delay = 0)

View File

@@ -1,45 +0,0 @@
<?php
namespace Illuminate\Queue;
use Carbon\Carbon;
use DateTimeInterface;
trait InteractsWithTime
{
/**
* Get the number of seconds until the given DateTime.
*
* @param \DateTimeInterface $delay
* @return int
*/
protected function secondsUntil($delay)
{
return $delay instanceof DateTimeInterface
? max(0, $delay->getTimestamp() - $this->currentTime())
: (int) $delay;
}
/**
* Get the "available at" UNIX timestamp.
*
* @param \DateTimeInterface|int $delay
* @return int
*/
protected function availableAt($delay = 0)
{
return $delay instanceof DateTimeInterface
? $delay->getTimestamp()
: Carbon::now()->addSeconds($delay)->getTimestamp();
}
/**
* Get the current system time as a UNIX timestamp.
*
* @return int
*/
protected function currentTime()
{
return Carbon::now()->getTimestamp();
}
}

View File

@@ -2,10 +2,10 @@
namespace Illuminate\Queue\Jobs;
use Pheanstalk\Pheanstalk;
use Illuminate\Container\Container;
use Pheanstalk\Job as PheanstalkJob;
use Illuminate\Contracts\Queue\Job as JobContract;
use Pheanstalk\Job as PheanstalkJob;
use Pheanstalk\Pheanstalk;
class BeanstalkdJob extends Job implements JobContract
{
@@ -43,9 +43,9 @@ class BeanstalkdJob extends Job implements JobContract
}
/**
* Release the job back into the queue.
* Release the job back into the queue after (n) seconds.
*
* @param int $delay
* @param int $delay
* @return void
*/
public function release($delay = 0)
@@ -96,7 +96,7 @@ class BeanstalkdJob extends Job implements JobContract
/**
* Get the job identifier.
*
* @return string
* @return int
*/
public function getJobId()
{

View File

@@ -3,8 +3,8 @@
namespace Illuminate\Queue\Jobs;
use Illuminate\Container\Container;
use Illuminate\Queue\DatabaseQueue;
use Illuminate\Contracts\Queue\Job as JobContract;
use Illuminate\Queue\DatabaseQueue;
class DatabaseJob extends Job implements JobContract
{
@@ -42,18 +42,16 @@ class DatabaseJob extends Job implements JobContract
}
/**
* Release the job back into the queue.
* Release the job back into the queue after (n) seconds.
*
* @param int $delay
* @return mixed
* @return void
*/
public function release($delay = 0)
{
parent::release($delay);
$this->delete();
return $this->database->release($this->queue, $this->job, $delay);
$this->database->deleteAndRelease($this->queue, $this, $delay);
}
/**
@@ -97,4 +95,14 @@ class DatabaseJob extends Job implements JobContract
{
return $this->job->payload;
}
/**
* Get the database job record.
*
* @return \Illuminate\Queue\Jobs\DatabaseJobRecord
*/
public function getJobRecord()
{
return $this->job;
}
}

View File

@@ -2,7 +2,7 @@
namespace Illuminate\Queue\Jobs;
use Illuminate\Queue\InteractsWithTime;
use Illuminate\Support\InteractsWithTime;
class DatabaseJobRecord
{

View File

@@ -2,7 +2,10 @@
namespace Illuminate\Queue\Jobs;
use Illuminate\Queue\InteractsWithTime;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Queue\Events\JobFailed;
use Illuminate\Queue\ManuallyFailedException;
use Illuminate\Support\InteractsWithTime;
abstract class Job
{
@@ -45,6 +48,8 @@ abstract class Job
/**
* The name of the connection the job belongs to.
*
* @var string
*/
protected $connectionName;
@@ -55,6 +60,30 @@ abstract class Job
*/
protected $queue;
/**
* Get the job identifier.
*
* @return string
*/
abstract public function getJobId();
/**
* Get the raw body of the job.
*
* @return string
*/
abstract public function getRawBody();
/**
* Get the UUID of the job.
*
* @return string|null
*/
public function uuid()
{
return $this->payload()['uuid'] ?? null;
}
/**
* Fire the job.
*
@@ -64,9 +93,9 @@ abstract class Job
{
$payload = $this->payload();
list($class, $method) = JobName::parse($payload['job']);
[$class, $method] = JobName::parse($payload['job']);
with($this->instance = $this->resolve($class))->{$method}($this, $payload['data']);
($this->instance = $this->resolve($class))->{$method}($this, $payload['data']);
}
/**
@@ -90,9 +119,9 @@ abstract class Job
}
/**
* Release the job back into the queue.
* Release the job back into the queue after (n) seconds.
*
* @param int $delay
* @param int $delay
* @return void
*/
public function release($delay = 0)
@@ -141,21 +170,47 @@ abstract class Job
}
/**
* Process an exception that caused the job to fail.
* Delete the job, call the "failed" method, and raise the failed job event.
*
* @param \Exception $e
* @param \Throwable|null $e
* @return void
*/
public function failed($e)
public function fail($e = null)
{
$this->markAsFailed();
if ($this->isDeleted()) {
return;
}
try {
// If the job has failed, we will delete it, call the "failed" method and then call
// an event indicating the job has failed so it can be logged if needed. This is
// to allow every developer to better keep monitor of their failed queue jobs.
$this->delete();
$this->failed($e);
} finally {
$this->resolve(Dispatcher::class)->dispatch(new JobFailed(
$this->connectionName, $this, $e ?: new ManuallyFailedException
));
}
}
/**
* Process an exception that caused the job to fail.
*
* @param \Throwable|null $e
* @return void
*/
protected function failed($e)
{
$payload = $this->payload();
list($class, $method) = JobName::parse($payload['job']);
[$class, $method] = JobName::parse($payload['job']);
if (method_exists($this->instance = $this->resolve($class), 'failed')) {
$this->instance->failed($payload['data'], $e);
$this->instance->failed($payload['data'], $e, $payload['uuid'] ?? '');
}
}
@@ -170,6 +225,16 @@ abstract class Job
return $this->container->make($class);
}
/**
* Get the resolved job handler instance.
*
* @return mixed
*/
public function getResolvedJob()
{
return $this->instance;
}
/**
* Get the decoded body of the job.
*
@@ -181,23 +246,63 @@ abstract class Job
}
/**
* The number of times to attempt a job.
* Get the number of times to attempt a job.
*
* @return int|null
*/
public function maxTries()
{
return array_get($this->payload(), 'maxTries');
return $this->payload()['maxTries'] ?? null;
}
/**
* The number of seconds the job can run.
* Get the number of times to attempt a job after an exception.
*
* @return int|null
*/
public function maxExceptions()
{
return $this->payload()['maxExceptions'] ?? null;
}
/**
* Determine if the job should fail when it timeouts.
*
* @return bool
*/
public function shouldFailOnTimeout()
{
return $this->payload()['failOnTimeout'] ?? false;
}
/**
* The number of seconds to wait before retrying a job that encountered an uncaught exception.
*
* @return int|int[]|null
*/
public function backoff()
{
return $this->payload()['backoff'] ?? $this->payload()['delay'] ?? null;
}
/**
* Get the number of seconds the job can run.
*
* @return int|null
*/
public function timeout()
{
return array_get($this->payload(), 'timeout');
return $this->payload()['timeout'] ?? null;
}
/**
* Get the timestamp indicating when the job should timeout.
*
* @return int|null
*/
public function retryUntil()
{
return $this->payload()['retryUntil'] ?? null;
}
/**

View File

@@ -2,7 +2,6 @@
namespace Illuminate\Queue\Jobs;
use Illuminate\Support\Arr;
use Illuminate\Support\Str;
class JobName
@@ -31,14 +30,6 @@ class JobName
return $payload['displayName'];
}
if ($name === 'Illuminate\Queue\CallQueuedHandler@call') {
return Arr::get($payload, 'data.commandName', $name);
}
if ($name === 'Illuminate\Events\CallQueuedHandler@call') {
return $payload['data']['class'].'@'.$payload['data']['method'];
}
return $name;
}
}

View File

@@ -2,10 +2,9 @@
namespace Illuminate\Queue\Jobs;
use Illuminate\Support\Arr;
use Illuminate\Queue\RedisQueue;
use Illuminate\Container\Container;
use Illuminate\Contracts\Queue\Job as JobContract;
use Illuminate\Queue\RedisQueue;
class RedisJob extends Job implements JobContract
{
@@ -86,9 +85,9 @@ class RedisJob extends Job implements JobContract
}
/**
* Release the job back into the queue.
* Release the job back into the queue after (n) seconds.
*
* @param int $delay
* @param int $delay
* @return void
*/
public function release($delay = 0)
@@ -105,23 +104,23 @@ class RedisJob extends Job implements JobContract
*/
public function attempts()
{
return Arr::get($this->decoded, 'attempts') + 1;
return ($this->decoded['attempts'] ?? null) + 1;
}
/**
* Get the job identifier.
*
* @return string
* @return string|null
*/
public function getJobId()
{
return Arr::get($this->decoded, 'id');
return $this->decoded['id'] ?? null;
}
/**
* Get the underlying Redis factory implementation.
*
* @return \Illuminate\Contracts\Redis\Factory
* @return \Illuminate\Queue\RedisQueue
*/
public function getRedisQueue()
{

View File

@@ -27,7 +27,7 @@ class SqsJob extends Job implements JobContract
*
* @param \Illuminate\Container\Container $container
* @param \Aws\Sqs\SqsClient $sqs
* @param array $job
* @param array $job
* @param string $connectionName
* @param string $queue
* @return void
@@ -42,9 +42,9 @@ class SqsJob extends Job implements JobContract
}
/**
* Release the job back into the queue.
* Release the job back into the queue after (n) seconds.
*
* @param int $delay
* @param int $delay
* @return void
*/
public function release($delay = 0)

View File

@@ -39,9 +39,9 @@ class SyncJob extends Job implements JobContract
}
/**
* Release the job back into the queue.
* Release the job back into the queue after (n) seconds.
*
* @param int $delay
* @param int $delay
* @return void
*/
public function release($delay = 0)

View File

@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) Taylor Otwell
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@@ -3,9 +3,8 @@
namespace Illuminate\Queue;
use Closure;
use Symfony\Component\Process\Process;
use Symfony\Component\Process\ProcessUtils;
use Symfony\Component\Process\PhpExecutableFinder;
use Symfony\Component\Process\Process;
class Listener
{
@@ -31,19 +30,12 @@ class Listener
protected $sleep = 3;
/**
* The amount of times to try a job before logging it failed.
* The number of times to try a job before logging it failed.
*
* @var int
*/
protected $maxTries = 0;
/**
* The queue worker command line.
*
* @var string
*/
protected $workerCommand;
/**
* The output handler callback.
*
@@ -60,19 +52,6 @@ class Listener
public function __construct($commandPath)
{
$this->commandPath = $commandPath;
$this->workerCommand = $this->buildCommandTemplate();
}
/**
* Build the environment specific worker command.
*
* @return string
*/
protected function buildCommandTemplate()
{
$command = 'queue:work %s --once --queue=%s --delay=%s --memory=%s --sleep=%s --tries=%s';
return "{$this->phpBinary()} {$this->artisanBinary()} {$command}";
}
/**
@@ -82,9 +61,7 @@ class Listener
*/
protected function phpBinary()
{
return ProcessUtils::escapeArgument(
(new PhpExecutableFinder)->find(false)
);
return (new PhpExecutableFinder)->find(false);
}
/**
@@ -94,9 +71,7 @@ class Listener
*/
protected function artisanBinary()
{
return defined('ARTISAN_BINARY')
? ProcessUtils::escapeArgument(ARTISAN_BINARY)
: 'artisan';
return defined('ARTISAN_BINARY') ? ARTISAN_BINARY : 'artisan';
}
/**
@@ -113,6 +88,10 @@ class Listener
while (true) {
$this->runProcess($process, $options->memory);
if ($options->rest) {
sleep($options->rest);
}
}
}
@@ -126,57 +105,66 @@ class Listener
*/
public function makeProcess($connection, $queue, ListenerOptions $options)
{
$command = $this->workerCommand;
$command = $this->createCommand(
$connection,
$queue,
$options
);
// If the environment is set, we will append it to the command string so the
// If the environment is set, we will append it to the command array so the
// workers will run under the specified environment. Otherwise, they will
// just run under the production environment which is not always right.
if (isset($options->environment)) {
$command = $this->addEnvironment($command, $options);
}
// Next, we will just format out the worker commands with all of the various
// options available for the command. This will produce the final command
// line that we will pass into a Symfony process object for processing.
$command = $this->formatCommand(
$command, $connection, $queue, $options
);
return new Process(
$command, $this->commandPath, null, null, $options->timeout
$command,
$this->commandPath,
null,
null,
$options->timeout
);
}
/**
* Add the environment option to the given command.
*
* @param string $command
* @param array $command
* @param \Illuminate\Queue\ListenerOptions $options
* @return string
* @return array
*/
protected function addEnvironment($command, ListenerOptions $options)
{
return $command.' --env='.ProcessUtils::escapeArgument($options->environment);
return array_merge($command, ["--env={$options->environment}"]);
}
/**
* Format the given command with the listener options.
* Create the command with the listener options.
*
* @param string $command
* @param string $connection
* @param string $queue
* @param \Illuminate\Queue\ListenerOptions $options
* @return string
* @return array
*/
protected function formatCommand($command, $connection, $queue, ListenerOptions $options)
protected function createCommand($connection, $queue, ListenerOptions $options)
{
return sprintf(
$command,
ProcessUtils::escapeArgument($connection),
ProcessUtils::escapeArgument($queue),
$options->delay, $options->memory,
$options->sleep, $options->maxTries
);
return array_filter([
$this->phpBinary(),
$this->artisanBinary(),
'queue:work',
$connection,
'--once',
"--name={$options->name}",
"--queue={$queue}",
"--backoff={$options->backoff}",
"--memory={$options->memory}",
"--sleep={$options->sleep}",
"--tries={$options->maxTries}",
$options->force ? '--force' : null,
], function ($value) {
return ! is_null($value);
});
}
/**
@@ -222,7 +210,7 @@ class Listener
*/
public function memoryExceeded($memoryLimit)
{
return (memory_get_usage() / 1024 / 1024) >= $memoryLimit;
return (memory_get_usage(true) / 1024 / 1024) >= $memoryLimit;
}
/**
@@ -232,7 +220,7 @@ class Listener
*/
public function stop()
{
die;
exit;
}
/**

View File

@@ -14,18 +14,21 @@ class ListenerOptions extends WorkerOptions
/**
* Create a new listener options instance.
*
* @param string $environment
* @param int $delay
* @param string $name
* @param string|null $environment
* @param int|int[] $backoff
* @param int $memory
* @param int $timeout
* @param int $sleep
* @param int $maxTries
* @param bool $force
* @param int $rest
* @return void
*/
public function __construct($environment = null, $delay = 0, $memory = 128, $timeout = 60, $sleep = 3, $maxTries = 0, $force = false)
public function __construct($name = 'default', $environment = null, $backoff = 0, $memory = 128, $timeout = 60, $sleep = 3, $maxTries = 1, $force = false, $rest = 0)
{
$this->environment = $environment;
parent::__construct($delay, $memory, $timeout, $sleep, $maxTries, $force);
parent::__construct($name, $backoff, $memory, $timeout, $sleep, $maxTries, $force, false, 0, 0, $rest);
}
}

View File

@@ -20,11 +20,31 @@ return redis.call('llen', KEYS[1]) + redis.call('zcard', KEYS[2]) + redis.call('
LUA;
}
/**
* Get the Lua script for pushing jobs onto the queue.
*
* KEYS[1] - The queue to push the job onto, for example: queues:foo
* KEYS[2] - The notification list for the queue we are pushing jobs onto, for example: queues:foo:notify
* ARGV[1] - The job payload
*
* @return string
*/
public static function push()
{
return <<<'LUA'
-- Push the job onto the queue...
redis.call('rpush', KEYS[1], ARGV[1])
-- Push a notification onto the "notify" queue...
redis.call('rpush', KEYS[2], 1)
LUA;
}
/**
* Get the Lua script for popping the next job off of the queue.
*
* KEYS[1] - The queue to pop jobs from, for example: queues:foo
* KEYS[2] - The queue to place reserved jobs on, for example: queues:foo:reserved
* KEYS[3] - The notify queue
* ARGV[1] - The time at which the reserved job will expire
*
* @return string
@@ -42,6 +62,7 @@ if(job ~= false) then
reserved['attempts'] = reserved['attempts'] + 1
reserved = cjson.encode(reserved)
redis.call('zadd', KEYS[2], ARGV[1], reserved)
redis.call('lpop', KEYS[3])
end
return {job, reserved}
@@ -76,6 +97,7 @@ LUA;
*
* KEYS[1] - The queue we are removing jobs from, for example: queues:foo:reserved
* KEYS[2] - The queue we are moving jobs to, for example: queues:foo
* KEYS[3] - The notification list for the queue we are moving jobs to, for example queues:foo:notify
* ARGV[1] - The current UNIX timestamp
*
* @return string
@@ -84,7 +106,7 @@ LUA;
{
return <<<'LUA'
-- Get all of the jobs with an expired "score"...
local val = redis.call('zrangebyscore', KEYS[1], '-inf', ARGV[1])
local val = redis.call('zrangebyscore', KEYS[1], '-inf', ARGV[1], 'limit', 0, ARGV[2])
-- If we have values in the array, we will remove them from the first queue
-- and add them onto the destination queue in chunks of 100, which moves
@@ -94,10 +116,33 @@ if(next(val) ~= nil) then
for i = 1, #val, 100 do
redis.call('rpush', KEYS[2], unpack(val, i, math.min(i+99, #val)))
-- Push a notification for every job that was migrated...
for j = i, math.min(i+99, #val) do
redis.call('rpush', KEYS[3], 1)
end
end
end
return val
LUA;
}
/**
* Get the Lua script for removing all jobs from the queue.
*
* KEYS[1] - The name of the primary queue
* KEYS[2] - The name of the "delayed" queue
* KEYS[3] - The name of the "reserved" queue
* KEYS[4] - The name of the "notify" queue
*
* @return string
*/
public static function clear()
{
return <<<'LUA'
local size = redis.call('llen', KEYS[1]) + redis.call('zcard', KEYS[2]) + redis.call('zcard', KEYS[3])
redis.call('del', KEYS[1], KEYS[2], KEYS[3], KEYS[4])
return size
LUA;
}
}

View File

@@ -0,0 +1,146 @@
<?php
namespace Illuminate\Queue\Middleware;
use Illuminate\Cache\RateLimiter;
use Illuminate\Cache\RateLimiting\Unlimited;
use Illuminate\Container\Container;
use Illuminate\Support\Arr;
class RateLimited
{
/**
* The rate limiter instance.
*
* @var \Illuminate\Cache\RateLimiter
*/
protected $limiter;
/**
* The name of the rate limiter.
*
* @var string
*/
protected $limiterName;
/**
* Indicates if the job should be released if the limit is exceeded.
*
* @var bool
*/
public $shouldRelease = true;
/**
* Create a new middleware instance.
*
* @param string $limiterName
* @return void
*/
public function __construct($limiterName)
{
$this->limiter = Container::getInstance()->make(RateLimiter::class);
$this->limiterName = $limiterName;
}
/**
* Process the job.
*
* @param mixed $job
* @param callable $next
* @return mixed
*/
public function handle($job, $next)
{
if (is_null($limiter = $this->limiter->limiter($this->limiterName))) {
return $next($job);
}
$limiterResponse = $limiter($job);
if ($limiterResponse instanceof Unlimited) {
return $next($job);
}
return $this->handleJob(
$job,
$next,
collect(Arr::wrap($limiterResponse))->map(function ($limit) {
return (object) [
'key' => md5($this->limiterName.$limit->key),
'maxAttempts' => $limit->maxAttempts,
'decayMinutes' => $limit->decayMinutes,
];
})->all()
);
}
/**
* Handle a rate limited job.
*
* @param mixed $job
* @param callable $next
* @param array $limits
* @return mixed
*/
protected function handleJob($job, $next, array $limits)
{
foreach ($limits as $limit) {
if ($this->limiter->tooManyAttempts($limit->key, $limit->maxAttempts)) {
return $this->shouldRelease
? $job->release($this->getTimeUntilNextRetry($limit->key))
: false;
}
$this->limiter->hit($limit->key, $limit->decayMinutes * 60);
}
return $next($job);
}
/**
* Do not release the job back to the queue if the limit is exceeded.
*
* @return $this
*/
public function dontRelease()
{
$this->shouldRelease = false;
return $this;
}
/**
* Get the number of seconds that should elapse before the job is retried.
*
* @param string $key
* @return int
*/
protected function getTimeUntilNextRetry($key)
{
return $this->limiter->availableIn($key) + 3;
}
/**
* Prepare the object for serialization.
*
* @return array
*/
public function __sleep()
{
return [
'limiterName',
'shouldRelease',
];
}
/**
* Prepare the object after unserialization.
*
* @return void
*/
public function __wakeup()
{
$this->limiter = Container::getInstance()->make(RateLimiter::class);
}
}

View File

@@ -0,0 +1,103 @@
<?php
namespace Illuminate\Queue\Middleware;
use Illuminate\Container\Container;
use Illuminate\Contracts\Redis\Factory as Redis;
use Illuminate\Redis\Limiters\DurationLimiter;
use Illuminate\Support\InteractsWithTime;
class RateLimitedWithRedis extends RateLimited
{
use InteractsWithTime;
/**
* The Redis factory implementation.
*
* @var \Illuminate\Contracts\Redis\Factory
*/
protected $redis;
/**
* The timestamp of the end of the current duration by key.
*
* @var array
*/
public $decaysAt = [];
/**
* Create a new middleware instance.
*
* @param string $limiterName
* @return void
*/
public function __construct($limiterName)
{
parent::__construct($limiterName);
$this->redis = Container::getInstance()->make(Redis::class);
}
/**
* Handle a rate limited job.
*
* @param mixed $job
* @param callable $next
* @param array $limits
* @return mixed
*/
protected function handleJob($job, $next, array $limits)
{
foreach ($limits as $limit) {
if ($this->tooManyAttempts($limit->key, $limit->maxAttempts, $limit->decayMinutes)) {
return $this->shouldRelease
? $job->release($this->getTimeUntilNextRetry($limit->key))
: false;
}
}
return $next($job);
}
/**
* Determine if the given key has been "accessed" too many times.
*
* @param string $key
* @param int $maxAttempts
* @param int $decayMinutes
* @return bool
*/
protected function tooManyAttempts($key, $maxAttempts, $decayMinutes)
{
$limiter = new DurationLimiter(
$this->redis, $key, $maxAttempts, $decayMinutes * 60
);
return tap(! $limiter->acquire(), function () use ($key, $limiter) {
$this->decaysAt[$key] = $limiter->decaysAt;
});
}
/**
* Get the number of seconds that should elapse before the job is retried.
*
* @param string $key
* @return int
*/
protected function getTimeUntilNextRetry($key)
{
return ($this->decaysAt[$key] - $this->currentTime()) + 3;
}
/**
* Prepare the object after unserialization.
*
* @return void
*/
public function __wakeup()
{
parent::__wakeup();
$this->redis = Container::getInstance()->make(Redis::class);
}
}

View File

@@ -0,0 +1,201 @@
<?php
namespace Illuminate\Queue\Middleware;
use Illuminate\Cache\RateLimiter;
use Illuminate\Container\Container;
use Throwable;
class ThrottlesExceptions
{
/**
* The developer specified key that the rate limiter should use.
*
* @var string
*/
protected $key;
/**
* Indicates whether the throttle key should use the job's UUID.
*
* @var bool
*/
protected $byJob = false;
/**
* The maximum number of attempts allowed before rate limiting applies.
*
* @var int
*/
protected $maxAttempts;
/**
* The number of minutes until the maximum attempts are reset.
*
* @var int
*/
protected $decayMinutes;
/**
* The number of minutes to wait before retrying the job after an exception.
*
* @var int
*/
protected $retryAfterMinutes = 0;
/**
* The callback that determines if rate limiting should apply.
*
* @var callable
*/
protected $whenCallback;
/**
* The prefix of the rate limiter key.
*
* @var string
*/
protected $prefix = 'laravel_throttles_exceptions:';
/**
* The rate limiter instance.
*
* @var \Illuminate\Cache\RateLimiter
*/
protected $limiter;
/**
* Create a new middleware instance.
*
* @param int $maxAttempts
* @param int $decayMinutes
* @return void
*/
public function __construct($maxAttempts = 10, $decayMinutes = 10)
{
$this->maxAttempts = $maxAttempts;
$this->decayMinutes = $decayMinutes;
}
/**
* Process the job.
*
* @param mixed $job
* @param callable $next
* @return mixed
*/
public function handle($job, $next)
{
$this->limiter = Container::getInstance()->make(RateLimiter::class);
if ($this->limiter->tooManyAttempts($jobKey = $this->getKey($job), $this->maxAttempts)) {
return $job->release($this->getTimeUntilNextRetry($jobKey));
}
try {
$next($job);
$this->limiter->clear($jobKey);
} catch (Throwable $throwable) {
if ($this->whenCallback && ! call_user_func($this->whenCallback, $throwable)) {
throw $throwable;
}
$this->limiter->hit($jobKey, $this->decayMinutes * 60);
return $job->release($this->retryAfterMinutes * 60);
}
}
/**
* Specify a callback that should determine if rate limiting behavior should apply.
*
* @param callable $callback
* @return $this
*/
public function when(callable $callback)
{
$this->whenCallback = $callback;
return $this;
}
/**
* Set the prefix of the rate limiter key.
*
* @param string $prefix
* @return $this
*/
public function withPrefix(string $prefix)
{
$this->prefix = $prefix;
return $this;
}
/**
* Specify the number of minutes a job should be delayed when it is released (before it has reached its max exceptions).
*
* @param int $backoff
* @return $this
*/
public function backoff($backoff)
{
$this->retryAfterMinutes = $backoff;
return $this;
}
/**
* Get the cache key associated for the rate limiter.
*
* @param mixed $job
* @return string
*/
protected function getKey($job)
{
if ($this->key) {
return $this->prefix.$this->key;
} elseif ($this->byJob) {
return $this->prefix.$job->job->uuid();
}
return $this->prefix.md5(get_class($job));
}
/**
* Set the value that the rate limiter should be keyed by.
*
* @param string $key
* @return $this
*/
public function by($key)
{
$this->key = $key;
return $this;
}
/**
* Indicate that the throttle key should use the job's UUID.
*
* @return $this
*/
public function byJob()
{
$this->byJob = true;
return $this;
}
/**
* Get the number of seconds that should elapse before the job is retried.
*
* @param string $key
* @return int
*/
protected function getTimeUntilNextRetry($key)
{
return $this->limiter->availableIn($key) + 3;
}
}

View File

@@ -0,0 +1,62 @@
<?php
namespace Illuminate\Queue\Middleware;
use Illuminate\Container\Container;
use Illuminate\Contracts\Redis\Factory as Redis;
use Illuminate\Redis\Limiters\DurationLimiter;
use Illuminate\Support\InteractsWithTime;
use Throwable;
class ThrottlesExceptionsWithRedis extends ThrottlesExceptions
{
use InteractsWithTime;
/**
* The Redis factory implementation.
*
* @var \Illuminate\Contracts\Redis\Factory
*/
protected $redis;
/**
* The rate limiter instance.
*
* @var \Illuminate\Redis\Limiters\DurationLimiter
*/
protected $limiter;
/**
* Process the job.
*
* @param mixed $job
* @param callable $next
* @return mixed
*/
public function handle($job, $next)
{
$this->redis = Container::getInstance()->make(Redis::class);
$this->limiter = new DurationLimiter(
$this->redis, $this->getKey($job), $this->maxAttempts, $this->decayMinutes * 60
);
if ($this->limiter->tooManyAttempts()) {
return $job->release($this->limiter->decaysAt - $this->currentTime());
}
try {
$next($job);
$this->limiter->clear();
} catch (Throwable $throwable) {
if ($this->whenCallback && ! call_user_func($this->whenCallback, $throwable)) {
throw $throwable;
}
$this->limiter->acquire();
return $job->release($this->retryAfterMinutes * 60);
}
}
}

View File

@@ -0,0 +1,162 @@
<?php
namespace Illuminate\Queue\Middleware;
use Illuminate\Container\Container;
use Illuminate\Contracts\Cache\Repository as Cache;
use Illuminate\Support\InteractsWithTime;
class WithoutOverlapping
{
use InteractsWithTime;
/**
* The job's unique key used for preventing overlaps.
*
* @var string
*/
public $key;
/**
* The number of seconds before a job should be available again if no lock was acquired.
*
* @var \DateTimeInterface|int|null
*/
public $releaseAfter;
/**
* The number of seconds before the lock should expire.
*
* @var int
*/
public $expiresAfter;
/**
* The prefix of the lock key.
*
* @var string
*/
public $prefix = 'laravel-queue-overlap:';
/**
* Share the key across different jobs.
*
* @var bool
*/
public $shareKey = false;
/**
* Create a new middleware instance.
*
* @param string $key
* @param \DateTimeInterface|int|null $releaseAfter
* @param \DateTimeInterface|int $expiresAfter
* @return void
*/
public function __construct($key = '', $releaseAfter = 0, $expiresAfter = 0)
{
$this->key = $key;
$this->releaseAfter = $releaseAfter;
$this->expiresAfter = $this->secondsUntil($expiresAfter);
}
/**
* Process the job.
*
* @param mixed $job
* @param callable $next
* @return mixed
*/
public function handle($job, $next)
{
$lock = Container::getInstance()->make(Cache::class)->lock(
$this->getLockKey($job), $this->expiresAfter
);
if ($lock->get()) {
try {
$next($job);
} finally {
$lock->release();
}
} elseif (! is_null($this->releaseAfter)) {
$job->release($this->releaseAfter);
}
}
/**
* Set the delay (in seconds) to release the job back to the queue.
*
* @param \DateTimeInterface|int $releaseAfter
* @return $this
*/
public function releaseAfter($releaseAfter)
{
$this->releaseAfter = $releaseAfter;
return $this;
}
/**
* Do not release the job back to the queue if no lock can be acquired.
*
* @return $this
*/
public function dontRelease()
{
$this->releaseAfter = null;
return $this;
}
/**
* Set the maximum number of seconds that can elapse before the lock is released.
*
* @param \DateTimeInterface|\DateInterval|int $expiresAfter
* @return $this
*/
public function expireAfter($expiresAfter)
{
$this->expiresAfter = $this->secondsUntil($expiresAfter);
return $this;
}
/**
* Set the prefix of the lock key.
*
* @param string $prefix
* @return $this
*/
public function withPrefix(string $prefix)
{
$this->prefix = $prefix;
return $this;
}
/**
* Indicate that the lock key should be shared across job classes.
*
* @return $this
*/
public function shared()
{
$this->shareKey = true;
return $this;
}
/**
* Get the lock key for the given job.
*
* @param mixed $job
* @return string
*/
public function getLockKey($job)
{
return $this->shareKey
? $this->prefix.$this->key
: $this->prefix.get_class($job).':'.$this->key;
}
}

View File

@@ -9,7 +9,7 @@ class NullQueue extends Queue implements QueueContract
/**
* Get the size of the queue.
*
* @param string $queue
* @param string|null $queue
* @return int
*/
public function size($queue = null)
@@ -21,8 +21,8 @@ class NullQueue extends Queue implements QueueContract
* Push a new job onto the queue.
*
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function push($job, $data = '', $queue = null)
@@ -34,8 +34,8 @@ class NullQueue extends Queue implements QueueContract
* Push a raw payload onto the queue.
*
* @param string $payload
* @param string $queue
* @param array $options
* @param string|null $queue
* @param array $options
* @return mixed
*/
public function pushRaw($payload, $queue = null, array $options = [])
@@ -44,12 +44,12 @@ class NullQueue extends Queue implements QueueContract
}
/**
* Push a new job onto the queue after a delay.
* Push a new job onto the queue after (n) seconds.
*
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function later($delay, $job, $data = '', $queue = null)
@@ -60,7 +60,7 @@ class NullQueue extends Queue implements QueueContract
/**
* Pop the next job off of the queue.
*
* @param string $queue
* @param string|null $queue
* @return \Illuminate\Contracts\Queue\Job|null
*/
public function pop($queue = null)

View File

@@ -2,7 +2,15 @@
namespace Illuminate\Queue;
use Closure;
use DateTimeInterface;
use Illuminate\Container\Container;
use Illuminate\Contracts\Encryption\Encrypter;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Queue\Events\JobQueued;
use Illuminate\Support\Arr;
use Illuminate\Support\InteractsWithTime;
use Illuminate\Support\Str;
abstract class Queue
{
@@ -15,13 +23,6 @@ abstract class Queue
*/
protected $container;
/**
* The encrypter implementation.
*
* @var \Illuminate\Contracts\Encryption\Encrypter
*/
protected $encrypter;
/**
* The connection name for the queue.
*
@@ -29,12 +30,26 @@ abstract class Queue
*/
protected $connectionName;
/**
* Indicates that jobs should be dispatched after all database transactions have committed.
*
* @return $this
*/
protected $dispatchAfterCommit;
/**
* The create payload callbacks.
*
* @var callable[]
*/
protected static $createPayloadCallbacks = [];
/**
* Push a new job onto the queue.
*
* @param string $queue
* @param string $job
* @param mixed $data
* @param mixed $data
* @return mixed
*/
public function pushOn($queue, $job, $data = '')
@@ -43,12 +58,12 @@ abstract class Queue
}
/**
* Push a new job onto the queue after a delay.
* Push a new job onto a specific queue after (n) seconds.
*
* @param string $queue
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param string $job
* @param mixed $data
* @param mixed $data
* @return mixed
*/
public function laterOn($queue, $delay, $job, $data = '')
@@ -59,10 +74,10 @@ abstract class Queue
/**
* Push an array of jobs onto the queue.
*
* @param array $jobs
* @param mixed $data
* @param string $queue
* @return mixed
* @param array $jobs
* @param mixed $data
* @param string|null $queue
* @return void
*/
public function bulk($jobs, $data = '', $queue = null)
{
@@ -74,18 +89,22 @@ abstract class Queue
/**
* Create a payload string from the given job and data.
*
* @param string $job
* @param mixed $data
* @param \Closure|string|object $job
* @param string $queue
* @param mixed $data
* @return string
*
* @throws \Illuminate\Queue\InvalidPayloadException
*/
protected function createPayload($job, $data = '', $queue = null)
protected function createPayload($job, $queue, $data = '')
{
$payload = json_encode($this->createPayloadArray($job, $data, $queue));
if ($job instanceof Closure) {
$job = CallQueuedClosure::create($job);
}
if (JSON_ERROR_NONE !== json_last_error()) {
$payload = json_encode($this->createPayloadArray($job, $queue, $data), \JSON_UNESCAPED_UNICODE);
if (json_last_error() !== JSON_ERROR_NONE) {
throw new InvalidPayloadException(
'Unable to JSON encode payload. Error code: '.json_last_error()
);
@@ -97,42 +116,59 @@ abstract class Queue
/**
* Create a payload array from the given job and data.
*
* @param string $job
* @param mixed $data
* @param string|object $job
* @param string $queue
* @param mixed $data
* @return array
*/
protected function createPayloadArray($job, $data = '', $queue = null)
protected function createPayloadArray($job, $queue, $data = '')
{
return is_object($job)
? $this->createObjectPayload($job)
: $this->createStringPayload($job, $data);
? $this->createObjectPayload($job, $queue)
: $this->createStringPayload($job, $queue, $data);
}
/**
* Create a payload for an object-based queue handler.
*
* @param mixed $job
* @param object $job
* @param string $queue
* @return array
*/
protected function createObjectPayload($job)
protected function createObjectPayload($job, $queue)
{
return [
$payload = $this->withCreatePayloadHooks($queue, [
'uuid' => (string) Str::uuid(),
'displayName' => $this->getDisplayName($job),
'job' => 'Illuminate\Queue\CallQueuedHandler@call',
'maxTries' => isset($job->tries) ? $job->tries : null,
'timeout' => isset($job->timeout) ? $job->timeout : null,
'maxTries' => $job->tries ?? null,
'maxExceptions' => $job->maxExceptions ?? null,
'failOnTimeout' => $job->failOnTimeout ?? false,
'backoff' => $this->getJobBackoff($job),
'timeout' => $job->timeout ?? null,
'retryUntil' => $this->getJobExpiration($job),
'data' => [
'commandName' => get_class($job),
'command' => serialize(clone $job),
'commandName' => $job,
'command' => $job,
],
];
]);
$command = $this->jobShouldBeEncrypted($job) && $this->container->bound(Encrypter::class)
? $this->container[Encrypter::class]->encrypt(serialize(clone $job))
: serialize(clone $job);
return array_merge($payload, [
'data' => array_merge($payload['data'], [
'commandName' => get_class($job),
'command' => $command,
]),
]);
}
/**
* Get the display name for the given job.
*
* @param mixed $job
* @param object $job
* @return string
*/
protected function getDisplayName($job)
@@ -141,20 +177,177 @@ abstract class Queue
? $job->displayName() : get_class($job);
}
/**
* Get the backoff for an object-based queue handler.
*
* @param mixed $job
* @return mixed
*/
public function getJobBackoff($job)
{
if (! method_exists($job, 'backoff') && ! isset($job->backoff)) {
return;
}
if (is_null($backoff = $job->backoff ?? $job->backoff())) {
return;
}
return collect(Arr::wrap($backoff))
->map(function ($backoff) {
return $backoff instanceof DateTimeInterface
? $this->secondsUntil($backoff) : $backoff;
})->implode(',');
}
/**
* Get the expiration timestamp for an object-based queue handler.
*
* @param mixed $job
* @return mixed
*/
public function getJobExpiration($job)
{
if (! method_exists($job, 'retryUntil') && ! isset($job->retryUntil)) {
return;
}
$expiration = $job->retryUntil ?? $job->retryUntil();
return $expiration instanceof DateTimeInterface
? $expiration->getTimestamp() : $expiration;
}
/**
* Determine if the job should be encrypted.
*
* @param object $job
* @return bool
*/
protected function jobShouldBeEncrypted($job)
{
if ($job instanceof ShouldBeEncrypted) {
return true;
}
return isset($job->shouldBeEncrypted) && $job->shouldBeEncrypted;
}
/**
* Create a typical, string based queue payload array.
*
* @param string $job
* @param string $queue
* @param mixed $data
* @return array
*/
protected function createStringPayload($job, $data)
protected function createStringPayload($job, $queue, $data)
{
return [
return $this->withCreatePayloadHooks($queue, [
'uuid' => (string) Str::uuid(),
'displayName' => is_string($job) ? explode('@', $job)[0] : null,
'job' => $job, 'maxTries' => null,
'timeout' => null, 'data' => $data,
];
'job' => $job,
'maxTries' => null,
'maxExceptions' => null,
'failOnTimeout' => false,
'backoff' => null,
'timeout' => null,
'data' => $data,
]);
}
/**
* Register a callback to be executed when creating job payloads.
*
* @param callable|null $callback
* @return void
*/
public static function createPayloadUsing($callback)
{
if (is_null($callback)) {
static::$createPayloadCallbacks = [];
} else {
static::$createPayloadCallbacks[] = $callback;
}
}
/**
* Create the given payload using any registered payload hooks.
*
* @param string $queue
* @param array $payload
* @return array
*/
protected function withCreatePayloadHooks($queue, array $payload)
{
if (! empty(static::$createPayloadCallbacks)) {
foreach (static::$createPayloadCallbacks as $callback) {
$payload = array_merge($payload, $callback($this->getConnectionName(), $queue, $payload));
}
}
return $payload;
}
/**
* Enqueue a job using the given callback.
*
* @param \Closure|string|object $job
* @param string $payload
* @param string $queue
* @param \DateTimeInterface|\DateInterval|int|null $delay
* @param callable $callback
* @return mixed
*/
protected function enqueueUsing($job, $payload, $queue, $delay, $callback)
{
if ($this->shouldDispatchAfterCommit($job) &&
$this->container->bound('db.transactions')) {
return $this->container->make('db.transactions')->addCallback(
function () use ($payload, $queue, $delay, $callback, $job) {
return tap($callback($payload, $queue, $delay), function ($jobId) use ($job) {
$this->raiseJobQueuedEvent($jobId, $job);
});
}
);
}
return tap($callback($payload, $queue, $delay), function ($jobId) use ($job) {
$this->raiseJobQueuedEvent($jobId, $job);
});
}
/**
* Determine if the job should be dispatched after all database transactions have committed.
*
* @param \Closure|string|object $job
* @return bool
*/
protected function shouldDispatchAfterCommit($job)
{
if (is_object($job) && isset($job->afterCommit)) {
return $job->afterCommit;
}
if (isset($this->dispatchAfterCommit)) {
return $this->dispatchAfterCommit;
}
return false;
}
/**
* Raise the job queued event.
*
* @param string|int|null $jobId
* @param \Closure|string|object $job
* @return void
*/
protected function raiseJobQueuedEvent($jobId, $job)
{
if ($this->container->bound('events')) {
$this->container['events']->dispatch(new JobQueued($this->connectionName, $jobId, $job));
}
}
/**
@@ -180,6 +373,16 @@ abstract class Queue
return $this;
}
/**
* Get the container instance being used by the connection.
*
* @return \Illuminate\Container\Container
*/
public function getContainer()
{
return $this->container;
}
/**
* Set the IoC container instance.
*

View File

@@ -3,16 +3,19 @@
namespace Illuminate\Queue;
use Closure;
use InvalidArgumentException;
use Illuminate\Contracts\Queue\Factory as FactoryContract;
use Illuminate\Contracts\Queue\Monitor as MonitorContract;
use InvalidArgumentException;
/**
* @mixin \Illuminate\Contracts\Queue\Queue
*/
class QueueManager implements FactoryContract, MonitorContract
{
/**
* The application instance.
*
* @var \Illuminate\Foundation\Application
* @var \Illuminate\Contracts\Foundation\Application
*/
protected $app;
@@ -33,7 +36,7 @@ class QueueManager implements FactoryContract, MonitorContract
/**
* Create a new queue manager instance.
*
* @param \Illuminate\Foundation\Application $app
* @param \Illuminate\Contracts\Foundation\Application $app
* @return void
*/
public function __construct($app)
@@ -110,7 +113,7 @@ class QueueManager implements FactoryContract, MonitorContract
/**
* Determine if the driver is connected.
*
* @param string $name
* @param string|null $name
* @return bool
*/
public function connected($name = null)
@@ -121,7 +124,7 @@ class QueueManager implements FactoryContract, MonitorContract
/**
* Resolve a queue connection instance.
*
* @param string $name
* @param string|null $name
* @return \Illuminate\Contracts\Queue\Queue
*/
public function connection($name = null)
@@ -145,11 +148,17 @@ class QueueManager implements FactoryContract, MonitorContract
*
* @param string $name
* @return \Illuminate\Contracts\Queue\Queue
*
* @throws \InvalidArgumentException
*/
protected function resolve($name)
{
$config = $this->getConfig($name);
if (is_null($config)) {
throw new InvalidArgumentException("The [{$name}] queue connection has not been configured.");
}
return $this->getConnector($config['driver'])
->connect($config)
->setConnectionName($name);
@@ -166,7 +175,7 @@ class QueueManager implements FactoryContract, MonitorContract
protected function getConnector($driver)
{
if (! isset($this->connectors[$driver])) {
throw new InvalidArgumentException("No connector for [$driver]");
throw new InvalidArgumentException("No connector for [$driver].");
}
return call_user_func($this->connectors[$driver]);
@@ -175,7 +184,7 @@ class QueueManager implements FactoryContract, MonitorContract
/**
* Add a queue connection resolver.
*
* @param string $driver
* @param string $driver
* @param \Closure $resolver
* @return void
*/
@@ -187,7 +196,7 @@ class QueueManager implements FactoryContract, MonitorContract
/**
* Add a queue connection resolver.
*
* @param string $driver
* @param string $driver
* @param \Closure $resolver
* @return void
*/
@@ -200,7 +209,7 @@ class QueueManager implements FactoryContract, MonitorContract
* Get the queue connection configuration.
*
* @param string $name
* @return array
* @return array|null
*/
protected function getConfig($name)
{
@@ -235,7 +244,7 @@ class QueueManager implements FactoryContract, MonitorContract
/**
* Get the full name for the given connection.
*
* @param string $connection
* @param string|null $connection
* @return string
*/
public function getName($connection = null)
@@ -244,20 +253,37 @@ class QueueManager implements FactoryContract, MonitorContract
}
/**
* Determine if the application is in maintenance mode.
* Get the application instance used by the manager.
*
* @return bool
* @return \Illuminate\Contracts\Foundation\Application
*/
public function isDownForMaintenance()
public function getApplication()
{
return $this->app->isDownForMaintenance();
return $this->app;
}
/**
* Set the application instance used by the manager.
*
* @param \Illuminate\Contracts\Foundation\Application $app
* @return $this
*/
public function setApplication($app)
{
$this->app = $app;
foreach ($this->connections as $connection) {
$connection->setContainer($app);
}
return $this;
}
/**
* Dynamically pass calls to the default connection.
*
* @param string $method
* @param array $parameters
* @param array $parameters
* @return mixed
*/
public function __call($method, $parameters)

View File

@@ -2,25 +2,27 @@
namespace Illuminate\Queue;
use Illuminate\Support\ServiceProvider;
use Illuminate\Queue\Connectors\SqsConnector;
use Illuminate\Queue\Connectors\NullConnector;
use Illuminate\Queue\Connectors\SyncConnector;
use Illuminate\Queue\Connectors\RedisConnector;
use Aws\DynamoDb\DynamoDbClient;
use Illuminate\Contracts\Debug\ExceptionHandler;
use Illuminate\Queue\Connectors\DatabaseConnector;
use Illuminate\Queue\Failed\NullFailedJobProvider;
use Illuminate\Contracts\Support\DeferrableProvider;
use Illuminate\Queue\Connectors\BeanstalkdConnector;
use Illuminate\Queue\Connectors\DatabaseConnector;
use Illuminate\Queue\Connectors\NullConnector;
use Illuminate\Queue\Connectors\RedisConnector;
use Illuminate\Queue\Connectors\SqsConnector;
use Illuminate\Queue\Connectors\SyncConnector;
use Illuminate\Queue\Failed\DatabaseFailedJobProvider;
use Illuminate\Queue\Failed\DatabaseUuidFailedJobProvider;
use Illuminate\Queue\Failed\DynamoDbFailedJobProvider;
use Illuminate\Queue\Failed\NullFailedJobProvider;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\Facade;
use Illuminate\Support\ServiceProvider;
use Laravel\SerializableClosure\SerializableClosure;
class QueueServiceProvider extends ServiceProvider
class QueueServiceProvider extends ServiceProvider implements DeferrableProvider
{
/**
* Indicates if loading of the provider is deferred.
*
* @var bool
*/
protected $defer = true;
use SerializesAndRestoresModelIdentifiers;
/**
* Register the service provider.
@@ -29,17 +31,39 @@ class QueueServiceProvider extends ServiceProvider
*/
public function register()
{
$this->configureSerializableClosureUses();
$this->registerManager();
$this->registerConnection();
$this->registerWorker();
$this->registerListener();
$this->registerFailedJobServices();
}
/**
* Configure serializable closures uses.
*
* @return void
*/
protected function configureSerializableClosureUses()
{
SerializableClosure::transformUseVariablesUsing(function ($data) {
foreach ($data as $key => $value) {
$data[$key] = $this->getSerializedPropertyValue($value);
}
return $data;
});
SerializableClosure::resolveUseVariablesUsing(function ($data) {
foreach ($data as $key => $value) {
$data[$key] = $this->getRestoredPropertyValue($value);
}
return $data;
});
}
/**
* Register the queue manager.
*
@@ -167,9 +191,34 @@ class QueueServiceProvider extends ServiceProvider
*/
protected function registerWorker()
{
$this->app->singleton('queue.worker', function () {
$this->app->singleton('queue.worker', function ($app) {
$isDownForMaintenance = function () {
return $this->app->isDownForMaintenance();
};
$resetScope = function () use ($app) {
if (method_exists($app['log']->driver(), 'withoutContext')) {
$app['log']->withoutContext();
}
if (method_exists($app['db'], 'getConnections')) {
foreach ($app['db']->getConnections() as $connection) {
$connection->resetTotalQueryDuration();
$connection->allowQueryDurationHandlersToRunAgain();
}
}
$app->forgetScopedInstances();
return Facade::clearResolvedInstances();
};
return new Worker(
$this->app['queue'], $this->app['events'], $this->app[ExceptionHandler::class]
$app['queue'],
$app['events'],
$app[ExceptionHandler::class],
$isDownForMaintenance,
$resetScope
);
});
}
@@ -181,8 +230,8 @@ class QueueServiceProvider extends ServiceProvider
*/
protected function registerListener()
{
$this->app->singleton('queue.listener', function () {
return new Listener($this->app->basePath());
$this->app->singleton('queue.listener', function ($app) {
return new Listener($app->basePath());
});
}
@@ -193,12 +242,23 @@ class QueueServiceProvider extends ServiceProvider
*/
protected function registerFailedJobServices()
{
$this->app->singleton('queue.failer', function () {
$config = $this->app['config']['queue.failed'];
$this->app->singleton('queue.failer', function ($app) {
$config = $app['config']['queue.failed'];
return isset($config['table'])
? $this->databaseFailedJobProvider($config)
: new NullFailedJobProvider;
if (array_key_exists('driver', $config) &&
(is_null($config['driver']) || $config['driver'] === 'null')) {
return new NullFailedJobProvider;
}
if (isset($config['driver']) && $config['driver'] === 'dynamodb') {
return $this->dynamoFailedJobProvider($config);
} elseif (isset($config['driver']) && $config['driver'] === 'database-uuids') {
return $this->databaseUuidFailedJobProvider($config);
} elseif (isset($config['table'])) {
return $this->databaseFailedJobProvider($config);
} else {
return new NullFailedJobProvider;
}
});
}
@@ -215,6 +275,46 @@ class QueueServiceProvider extends ServiceProvider
);
}
/**
* Create a new database failed job provider that uses UUIDs as IDs.
*
* @param array $config
* @return \Illuminate\Queue\Failed\DatabaseUuidFailedJobProvider
*/
protected function databaseUuidFailedJobProvider($config)
{
return new DatabaseUuidFailedJobProvider(
$this->app['db'], $config['database'], $config['table']
);
}
/**
* Create a new DynamoDb failed job provider.
*
* @param array $config
* @return \Illuminate\Queue\Failed\DynamoDbFailedJobProvider
*/
protected function dynamoFailedJobProvider($config)
{
$dynamoConfig = [
'region' => $config['region'],
'version' => 'latest',
'endpoint' => $config['endpoint'] ?? null,
];
if (! empty($config['key']) && ! empty($config['secret'])) {
$dynamoConfig['credentials'] = Arr::only(
$config, ['key', 'secret', 'token']
);
}
return new DynamoDbFailedJobProvider(
new DynamoDbClient($dynamoConfig),
$this->app['config']['app.name'],
$config['table']
);
}
/**
* Get the services provided by the provider.
*
@@ -223,8 +323,11 @@ class QueueServiceProvider extends ServiceProvider
public function provides()
{
return [
'queue', 'queue.worker', 'queue.listener',
'queue.failer', 'queue.connection',
'queue',
'queue.connection',
'queue.failer',
'queue.listener',
'queue.worker',
];
}
}

View File

@@ -25,10 +25,10 @@ Once the Capsule instance has been registered. You may use it like so:
```PHP
// As an instance...
$queue->push('SendEmail', array('message' => $message));
$queue->push('SendEmail', ['message' => $message]);
// If setAsGlobal has been called...
Queue::push('SendEmail', array('message' => $message));
Queue::push('SendEmail', ['message' => $message]);
```
For further documentation on using the queue, consult the [Laravel framework documentation](https://laravel.com/docs).

View File

@@ -2,13 +2,13 @@
namespace Illuminate\Queue;
use Illuminate\Support\Arr;
use Illuminate\Support\Str;
use Illuminate\Queue\Jobs\RedisJob;
use Illuminate\Contracts\Redis\Factory as Redis;
use Illuminate\Contracts\Queue\ClearableQueue;
use Illuminate\Contracts\Queue\Queue as QueueContract;
use Illuminate\Contracts\Redis\Factory as Redis;
use Illuminate\Queue\Jobs\RedisJob;
use Illuminate\Support\Str;
class RedisQueue extends Queue implements QueueContract
class RedisQueue extends Queue implements QueueContract, ClearableQueue
{
/**
* The Redis factory implementation.
@@ -38,27 +38,55 @@ class RedisQueue extends Queue implements QueueContract
*/
protected $retryAfter = 60;
/**
* The maximum number of seconds to block for a job.
*
* @var int|null
*/
protected $blockFor = null;
/**
* The batch size to use when migrating delayed / expired jobs onto the primary queue.
*
* Negative values are infinite.
*
* @var int
*/
protected $migrationBatchSize = -1;
/**
* Create a new Redis queue instance.
*
* @param \Illuminate\Contracts\Redis\Factory $redis
* @param string $default
* @param string $connection
* @param string|null $connection
* @param int $retryAfter
* @param int|null $blockFor
* @param bool $dispatchAfterCommit
* @param int $migrationBatchSize
* @return void
*/
public function __construct(Redis $redis, $default = 'default', $connection = null, $retryAfter = 60)
public function __construct(Redis $redis,
$default = 'default',
$connection = null,
$retryAfter = 60,
$blockFor = null,
$dispatchAfterCommit = false,
$migrationBatchSize = -1)
{
$this->redis = $redis;
$this->default = $default;
$this->blockFor = $blockFor;
$this->connection = $connection;
$this->retryAfter = $retryAfter;
$this->dispatchAfterCommit = $dispatchAfterCommit;
$this->migrationBatchSize = $migrationBatchSize;
}
/**
* Get the size of the queue.
*
* @param string $queue
* @param string|null $queue
* @return int
*/
public function size($queue = null)
@@ -70,54 +98,96 @@ class RedisQueue extends Queue implements QueueContract
);
}
/**
* Push an array of jobs onto the queue.
*
* @param array $jobs
* @param mixed $data
* @param string|null $queue
* @return void
*/
public function bulk($jobs, $data = '', $queue = null)
{
$this->getConnection()->pipeline(function () use ($jobs, $data, $queue) {
$this->getConnection()->transaction(function () use ($jobs, $data, $queue) {
foreach ((array) $jobs as $job) {
if (isset($job->delay)) {
$this->later($job->delay, $job, $data, $queue);
} else {
$this->push($job, $data, $queue);
}
}
});
});
}
/**
* Push a new job onto the queue.
*
* @param object|string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function push($job, $data = '', $queue = null)
{
return $this->pushRaw($this->createPayload($job, $data), $queue);
return $this->enqueueUsing(
$job,
$this->createPayload($job, $this->getQueue($queue), $data),
$queue,
null,
function ($payload, $queue) {
return $this->pushRaw($payload, $queue);
}
);
}
/**
* Push a raw payload onto the queue.
*
* @param string $payload
* @param string $queue
* @param array $options
* @param string|null $queue
* @param array $options
* @return mixed
*/
public function pushRaw($payload, $queue = null, array $options = [])
{
$this->getConnection()->rpush($this->getQueue($queue), $payload);
$this->getConnection()->eval(
LuaScripts::push(), 2, $this->getQueue($queue),
$this->getQueue($queue).':notify', $payload
);
return Arr::get(json_decode($payload, true), 'id');
return json_decode($payload, true)['id'] ?? null;
}
/**
* Push a new job onto the queue after a delay.
*
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param object|string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function later($delay, $job, $data = '', $queue = null)
{
return $this->laterRaw($delay, $this->createPayload($job, $data), $queue);
return $this->enqueueUsing(
$job,
$this->createPayload($job, $this->getQueue($queue), $data),
$queue,
$delay,
function ($payload, $queue, $delay) {
return $this->laterRaw($delay, $payload, $queue);
}
);
}
/**
* Push a raw job onto the queue after a delay.
* Push a raw job onto the queue after (n) seconds.
*
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param string $payload
* @param string $queue
* @param string|null $queue
* @return mixed
*/
protected function laterRaw($delay, $payload, $queue = null)
@@ -126,20 +196,20 @@ class RedisQueue extends Queue implements QueueContract
$this->getQueue($queue).':delayed', $this->availableAt($delay), $payload
);
return Arr::get(json_decode($payload, true), 'id');
return json_decode($payload, true)['id'] ?? null;
}
/**
* Create a payload string from the given job and data.
*
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @return array
*/
protected function createPayloadArray($job, $data = '', $queue = null)
protected function createPayloadArray($job, $queue, $data = '')
{
return array_merge(parent::createPayloadArray($job, $data, $queue), [
return array_merge(parent::createPayloadArray($job, $queue, $data), [
'id' => $this->getRandomId(),
'attempts' => 0,
]);
@@ -148,14 +218,14 @@ class RedisQueue extends Queue implements QueueContract
/**
* Pop the next job off of the queue.
*
* @param string $queue
* @param string|null $queue
* @return \Illuminate\Contracts\Queue\Job|null
*/
public function pop($queue = null)
{
$this->migrate($prefixed = $this->getQueue($queue));
list($job, $reserved) = $this->retrieveNextJob($prefixed);
[$job, $reserved] = $this->retrieveNextJob($prefixed);
if ($reserved) {
return new RedisJob(
@@ -185,12 +255,13 @@ class RedisQueue extends Queue implements QueueContract
*
* @param string $from
* @param string $to
* @param int $limit
* @return array
*/
public function migrateExpiredJobs($from, $to)
{
return $this->getConnection()->eval(
LuaScripts::migrateExpiredJobs(), 2, $from, $to, $this->currentTime()
LuaScripts::migrateExpiredJobs(), 3, $from, $to, $to.':notify', $this->currentTime(), $this->migrationBatchSize
);
}
@@ -198,14 +269,28 @@ class RedisQueue extends Queue implements QueueContract
* Retrieve the next job from the queue.
*
* @param string $queue
* @param bool $block
* @return array
*/
protected function retrieveNextJob($queue)
protected function retrieveNextJob($queue, $block = true)
{
return $this->getConnection()->eval(
LuaScripts::pop(), 2, $queue, $queue.':reserved',
$nextJob = $this->getConnection()->eval(
LuaScripts::pop(), 3, $queue, $queue.':reserved', $queue.':notify',
$this->availableAt($this->retryAfter)
);
if (empty($nextJob)) {
return [null, null];
}
[$job, $reserved] = $nextJob;
if (! $job && ! is_null($this->blockFor) && $block &&
$this->getConnection()->blpop([$queue.':notify'], $this->blockFor)) {
return $this->retrieveNextJob($queue, false);
}
return [$job, $reserved];
}
/**
@@ -238,6 +323,22 @@ class RedisQueue extends Queue implements QueueContract
);
}
/**
* Delete all of the jobs from the queue.
*
* @param string $queue
* @return int
*/
public function clear($queue)
{
$queue = $this->getQueue($queue);
return $this->getConnection()->eval(
LuaScripts::clear(), 4, $queue, $queue.':delayed',
$queue.':reserved', $queue.':notify'
);
}
/**
* Get a random ID string.
*
@@ -254,7 +355,7 @@ class RedisQueue extends Queue implements QueueContract
* @param string|null $queue
* @return string
*/
protected function getQueue($queue)
public function getQueue($queue)
{
return 'queues:'.($queue ?: $this->default);
}
@@ -262,9 +363,9 @@ class RedisQueue extends Queue implements QueueContract
/**
* Get the connection for the queue.
*
* @return \Predis\ClientInterface
* @return \Illuminate\Redis\Connections\Connection
*/
protected function getConnection()
public function getConnection()
{
return $this->redis->connection($this->connection);
}

View File

@@ -2,10 +2,12 @@
namespace Illuminate\Queue;
use Illuminate\Contracts\Queue\QueueableEntity;
use Illuminate\Contracts\Database\ModelIdentifier;
use Illuminate\Contracts\Queue\QueueableCollection;
use Illuminate\Contracts\Queue\QueueableEntity;
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
use Illuminate\Database\Eloquent\Relations\Concerns\AsPivot;
use Illuminate\Database\Eloquent\Relations\Pivot;
trait SerializesAndRestoresModelIdentifiers
{
@@ -18,11 +20,25 @@ trait SerializesAndRestoresModelIdentifiers
protected function getSerializedPropertyValue($value)
{
if ($value instanceof QueueableCollection) {
return new ModelIdentifier($value->getQueueableClass(), $value->getQueueableIds());
return (new ModelIdentifier(
$value->getQueueableClass(),
$value->getQueueableIds(),
$value->getQueueableRelations(),
$value->getQueueableConnection()
))->useCollectionClass(
($collectionClass = get_class($value)) !== EloquentCollection::class
? $collectionClass
: null
);
}
if ($value instanceof QueueableEntity) {
return new ModelIdentifier(get_class($value), $value->getQueueableId());
return new ModelIdentifier(
get_class($value),
$value->getQueueableId(),
$value->getQueueableRelations(),
$value->getQueueableConnection()
);
}
return $value;
@@ -42,8 +58,7 @@ trait SerializesAndRestoresModelIdentifiers
return is_array($value->id)
? $this->restoreCollection($value)
: $this->getQueryForModelRestoration(new $value->class)
->useWritePdo()->findOrFail($value->id);
: $this->restoreModel($value);
}
/**
@@ -55,23 +70,53 @@ trait SerializesAndRestoresModelIdentifiers
protected function restoreCollection($value)
{
if (! $value->class || count($value->id) === 0) {
return new EloquentCollection;
return ! is_null($value->collectionClass ?? null)
? new $value->collectionClass
: new EloquentCollection;
}
$model = new $value->class;
$collection = $this->getQueryForModelRestoration(
(new $value->class)->setConnection($value->connection), $value->id
)->useWritePdo()->get();
return $this->getQueryForModelRestoration($model)->useWritePdo()
->whereIn($model->getQualifiedKeyName(), $value->id)->get();
if (is_a($value->class, Pivot::class, true) ||
in_array(AsPivot::class, class_uses($value->class))) {
return $collection;
}
$collection = $collection->keyBy->getKey();
$collectionClass = get_class($collection);
return new $collectionClass(
collect($value->id)->map(function ($id) use ($collection) {
return $collection[$id] ?? null;
})->filter()
);
}
/**
* Get the query for restoration.
* Restore the model from the model identifier instance.
*
* @param \Illuminate\Contracts\Database\ModelIdentifier $value
* @return \Illuminate\Database\Eloquent\Model
*/
public function restoreModel($value)
{
return $this->getQueryForModelRestoration(
(new $value->class)->setConnection($value->connection), $value->id
)->useWritePdo()->firstOrFail()->load($value->relations ?? []);
}
/**
* Get the query for model restoration.
*
* @param \Illuminate\Database\Eloquent\Model $model
* @param array|int $ids
* @return \Illuminate\Database\Eloquent\Builder
*/
protected function getQueryForModelRestoration($model)
protected function getQueryForModelRestoration($model, $ids)
{
return $model->newQueryWithoutScopes();
return $model->newQueryForRestoration($ids);
}
}

View File

@@ -10,36 +10,83 @@ trait SerializesModels
use SerializesAndRestoresModelIdentifiers;
/**
* Prepare the instance for serialization.
* Prepare the instance values for serialization.
*
* @return array
*/
public function __sleep()
public function __serialize()
{
$values = [];
$properties = (new ReflectionClass($this))->getProperties();
$class = get_class($this);
foreach ($properties as $property) {
$property->setValue($this, $this->getSerializedPropertyValue(
$this->getPropertyValue($property)
));
if ($property->isStatic()) {
continue;
}
$property->setAccessible(true);
if (! $property->isInitialized($this)) {
continue;
}
$value = $this->getPropertyValue($property);
if ($property->hasDefaultValue() && $value === $property->getDefaultValue()) {
continue;
}
$name = $property->getName();
if ($property->isPrivate()) {
$name = "\0{$class}\0{$name}";
} elseif ($property->isProtected()) {
$name = "\0*\0{$name}";
}
$values[$name] = $this->getSerializedPropertyValue($value);
}
return array_map(function ($p) {
return $p->getName();
}, $properties);
return $values;
}
/**
* Restore the model after serialization.
*
* @param array $values
* @return void
*/
public function __wakeup()
public function __unserialize(array $values)
{
foreach ((new ReflectionClass($this))->getProperties() as $property) {
$property->setValue($this, $this->getRestoredPropertyValue(
$this->getPropertyValue($property)
));
$properties = (new ReflectionClass($this))->getProperties();
$class = get_class($this);
foreach ($properties as $property) {
if ($property->isStatic()) {
continue;
}
$name = $property->getName();
if ($property->isPrivate()) {
$name = "\0{$class}\0{$name}";
} elseif ($property->isProtected()) {
$name = "\0*\0{$name}";
}
if (! array_key_exists($name, $values)) {
continue;
}
$property->setAccessible(true);
$property->setValue(
$this, $this->getRestoredPropertyValue($values[$name])
);
}
}

View File

@@ -3,10 +3,12 @@
namespace Illuminate\Queue;
use Aws\Sqs\SqsClient;
use Illuminate\Queue\Jobs\SqsJob;
use Illuminate\Contracts\Queue\ClearableQueue;
use Illuminate\Contracts\Queue\Queue as QueueContract;
use Illuminate\Queue\Jobs\SqsJob;
use Illuminate\Support\Str;
class SqsQueue extends Queue implements QueueContract
class SqsQueue extends Queue implements QueueContract, ClearableQueue
{
/**
* The Amazon SQS instance.
@@ -29,25 +31,40 @@ class SqsQueue extends Queue implements QueueContract
*/
protected $prefix;
/**
* The queue name suffix.
*
* @var string
*/
private $suffix;
/**
* Create a new Amazon SQS queue instance.
*
* @param \Aws\Sqs\SqsClient $sqs
* @param string $default
* @param string $prefix
* @param string $suffix
* @param bool $dispatchAfterCommit
* @return void
*/
public function __construct(SqsClient $sqs, $default, $prefix = '')
public function __construct(SqsClient $sqs,
$default,
$prefix = '',
$suffix = '',
$dispatchAfterCommit = false)
{
$this->sqs = $sqs;
$this->prefix = $prefix;
$this->default = $default;
$this->suffix = $suffix;
$this->dispatchAfterCommit = $dispatchAfterCommit;
}
/**
* Get the size of the queue.
*
* @param string $queue
* @param string|null $queue
* @return int
*/
public function size($queue = null)
@@ -66,21 +83,29 @@ class SqsQueue extends Queue implements QueueContract
* Push a new job onto the queue.
*
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function push($job, $data = '', $queue = null)
{
return $this->pushRaw($this->createPayload($job, $data), $queue);
return $this->enqueueUsing(
$job,
$this->createPayload($job, $queue ?: $this->default, $data),
$queue,
null,
function ($payload, $queue) {
return $this->pushRaw($payload, $queue);
}
);
}
/**
* Push a raw payload onto the queue.
*
* @param string $payload
* @param string $queue
* @param array $options
* @param string|null $queue
* @param array $options
* @return mixed
*/
public function pushRaw($payload, $queue = null, array $options = [])
@@ -91,27 +116,54 @@ class SqsQueue extends Queue implements QueueContract
}
/**
* Push a new job onto the queue after a delay.
* Push a new job onto the queue after (n) seconds.
*
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function later($delay, $job, $data = '', $queue = null)
{
return $this->sqs->sendMessage([
'QueueUrl' => $this->getQueue($queue),
'MessageBody' => $this->createPayload($job, $data),
'DelaySeconds' => $this->secondsUntil($delay),
])->get('MessageId');
return $this->enqueueUsing(
$job,
$this->createPayload($job, $queue ?: $this->default, $data),
$queue,
$delay,
function ($payload, $queue, $delay) {
return $this->sqs->sendMessage([
'QueueUrl' => $this->getQueue($queue),
'MessageBody' => $payload,
'DelaySeconds' => $this->secondsUntil($delay),
])->get('MessageId');
}
);
}
/**
* Push an array of jobs onto the queue.
*
* @param array $jobs
* @param mixed $data
* @param string|null $queue
* @return void
*/
public function bulk($jobs, $data = '', $queue = null)
{
foreach ((array) $jobs as $job) {
if (isset($job->delay)) {
$this->later($job->delay, $job, $data, $queue);
} else {
$this->push($job, $data, $queue);
}
}
}
/**
* Pop the next job off of the queue.
*
* @param string $queue
* @param string|null $queue
* @return \Illuminate\Contracts\Queue\Job|null
*/
public function pop($queue = null)
@@ -121,7 +173,7 @@ class SqsQueue extends Queue implements QueueContract
'AttributeNames' => ['ApproximateReceiveCount'],
]);
if (count($response['Messages']) > 0) {
if (! is_null($response['Messages']) && count($response['Messages']) > 0) {
return new SqsJob(
$this->container, $this->sqs, $response['Messages'][0],
$this->connectionName, $queue
@@ -129,6 +181,21 @@ class SqsQueue extends Queue implements QueueContract
}
}
/**
* Delete all of the jobs from the queue.
*
* @param string $queue
* @return int
*/
public function clear($queue)
{
return tap($this->size($queue), function () use ($queue) {
$this->sqs->purgeQueue([
'QueueUrl' => $this->getQueue($queue),
]);
});
}
/**
* Get the queue or return the default.
*
@@ -140,7 +207,26 @@ class SqsQueue extends Queue implements QueueContract
$queue = $queue ?: $this->default;
return filter_var($queue, FILTER_VALIDATE_URL) === false
? rtrim($this->prefix, '/').'/'.$queue : $queue;
? $this->suffixQueue($queue, $this->suffix)
: $queue;
}
/**
* Add the given suffix to the given queue name.
*
* @param string $queue
* @param string $suffix
* @return string
*/
protected function suffixQueue($queue, $suffix = '')
{
if (str_ends_with($queue, '.fifo')) {
$queue = Str::beforeLast($queue, '.fifo');
return rtrim($this->prefix, '/').'/'.Str::finish($queue, $suffix).'.fifo';
}
return rtrim($this->prefix, '/').'/'.Str::finish($queue, $this->suffix);
}
/**

View File

@@ -2,19 +2,20 @@
namespace Illuminate\Queue;
use Exception;
use Throwable;
use Illuminate\Queue\Jobs\SyncJob;
use Illuminate\Contracts\Queue\Job;
use Illuminate\Contracts\Queue\Queue as QueueContract;
use Symfony\Component\Debug\Exception\FatalThrowableError;
use Illuminate\Queue\Events\JobExceptionOccurred;
use Illuminate\Queue\Events\JobProcessed;
use Illuminate\Queue\Events\JobProcessing;
use Illuminate\Queue\Jobs\SyncJob;
use Throwable;
class SyncQueue extends Queue implements QueueContract
{
/**
* Get the size of the queue.
*
* @param string $queue
* @param string|null $queue
* @return int
*/
public function size($queue = null)
@@ -26,15 +27,15 @@ class SyncQueue extends Queue implements QueueContract
* Push a new job onto the queue.
*
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*
* @throws \Exception|\Throwable
* @throws \Throwable
*/
public function push($job, $data = '', $queue = null)
{
$queueJob = $this->resolveJob($this->createPayload($job, $data, $queue), $queue);
$queueJob = $this->resolveJob($this->createPayload($job, $queue, $data), $queue);
try {
$this->raiseBeforeJobEvent($queueJob);
@@ -42,10 +43,8 @@ class SyncQueue extends Queue implements QueueContract
$queueJob->fire();
$this->raiseAfterJobEvent($queueJob);
} catch (Exception $e) {
$this->handleException($queueJob, $e);
} catch (Throwable $e) {
$this->handleException($queueJob, new FatalThrowableError($e));
$this->handleException($queueJob, $e);
}
return 0;
@@ -72,7 +71,7 @@ class SyncQueue extends Queue implements QueueContract
protected function raiseBeforeJobEvent(Job $job)
{
if ($this->container->bound('events')) {
$this->container['events']->fire(new Events\JobProcessing($this->connectionName, $job));
$this->container['events']->dispatch(new JobProcessing($this->connectionName, $job));
}
}
@@ -85,7 +84,7 @@ class SyncQueue extends Queue implements QueueContract
protected function raiseAfterJobEvent(Job $job)
{
if ($this->container->bound('events')) {
$this->container['events']->fire(new Events\JobProcessed($this->connectionName, $job));
$this->container['events']->dispatch(new JobProcessed($this->connectionName, $job));
}
}
@@ -93,30 +92,30 @@ class SyncQueue extends Queue implements QueueContract
* Raise the exception occurred queue job event.
*
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Exception $e
* @param \Throwable $e
* @return void
*/
protected function raiseExceptionOccurredJobEvent(Job $job, $e)
protected function raiseExceptionOccurredJobEvent(Job $job, Throwable $e)
{
if ($this->container->bound('events')) {
$this->container['events']->fire(new Events\JobExceptionOccurred($this->connectionName, $job, $e));
$this->container['events']->dispatch(new JobExceptionOccurred($this->connectionName, $job, $e));
}
}
/**
* Handle an exception that occurred while processing a job.
*
* @param \Illuminate\Queue\Jobs\Job $queueJob
* @param \Exception $e
* @param \Illuminate\Contracts\Queue\Job $queueJob
* @param \Throwable $e
* @return void
*
* @throws \Exception
* @throws \Throwable
*/
protected function handleException($queueJob, $e)
protected function handleException(Job $queueJob, Throwable $e)
{
$this->raiseExceptionOccurredJobEvent($queueJob, $e);
FailingJob::handle($this->connectionName, $queueJob, $e);
$queueJob->fail($e);
throw $e;
}
@@ -125,8 +124,8 @@ class SyncQueue extends Queue implements QueueContract
* Push a raw payload onto the queue.
*
* @param string $payload
* @param string $queue
* @param array $options
* @param string|null $queue
* @param array $options
* @return mixed
*/
public function pushRaw($payload, $queue = null, array $options = [])
@@ -135,12 +134,12 @@ class SyncQueue extends Queue implements QueueContract
}
/**
* Push a new job onto the queue after a delay.
* Push a new job onto the queue after (n) seconds.
*
* @param \DateTime|int $delay
* @param \DateTimeInterface|\DateInterval|int $delay
* @param string $job
* @param mixed $data
* @param string $queue
* @param mixed $data
* @param string|null $queue
* @return mixed
*/
public function later($delay, $job, $data = '', $queue = null)
@@ -151,7 +150,7 @@ class SyncQueue extends Queue implements QueueContract
/**
* Pop the next job off of the queue.
*
* @param string $queue
* @param string|null $queue
* @return \Illuminate\Contracts\Queue\Job|null
*/
public function pop($queue = null)

View File

@@ -2,22 +2,39 @@
namespace Illuminate\Queue;
use Exception;
use Throwable;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Database\DetectsLostConnections;
use Illuminate\Contracts\Debug\ExceptionHandler;
use Symfony\Component\Debug\Exception\FatalThrowableError;
use Illuminate\Contracts\Cache\Repository as CacheContract;
use Illuminate\Contracts\Debug\ExceptionHandler;
use Illuminate\Contracts\Events\Dispatcher;
use Illuminate\Contracts\Queue\Factory as QueueManager;
use Illuminate\Database\DetectsLostConnections;
use Illuminate\Queue\Events\JobExceptionOccurred;
use Illuminate\Queue\Events\JobProcessed;
use Illuminate\Queue\Events\JobProcessing;
use Illuminate\Queue\Events\JobReleasedAfterException;
use Illuminate\Queue\Events\Looping;
use Illuminate\Queue\Events\WorkerStopping;
use Illuminate\Support\Carbon;
use Throwable;
class Worker
{
use DetectsLostConnections;
const EXIT_SUCCESS = 0;
const EXIT_ERROR = 1;
const EXIT_MEMORY_LIMIT = 12;
/**
* The name of the worker.
*
* @var string
*/
protected $name;
/**
* The queue manager instance.
*
* @var \Illuminate\Queue\QueueManager
* @var \Illuminate\Contracts\Queue\Factory
*/
protected $manager;
@@ -38,10 +55,24 @@ class Worker
/**
* The exception handler instance.
*
* @var \Illuminate\Foundation\Exceptions\Handler
* @var \Illuminate\Contracts\Debug\ExceptionHandler
*/
protected $exceptions;
/**
* The callback used to determine if the application is in maintenance mode.
*
* @var callable
*/
protected $isDownForMaintenance;
/**
* The callback used to reset the application's scope.
*
* @var callable
*/
protected $resetScope;
/**
* Indicates if the worker should exit.
*
@@ -56,21 +87,34 @@ class Worker
*/
public $paused = false;
/**
* The callbacks used to pop jobs from queues.
*
* @var callable[]
*/
protected static $popCallbacks = [];
/**
* Create a new queue worker.
*
* @param \Illuminate\Queue\QueueManager $manager
* @param \Illuminate\Contracts\Queue\Factory $manager
* @param \Illuminate\Contracts\Events\Dispatcher $events
* @param \Illuminate\Contracts\Debug\ExceptionHandler $exceptions
* @param callable $isDownForMaintenance
* @param callable|null $resetScope
* @return void
*/
public function __construct(QueueManager $manager,
Dispatcher $events,
ExceptionHandler $exceptions)
ExceptionHandler $exceptions,
callable $isDownForMaintenance,
callable $resetScope = null)
{
$this->events = $events;
$this->manager = $manager;
$this->exceptions = $exceptions;
$this->isDownForMaintenance = $isDownForMaintenance;
$this->resetScope = $resetScope;
}
/**
@@ -79,24 +123,36 @@ class Worker
* @param string $connectionName
* @param string $queue
* @param \Illuminate\Queue\WorkerOptions $options
* @return void
* @return int
*/
public function daemon($connectionName, $queue, WorkerOptions $options)
{
$this->listenForSignals();
if ($supportsAsyncSignals = $this->supportsAsyncSignals()) {
$this->listenForSignals();
}
$lastRestart = $this->getTimestampOfLastQueueRestart();
[$startTime, $jobsProcessed] = [hrtime(true) / 1e9, 0];
while (true) {
// Before reserving any jobs, we will make sure this queue is not paused and
// if it is we will just pause this worker for a given amount of time and
// make sure we do not need to kill this worker process off completely.
if (! $this->daemonShouldRun($options)) {
$this->pauseWorker($options, $lastRestart);
if (! $this->daemonShouldRun($options, $connectionName, $queue)) {
$status = $this->pauseWorker($options, $lastRestart);
if (! is_null($status)) {
return $this->stop($status, $options);
}
continue;
}
if (isset($this->resetScope)) {
($this->resetScope)();
}
// First, we will attempt to get the next job off of the queue. We will also
// register the timeout handler and reset the alarm for this job so it is
// not stuck in a frozen state forever. Then, we can fire off this job.
@@ -104,52 +160,92 @@ class Worker
$this->manager->connection($connectionName), $queue
);
$this->registerTimeoutHandler($job, $options);
if ($supportsAsyncSignals) {
$this->registerTimeoutHandler($job, $options);
}
// If the daemon should run (not in maintenance mode, etc.), then we can run
// fire off this job for processing. Otherwise, we will need to sleep the
// worker so no more jobs are processed until they should be processed.
if ($job) {
$jobsProcessed++;
$this->runJob($job, $connectionName, $options);
if ($options->rest > 0) {
$this->sleep($options->rest);
}
} else {
$this->sleep($options->sleep);
}
if ($supportsAsyncSignals) {
$this->resetTimeoutHandler();
}
// Finally, we will check to see if we have exceeded our memory limits or if
// the queue should restart based on other indications. If so, we'll stop
// this worker and let whatever is "monitoring" it restart the process.
$this->stopIfNecessary($options, $lastRestart);
$status = $this->stopIfNecessary(
$options, $lastRestart, $startTime, $jobsProcessed, $job
);
if (! is_null($status)) {
return $this->stop($status, $options);
}
}
}
/**
* Register the worker timeout handler (PHP 7.1+).
* Register the worker timeout handler.
*
* @param \Illuminate\Contracts\Queue\Job|null $job
* @param WorkerOptions $options
* @param \Illuminate\Queue\WorkerOptions $options
* @return void
*/
protected function registerTimeoutHandler($job, WorkerOptions $options)
{
if ($this->supportsAsyncSignals()) {
// We will register a signal handler for the alarm signal so that we can kill this
// process if it is running too long because it has frozen. This uses the async
// signals supported in recent versions of PHP to accomplish it conveniently.
pcntl_signal(SIGALRM, function () {
$this->kill(1);
});
// We will register a signal handler for the alarm signal so that we can kill this
// process if it is running too long because it has frozen. This uses the async
// signals supported in recent versions of PHP to accomplish it conveniently.
pcntl_signal(SIGALRM, function () use ($job, $options) {
if ($job) {
$this->markJobAsFailedIfWillExceedMaxAttempts(
$job->getConnectionName(), $job, (int) $options->maxTries, $e = $this->maxAttemptsExceededException($job)
);
pcntl_alarm(
max($this->timeoutForJob($job, $options), 0)
);
}
$this->markJobAsFailedIfWillExceedMaxExceptions(
$job->getConnectionName(), $job, $e
);
$this->markJobAsFailedIfItShouldFailOnTimeout(
$job->getConnectionName(), $job, $e
);
}
$this->kill(static::EXIT_ERROR, $options);
});
pcntl_alarm(
max($this->timeoutForJob($job, $options), 0)
);
}
/**
* Reset the worker timeout handler.
*
* @return void
*/
protected function resetTimeoutHandler()
{
pcntl_alarm(0);
}
/**
* Get the appropriate timeout for the given job.
*
* @param \Illuminate\Contracts\Queue\Job|null $job
* @param WorkerOptions $options
* @param \Illuminate\Queue\WorkerOptions $options
* @return int
*/
protected function timeoutForJob($job, WorkerOptions $options)
@@ -160,47 +256,53 @@ class Worker
/**
* Determine if the daemon should process on this iteration.
*
* @param WorkerOptions $options
* @param \Illuminate\Queue\WorkerOptions $options
* @param string $connectionName
* @param string $queue
* @return bool
*/
protected function daemonShouldRun(WorkerOptions $options)
protected function daemonShouldRun(WorkerOptions $options, $connectionName, $queue)
{
return ! (($this->manager->isDownForMaintenance() && ! $options->force) ||
return ! ((($this->isDownForMaintenance)() && ! $options->force) ||
$this->paused ||
$this->events->until(new Events\Looping) === false);
$this->events->until(new Looping($connectionName, $queue)) === false);
}
/**
* Pause the worker for the current loop.
*
* @param WorkerOptions $options
* @param \Illuminate\Queue\WorkerOptions $options
* @param int $lastRestart
* @return void
* @return int|null
*/
protected function pauseWorker(WorkerOptions $options, $lastRestart)
{
$this->sleep($options->sleep > 0 ? $options->sleep : 1);
$this->stopIfNecessary($options, $lastRestart);
return $this->stopIfNecessary($options, $lastRestart);
}
/**
* Stop the process if necessary.
* Determine the exit code to stop the process if necessary.
*
* @param WorkerOptions $options
* @param \Illuminate\Queue\WorkerOptions $options
* @param int $lastRestart
* @param int $startTime
* @param int $jobsProcessed
* @param mixed $job
* @return int|null
*/
protected function stopIfNecessary(WorkerOptions $options, $lastRestart)
protected function stopIfNecessary(WorkerOptions $options, $lastRestart, $startTime = 0, $jobsProcessed = 0, $job = null)
{
if ($this->shouldQuit) {
$this->kill();
}
if ($this->memoryExceeded($options->memory)) {
$this->stop(12);
} elseif ($this->queueShouldRestart($lastRestart)) {
$this->stop();
}
return match (true) {
$this->shouldQuit => static::EXIT_SUCCESS,
$this->memoryExceeded($options->memory) => static::EXIT_MEMORY_LIMIT,
$this->queueShouldRestart($lastRestart) => static::EXIT_SUCCESS,
$options->stopWhenEmpty && is_null($job) => static::EXIT_SUCCESS,
$options->maxTime && hrtime(true) / 1e9 - $startTime >= $options->maxTime => static::EXIT_SUCCESS,
$options->maxJobs && $jobsProcessed >= $options->maxJobs => static::EXIT_SUCCESS,
default => null
};
}
/**
@@ -236,20 +338,26 @@ class Worker
*/
protected function getNextJob($connection, $queue)
{
$popJobCallback = function ($queue) use ($connection) {
return $connection->pop($queue);
};
try {
if (isset(static::$popCallbacks[$this->name])) {
return (static::$popCallbacks[$this->name])($popJobCallback, $queue);
}
foreach (explode(',', $queue) as $queue) {
if (! is_null($job = $connection->pop($queue))) {
if (! is_null($job = $popJobCallback($queue))) {
return $job;
}
}
} catch (Exception $e) {
} catch (Throwable $e) {
$this->exceptions->report($e);
$this->stopWorkerIfLostConnection($e);
} catch (Throwable $e) {
$this->exceptions->report($e = new FatalThrowableError($e));
$this->stopWorkerIfLostConnection($e);
$this->sleep(1);
}
}
@@ -265,12 +373,8 @@ class Worker
{
try {
return $this->process($connectionName, $job, $options);
} catch (Exception $e) {
$this->exceptions->report($e);
$this->stopWorkerIfLostConnection($e);
} catch (Throwable $e) {
$this->exceptions->report($e = new FatalThrowableError($e));
$this->exceptions->report($e);
$this->stopWorkerIfLostConnection($e);
}
@@ -279,7 +383,7 @@ class Worker
/**
* Stop the worker if we have lost connection to a database.
*
* @param \Exception $e
* @param \Throwable $e
* @return void
*/
protected function stopWorkerIfLostConnection($e)
@@ -302,8 +406,8 @@ class Worker
public function process($connectionName, $job, WorkerOptions $options)
{
try {
// First we will raise the before job event and determine if the job has already ran
// over the its maximum attempt limit, which could primarily happen if the job is
// First we will raise the before job event and determine if the job has already run
// over its maximum attempt limits, which could primarily happen when this job is
// continually timing out and not actually throwing any exceptions from itself.
$this->raiseBeforeJobEvent($connectionName, $job);
@@ -311,18 +415,18 @@ class Worker
$connectionName, $job, (int) $options->maxTries
);
// Here we will fire off the job and let it process. We will catch any exceptions so
// they can be reported to the developers logs, etc. Once the job is finished the
// proper events will be fired to let any listeners know this job has finished.
if ($job->isDeleted()) {
return $this->raiseAfterJobEvent($connectionName, $job);
}
// Here we will fire off the job and let it process. We will catch any exceptions, so
// they can be reported to the developer's logs, etc. Once the job is finished the
// proper events will be fired to let any listeners know this job has completed.
$job->fire();
$this->raiseAfterJobEvent($connectionName, $job);
} catch (Exception $e) {
$this->handleJobException($connectionName, $job, $options, $e);
} catch (Throwable $e) {
$this->handleJobException(
$connectionName, $job, $options, new FatalThrowableError($e)
);
$this->handleJobException($connectionName, $job, $options, $e);
}
}
@@ -332,20 +436,26 @@ class Worker
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Illuminate\Queue\WorkerOptions $options
* @param \Exception $e
* @param \Throwable $e
* @return void
*
* @throws \Exception
* @throws \Throwable
*/
protected function handleJobException($connectionName, $job, WorkerOptions $options, $e)
protected function handleJobException($connectionName, $job, WorkerOptions $options, Throwable $e)
{
try {
// First, we will go ahead and mark the job as failed if it will exceed the maximum
// attempts it is allowed to run the next time we process it. If so we will just
// go ahead and mark it as failed now so we do not have to release this again.
$this->markJobAsFailedIfWillExceedMaxAttempts(
$connectionName, $job, (int) $options->maxTries, $e
);
if (! $job->hasFailed()) {
$this->markJobAsFailedIfWillExceedMaxAttempts(
$connectionName, $job, (int) $options->maxTries, $e
);
$this->markJobAsFailedIfWillExceedMaxExceptions(
$connectionName, $job, $e
);
}
$this->raiseExceptionOccurredJobEvent(
$connectionName, $job, $e
@@ -355,7 +465,11 @@ class Worker
// so it is not lost entirely. This'll let the job be retried at a later time by
// another listener (or this same one). We will re-throw this exception after.
if (! $job->isDeleted() && ! $job->isReleased() && ! $job->hasFailed()) {
$job->release($options->delay);
$job->release($this->calculateBackoff($job, $options));
$this->events->dispatch(new JobReleasedAfterException(
$connectionName, $job
));
}
}
@@ -371,18 +485,24 @@ class Worker
* @param \Illuminate\Contracts\Queue\Job $job
* @param int $maxTries
* @return void
*
* @throws \Throwable
*/
protected function markJobAsFailedIfAlreadyExceedsMaxAttempts($connectionName, $job, $maxTries)
{
$maxTries = ! is_null($job->maxTries()) ? $job->maxTries() : $maxTries;
if ($maxTries === 0 || $job->attempts() <= $maxTries) {
$retryUntil = $job->retryUntil();
if ($retryUntil && Carbon::now()->getTimestamp() <= $retryUntil) {
return;
}
$this->failJob($connectionName, $job, $e = new MaxAttemptsExceededException(
'A queued job has been attempted too many times. The job may have previously timed out.'
));
if (! $retryUntil && ($maxTries === 0 || $job->attempts() <= $maxTries)) {
return;
}
$this->failJob($job, $e = $this->maxAttemptsExceededException($job));
throw $e;
}
@@ -393,29 +513,92 @@ class Worker
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param int $maxTries
* @param \Exception $e
* @param \Throwable $e
* @return void
*/
protected function markJobAsFailedIfWillExceedMaxAttempts($connectionName, $job, $maxTries, $e)
protected function markJobAsFailedIfWillExceedMaxAttempts($connectionName, $job, $maxTries, Throwable $e)
{
$maxTries = ! is_null($job->maxTries()) ? $job->maxTries() : $maxTries;
if ($maxTries > 0 && $job->attempts() >= $maxTries) {
$this->failJob($connectionName, $job, $e);
if ($job->retryUntil() && $job->retryUntil() <= Carbon::now()->getTimestamp()) {
$this->failJob($job, $e);
}
if (! $job->retryUntil() && $maxTries > 0 && $job->attempts() >= $maxTries) {
$this->failJob($job, $e);
}
}
/**
* Mark the given job as failed if it has exceeded the maximum allowed attempts.
*
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Throwable $e
* @return void
*/
protected function markJobAsFailedIfWillExceedMaxExceptions($connectionName, $job, Throwable $e)
{
if (! $this->cache || is_null($uuid = $job->uuid()) ||
is_null($maxExceptions = $job->maxExceptions())) {
return;
}
if (! $this->cache->get('job-exceptions:'.$uuid)) {
$this->cache->put('job-exceptions:'.$uuid, 0, Carbon::now()->addDay());
}
if ($maxExceptions <= $this->cache->increment('job-exceptions:'.$uuid)) {
$this->cache->forget('job-exceptions:'.$uuid);
$this->failJob($job, $e);
}
}
/**
* Mark the given job as failed if it should fail on timeouts.
*
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Throwable $e
* @return void
*/
protected function markJobAsFailedIfItShouldFailOnTimeout($connectionName, $job, Throwable $e)
{
if (method_exists($job, 'shouldFailOnTimeout') ? $job->shouldFailOnTimeout() : false) {
$this->failJob($job, $e);
}
}
/**
* Mark the given job as failed and raise the relevant event.
*
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Exception $e
* @param \Throwable $e
* @return void
*/
protected function failJob($connectionName, $job, $e)
protected function failJob($job, Throwable $e)
{
return FailingJob::handle($connectionName, $job, $e);
$job->fail($e);
}
/**
* Calculate the backoff for the given job.
*
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Illuminate\Queue\WorkerOptions $options
* @return int
*/
protected function calculateBackoff($job, WorkerOptions $options)
{
$backoff = explode(
',',
method_exists($job, 'backoff') && ! is_null($job->backoff())
? $job->backoff()
: $options->backoff
);
return (int) ($backoff[$job->attempts() - 1] ?? last($backoff));
}
/**
@@ -427,7 +610,7 @@ class Worker
*/
protected function raiseBeforeJobEvent($connectionName, $job)
{
$this->events->fire(new Events\JobProcessing(
$this->events->dispatch(new JobProcessing(
$connectionName, $job
));
}
@@ -441,7 +624,7 @@ class Worker
*/
protected function raiseAfterJobEvent($connectionName, $job)
{
$this->events->fire(new Events\JobProcessed(
$this->events->dispatch(new JobProcessed(
$connectionName, $job
));
}
@@ -451,27 +634,12 @@ class Worker
*
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Exception $e
* @param \Throwable $e
* @return void
*/
protected function raiseExceptionOccurredJobEvent($connectionName, $job, $e)
protected function raiseExceptionOccurredJobEvent($connectionName, $job, Throwable $e)
{
$this->events->fire(new Events\JobExceptionOccurred(
$connectionName, $job, $e
));
}
/**
* Raise the failed queue job event.
*
* @param string $connectionName
* @param \Illuminate\Contracts\Queue\Job $job
* @param \Exception $e
* @return void
*/
protected function raiseFailedJobEvent($connectionName, $job, $e)
{
$this->events->fire(new Events\JobFailed(
$this->events->dispatch(new JobExceptionOccurred(
$connectionName, $job, $e
));
}
@@ -506,21 +674,12 @@ class Worker
*/
protected function listenForSignals()
{
if ($this->supportsAsyncSignals()) {
pcntl_async_signals(true);
pcntl_async_signals(true);
pcntl_signal(SIGTERM, function () {
$this->shouldQuit = true;
});
pcntl_signal(SIGUSR2, function () {
$this->paused = true;
});
pcntl_signal(SIGCONT, function () {
$this->paused = false;
});
}
pcntl_signal(SIGQUIT, fn () => $this->shouldQuit = true);
pcntl_signal(SIGTERM, fn () => $this->shouldQuit = true);
pcntl_signal(SIGUSR2, fn () => $this->paused = true);
pcntl_signal(SIGCONT, fn () => $this->paused = false);
}
/**
@@ -530,42 +689,45 @@ class Worker
*/
protected function supportsAsyncSignals()
{
return version_compare(PHP_VERSION, '7.1.0') >= 0 &&
extension_loaded('pcntl');
return extension_loaded('pcntl');
}
/**
* Determine if the memory limit has been exceeded.
*
* @param int $memoryLimit
* @param int $memoryLimit
* @return bool
*/
public function memoryExceeded($memoryLimit)
{
return (memory_get_usage() / 1024 / 1024) >= $memoryLimit;
return (memory_get_usage(true) / 1024 / 1024) >= $memoryLimit;
}
/**
* Stop listening and bail out of the script.
*
* @param int $status
* @return void
* @param WorkerOptions|null $options
* @return int
*/
public function stop($status = 0)
public function stop($status = 0, $options = null)
{
$this->events->fire(new Events\WorkerStopping);
$this->events->dispatch(new WorkerStopping($status, $options));
exit($status);
return $status;
}
/**
* Kill the process.
*
* @param int $status
* @return void
* @param \Illuminate\Queue\WorkerOptions|null $options
* @return never
*/
public function kill($status = 0)
public function kill($status = 0, $options = null)
{
$this->events->dispatch(new WorkerStopping($status, $options));
if (extension_loaded('posix')) {
posix_kill(getmypid(), SIGKILL);
}
@@ -573,32 +735,80 @@ class Worker
exit($status);
}
/**
* Create an instance of MaxAttemptsExceededException.
*
* @param \Illuminate\Contracts\Queue\Job $job
* @return \Illuminate\Queue\MaxAttemptsExceededException
*/
protected function maxAttemptsExceededException($job)
{
return new MaxAttemptsExceededException(
$job->resolveName().' has been attempted too many times or run too long. The job may have previously timed out.'
);
}
/**
* Sleep the script for a given number of seconds.
*
* @param int $seconds
* @param int|float $seconds
* @return void
*/
public function sleep($seconds)
{
sleep($seconds);
if ($seconds < 1) {
usleep($seconds * 1000000);
} else {
sleep($seconds);
}
}
/**
* Set the cache repository implementation.
*
* @param \Illuminate\Contracts\Cache\Repository $cache
* @return void
* @return $this
*/
public function setCache(CacheContract $cache)
{
$this->cache = $cache;
return $this;
}
/**
* Set the name of the worker.
*
* @param string $name
* @return $this
*/
public function setName($name)
{
$this->name = $name;
return $this;
}
/**
* Register a callback to be executed to pick jobs.
*
* @param string $workerName
* @param callable $callback
* @return void
*/
public static function popUsing($workerName, $callback)
{
if (is_null($callback)) {
unset(static::$popCallbacks[$workerName]);
} else {
static::$popCallbacks[$workerName] = $callback;
}
}
/**
* Get the queue manager instance.
*
* @return \Illuminate\Queue\QueueManager
* @return \Illuminate\Contracts\Queue\Factory
*/
public function getManager()
{
@@ -608,7 +818,7 @@ class Worker
/**
* Set the queue manager instance.
*
* @param \Illuminate\Queue\QueueManager $manager
* @param \Illuminate\Contracts\Queue\Factory $manager
* @return void
*/
public function setManager(QueueManager $manager)

View File

@@ -5,11 +5,18 @@ namespace Illuminate\Queue;
class WorkerOptions
{
/**
* The number of seconds before a released job will be available.
* The name of the worker.
*
* @var int
*/
public $delay;
public $name;
/**
* The number of seconds to wait before retrying a job that encountered an uncaught exception.
*
* @var int|int[]
*/
public $backoff;
/**
* The maximum amount of RAM the worker may consume.
@@ -33,7 +40,14 @@ class WorkerOptions
public $sleep;
/**
* The maximum amount of times a job may be attempted.
* The number of seconds to rest between jobs.
*
* @var int
*/
public $rest;
/**
* The maximum number of times a job may be attempted.
*
* @var int
*/
@@ -46,23 +60,56 @@ class WorkerOptions
*/
public $force;
/**
* Indicates if the worker should stop when the queue is empty.
*
* @var bool
*/
public $stopWhenEmpty;
/**
* The maximum number of jobs to run.
*
* @var int
*/
public $maxJobs;
/**
* The maximum number of seconds a worker may live.
*
* @var int
*/
public $maxTime;
/**
* Create a new worker options instance.
*
* @param int $delay
* @param string $name
* @param int|int[] $backoff
* @param int $memory
* @param int $timeout
* @param int $sleep
* @param int $maxTries
* @param bool $force
* @param bool $stopWhenEmpty
* @param int $maxJobs
* @param int $maxTime
* @param int $rest
* @return void
*/
public function __construct($delay = 0, $memory = 128, $timeout = 60, $sleep = 3, $maxTries = 0, $force = false)
public function __construct($name = 'default', $backoff = 0, $memory = 128, $timeout = 60, $sleep = 3, $maxTries = 1,
$force = false, $stopWhenEmpty = false, $maxJobs = 0, $maxTime = 0, $rest = 0)
{
$this->delay = $delay;
$this->name = $name;
$this->backoff = $backoff;
$this->sleep = $sleep;
$this->rest = $rest;
$this->force = $force;
$this->memory = $memory;
$this->timeout = $timeout;
$this->maxTries = $maxTries;
$this->stopWhenEmpty = $stopWhenEmpty;
$this->maxJobs = $maxJobs;
$this->maxTime = $maxTime;
}
}

View File

@@ -14,16 +14,19 @@
}
],
"require": {
"php": ">=5.6.4",
"illuminate/console": "5.4.*",
"illuminate/container": "5.4.*",
"illuminate/contracts": "5.4.*",
"illuminate/database": "5.4.*",
"illuminate/filesystem": "5.4.*",
"illuminate/support": "5.4.*",
"nesbot/carbon": "~1.20",
"symfony/debug": "~3.2",
"symfony/process": "~3.2"
"php": "^8.0.2",
"ext-json": "*",
"illuminate/collections": "^9.0",
"illuminate/console": "^9.0",
"illuminate/container": "^9.0",
"illuminate/contracts": "^9.0",
"illuminate/database": "^9.0",
"illuminate/filesystem": "^9.0",
"illuminate/pipeline": "^9.0",
"illuminate/support": "^9.0",
"laravel/serializable-closure": "^1.2.2",
"ramsey/uuid": "^4.7",
"symfony/process": "^6.0"
},
"autoload": {
"psr-4": {
@@ -32,13 +35,15 @@
},
"extra": {
"branch-alias": {
"dev-master": "5.4-dev"
"dev-master": "9.x-dev"
}
},
"suggest": {
"aws/aws-sdk-php": "Required to use the SQS queue driver (~3.0).",
"illuminate/redis": "Required to use the Redis queue driver (5.4.*).",
"pda/pheanstalk": "Required to use the Beanstalk queue driver (~3.0)."
"ext-pcntl": "Required to use all features of the queue worker.",
"ext-posix": "Required to use all features of the queue worker.",
"aws/aws-sdk-php": "Required to use the SQS queue driver and DynamoDb failed job storage (^3.235.5).",
"illuminate/redis": "Required to use the Redis queue driver (^9.0).",
"pda/pheanstalk": "Required to use the Beanstalk queue driver (^4.0)."
},
"config": {
"sort-packages": true