feat: Passwordless cross-device authentication

- Arkitektur: docs/auth/passwordless-architecture.md
- Backend: iom/quixzoom-auth-service/ (FastAPI + Redis)
- Webb: quixzoom-market-pages/se/login/ (QR-kod + polling)
- App: iom/quixzoom-app/src/features/auth/ (push + deep links)

Flöde: QR-kod → app-godkännande → webb-inloggad
This commit is contained in:
Bernt
2026-07-07 07:11:50 +00:00
parent 4aa984ad74
commit 6989a98d75
61843 changed files with 5491611 additions and 872231 deletions
+56
View File
@@ -0,0 +1,56 @@
'use strict';
const _ = require('lodash');
const builtinStrategies = {
fixed(delay) {
return function() {
return delay;
};
},
exponential(delay) {
return function(attemptsMade) {
return Math.round((Math.pow(2, attemptsMade) - 1) * delay);
};
}
};
function lookupStrategy(backoff, customStrategies) {
if (backoff.type in customStrategies) {
return customStrategies[backoff.type];
} else if (backoff.type in builtinStrategies) {
return builtinStrategies[backoff.type](backoff.delay);
} else {
throw new Error(
'Unknown backoff strategy ' +
backoff.type +
'. If a custom backoff strategy is used, specify it when the queue is created.'
);
}
}
module.exports = {
normalize(backoff) {
if (_.isFinite(backoff)) {
return {
type: 'fixed',
delay: backoff
};
} else if (backoff) {
return backoff;
}
},
calculate(backoff, attemptsMade, customStrategies, err, strategyOptions) {
if (backoff) {
const strategy = lookupStrategy(
backoff,
customStrategies,
strategyOptions
);
return strategy(attemptsMade, err, strategyOptions);
}
}
};
+112
View File
@@ -0,0 +1,112 @@
--[[
Adds a job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- if delayed:
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
- if not delayed
- Adds the jobId to the wait/paused list in one of three ways:
- LIFO
- FIFO
- prioritized.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta-paused'
KEYS[4] 'id'
KEYS[5] 'delayed'
KEYS[6] 'priority'
ARGV[1] key prefix,
ARGV[2] custom id (will not generate one automatically)
ARGV[3] name
ARGV[4] data (json stringified job data)
ARGV[5] opts (json stringified job opts)
ARGV[6] timestamp
ARGV[7] delay
ARGV[8] delayedTimestamp
ARGV[9] priority
ARGV[10] LIFO
ARGV[11] token
ARGV[12] debounce key
ARGV[13] debounceId
ARGV[14] debounceTtl
]]
local jobId
local jobIdKey
local rcall = redis.call
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/debounceJob"
--- @include "includes/getTargetQueueList"
local jobCounter = rcall("INCR", KEYS[4])
if ARGV[2] == "" then
jobId = jobCounter
jobIdKey = ARGV[1] .. jobId
else
jobId = ARGV[2]
jobIdKey = ARGV[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
rcall("PUBLISH", ARGV[1] .. "duplicated@" .. ARGV[11], jobId)
return jobId .. "" -- convert to string
end
end
local debounceKey = ARGV[12]
local opts = cmsgpack.unpack(ARGV[5])
local debouncedJobId = debounceJob(ARGV[1], ARGV[13], ARGV[14],
jobId, debounceKey, ARGV[11])
if debouncedJobId then
return debouncedJobId
end
local debounceId = ARGV[13]
local optionalValues = {}
if debounceId ~= "" then
table.insert(optionalValues, "deid")
table.insert(optionalValues, debounceId)
end
-- Store the job.
rcall("HMSET", jobIdKey, "name", ARGV[3], "data", ARGV[4], "opts", opts, "timestamp",
ARGV[6], "delay", ARGV[7], "priority", ARGV[9], unpack(optionalValues))
-- Check if job is delayed
local delayedTimestamp = tonumber(ARGV[8])
if(delayedTimestamp ~= 0) then
local timestamp = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
rcall("ZADD", KEYS[5], timestamp, jobId)
rcall("PUBLISH", KEYS[5], delayedTimestamp)
else
local target
-- Whe check for the meta-paused key to decide if we are paused or not
-- (since an empty list and !EXISTS are not really the same)
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
-- Standard or priority add
local priority = tonumber(ARGV[9])
if priority == 0 then
-- LIFO or FIFO
rcall(ARGV[10], target, jobId)
else
addJobWithPriority(KEYS[6], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[1] .. "ing@" .. ARGV[11], jobId)
end
return jobId .. "" -- convert to string
+30
View File
@@ -0,0 +1,30 @@
--[[
Add job log
Input:
KEYS[1] job id key
KEYS[2] job logs key
ARGV[1] id
ARGV[2] log
ARGV[3] keepLogs
Output:
-1 - Missing job.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
local logCount = rcall("RPUSH", KEYS[2], ARGV[2])
if ARGV[3] ~= '' then
local keepLogs = tonumber(ARGV[3])
rcall("LTRIM", KEYS[2], -keepLogs, -1)
return math.min(keepLogs, logCount)
end
return logCount
else
return -1
end
+146
View File
@@ -0,0 +1,146 @@
--[[
Remove jobs from the specific set.
Input:
KEYS[1] set key,
KEYS[2] priority key
KEYS[3] rate limiter key
ARGV[1] prefix key
ARGV[2] maxTimestamp
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
]]
local setKey = KEYS[1]
local priorityKey = KEYS[2]
local rateLimiterKey = KEYS[3]
local prefixKey = ARGV[1]
local maxTimestamp = ARGV[2]
local limitStr = ARGV[3]
local setName = ARGV[4]
local isList = false
local rcall = redis.call
-- Includes
--- @include "includes/removeDebounceKey"
if setName == "wait" or setName == "active" or setName == "paused" then
isList = true
end
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
-- of items in a sorted set only run a single iteration. If we simply used
-- ZRANGE, we may take a long time traversing through jobs that are within the
-- grace period.
local function shouldUseZRangeByScore(isList, limit)
return not isList and limit > 0
end
local function getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
if isList then
return rcall("LRANGE", setKey, rangeStart, rangeEnd)
elseif shouldUseZRangeByScore(isList, limit) then
return rcall("ZRANGEBYSCORE", setKey, 0, maxTimestamp, "LIMIT", 0, limit)
else
return rcall("ZRANGE", setKey, rangeStart, rangeEnd)
end
end
local limit = tonumber(limitStr)
local rangeStart = 0
local rangeEnd = -1
-- If we're only deleting _n_ items, avoid retrieving all items
-- for faster performance
--
-- Start from the tail of the list, since that's where oldest elements
-- are generally added for FIFO lists
if limit > 0 then
rangeStart = -1 - limit + 1
rangeEnd = -1
end
local jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
local deleted = {}
local deletedCount = 0
local jobTS
-- Run this loop:
-- - Once, if limit is -1 or 0
-- - As many times as needed if limit is positive
while ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do
local jobIdsLen = #jobIds
for i, jobId in ipairs(jobIds) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = prefixKey .. jobId
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Find the right timestamp of the job to compare to maxTimestamp:
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
-- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
-- * timestamp is the original job submission time
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
for _, ts in ipairs(rcall("HMGET", jobKey, "finishedOn", "processedOn", "timestamp")) do
if (ts) then
jobTS = ts
break
end
end
if (not jobTS or jobTS < maxTimestamp) then
if isList then
-- Job ids can't be the empty string. Use the empty string as a
-- deletion marker. The actual deletion will occur at the end of the
-- script.
rcall("LSET", setKey, rangeEnd - jobIdsLen + i, "")
else
rcall("ZREM", setKey, jobId)
end
rcall("ZREM", priorityKey, jobId)
if setName ~= "completed" and setName ~= "failed" then
removeDebounceKey(prefixKey, jobKey)
end
rcall("DEL", jobKey)
rcall("DEL", jobKey .. ":logs")
-- delete keys related to rate limiter
-- NOTE: this code is unncessary for other sets than wait, paused and delayed.
local limiterIndexTable = rateLimiterKey .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
deletedCount = deletedCount + 1
table.insert(deleted, jobId)
end
end
end
-- If we didn't have a limit or used the single-iteration ZRANGEBYSCORE
-- function, return immediately. We should have deleted all the jobs we can
if limit <= 0 or shouldUseZRangeByScore(isList, limit) then
break
end
if deletedCount < limit then
-- We didn't delete enough. Look for more to delete
rangeStart = rangeStart - limit
rangeEnd = rangeEnd - limit
jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
end
end
if isList then
rcall("LREM", setKey, 0, "")
end
return deleted
+22
View File
@@ -0,0 +1,22 @@
--[[
Extend lock and removes the job from the stalled set.
Input:
KEYS[1] 'lock',
KEYS[2] 'stalled'
ARGV[1] token
ARGV[2] lock duration in milliseconds
ARGV[3] jobid
Output:
"1" if lock extended succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
rcall("SREM", KEYS[2], ARGV[3])
return 1
end
end
return 0
+37
View File
@@ -0,0 +1,37 @@
--[[
Get counts per provided states
Input:
KEYS[1] wait key
KEYS[2] paused key
KEYS[3] meta-paused key
KEYS[4] priority key
ARGV[1...] priorities
]]
local rcall = redis.call
local results = {}
local prioritizedKey = KEYS[4]
-- Includes
--- @include "includes/getTargetQueueList"
for i = 1, #ARGV do
local priority = tonumber(ARGV[i])
if priority == 0 then
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
local count = rcall("LLEN", target) - rcall("ZCARD", prioritizedKey)
if count < 0 then
-- considering when last waiting job is moved to active before
-- removing priority reference
results[#results+1] = 0
else
results[#results+1] = count
end
else
results[#results+1] = rcall("ZCOUNT", prioritizedKey,
priority, priority)
end
end
return results
+16
View File
@@ -0,0 +1,16 @@
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
+18
View File
@@ -0,0 +1,18 @@
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
+46
View File
@@ -0,0 +1,46 @@
--[[
Functions to collect metrics based on a current and previous count of jobs.
Granualarity is fixed at 1 minute.
]]
-- Includes
--- @include "batches"
local function collectMetrics(metaKey, dataPointsList, maxDataPoints, timestamp)
-- Increment current count
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
-- Compute how many data points we need to add to the list, N.
local prevTS = rcall("HGET", metaKey, "prevTS")
if not prevTS then
-- If prevTS is nil, set it to the current timestamp
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
return
end
local N = math.min(math.floor(timestamp / 60000) - math.floor(prevTS / 60000), tonumber(maxDataPoints))
if N > 0 then
local delta = count - rcall("HGET", metaKey, "prevCount")
-- If N > 1, add N-1 zeros to the list
if N > 1 then
local points = {}
points[1] = delta
for i = 2, N do points[i] = 0 end
for from, to in batches(#points, 7000) do
rcall("LPUSH", dataPointsList, unpack(points, from, to))
end
else
-- LPUSH delta to the list
rcall("LPUSH", dataPointsList, delta)
end
-- LTRIM to keep list to its max size
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
-- update prev count with current count
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
end
end
+20
View File
@@ -0,0 +1,20 @@
--[[
Function to debounce a job.
]]
local function debounceJob(prefixKey, debounceId, ttl, jobId, debounceKey, token)
if debounceId ~= "" then
local debounceKeyExists
if ttl ~= "" then
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
else
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
end
if debounceKeyExists then
local currentDebounceJobId = rcall('GET', debounceKey)
rcall("PUBLISH", prefixKey .. "debounced@" .. token, currentDebounceJobId)
return currentDebounceJobId
end
end
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
+12
View File
@@ -0,0 +1,12 @@
--[[
Function to remove debounce key.
]]
local function removeDebounceKey(prefixKey, jobKey)
local debounceId = rcall("HGET", jobKey, "deid")
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
rcall("DEL", debounceKey)
end
end
@@ -0,0 +1,14 @@
--[[
Function to remove debounce key if needed.
]]
local function removeDebounceKeyIfNeeded(prefixKey, debounceId)
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
local pttl = rcall("PTTL", debounceKey)
if pttl == 0 or pttl == -1 then
rcall("DEL", debounceKey)
end
end
end
+19
View File
@@ -0,0 +1,19 @@
local function removeLock(jobKey, stalledKey, token, jobId)
if token ~= "0" then
local lockKey = jobKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", stalledKey, jobId)
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
return 0
end
+22
View File
@@ -0,0 +1,22 @@
--[[
Checks if a job is finished (.i.e. is in the completed or failed set)
Input:
KEYS[1] completed key
KEYS[2] failed key
ARGV[1] job id
Output:
0 - not finished.
1 - completed.
2 - failed.
]]
if redis.call("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return 1
end
if redis.call("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return 2
end
return redis.call("ZSCORE", KEYS[2], ARGV[1])
+20
View File
@@ -0,0 +1,20 @@
--[[
Checks if job is in a given list.
Input:
KEYS[1]
ARGV[1]
Output:
1 if element found in the list.
]]
local function item_in_list (list, item)
for _, v in pairs(list) do
if v == item then
return 1
end
end
return nil
end
local items = redis.call("LRANGE", KEYS[1] , 0, -1)
return item_in_list(items, ARGV[1])
+137
View File
@@ -0,0 +1,137 @@
--[[
Move stalled jobs to wait.
Input:
KEYS[1] 'stalled' (SET)
KEYS[2] 'wait', (LIST)
KEYS[3] 'active', (LIST)
KEYS[4] 'failed', (ZSET)
KEYS[5] 'stalled-check', (KEY)
KEYS[6] 'meta-paused', (KEY)
KEYS[7] 'paused', (LIST)
ARGV[1] Max stalled job count
ARGV[2] queue.toKey('')
ARGV[3] timestamp
ARGV[4] max check time
Events:
'stalled' with stalled job id.
]]
local rcall = redis.call
-- Includes
--- @include "includes/batches"
--- @include "includes/getTargetQueueList"
--- @include "includes/removeDebounceKeyIfNeeded"
local function removeJob(jobId, baseKey)
local jobKey = baseKey .. jobId
rcall("DEL", jobKey, jobKey .. ':logs')
end
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
for i, jobId in ipairs(jobIds) do
removeJob(jobId, prefix)
end
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
for i, jobId in ipairs(jobIds) do
removeJob(jobId, prefix)
end
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
end
-- Check if we need to check for stalled jobs now.
if rcall("EXISTS", KEYS[5]) == 1 then
return {{}, {}}
end
rcall("SET", KEYS[5], ARGV[3], "PX", ARGV[4])
-- Move all stalled jobs to wait
local stalling = rcall('SMEMBERS', KEYS[1])
local stalled = {}
local failed = {}
if(#stalling > 0) then
rcall('DEL', KEYS[1])
local MAX_STALLED_JOB_COUNT = tonumber(ARGV[1])
-- Remove from active list
for i, jobId in ipairs(stalling) do
local jobKey = ARGV[2] .. jobId
-- Check that the lock is also missing, then we can handle this job as really stalled.
if(rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Remove from the active queue.
local removed = rcall("LREM", KEYS[3], 1, jobId)
if(removed > 0) then
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter", 1)
if(stalledCount > MAX_STALLED_JOB_COUNT) then
local jobAttributes = rcall("HMGET", jobKey, "opts", "deid")
local opts = cjson.decode(jobAttributes[1])
local removeOnFailType = type(opts["removeOnFail"])
rcall("ZADD", KEYS[4], ARGV[3], jobId)
rcall("HMSET", jobKey, "failedReason", "job stalled more than allowable limit",
"finishedOn", ARGV[3])
removeDebounceKeyIfNeeded(ARGV[2], jobAttributes[2])
rcall("PUBLISH", KEYS[4], '{"jobId":"' .. jobId .. '", "val": "job stalled more than maxStalledCount"}')
if removeOnFailType == "number" then
removeJobsByMaxCount(opts["removeOnFail"],
KEYS[4], ARGV[2])
elseif removeOnFailType == "boolean" then
if opts["removeOnFail"] then
removeJob(jobId, ARGV[2])
rcall("ZREM", KEYS[4], jobId)
end
elseif removeOnFailType ~= "nil" then
local maxAge = opts["removeOnFail"]["age"]
local maxCount = opts["removeOnFail"]["count"]
if maxAge ~= nil then
removeJobsByMaxAge(ARGV[3], maxAge,
KEYS[4], ARGV[2])
end
if maxCount ~= nil and maxCount > 0 then
removeJobsByMaxCount(maxCount, KEYS[4],
ARGV[2])
end
end
table.insert(failed, jobId)
else
local target = getTargetQueueList(KEYS[6], KEYS[2], KEYS[7])
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
rcall("RPUSH", target, jobId)
rcall('PUBLISH', KEYS[1] .. '@', jobId)
table.insert(stalled, jobId)
end
end
end
end
end
-- Mark potentially stalled jobs
local active = rcall('LRANGE', KEYS[3], 0, -1)
if (#active > 0) then
for from, to in batches(#active, 7000) do
rcall('SADD', KEYS[1], unpack(active, from, to))
end
end
return {failed, stalled}
+149
View File
@@ -0,0 +1,149 @@
--[[
Move next job to be processed to active, lock it and fetch its data. The job
may be delayed, in that case we need to move it to the delayed set instead.
This operation guarantees that the worker owns the job during the locks
expiration time. The worker is responsible of keeping the lock fresh
so that no other worker picks this job again.
Input:
KEYS[1] wait key
KEYS[2] active key
KEYS[3] priority key
KEYS[4] active event key
KEYS[5] stalled key
-- Rate limiting
KEYS[6] rate limiter key
KEYS[7] delayed key
--
KEYS[8] drained key
ARGV[1] key prefix
ARGV[2] lock token
ARGV[3] lock duration in milliseconds
ARGV[4] timestamp
ARGV[5] optional jobid
ARGV[6] optional jobs per time unit (rate limiter)
ARGV[7] optional time unit (rate limiter)
ARGV[8] optional do not do anything with job if rate limit hit
ARGV[9] optional rate limit by key
]]
local rcall = redis.call
local rateLimit = function(jobId, maxJobs)
local rateLimiterKey = KEYS[6];
local limiterIndexTable = rateLimiterKey .. ":index"
-- Rate limit by group?
if(ARGV[9]) then
local group = string.match(jobId, "[^:]+$")
if group ~= nil then
rateLimiterKey = rateLimiterKey .. ":" .. group
end
end
-- -- key for storing rate limited jobs
-- When a job has been previously rate limited it should be part of this set
-- if the job is back here means that the delay time for this job has passed and now we should
-- be able to process it again.
local limitedSetKey = rateLimiterKey .. ":limited"
local delay = 0
-- -- Check if job was already limited
local isLimited = rcall("SISMEMBER", limitedSetKey, jobId);
if isLimited == 1 then
-- Remove from limited zset since we are going to try to process it
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
else
-- If not, check if there are any limited jobs
-- If the job has not been rate limited, we should check if there are any other rate limited jobs, because if that
-- is the case we do not want to process this job, just calculate a delay for it and put it to "sleep".
local numLimitedJobs = rcall("SCARD", limitedSetKey)
if numLimitedJobs > 0 then
-- Note, add some slack to compensate for drift.
delay = ((numLimitedJobs * ARGV[7] * 1.1) / maxJobs) + tonumber(rcall("PTTL", rateLimiterKey))
end
end
local jobCounter = tonumber(rcall("GET", rateLimiterKey))
if(jobCounter == nil) then
jobCounter = 0
end
-- check if rate limit hit
if (delay == 0) and (jobCounter >= maxJobs) then
-- Seems like there are no current rated limited jobs, but the jobCounter has exceeded the number of jobs for this unit of time so we need to rate limit this job.
local exceedingJobs = jobCounter - maxJobs
delay = tonumber(rcall("PTTL", rateLimiterKey)) + ((exceedingJobs) * ARGV[7]) / maxJobs
end
if delay > 0 then
local bounceBack = ARGV[8]
if bounceBack == 'false' then
local timestamp = delay + tonumber(ARGV[4])
-- put job into delayed queue
rcall("ZADD", KEYS[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId)
rcall("PUBLISH", KEYS[7], timestamp)
rcall("SADD", limitedSetKey, jobId)
-- store index so that we can delete rate limited data
rcall("HSET", limiterIndexTable, jobId, limitedSetKey)
end
-- remove from active queue
rcall("LREM", KEYS[2], 1, jobId)
return true
else
-- false indicates not rate limited
-- increment jobCounter only when a job is not rate limited
if (jobCounter == 0) then
rcall("PSETEX", rateLimiterKey, ARGV[7], 1)
else
rcall("INCR", rateLimiterKey)
end
return false
end
end
local jobId = ARGV[5]
if jobId ~= '' then
-- clean stalled key
rcall("SREM", KEYS[5], jobId)
else
-- move from wait to active
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
end
if jobId then
-- Check if we need to perform rate limiting.
local maxJobs = tonumber(ARGV[6])
if maxJobs then
if rateLimit(jobId, maxJobs) then
return
end
end
-- get a lock
local jobKey = ARGV[1] .. jobId
local lockKey = jobKey .. ':lock'
rcall("SET", lockKey, ARGV[2], "PX", ARGV[3])
-- remove from priority
rcall("ZREM", KEYS[3], jobId)
rcall("PUBLISH", KEYS[4], jobId)
rcall("HSET", jobKey, "processedOn", ARGV[4])
return {rcall("HGETALL", jobKey), jobId} -- get job data
else
rcall("PUBLISH", KEYS[8], "")
end
+43
View File
@@ -0,0 +1,43 @@
--[[
Moves job from active to delayed set.
Input:
KEYS[1] active key
KEYS[2] delayed key
KEYS[3] job key
KEYS[4] stalled key
ARGV[1] delayedTimestamp
ARGV[2] the id of the job
ARGV[3] queue token
Output:
0 - OK
-1 - Missing job.
-2 - Job is locked.
Events:
- delayed key.
]]
local rcall = redis.call
-- Includes
--- @include "includes/removeLock"
if rcall("EXISTS", KEYS[3]) == 1 then
local errorCode = removeLock(KEYS[3], KEYS[4], ARGV[3], ARGV[2])
if errorCode < 0 then
return errorCode
end
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[2])
if numRemovedElements < 1 then return -3 end
local score = tonumber(ARGV[1])
rcall("ZADD", KEYS[2], score, ARGV[2])
rcall("PUBLISH", KEYS[2], (score / 0x1000))
return 0
else
return -1
end
+134
View File
@@ -0,0 +1,134 @@
--[[
Move job from active to a finished status (completed or failed)
A job can only be moved to completed if it was active.
The job must be locked before it can be moved to a finished status,
and the lock must be released in this script.
Input:
KEYS[1] active key
KEYS[2] completed/failed key
KEYS[3] jobId key
KEYS[4] wait key
KEYS[5] priority key
KEYS[6] active event key
KEYS[7] delayed key
KEYS[8] stalled key
KEYS[9] metrics key
ARGV[1] jobId
ARGV[2] timestamp
ARGV[3] msg property
ARGV[4] return value / failed reason
ARGV[5] token
ARGV[6] shouldRemove
ARGV[7] event data (? maybe just send jobid).
ARGV[8] should fetch next job
ARGV[9] base key
ARGV[10] lock token
ARGV[11] lock duration in milliseconds
ARGV[12] maxMetricsSize
Output:
0 OK
-1 Missing key.
-2 Missing lock.
-3 - Job not in active set.
Events:
'completed/failed'
]]
local rcall = redis.call
-- Includes
--- @include "includes/collectMetrics"
--- @include "includes/removeLock"
--- @include "includes/removeDebounceKeyIfNeeded"
if rcall("EXISTS", KEYS[3]) == 1 then -- // Make sure job exists
local errorCode = removeLock(KEYS[3], KEYS[8], ARGV[5], ARGV[1])
if errorCode < 0 then
return errorCode
end
-- Remove from active list (if not active we shall return error)
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[1])
if numRemovedElements < 1 then return -3 end
local debounceId = rcall("HGET", KEYS[3], "deid")
removeDebounceKeyIfNeeded(ARGV[9], debounceId)
-- Remove job?
local keepJobs = cmsgpack.unpack(ARGV[6])
local maxCount = keepJobs['count']
local maxAge = keepJobs['age']
local targetSet = KEYS[2]
local timestamp = ARGV[2]
if maxCount ~= 0 then
-- Add to complete/failed set
rcall("ZADD", targetSet, timestamp, ARGV[1])
rcall("HMSET", KEYS[3], ARGV[3], ARGV[4], "finishedOn", timestamp) -- "returnvalue" / "failedReason" and "finishedOn"
local function removeJobs(jobIds)
for i, jobId in ipairs(jobIds) do
local jobKey = ARGV[9] .. jobId
local jobLogKey = jobKey .. ':logs'
rcall("DEL", jobKey, jobLogKey)
end
end
-- Remove old jobs?
if maxAge ~= nil then
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
removeJobs(jobIds)
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
if maxCount ~= nil and maxCount > 0 then
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
removeJobs(jobIds)
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1));
end
else
local jobLogKey = KEYS[3] .. ':logs'
rcall("DEL", KEYS[3], jobLogKey)
end
-- Collect metrics
if ARGV[12] ~= "" then
collectMetrics(KEYS[9], KEYS[9]..':data', ARGV[12], timestamp)
end
rcall("PUBLISH", targetSet, ARGV[7])
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
-- and not rate limited.
if (ARGV[8] == "1") then
-- move from wait to active
local jobId = rcall("RPOPLPUSH", KEYS[4], KEYS[1])
if jobId then
local jobKey = ARGV[9] .. jobId
local lockKey = jobKey .. ':lock'
-- get a lock
rcall("SET", lockKey, ARGV[11], "PX", ARGV[10])
rcall("ZREM", KEYS[5], jobId) -- remove from priority
rcall("PUBLISH", KEYS[6], jobId)
rcall("HSET", jobKey, "processedOn", ARGV[2])
return {rcall("HGETALL", jobKey), jobId} -- get job data
end
end
return 0
else
return -1
end
+108
View File
@@ -0,0 +1,108 @@
--[[
Completely obliterates a queue and all of its contents
Input:
KEYS[1] meta-paused
KEYS[2] base
ARGV[1] count
ARGV[2] force
]]
-- This command completely destroys a queue including all of its jobs, current or past
-- leaving no trace of its existence. Since this script needs to iterate to find all the job
-- keys, consider that this call may be slow for very large queues.
-- The queue needs to be "paused" or it will return an error
-- If the queue has currently active jobs then the script by default will return error,
-- however this behaviour can be overrided using the 'force' option.
local maxCount = tonumber(ARGV[1])
local baseKey = KEYS[2]
local rcall = redis.call
-- Includes
--- @include "includes/removeDebounceKey"
local function getListItems(keyName, max)
return rcall('LRANGE', keyName, 0, max - 1)
end
local function getZSetItems(keyName, max)
return rcall('ZRANGE', keyName, 0, max - 1)
end
local function removeJobs(baseKey, keys)
for i, key in ipairs(keys) do
local jobKey = baseKey .. key
rcall("DEL", jobKey, jobKey .. ':logs')
removeDebounceKey(baseKey, jobKey)
end
maxCount = maxCount - #keys
end
local function removeListJobs(keyName, max)
local jobs = getListItems(keyName, max)
removeJobs(baseKey, jobs)
rcall("LTRIM", keyName, #jobs, -1)
end
local function removeZSetJobs(keyName, max)
local jobs = getZSetItems(keyName, max)
removeJobs(baseKey, jobs)
if (#jobs > 0) then rcall("ZREM", keyName, unpack(jobs)) end
end
local function removeLockKeys(keys)
for i, key in ipairs(keys) do rcall("DEL", baseKey .. key .. ':lock') end
end
-- 1) Check if paused, if not return with error.
if rcall("EXISTS", KEYS[1]) ~= 1 then
return -1 -- Error, NotPaused
end
-- 2) Check if there are active jobs, if there are and not "force" return error.
local activeKey = baseKey .. 'active'
local activeJobs = getListItems(activeKey, maxCount)
if (#activeJobs > 0) then
if (ARGV[2] == "") then
return -2 -- Error, ExistsActiveJobs
end
end
removeLockKeys(activeJobs)
removeJobs(baseKey, activeJobs)
rcall("LTRIM", activeKey, #activeJobs, -1)
if (maxCount <= 0) then return 1 end
local waitKey = baseKey .. 'paused'
removeListJobs(waitKey, maxCount)
if (maxCount <= 0) then return 1 end
local delayedKey = baseKey .. 'delayed'
removeZSetJobs(delayedKey, maxCount)
if (maxCount <= 0) then return 1 end
local completedKey = baseKey .. 'completed'
removeZSetJobs(completedKey, maxCount)
if (maxCount <= 0) then return 1 end
local failedKey = baseKey .. 'failed'
removeZSetJobs(failedKey, maxCount)
if (maxCount <= 0) then return 1 end
if (maxCount > 0) then
rcall("DEL", baseKey .. 'priority')
rcall("DEL", baseKey .. 'stalled-check')
rcall("DEL", baseKey .. 'stalled')
rcall("DEL", baseKey .. 'meta-paused')
rcall("DEL", baseKey .. 'meta')
rcall("DEL", baseKey .. 'id')
rcall("DEL", baseKey .. 'repeat')
rcall("DEL", baseKey .. 'metrics:completed')
rcall("DEL", baseKey .. 'metrics:completed:data')
rcall("DEL", baseKey .. 'metrics:failed')
rcall("DEL", baseKey .. 'metrics:failed:data')
return 0
else
return 1
end
+35
View File
@@ -0,0 +1,35 @@
--[[
Pauses or resumes a queue globably.
Input:
KEYS[1] 'wait' or 'paused''
KEYS[2] 'paused' or 'wait'
KEYS[3] 'meta-paused'
KEYS[4] 'paused' o 'resumed' event.
KEYS[5] 'meta' this key is only used in BullMQ and above.
ARGV[1] 'paused' or 'resumed'
Event:
publish paused or resumed event.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then
rcall("RENAME", KEYS[1], KEYS[2])
end
if ARGV[1] == "paused" then
rcall("SET", KEYS[3], 1)
-- for forwards compatibility
rcall("HSET", KEYS[5], "paused", 1)
else
rcall("DEL", KEYS[3])
-- for forwards compatibility
rcall("HDEL", KEYS[5], "paused")
end
rcall("PUBLISH", KEYS[4], ARGV[1])
+45
View File
@@ -0,0 +1,45 @@
--[[
Promotes a job that is currently "delayed" to the "waiting" state
Input:
KEYS[1] 'delayed'
KEYS[2] 'wait'
KEYS[3] 'paused'
KEYS[4] 'meta-paused'
KEYS[5] 'priority'
ARGV[1] queue.toKey('')
ARGV[2] jobId
ARGV[3] queue token
Events:
'waiting'
]]
local rcall = redis.call;
local jobId = ARGV[2]
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
if rcall("ZREM", KEYS[1], jobId) == 1 then
local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", target, jobId)
else
addJobWithPriority(KEYS[5], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[2] .. "ing@" .. ARGV[3], jobId)
rcall("HSET", ARGV[1] .. jobId, "delay", 0)
return 0
else
return -1
end
+19
View File
@@ -0,0 +1,19 @@
--[[
Release lock
Input:
KEYS[1] 'lock',
ARGV[1] token
ARGV[2] lock duration in milliseconds
Output:
"OK" if lock extented succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
return rcall("DEL", KEYS[1])
else
return 0
end
+59
View File
@@ -0,0 +1,59 @@
--[[
Remove a job from all the queues it may be in as well as all its data.
In order to be able to remove a job, it must be unlocked.
Input:
KEYS[1] 'active',
KEYS[2] 'wait',
KEYS[3] 'delayed',
KEYS[4] 'paused',
KEYS[5] 'completed',
KEYS[6] 'failed',
KEYS[7] 'priority',
KEYS[8] jobId key
KEYS[9] job logs
KEYS[10] rate limiter index table
KEYS[11] prefix key
ARGV[1] jobId
ARGV[2] lock token
Events:
'removed'
]]
-- TODO PUBLISH global event 'removed'
local rcall = redis.call
-- Includes
--- @include "includes/removeDebounceKey"
local lockKey = KEYS[8] .. ':lock'
local lock = rcall("GET", lockKey)
if not lock then -- or (lock == ARGV[2])) then
local jobId = ARGV[1]
rcall("LREM", KEYS[1], 0, jobId)
rcall("LREM", KEYS[2], 0, jobId)
rcall("ZREM", KEYS[3], jobId)
rcall("LREM", KEYS[4], 0, jobId)
rcall("ZREM", KEYS[5], jobId)
rcall("ZREM", KEYS[6], jobId)
rcall("ZREM", KEYS[7], jobId)
removeDebounceKey(KEYS[11], KEYS[8])
rcall("DEL", KEYS[8])
rcall("DEL", KEYS[9])
-- delete keys related to rate limiter
local limiterIndexTable = KEYS[10] .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
return 1
else
return 0
end
+61
View File
@@ -0,0 +1,61 @@
--[[
Remove all jobs matching a given pattern from all the queues they may be in as well as all its data.
In order to be able to remove any job, they must be unlocked.
Input:
KEYS[1] 'active',
KEYS[2] 'wait',
KEYS[3] 'delayed',
KEYS[4] 'paused',
KEYS[5] 'completed',
KEYS[6] 'failed',
KEYS[7] 'priority',
KEYS[8] 'rate-limiter'
ARGV[1] prefix
ARGV[2] pattern
ARGV[3] cursor
Events:
'removed'
]]
-- TODO PUBLISH global events 'removed'
local rcall = redis.call
local result = rcall("SCAN", ARGV[3], "MATCH", ARGV[1] .. ARGV[2])
local cursor = result[1];
local jobKeys = result[2];
local removed = {}
local prefixLen = string.len(ARGV[1]) + 1
for i, jobKey in ipairs(jobKeys) do
local keyTypeResp = rcall("TYPE", jobKey)
if keyTypeResp["ok"] == "hash" then
local jobId = string.sub(jobKey, prefixLen)
local lockKey = jobKey .. ':lock'
local lock = redis.call("GET", lockKey)
if not lock then
rcall("LREM", KEYS[1], 0, jobId)
rcall("LREM", KEYS[2], 0, jobId)
rcall("ZREM", KEYS[3], jobId)
rcall("LREM", KEYS[4], 0, jobId)
rcall("ZREM", KEYS[5], jobId)
rcall("ZREM", KEYS[6], jobId)
rcall("ZREM", KEYS[7], jobId)
rcall("DEL", jobKey)
rcall("DEL", jobKey .. ':logs')
-- delete keys related to rate limiter
local limiterIndexTable = KEYS[8] .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
table.insert(removed, jobId)
end
end
end
return {cursor, removed}
+22
View File
@@ -0,0 +1,22 @@
--[[
Removes a repeatable job
Input:
KEYS[1] repeat jobs key
KEYS[2] delayed jobs key
ARGV[1] repeat job id
ARGV[2] repeat job key
ARGV[3] queue key
]]
local millis = redis.call("ZSCORE", KEYS[1], ARGV[2])
if(millis) then
-- Delete next programmed job.
local repeatJobId = ARGV[1] .. millis
if(redis.call("ZREM", KEYS[2], repeatJobId) == 1) then
redis.call("DEL", ARGV[3] .. repeatJobId)
end
end
redis.call("ZREM", KEYS[1], ARGV[2]);
+52
View File
@@ -0,0 +1,52 @@
--[[
Attempts to reprocess a job
Input:
KEYS[1] job key
KEYS[2] job lock key
KEYS[3] job state
KEYS[4] wait key
KEYS[5] meta-pause
KEYS[6] paused key
ARGV[1] job.id,
ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
ARGV[3] token
ARGV[4] timestamp
Output:
1 means the operation was a success
0 means the job does not exist
-1 means the job is currently locked and can't be retried.
-2 means the job was not found in the expected set.
]]
local rcall = redis.call;
if (rcall("EXISTS", KEYS[1]) == 1) then
if (rcall("EXISTS", KEYS[2]) == 0) then
rcall("HDEL", KEYS[1], "finishedOn", "processedOn", "failedReason")
rcall("HSET", KEYS[1], "retriedOn", ARGV[4])
if (rcall("ZREM", KEYS[3], ARGV[1]) == 1) then
local target
if rcall("EXISTS", KEYS[5]) ~= 1 then
target = KEYS[4]
else
target = KEYS[6]
end
rcall(ARGV[2], target, ARGV[1])
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[4] .. "ing@" .. ARGV[3], ARGV[1])
return 1
else
return -2
end
else
return -1
end
else
return 0
end
+56
View File
@@ -0,0 +1,56 @@
--[[
Retries a failed job by moving it back to the wait queue.
Input:
KEYS[1] 'active',
KEYS[2] 'wait'
KEYS[3] jobId key
KEYS[4] 'meta-paused'
KEYS[5] 'paused'
KEYS[6] stalled key
KEYS[7] 'priority'
ARGV[1] pushCmd
ARGV[2] jobId
ARGV[3] token
Events:
'prefix:added'
Output:
0 - OK
-1 - Missing key
-2 - Job Not locked
-3 - Job not in active set
]]
local rcall = redis.call
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
--- @include "includes/removeLock"
if rcall("EXISTS", KEYS[3]) == 1 then
local errorCode = removeLock(KEYS[3], KEYS[6], ARGV[3], ARGV[2])
if errorCode < 0 then
return errorCode
end
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[2])
if numRemovedElements < 1 then return -3 end
local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[5])
local priority = tonumber(rcall("HGET", KEYS[3], "priority")) or 0
if priority == 0 then
-- LIFO or FIFO
rcall(ARGV[1], target, ARGV[2])
else
addJobWithPriority(KEYS[7], priority, ARGV[2], target)
end
return 0
else
return -1
end
+54
View File
@@ -0,0 +1,54 @@
--[[
Attempts to retry all failed jobs
Input:
KEYS[1] base key
KEYS[2] failed state key
KEYS[3] wait state key
KEYS[4] 'meta-paused'
KEYS[5] 'paused'
ARGV[1] count
Output:
1 means the operation is not completed
0 means the operation is completed
]]
local baseKey = KEYS[1]
local maxCount = tonumber(ARGV[1])
local rcall = redis.call;
-- Includes
--- @include "includes/batches"
local function getZSetItems(keyName, max)
return rcall('ZRANGE', keyName, 0, max - 1)
end
local jobs = getZSetItems(KEYS[2], maxCount)
if (#jobs > 0) then
for i, key in ipairs(jobs) do
local jobKey = baseKey .. key
rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason")
end
local target
if rcall("EXISTS", KEYS[4]) ~= 1 then
target = KEYS[3]
else
target = KEYS[5]
end
for from, to in batches(#jobs, 7000) do
rcall("ZREM", KEYS[2], unpack(jobs, from, to))
rcall("LPUSH", target, unpack(jobs, from, to))
end
end
maxCount = maxCount - #jobs
if (maxCount <= 0) then return 1 end
return 0
+24
View File
@@ -0,0 +1,24 @@
--[[
Save stacktrace and failedReason.
Input:
KEYS[1] job key
ARGV[1] stacktrace
ARGV[2] failedReason
ARGV[3] attemptsMade
Output:
0 - OK
-1 - Missing key
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then
rcall("HMSET", KEYS[1], "stacktrace", ARGV[1], "failedReason", ARGV[2],
"attemptsMade", ARGV[3])
return 0
else
return -1
end
+17
View File
@@ -0,0 +1,17 @@
--[[
Takes a lock
Input:
KEYS[1] 'lock',
ARGV[1] token
ARGV[2] lock duration in milliseconds
Output:
"OK" if lock taken successfully.
]]
if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", ARGV[2]) then
return 1
else
return 0
end
+20
View File
@@ -0,0 +1,20 @@
--[[
Update job data
Input:
KEYS[1] Job id key
ARGV[1] data
Output:
0 - OK
-1 - Missing job.
]]
local rcall = redis.call
if rcall("EXISTS",KEYS[1]) == 1 then -- // Make sure job exists
rcall("HSET", KEYS[1], "data", ARGV[1])
return 0
else
return -1
end
+59
View File
@@ -0,0 +1,59 @@
--[[
Updates the delay set, by picking a delayed job that should
be processed now.
Input:
KEYS[1] 'delayed'
KEYS[2] 'active'
KEYS[3] 'wait'
KEYS[4] 'priority'
KEYS[5] 'paused'
KEYS[6] 'meta-paused'
ARGV[1] queue.toKey('')
ARGV[2] delayed timestamp
ARGV[3] queue token
Events:
'removed'
]]
local rcall = redis.call;
-- Includes
--- @include "includes/addJobWithPriority"
--- @include "includes/getTargetQueueList"
-- Try to get as much as 1000 jobs at once
local jobs = rcall("ZRANGEBYSCORE", KEYS[1], 0, tonumber(ARGV[2]) * 0x1000, "LIMIT", 0, 1000)
if(#jobs > 0) then
rcall("ZREM", KEYS[1], unpack(jobs))
-- check if we need to use push in paused instead of waiting
local target = getTargetQueueList(KEYS[6], KEYS[3], KEYS[5])
for _, jobId in ipairs(jobs) do
-- Is this really needed?
rcall("LREM", KEYS[2], 0, jobId)
local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", target, jobId)
else
addJobWithPriority(KEYS[4], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[3] .. "ing@" .. ARGV[3], jobId)
rcall("HSET", ARGV[1] .. jobId, "delay", 0)
end
end
local nextTimestamp = rcall("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")[2]
if(nextTimestamp ~= nil) then
rcall("PUBLISH", KEYS[1], nextTimestamp / 0x1000)
end
return nextTimestamp
+21
View File
@@ -0,0 +1,21 @@
--[[
Update job progress
Input:
KEYS[1] Job id key
KEYS[2] progress event key
ARGV[1] progress
ARGV[2] event data
Event:
progress(jobId, progress)
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
rcall("HSET", KEYS[1], "progress", ARGV[1])
rcall("PUBLISH", KEYS[2], ARGV[2])
return 0
else
return -1
end
+11
View File
@@ -0,0 +1,11 @@
'use strict';
module.exports.Messages = {
RETRY_JOB_NOT_EXIST: "Couldn't retry job: The job doesn't exist",
RETRY_JOB_IS_LOCKED: "Couldn't retry job: The job is locked",
RETRY_JOB_NOT_FAILED:
"Couldn't retry job: The job has been already retried or has not failed",
MISSING_REDIS_OPTS: `Using a redis instance with enableReadyCheck or maxRetriesPerRequest for bclient/subscriber is not permitted.
see https://github.com/OptimalBits/bull/issues/1873
`
};
+273
View File
@@ -0,0 +1,273 @@
'use strict';
const _ = require('lodash');
const Job = require('./job');
const scripts = require('./scripts');
module.exports = function(Queue) {
Queue.prototype.getJob = async function(jobId) {
await this.isReady();
return Job.fromId(this, jobId);
};
Queue.prototype.getCountsPerPriority = async function(priorities) {
const uniquePriorities = [...new Set(priorities)];
const responses = await scripts.getCountsPerPriority(
this,
uniquePriorities
);
const counts = {};
responses.forEach((res, index) => {
counts[`${uniquePriorities[index]}`] = res || 0;
});
return counts;
};
Queue.prototype._commandByType = function(types, count, callback) {
return _.map(types, type => {
type = type === 'waiting' ? 'wait' : type; // alias
const key = this.toKey(type);
switch (type) {
case 'completed':
case 'failed':
case 'delayed':
case 'repeat':
return callback(key, count ? 'zcard' : 'zrange');
case 'active':
case 'wait':
case 'paused':
return callback(key, count ? 'llen' : 'lrange');
}
});
};
/**
Returns the number of jobs waiting to be processed.
*/
Queue.prototype.count = function() {
return this.getJobCountByTypes('wait', 'paused', 'delayed');
};
// Job counts by type
// Queue#getJobCountByTypes('completed') => completed count
// Queue#getJobCountByTypes('completed,failed') => completed + failed count
// Queue#getJobCountByTypes('completed', 'failed') => completed + failed count
// Queue#getJobCountByTypes('completed,waiting', 'failed') => completed + waiting + failed count
Queue.prototype.getJobCountByTypes = function() {
return this.getJobCounts.apply(this, arguments).then(result => {
return _.chain(result)
.values()
.sum()
.value();
});
};
/**
* Returns the job counts for each type specified or every list/set in the queue by default.
*
*/
Queue.prototype.getJobCounts = function() {
const types = parseTypeArg(arguments);
const multi = this.multi();
this._commandByType(types, true, (key, command) => {
multi[command](key);
});
return multi.exec().then(res => {
const counts = {};
res.forEach((res, index) => {
counts[types[index]] = res[1] || 0;
});
return counts;
});
};
Queue.prototype.getCompletedCount = function() {
return this.getJobCountByTypes('completed');
};
Queue.prototype.getFailedCount = function() {
return this.getJobCountByTypes('failed');
};
Queue.prototype.getDelayedCount = function() {
return this.getJobCountByTypes('delayed');
};
Queue.prototype.getActiveCount = function() {
return this.getJobCountByTypes('active');
};
Queue.prototype.getWaitingCount = function() {
return this.getJobCountByTypes('wait', 'paused');
};
/**
*
* @returns the potential stalled jobs. Only useful for tests.
*/
Queue.prototype.getStalledCount = function() {
const key = this.toKey('stalled');
return this.client.scard(key);
};
// TO BE DEPRECATED --->
Queue.prototype.getPausedCount = function() {
return this.getJobCountByTypes('paused');
};
// <-----
Queue.prototype.getWaiting = function(start, end, opts) {
return this.getJobs(['wait', 'paused'], start, end, true, opts);
};
Queue.prototype.getActive = function(start, end, opts) {
return this.getJobs('active', start, end, true, opts);
};
Queue.prototype.getDelayed = function(start, end, opts) {
return this.getJobs('delayed', start, end, true, opts);
};
Queue.prototype.getCompleted = function(start, end, opts) {
return this.getJobs('completed', start, end, false, opts);
};
Queue.prototype.getFailed = function(start, end, opts) {
return this.getJobs('failed', start, end, false, opts);
};
Queue.prototype.getRanges = function(types, start, end, asc) {
start = _.isUndefined(start) ? 0 : start;
end = _.isUndefined(end) ? -1 : end;
const multi = this.multi();
const multiCommands = [];
this._commandByType(parseTypeArg(types), false, (key, command) => {
switch (command) {
case 'lrange':
if (asc) {
multiCommands.push('lrange');
multi.lrange(key, -(end + 1), -(start + 1));
} else {
multi.lrange(key, start, end);
}
break;
case 'zrange':
multiCommands.push('zrange');
if (asc) {
multi.zrange(key, start, end);
} else {
multi.zrevrange(key, start, end);
}
break;
}
});
return multi.exec().then(responses => {
let results = [];
responses.forEach((response, index) => {
const result = response[1] || [];
if (asc && multiCommands[index] === 'lrange') {
results = results.concat(result.reverse());
} else {
results = results.concat(result);
}
});
return results;
});
};
Queue.prototype.getJobs = function(types, start, end, asc, opts) {
return this.getRanges(types, start, end, asc).then(jobIds => {
return Promise.all(jobIds.map(jobId => this.getJobFromId(jobId, opts)));
});
};
Queue.prototype.getJobLogs = function(jobId, start, end, asc = true) {
start = _.isUndefined(start) ? 0 : start;
end = _.isUndefined(end) ? -1 : end;
const multi = this.multi();
const logsKey = this.toKey(jobId + ':logs');
if (asc) {
multi.lrange(logsKey, start, end);
} else {
multi.lrange(logsKey, -(end + 1), -(start + 1));
}
multi.llen(logsKey);
return multi.exec().then(result => {
if (!asc) {
result[0][1].reverse();
}
return {
logs: result[0][1],
count: result[1][1]
};
});
};
/**
* Get queue metrics related to the queue.
*
* This method returns the gathered metrics for the queue.
* The metrics are represented as an array of job counts
* per unit of time (1 minute).
*
* @param start - Start point of the metrics, where 0
* is the newest point to be returned.
* @param end - End poinf of the metrics, where -1 is the
* oldest point to be returned.
*
* @returns - Returns an object with queue metrics.
*/
Queue.prototype.getMetrics = async function(type, start = 0, end = -1) {
const metricsKey = this.toKey(`metrics:${type}`);
const dataKey = `${metricsKey}:data`;
const multi = this.multi();
multi.hmget(metricsKey, 'count', 'prevTS', 'prevCount');
multi.lrange(dataKey, start, end);
multi.llen(dataKey);
const [hmget, range, len] = await multi.exec();
const [err, [count, prevTS, prevCount]] = hmget;
const [err2, data] = range;
const [err3, numPoints] = len;
if (err || err2) {
throw err || err2 || err3;
}
return {
meta: {
count: parseInt(count || '0', 10),
prevTS: parseInt(prevTS || '0', 10),
prevCount: parseInt(prevCount || '0', 10)
},
data,
count: numPoints
};
};
};
function parseTypeArg(args) {
const types = _.chain([])
.concat(args)
.join(',')
.split(/\s*,\s*/g)
.compact()
.value();
return types.length
? types
: ['waiting', 'active', 'completed', 'failed', 'delayed', 'paused'];
}
+673
View File
@@ -0,0 +1,673 @@
'use strict';
const _ = require('lodash');
const utils = require('./utils');
const scripts = require('./scripts');
const debuglog = require('util').debuglog('bull');
const errors = require('./errors');
const backoffs = require('./backoffs');
const FINISHED_WATCHDOG = 5000;
const DEFAULT_JOB_NAME = '__default__';
/**
interface JobOptions
{
priority: Priority;
attempts: number;
delay: number;
}
*/
const jobFields = [
'opts',
'name',
'id',
'progress',
'delay',
'timestamp',
'finishedOn',
'processedOn',
'retriedOn',
'failedReason',
'attemptsMade',
'stacktrace',
'returnvalue'
];
// queue: Queue, data: {}, opts: JobOptions
const Job = function(queue, name, data, opts) {
if (typeof name !== 'string') {
opts = data;
data = name;
name = DEFAULT_JOB_NAME;
}
// defaults
this.opts = setDefaultOpts(opts);
this.name = name;
this.queue = queue;
this.data = data;
this._progress = 0;
this.delay = this.opts.delay < 0 ? 0 : this.opts.delay;
this.timestamp = this.opts.timestamp;
this.stacktrace = [];
this.returnvalue = null;
this.attemptsMade = 0;
this.toKey = _.bind(queue.toKey, queue);
this.debounceId = this.opts.debounce ? this.opts.debounce.id : undefined;
};
function setDefaultOpts(opts) {
const _opts = Object.assign({}, opts);
_opts.attempts = typeof _opts.attempts == 'undefined' ? 1 : _opts.attempts;
_opts.delay = typeof _opts.delay == 'undefined' ? 0 : Number(_opts.delay);
_opts.timestamp =
typeof _opts.timestamp == 'undefined' ? Date.now() : _opts.timestamp;
_opts.attempts = parseInt(_opts.attempts);
_opts.backoff = backoffs.normalize(_opts.backoff);
return _opts;
}
Job.DEFAULT_JOB_NAME = DEFAULT_JOB_NAME;
function addJob(queue, client, job) {
const opts = job.opts;
const jobData = job.toData();
return scripts.addJob(client, queue, jobData, {
lifo: opts.lifo,
customJobId: opts.jobId,
priority: opts.priority,
debounce: opts.debounce
});
}
Job.create = function(queue, name, data, opts) {
const job = new Job(queue, name, data, opts);
return queue
.isReady()
.then(() => {
return addJob(queue, queue.client, job);
})
.then(jobId => {
job.id = jobId;
debuglog('Job added', jobId);
return job;
});
};
Job.createBulk = function(queue, jobs) {
jobs = jobs.map(job => new Job(queue, job.name, job.data, job.opts));
return queue
.isReady()
.then(() => {
const multi = queue.client.multi();
for (const job of jobs) {
addJob(queue, multi, job);
}
return multi.exec();
})
.then(res => {
res.forEach((res, index) => {
jobs[index].id = res[1];
debuglog('Job added', res[1]);
});
return jobs;
});
};
Job.fromId = async function(queue, jobId, opts) {
// jobId can be undefined if moveJob returns undefined
if (!jobId) {
return Promise.resolve();
}
const jobKey = queue.toKey(jobId);
let rawJob;
if (opts && opts.excludeData) {
rawJob = _.zipObject(
jobFields,
await queue.client.hmget(jobKey, jobFields)
);
} else {
rawJob = await queue.client.hgetall(jobKey);
}
return _.isEmpty(rawJob) ? null : Job.fromJSON(queue, rawJob, jobId);
};
Job.remove = async function(queue, pattern) {
await queue.isReady();
const removed = await scripts.removeWithPattern(queue, pattern);
removed.forEach(jobId => queue.emit('removed', jobId));
};
Job.prototype.progress = function(progress) {
if (_.isUndefined(progress)) {
return this._progress;
}
this._progress = progress;
return scripts.updateProgress(this, progress);
};
Job.prototype.update = async function(data) {
this.data = data;
const code = await scripts.updateData(this, data);
if (code < 0) {
throw scripts.finishedErrors(code, this.id, 'updateData');
}
};
Job.prototype.toJSON = function() {
const opts = Object.assign({}, this.opts);
return {
id: this.id,
name: this.name,
data: this.data || {},
opts: opts,
progress: this._progress,
delay: this.delay, // Move to opts
timestamp: this.timestamp,
attemptsMade: this.attemptsMade,
failedReason: this.failedReason,
stacktrace: this.stacktrace || null,
returnvalue: this.returnvalue || null,
debounceId: this.debounceId || null,
finishedOn: this.finishedOn || null,
processedOn: this.processedOn || null
};
};
Job.prototype.toData = function() {
const json = this.toJSON();
json.data = JSON.stringify(json.data);
json.opts = JSON.stringify(json.opts);
json.stacktrace = JSON.stringify(json.stacktrace);
json.failedReason = JSON.stringify(json.failedReason);
json.returnvalue = JSON.stringify(json.returnvalue);
return json;
};
/**
Return a unique key representing a lock for this Job
*/
Job.prototype.lockKey = function() {
return this.toKey(this.id) + ':lock';
};
/**
Takes a lock for this job so that no other queue worker can process it at the
same time.
*/
Job.prototype.takeLock = function() {
return scripts.takeLock(this.queue, this).then(lock => {
return lock || false;
});
};
/**
Releases the lock. Only locks owned by the queue instance can be released.
*/
Job.prototype.releaseLock = function() {
return scripts.releaseLock(this.queue, this.id).then(unlocked => {
if (unlocked != 1) {
throw new Error('Could not release lock for job ' + this.id);
}
});
};
/**
* Extend the lock for this job.
*
* @param duration lock duration in milliseconds
*/
Job.prototype.extendLock = function(duration) {
return scripts.extendLock(this.queue, this.id, duration);
};
/**
* Moves a job to the completed queue.
* Returned job to be used with Queue.prototype.nextJobFromJobData.
* @param returnValue {string} The jobs success message.
* @param ignoreLock {boolean} True when wanting to ignore the redis lock on this job.
* @param notFetch {boolean} True when should not fetch next job from queue.
* @returns {Promise} Returns the jobData of the next job in the waiting queue.
*/
Job.prototype.moveToCompleted = function(
returnValue,
ignoreLock,
notFetch = false
) {
return this.queue.isReady().then(() => {
this.returnvalue = returnValue || 0;
returnValue = utils.tryCatch(JSON.stringify, JSON, [returnValue]);
if (returnValue === utils.errorObject) {
const err = utils.errorObject.value;
return Promise.reject(err);
}
this.finishedOn = Date.now();
return scripts.moveToCompleted(
this,
returnValue,
this.opts.removeOnComplete,
ignoreLock,
notFetch
);
});
};
Job.prototype.discard = function() {
this._discarded = true;
};
/**
* Moves a job to the failed queue.
* @param err {string} The jobs error message.
* @param ignoreLock {boolean} True when wanting to ignore the redis lock on this job.
* @returns void
*/
Job.prototype.moveToFailed = async function(err, ignoreLock) {
err = err || { message: 'Unknown reason' };
this.failedReason = err.message;
await this.queue.isReady();
let command;
const multi = this.queue.client.multi();
this._saveAttempt(multi, err);
// Check if an automatic retry should be performed
let moveToFailed = false;
if (this.attemptsMade < this.opts.attempts && !this._discarded) {
// Check if backoff is needed
const delay = await backoffs.calculate(
this.opts.backoff,
this.attemptsMade,
this.queue.settings.backoffStrategies,
err,
_.get(this, 'opts.backoff.options', null)
);
if (delay === -1) {
// If delay is -1, we should no continue retrying
moveToFailed = true;
} else if (delay) {
// If so, move to delayed (need to unlock job in this case!)
const args = scripts.moveToDelayedArgs(
this.queue,
this.id,
Date.now() + delay,
ignoreLock
);
multi.moveToDelayed(args);
command = 'delayed';
} else {
// If not, retry immediately
multi.retryJob(scripts.retryJobArgs(this, ignoreLock));
command = 'retry';
}
} else {
// If not, move to failed
moveToFailed = true;
}
if (moveToFailed) {
this.finishedOn = Date.now();
const args = scripts.moveToFailedArgs(
this,
err.message,
this.opts.removeOnFail,
ignoreLock
);
multi.moveToFinished(args);
command = 'failed';
}
const results = await multi.exec();
const code = _.last(results)[1];
if (code < 0) {
throw scripts.finishedErrors(code, this.id, command, 'active');
}
};
Job.prototype.moveToDelayed = function(timestamp, ignoreLock) {
return scripts.moveToDelayed(this.queue, this.id, timestamp, ignoreLock);
};
Job.prototype.promote = function() {
const queue = this.queue;
const jobId = this.id;
return queue.isReady().then(() =>
scripts.promote(queue, jobId).then(result => {
if (result === -1) {
throw new Error('Job ' + jobId + ' is not in a delayed state');
}
})
);
};
/**
* Attempts to retry the job. Only a job that has failed can be retried.
*
* @return {Promise} If resolved and return code is 1, then the queue emits a waiting event
* otherwise the operation was not a success and throw the corresponding error. If the promise
* rejects, it indicates that the script failed to execute
*/
Job.prototype.retry = function() {
return this.queue.isReady().then(() => {
this.failedReason = null;
this.finishedOn = null;
this.processedOn = null;
this.retriedOn = Date.now();
return scripts.reprocessJob(this, { state: 'failed' }).then(result => {
if (result === 1) {
return;
} else if (result === 0) {
throw new Error(errors.Messages.RETRY_JOB_NOT_EXIST);
} else if (result === -1) {
throw new Error(errors.Messages.RETRY_JOB_IS_LOCKED);
} else if (result === -2) {
throw new Error(errors.Messages.RETRY_JOB_NOT_FAILED);
}
});
});
};
/**
* Logs one row of log data.
*
* @params logRow: string String with log data to be logged.
*
*/
Job.prototype.log = function(logRow) {
return scripts.addLog(this.queue, this.id, logRow);
};
Job.prototype.isCompleted = function() {
return this._isDone('completed');
};
Job.prototype.isFailed = function() {
return this._isDone('failed');
};
Job.prototype.isDelayed = function() {
return this._isDone('delayed');
};
Job.prototype.isActive = function() {
return this._isInList('active');
};
Job.prototype.isWaiting = function() {
return this._isInList('wait');
};
Job.prototype.isPaused = function() {
return this._isInList('paused');
};
Job.prototype.isStuck = function() {
return this.getState().then(state => {
return state === 'stuck';
});
};
Job.prototype.isDiscarded = function() {
return this._discarded;
};
Job.prototype.getState = function() {
const fns = [
{ fn: 'isCompleted', state: 'completed' },
{ fn: 'isFailed', state: 'failed' },
{ fn: 'isDelayed', state: 'delayed' },
{ fn: 'isActive', state: 'active' },
{ fn: 'isWaiting', state: 'waiting' },
{ fn: 'isPaused', state: 'paused' }
];
return fns
.reduce((result, fn) => {
return result.then(state => {
if (state) {
return state;
}
return this[fn.fn]().then(result => {
return result ? fn.state : null;
});
});
}, Promise.resolve())
.then(result => {
return result ? result : 'stuck';
});
};
Job.prototype.remove = function() {
const queue = this.queue;
const job = this;
return queue.isReady().then(() => {
return scripts.remove(queue, job.id).then(removed => {
if (removed) {
queue.emit('removed', job);
} else {
throw new Error('Could not remove job ' + job.id);
}
});
});
};
/**
* Returns a promise the resolves when the job has finished. (completed or failed).
*/
Job.prototype.finished = async function() {
await Promise.all([
this.queue._registerEvent('global:completed'),
this.queue._registerEvent('global:failed')
]);
await this.queue.isReady();
const status = await scripts.isFinished(this);
const finished = status > 0;
if (finished) {
const job = await Job.fromId(this.queue, this.id);
if (status == 2) {
throw new Error(job.failedReason);
} else {
return job.returnvalue;
}
} else {
return new Promise((resolve, reject) => {
const onCompleted = (jobId, resultValue) => {
if (String(jobId) === String(this.id)) {
let result = void 0;
try {
if (typeof resultValue === 'string') {
result = JSON.parse(resultValue);
}
} catch (err) {
//swallow exception because the resultValue got corrupted somehow.
debuglog('corrupted resultValue: ' + resultValue, err);
}
resolve(result);
removeListeners();
}
};
const onFailed = (jobId, failedReason) => {
if (String(jobId) === String(this.id)) {
reject(new Error(failedReason));
removeListeners();
}
};
this.queue.on('global:completed', onCompleted);
this.queue.on('global:failed', onFailed);
const removeListeners = () => {
clearInterval(interval);
this.queue.removeListener('global:completed', onCompleted);
this.queue.removeListener('global:failed', onFailed);
};
//
// Watchdog
//
const interval = setInterval(() => {
if (this._isQueueClosing()) {
removeListeners();
// TODO(manast) maybe we would need a more graceful way to get out of this interval.
reject(
new Error('cannot check if job is finished in a closing queue.')
);
} else {
scripts.isFinished(this).then(status => {
const finished = status > 0;
if (finished) {
Job.fromId(this.queue, this.id).then(job => {
removeListeners();
if (status == 2) {
reject(new Error(job.failedReason));
} else {
resolve(job.returnvalue);
}
});
}
});
}
}, FINISHED_WATCHDOG);
});
}
};
// -----------------------------------------------------------------------------
// Private methods
// -----------------------------------------------------------------------------
Job.prototype._isQueueClosing = function() {
return this.queue.closing;
};
Job.prototype._isDone = function(list) {
return this.queue.client
.zscore(this.queue.toKey(list), this.id)
.then(score => {
return score !== null;
});
};
Job.prototype._isInList = function(list) {
return scripts.isJobInList(
this.queue.client,
this.queue.toKey(list),
this.id
);
};
Job.prototype._saveAttempt = function(multi, err) {
this.attemptsMade++;
this.stacktrace = this.stacktrace || [];
if (err && err.stack) {
this.stacktrace.push(err.stack);
if (this.opts.stackTraceLimit) {
this.stacktrace = this.stacktrace.slice(-this.opts.stackTraceLimit);
}
}
const args = scripts.saveStacktraceArgs(
this,
JSON.stringify(this.stacktrace),
err && err.message,
);
multi.saveStacktrace(args);
};
Job.fromJSON = function(queue, json, jobId) {
const opts = JSON.parse(json.opts || '{}');
const data = opts.preventParsingData
? json.data
: JSON.parse(json.data || '{}');
const job = new Job(queue, json.name || Job.DEFAULT_JOB_NAME, data, opts);
job.id = json.id || jobId;
try {
job._progress = JSON.parse(json.progress || 0);
} catch (err) {
console.error(
`Error parsing progress ${json.progress} with ${err.message}`
);
}
job.delay = parseInt(json.delay);
job.timestamp = parseInt(json.timestamp);
if (json.finishedOn) {
job.finishedOn = parseInt(json.finishedOn);
}
if (json.processedOn) {
job.processedOn = parseInt(json.processedOn);
}
if (json.retriedOn) {
job.retriedOn = parseInt(json.retriedOn);
}
job.failedReason = json.failedReason;
job.attemptsMade = parseInt(json.attemptsMade || 0);
job.stacktrace = getTraces(json.stacktrace);
if (typeof json.returnvalue === 'string') {
job.returnvalue = getReturnValue(json.returnvalue);
}
if (json.deid) {
job.debounceId = json.deid;
}
return job;
};
function getTraces(stacktrace) {
const _traces = utils.tryCatch(JSON.parse, JSON, [stacktrace]);
if (_traces === utils.errorObject || !(_traces instanceof Array)) {
return [];
} else {
return _traces;
}
}
function getReturnValue(_value) {
const value = utils.tryCatch(JSON.parse, JSON, [_value]);
if (value !== utils.errorObject) {
return value;
} else {
debuglog('corrupted returnvalue: ' + _value, value);
}
}
module.exports = Job;
+84
View File
@@ -0,0 +1,84 @@
// Extracted from p-timeout https://github.com/sindresorhus/p-timeout
// as it is not commonjs compatible. This is version 5.0.2
'use strict';
class TimeoutError extends Error {
constructor(message) {
super(message);
this.name = 'TimeoutError';
}
}
module.exports.TimeoutError = TimeoutError;
module.exports.pTimeout = function pTimeout(
promise,
milliseconds,
fallback,
options
) {
let timer;
const cancelablePromise = new Promise((resolve, reject) => {
if (typeof milliseconds !== 'number' || Math.sign(milliseconds) !== 1) {
throw new TypeError(
`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``
);
}
if (milliseconds === Number.POSITIVE_INFINITY) {
resolve(promise);
return;
}
options = {
customTimers: { setTimeout, clearTimeout },
...options
};
timer = options.customTimers.setTimeout.call(
undefined,
() => {
if (typeof fallback === 'function') {
try {
resolve(fallback());
} catch (error) {
reject(error);
}
return;
}
const message =
typeof fallback === 'string'
? fallback
: `Promise timed out after ${milliseconds} milliseconds`;
const timeoutError =
fallback instanceof Error ? fallback : new TimeoutError(message);
if (typeof promise.cancel === 'function') {
promise.cancel();
}
reject(timeoutError);
},
milliseconds
);
(async () => {
try {
resolve(await promise);
} catch (error) {
reject(error);
} finally {
options.customTimers.clearTimeout.call(undefined, timer);
}
})();
});
cancelablePromise['clear'] = () => {
clearTimeout(timer);
timer = undefined;
};
return cancelablePromise;
};
+144
View File
@@ -0,0 +1,144 @@
'use strict';
const fork = require('child_process').fork;
const path = require('path');
const _ = require('lodash');
const getPort = require('get-port');
const { killAsync } = require('./utils');
const CHILD_KILL_TIMEOUT = 30000;
const ChildPool = function ChildPool() {
if (!(this instanceof ChildPool)) {
return new ChildPool();
}
this.retained = {};
this.free = {};
};
const convertExecArgv = function(execArgv) {
const standard = [];
const promises = [];
_.forEach(execArgv, arg => {
if (arg.indexOf('--inspect') === -1) {
standard.push(arg);
} else {
const argName = arg.split('=')[0];
promises.push(
getPort().then(port => {
return `${argName}=${port}`;
})
);
}
});
return Promise.all(promises).then(convertedArgs => {
return standard.concat(convertedArgs);
});
};
ChildPool.prototype.retain = function(processFile) {
const _this = this;
let child = _this.getFree(processFile).pop();
if (child) {
_this.retained[child.pid] = child;
return Promise.resolve(child);
}
return convertExecArgv(process.execArgv).then(execArgv => {
child = fork(path.join(__dirname, './master.js'), {
execArgv
});
child.processFile = processFile;
_this.retained[child.pid] = child;
child.on('exit', _this.remove.bind(_this, child));
return initChild(child, child.processFile)
.then(() => {
return child;
})
.catch(err => {
this.remove(child);
throw err;
});
});
};
ChildPool.prototype.release = function(child) {
delete this.retained[child.pid];
this.getFree(child.processFile).push(child);
};
ChildPool.prototype.remove = function(child) {
delete this.retained[child.pid];
const free = this.getFree(child.processFile);
const childIndex = free.indexOf(child);
if (childIndex > -1) {
free.splice(childIndex, 1);
}
};
ChildPool.prototype.kill = function(child, signal) {
this.remove(child);
return killAsync(child, signal || 'SIGKILL', CHILD_KILL_TIMEOUT);
};
ChildPool.prototype.clean = function() {
const children = _.values(this.retained).concat(this.getAllFree());
this.retained = {};
this.free = {};
const allKillPromises = [];
children.forEach(child => {
allKillPromises.push(this.kill(child, 'SIGTERM'));
});
return Promise.all(allKillPromises).then(() => {});
};
ChildPool.prototype.getFree = function(id) {
return (this.free[id] = this.free[id] || []);
};
ChildPool.prototype.getAllFree = function() {
return _.flatten(_.values(this.free));
};
async function initChild(child, processFile) {
const onComplete = new Promise((resolve, reject) => {
const onMessageHandler = msg => {
if (msg.cmd === 'init-complete') {
resolve();
} else if (msg.cmd === 'error') {
reject(msg.error);
}
child.off('message', onMessageHandler);
};
child.on('message', onMessageHandler);
});
await new Promise(resolve =>
child.send({ cmd: 'init', value: processFile }, resolve)
);
await onComplete;
}
function ChildPoolSingleton(isSharedChildPool = false) {
if (isSharedChildPool === false) {
return new ChildPool();
} else if (
!(this instanceof ChildPool) &&
ChildPoolSingleton.instance === undefined
) {
ChildPoolSingleton.instance = new ChildPool();
}
return ChildPoolSingleton.instance;
}
module.exports = ChildPoolSingleton;
+200
View File
@@ -0,0 +1,200 @@
/**
* Master of child processes. Handles communication between the
* processor and the main process.
*
*/
'use strict';
let status;
let processor;
let currentJobPromise;
const { promisify } = require('util');
const { asyncSend } = require('./utils');
// https://stackoverflow.com/questions/18391212/is-it-not-possible-to-stringify-an-error-using-json-stringify
if (!('toJSON' in Error.prototype)) {
Object.defineProperty(Error.prototype, 'toJSON', {
value: function() {
const alt = {};
Object.getOwnPropertyNames(this).forEach(function(key) {
alt[key] = this[key];
}, this);
return alt;
},
configurable: true,
writable: true
});
}
async function waitForCurrentJobAndExit() {
status = 'TERMINATING';
try {
await currentJobPromise;
} finally {
// it's an exit handler
// eslint-disable-next-line no-process-exit
process.exit(process.exitCode || 0);
}
}
process.on('SIGTERM', waitForCurrentJobAndExit);
process.on('SIGINT', waitForCurrentJobAndExit);
process.on('message', msg => {
switch (msg.cmd) {
case 'init':
try {
processor = require(msg.value);
} catch (err) {
status = 'Errored';
err.message = `Error loading process file ${msg.value}. ${err.message}`;
return process.send({
cmd: 'error',
error: err
});
}
if (processor.default) {
// support es2015 module.
processor = processor.default;
}
if (processor.length > 1) {
processor = promisify(processor);
} else {
const origProcessor = processor;
processor = function() {
try {
return Promise.resolve(origProcessor.apply(null, arguments));
} catch (err) {
return Promise.reject(err);
}
};
}
status = 'IDLE';
process.send({
cmd: 'init-complete'
});
break;
case 'start':
if (status !== 'IDLE') {
return process.send({
cmd: 'error',
err: new Error('cannot start a not idling child process')
});
}
status = 'STARTED';
currentJobPromise = (async () => {
try {
const result = (await processor(wrapJob(msg.job))) || {};
await asyncSend(process, {
cmd: 'completed',
value: result
});
} catch (err) {
if (!err.message) {
// eslint-disable-next-line no-ex-assign
err = new Error(err);
}
await asyncSend(process, {
cmd: 'failed',
value: err
});
} finally {
status = 'IDLE';
currentJobPromise = null;
}
})();
break;
case 'stop':
break;
}
});
/*eslint no-process-exit: "off"*/
process.on('uncaughtException', err => {
if (!err.message) {
err = new Error(err);
}
process.send({
cmd: 'failed',
value: err
});
// An uncaughException leaves this process in a potentially undetermined state so
// we must exit
process.exit(-1);
});
/**
* Enhance the given job argument with some functions
* that can be called from the sandboxed job processor.
*
* Note, the `job` argument is a JSON deserialized message
* from the main node process to this forked child process,
* the functions on the original job object are not in tact.
* The wrapped job adds back some of those original functions.
*/
function wrapJob(job) {
/*
* Emulate the real job `progress` function.
* If no argument is given, it behaves as a sync getter.
* If an argument is given, it behaves as an async setter.
*/
let progressValue = job.progress;
job.progress = function(progress) {
if (progress) {
// Locally store reference to new progress value
// so that we can return it from this process synchronously.
progressValue = progress;
// Send message to update job progress.
return asyncSend(process, {
cmd: 'progress',
value: progress
});
} else {
// Return the last known progress value.
return progressValue;
}
};
/**
* Update job info
*/
job.update = function(data) {
process.send({
cmd: 'update',
value: data
});
};
/*
* Emulate the real job `log` function.
*/
job.log = function(row) {
return asyncSend(process, {
cmd: 'log',
value: row
});
};
/*
* Emulate the real job `update` function.
*/
job.update = function(data) {
process.send({
cmd: 'update',
value: data
});
job.data = data;
};
/*
* Emulate the real job `discard` function.
*/
job.discard = function() {
process.send({
cmd: 'discard'
});
};
return job;
}
+68
View File
@@ -0,0 +1,68 @@
'use strict';
const { asyncSend } = require('./utils');
module.exports = function(processFile, childPool) {
return function process(job) {
return childPool.retain(processFile).then(async child => {
let msgHandler;
let exitHandler;
await asyncSend(child, {
cmd: 'start',
job: job
});
const done = new Promise((resolve, reject) => {
msgHandler = function(msg) {
switch (msg.cmd) {
case 'completed':
resolve(msg.value);
break;
case 'failed':
case 'error': {
const err = new Error();
Object.assign(err, msg.value);
reject(err);
break;
}
case 'progress':
job.progress(msg.value);
break;
case 'update':
job.update(msg.value);
break;
case 'discard':
job.discard();
break;
case 'log':
job.log(msg.value);
break;
}
};
exitHandler = (exitCode, signal) => {
reject(
new Error(
'Unexpected exit code: ' + exitCode + ' signal: ' + signal
)
);
};
child.on('message', msgHandler);
child.on('exit', exitHandler);
});
return done.finally(() => {
child.removeListener('message', msgHandler);
child.removeListener('exit', exitHandler);
if (child.exitCode !== null || /SIG.*/.test(child.signalCode)) {
childPool.remove(child);
} else {
childPool.release(child);
}
});
});
};
};
+70
View File
@@ -0,0 +1,70 @@
'use strict';
function hasProcessExited(child) {
return !!(child.exitCode !== null || child.signalCode);
}
function onExitOnce(child) {
return new Promise(resolve => {
child.once('exit', () => resolve());
});
}
/**
* Sends a kill signal to a child resolving when the child has exited,
* resorting to SIGKILL if the given timeout is reached
*
* @param {ChildProcess} child
* @param {'SIGTERM' | 'SIGKILL'} [signal] initial signal to use
* @param {number} [timeoutMs] time to wait until sending SIGKILL
*
* @returns {Promise<void>} the killed child
*/
function killAsync(child, signal, timeoutMs) {
if (hasProcessExited(child)) {
return Promise.resolve(child);
}
// catch any new on exit
let onExit = onExitOnce(child);
child.kill(signal || 'SIGKILL');
if (timeoutMs === 0 || isFinite(timeoutMs)) {
const timeout = setTimeout(() => {
if (!hasProcessExited(child)) {
child.kill('SIGKILL');
}
}, timeoutMs);
onExit = onExit.then(() => {
clearTimeout(timeout);
});
}
return onExit;
}
/*
asyncSend
Same as process.send but waits until the send is complete
the async version is used below because otherwise
the termination handler may exit before the parent
process has recived the messages it requires
*/
const asyncSend = (proc, msg) => {
return new Promise((resolve, reject) => {
proc.send(msg, err => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
};
module.exports = {
killAsync,
asyncSend
};
Generated Vendored Executable
+1417
View File
@@ -0,0 +1,1417 @@
'use strict';
const Redis = require('ioredis');
const EventEmitter = require('events');
const _ = require('lodash');
const fs = require('fs');
const path = require('path');
const util = require('util');
const url = require('url');
const Job = require('./job');
const scripts = require('./scripts');
const errors = require('./errors');
const utils = require('./utils');
const TimerManager = require('./timer-manager');
const { promisify } = require('util');
const { pTimeout } = require('./p-timeout');
const semver = require('semver');
const debuglog = require('util').debuglog('bull');
const uuid = require('uuid');
const commands = require('./scripts/');
/**
Gets or creates a new Queue with the given name.
The Queue keeps 6 data structures:
- wait (list)
- active (list)
- delayed (zset)
- priority (zset)
- completed (zset)
- failed (zset)
--> priorities -- > completed
/ | /
job -> wait -> active
\ ^ \
v | -- > failed
delayed
*/
/**
Delayed jobs are jobs that cannot be executed until a certain time in
ms has passed since they were added to the queue.
The mechanism is simple, a delayedTimestamp variable holds the next
known timestamp that is on the delayed set (or MAX_TIMEOUT_MS if none).
When the current job has finalized the variable is checked, if
no delayed job has to be executed yet a setTimeout is set so that a
delayed job is processed after timing out.
*/
const MINIMUM_REDIS_VERSION = '2.8.18';
/*
interface QueueOptions {
prefix?: string = 'bull',
limiter?: RateLimiter,
redis : RedisOpts, // ioredis defaults,
createClient?: (type: enum('client', 'subscriber'), redisOpts?: RedisOpts) => redisClient,
defaultJobOptions?: JobOptions,
// Advanced settings
settings?: QueueSettings {
lockDuration?: number = 30000,
lockRenewTime?: number = lockDuration / 2,
stalledInterval?: number = 30000,
maxStalledCount?: number = 1, // The maximum number of times a job can be recovered from the 'stalled' state
guardInterval?: number = 5000,
retryProcessDelay?: number = 5000,
drainDelay?: number = 5
isSharedChildPool?: boolean = false
}
}
interface RateLimiter {
max: number, // Number of jobs
duration: number, // per duration milliseconds
}
*/
// Queue(name: string, url?, opts?)
const Queue = function Queue(name, url, opts) {
if (!(this instanceof Queue)) {
return new Queue(name, url, opts);
}
if (_.isString(url)) {
const clonedOpts = _.cloneDeep(opts || {});
opts = {
...clonedOpts,
redis: {
...redisOptsFromUrl(url),
...clonedOpts.redis
}
};
} else {
opts = _.cloneDeep(url || {});
}
if (!_.isObject(opts)) {
throw TypeError('Options must be a valid object');
}
if (opts.limiter) {
if (opts.limiter.max && opts.limiter.duration) {
this.limiter = opts.limiter;
} else {
throw new TypeError('Limiter requires `max` and `duration` options');
}
}
if (opts.defaultJobOptions) {
this.defaultJobOptions = opts.defaultJobOptions;
}
this.name = name;
this.token = uuid.v4();
opts.redis = {
enableReadyCheck: false,
...(_.isString(opts.redis)
? { ...redisOptsFromUrl(opts.redis) }
: opts.redis)
};
_.defaults(opts.redis, {
port: 6379,
host: '127.0.0.1',
db: opts.redis.db || opts.redis.DB,
retryStrategy: function(times) {
return Math.min(Math.exp(times), 20000);
}
});
this.keyPrefix = opts.redis.keyPrefix || opts.prefix || 'bull';
//
// We cannot use ioredis keyPrefix feature since we
// create keys dynamically in lua scripts.
//
delete opts.redis.keyPrefix;
this.clients = [];
const loadCommands = (providedScripts, client) => {
const finalScripts = providedScripts || scripts;
for (const property in finalScripts) {
// Only define the command if not already defined
if (!client[finalScripts[property].name]) {
client.defineCommand(finalScripts[property].name, {
numberOfKeys: finalScripts[property].keys,
lua: finalScripts[property].content
});
}
}
};
const lazyClient = redisClientGetter(this, opts, (type, client) => {
// bubble up Redis error events
const handler = this.emit.bind(this, 'error');
client.on('error', handler);
this.once('close', () => client.removeListener('error', handler));
if (type === 'client') {
this._initializing = (async () => loadCommands(commands, client))().then(
() => {
debuglog(name + ' queue ready');
},
err => {
this.emit('error', new Error('Error initializing Lua scripts'));
throw err;
}
);
this._initializing.catch((/*err*/) => {});
}
});
Object.defineProperties(this, {
//
// Queue client (used to add jobs, pause queues, etc);
//
client: {
get: lazyClient('client')
},
//
// Event subscriber client (receive messages from other instance of the queue)
//
eclient: {
get: lazyClient('subscriber')
},
bclient: {
get: lazyClient('bclient')
}
});
if (opts.skipVersionCheck !== true) {
getRedisVersion(this.client)
.then(version => {
if (semver.lt(version, MINIMUM_REDIS_VERSION)) {
this.emit(
'error',
new Error(
'Redis version needs to be greater than ' +
MINIMUM_REDIS_VERSION +
'. Current: ' +
version
)
);
}
})
.catch((/*err*/) => {
// Ignore this error.
});
}
this.handlers = {};
this.delayTimer;
this.processing = [];
this.retrieving = 0;
this.drained = true;
this.settings = _.defaults(opts.settings, {
lockDuration: 30000,
stalledInterval: 30000,
maxStalledCount: 1,
guardInterval: 5000,
retryProcessDelay: 5000,
drainDelay: 5,
backoffStrategies: {},
isSharedChildPool: false
});
this.metrics = opts.metrics;
this.settings.lockRenewTime =
this.settings.lockRenewTime || this.settings.lockDuration / 2;
this.on('error', () => {
// Dummy handler to avoid process to exit with an unhandled exception.
});
// keeps track of active timers. used by close() to
// ensure that disconnect() is deferred until all
// scheduled redis commands have been executed
this.timers = new TimerManager();
// Bind these methods to avoid constant rebinding and/or creating closures
// in processJobs etc.
this.moveUnlockedJobsToWait = this.moveUnlockedJobsToWait.bind(this);
this.processJob = this.processJob.bind(this);
this.getJobFromId = Job.fromId.bind(null, this);
const keys = {};
_.each(
[
'',
'active',
'wait',
'waiting',
'paused',
'resumed',
'meta-paused',
'active',
'id',
'delayed',
'priority',
'stalled-check',
'completed',
'failed',
'stalled',
'repeat',
'limiter',
'drained',
'duplicated',
'progress',
'de' // debounce key
],
key => {
keys[key] = this.toKey(key);
}
);
this.keys = keys;
};
function redisClientGetter(queue, options, initCallback) {
const createClient = _.isFunction(options.createClient)
? options.createClient
: function(type, config) {
if (['bclient', 'subscriber'].includes(type)) {
return new Redis({ ...config, maxRetriesPerRequest: null });
} else {
return new Redis(config);
}
};
const connections = {};
return function(type) {
return function() {
// Memoized connection
if (connections[type] != null) {
return connections[type];
}
const clientOptions = _.assign({}, options.redis);
const client = (connections[type] = createClient(type, clientOptions));
const opts = client.options.redisOptions || client.options;
if (
['bclient', 'subscriber'].includes(type) &&
(opts.enableReadyCheck || opts.maxRetriesPerRequest)
) {
throw new Error(errors.Messages.MISSING_REDIS_OPTS);
}
// Since connections are lazily initialized, we can't check queue.client
// without initializing a connection. So expose a boolean we can safely
// query.
queue[type + 'Initialized'] = true;
if (!options.createClient) {
queue.clients.push(client);
}
return initCallback(type, client), client;
};
};
}
function redisOptsFromUrl(urlString) {
let redisOpts = {};
try {
const redisUrl = url.parse(urlString, true, true);
redisOpts.port = parseInt(redisUrl.port || '6379', 10);
redisOpts.host = redisUrl.hostname;
redisOpts.db = redisUrl.pathname ? redisUrl.pathname.split('/')[1] : 0;
if (redisUrl.auth) {
const columnIndex = redisUrl.auth.indexOf(':');
redisOpts.password = redisUrl.auth.slice(columnIndex + 1);
if (columnIndex > 0) {
redisOpts.username = redisUrl.auth.slice(0, columnIndex);
}
}
if (redisUrl.query) {
redisOpts = { ...redisOpts, ...redisUrl.query };
}
} catch (e) {
throw new Error(e.message);
}
return redisOpts;
}
util.inherits(Queue, EventEmitter);
//
// Extend Queue with "aspects"
//
require('./getters')(Queue);
require('./worker')(Queue);
require('./repeatable')(Queue);
// --
Queue.prototype.off = Queue.prototype.removeListener;
const _on = Queue.prototype.on;
Queue.prototype.on = function(eventName) {
this._registerEvent(eventName);
return _on.apply(this, arguments);
};
const _once = Queue.prototype.once;
Queue.prototype.once = function(eventName) {
this._registerEvent(eventName);
return _once.apply(this, arguments);
};
Queue.prototype._initProcess = function() {
if (!this._initializingProcess) {
//
// Only setup listeners if .on/.addEventListener called, or process function defined.
//
this.delayedTimestamp = Number.MAX_VALUE;
this._initializingProcess = this.isReady()
.then(() => {
return this._registerEvent('delayed');
})
.then(() => {
return this.updateDelayTimer();
});
this.errorRetryTimer = {};
}
return this._initializingProcess;
};
Queue.prototype._setupQueueEventListeners = function() {
/*
if(eventName !== 'cleaned' && eventName !== 'error'){
args[0] = Job.fromJSON(this, args[0]);
}
*/
const activeKey = this.keys.active;
const stalledKey = this.keys.stalled;
const progressKey = this.keys.progress;
const delayedKey = this.keys.delayed;
const pausedKey = this.keys.paused;
const resumedKey = this.keys.resumed;
const waitingKey = this.keys.waiting;
const completedKey = this.keys.completed;
const failedKey = this.keys.failed;
const drainedKey = this.keys.drained;
const duplicatedKey = this.keys.duplicated;
const debouncedKey = this.keys.de + 'bounced';
const pmessageHandler = (pattern, channel, message) => {
const keyAndToken = channel.split('@');
const key = keyAndToken[0];
const token = keyAndToken[1];
switch (key) {
case activeKey:
utils.emitSafe(this, 'global:active', message, 'waiting');
break;
case waitingKey:
if (this.token === token) {
utils.emitSafe(this, 'waiting', message, null);
}
token && utils.emitSafe(this, 'global:waiting', message, null);
break;
case stalledKey:
if (this.token === token) {
utils.emitSafe(this, 'stalled', message);
}
utils.emitSafe(this, 'global:stalled', message);
break;
case duplicatedKey:
if (this.token === token) {
utils.emitSafe(this, 'duplicated', message);
}
utils.emitSafe(this, 'global:duplicated', message);
break;
case debouncedKey:
if (this.token === token) {
utils.emitSafe(this, 'debounced', message);
}
utils.emitSafe(this, 'global:debounced', message);
break;
}
};
const messageHandler = (channel, message) => {
const key = channel.split('@')[0];
switch (key) {
case progressKey: {
// New way to send progress message data
try {
const { progress, jobId } = JSON.parse(message);
utils.emitSafe(this, 'global:progress', jobId, progress);
} catch (err) {
// If we fail we should try to parse the data using the deprecated method
const commaPos = message.indexOf(',');
const jobId = message.substring(0, commaPos);
const progress = message.substring(commaPos + 1);
utils.emitSafe(this, 'global:progress', jobId, JSON.parse(progress));
}
break;
}
case delayedKey: {
const newDelayedTimestamp = _.ceil(message);
if (newDelayedTimestamp < this.delayedTimestamp) {
// The new delayed timestamp is before the currently newest known delayed timestamp
// Assume this is the new delayed timestamp and call `updateDelayTimer()` to process any delayed jobs
// This will also update the `delayedTimestamp`
this.delayedTimestamp = newDelayedTimestamp;
this.updateDelayTimer();
}
break;
}
case pausedKey:
case resumedKey:
utils.emitSafe(this, 'global:' + message);
break;
case completedKey: {
const data = JSON.parse(message);
utils.emitSafe(
this,
'global:completed',
data.jobId,
data.val,
'active'
);
break;
}
case failedKey: {
const data = JSON.parse(message);
utils.emitSafe(this, 'global:failed', data.jobId, data.val, 'active');
break;
}
case drainedKey:
utils.emitSafe(this, 'global:drained');
break;
}
};
this.eclient.on('pmessage', pmessageHandler);
this.eclient.on('message', messageHandler);
this.once('close', () => {
this.eclient.removeListener('pmessage', pmessageHandler);
this.eclient.removeListener('message', messageHandler);
});
};
Queue.prototype._registerEvent = function(eventName) {
const internalEvents = ['waiting', 'delayed', 'duplicated', 'debounced'];
if (
eventName.startsWith('global:') ||
internalEvents.indexOf(eventName) !== -1
) {
if (!this.registeredEvents) {
this._setupQueueEventListeners();
this.registeredEvents = this.registeredEvents || {};
}
const _eventName = eventName.replace('global:', '');
if (!this.registeredEvents[_eventName]) {
return utils
.isRedisReady(this.eclient)
.then(() => {
const channel = this.toKey(_eventName);
if (['active', 'waiting', 'stalled', 'duplicated', 'debounced'].indexOf(_eventName) !== -1) {
return (this.registeredEvents[_eventName] = this.eclient.psubscribe(
channel + '*'
));
} else {
return (this.registeredEvents[_eventName] = this.eclient.subscribe(
channel
));
}
})
.then(() => {
utils.emitSafe(this, 'registered:' + eventName);
});
} else {
return this.registeredEvents[_eventName];
}
}
return Promise.resolve();
};
Queue.ErrorMessages = errors.Messages;
Queue.prototype.isReady = async function() {
await this._initializing;
return this;
};
async function redisClientDisconnect(client) {
if (client.status !== 'end') {
let _resolve, _reject;
return new Promise((resolve, reject) => {
_resolve = resolve;
_reject = reject;
client.once('end', _resolve);
pTimeout(
client.quit().catch(err => {
if (err.message !== 'Connection is closed.') {
throw err;
}
}),
500
)
.catch(() => {
// Ignore timeout error
})
.finally(() => {
client.once('error', _reject);
client.disconnect();
if (['connecting', 'reconnecting'].includes(client.status)) {
resolve();
}
});
}).finally(() => {
client.removeListener('end', _resolve);
client.removeListener('error', _reject);
});
}
}
Queue.prototype.disconnect = async function() {
await Promise.all(
this.clients.map(client =>
client.blocked ? client.disconnect() : redisClientDisconnect(client)
)
);
};
Queue.prototype.removeJobs = function(pattern) {
return Job.remove(this, pattern);
};
Queue.prototype.close = function(doNotWaitJobs) {
let isReady = true;
if (this.closing) {
return this.closing;
}
return (this.closing = this.isReady()
.then(this._initializingProcess)
.catch(() => {
isReady = false;
})
.then(() => isReady && this.pause(true, doNotWaitJobs))
.catch(() => void 0) // Ignore possible error from pause
.finally(() => this._clearTimers())
.then(() => {
if (!this.childPool) {
return;
}
const cleanPromise = this.childPool.clean().catch(() => {
// Ignore this error and try to close anyway.
});
if (doNotWaitJobs) {
return;
}
return cleanPromise;
})
.then(
async () => this.disconnect(),
err => console.error(err)
)
.finally(() => {
this.closed = true;
utils.emitSafe(this, 'close');
}));
};
Queue.prototype._clearTimers = function() {
_.each(this.errorRetryTimer, timer => {
clearTimeout(timer);
});
clearTimeout(this.delayTimer);
clearInterval(this.guardianTimer);
clearInterval(this.moveUnlockedJobsToWaitInterval);
this.timers.clearAll();
return this.timers.whenIdle();
};
/**
Processes a job from the queue. The callback is called for every job that
is dequeued.
@method process
*/
Queue.prototype.process = function(name, concurrency, handler) {
switch (arguments.length) {
case 1:
handler = name;
concurrency = 1;
name = Job.DEFAULT_JOB_NAME;
break;
case 2: // (string, function) or (string, string) or (number, function) or (number, string)
handler = concurrency;
if (typeof name === 'string') {
concurrency = 1;
} else {
concurrency = name;
name = Job.DEFAULT_JOB_NAME;
}
break;
}
this.setHandler(name, handler);
return this._initProcess().then(() => {
return this.start(concurrency, name);
});
};
Queue.prototype.start = function(concurrency, name) {
return this.run(concurrency, name).catch(err => {
utils.emitSafe(this, 'error', err, 'error running queue');
throw err;
});
};
Queue.prototype.setHandler = function(name, handler) {
if (!handler) {
throw new Error('Cannot set an undefined handler');
}
if (this.handlers[name]) {
throw new Error('Cannot define the same handler twice ' + name);
}
this.setWorkerName();
if (typeof handler === 'string') {
const supportedFileTypes = ['.js', '.ts', '.flow', '.cjs'];
const processorFile =
handler +
(supportedFileTypes.includes(path.extname(handler)) ? '' : '.js');
if (!fs.existsSync(processorFile)) {
throw new Error('File ' + processorFile + ' does not exist');
}
const isSharedChildPool = this.settings.isSharedChildPool;
this.childPool =
this.childPool || require('./process/child-pool')(isSharedChildPool);
const sandbox = require('./process/sandbox');
this.handlers[name] = sandbox(handler, this.childPool).bind(this);
} else {
handler = handler.bind(this);
if (handler.length > 1) {
this.handlers[name] = promisify(handler);
} else {
this.handlers[name] = function() {
try {
return Promise.resolve(handler.apply(null, arguments));
} catch (err) {
return Promise.reject(err);
}
};
}
}
};
/**
interface JobOptions
{
attempts: number;
repeat: {
tz?: string,
endDate?: Date | string | number
},
preventParsingData: boolean;
}
*/
/**
Adds a job to the queue.
@method add
@param data: {} Custom data to store for this job. Should be JSON serializable.
@param opts: JobOptions Options for this job.
*/
Queue.prototype.add = function(name, data, opts) {
if (typeof name !== 'string') {
opts = data;
data = name;
name = Job.DEFAULT_JOB_NAME;
}
opts = _.cloneDeep({ ...this.defaultJobOptions, ...opts });
opts.jobId = jobIdForGroup(this.limiter, opts, data);
if (opts.repeat) {
return this.isReady().then(() => {
return this.nextRepeatableJob(name, data, opts, true);
});
} else {
return Job.create(this, name, data, opts);
}
};
/**
* Retry all the failed jobs.
*
* @param opts.count - number to limit how many jobs will be moved to wait status per iteration
* @returns
*/
Queue.prototype.retryJobs = async function(opts = {}) {
let cursor = 0;
do {
cursor = await scripts.retryJobs(this, opts.count);
} while (cursor);
};
/**
* Removes a debounce key.
*
* @param id - identifier
*/
Queue.prototype.removeDebounceKey = (id) => {
return this.client.del(`${this.keys.de}:${id}`);
}
/**
Adds an array of jobs to the queue.
@method add
@param jobs: [] The array of jobs to add to the queue. Each job is defined by 3 properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add'.
*/
Queue.prototype.addBulk = function(jobs) {
const decoratedJobs = jobs.map(job => {
const jobId = jobIdForGroup(this.limiter, job.opts, job.data);
return {
...job,
name: typeof job.name !== 'string' ? Job.DEFAULT_JOB_NAME : job.name,
opts: {
...this.defaultJobOptions,
...job.opts,
jobId
}
};
});
return Job.createBulk(this, decoratedJobs);
};
/**
Empties the queue.
Returns a promise that is resolved after the operation has been completed.
Note that if some other process is adding jobs at the same time as emptying,
the queues may not be really empty after this method has executed completely.
Also, if the method does error between emptying the lists and removing all the
jobs, there will be zombie jobs left in redis.
TODO: Use EVAL to make this operation fully atomic.
*/
Queue.prototype.empty = function() {
const queueKeys = this.keys;
let multi = this.multi();
multi.lrange(queueKeys.wait, 0, -1);
multi.lrange(queueKeys.paused, 0, -1);
multi.keys(this.toKey('*:limited'));
multi.del(
queueKeys.wait,
queueKeys.paused,
queueKeys['meta-paused'],
queueKeys.delayed,
queueKeys.priority,
queueKeys.limiter,
`${queueKeys.limiter}:index`
);
return multi.exec().then(res => {
let [waiting, paused, limited] = res;
waiting = waiting[1];
paused = paused[1];
limited = limited[1];
const jobKeys = paused.concat(waiting).map(this.toKey, this);
if (jobKeys.length || limited.length) {
multi = this.multi();
for (let i = 0; i < jobKeys.length; i += 10000) {
multi.del.apply(multi, jobKeys.slice(i, i + 10000));
}
for (let i = 0; i < limited.length; i += 10000) {
multi.del.apply(multi, limited.slice(i, i + 10000));
}
return multi.exec();
}
});
};
/**
Pauses the processing of this queue, locally if true passed, otherwise globally.
For global pause, we use an atomic RENAME operation on the wait queue. Since
we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue
is renamed to 'paused', no new jobs will be processed (the current ones
will run until finalized).
Adding jobs requires a LUA script to check first if the paused list exist
and in that case it will add it there instead of the wait list.
*/
Queue.prototype.pause = function(isLocal, doNotWaitActive) {
return this.isReady()
.then(() => {
if (isLocal) {
if (!this.paused) {
this.paused = new Promise(resolve => {
this.resumeLocal = function() {
this.paused = null; // Allow pause to be checked externally for paused state.
resolve();
};
});
}
if (!this.bclientInitialized) {
// bclient not yet initialized, so no jobs to wait for
return;
}
if (doNotWaitActive) {
// Force reconnection of blocking connection to abort blocking redis call immediately.
return redisClientDisconnect(this.bclient).then(() =>
this.bclient.connect()
);
}
return this.whenCurrentJobsFinished();
} else {
return scripts.pause(this, true);
}
})
.then(() => {
return utils.emitSafe(this, 'paused');
});
};
Queue.prototype.resume = function(isLocal /* Optional */) {
return this.isReady()
.then(() => {
if (isLocal) {
if (this.resumeLocal) {
this.resumeLocal();
}
} else {
return scripts.pause(this, false);
}
})
.then(() => {
utils.emitSafe(this, 'resumed');
});
};
Queue.prototype.isPaused = async function(isLocal) {
if (isLocal) {
return !!this.paused;
} else {
await this.isReady();
const multi = this.multi();
multi.exists(this.keys['meta-paused']);
// For forward compatibility with BullMQ.
multi.hexists(this.toKey('meta'), 'paused');
const [[, isPaused], [, isPausedNew]] = await multi.exec();
return !!(isPaused || isPausedNew);
}
};
Queue.prototype.run = function(concurrency, handlerName) {
if (!Number.isInteger(concurrency)) {
throw new Error('Cannot set Float as concurrency');
}
const promises = [];
return this.isReady()
.then(() => {
return this.moveUnlockedJobsToWait();
})
.then(() => {
return utils.isRedisReady(this.bclient);
})
.then(() => {
while (concurrency--) {
promises.push(
new Promise(resolve => {
this.processJobs(`${handlerName}:${concurrency}`, resolve);
})
);
}
this.startMoveUnlockedJobsToWait();
return Promise.all(promises);
});
};
// ---------------------------------------------------------------------
// Private methods
// ---------------------------------------------------------------------
/**
This function updates the delay timer, which is a timer that timeouts
at the next known delayed job.
*/
Queue.prototype.updateDelayTimer = function() {
if (this.closing) {
return Promise.resolve();
}
return scripts
.updateDelaySet(this, Date.now())
.then(nextTimestamp => {
this.delayedTimestamp = nextTimestamp
? nextTimestamp / 4096
: Number.MAX_VALUE;
// Clear any existing update delay timer
if (this.delayTimer) {
clearTimeout(this.delayTimer);
}
// Delay for the next update of delay set
const delay = _.min([
this.delayedTimestamp - Date.now(),
this.settings.guardInterval
]);
// Schedule next processing of the delayed jobs
if (delay <= 0) {
// Next set of jobs are due right now, process them also
this.updateDelayTimer();
} else {
// Update the delay set when the next job is due
// or the next guard time
this.delayTimer = setTimeout(() => this.updateDelayTimer(), delay);
}
// Silence warnings about promise created but not returned.
// This isn't an issue since we emit errors.
// See http://bluebirdjs.com/docs/warning-explanations.html#warning-a-promise-was-created-in-a-handler-but-was-not-returned-from-it
return null;
})
.catch(err => {
utils.emitSafe(this, 'error', err, 'Error updating the delay timer');
if (this.delayTimer) {
clearTimeout(this.delayTimer);
}
this.delayTimer = setTimeout(
() => this.updateDelayTimer(),
this.settings.guardInterval
);
});
};
/**
* Process jobs that have been added to the active list but are not being
* processed properly. This can happen due to a process crash in the middle
* of processing a job, leaving it in 'active' but without a job lock.
*/
Queue.prototype.moveUnlockedJobsToWait = function() {
if (this.closing) {
return Promise.resolve();
}
return scripts
.moveUnlockedJobsToWait(this)
.then(([failed, stalled]) => {
const handleFailedJobs = failed.map(jobId => {
return this.getJobFromId(jobId).then(job => {
utils.emitSafe(
this,
'failed',
job,
new Error('job stalled more than allowable limit'),
'active'
);
return null;
});
});
const handleStalledJobs = stalled.map(jobId => {
return this.getJobFromId(jobId).then(job => {
// Do not emit the event if the job was completed by another worker
if (job !== null) {
utils.emitSafe(this, 'stalled', job);
}
return null;
});
});
return Promise.all(handleFailedJobs.concat(handleStalledJobs));
})
.catch(err => {
utils.emitSafe(
this,
'error',
err,
'Failed to handle unlocked job in active'
);
});
};
Queue.prototype.startMoveUnlockedJobsToWait = function() {
clearInterval(this.moveUnlockedJobsToWaitInterval);
if (this.settings.stalledInterval > 0 && !this.closing) {
this.moveUnlockedJobsToWaitInterval = setInterval(
this.moveUnlockedJobsToWait,
this.settings.stalledInterval
);
}
};
/*
Process jobs. Note last argument 'job' is optional.
*/
Queue.prototype.processJobs = function(index, resolve, job) {
const processJobs = this.processJobs.bind(this, index, resolve);
process.nextTick(() => {
this._processJobOnNextTick(processJobs, index, resolve, job);
});
};
Queue.prototype._processJobOnNextTick = function(
processJobs,
index,
resolve,
job
) {
if (!this.closing) {
(this.paused || Promise.resolve())
.then(() => {
const gettingNextJob = job ? Promise.resolve(job) : this.getNextJob();
return (this.processing[index] = gettingNextJob
.then(this.processJob)
.then(processJobs, err => {
if (!(this.closing && err.message === 'Connection is closed.')) {
utils.emitSafe(this, 'error', err, 'Error processing job');
//
// Wait before trying to process again.
//
clearTimeout(this.errorRetryTimer[index]);
this.errorRetryTimer[index] = setTimeout(() => {
processJobs();
}, this.settings.retryProcessDelay);
}
return null;
}));
})
.catch(err => {
utils.emitSafe(this, 'error', err, 'Error processing job');
});
} else {
resolve(this.closing);
}
};
Queue.prototype.processJob = function(job, notFetch = false) {
let lockRenewId;
let timerStopped = false;
if (!job) {
return Promise.resolve();
}
//
// There are two cases to take into consideration regarding locks.
// 1) The lock renewer fails to renew a lock, this should make this job
// unable to complete, since some other worker is also working on it.
// 2) The lock renewer is called more seldom than the check for stalled
// jobs, so we can assume the job has been stalled and is already being processed
// by another worker. See #308
//
const lockExtender = () => {
lockRenewId = this.timers.set(
'lockExtender',
this.settings.lockRenewTime,
() => {
scripts
.extendLock(this, job.id, this.settings.lockDuration)
.then(lock => {
if (lock && !timerStopped) {
lockExtender();
}
})
.catch(err => {
utils.emitSafe(this, 'lock-extension-failed', job, err);
});
}
);
};
const timeoutMs = job.opts.timeout;
const stopTimer = () => {
timerStopped = true;
this.timers.clear(lockRenewId);
};
const handleCompleted = result => {
return job.moveToCompleted(result, undefined, notFetch).then(jobData => {
utils.emitSafe(this, 'completed', job, result, 'active');
return jobData ? this.nextJobFromJobData(jobData[0], jobData[1]) : null;
});
};
const handleFailed = err => {
const error = err;
return job.moveToFailed(err).then(jobData => {
utils.emitSafe(this, 'failed', job, error, 'active');
return jobData ? this.nextJobFromJobData(jobData[0], jobData[1]) : null;
});
};
lockExtender();
const handler = this.handlers[job.name] || this.handlers['*'];
if (!handler) {
return handleFailed(
new Error('Missing process handler for job type ' + job.name)
);
} else {
let jobPromise = handler(job);
if (timeoutMs) {
jobPromise = pTimeout(jobPromise, timeoutMs);
}
// Local event with jobPromise so that we can cancel job.
utils.emitSafe(this, 'active', job, jobPromise, 'waiting');
return jobPromise
.then(handleCompleted)
.catch(handleFailed)
.finally(() => {
stopTimer();
});
}
};
Queue.prototype.multi = function() {
return this.client.multi();
};
/**
Returns a promise that resolves to the next job in queue.
*/
Queue.prototype.getNextJob = async function() {
if (this.closing) {
return Promise.resolve();
}
if (this.drained) {
//
// Waiting for new jobs to arrive
//
try {
this.bclient.blocked = true;
const jobId = await this.bclient.brpoplpush(
this.keys.wait,
this.keys.active,
this.settings.drainDelay
);
this.bclient.blocked = false;
if (jobId) {
return this.moveToActive(jobId);
}
} catch (err) {
// Swallow error if locally paused since we did force a disconnection
if (!(this.paused && err.message === 'Connection is closed.')) {
throw err;
}
}
} else {
return this.moveToActive();
}
};
Queue.prototype.moveToActive = async function(jobId) {
// For manual retrieving jobs we need to wait for the queue to be ready.
await this.isReady();
return scripts.moveToActive(this, jobId).then(([jobData, jobId]) => {
return this.nextJobFromJobData(jobData, jobId);
});
};
Queue.prototype.nextJobFromJobData = function(jobData, jobId) {
if (jobData) {
this.drained = false;
const job = Job.fromJSON(this, jobData, jobId);
if (job.opts.repeat) {
return this.nextRepeatableJob(job.name, job.data, job.opts).then(() => {
return job;
});
}
return job;
} else {
this.drained = true;
utils.emitSafe(this, 'drained');
return null;
}
};
Queue.prototype.retryJob = function(job) {
return job.retry();
};
Queue.prototype.toKey = function(queueType) {
return [this.keyPrefix, this.name, queueType].join(':');
};
/*@function clean
*
* Cleans jobs from a queue. Similar to remove but keeps jobs within a certain
* grace period.
*
* @param {int} grace - The grace period
* @param {string} [type=completed] - The type of job to clean. Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.
* @param {int} The max number of jobs to clean
*/
Queue.prototype.clean = function(grace, type, limit) {
return this.isReady().then(() => {
if (grace === undefined || grace === null) {
throw new Error('You must define a grace period.');
}
if (!type) {
type = 'completed';
}
if (
_.indexOf(
['completed', 'wait', 'active', 'paused', 'delayed', 'failed'],
type
) === -1
) {
throw new Error('Cannot clean unknown queue type ' + type);
}
return scripts
.cleanJobsInSet(this, type, Date.now() - grace, limit)
.then(jobs => {
utils.emitSafe(this, 'cleaned', jobs, type);
return jobs;
})
.catch(err => {
utils.emitSafe(this, 'error', err);
throw err;
});
});
};
/* @method obliterate
*
* Completely destroys the queue and all of its contents irreversibly.
* This method will the *pause* the queue and requires that there are no
* active jobs. It is possible to bypass this requirement, i.e. not
* having active jobs using the "force" option.
*
* Note: This operation requires to iterate on all the jobs stored in the queue
* and can be slow for very large queues.
*
* @param { { force: boolean, count: number }} opts. Use force = true to force obliteration even
* with active jobs in the queue. Use count with the maximun number of deleted keys per iteration,
* 1000 is the default.
*/
Queue.prototype.obliterate = async function(opts) {
await this.pause();
let cursor = 0;
do {
cursor = await scripts.obliterate(this, {
force: false,
count: 1000,
...opts
});
} while (cursor);
};
/**
* Returns a promise that resolves when active jobs are finished
*
* @returns {Promise}
*/
Queue.prototype.whenCurrentJobsFinished = function() {
if (!this.bclientInitialized) {
// bclient not yet initialized, so no jobs to wait for
return Promise.resolve();
}
//
// Force reconnection of blocking connection to abort blocking redis call immediately.
//
const forcedReconnection = redisClientDisconnect(this.bclient).then(() => {
return this.bclient.connect();
});
return Promise.all(Object.values(this.processing)).then(
() => forcedReconnection
);
};
//
// Private local functions
//
function getRedisVersion(client) {
return client.info().then(doc => {
const prefix = 'redis_version:';
const lines = doc.split('\r\n');
for (let i = 0; i < lines.length; i++) {
if (lines[i].indexOf(prefix) === 0) {
return lines[i].substr(prefix.length);
}
}
});
}
function jobIdForGroup(limiter, opts, data) {
const jobId = opts && opts.jobId;
const groupKey = _.get(limiter, 'groupKey');
if (groupKey) {
return `${jobId || uuid.v4()}:${_.get(data, groupKey)}`;
}
return jobId;
}
module.exports = Queue;
+237
View File
@@ -0,0 +1,237 @@
'use strict';
const _ = require('lodash');
const parser = require('cron-parser');
const crypto = require('crypto');
const Job = require('./job');
module.exports = function(Queue) {
Queue.prototype.nextRepeatableJob = function(
name,
data,
opts,
skipCheckExists
) {
const client = this.client;
const repeat = opts.repeat;
const prevMillis = opts.prevMillis || 0;
if (!prevMillis && opts.jobId) {
repeat.jobId = opts.jobId;
}
const currentCount = repeat.count ? repeat.count + 1 : 1;
if (!_.isUndefined(repeat.limit) && currentCount > repeat.limit) {
return Promise.resolve();
}
let now = Date.now();
if (!_.isUndefined(repeat.endDate) && now > new Date(repeat.endDate)) {
return Promise.resolve();
}
now = prevMillis < now ? now : prevMillis;
const nextMillis = getNextMillis(now, repeat);
if (nextMillis) {
const jobId = repeat.jobId ? repeat.jobId + ':' : ':';
const repeatKey = getRepeatKey(name, repeat, jobId);
const createNextJob = () => {
return client.zadd(this.keys.repeat, nextMillis, repeatKey).then(() => {
//
// Generate unique job id for this iteration.
//
const customId = getRepeatJobId(
name,
jobId,
nextMillis,
md5(repeatKey)
);
now = Date.now();
const delay = nextMillis - now;
return Job.create(
this,
name,
data,
_.defaultsDeep(
{
repeat: {
count: currentCount,
key: repeatKey
},
jobId: customId,
delay: delay < 0 ? 0 : delay,
timestamp: now,
prevMillis: nextMillis
},
opts
)
);
});
};
if (skipCheckExists) {
return createNextJob();
}
// Check that the repeatable job hasn't been removed
// TODO: a lua script would be better here
return client
.zscore(this.keys.repeat, repeatKey)
.then(repeatableExists => {
// The job could have been deleted since this check
if (repeatableExists) {
return createNextJob();
}
return Promise.resolve();
});
} else {
return Promise.resolve();
}
};
Queue.prototype.removeRepeatable = function(name, repeat) {
if (typeof name !== 'string') {
repeat = name;
name = Job.DEFAULT_JOB_NAME;
}
return this.isReady().then(() => {
const jobId = repeat.jobId ? repeat.jobId + ':' : ':';
const repeatJobKey = getRepeatKey(name, repeat, jobId);
const repeatJobId = getRepeatJobId(name, jobId, '', md5(repeatJobKey));
const queueKey = this.keys[''];
return this.client.removeRepeatable(
this.keys.repeat,
this.keys.delayed,
repeatJobId,
repeatJobKey,
queueKey
);
});
};
Queue.prototype.removeRepeatableByKey = function(repeatJobKey) {
const repeatMeta = this._keyToData(repeatJobKey);
const queueKey = this.keys[''];
const jobId = repeatMeta.id ? repeatMeta.id + ':' : ':';
const repeatJobId = getRepeatJobId(
repeatMeta.name || Job.DEFAULT_JOB_NAME,
jobId,
'',
md5(repeatJobKey)
);
return this.isReady().then(() => {
return this.client.removeRepeatable(
this.keys.repeat,
this.keys.delayed,
repeatJobId,
repeatJobKey,
queueKey
);
});
};
Queue.prototype._keyToData = function(key) {
const data = key.split(':');
return {
key: key,
name: data[0],
id: data[1] || null,
endDate: parseInt(data[2]) || null,
tz: data[3] || null,
cron: data[4]
};
};
Queue.prototype.getRepeatableJobs = function(start, end, asc) {
const key = this.keys.repeat;
start = start || 0;
end = end || -1;
return (asc
? this.client.zrange(key, start, end, 'WITHSCORES')
: this.client.zrevrange(key, start, end, 'WITHSCORES')
).then(result => {
const jobs = [];
for (let i = 0; i < result.length; i += 2) {
const data = this._keyToData(result[i]);
jobs.push({
key: data.key,
name: data.name,
id: data.id,
endDate: data.endDate,
tz: data.cron ? data.tz : null,
cron: data.cron || null,
every: !data.cron ? parseInt(data.tz) : null,
next: parseInt(result[i + 1])
});
}
return jobs;
});
};
Queue.prototype.getRepeatableCount = function() {
return this.client.zcard(this.toKey('repeat'));
};
function getRepeatJobId(name, jobId, nextMillis, namespace) {
return 'repeat:' + md5(name + jobId + namespace) + ':' + nextMillis;
}
function getRepeatKey(name, repeat, jobId) {
const endDate = repeat.endDate
? new Date(repeat.endDate).getTime() + ':'
: ':';
const tz = repeat.tz ? repeat.tz + ':' : ':';
const suffix = repeat.cron ? tz + repeat.cron : String(repeat.every);
return name + ':' + jobId + endDate + suffix;
}
function getNextMillis(millis, opts) {
if (opts.cron && opts.every) {
throw new Error(
'Both .cron and .every options are defined for this repeatable job'
);
}
if (opts.every) {
return Math.floor(millis / opts.every) * opts.every + opts.every;
}
const currentDate =
opts.startDate && new Date(opts.startDate) > new Date(millis)
? new Date(opts.startDate)
: new Date(millis);
const interval = parser.parseExpression(
opts.cron,
_.defaults(
{
currentDate
},
opts
)
);
try {
return interval.next().getTime();
} catch (e) {
// Ignore error
}
}
function md5(str) {
return crypto
.createHash('md5')
.update(str)
.digest('hex');
}
};
+625
View File
@@ -0,0 +1,625 @@
/**
* Includes all the scripts needed by the queue and jobs.
*/
'use strict';
const _ = require('lodash');
const msgpackr = require('msgpackr');
const packer = new msgpackr.Packr({
useRecords: false,
encodeUndefinedAsNil: true
});
const pack = packer.pack;
const scripts = {
isJobInList(client, listKey, jobId) {
return client.isJobInList([listKey, jobId]).then(result => {
return result === 1;
});
},
addJob(client, queue, job, opts) {
const queueKeys = queue.keys;
let keys = [
queueKeys.wait,
queueKeys.paused,
queueKeys['meta-paused'],
queueKeys.id,
queueKeys.delayed,
queueKeys.priority
];
const args = [
queueKeys[''],
_.isUndefined(opts.customJobId) ? '' : opts.customJobId,
job.name,
job.data,
pack(job.opts),
job.timestamp,
job.delay,
job.delay ? job.timestamp + job.delay : 0,
opts.priority || 0,
opts.lifo ? 'RPUSH' : 'LPUSH',
queue.token,
job.debounceId ? `${queueKeys.de}:${job.debounceId}` : null,
opts.debounce ? opts.debounce.id : null,
opts.debounce ? opts.debounce.ttl : null,
];
keys = keys.concat(args);
return client.addJob(keys);
},
pause(queue, pause) {
let src = 'wait',
dst = 'paused';
if (!pause) {
src = 'paused';
dst = 'wait';
}
const keys = _.map(
[src, dst, 'meta-paused', pause ? 'paused' : 'resumed', 'meta'],
name => {
return queue.toKey(name);
}
);
return queue.client.pause(keys.concat([pause ? 'paused' : 'resumed']));
},
async addLog(queue, jobId, logRow, keepLogs) {
const client = await queue.client;
const keys = [queue.toKey(jobId), queue.toKey(jobId) + ':logs'];
const result = await client.addLog(
keys.concat([jobId, logRow, keepLogs ? keepLogs : ''])
);
if (result < 0) {
throw scripts.finishedErrors(result, jobId, 'addLog');
}
return result;
},
getCountsPerPriorityArgs(queue, priorities) {
const keys = [
queue.keys.wait,
queue.keys.paused,
queue.keys['meta-paused'],
queue.keys.priority
];
const args = priorities;
return keys.concat(args);
},
async getCountsPerPriority(queue, priorities) {
const client = await queue.client;
const args = this.getCountsPerPriorityArgs(queue, priorities);
return client.getCountsPerPriority(args);
},
moveToActive(queue, jobId) {
const queueKeys = queue.keys;
const keys = [queueKeys.wait, queueKeys.active, queueKeys.priority];
keys[3] = keys[1] + '@' + queue.token;
keys[4] = queueKeys.stalled;
keys[5] = queueKeys.limiter;
keys[6] = queueKeys.delayed;
keys[7] = queueKeys.drained;
const args = [
queueKeys[''],
queue.token,
queue.settings.lockDuration,
Date.now(),
jobId
];
if (queue.limiter) {
args.push(
queue.limiter.max,
queue.limiter.duration,
!!queue.limiter.bounceBack
);
queue.limiter.groupKey && args.push(true);
}
return queue.client.moveToActive(keys.concat(args)).then(raw2jobData);
},
updateProgress(job, progress) {
const queue = job.queue;
const keys = [job.id, 'progress'].map(name => {
return queue.toKey(name);
});
const progressJson = JSON.stringify(progress);
return queue.client
.updateProgress(keys, [
progressJson,
JSON.stringify({ jobId: job.id, progress })
])
.then(code => {
if (code < 0) {
throw scripts.finishedErrors(code, job.id, 'updateProgress');
}
queue.emit('progress', job, progress);
});
},
updateData(job, data) {
const queue = job.queue;
const keys = [job.id].map(name => {
return queue.toKey(name);
});
const dataJson = JSON.stringify(data);
return queue.client.updateData(keys, [dataJson]);
},
saveStacktraceArgs(
job,
stacktrace,
failedReason
) {
const queue = job.queue;
const keys = [queue.toKey(job.id)];
return keys.concat([stacktrace, failedReason, job.attemptsMade]);
},
retryJobsArgs(queue, count) {
const keys = [
queue.toKey(''),
queue.toKey('failed'),
queue.toKey('wait'),
queue.toKey('meta-paused'),
queue.toKey('paused')
];
const args = [count];
return keys.concat(args);
},
async retryJobs(queue, count = 1000) {
const client = await queue.client;
const args = this.retryJobsArgs(queue, count);
return client.retryJobs(args);
},
moveToFinishedArgs(
job,
val,
propVal,
shouldRemove,
target,
ignoreLock,
notFetch
) {
const queue = job.queue;
const queueKeys = queue.keys;
const metricsKey = queue.toKey(`metrics:${target}`);
const keys = [
queueKeys.active,
queueKeys[target],
queue.toKey(job.id),
queueKeys.wait,
queueKeys.priority,
queueKeys.active + '@' + queue.token,
queueKeys.delayed,
queueKeys.stalled,
metricsKey
];
const keepJobs = pack(
typeof shouldRemove === 'object'
? shouldRemove
: typeof shouldRemove === 'number'
? { count: shouldRemove }
: { count: shouldRemove ? 0 : -1 }
);
const args = [
job.id,
job.finishedOn,
propVal,
_.isUndefined(val) ? 'null' : val,
ignoreLock ? '0' : queue.token,
keepJobs,
JSON.stringify({ jobId: job.id, val: val }),
notFetch || queue.paused || queue.closing || queue.limiter ? 0 : 1,
queueKeys[''],
queue.settings.lockDuration,
queue.token,
queue.metrics && queue.metrics.maxDataPoints
];
return keys.concat(args);
},
moveToFinished(
job,
val,
propVal,
shouldRemove,
target,
ignoreLock,
notFetch = false
) {
const args = scripts.moveToFinishedArgs(
job,
val,
propVal,
shouldRemove,
target,
ignoreLock,
notFetch,
job.queue.toKey('')
);
return job.queue.client.moveToFinished(args).then(result => {
if (result < 0) {
throw scripts.finishedErrors(result, job.id, 'finished', 'active');
} else if (result) {
return raw2jobData(result);
}
return 0;
});
},
finishedErrors(code, jobId, command, state) {
switch (code) {
case -1:
return new Error('Missing key for job ' + jobId + ' ' + command);
case -2:
return new Error('Missing lock for job ' + jobId + ' ' + command);
case -3:
return new Error(
`Job ${jobId} is not in the ${state} state. ${command}`
);
case -6:
return new Error(
`Lock mismatch for job ${jobId}. Cmd ${command} from ${state}`
);
}
},
// TODO: add a retention argument for completed and finished jobs (in time).
moveToCompleted(
job,
returnvalue,
removeOnComplete,
ignoreLock,
notFetch = false
) {
return scripts.moveToFinished(
job,
returnvalue,
'returnvalue',
removeOnComplete,
'completed',
ignoreLock,
notFetch
);
},
moveToFailedArgs(job, failedReason, removeOnFailed, ignoreLock) {
return scripts.moveToFinishedArgs(
job,
failedReason,
'failedReason',
removeOnFailed,
'failed',
ignoreLock,
true
);
},
moveToFailed(job, failedReason, removeOnFailed, ignoreLock) {
const args = scripts.moveToFailedArgs(
job,
failedReason,
removeOnFailed,
ignoreLock
);
return scripts.moveToFinished(args);
},
isFinished(job) {
const keys = _.map(['completed', 'failed'], key => {
return job.queue.toKey(key);
});
return job.queue.client.isFinished(keys.concat([job.id]));
},
moveToDelayedArgs(queue, jobId, timestamp, ignoreLock) {
//
// Bake in the job id first 12 bits into the timestamp
// to guarantee correct execution order of delayed jobs
// (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
//
// WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
//
timestamp = _.isUndefined(timestamp) ? 0 : timestamp;
timestamp = +timestamp || 0;
timestamp = timestamp < 0 ? 0 : timestamp;
if (timestamp > 0) {
timestamp = timestamp * 0x1000 + (jobId & 0xfff);
}
const keys = _.map(['active', 'delayed', jobId, 'stalled'], name => {
return queue.toKey(name);
});
return keys.concat([
JSON.stringify(timestamp),
jobId,
ignoreLock ? '0' : queue.token
]);
},
moveToDelayed(queue, jobId, timestamp, ignoreLock) {
const args = scripts.moveToDelayedArgs(queue, jobId, timestamp, ignoreLock);
return queue.client.moveToDelayed(args).then(result => {
switch (result) {
case -1:
throw new Error(
'Missing Job ' +
jobId +
' when trying to move from active to delayed'
);
case -2:
throw new Error(
'Job ' +
jobId +
' was locked when trying to move from active to delayed'
);
}
});
},
remove(queue, jobId) {
const keys = [
queue.keys.active,
queue.keys.wait,
queue.keys.delayed,
queue.keys.paused,
queue.keys.completed,
queue.keys.failed,
queue.keys.priority,
queue.toKey(jobId),
queue.toKey(`${jobId}:logs`),
queue.keys.limiter,
queue.toKey(''),
];
return queue.client.removeJob(keys.concat([jobId, queue.token]));
},
async removeWithPattern(queue, pattern) {
const keys = [
queue.keys.active,
queue.keys.wait,
queue.keys.delayed,
queue.keys.paused,
queue.keys.completed,
queue.keys.failed,
queue.keys.priority,
queue.keys.limiter
];
const allRemoved = [];
let cursor = '0',
removed;
do {
[cursor, removed] = await queue.client.removeJobs(
keys.concat([queue.toKey(''), pattern, cursor])
);
allRemoved.push.apply(allRemoved, removed);
} while (cursor !== '0');
return allRemoved;
},
extendLock(queue, jobId, duration) {
return queue.client.extendLock([
queue.toKey(jobId) + ':lock',
queue.keys.stalled,
queue.token,
duration,
jobId
]);
},
releaseLock(queue, jobId) {
return queue.client.releaseLock([
queue.toKey(jobId) + ':lock',
queue.token
]);
},
takeLock(queue, job) {
return queue.client.takeLock([
job.lockKey(),
queue.token,
queue.settings.lockDuration
]);
},
/**
It checks if the job in the top of the delay set should be moved back to the
top of the wait queue (so that it will be processed as soon as possible)
*/
updateDelaySet(queue, delayedTimestamp) {
const keys = [
queue.keys.delayed,
queue.keys.active,
queue.keys.wait,
queue.keys.priority,
queue.keys.paused,
queue.keys['meta-paused']
];
const args = [queue.toKey(''), delayedTimestamp, queue.token];
return queue.client.updateDelaySet(keys.concat(args));
},
promote(queue, jobId) {
const keys = [
queue.keys.delayed,
queue.keys.wait,
queue.keys.paused,
queue.keys['meta-paused'],
queue.keys.priority
];
const args = [queue.toKey(''), jobId, queue.token];
return queue.client.promote(keys.concat(args));
},
/**
* Looks for unlocked jobs in the active queue.
*
* The job was being worked on, but the worker process died and it failed to renew the lock.
* We call these jobs 'stalled'. This is the most common case. We resolve these by moving them
* back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,
* (e.g. if the job handler keeps crashing), we limit the number stalled job recoveries to settings.maxStalledCount.
*/
moveUnlockedJobsToWait(queue) {
const keys = [
queue.keys.stalled,
queue.keys.wait,
queue.keys.active,
queue.keys.failed,
queue.keys['stalled-check'],
queue.keys['meta-paused'],
queue.keys.paused
];
const args = [
queue.settings.maxStalledCount,
queue.toKey(''),
Date.now(),
queue.settings.stalledInterval
];
return queue.client.moveStalledJobsToWait(keys.concat(args));
},
cleanJobsInSet(queue, set, ts, limit) {
return queue.client.cleanJobsInSet([
queue.toKey(set),
queue.toKey('priority'),
queue.keys.limiter,
queue.toKey(''),
ts,
limit || 0,
set
]);
},
retryJobArgs(job, ignoreLock) {
const queue = job.queue;
const jobId = job.id;
const keys = _.map(
['active', 'wait', jobId, 'meta-paused', 'paused', 'stalled', 'priority'],
name => {
return queue.toKey(name);
}
);
const pushCmd = (job.opts.lifo ? 'R' : 'L') + 'PUSH';
return keys.concat([pushCmd, jobId, ignoreLock ? '0' : job.queue.token]);
},
/**
* Attempts to reprocess a job
*
* @param {Job} job
* @param {Object} options
* @param {String} options.state The expected job state. If the job is not found
* on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'
*
* @return {Promise<Number>} Returns a promise that evaluates to a return code:
* 1 means the operation was a success
* 0 means the job does not exist
* -1 means the job is currently locked and can't be retried.
* -2 means the job was not found in the expected set
*/
reprocessJob(job, options) {
const queue = job.queue;
const keys = [
queue.toKey(job.id),
queue.toKey(job.id) + ':lock',
queue.toKey(options.state),
queue.toKey('wait'),
queue.toKey('meta-paused'),
queue.toKey('paused')
];
const args = [
job.id,
(job.opts.lifo ? 'R' : 'L') + 'PUSH',
queue.token,
Date.now()
];
return queue.client.reprocessJob(keys.concat(args));
},
obliterate(queue, opts) {
const client = queue.client;
const keys = [queue.keys['meta-paused'], queue.toKey('')];
const args = [opts.count, opts.force ? 'force' : null];
return client.obliterate(keys.concat(args)).then(result => {
if (result < 0) {
switch (result) {
case -1:
throw new Error('Cannot obliterate non-paused queue');
case -2:
throw new Error('Cannot obliterate queue with active jobs');
}
}
return result;
});
}
};
module.exports = scripts;
function array2obj(arr) {
const obj = {};
for (let i = 0; i < arr.length; i += 2) {
obj[arr[i]] = arr[i + 1];
}
return obj;
}
function raw2jobData(raw) {
if (raw) {
const jobData = raw[0];
if (jobData.length) {
const job = array2obj(jobData);
return [job, raw[1]];
}
}
return [];
}
+141
View File
@@ -0,0 +1,141 @@
'use strict';
const content = `--[[
Adds a job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- if delayed:
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
- if not delayed
- Adds the jobId to the wait/paused list in one of three ways:
- LIFO
- FIFO
- prioritized.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta-paused'
KEYS[4] 'id'
KEYS[5] 'delayed'
KEYS[6] 'priority'
ARGV[1] key prefix,
ARGV[2] custom id (will not generate one automatically)
ARGV[3] name
ARGV[4] data (json stringified job data)
ARGV[5] opts (json stringified job opts)
ARGV[6] timestamp
ARGV[7] delay
ARGV[8] delayedTimestamp
ARGV[9] priority
ARGV[10] LIFO
ARGV[11] token
ARGV[12] debounce key
ARGV[13] debounceId
ARGV[14] debounceTtl
]]
local jobId
local jobIdKey
local rcall = redis.call
-- Includes
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
--[[
Function to debounce a job.
]]
local function debounceJob(prefixKey, debounceId, ttl, jobId, debounceKey, token)
if debounceId ~= "" then
local debounceKeyExists
if ttl ~= "" then
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
else
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
end
if debounceKeyExists then
local currentDebounceJobId = rcall('GET', debounceKey)
rcall("PUBLISH", prefixKey .. "debounced@" .. token, currentDebounceJobId)
return currentDebounceJobId
end
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
local jobCounter = rcall("INCR", KEYS[4])
if ARGV[2] == "" then
jobId = jobCounter
jobIdKey = ARGV[1] .. jobId
else
jobId = ARGV[2]
jobIdKey = ARGV[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
rcall("PUBLISH", ARGV[1] .. "duplicated@" .. ARGV[11], jobId)
return jobId .. "" -- convert to string
end
end
local debounceKey = ARGV[12]
local opts = cmsgpack.unpack(ARGV[5])
local debouncedJobId = debounceJob(ARGV[1], ARGV[13], ARGV[14],
jobId, debounceKey, ARGV[11])
if debouncedJobId then
return debouncedJobId
end
local debounceId = ARGV[13]
local optionalValues = {}
if debounceId ~= "" then
table.insert(optionalValues, "deid")
table.insert(optionalValues, debounceId)
end
-- Store the job.
rcall("HMSET", jobIdKey, "name", ARGV[3], "data", ARGV[4], "opts", opts, "timestamp",
ARGV[6], "delay", ARGV[7], "priority", ARGV[9], unpack(optionalValues))
-- Check if job is delayed
local delayedTimestamp = tonumber(ARGV[8])
if(delayedTimestamp ~= 0) then
local timestamp = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
rcall("ZADD", KEYS[5], timestamp, jobId)
rcall("PUBLISH", KEYS[5], delayedTimestamp)
else
local target
-- Whe check for the meta-paused key to decide if we are paused or not
-- (since an empty list and !EXISTS are not really the same)
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
-- Standard or priority add
local priority = tonumber(ARGV[9])
if priority == 0 then
-- LIFO or FIFO
rcall(ARGV[10], target, jobId)
else
addJobWithPriority(KEYS[6], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[1] .. "ing@" .. ARGV[11], jobId)
end
return jobId .. "" -- convert to string
`;
module.exports = {
name: 'addJob',
content,
keys: 6,
};
+30
View File
@@ -0,0 +1,30 @@
'use strict';
const content = `--[[
Add job log
Input:
KEYS[1] job id key
KEYS[2] job logs key
ARGV[1] id
ARGV[2] log
ARGV[3] keepLogs
Output:
-1 - Missing job.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
local logCount = rcall("RPUSH", KEYS[2], ARGV[2])
if ARGV[3] ~= '' then
local keepLogs = tonumber(ARGV[3])
rcall("LTRIM", KEYS[2], -keepLogs, -1)
return math.min(keepLogs, logCount)
end
return logCount
else
return -1
end
`;
module.exports = {
name: 'addLog',
content,
keys: 2,
};
+139
View File
@@ -0,0 +1,139 @@
'use strict';
const content = `--[[
Remove jobs from the specific set.
Input:
KEYS[1] set key,
KEYS[2] priority key
KEYS[3] rate limiter key
ARGV[1] prefix key
ARGV[2] maxTimestamp
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
]]
local setKey = KEYS[1]
local priorityKey = KEYS[2]
local rateLimiterKey = KEYS[3]
local prefixKey = ARGV[1]
local maxTimestamp = ARGV[2]
local limitStr = ARGV[3]
local setName = ARGV[4]
local isList = false
local rcall = redis.call
-- Includes
--[[
Function to remove debounce key.
]]
local function removeDebounceKey(prefixKey, jobKey)
local debounceId = rcall("HGET", jobKey, "deid")
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
rcall("DEL", debounceKey)
end
end
if setName == "wait" or setName == "active" or setName == "paused" then
isList = true
end
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
-- of items in a sorted set only run a single iteration. If we simply used
-- ZRANGE, we may take a long time traversing through jobs that are within the
-- grace period.
local function shouldUseZRangeByScore(isList, limit)
return not isList and limit > 0
end
local function getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
if isList then
return rcall("LRANGE", setKey, rangeStart, rangeEnd)
elseif shouldUseZRangeByScore(isList, limit) then
return rcall("ZRANGEBYSCORE", setKey, 0, maxTimestamp, "LIMIT", 0, limit)
else
return rcall("ZRANGE", setKey, rangeStart, rangeEnd)
end
end
local limit = tonumber(limitStr)
local rangeStart = 0
local rangeEnd = -1
-- If we're only deleting _n_ items, avoid retrieving all items
-- for faster performance
--
-- Start from the tail of the list, since that's where oldest elements
-- are generally added for FIFO lists
if limit > 0 then
rangeStart = -1 - limit + 1
rangeEnd = -1
end
local jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
local deleted = {}
local deletedCount = 0
local jobTS
-- Run this loop:
-- - Once, if limit is -1 or 0
-- - As many times as needed if limit is positive
while ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do
local jobIdsLen = #jobIds
for i, jobId in ipairs(jobIds) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = prefixKey .. jobId
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Find the right timestamp of the job to compare to maxTimestamp:
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
-- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
-- * timestamp is the original job submission time
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
for _, ts in ipairs(rcall("HMGET", jobKey, "finishedOn", "processedOn", "timestamp")) do
if (ts) then
jobTS = ts
break
end
end
if (not jobTS or jobTS < maxTimestamp) then
if isList then
-- Job ids can't be the empty string. Use the empty string as a
-- deletion marker. The actual deletion will occur at the end of the
-- script.
rcall("LSET", setKey, rangeEnd - jobIdsLen + i, "")
else
rcall("ZREM", setKey, jobId)
end
rcall("ZREM", priorityKey, jobId)
if setName ~= "completed" and setName ~= "failed" then
removeDebounceKey(prefixKey, jobKey)
end
rcall("DEL", jobKey)
rcall("DEL", jobKey .. ":logs")
-- delete keys related to rate limiter
-- NOTE: this code is unncessary for other sets than wait, paused and delayed.
local limiterIndexTable = rateLimiterKey .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
deletedCount = deletedCount + 1
table.insert(deleted, jobId)
end
end
end
-- If we didn't have a limit or used the single-iteration ZRANGEBYSCORE
-- function, return immediately. We should have deleted all the jobs we can
if limit <= 0 or shouldUseZRangeByScore(isList, limit) then
break
end
if deletedCount < limit then
-- We didn't delete enough. Look for more to delete
rangeStart = rangeStart - limit
rangeEnd = rangeEnd - limit
jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
end
end
if isList then
rcall("LREM", setKey, 0, "")
end
return deleted
`;
module.exports = {
name: 'cleanJobsInSet',
content,
keys: 3,
};
+26
View File
@@ -0,0 +1,26 @@
'use strict';
const content = `--[[
Extend lock and removes the job from the stalled set.
Input:
KEYS[1] 'lock',
KEYS[2] 'stalled'
ARGV[1] token
ARGV[2] lock duration in milliseconds
ARGV[3] jobid
Output:
"1" if lock extended succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
rcall("SREM", KEYS[2], ARGV[3])
return 1
end
end
return 0
`;
module.exports = {
name: 'extendLock',
content,
keys: 2,
};
+49
View File
@@ -0,0 +1,49 @@
'use strict';
const content = `--[[
Get counts per provided states
Input:
KEYS[1] wait key
KEYS[2] paused key
KEYS[3] meta-paused key
KEYS[4] priority key
ARGV[1...] priorities
]]
local rcall = redis.call
local results = {}
local prioritizedKey = KEYS[4]
-- Includes
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
for i = 1, #ARGV do
local priority = tonumber(ARGV[i])
if priority == 0 then
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
local count = rcall("LLEN", target) - rcall("ZCARD", prioritizedKey)
if count < 0 then
-- considering when last waiting job is moved to active before
-- removing priority reference
results[#results+1] = 0
else
results[#results+1] = count
end
else
results[#results+1] = rcall("ZCOUNT", prioritizedKey,
priority, priority)
end
end
return results
`;
module.exports = {
name: 'getCountsPerPriority',
content,
keys: 4,
};
+29
View File
@@ -0,0 +1,29 @@
'use strict';
module.exports = {
["addJob-6"]: require('./addJob-6'),
["addLog-2"]: require('./addLog-2'),
["cleanJobsInSet-3"]: require('./cleanJobsInSet-3'),
["extendLock-2"]: require('./extendLock-2'),
["getCountsPerPriority-4"]: require('./getCountsPerPriority-4'),
["isFinished-2"]: require('./isFinished-2'),
["isJobInList-1"]: require('./isJobInList-1'),
["moveStalledJobsToWait-7"]: require('./moveStalledJobsToWait-7'),
["moveToActive-8"]: require('./moveToActive-8'),
["moveToDelayed-4"]: require('./moveToDelayed-4'),
["moveToFinished-9"]: require('./moveToFinished-9'),
["obliterate-2"]: require('./obliterate-2'),
["pause-5"]: require('./pause-5'),
["promote-5"]: require('./promote-5'),
["releaseLock-1"]: require('./releaseLock-1'),
["removeJob-11"]: require('./removeJob-11'),
["removeJobs-8"]: require('./removeJobs-8'),
["removeRepeatable-2"]: require('./removeRepeatable-2'),
["reprocessJob-6"]: require('./reprocessJob-6'),
["retryJob-7"]: require('./retryJob-7'),
["retryJobs-5"]: require('./retryJobs-5'),
["saveStacktrace-1"]: require('./saveStacktrace-1'),
["takeLock-1"]: require('./takeLock-1'),
["updateData-1"]: require('./updateData-1'),
["updateDelaySet-6"]: require('./updateDelaySet-6'),
["updateProgress-2"]: require('./updateProgress-2'),
}
+25
View File
@@ -0,0 +1,25 @@
'use strict';
const content = `--[[
Checks if a job is finished (.i.e. is in the completed or failed set)
Input:
KEYS[1] completed key
KEYS[2] failed key
ARGV[1] job id
Output:
0 - not finished.
1 - completed.
2 - failed.
]]
if redis.call("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return 1
end
if redis.call("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return 2
end
return redis.call("ZSCORE", KEYS[2], ARGV[1])
`;
module.exports = {
name: 'isFinished',
content,
keys: 2,
};
+25
View File
@@ -0,0 +1,25 @@
'use strict';
const content = `--[[
Checks if job is in a given list.
Input:
KEYS[1]
ARGV[1]
Output:
1 if element found in the list.
]]
local function item_in_list (list, item)
for _, v in pairs(list) do
if v == item then
return 1
end
end
return nil
end
local items = redis.call("LRANGE", KEYS[1] , 0, -1)
return item_in_list(items, ARGV[1])
`;
module.exports = {
name: 'isJobInList',
content,
keys: 1,
};
+156
View File
@@ -0,0 +1,156 @@
'use strict';
const content = `--[[
Move stalled jobs to wait.
Input:
KEYS[1] 'stalled' (SET)
KEYS[2] 'wait', (LIST)
KEYS[3] 'active', (LIST)
KEYS[4] 'failed', (ZSET)
KEYS[5] 'stalled-check', (KEY)
KEYS[6] 'meta-paused', (KEY)
KEYS[7] 'paused', (LIST)
ARGV[1] Max stalled job count
ARGV[2] queue.toKey('')
ARGV[3] timestamp
ARGV[4] max check time
Events:
'stalled' with stalled job id.
]]
local rcall = redis.call
-- Includes
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
--[[
Function to remove debounce key if needed.
]]
local function removeDebounceKeyIfNeeded(prefixKey, debounceId)
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
local pttl = rcall("PTTL", debounceKey)
if pttl == 0 or pttl == -1 then
rcall("DEL", debounceKey)
end
end
end
local function removeJob(jobId, baseKey)
local jobKey = baseKey .. jobId
rcall("DEL", jobKey, jobKey .. ':logs')
end
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
for i, jobId in ipairs(jobIds) do
removeJob(jobId, prefix)
end
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
for i, jobId in ipairs(jobIds) do
removeJob(jobId, prefix)
end
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
end
-- Check if we need to check for stalled jobs now.
if rcall("EXISTS", KEYS[5]) == 1 then
return {{}, {}}
end
rcall("SET", KEYS[5], ARGV[3], "PX", ARGV[4])
-- Move all stalled jobs to wait
local stalling = rcall('SMEMBERS', KEYS[1])
local stalled = {}
local failed = {}
if(#stalling > 0) then
rcall('DEL', KEYS[1])
local MAX_STALLED_JOB_COUNT = tonumber(ARGV[1])
-- Remove from active list
for i, jobId in ipairs(stalling) do
local jobKey = ARGV[2] .. jobId
-- Check that the lock is also missing, then we can handle this job as really stalled.
if(rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Remove from the active queue.
local removed = rcall("LREM", KEYS[3], 1, jobId)
if(removed > 0) then
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter", 1)
if(stalledCount > MAX_STALLED_JOB_COUNT) then
local jobAttributes = rcall("HMGET", jobKey, "opts", "deid")
local opts = cjson.decode(jobAttributes[1])
local removeOnFailType = type(opts["removeOnFail"])
rcall("ZADD", KEYS[4], ARGV[3], jobId)
rcall("HMSET", jobKey, "failedReason", "job stalled more than allowable limit",
"finishedOn", ARGV[3])
removeDebounceKeyIfNeeded(ARGV[2], jobAttributes[2])
rcall("PUBLISH", KEYS[4], '{"jobId":"' .. jobId .. '", "val": "job stalled more than maxStalledCount"}')
if removeOnFailType == "number" then
removeJobsByMaxCount(opts["removeOnFail"],
KEYS[4], ARGV[2])
elseif removeOnFailType == "boolean" then
if opts["removeOnFail"] then
removeJob(jobId, ARGV[2])
rcall("ZREM", KEYS[4], jobId)
end
elseif removeOnFailType ~= "nil" then
local maxAge = opts["removeOnFail"]["age"]
local maxCount = opts["removeOnFail"]["count"]
if maxAge ~= nil then
removeJobsByMaxAge(ARGV[3], maxAge,
KEYS[4], ARGV[2])
end
if maxCount ~= nil and maxCount > 0 then
removeJobsByMaxCount(maxCount, KEYS[4],
ARGV[2])
end
end
table.insert(failed, jobId)
else
local target = getTargetQueueList(KEYS[6], KEYS[2], KEYS[7])
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
rcall("RPUSH", target, jobId)
rcall('PUBLISH', KEYS[1] .. '@', jobId)
table.insert(stalled, jobId)
end
end
end
end
end
-- Mark potentially stalled jobs
local active = rcall('LRANGE', KEYS[3], 0, -1)
if (#active > 0) then
for from, to in batches(#active, 7000) do
rcall('SADD', KEYS[1], unpack(active, from, to))
end
end
return {failed, stalled}
`;
module.exports = {
name: 'moveStalledJobsToWait',
content,
keys: 7,
};
+130
View File
@@ -0,0 +1,130 @@
'use strict';
const content = `--[[
Move next job to be processed to active, lock it and fetch its data. The job
may be delayed, in that case we need to move it to the delayed set instead.
This operation guarantees that the worker owns the job during the locks
expiration time. The worker is responsible of keeping the lock fresh
so that no other worker picks this job again.
Input:
KEYS[1] wait key
KEYS[2] active key
KEYS[3] priority key
KEYS[4] active event key
KEYS[5] stalled key
-- Rate limiting
KEYS[6] rate limiter key
KEYS[7] delayed key
--
KEYS[8] drained key
ARGV[1] key prefix
ARGV[2] lock token
ARGV[3] lock duration in milliseconds
ARGV[4] timestamp
ARGV[5] optional jobid
ARGV[6] optional jobs per time unit (rate limiter)
ARGV[7] optional time unit (rate limiter)
ARGV[8] optional do not do anything with job if rate limit hit
ARGV[9] optional rate limit by key
]]
local rcall = redis.call
local rateLimit = function(jobId, maxJobs)
local rateLimiterKey = KEYS[6];
local limiterIndexTable = rateLimiterKey .. ":index"
-- Rate limit by group?
if(ARGV[9]) then
local group = string.match(jobId, "[^:]+$")
if group ~= nil then
rateLimiterKey = rateLimiterKey .. ":" .. group
end
end
-- -- key for storing rate limited jobs
-- When a job has been previously rate limited it should be part of this set
-- if the job is back here means that the delay time for this job has passed and now we should
-- be able to process it again.
local limitedSetKey = rateLimiterKey .. ":limited"
local delay = 0
-- -- Check if job was already limited
local isLimited = rcall("SISMEMBER", limitedSetKey, jobId);
if isLimited == 1 then
-- Remove from limited zset since we are going to try to process it
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
else
-- If not, check if there are any limited jobs
-- If the job has not been rate limited, we should check if there are any other rate limited jobs, because if that
-- is the case we do not want to process this job, just calculate a delay for it and put it to "sleep".
local numLimitedJobs = rcall("SCARD", limitedSetKey)
if numLimitedJobs > 0 then
-- Note, add some slack to compensate for drift.
delay = ((numLimitedJobs * ARGV[7] * 1.1) / maxJobs) + tonumber(rcall("PTTL", rateLimiterKey))
end
end
local jobCounter = tonumber(rcall("GET", rateLimiterKey))
if(jobCounter == nil) then
jobCounter = 0
end
-- check if rate limit hit
if (delay == 0) and (jobCounter >= maxJobs) then
-- Seems like there are no current rated limited jobs, but the jobCounter has exceeded the number of jobs for this unit of time so we need to rate limit this job.
local exceedingJobs = jobCounter - maxJobs
delay = tonumber(rcall("PTTL", rateLimiterKey)) + ((exceedingJobs) * ARGV[7]) / maxJobs
end
if delay > 0 then
local bounceBack = ARGV[8]
if bounceBack == 'false' then
local timestamp = delay + tonumber(ARGV[4])
-- put job into delayed queue
rcall("ZADD", KEYS[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId)
rcall("PUBLISH", KEYS[7], timestamp)
rcall("SADD", limitedSetKey, jobId)
-- store index so that we can delete rate limited data
rcall("HSET", limiterIndexTable, jobId, limitedSetKey)
end
-- remove from active queue
rcall("LREM", KEYS[2], 1, jobId)
return true
else
-- false indicates not rate limited
-- increment jobCounter only when a job is not rate limited
if (jobCounter == 0) then
rcall("PSETEX", rateLimiterKey, ARGV[7], 1)
else
rcall("INCR", rateLimiterKey)
end
return false
end
end
local jobId = ARGV[5]
if jobId ~= '' then
-- clean stalled key
rcall("SREM", KEYS[5], jobId)
else
-- move from wait to active
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
end
if jobId then
-- Check if we need to perform rate limiting.
local maxJobs = tonumber(ARGV[6])
if maxJobs then
if rateLimit(jobId, maxJobs) then
return
end
end
-- get a lock
local jobKey = ARGV[1] .. jobId
local lockKey = jobKey .. ':lock'
rcall("SET", lockKey, ARGV[2], "PX", ARGV[3])
-- remove from priority
rcall("ZREM", KEYS[3], jobId)
rcall("PUBLISH", KEYS[4], jobId)
rcall("HSET", jobKey, "processedOn", ARGV[4])
return {rcall("HGETALL", jobKey), jobId} -- get job data
else
rcall("PUBLISH", KEYS[8], "")
end
`;
module.exports = {
name: 'moveToActive',
content,
keys: 8,
};
+59
View File
@@ -0,0 +1,59 @@
'use strict';
const content = `--[[
Moves job from active to delayed set.
Input:
KEYS[1] active key
KEYS[2] delayed key
KEYS[3] job key
KEYS[4] stalled key
ARGV[1] delayedTimestamp
ARGV[2] the id of the job
ARGV[3] queue token
Output:
0 - OK
-1 - Missing job.
-2 - Job is locked.
Events:
- delayed key.
]]
local rcall = redis.call
-- Includes
local function removeLock(jobKey, stalledKey, token, jobId)
if token ~= "0" then
local lockKey = jobKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", stalledKey, jobId)
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
return 0
end
if rcall("EXISTS", KEYS[3]) == 1 then
local errorCode = removeLock(KEYS[3], KEYS[4], ARGV[3], ARGV[2])
if errorCode < 0 then
return errorCode
end
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[2])
if numRemovedElements < 1 then return -3 end
local score = tonumber(ARGV[1])
rcall("ZADD", KEYS[2], score, ARGV[2])
rcall("PUBLISH", KEYS[2], (score / 0x1000))
return 0
else
return -1
end
`;
module.exports = {
name: 'moveToDelayed',
content,
keys: 4,
};
+196
View File
@@ -0,0 +1,196 @@
'use strict';
const content = `--[[
Move job from active to a finished status (completed or failed)
A job can only be moved to completed if it was active.
The job must be locked before it can be moved to a finished status,
and the lock must be released in this script.
Input:
KEYS[1] active key
KEYS[2] completed/failed key
KEYS[3] jobId key
KEYS[4] wait key
KEYS[5] priority key
KEYS[6] active event key
KEYS[7] delayed key
KEYS[8] stalled key
KEYS[9] metrics key
ARGV[1] jobId
ARGV[2] timestamp
ARGV[3] msg property
ARGV[4] return value / failed reason
ARGV[5] token
ARGV[6] shouldRemove
ARGV[7] event data (? maybe just send jobid).
ARGV[8] should fetch next job
ARGV[9] base key
ARGV[10] lock token
ARGV[11] lock duration in milliseconds
ARGV[12] maxMetricsSize
Output:
0 OK
-1 Missing key.
-2 Missing lock.
-3 - Job not in active set.
Events:
'completed/failed'
]]
local rcall = redis.call
-- Includes
--[[
Functions to collect metrics based on a current and previous count of jobs.
Granualarity is fixed at 1 minute.
]]
-- Includes
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
local function collectMetrics(metaKey, dataPointsList, maxDataPoints, timestamp)
-- Increment current count
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
-- Compute how many data points we need to add to the list, N.
local prevTS = rcall("HGET", metaKey, "prevTS")
if not prevTS then
-- If prevTS is nil, set it to the current timestamp
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
return
end
local N = math.min(math.floor(timestamp / 60000) - math.floor(prevTS / 60000), tonumber(maxDataPoints))
if N > 0 then
local delta = count - rcall("HGET", metaKey, "prevCount")
-- If N > 1, add N-1 zeros to the list
if N > 1 then
local points = {}
points[1] = delta
for i = 2, N do points[i] = 0 end
for from, to in batches(#points, 7000) do
rcall("LPUSH", dataPointsList, unpack(points, from, to))
end
else
-- LPUSH delta to the list
rcall("LPUSH", dataPointsList, delta)
end
-- LTRIM to keep list to its max size
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
-- update prev count with current count
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
end
end
local function removeLock(jobKey, stalledKey, token, jobId)
if token ~= "0" then
local lockKey = jobKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", stalledKey, jobId)
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
return 0
end
--[[
Function to remove debounce key if needed.
]]
local function removeDebounceKeyIfNeeded(prefixKey, debounceId)
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
local pttl = rcall("PTTL", debounceKey)
if pttl == 0 or pttl == -1 then
rcall("DEL", debounceKey)
end
end
end
if rcall("EXISTS", KEYS[3]) == 1 then -- // Make sure job exists
local errorCode = removeLock(KEYS[3], KEYS[8], ARGV[5], ARGV[1])
if errorCode < 0 then
return errorCode
end
-- Remove from active list (if not active we shall return error)
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[1])
if numRemovedElements < 1 then return -3 end
local debounceId = rcall("HGET", KEYS[3], "deid")
removeDebounceKeyIfNeeded(ARGV[9], debounceId)
-- Remove job?
local keepJobs = cmsgpack.unpack(ARGV[6])
local maxCount = keepJobs['count']
local maxAge = keepJobs['age']
local targetSet = KEYS[2]
local timestamp = ARGV[2]
if maxCount ~= 0 then
-- Add to complete/failed set
rcall("ZADD", targetSet, timestamp, ARGV[1])
rcall("HMSET", KEYS[3], ARGV[3], ARGV[4], "finishedOn", timestamp) -- "returnvalue" / "failedReason" and "finishedOn"
local function removeJobs(jobIds)
for i, jobId in ipairs(jobIds) do
local jobKey = ARGV[9] .. jobId
local jobLogKey = jobKey .. ':logs'
rcall("DEL", jobKey, jobLogKey)
end
end
-- Remove old jobs?
if maxAge ~= nil then
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
removeJobs(jobIds)
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
if maxCount ~= nil and maxCount > 0 then
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
removeJobs(jobIds)
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1));
end
else
local jobLogKey = KEYS[3] .. ':logs'
rcall("DEL", KEYS[3], jobLogKey)
end
-- Collect metrics
if ARGV[12] ~= "" then
collectMetrics(KEYS[9], KEYS[9]..':data', ARGV[12], timestamp)
end
rcall("PUBLISH", targetSet, ARGV[7])
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
-- and not rate limited.
if (ARGV[8] == "1") then
-- move from wait to active
local jobId = rcall("RPOPLPUSH", KEYS[4], KEYS[1])
if jobId then
local jobKey = ARGV[9] .. jobId
local lockKey = jobKey .. ':lock'
-- get a lock
rcall("SET", lockKey, ARGV[11], "PX", ARGV[10])
rcall("ZREM", KEYS[5], jobId) -- remove from priority
rcall("PUBLISH", KEYS[6], jobId)
rcall("HSET", jobKey, "processedOn", ARGV[2])
return {rcall("HGETALL", jobKey), jobId} -- get job data
end
end
return 0
else
return -1
end
`;
module.exports = {
name: 'moveToFinished',
content,
keys: 9,
};
+106
View File
@@ -0,0 +1,106 @@
'use strict';
const content = `--[[
Completely obliterates a queue and all of its contents
Input:
KEYS[1] meta-paused
KEYS[2] base
ARGV[1] count
ARGV[2] force
]]
-- This command completely destroys a queue including all of its jobs, current or past
-- leaving no trace of its existence. Since this script needs to iterate to find all the job
-- keys, consider that this call may be slow for very large queues.
-- The queue needs to be "paused" or it will return an error
-- If the queue has currently active jobs then the script by default will return error,
-- however this behaviour can be overrided using the 'force' option.
local maxCount = tonumber(ARGV[1])
local baseKey = KEYS[2]
local rcall = redis.call
-- Includes
--[[
Function to remove debounce key.
]]
local function removeDebounceKey(prefixKey, jobKey)
local debounceId = rcall("HGET", jobKey, "deid")
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
rcall("DEL", debounceKey)
end
end
local function getListItems(keyName, max)
return rcall('LRANGE', keyName, 0, max - 1)
end
local function getZSetItems(keyName, max)
return rcall('ZRANGE', keyName, 0, max - 1)
end
local function removeJobs(baseKey, keys)
for i, key in ipairs(keys) do
local jobKey = baseKey .. key
rcall("DEL", jobKey, jobKey .. ':logs')
removeDebounceKey(baseKey, jobKey)
end
maxCount = maxCount - #keys
end
local function removeListJobs(keyName, max)
local jobs = getListItems(keyName, max)
removeJobs(baseKey, jobs)
rcall("LTRIM", keyName, #jobs, -1)
end
local function removeZSetJobs(keyName, max)
local jobs = getZSetItems(keyName, max)
removeJobs(baseKey, jobs)
if (#jobs > 0) then rcall("ZREM", keyName, unpack(jobs)) end
end
local function removeLockKeys(keys)
for i, key in ipairs(keys) do rcall("DEL", baseKey .. key .. ':lock') end
end
-- 1) Check if paused, if not return with error.
if rcall("EXISTS", KEYS[1]) ~= 1 then
return -1 -- Error, NotPaused
end
-- 2) Check if there are active jobs, if there are and not "force" return error.
local activeKey = baseKey .. 'active'
local activeJobs = getListItems(activeKey, maxCount)
if (#activeJobs > 0) then
if (ARGV[2] == "") then
return -2 -- Error, ExistsActiveJobs
end
end
removeLockKeys(activeJobs)
removeJobs(baseKey, activeJobs)
rcall("LTRIM", activeKey, #activeJobs, -1)
if (maxCount <= 0) then return 1 end
local waitKey = baseKey .. 'paused'
removeListJobs(waitKey, maxCount)
if (maxCount <= 0) then return 1 end
local delayedKey = baseKey .. 'delayed'
removeZSetJobs(delayedKey, maxCount)
if (maxCount <= 0) then return 1 end
local completedKey = baseKey .. 'completed'
removeZSetJobs(completedKey, maxCount)
if (maxCount <= 0) then return 1 end
local failedKey = baseKey .. 'failed'
removeZSetJobs(failedKey, maxCount)
if (maxCount <= 0) then return 1 end
if (maxCount > 0) then
rcall("DEL", baseKey .. 'priority')
rcall("DEL", baseKey .. 'stalled-check')
rcall("DEL", baseKey .. 'stalled')
rcall("DEL", baseKey .. 'meta-paused')
rcall("DEL", baseKey .. 'meta')
rcall("DEL", baseKey .. 'id')
rcall("DEL", baseKey .. 'repeat')
rcall("DEL", baseKey .. 'metrics:completed')
rcall("DEL", baseKey .. 'metrics:completed:data')
rcall("DEL", baseKey .. 'metrics:failed')
rcall("DEL", baseKey .. 'metrics:failed:data')
return 0
else
return 1
end
`;
module.exports = {
name: 'obliterate',
content,
keys: 2,
};
+33
View File
@@ -0,0 +1,33 @@
'use strict';
const content = `--[[
Pauses or resumes a queue globably.
Input:
KEYS[1] 'wait' or 'paused''
KEYS[2] 'paused' or 'wait'
KEYS[3] 'meta-paused'
KEYS[4] 'paused' o 'resumed' event.
KEYS[5] 'meta' this key is only used in BullMQ and above.
ARGV[1] 'paused' or 'resumed'
Event:
publish paused or resumed event.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then
rcall("RENAME", KEYS[1], KEYS[2])
end
if ARGV[1] == "paused" then
rcall("SET", KEYS[3], 1)
-- for forwards compatibility
rcall("HSET", KEYS[5], "paused", 1)
else
rcall("DEL", KEYS[3])
-- for forwards compatibility
rcall("HDEL", KEYS[5], "paused")
end
rcall("PUBLISH", KEYS[4], ARGV[1])
`;
module.exports = {
name: 'pause',
content,
keys: 5,
};
+65
View File
@@ -0,0 +1,65 @@
'use strict';
const content = `--[[
Promotes a job that is currently "delayed" to the "waiting" state
Input:
KEYS[1] 'delayed'
KEYS[2] 'wait'
KEYS[3] 'paused'
KEYS[4] 'meta-paused'
KEYS[5] 'priority'
ARGV[1] queue.toKey('')
ARGV[2] jobId
ARGV[3] queue token
Events:
'waiting'
]]
local rcall = redis.call;
local jobId = ARGV[2]
-- Includes
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
if rcall("ZREM", KEYS[1], jobId) == 1 then
local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", target, jobId)
else
addJobWithPriority(KEYS[5], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[2] .. "ing@" .. ARGV[3], jobId)
rcall("HSET", ARGV[1] .. jobId, "delay", 0)
return 0
else
return -1
end
`;
module.exports = {
name: 'promote',
content,
keys: 5,
};
+22
View File
@@ -0,0 +1,22 @@
'use strict';
const content = `--[[
Release lock
Input:
KEYS[1] 'lock',
ARGV[1] token
ARGV[2] lock duration in milliseconds
Output:
"OK" if lock extented succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
return rcall("DEL", KEYS[1])
else
return 0
end
`;
module.exports = {
name: 'releaseLock',
content,
keys: 1,
};
+65
View File
@@ -0,0 +1,65 @@
'use strict';
const content = `--[[
Remove a job from all the queues it may be in as well as all its data.
In order to be able to remove a job, it must be unlocked.
Input:
KEYS[1] 'active',
KEYS[2] 'wait',
KEYS[3] 'delayed',
KEYS[4] 'paused',
KEYS[5] 'completed',
KEYS[6] 'failed',
KEYS[7] 'priority',
KEYS[8] jobId key
KEYS[9] job logs
KEYS[10] rate limiter index table
KEYS[11] prefix key
ARGV[1] jobId
ARGV[2] lock token
Events:
'removed'
]]
-- TODO PUBLISH global event 'removed'
local rcall = redis.call
-- Includes
--[[
Function to remove debounce key.
]]
local function removeDebounceKey(prefixKey, jobKey)
local debounceId = rcall("HGET", jobKey, "deid")
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
rcall("DEL", debounceKey)
end
end
local lockKey = KEYS[8] .. ':lock'
local lock = rcall("GET", lockKey)
if not lock then -- or (lock == ARGV[2])) then
local jobId = ARGV[1]
rcall("LREM", KEYS[1], 0, jobId)
rcall("LREM", KEYS[2], 0, jobId)
rcall("ZREM", KEYS[3], jobId)
rcall("LREM", KEYS[4], 0, jobId)
rcall("ZREM", KEYS[5], jobId)
rcall("ZREM", KEYS[6], jobId)
rcall("ZREM", KEYS[7], jobId)
removeDebounceKey(KEYS[11], KEYS[8])
rcall("DEL", KEYS[8])
rcall("DEL", KEYS[9])
-- delete keys related to rate limiter
local limiterIndexTable = KEYS[10] .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
return 1
else
return 0
end
`;
module.exports = {
name: 'removeJob',
content,
keys: 11,
};
+60
View File
@@ -0,0 +1,60 @@
'use strict';
const content = `--[[
Remove all jobs matching a given pattern from all the queues they may be in as well as all its data.
In order to be able to remove any job, they must be unlocked.
Input:
KEYS[1] 'active',
KEYS[2] 'wait',
KEYS[3] 'delayed',
KEYS[4] 'paused',
KEYS[5] 'completed',
KEYS[6] 'failed',
KEYS[7] 'priority',
KEYS[8] 'rate-limiter'
ARGV[1] prefix
ARGV[2] pattern
ARGV[3] cursor
Events:
'removed'
]]
-- TODO PUBLISH global events 'removed'
local rcall = redis.call
local result = rcall("SCAN", ARGV[3], "MATCH", ARGV[1] .. ARGV[2])
local cursor = result[1];
local jobKeys = result[2];
local removed = {}
local prefixLen = string.len(ARGV[1]) + 1
for i, jobKey in ipairs(jobKeys) do
local keyTypeResp = rcall("TYPE", jobKey)
if keyTypeResp["ok"] == "hash" then
local jobId = string.sub(jobKey, prefixLen)
local lockKey = jobKey .. ':lock'
local lock = redis.call("GET", lockKey)
if not lock then
rcall("LREM", KEYS[1], 0, jobId)
rcall("LREM", KEYS[2], 0, jobId)
rcall("ZREM", KEYS[3], jobId)
rcall("LREM", KEYS[4], 0, jobId)
rcall("ZREM", KEYS[5], jobId)
rcall("ZREM", KEYS[6], jobId)
rcall("ZREM", KEYS[7], jobId)
rcall("DEL", jobKey)
rcall("DEL", jobKey .. ':logs')
-- delete keys related to rate limiter
local limiterIndexTable = KEYS[8] .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
table.insert(removed, jobId)
end
end
end
return {cursor, removed}
`;
module.exports = {
name: 'removeJobs',
content,
keys: 8,
};
+25
View File
@@ -0,0 +1,25 @@
'use strict';
const content = `--[[
Removes a repeatable job
Input:
KEYS[1] repeat jobs key
KEYS[2] delayed jobs key
ARGV[1] repeat job id
ARGV[2] repeat job key
ARGV[3] queue key
]]
local millis = redis.call("ZSCORE", KEYS[1], ARGV[2])
if(millis) then
-- Delete next programmed job.
local repeatJobId = ARGV[1] .. millis
if(redis.call("ZREM", KEYS[2], repeatJobId) == 1) then
redis.call("DEL", ARGV[3] .. repeatJobId)
end
end
redis.call("ZREM", KEYS[1], ARGV[2]);
`;
module.exports = {
name: 'removeRepeatable',
content,
keys: 2,
};
+51
View File
@@ -0,0 +1,51 @@
'use strict';
const content = `--[[
Attempts to reprocess a job
Input:
KEYS[1] job key
KEYS[2] job lock key
KEYS[3] job state
KEYS[4] wait key
KEYS[5] meta-pause
KEYS[6] paused key
ARGV[1] job.id,
ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
ARGV[3] token
ARGV[4] timestamp
Output:
1 means the operation was a success
0 means the job does not exist
-1 means the job is currently locked and can't be retried.
-2 means the job was not found in the expected set.
]]
local rcall = redis.call;
if (rcall("EXISTS", KEYS[1]) == 1) then
if (rcall("EXISTS", KEYS[2]) == 0) then
rcall("HDEL", KEYS[1], "finishedOn", "processedOn", "failedReason")
rcall("HSET", KEYS[1], "retriedOn", ARGV[4])
if (rcall("ZREM", KEYS[3], ARGV[1]) == 1) then
local target
if rcall("EXISTS", KEYS[5]) ~= 1 then
target = KEYS[4]
else
target = KEYS[6]
end
rcall(ARGV[2], target, ARGV[1])
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[4] .. "ing@" .. ARGV[3], ARGV[1])
return 1
else
return -2
end
else
return -1
end
else
return 0
end
`;
module.exports = {
name: 'reprocessJob',
content,
keys: 6,
};
+93
View File
@@ -0,0 +1,93 @@
'use strict';
const content = `--[[
Retries a failed job by moving it back to the wait queue.
Input:
KEYS[1] 'active',
KEYS[2] 'wait'
KEYS[3] jobId key
KEYS[4] 'meta-paused'
KEYS[5] 'paused'
KEYS[6] stalled key
KEYS[7] 'priority'
ARGV[1] pushCmd
ARGV[2] jobId
ARGV[3] token
Events:
'prefix:added'
Output:
0 - OK
-1 - Missing key
-2 - Job Not locked
-3 - Job not in active set
]]
local rcall = redis.call
-- Includes
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
local function removeLock(jobKey, stalledKey, token, jobId)
if token ~= "0" then
local lockKey = jobKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", stalledKey, jobId)
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
return 0
end
if rcall("EXISTS", KEYS[3]) == 1 then
local errorCode = removeLock(KEYS[3], KEYS[6], ARGV[3], ARGV[2])
if errorCode < 0 then
return errorCode
end
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[2])
if numRemovedElements < 1 then return -3 end
local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[5])
local priority = tonumber(rcall("HGET", KEYS[3], "priority")) or 0
if priority == 0 then
-- LIFO or FIFO
rcall(ARGV[1], target, ARGV[2])
else
addJobWithPriority(KEYS[7], priority, ARGV[2], target)
end
return 0
else
return -1
end
`;
module.exports = {
name: 'retryJob',
content,
keys: 7,
};
+63
View File
@@ -0,0 +1,63 @@
'use strict';
const content = `--[[
Attempts to retry all failed jobs
Input:
KEYS[1] base key
KEYS[2] failed state key
KEYS[3] wait state key
KEYS[4] 'meta-paused'
KEYS[5] 'paused'
ARGV[1] count
Output:
1 means the operation is not completed
0 means the operation is completed
]]
local baseKey = KEYS[1]
local maxCount = tonumber(ARGV[1])
local rcall = redis.call;
-- Includes
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
local function getZSetItems(keyName, max)
return rcall('ZRANGE', keyName, 0, max - 1)
end
local jobs = getZSetItems(KEYS[2], maxCount)
if (#jobs > 0) then
for i, key in ipairs(jobs) do
local jobKey = baseKey .. key
rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason")
end
local target
if rcall("EXISTS", KEYS[4]) ~= 1 then
target = KEYS[3]
else
target = KEYS[5]
end
for from, to in batches(#jobs, 7000) do
rcall("ZREM", KEYS[2], unpack(jobs, from, to))
rcall("LPUSH", target, unpack(jobs, from, to))
end
end
maxCount = maxCount - #jobs
if (maxCount <= 0) then return 1 end
return 0
`;
module.exports = {
name: 'retryJobs',
content,
keys: 5,
};
+26
View File
@@ -0,0 +1,26 @@
'use strict';
const content = `--[[
Save stacktrace and failedReason.
Input:
KEYS[1] job key
ARGV[1] stacktrace
ARGV[2] failedReason
ARGV[3] attemptsMade
Output:
0 - OK
-1 - Missing key
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then
rcall("HMSET", KEYS[1], "stacktrace", ARGV[1], "failedReason", ARGV[2],
"attemptsMade", ARGV[3])
return 0
else
return -1
end
`;
module.exports = {
name: 'saveStacktrace',
content,
keys: 1,
};
+21
View File
@@ -0,0 +1,21 @@
'use strict';
const content = `--[[
Takes a lock
Input:
KEYS[1] 'lock',
ARGV[1] token
ARGV[2] lock duration in milliseconds
Output:
"OK" if lock taken successfully.
]]
if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", ARGV[2]) then
return 1
else
return 0
end
`;
module.exports = {
name: 'takeLock',
content,
keys: 1,
};
+23
View File
@@ -0,0 +1,23 @@
'use strict';
const content = `--[[
Update job data
Input:
KEYS[1] Job id key
ARGV[1] data
Output:
0 - OK
-1 - Missing job.
]]
local rcall = redis.call
if rcall("EXISTS",KEYS[1]) == 1 then -- // Make sure job exists
rcall("HSET", KEYS[1], "data", ARGV[1])
return 0
else
return -1
end
`;
module.exports = {
name: 'updateData',
content,
keys: 1,
};
+76
View File
@@ -0,0 +1,76 @@
'use strict';
const content = `--[[
Updates the delay set, by picking a delayed job that should
be processed now.
Input:
KEYS[1] 'delayed'
KEYS[2] 'active'
KEYS[3] 'wait'
KEYS[4] 'priority'
KEYS[5] 'paused'
KEYS[6] 'meta-paused'
ARGV[1] queue.toKey('')
ARGV[2] delayed timestamp
ARGV[3] queue token
Events:
'removed'
]]
local rcall = redis.call;
-- Includes
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
-- Try to get as much as 1000 jobs at once
local jobs = rcall("ZRANGEBYSCORE", KEYS[1], 0, tonumber(ARGV[2]) * 0x1000, "LIMIT", 0, 1000)
if(#jobs > 0) then
rcall("ZREM", KEYS[1], unpack(jobs))
-- check if we need to use push in paused instead of waiting
local target = getTargetQueueList(KEYS[6], KEYS[3], KEYS[5])
for _, jobId in ipairs(jobs) do
-- Is this really needed?
rcall("LREM", KEYS[2], 0, jobId)
local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", target, jobId)
else
addJobWithPriority(KEYS[4], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[3] .. "ing@" .. ARGV[3], jobId)
rcall("HSET", ARGV[1] .. jobId, "delay", 0)
end
end
local nextTimestamp = rcall("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")[2]
if(nextTimestamp ~= nil) then
rcall("PUBLISH", KEYS[1], nextTimestamp / 0x1000)
end
return nextTimestamp
`;
module.exports = {
name: 'updateDelaySet',
content,
keys: 6,
};
+25
View File
@@ -0,0 +1,25 @@
'use strict';
const content = `--[[
Update job progress
Input:
KEYS[1] Job id key
KEYS[2] progress event key
ARGV[1] progress
ARGV[2] event data
Event:
progress(jobId, progress)
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
rcall("HSET", KEYS[1], "progress", ARGV[1])
rcall("PUBLISH", KEYS[2], ARGV[2])
return 0
else
return -1
end
`;
module.exports = {
name: 'updateProgress',
content,
keys: 2,
};
+142
View File
@@ -0,0 +1,142 @@
'use strict';
const _ = require('lodash');
const uuid = require('uuid');
/**
Timer Manager
Keep track of timers to ensure that disconnect() is
only called (via close()) at a time when it's safe
to do so.
Queues currently use two timers:
- The first one is used for delayed jobs and is
preemptible i.e. it is possible to close a queue
while delayed jobs are still pending (they will
be processed when the queue is resumed). This timer
is cleared by close() and is not managed here.
- The second one is used to lock Redis while
processing jobs. These timers are short-lived,
and there can be more than one active at a
time.
The lock timer executes Redis commands, which
means we can't close queues while it's active i.e.
this won't work:
queue.process(function (job, jobDone) {
handle(job);
queue.disconnect().then(jobDone);
})
The disconnect() call closes the Redis connections; then, when
a queue tries to perform the scheduled Redis commands,
they block until a Redis connection becomes available...
The solution is to close the Redis connections when there are no
active timers i.e. when the queue is idle. This helper class keeps
track of the active timers and executes any queued listeners
whenever that count goes to zero.
Since disconnect() simply can't work if there are active handles,
its close() wrapper postpones closing the Redis connections
until the next idle state. This means that close() can safely
be called from anywhere at any time, even from within a job
handler:
queue.process(function (job, jobDone) {
handle(job);
queue.close();
jobDone();
})
*/
function TimerManager() {
this.idle = true;
this.listeners = [];
this.timers = {};
}
/**
Create a new timer (setTimeout).
Expired timers are automatically cleared
@param {String} name - Name of a timer key. Used only for debugging.
@param {Number} delay - delay of timeout
@param {Function} fn - Function to execute after delay
@returns {Number} id - The timer id. Used to clear the timer
*/
TimerManager.prototype.set = function(name, delay, fn) {
const id = uuid.v4();
const timer = setTimeout(
(timerInstance, timeoutId) => {
timerInstance.clear(timeoutId);
try {
fn();
} catch (err) {
console.error(err);
}
},
delay,
this,
id
);
// XXX only the timer is used, but the
// other fields are useful for
// troubleshooting/debugging
this.timers[id] = {
name,
timer
};
this.idle = false;
return id;
};
/**
Clear a timer (clearTimeout).
Queued listeners are executed if there are no
remaining timers
*/
TimerManager.prototype.clear = function(id) {
const timers = this.timers;
const timer = timers[id];
if (!timer) {
return;
}
clearTimeout(timer.timer);
delete timers[id];
if (!this.idle && _.size(timers) === 0) {
while (this.listeners.length) {
this.listeners.pop()();
}
this.idle = true;
}
};
TimerManager.prototype.clearAll = function() {
_.each(this.timers, (timer, id) => {
this.clear(id);
});
};
/**
* Returns a promise that resolves when there are no active timers.
*/
TimerManager.prototype.whenIdle = function() {
return new Promise(resolve => {
if (this.idle) {
resolve();
} else {
this.listeners.unshift(resolve);
}
});
};
module.exports = TimerManager;
+70
View File
@@ -0,0 +1,70 @@
'use strict';
const errorObject = { value: null };
function tryCatch(fn, ctx, args) {
try {
return fn.apply(ctx, args);
} catch (e) {
errorObject.value = e;
return errorObject;
}
}
/**
* Waits for a redis client to be ready.
* @param {Redis} redis client
*/
function isRedisReady(client) {
return new Promise((resolve, reject) => {
if (client.status === 'ready') {
resolve();
} else {
function handleReady() {
client.removeListener('end', handleEnd);
client.removeListener('error', handleError);
resolve();
}
let lastError;
function handleError(err) {
lastError = err;
}
function handleEnd() {
client.removeListener('ready', handleReady);
client.removeListener('error', handleError);
reject(lastError);
}
client.once('ready', handleReady);
client.on('error', handleError);
client.once('end', handleEnd);
}
});
}
module.exports.errorObject = errorObject;
module.exports.tryCatch = tryCatch;
module.exports.isRedisReady = isRedisReady;
module.exports.emitSafe = function(emitter, event, ...args) {
try {
return emitter.emit(event, ...args);
} catch (err) {
try {
return emitter.emit('error', err);
} catch (err) {
// We give up if the error event also throws an exception.
console.error(err);
}
}
};
module.exports.MetricsTime = {
ONE_MINUTE: 1,
FIVE_MINUTES: 5,
FIFTEEN_MINUTES: 15,
THIRTY_MINUTES: 30,
ONE_HOUR: 60,
ONE_WEEK: 60 * 24 * 7,
TWO_WEEKS: 60 * 24 * 7 * 2,
ONE_MONTH: 60 * 24 * 7 * 2 * 4
};
+68
View File
@@ -0,0 +1,68 @@
'use strict';
const utils = require('./utils');
const clientCommandMessageReg = /ERR unknown command ['`]\s*client\s*['`]/;
module.exports = function(Queue) {
// IDEA, How to store metadata associated to a worker.
// create a key from the worker ID associated to the given name.
// We keep a hash table bull:myqueue:workers where every worker is a hash key workername:workerId with json holding
// metadata of the worker. The worker key gets expired every 30 seconds or so, we renew the worker metadata.
//
Queue.prototype.setWorkerName = function() {
return utils
.isRedisReady(this.client)
.then(() => {
const connectionName = this.clientName();
this.bclient.options.connectionName = connectionName;
return this.bclient.client('setname', connectionName);
})
.catch(err => {
if (!clientCommandMessageReg.test(err.message)) throw err;
});
};
Queue.prototype.getWorkers = function() {
return utils
.isRedisReady(this.client)
.then(() => {
return this.client.client('list');
})
.then(clients => {
return this.parseClientList(clients);
})
.catch(err => {
if (!clientCommandMessageReg.test(err.message)) throw err;
});
};
Queue.prototype.base64Name = function() {
return Buffer.from(this.name).toString('base64');
};
Queue.prototype.clientName = function() {
return this.keyPrefix + ':' + this.base64Name();
};
Queue.prototype.parseClientList = function(list) {
const lines = list.split('\n');
const clients = [];
lines.forEach(line => {
const client = {};
const keyValues = line.split(' ');
keyValues.forEach(keyValue => {
const index = keyValue.indexOf('=');
const key = keyValue.substring(0, index);
const value = keyValue.substring(index + 1);
client[key] = value;
});
const name = client['name'];
if (name && name.startsWith(this.clientName())) {
client['name'] = this.name;
clients.push(client);
}
});
return clients;
};
};