feat: Passwordless cross-device authentication

- Arkitektur: docs/auth/passwordless-architecture.md
- Backend: iom/quixzoom-auth-service/ (FastAPI + Redis)
- Webb: quixzoom-market-pages/se/login/ (QR-kod + polling)
- App: iom/quixzoom-app/src/features/auth/ (push + deep links)

Flöde: QR-kod → app-godkännande → webb-inloggad
This commit is contained in:
Bernt
2026-07-07 07:11:50 +00:00
parent 4aa984ad74
commit 6989a98d75
61843 changed files with 5491611 additions and 872231 deletions
+141
View File
@@ -0,0 +1,141 @@
'use strict';
const content = `--[[
Adds a job to the queue by doing the following:
- Increases the job counter if needed.
- Creates a new job key with the job data.
- if delayed:
- computes timestamp.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
- if not delayed
- Adds the jobId to the wait/paused list in one of three ways:
- LIFO
- FIFO
- prioritized.
- Adds the job to the "added" list so that workers gets notified.
Input:
KEYS[1] 'wait',
KEYS[2] 'paused'
KEYS[3] 'meta-paused'
KEYS[4] 'id'
KEYS[5] 'delayed'
KEYS[6] 'priority'
ARGV[1] key prefix,
ARGV[2] custom id (will not generate one automatically)
ARGV[3] name
ARGV[4] data (json stringified job data)
ARGV[5] opts (json stringified job opts)
ARGV[6] timestamp
ARGV[7] delay
ARGV[8] delayedTimestamp
ARGV[9] priority
ARGV[10] LIFO
ARGV[11] token
ARGV[12] debounce key
ARGV[13] debounceId
ARGV[14] debounceTtl
]]
local jobId
local jobIdKey
local rcall = redis.call
-- Includes
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
--[[
Function to debounce a job.
]]
local function debounceJob(prefixKey, debounceId, ttl, jobId, debounceKey, token)
if debounceId ~= "" then
local debounceKeyExists
if ttl ~= "" then
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
else
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
end
if debounceKeyExists then
local currentDebounceJobId = rcall('GET', debounceKey)
rcall("PUBLISH", prefixKey .. "debounced@" .. token, currentDebounceJobId)
return currentDebounceJobId
end
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
local jobCounter = rcall("INCR", KEYS[4])
if ARGV[2] == "" then
jobId = jobCounter
jobIdKey = ARGV[1] .. jobId
else
jobId = ARGV[2]
jobIdKey = ARGV[1] .. jobId
if rcall("EXISTS", jobIdKey) == 1 then
rcall("PUBLISH", ARGV[1] .. "duplicated@" .. ARGV[11], jobId)
return jobId .. "" -- convert to string
end
end
local debounceKey = ARGV[12]
local opts = cmsgpack.unpack(ARGV[5])
local debouncedJobId = debounceJob(ARGV[1], ARGV[13], ARGV[14],
jobId, debounceKey, ARGV[11])
if debouncedJobId then
return debouncedJobId
end
local debounceId = ARGV[13]
local optionalValues = {}
if debounceId ~= "" then
table.insert(optionalValues, "deid")
table.insert(optionalValues, debounceId)
end
-- Store the job.
rcall("HMSET", jobIdKey, "name", ARGV[3], "data", ARGV[4], "opts", opts, "timestamp",
ARGV[6], "delay", ARGV[7], "priority", ARGV[9], unpack(optionalValues))
-- Check if job is delayed
local delayedTimestamp = tonumber(ARGV[8])
if(delayedTimestamp ~= 0) then
local timestamp = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
rcall("ZADD", KEYS[5], timestamp, jobId)
rcall("PUBLISH", KEYS[5], delayedTimestamp)
else
local target
-- Whe check for the meta-paused key to decide if we are paused or not
-- (since an empty list and !EXISTS are not really the same)
local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
-- Standard or priority add
local priority = tonumber(ARGV[9])
if priority == 0 then
-- LIFO or FIFO
rcall(ARGV[10], target, jobId)
else
addJobWithPriority(KEYS[6], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[1] .. "ing@" .. ARGV[11], jobId)
end
return jobId .. "" -- convert to string
`;
module.exports = {
name: 'addJob',
content,
keys: 6,
};
+30
View File
@@ -0,0 +1,30 @@
'use strict';
const content = `--[[
Add job log
Input:
KEYS[1] job id key
KEYS[2] job logs key
ARGV[1] id
ARGV[2] log
ARGV[3] keepLogs
Output:
-1 - Missing job.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
local logCount = rcall("RPUSH", KEYS[2], ARGV[2])
if ARGV[3] ~= '' then
local keepLogs = tonumber(ARGV[3])
rcall("LTRIM", KEYS[2], -keepLogs, -1)
return math.min(keepLogs, logCount)
end
return logCount
else
return -1
end
`;
module.exports = {
name: 'addLog',
content,
keys: 2,
};
+139
View File
@@ -0,0 +1,139 @@
'use strict';
const content = `--[[
Remove jobs from the specific set.
Input:
KEYS[1] set key,
KEYS[2] priority key
KEYS[3] rate limiter key
ARGV[1] prefix key
ARGV[2] maxTimestamp
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
]]
local setKey = KEYS[1]
local priorityKey = KEYS[2]
local rateLimiterKey = KEYS[3]
local prefixKey = ARGV[1]
local maxTimestamp = ARGV[2]
local limitStr = ARGV[3]
local setName = ARGV[4]
local isList = false
local rcall = redis.call
-- Includes
--[[
Function to remove debounce key.
]]
local function removeDebounceKey(prefixKey, jobKey)
local debounceId = rcall("HGET", jobKey, "deid")
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
rcall("DEL", debounceKey)
end
end
if setName == "wait" or setName == "active" or setName == "paused" then
isList = true
end
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
-- of items in a sorted set only run a single iteration. If we simply used
-- ZRANGE, we may take a long time traversing through jobs that are within the
-- grace period.
local function shouldUseZRangeByScore(isList, limit)
return not isList and limit > 0
end
local function getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
if isList then
return rcall("LRANGE", setKey, rangeStart, rangeEnd)
elseif shouldUseZRangeByScore(isList, limit) then
return rcall("ZRANGEBYSCORE", setKey, 0, maxTimestamp, "LIMIT", 0, limit)
else
return rcall("ZRANGE", setKey, rangeStart, rangeEnd)
end
end
local limit = tonumber(limitStr)
local rangeStart = 0
local rangeEnd = -1
-- If we're only deleting _n_ items, avoid retrieving all items
-- for faster performance
--
-- Start from the tail of the list, since that's where oldest elements
-- are generally added for FIFO lists
if limit > 0 then
rangeStart = -1 - limit + 1
rangeEnd = -1
end
local jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
local deleted = {}
local deletedCount = 0
local jobTS
-- Run this loop:
-- - Once, if limit is -1 or 0
-- - As many times as needed if limit is positive
while ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do
local jobIdsLen = #jobIds
for i, jobId in ipairs(jobIds) do
if limit > 0 and deletedCount >= limit then
break
end
local jobKey = prefixKey .. jobId
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Find the right timestamp of the job to compare to maxTimestamp:
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
-- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
-- * timestamp is the original job submission time
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
for _, ts in ipairs(rcall("HMGET", jobKey, "finishedOn", "processedOn", "timestamp")) do
if (ts) then
jobTS = ts
break
end
end
if (not jobTS or jobTS < maxTimestamp) then
if isList then
-- Job ids can't be the empty string. Use the empty string as a
-- deletion marker. The actual deletion will occur at the end of the
-- script.
rcall("LSET", setKey, rangeEnd - jobIdsLen + i, "")
else
rcall("ZREM", setKey, jobId)
end
rcall("ZREM", priorityKey, jobId)
if setName ~= "completed" and setName ~= "failed" then
removeDebounceKey(prefixKey, jobKey)
end
rcall("DEL", jobKey)
rcall("DEL", jobKey .. ":logs")
-- delete keys related to rate limiter
-- NOTE: this code is unncessary for other sets than wait, paused and delayed.
local limiterIndexTable = rateLimiterKey .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
deletedCount = deletedCount + 1
table.insert(deleted, jobId)
end
end
end
-- If we didn't have a limit or used the single-iteration ZRANGEBYSCORE
-- function, return immediately. We should have deleted all the jobs we can
if limit <= 0 or shouldUseZRangeByScore(isList, limit) then
break
end
if deletedCount < limit then
-- We didn't delete enough. Look for more to delete
rangeStart = rangeStart - limit
rangeEnd = rangeEnd - limit
jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
end
end
if isList then
rcall("LREM", setKey, 0, "")
end
return deleted
`;
module.exports = {
name: 'cleanJobsInSet',
content,
keys: 3,
};
+26
View File
@@ -0,0 +1,26 @@
'use strict';
const content = `--[[
Extend lock and removes the job from the stalled set.
Input:
KEYS[1] 'lock',
KEYS[2] 'stalled'
ARGV[1] token
ARGV[2] lock duration in milliseconds
ARGV[3] jobid
Output:
"1" if lock extended succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
rcall("SREM", KEYS[2], ARGV[3])
return 1
end
end
return 0
`;
module.exports = {
name: 'extendLock',
content,
keys: 2,
};
+49
View File
@@ -0,0 +1,49 @@
'use strict';
const content = `--[[
Get counts per provided states
Input:
KEYS[1] wait key
KEYS[2] paused key
KEYS[3] meta-paused key
KEYS[4] priority key
ARGV[1...] priorities
]]
local rcall = redis.call
local results = {}
local prioritizedKey = KEYS[4]
-- Includes
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
for i = 1, #ARGV do
local priority = tonumber(ARGV[i])
if priority == 0 then
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
local count = rcall("LLEN", target) - rcall("ZCARD", prioritizedKey)
if count < 0 then
-- considering when last waiting job is moved to active before
-- removing priority reference
results[#results+1] = 0
else
results[#results+1] = count
end
else
results[#results+1] = rcall("ZCOUNT", prioritizedKey,
priority, priority)
end
end
return results
`;
module.exports = {
name: 'getCountsPerPriority',
content,
keys: 4,
};
+29
View File
@@ -0,0 +1,29 @@
'use strict';
module.exports = {
["addJob-6"]: require('./addJob-6'),
["addLog-2"]: require('./addLog-2'),
["cleanJobsInSet-3"]: require('./cleanJobsInSet-3'),
["extendLock-2"]: require('./extendLock-2'),
["getCountsPerPriority-4"]: require('./getCountsPerPriority-4'),
["isFinished-2"]: require('./isFinished-2'),
["isJobInList-1"]: require('./isJobInList-1'),
["moveStalledJobsToWait-7"]: require('./moveStalledJobsToWait-7'),
["moveToActive-8"]: require('./moveToActive-8'),
["moveToDelayed-4"]: require('./moveToDelayed-4'),
["moveToFinished-9"]: require('./moveToFinished-9'),
["obliterate-2"]: require('./obliterate-2'),
["pause-5"]: require('./pause-5'),
["promote-5"]: require('./promote-5'),
["releaseLock-1"]: require('./releaseLock-1'),
["removeJob-11"]: require('./removeJob-11'),
["removeJobs-8"]: require('./removeJobs-8'),
["removeRepeatable-2"]: require('./removeRepeatable-2'),
["reprocessJob-6"]: require('./reprocessJob-6'),
["retryJob-7"]: require('./retryJob-7'),
["retryJobs-5"]: require('./retryJobs-5'),
["saveStacktrace-1"]: require('./saveStacktrace-1'),
["takeLock-1"]: require('./takeLock-1'),
["updateData-1"]: require('./updateData-1'),
["updateDelaySet-6"]: require('./updateDelaySet-6'),
["updateProgress-2"]: require('./updateProgress-2'),
}
+25
View File
@@ -0,0 +1,25 @@
'use strict';
const content = `--[[
Checks if a job is finished (.i.e. is in the completed or failed set)
Input:
KEYS[1] completed key
KEYS[2] failed key
ARGV[1] job id
Output:
0 - not finished.
1 - completed.
2 - failed.
]]
if redis.call("ZSCORE", KEYS[1], ARGV[1]) ~= false then
return 1
end
if redis.call("ZSCORE", KEYS[2], ARGV[1]) ~= false then
return 2
end
return redis.call("ZSCORE", KEYS[2], ARGV[1])
`;
module.exports = {
name: 'isFinished',
content,
keys: 2,
};
+25
View File
@@ -0,0 +1,25 @@
'use strict';
const content = `--[[
Checks if job is in a given list.
Input:
KEYS[1]
ARGV[1]
Output:
1 if element found in the list.
]]
local function item_in_list (list, item)
for _, v in pairs(list) do
if v == item then
return 1
end
end
return nil
end
local items = redis.call("LRANGE", KEYS[1] , 0, -1)
return item_in_list(items, ARGV[1])
`;
module.exports = {
name: 'isJobInList',
content,
keys: 1,
};
+156
View File
@@ -0,0 +1,156 @@
'use strict';
const content = `--[[
Move stalled jobs to wait.
Input:
KEYS[1] 'stalled' (SET)
KEYS[2] 'wait', (LIST)
KEYS[3] 'active', (LIST)
KEYS[4] 'failed', (ZSET)
KEYS[5] 'stalled-check', (KEY)
KEYS[6] 'meta-paused', (KEY)
KEYS[7] 'paused', (LIST)
ARGV[1] Max stalled job count
ARGV[2] queue.toKey('')
ARGV[3] timestamp
ARGV[4] max check time
Events:
'stalled' with stalled job id.
]]
local rcall = redis.call
-- Includes
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
--[[
Function to remove debounce key if needed.
]]
local function removeDebounceKeyIfNeeded(prefixKey, debounceId)
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
local pttl = rcall("PTTL", debounceKey)
if pttl == 0 or pttl == -1 then
rcall("DEL", debounceKey)
end
end
end
local function removeJob(jobId, baseKey)
local jobKey = baseKey .. jobId
rcall("DEL", jobKey, jobKey .. ':logs')
end
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
for i, jobId in ipairs(jobIds) do
removeJob(jobId, prefix)
end
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
for i, jobId in ipairs(jobIds) do
removeJob(jobId, prefix)
end
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
end
-- Check if we need to check for stalled jobs now.
if rcall("EXISTS", KEYS[5]) == 1 then
return {{}, {}}
end
rcall("SET", KEYS[5], ARGV[3], "PX", ARGV[4])
-- Move all stalled jobs to wait
local stalling = rcall('SMEMBERS', KEYS[1])
local stalled = {}
local failed = {}
if(#stalling > 0) then
rcall('DEL', KEYS[1])
local MAX_STALLED_JOB_COUNT = tonumber(ARGV[1])
-- Remove from active list
for i, jobId in ipairs(stalling) do
local jobKey = ARGV[2] .. jobId
-- Check that the lock is also missing, then we can handle this job as really stalled.
if(rcall("EXISTS", jobKey .. ":lock") == 0) then
-- Remove from the active queue.
local removed = rcall("LREM", KEYS[3], 1, jobId)
if(removed > 0) then
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter", 1)
if(stalledCount > MAX_STALLED_JOB_COUNT) then
local jobAttributes = rcall("HMGET", jobKey, "opts", "deid")
local opts = cjson.decode(jobAttributes[1])
local removeOnFailType = type(opts["removeOnFail"])
rcall("ZADD", KEYS[4], ARGV[3], jobId)
rcall("HMSET", jobKey, "failedReason", "job stalled more than allowable limit",
"finishedOn", ARGV[3])
removeDebounceKeyIfNeeded(ARGV[2], jobAttributes[2])
rcall("PUBLISH", KEYS[4], '{"jobId":"' .. jobId .. '", "val": "job stalled more than maxStalledCount"}')
if removeOnFailType == "number" then
removeJobsByMaxCount(opts["removeOnFail"],
KEYS[4], ARGV[2])
elseif removeOnFailType == "boolean" then
if opts["removeOnFail"] then
removeJob(jobId, ARGV[2])
rcall("ZREM", KEYS[4], jobId)
end
elseif removeOnFailType ~= "nil" then
local maxAge = opts["removeOnFail"]["age"]
local maxCount = opts["removeOnFail"]["count"]
if maxAge ~= nil then
removeJobsByMaxAge(ARGV[3], maxAge,
KEYS[4], ARGV[2])
end
if maxCount ~= nil and maxCount > 0 then
removeJobsByMaxCount(maxCount, KEYS[4],
ARGV[2])
end
end
table.insert(failed, jobId)
else
local target = getTargetQueueList(KEYS[6], KEYS[2], KEYS[7])
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
rcall("RPUSH", target, jobId)
rcall('PUBLISH', KEYS[1] .. '@', jobId)
table.insert(stalled, jobId)
end
end
end
end
end
-- Mark potentially stalled jobs
local active = rcall('LRANGE', KEYS[3], 0, -1)
if (#active > 0) then
for from, to in batches(#active, 7000) do
rcall('SADD', KEYS[1], unpack(active, from, to))
end
end
return {failed, stalled}
`;
module.exports = {
name: 'moveStalledJobsToWait',
content,
keys: 7,
};
+130
View File
@@ -0,0 +1,130 @@
'use strict';
const content = `--[[
Move next job to be processed to active, lock it and fetch its data. The job
may be delayed, in that case we need to move it to the delayed set instead.
This operation guarantees that the worker owns the job during the locks
expiration time. The worker is responsible of keeping the lock fresh
so that no other worker picks this job again.
Input:
KEYS[1] wait key
KEYS[2] active key
KEYS[3] priority key
KEYS[4] active event key
KEYS[5] stalled key
-- Rate limiting
KEYS[6] rate limiter key
KEYS[7] delayed key
--
KEYS[8] drained key
ARGV[1] key prefix
ARGV[2] lock token
ARGV[3] lock duration in milliseconds
ARGV[4] timestamp
ARGV[5] optional jobid
ARGV[6] optional jobs per time unit (rate limiter)
ARGV[7] optional time unit (rate limiter)
ARGV[8] optional do not do anything with job if rate limit hit
ARGV[9] optional rate limit by key
]]
local rcall = redis.call
local rateLimit = function(jobId, maxJobs)
local rateLimiterKey = KEYS[6];
local limiterIndexTable = rateLimiterKey .. ":index"
-- Rate limit by group?
if(ARGV[9]) then
local group = string.match(jobId, "[^:]+$")
if group ~= nil then
rateLimiterKey = rateLimiterKey .. ":" .. group
end
end
-- -- key for storing rate limited jobs
-- When a job has been previously rate limited it should be part of this set
-- if the job is back here means that the delay time for this job has passed and now we should
-- be able to process it again.
local limitedSetKey = rateLimiterKey .. ":limited"
local delay = 0
-- -- Check if job was already limited
local isLimited = rcall("SISMEMBER", limitedSetKey, jobId);
if isLimited == 1 then
-- Remove from limited zset since we are going to try to process it
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
else
-- If not, check if there are any limited jobs
-- If the job has not been rate limited, we should check if there are any other rate limited jobs, because if that
-- is the case we do not want to process this job, just calculate a delay for it and put it to "sleep".
local numLimitedJobs = rcall("SCARD", limitedSetKey)
if numLimitedJobs > 0 then
-- Note, add some slack to compensate for drift.
delay = ((numLimitedJobs * ARGV[7] * 1.1) / maxJobs) + tonumber(rcall("PTTL", rateLimiterKey))
end
end
local jobCounter = tonumber(rcall("GET", rateLimiterKey))
if(jobCounter == nil) then
jobCounter = 0
end
-- check if rate limit hit
if (delay == 0) and (jobCounter >= maxJobs) then
-- Seems like there are no current rated limited jobs, but the jobCounter has exceeded the number of jobs for this unit of time so we need to rate limit this job.
local exceedingJobs = jobCounter - maxJobs
delay = tonumber(rcall("PTTL", rateLimiterKey)) + ((exceedingJobs) * ARGV[7]) / maxJobs
end
if delay > 0 then
local bounceBack = ARGV[8]
if bounceBack == 'false' then
local timestamp = delay + tonumber(ARGV[4])
-- put job into delayed queue
rcall("ZADD", KEYS[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId)
rcall("PUBLISH", KEYS[7], timestamp)
rcall("SADD", limitedSetKey, jobId)
-- store index so that we can delete rate limited data
rcall("HSET", limiterIndexTable, jobId, limitedSetKey)
end
-- remove from active queue
rcall("LREM", KEYS[2], 1, jobId)
return true
else
-- false indicates not rate limited
-- increment jobCounter only when a job is not rate limited
if (jobCounter == 0) then
rcall("PSETEX", rateLimiterKey, ARGV[7], 1)
else
rcall("INCR", rateLimiterKey)
end
return false
end
end
local jobId = ARGV[5]
if jobId ~= '' then
-- clean stalled key
rcall("SREM", KEYS[5], jobId)
else
-- move from wait to active
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
end
if jobId then
-- Check if we need to perform rate limiting.
local maxJobs = tonumber(ARGV[6])
if maxJobs then
if rateLimit(jobId, maxJobs) then
return
end
end
-- get a lock
local jobKey = ARGV[1] .. jobId
local lockKey = jobKey .. ':lock'
rcall("SET", lockKey, ARGV[2], "PX", ARGV[3])
-- remove from priority
rcall("ZREM", KEYS[3], jobId)
rcall("PUBLISH", KEYS[4], jobId)
rcall("HSET", jobKey, "processedOn", ARGV[4])
return {rcall("HGETALL", jobKey), jobId} -- get job data
else
rcall("PUBLISH", KEYS[8], "")
end
`;
module.exports = {
name: 'moveToActive',
content,
keys: 8,
};
+59
View File
@@ -0,0 +1,59 @@
'use strict';
const content = `--[[
Moves job from active to delayed set.
Input:
KEYS[1] active key
KEYS[2] delayed key
KEYS[3] job key
KEYS[4] stalled key
ARGV[1] delayedTimestamp
ARGV[2] the id of the job
ARGV[3] queue token
Output:
0 - OK
-1 - Missing job.
-2 - Job is locked.
Events:
- delayed key.
]]
local rcall = redis.call
-- Includes
local function removeLock(jobKey, stalledKey, token, jobId)
if token ~= "0" then
local lockKey = jobKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", stalledKey, jobId)
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
return 0
end
if rcall("EXISTS", KEYS[3]) == 1 then
local errorCode = removeLock(KEYS[3], KEYS[4], ARGV[3], ARGV[2])
if errorCode < 0 then
return errorCode
end
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[2])
if numRemovedElements < 1 then return -3 end
local score = tonumber(ARGV[1])
rcall("ZADD", KEYS[2], score, ARGV[2])
rcall("PUBLISH", KEYS[2], (score / 0x1000))
return 0
else
return -1
end
`;
module.exports = {
name: 'moveToDelayed',
content,
keys: 4,
};
+196
View File
@@ -0,0 +1,196 @@
'use strict';
const content = `--[[
Move job from active to a finished status (completed or failed)
A job can only be moved to completed if it was active.
The job must be locked before it can be moved to a finished status,
and the lock must be released in this script.
Input:
KEYS[1] active key
KEYS[2] completed/failed key
KEYS[3] jobId key
KEYS[4] wait key
KEYS[5] priority key
KEYS[6] active event key
KEYS[7] delayed key
KEYS[8] stalled key
KEYS[9] metrics key
ARGV[1] jobId
ARGV[2] timestamp
ARGV[3] msg property
ARGV[4] return value / failed reason
ARGV[5] token
ARGV[6] shouldRemove
ARGV[7] event data (? maybe just send jobid).
ARGV[8] should fetch next job
ARGV[9] base key
ARGV[10] lock token
ARGV[11] lock duration in milliseconds
ARGV[12] maxMetricsSize
Output:
0 OK
-1 Missing key.
-2 Missing lock.
-3 - Job not in active set.
Events:
'completed/failed'
]]
local rcall = redis.call
-- Includes
--[[
Functions to collect metrics based on a current and previous count of jobs.
Granualarity is fixed at 1 minute.
]]
-- Includes
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
local function collectMetrics(metaKey, dataPointsList, maxDataPoints, timestamp)
-- Increment current count
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
-- Compute how many data points we need to add to the list, N.
local prevTS = rcall("HGET", metaKey, "prevTS")
if not prevTS then
-- If prevTS is nil, set it to the current timestamp
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
return
end
local N = math.min(math.floor(timestamp / 60000) - math.floor(prevTS / 60000), tonumber(maxDataPoints))
if N > 0 then
local delta = count - rcall("HGET", metaKey, "prevCount")
-- If N > 1, add N-1 zeros to the list
if N > 1 then
local points = {}
points[1] = delta
for i = 2, N do points[i] = 0 end
for from, to in batches(#points, 7000) do
rcall("LPUSH", dataPointsList, unpack(points, from, to))
end
else
-- LPUSH delta to the list
rcall("LPUSH", dataPointsList, delta)
end
-- LTRIM to keep list to its max size
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
-- update prev count with current count
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
end
end
local function removeLock(jobKey, stalledKey, token, jobId)
if token ~= "0" then
local lockKey = jobKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", stalledKey, jobId)
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
return 0
end
--[[
Function to remove debounce key if needed.
]]
local function removeDebounceKeyIfNeeded(prefixKey, debounceId)
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
local pttl = rcall("PTTL", debounceKey)
if pttl == 0 or pttl == -1 then
rcall("DEL", debounceKey)
end
end
end
if rcall("EXISTS", KEYS[3]) == 1 then -- // Make sure job exists
local errorCode = removeLock(KEYS[3], KEYS[8], ARGV[5], ARGV[1])
if errorCode < 0 then
return errorCode
end
-- Remove from active list (if not active we shall return error)
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[1])
if numRemovedElements < 1 then return -3 end
local debounceId = rcall("HGET", KEYS[3], "deid")
removeDebounceKeyIfNeeded(ARGV[9], debounceId)
-- Remove job?
local keepJobs = cmsgpack.unpack(ARGV[6])
local maxCount = keepJobs['count']
local maxAge = keepJobs['age']
local targetSet = KEYS[2]
local timestamp = ARGV[2]
if maxCount ~= 0 then
-- Add to complete/failed set
rcall("ZADD", targetSet, timestamp, ARGV[1])
rcall("HMSET", KEYS[3], ARGV[3], ARGV[4], "finishedOn", timestamp) -- "returnvalue" / "failedReason" and "finishedOn"
local function removeJobs(jobIds)
for i, jobId in ipairs(jobIds) do
local jobKey = ARGV[9] .. jobId
local jobLogKey = jobKey .. ':logs'
rcall("DEL", jobKey, jobLogKey)
end
end
-- Remove old jobs?
if maxAge ~= nil then
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
removeJobs(jobIds)
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
if maxCount ~= nil and maxCount > 0 then
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
removeJobs(jobIds)
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1));
end
else
local jobLogKey = KEYS[3] .. ':logs'
rcall("DEL", KEYS[3], jobLogKey)
end
-- Collect metrics
if ARGV[12] ~= "" then
collectMetrics(KEYS[9], KEYS[9]..':data', ARGV[12], timestamp)
end
rcall("PUBLISH", targetSet, ARGV[7])
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
-- and not rate limited.
if (ARGV[8] == "1") then
-- move from wait to active
local jobId = rcall("RPOPLPUSH", KEYS[4], KEYS[1])
if jobId then
local jobKey = ARGV[9] .. jobId
local lockKey = jobKey .. ':lock'
-- get a lock
rcall("SET", lockKey, ARGV[11], "PX", ARGV[10])
rcall("ZREM", KEYS[5], jobId) -- remove from priority
rcall("PUBLISH", KEYS[6], jobId)
rcall("HSET", jobKey, "processedOn", ARGV[2])
return {rcall("HGETALL", jobKey), jobId} -- get job data
end
end
return 0
else
return -1
end
`;
module.exports = {
name: 'moveToFinished',
content,
keys: 9,
};
+106
View File
@@ -0,0 +1,106 @@
'use strict';
const content = `--[[
Completely obliterates a queue and all of its contents
Input:
KEYS[1] meta-paused
KEYS[2] base
ARGV[1] count
ARGV[2] force
]]
-- This command completely destroys a queue including all of its jobs, current or past
-- leaving no trace of its existence. Since this script needs to iterate to find all the job
-- keys, consider that this call may be slow for very large queues.
-- The queue needs to be "paused" or it will return an error
-- If the queue has currently active jobs then the script by default will return error,
-- however this behaviour can be overrided using the 'force' option.
local maxCount = tonumber(ARGV[1])
local baseKey = KEYS[2]
local rcall = redis.call
-- Includes
--[[
Function to remove debounce key.
]]
local function removeDebounceKey(prefixKey, jobKey)
local debounceId = rcall("HGET", jobKey, "deid")
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
rcall("DEL", debounceKey)
end
end
local function getListItems(keyName, max)
return rcall('LRANGE', keyName, 0, max - 1)
end
local function getZSetItems(keyName, max)
return rcall('ZRANGE', keyName, 0, max - 1)
end
local function removeJobs(baseKey, keys)
for i, key in ipairs(keys) do
local jobKey = baseKey .. key
rcall("DEL", jobKey, jobKey .. ':logs')
removeDebounceKey(baseKey, jobKey)
end
maxCount = maxCount - #keys
end
local function removeListJobs(keyName, max)
local jobs = getListItems(keyName, max)
removeJobs(baseKey, jobs)
rcall("LTRIM", keyName, #jobs, -1)
end
local function removeZSetJobs(keyName, max)
local jobs = getZSetItems(keyName, max)
removeJobs(baseKey, jobs)
if (#jobs > 0) then rcall("ZREM", keyName, unpack(jobs)) end
end
local function removeLockKeys(keys)
for i, key in ipairs(keys) do rcall("DEL", baseKey .. key .. ':lock') end
end
-- 1) Check if paused, if not return with error.
if rcall("EXISTS", KEYS[1]) ~= 1 then
return -1 -- Error, NotPaused
end
-- 2) Check if there are active jobs, if there are and not "force" return error.
local activeKey = baseKey .. 'active'
local activeJobs = getListItems(activeKey, maxCount)
if (#activeJobs > 0) then
if (ARGV[2] == "") then
return -2 -- Error, ExistsActiveJobs
end
end
removeLockKeys(activeJobs)
removeJobs(baseKey, activeJobs)
rcall("LTRIM", activeKey, #activeJobs, -1)
if (maxCount <= 0) then return 1 end
local waitKey = baseKey .. 'paused'
removeListJobs(waitKey, maxCount)
if (maxCount <= 0) then return 1 end
local delayedKey = baseKey .. 'delayed'
removeZSetJobs(delayedKey, maxCount)
if (maxCount <= 0) then return 1 end
local completedKey = baseKey .. 'completed'
removeZSetJobs(completedKey, maxCount)
if (maxCount <= 0) then return 1 end
local failedKey = baseKey .. 'failed'
removeZSetJobs(failedKey, maxCount)
if (maxCount <= 0) then return 1 end
if (maxCount > 0) then
rcall("DEL", baseKey .. 'priority')
rcall("DEL", baseKey .. 'stalled-check')
rcall("DEL", baseKey .. 'stalled')
rcall("DEL", baseKey .. 'meta-paused')
rcall("DEL", baseKey .. 'meta')
rcall("DEL", baseKey .. 'id')
rcall("DEL", baseKey .. 'repeat')
rcall("DEL", baseKey .. 'metrics:completed')
rcall("DEL", baseKey .. 'metrics:completed:data')
rcall("DEL", baseKey .. 'metrics:failed')
rcall("DEL", baseKey .. 'metrics:failed:data')
return 0
else
return 1
end
`;
module.exports = {
name: 'obliterate',
content,
keys: 2,
};
+33
View File
@@ -0,0 +1,33 @@
'use strict';
const content = `--[[
Pauses or resumes a queue globably.
Input:
KEYS[1] 'wait' or 'paused''
KEYS[2] 'paused' or 'wait'
KEYS[3] 'meta-paused'
KEYS[4] 'paused' o 'resumed' event.
KEYS[5] 'meta' this key is only used in BullMQ and above.
ARGV[1] 'paused' or 'resumed'
Event:
publish paused or resumed event.
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then
rcall("RENAME", KEYS[1], KEYS[2])
end
if ARGV[1] == "paused" then
rcall("SET", KEYS[3], 1)
-- for forwards compatibility
rcall("HSET", KEYS[5], "paused", 1)
else
rcall("DEL", KEYS[3])
-- for forwards compatibility
rcall("HDEL", KEYS[5], "paused")
end
rcall("PUBLISH", KEYS[4], ARGV[1])
`;
module.exports = {
name: 'pause',
content,
keys: 5,
};
+65
View File
@@ -0,0 +1,65 @@
'use strict';
const content = `--[[
Promotes a job that is currently "delayed" to the "waiting" state
Input:
KEYS[1] 'delayed'
KEYS[2] 'wait'
KEYS[3] 'paused'
KEYS[4] 'meta-paused'
KEYS[5] 'priority'
ARGV[1] queue.toKey('')
ARGV[2] jobId
ARGV[3] queue token
Events:
'waiting'
]]
local rcall = redis.call;
local jobId = ARGV[2]
-- Includes
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
if rcall("ZREM", KEYS[1], jobId) == 1 then
local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", target, jobId)
else
addJobWithPriority(KEYS[5], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[2] .. "ing@" .. ARGV[3], jobId)
rcall("HSET", ARGV[1] .. jobId, "delay", 0)
return 0
else
return -1
end
`;
module.exports = {
name: 'promote',
content,
keys: 5,
};
+22
View File
@@ -0,0 +1,22 @@
'use strict';
const content = `--[[
Release lock
Input:
KEYS[1] 'lock',
ARGV[1] token
ARGV[2] lock duration in milliseconds
Output:
"OK" if lock extented succesfully.
]]
local rcall = redis.call
if rcall("GET", KEYS[1]) == ARGV[1] then
return rcall("DEL", KEYS[1])
else
return 0
end
`;
module.exports = {
name: 'releaseLock',
content,
keys: 1,
};
+65
View File
@@ -0,0 +1,65 @@
'use strict';
const content = `--[[
Remove a job from all the queues it may be in as well as all its data.
In order to be able to remove a job, it must be unlocked.
Input:
KEYS[1] 'active',
KEYS[2] 'wait',
KEYS[3] 'delayed',
KEYS[4] 'paused',
KEYS[5] 'completed',
KEYS[6] 'failed',
KEYS[7] 'priority',
KEYS[8] jobId key
KEYS[9] job logs
KEYS[10] rate limiter index table
KEYS[11] prefix key
ARGV[1] jobId
ARGV[2] lock token
Events:
'removed'
]]
-- TODO PUBLISH global event 'removed'
local rcall = redis.call
-- Includes
--[[
Function to remove debounce key.
]]
local function removeDebounceKey(prefixKey, jobKey)
local debounceId = rcall("HGET", jobKey, "deid")
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
rcall("DEL", debounceKey)
end
end
local lockKey = KEYS[8] .. ':lock'
local lock = rcall("GET", lockKey)
if not lock then -- or (lock == ARGV[2])) then
local jobId = ARGV[1]
rcall("LREM", KEYS[1], 0, jobId)
rcall("LREM", KEYS[2], 0, jobId)
rcall("ZREM", KEYS[3], jobId)
rcall("LREM", KEYS[4], 0, jobId)
rcall("ZREM", KEYS[5], jobId)
rcall("ZREM", KEYS[6], jobId)
rcall("ZREM", KEYS[7], jobId)
removeDebounceKey(KEYS[11], KEYS[8])
rcall("DEL", KEYS[8])
rcall("DEL", KEYS[9])
-- delete keys related to rate limiter
local limiterIndexTable = KEYS[10] .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
return 1
else
return 0
end
`;
module.exports = {
name: 'removeJob',
content,
keys: 11,
};
+60
View File
@@ -0,0 +1,60 @@
'use strict';
const content = `--[[
Remove all jobs matching a given pattern from all the queues they may be in as well as all its data.
In order to be able to remove any job, they must be unlocked.
Input:
KEYS[1] 'active',
KEYS[2] 'wait',
KEYS[3] 'delayed',
KEYS[4] 'paused',
KEYS[5] 'completed',
KEYS[6] 'failed',
KEYS[7] 'priority',
KEYS[8] 'rate-limiter'
ARGV[1] prefix
ARGV[2] pattern
ARGV[3] cursor
Events:
'removed'
]]
-- TODO PUBLISH global events 'removed'
local rcall = redis.call
local result = rcall("SCAN", ARGV[3], "MATCH", ARGV[1] .. ARGV[2])
local cursor = result[1];
local jobKeys = result[2];
local removed = {}
local prefixLen = string.len(ARGV[1]) + 1
for i, jobKey in ipairs(jobKeys) do
local keyTypeResp = rcall("TYPE", jobKey)
if keyTypeResp["ok"] == "hash" then
local jobId = string.sub(jobKey, prefixLen)
local lockKey = jobKey .. ':lock'
local lock = redis.call("GET", lockKey)
if not lock then
rcall("LREM", KEYS[1], 0, jobId)
rcall("LREM", KEYS[2], 0, jobId)
rcall("ZREM", KEYS[3], jobId)
rcall("LREM", KEYS[4], 0, jobId)
rcall("ZREM", KEYS[5], jobId)
rcall("ZREM", KEYS[6], jobId)
rcall("ZREM", KEYS[7], jobId)
rcall("DEL", jobKey)
rcall("DEL", jobKey .. ':logs')
-- delete keys related to rate limiter
local limiterIndexTable = KEYS[8] .. ":index"
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
if limitedSetKey then
rcall("SREM", limitedSetKey, jobId)
rcall("HDEL", limiterIndexTable, jobId)
end
table.insert(removed, jobId)
end
end
end
return {cursor, removed}
`;
module.exports = {
name: 'removeJobs',
content,
keys: 8,
};
+25
View File
@@ -0,0 +1,25 @@
'use strict';
const content = `--[[
Removes a repeatable job
Input:
KEYS[1] repeat jobs key
KEYS[2] delayed jobs key
ARGV[1] repeat job id
ARGV[2] repeat job key
ARGV[3] queue key
]]
local millis = redis.call("ZSCORE", KEYS[1], ARGV[2])
if(millis) then
-- Delete next programmed job.
local repeatJobId = ARGV[1] .. millis
if(redis.call("ZREM", KEYS[2], repeatJobId) == 1) then
redis.call("DEL", ARGV[3] .. repeatJobId)
end
end
redis.call("ZREM", KEYS[1], ARGV[2]);
`;
module.exports = {
name: 'removeRepeatable',
content,
keys: 2,
};
+51
View File
@@ -0,0 +1,51 @@
'use strict';
const content = `--[[
Attempts to reprocess a job
Input:
KEYS[1] job key
KEYS[2] job lock key
KEYS[3] job state
KEYS[4] wait key
KEYS[5] meta-pause
KEYS[6] paused key
ARGV[1] job.id,
ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
ARGV[3] token
ARGV[4] timestamp
Output:
1 means the operation was a success
0 means the job does not exist
-1 means the job is currently locked and can't be retried.
-2 means the job was not found in the expected set.
]]
local rcall = redis.call;
if (rcall("EXISTS", KEYS[1]) == 1) then
if (rcall("EXISTS", KEYS[2]) == 0) then
rcall("HDEL", KEYS[1], "finishedOn", "processedOn", "failedReason")
rcall("HSET", KEYS[1], "retriedOn", ARGV[4])
if (rcall("ZREM", KEYS[3], ARGV[1]) == 1) then
local target
if rcall("EXISTS", KEYS[5]) ~= 1 then
target = KEYS[4]
else
target = KEYS[6]
end
rcall(ARGV[2], target, ARGV[1])
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[4] .. "ing@" .. ARGV[3], ARGV[1])
return 1
else
return -2
end
else
return -1
end
else
return 0
end
`;
module.exports = {
name: 'reprocessJob',
content,
keys: 6,
};
+93
View File
@@ -0,0 +1,93 @@
'use strict';
const content = `--[[
Retries a failed job by moving it back to the wait queue.
Input:
KEYS[1] 'active',
KEYS[2] 'wait'
KEYS[3] jobId key
KEYS[4] 'meta-paused'
KEYS[5] 'paused'
KEYS[6] stalled key
KEYS[7] 'priority'
ARGV[1] pushCmd
ARGV[2] jobId
ARGV[3] token
Events:
'prefix:added'
Output:
0 - OK
-1 - Missing key
-2 - Job Not locked
-3 - Job not in active set
]]
local rcall = redis.call
-- Includes
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
local function removeLock(jobKey, stalledKey, token, jobId)
if token ~= "0" then
local lockKey = jobKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", stalledKey, jobId)
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
return 0
end
if rcall("EXISTS", KEYS[3]) == 1 then
local errorCode = removeLock(KEYS[3], KEYS[6], ARGV[3], ARGV[2])
if errorCode < 0 then
return errorCode
end
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[2])
if numRemovedElements < 1 then return -3 end
local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[5])
local priority = tonumber(rcall("HGET", KEYS[3], "priority")) or 0
if priority == 0 then
-- LIFO or FIFO
rcall(ARGV[1], target, ARGV[2])
else
addJobWithPriority(KEYS[7], priority, ARGV[2], target)
end
return 0
else
return -1
end
`;
module.exports = {
name: 'retryJob',
content,
keys: 7,
};
+63
View File
@@ -0,0 +1,63 @@
'use strict';
const content = `--[[
Attempts to retry all failed jobs
Input:
KEYS[1] base key
KEYS[2] failed state key
KEYS[3] wait state key
KEYS[4] 'meta-paused'
KEYS[5] 'paused'
ARGV[1] count
Output:
1 means the operation is not completed
0 means the operation is completed
]]
local baseKey = KEYS[1]
local maxCount = tonumber(ARGV[1])
local rcall = redis.call;
-- Includes
--[[
Function to loop in batches.
Just a bit of warning, some commands as ZREM
could receive a maximum of 7000 parameters per call.
]]
local function batches(n, batchSize)
local i = 0
return function()
local from = i * batchSize + 1
i = i + 1
if (from <= n) then
local to = math.min(from + batchSize - 1, n)
return from, to
end
end
end
local function getZSetItems(keyName, max)
return rcall('ZRANGE', keyName, 0, max - 1)
end
local jobs = getZSetItems(KEYS[2], maxCount)
if (#jobs > 0) then
for i, key in ipairs(jobs) do
local jobKey = baseKey .. key
rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason")
end
local target
if rcall("EXISTS", KEYS[4]) ~= 1 then
target = KEYS[3]
else
target = KEYS[5]
end
for from, to in batches(#jobs, 7000) do
rcall("ZREM", KEYS[2], unpack(jobs, from, to))
rcall("LPUSH", target, unpack(jobs, from, to))
end
end
maxCount = maxCount - #jobs
if (maxCount <= 0) then return 1 end
return 0
`;
module.exports = {
name: 'retryJobs',
content,
keys: 5,
};
+26
View File
@@ -0,0 +1,26 @@
'use strict';
const content = `--[[
Save stacktrace and failedReason.
Input:
KEYS[1] job key
ARGV[1] stacktrace
ARGV[2] failedReason
ARGV[3] attemptsMade
Output:
0 - OK
-1 - Missing key
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then
rcall("HMSET", KEYS[1], "stacktrace", ARGV[1], "failedReason", ARGV[2],
"attemptsMade", ARGV[3])
return 0
else
return -1
end
`;
module.exports = {
name: 'saveStacktrace',
content,
keys: 1,
};
+21
View File
@@ -0,0 +1,21 @@
'use strict';
const content = `--[[
Takes a lock
Input:
KEYS[1] 'lock',
ARGV[1] token
ARGV[2] lock duration in milliseconds
Output:
"OK" if lock taken successfully.
]]
if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", ARGV[2]) then
return 1
else
return 0
end
`;
module.exports = {
name: 'takeLock',
content,
keys: 1,
};
+23
View File
@@ -0,0 +1,23 @@
'use strict';
const content = `--[[
Update job data
Input:
KEYS[1] Job id key
ARGV[1] data
Output:
0 - OK
-1 - Missing job.
]]
local rcall = redis.call
if rcall("EXISTS",KEYS[1]) == 1 then -- // Make sure job exists
rcall("HSET", KEYS[1], "data", ARGV[1])
return 0
else
return -1
end
`;
module.exports = {
name: 'updateData',
content,
keys: 1,
};
+76
View File
@@ -0,0 +1,76 @@
'use strict';
const content = `--[[
Updates the delay set, by picking a delayed job that should
be processed now.
Input:
KEYS[1] 'delayed'
KEYS[2] 'active'
KEYS[3] 'wait'
KEYS[4] 'priority'
KEYS[5] 'paused'
KEYS[6] 'meta-paused'
ARGV[1] queue.toKey('')
ARGV[2] delayed timestamp
ARGV[3] queue token
Events:
'removed'
]]
local rcall = redis.call;
-- Includes
--[[
Function to add job considering priority.
]]
local function addJobWithPriority(priorityKey, priority, jobId, targetKey)
rcall("ZADD", priorityKey, priority, jobId)
local count = rcall("ZCOUNT", priorityKey, 0, priority)
local len = rcall("LLEN", targetKey)
local id = rcall("LINDEX", targetKey, len - (count - 1))
if id then
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
else
rcall("RPUSH", targetKey, jobId)
end
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
if rcall("EXISTS", queueMetaKey) ~= 1 then
return waitKey, false
else
return pausedKey, true
end
end
-- Try to get as much as 1000 jobs at once
local jobs = rcall("ZRANGEBYSCORE", KEYS[1], 0, tonumber(ARGV[2]) * 0x1000, "LIMIT", 0, 1000)
if(#jobs > 0) then
rcall("ZREM", KEYS[1], unpack(jobs))
-- check if we need to use push in paused instead of waiting
local target = getTargetQueueList(KEYS[6], KEYS[3], KEYS[5])
for _, jobId in ipairs(jobs) do
-- Is this really needed?
rcall("LREM", KEYS[2], 0, jobId)
local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
if priority == 0 then
-- LIFO or FIFO
rcall("LPUSH", target, jobId)
else
addJobWithPriority(KEYS[4], priority, jobId, target)
end
-- Emit waiting event (wait..ing@token)
rcall("PUBLISH", KEYS[3] .. "ing@" .. ARGV[3], jobId)
rcall("HSET", ARGV[1] .. jobId, "delay", 0)
end
end
local nextTimestamp = rcall("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")[2]
if(nextTimestamp ~= nil) then
rcall("PUBLISH", KEYS[1], nextTimestamp / 0x1000)
end
return nextTimestamp
`;
module.exports = {
name: 'updateDelaySet',
content,
keys: 6,
};
+25
View File
@@ -0,0 +1,25 @@
'use strict';
const content = `--[[
Update job progress
Input:
KEYS[1] Job id key
KEYS[2] progress event key
ARGV[1] progress
ARGV[2] event data
Event:
progress(jobId, progress)
]]
local rcall = redis.call
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
rcall("HSET", KEYS[1], "progress", ARGV[1])
rcall("PUBLISH", KEYS[2], ARGV[2])
return 0
else
return -1
end
`;
module.exports = {
name: 'updateProgress',
content,
keys: 2,
};